Updates

daemon/obsidian.js

@@ -5,7 +5,8 @@ import { createHash, randomUUID } from 'node:crypto';
 
 const MARKER = '.jarvis-vault.json';
 const MAX_BYTES = 2 * 1024 * 1024;
-const MAX_ENTRIES = 10000;
+const PAGE_BYTES = 1024;
+const PAGE_ENTRIES = 20;
 const digest = data => createHash('sha256').update(data).digest('hex');
 export const defaultVaultPath = () => path.join(process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local/share'), 'jarvis/obsidian-agent');
 
@@ -62,7 +63,7 @@ export class ObsidianVault {
       this.ready(); result.ready = true;
       fs.accessSync(this.root, fs.constants.R_OK); result.readable = true;
       fs.accessSync(this.root, fs.constants.W_OK); result.writable = true;
-      const listing = this.list(); result.files = listing.entries.filter(e => e.type === 'file').length; result.truncated = listing.truncated;
+      const listing = this.list(); result.files = listing.totalFiles; result.truncated = false;
       result.uri = `obsidian://open?path=${encodeURIComponent(path.join(this.root, 'Welcome.md'))}`;
     } catch (error) { result.error = error.message; }
     return result;
@@ -84,32 +85,50 @@ export class ObsidianVault {
     }
     return result;
   }
-  list(relative = '') {
+  list(relative = '', { offset = 0, limit = PAGE_ENTRIES } = {}) {
     this.ready();
+    if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > PAGE_ENTRIES) throw new Error('Invalid listing page');
     const base = relative ? this.resolve(relative) : this.root;
-    const entries = []; let visited = 0; let truncated = false;
-    const walk = (dir, depth) => {
-      if (depth > 32) { truncated = true; return; }
+    const entries = []; let visited = 0; let totalFiles = 0;
+    const walk = dir => {
       for (const name of fs.readdirSync(dir).sort()) {
-        if (++visited > MAX_ENTRIES) { truncated = true; return; }
         if (name.startsWith('.')) continue;
         const file = path.join(dir, name); const rel = path.relative(this.root, file).split(path.sep).join('/');
         if (!this.memoryEnabled && (rel === 'memory' || rel.startsWith('memory/'))) continue;
         const stat = fs.lstatSync(file);
         if (stat.isSymbolicLink() || (stat.isFile() && stat.nlink > 1)) continue;
-        if (stat.isDirectory()) { entries.push({ path: rel, type: 'folder' }); walk(file, depth + 1); }
-        else if (stat.isFile()) entries.push({ path: rel, type: 'file', bytes: stat.size });
-        if (truncated) return;
+        if (!stat.isDirectory() && !stat.isFile()) continue;
+        if (stat.isFile()) totalFiles++;
+        if (visited++ >= offset && entries.length < limit) entries.push({ path: rel, type: stat.isDirectory() ? 'folder' : 'file', bytes: stat.size });
+        if (stat.isDirectory()) walk(file);
       }
     };
-    checked(base); walk(base, 0); return { entries, truncated };
+    checked(base); walk(base);
+    const nextOffset = offset + entries.length < visited ? offset + entries.length : null;
+    return { entries, totalFiles, totalEntries: visited, nextOffset, truncated: nextOffset !== null };
   }
-  read(relative, encoding = 'utf8') {
+  revision(relative) {
+    const file = this.resolve(relative); const hash = createHash('sha256');
+    const fd = fs.openSync(file, 'r'); const buffer = Buffer.alloc(64 * 1024);
+    try { let count; let position = 0; while ((count = fs.readSync(fd, buffer, 0, buffer.length, position)) > 0) { hash.update(buffer.subarray(0, count)); position += count; } }
+    finally { fs.closeSync(fd); }
+    return hash.digest('hex');
+  }
+  read(relative, encoding = 'utf8', { offset = 0, limit = PAGE_BYTES, revision } = {}) {
     this.ready(); const file = this.resolve(relative);
     if (!['utf8', 'base64'].includes(encoding)) throw new Error('Encoding must be utf8 or base64');
-    if (fs.statSync(file).size > MAX_BYTES) throw new Error('File exceeds 2 MiB bridge limit');
-    const data = fs.readFileSync(file);
-    return { path: relative, content: data.toString(encoding), encoding, revision: digest(data) };
+    if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 4 || limit > PAGE_BYTES) throw new Error('Invalid read page');
+    const current = this.revision(relative);
+    if (revision && revision !== current) throw new Error('Revision conflict: restart reading the changed file');
+    const bytes = fs.statSync(file).size;
+    if (offset > bytes) throw new Error('Read offset exceeds file size');
+    const fd = fs.openSync(file, 'r'); const buffer = Buffer.alloc(limit + 4); let count;
+    try { count = fs.readSync(fd, buffer, 0, buffer.length, offset); } finally { fs.closeSync(fd); }
+    let end = Math.min(limit, count);
+    // Do not split a UTF-8 code point between pages. Offsets are always bytes.
+    if (encoding === 'utf8' && end < count) while (end > 0 && (buffer[end] & 0xc0) === 0x80) end--;
+    const nextOffset = offset + end < bytes ? offset + end : null;
+    return { path: relative, encoding, revision: current, offset, bytes, nextOffset, complete: nextOffset === null, content: buffer.subarray(0, end).toString(encoding) };
   }
   write(relative, content, revision, encoding = 'utf8') {
     this.ready(); const file = this.resolve(relative);
@@ -117,7 +136,7 @@ export class ObsidianVault {
     const data = Buffer.from(content, encoding);
     if (data.length > MAX_BYTES) throw new Error('File exceeds 2 MiB bridge limit');
     if (fs.existsSync(file)) {
-      if (!revision || this.read(relative, 'base64').revision !== revision) throw new Error('Revision conflict: read the current file before replacing it');
+      if (!revision || this.revision(relative) !== revision) throw new Error('Revision conflict: read the current file before replacing it');
     } else if (revision) throw new Error('Revision conflict: file no longer exists');
     fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
     const temp = checked(path.join(path.dirname(file), `.jarvis-write-${randomUUID()}`));
@@ -129,7 +148,7 @@ export class ObsidianVault {
   move(relative, destination, revision) {
     this.ready(); const source = this.resolve(relative); const dest = this.resolve(destination);
     if (!fs.statSync(source).isFile()) throw new Error('Move individual files; remove empty folders separately');
-    if (!revision || this.read(relative, 'base64').revision !== revision) throw new Error('Revision conflict: read before moving');
+    if (!revision || this.revision(relative) !== revision) throw new Error('Revision conflict: read before moving');
     if (fs.existsSync(dest)) throw new Error('Destination already exists');
     fs.mkdirSync(path.dirname(dest), { recursive: true, mode: 0o700 }); fs.renameSync(source, dest);
     return { path: destination, revision, linksUpdated: false };
@@ -137,7 +156,7 @@ export class ObsidianVault {
   remove(relative, revision) {
     this.ready(); const file = this.resolve(relative);
     if (fs.statSync(file).isDirectory()) { fs.rmdirSync(file); return { removedEmptyFolder: relative }; }
-    if (!revision || this.read(relative, 'base64').revision !== revision) throw new Error('Revision conflict: read before deleting');
+    if (!revision || this.revision(relative) !== revision) throw new Error('Revision conflict: read before deleting');
     const id = randomUUID(); const trash = this.resolve(`.trash/${id}`, { internal: true });
     fs.mkdirSync(trash, { recursive: true, mode: 0o700 });
     fs.writeFileSync(path.join(trash, 'metadata.json'), JSON.stringify({ path: relative, deletedAt: new Date().toISOString() }), { flag: 'wx', mode: 0o600 });
@@ -160,33 +179,48 @@ export class ObsidianVault {
     fs.unlinkSync(this.resolve(`.trash/${id}/metadata.json`, { internal: true })); fs.rmdirSync(this.resolve(`.trash/${id}`, { internal: true }));
     return { path: relative };
   }
-  search(query, { memory = false } = {}) {
+  search(query, { memory = false, offset = 0 } = {}) {
     if (typeof query !== 'string' || query.length > 500) throw new Error('Query must be text up to 500 characters');
     this.ready();
-    if (memory && !fs.existsSync(this.resolve('memory'))) return { matches: [], truncated: false };
-    const listing = this.list(memory ? 'memory' : ''); const matches = []; let scannedBytes = 0; let truncated = listing.truncated;
+    if (memory && !this.memoryEnabled) throw new Error('Vault memory access is disabled');
+    // Durable knowledge can live anywhere in the vault, not only memory/.
+    const listing = this.list('', { offset }); const matches = [];
     for (const entry of listing.entries) {
-      if (entry.type !== 'file' || !entry.path.toLowerCase().endsWith('.md') || entry.bytes > MAX_BYTES) continue;
-      if ((scannedBytes += entry.bytes) > 16 * MAX_BYTES) { truncated = true; break; }
-      const note = this.read(entry.path); const index = note.content.toLowerCase().indexOf(query.toLowerCase());
-      if (index >= 0 || entry.path.toLowerCase().includes(query.toLowerCase())) matches.push({ path: entry.path, revision: note.revision, snippet: note.content.slice(Math.max(0, index - 100), Math.max(0, index) + 500) });
-      if (matches.length >= 50) { truncated = true; break; }
+      if (entry.type !== 'file' || !entry.path.toLowerCase().endsWith('.md')) continue;
+      const needle = query.toLowerCase(); let tail = '';
+      const fd = fs.openSync(this.resolve(entry.path), 'r'); const buffer = Buffer.alloc(64 * 1024);
+      try {
+        let count; let position = 0;
+        do {
+          count = fs.readSync(fd, buffer, 0, buffer.length, position);
+          position += count;
+          // Search bytes so even a multibyte character spanning chunks is preserved.
+          const text = tail + buffer.subarray(0, count).toString('latin1');
+          const decoded = Buffer.from(text, 'latin1').toString('utf8');
+          const index = decoded.toLowerCase().indexOf(needle);
+          if (index >= 0 || entry.path.toLowerCase().includes(needle)) {
+            matches.push({ path: entry.path, revision: this.revision(entry.path), snippet: decoded.slice(Math.max(0, index - 40), Math.max(0, index) + 160) }); break;
+          }
+          tail = text.slice(-Math.max(4, Buffer.byteLength(query) + 4));
+        } while (count > 0);
+      } finally { fs.closeSync(fd); }
+
     }
-    return { matches, truncated };
+    return { matches, nextOffset: listing.nextOffset, truncated: listing.truncated };
   }
   execute(args = {}) {
     switch (args.action) {
       case 'status': return this.status();
-      case 'list': return this.list(args.path);
-      case 'read': return this.read(args.path, args.encoding);
+      case 'list': return this.list(args.path, args);
+      case 'read': return this.read(args.path, args.encoding, args);
       case 'write': return this.write(args.path, args.content, args.revision, args.encoding);
       case 'mkdir': return this.mkdir(args.path);
       case 'move': return this.move(args.path, args.destination, args.revision);
       case 'delete': return this.remove(args.path, args.revision);
       case 'trash': return this.trash();
       case 'restore': return this.restore(args.trashId, args.destination);
-      case 'search': return this.search(args.query);
-      case 'memory_search': return this.search(args.query, { memory: true });
+      case 'search': return this.search(args.query, args);
+      case 'memory_search': return this.search(args.query, { ...args, memory: true });
       default: throw new Error('Unknown vault action');
     }
   }