Fixes

computer-use/atspi.js

@@ -1,9 +1,29 @@
 import { spawn } from 'node:child_process';
 
-export class AtspiProvider {
-  constructor({ python = 'python3', helper = new URL('./py/atspi_snapshot.py', import.meta.url).pathname, spawnImpl = spawn } = {}) { this.python = python; this.helper = helper; this.spawnImpl = spawnImpl; }
-  async tree({ focusedOnly = true, maxNodes = 400 } = {}) {
-    return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper, focusedOnly ? 'focused' : 'desktop', String(Math.min(400, maxNodes))], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => { if (code !== 0) return reject(new Error(err || `AT-SPI exited ${code}`)); try { resolve(JSON.parse(out)); } catch (error) { reject(error); } }); });
-  }
-  async action(target, action) { return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper.replace('atspi_snapshot.py', 'atspi_action.py'), JSON.stringify(target), String(action)], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => { if (code !== 0) return reject(new Error(err || `AT-SPI action exited ${code}`)); try { resolve(JSON.parse(out)); } catch (error) { reject(error); } }); }); }
+function runHelper({ spawnImpl, python, helper, args, timeoutMs, label }) {
+  return new Promise((resolve, reject) => {
+    const child = spawnImpl(python, [helper, ...args], { stdio: ['ignore', 'pipe', 'pipe'] });
+    let out = ''; let err = ''; let settled = false;
+    const timer = timeoutMs > 0 ? setTimeout(() => { if (settled) return; settled = true; child.kill('SIGTERM'); setTimeout(() => child.kill('SIGKILL'), 500); reject(new Error(`${label} timed out`)); }, timeoutMs) : null;
+    const done = (fn) => (value) => { if (settled) return; settled = true; if (timer) clearTimeout(timer); fn(value); };
+    child.stdout.on('data', (d) => { out += d; });
+    child.stderr.on('data', (d) => { err += d; });
+    child.on('error', done(reject));
+    child.on('close', (code) => done(code === 0 ? resolve : reject)(code === 0 ? out : new Error(err.trim() || `${label} exited ${code}`)));
+  });
+}
+
+export class AtspiProvider {
+  constructor({ python = 'python3', helper = new URL('./py/atspi_snapshot.py', import.meta.url).pathname, spawnImpl = spawn, timeoutMs = 5000 } = {}) {
+    this.python = python; this.helper = helper; this.spawnImpl = spawnImpl; this.timeoutMs = timeoutMs;
+  }
+  async tree({ focusedOnly = true, maxNodes = 400 } = {}) {
+    const out = await runHelper({ spawnImpl: this.spawnImpl, python: this.python, helper: this.helper, args: [focusedOnly ? 'focused' : 'desktop', String(Math.min(400, maxNodes))], timeoutMs: this.timeoutMs, label: 'AT-SPI' });
+    return JSON.parse(String(out));
+  }
+  async action(target, action) {
+    const helper = this.helper.replace('atspi_snapshot.py', 'atspi_action.py');
+    const out = await runHelper({ spawnImpl: this.spawnImpl, python: this.python, helper, args: [JSON.stringify(target), String(action)], timeoutMs: this.timeoutMs, label: 'AT-SPI action' });
+    return JSON.parse(String(out));
+  }
 }

computer-use/observer.js

@@ -24,22 +24,64 @@ export class DesktopObserver {
   constructor({ screenshot = new PortalScreenshot(), normalizer = new FrameNormalizer(), atspi = new AtspiProvider(), shell = new ShellProvider(), ocr, vision, tmpDir = '/tmp/jarvis-cu', timeouts = {} } = {}) {
     this.screenshot = screenshot; this.normalizer = normalizer; this.atspi = atspi; this.shell = shell; this.ocr = ocr; this.vision = vision; this.tmpDir = tmpDir;
     this.lastTree = [];
-    this.timeouts = { screenshot: timeouts.screenshot ?? 8000, tree: timeouts.tree ?? 5000, ocr: timeouts.ocr ?? 8000, vision: timeouts.vision ?? 8000 };
+    this.timeouts = { screenshot: timeouts.screenshot ?? 8000, tree: timeouts.tree ?? 5000, shell: timeouts.shell ?? 2000, ocr: timeouts.ocr ?? 8000, vision: timeouts.vision ?? 8000 };
   }
-  async tree({ focusedOnly = true, maxNodes = 400 } = {}) { const nodes = await timed(this.atspi.tree({ focusedOnly, maxNodes }), this.timeouts.tree, 'AT-SPI'); this.lastTree = nodes.map((node, index) => ({ ...node, ref: `r${index + 1}` })); return this.lastTree; }
+
+  async tree({ focusedOnly = true, maxNodes = 400 } = {}) {
+    const nodes = await timed(this.atspi.tree({ focusedOnly, maxNodes }), this.timeouts.tree, 'AT-SPI');
+    this.lastTree = nodes.map((node, index) => ({ ...node, ref: `r${index + 1}` }));
+    return this.lastTree;
+  }
+
+  async _captureFrame() {
+    const rawPath = await this.screenshot.capture(path.join(this.tmpDir, `observe-${Date.now()}.png`));
+    return this.normalizer.normalize(rawPath);
+  }
+
+  async _shellSnapshot() {
+    if (this.shell.connect) await this.shell.connect();
+    const [windows, focused] = await Promise.all([this.shell.windows(), this.shell.focused()]);
+    return { windows, focused };
+  }
+
   async observe({ includeTree = true, includeOcr = false, includeVision = false } = {}) {
     await mkdir(this.tmpDir, { recursive: true });
     const unavailable = [];
-    let rawPath; let frame = { path: null };
-    try { rawPath = await timed(this.screenshot.capture(path.join(this.tmpDir, `observe-${Date.now()}.png`)), this.timeouts.screenshot, 'screenshot'); frame = await this.normalizer.normalize(rawPath); } catch (error) { unavailable.push(`screenshot: ${error.message}`); }
-    await this.shell.connect?.().catch((error) => unavailable.push(`Shell helper: ${error.message}`));
-    const [windows, focused, tree] = await Promise.all([this.shell.windows(), this.shell.focused(), includeTree ? this.tree().catch((error) => { unavailable.push(`AT-SPI: ${error.message}`); return []; }) : Promise.resolve([])]);
-    const result = { monitor: null, focused, windows: windows.windows || [], tree, screenshot_path: frame.path, ocr_blocks: [], vision_hint: null, unavailable };
-    if (!windows.available) result.unavailable.push(windows.reason);
-    if (frame.path && includeOcr && this.ocr) result.ocr_blocks = await timed(this.ocr(frame.path), this.timeouts.ocr, 'OCR').catch((error) => { result.unavailable.push(`OCR: ${error.message}`); return []; });
-    if (frame.path && includeVision && this.vision) result.vision_hint = await timed(this.vision(frame.path), this.timeouts.vision, 'vision').catch((error) => { result.unavailable.push(`vision: ${error.message}`); return null; });
+    const note = (label) => (error) => { unavailable.push(`${label}: ${error.message}`); return null; };
+    const [frame, shell, tree] = await Promise.all([
+      timed(this._captureFrame(), this.timeouts.screenshot, 'screenshot').catch(note('screenshot')),
+      timed(this._shellSnapshot(), this.timeouts.shell, 'Shell helper').catch(note('Shell helper')),
+      includeTree ? this.tree().catch((error) => { unavailable.push(`AT-SPI: ${error.message}`); return []; }) : Promise.resolve([]),
+    ]);
+    const windows = shell?.windows || { available: false, windows: [] };
+    const result = {
+      monitor: null,
+      focused: shell?.focused || null,
+      windows: windows.windows || [],
+      tree,
+      screenshot_path: frame?.path || null,
+      ocr_blocks: [],
+      vision_hint: null,
+      unavailable,
+    };
+    if (shell && !windows.available && windows.reason) result.unavailable.push(windows.reason);
+    if (result.screenshot_path && includeOcr && this.ocr) result.ocr_blocks = await timed(this.ocr(result.screenshot_path), this.timeouts.ocr, 'OCR').catch((error) => { result.unavailable.push(`OCR: ${error.message}`); return []; });
+    if (result.screenshot_path && includeVision && this.vision) result.vision_hint = await timed(this.vision(result.screenshot_path), this.timeouts.vision, 'vision').catch((error) => { result.unavailable.push(`vision: ${error.message}`); return null; });
     return result;
   }
-  async zoom({ rect, ref } = {}) { await mkdir(this.tmpDir, { recursive: true }); const node = ref ? this.lastTree.find((item) => item.ref === ref) : null; const target = rect || node?.rect; if (!target) throw new Error('rect or current tree ref is required'); const raw = await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`)); const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), target); return { rect: target, screenshot_path: frame.path }; }
-  async find({ query, role } = {}) { const nodes = this.lastTree.length ? this.lastTree : await this.tree(); return nodes.map((node) => ({ ...node, score: score(query, node) })).filter((node) => node.score && (!role || normalize(node.role) === normalize(role))).sort((a, b) => b.score - a.score).slice(0, 20); }
+
+  async zoom({ rect, ref } = {}) {
+    await mkdir(this.tmpDir, { recursive: true });
+    const node = ref ? this.lastTree.find((item) => item.ref === ref) : null;
+    const target = rect || node?.rect;
+    if (!target) throw new Error('rect or current tree ref is required');
+    const raw = await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`));
+    const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), target);
+    return { rect: target, screenshot_path: frame.path };
+  }
+
+  async find({ query, role } = {}) {
+    const nodes = this.lastTree.length ? this.lastTree : await this.tree();
+    return nodes.map((node) => ({ ...node, score: score(query, node) })).filter((node) => node.score && (!role || normalize(node.role) === normalize(role))).sort((a, b) => b.score - a.score).slice(0, 20);
+  }
 }

computer-use/py/atspi_snapshot.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 import json
 import sys
+import gi
+gi.require_version('Atspi', '2.0')
 from gi.repository import Atspi
 
 mode, limit = sys.argv[1], int(sys.argv[2])

computer-use/py/portal_screenshot.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Capture one frame through GNOME Shell or the Screenshot portal and print its path."""
+"""Capture one frame through the Jarvis GNOME Shell helper, then the Screenshot portal."""
 import sys
 from gi.repository import Gio, GLib
 
@@ -7,22 +7,22 @@ target = sys.argv[1]
 bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
 
 
-def gnome_shell_screenshot():
+def jarvis_shell_screenshot():
     proxy = Gio.DBusProxy.new_sync(
         bus, Gio.DBusProxyFlags.NONE, None,
-        'org.gnome.Shell.Screenshot', '/org/gnome/Shell/Screenshot',
-        'org.gnome.Shell.Screenshot', None,
+        'io.qvac.Jarvis.Shell', '/io/qvac/Jarvis/Shell',
+        'io.qvac.Jarvis.Shell', None,
     )
-    ok, path = proxy.call_sync(
+    path = proxy.call_sync(
         'Screenshot',
-        GLib.Variant('(bbs)', (False, False, target)),
+        GLib.Variant('(s)', (target,)),
         Gio.DBusCallFlags.NONE,
         5000,
         None,
-    ).unpack()
-    if not ok:
-        raise RuntimeError('GNOME Shell screenshot failed')
-    return path or target
+    ).unpack()[0]
+    if not path:
+        raise RuntimeError('Jarvis Shell screenshot returned no path')
+    return path
 
 
 def portal_screenshot():
@@ -46,7 +46,7 @@ def portal_screenshot():
         loop.quit()
 
     sub = bus.signal_subscribe(None, 'org.freedesktop.portal.Request', 'Response', request, None, Gio.DBusSignalFlags.NONE, response)
-    GLib.timeout_add(7000, loop.quit)
+    GLib.timeout_add(4000, loop.quit)
     loop.run()
     bus.signal_unsubscribe(sub)
     if not result['uri']:
@@ -59,7 +59,7 @@ def portal_screenshot():
 
 
 errors = []
-for capture in (gnome_shell_screenshot, portal_screenshot):
+for capture in (jarvis_shell_screenshot, portal_screenshot):
     try:
         print(capture())
         raise SystemExit(0)

computer-use/shell-provider.js

@@ -1,7 +1,55 @@
-export class ShellProvider {
-  constructor({ listWindows, focusedWindow, focusWindow, proxy } = {}) { this.listWindowsImpl = listWindows; this.focusedWindowImpl = focusedWindow; this.focusWindowImpl = focusWindow; this.proxy = proxy; }
-  async connect() { if (this.proxy) return this; const dbus = await import('dbus-next'); const bus = dbus.sessionBus(); const object = await bus.getProxyObject('io.qvac.Jarvis.Shell', '/io/qvac/Jarvis/Shell'); this.proxy = object.getInterface('io.qvac.Jarvis.Shell'); return this; }
-  async windows() { if (this.proxy) return { available: true, windows: JSON.parse(await this.proxy.ListWindows()) }; if (!this.listWindowsImpl) return { available: false, reason: 'GNOME Shell helper is not connected', windows: [] }; return { available: true, windows: await this.listWindowsImpl() }; }
-  async focused() { if (this.proxy) return JSON.parse(await this.proxy.FocusedWindow()); if (!this.focusedWindowImpl) return null; return this.focusedWindowImpl(); }
-  async focus(id) { if (this.proxy) return this.proxy.FocusWindow(BigInt(id)); if (!this.focusWindowImpl) throw new Error('GNOME Shell focus helper is unavailable'); return this.focusWindowImpl(id); }
+import { spawn } from 'node:child_process';
+
+const DEST = 'io.qvac.Jarvis.Shell';
+const PATH = '/io/qvac/Jarvis/Shell';
+const IFACE = 'io.qvac.Jarvis.Shell';
+
+function parseGdbusString(out) {
+  const match = String(out).match(/'((?:\\.|[^'\\])*)'/);
+  if (!match) throw new Error('GNOME Shell helper returned no string');
+  return match[1].replace(/\\(.)/g, '$1');
+}
+
+export class ShellProvider {
+  constructor({ listWindows, focusedWindow, focusWindow, proxy, spawnImpl = spawn, timeoutMs = 2000 } = {}) {
+    this.listWindowsImpl = listWindows; this.focusedWindowImpl = focusedWindow; this.focusWindowImpl = focusWindow; this.proxy = proxy;
+    this.spawnImpl = spawnImpl; this.timeoutMs = timeoutMs;
+  }
+
+  _call(method, args = []) {
+    return new Promise((resolve, reject) => {
+      const child = this.spawnImpl('gdbus', ['call', '--session', '--dest', DEST, '--object-path', PATH, '--method', `${IFACE}.${method}`, ...args], { stdio: ['ignore', 'pipe', 'pipe'] });
+      let out = ''; let err = ''; let settled = false;
+      const timer = setTimeout(() => { if (settled) return; settled = true; child.kill('SIGTERM'); reject(new Error('GNOME Shell helper timed out')); }, this.timeoutMs);
+      const done = (fn) => (value) => { if (settled) return; settled = true; clearTimeout(timer); fn(value); };
+      child.stdout.on('data', (d) => { out += d; });
+      child.stderr.on('data', (d) => { err += d; });
+      child.on('error', done(reject));
+      child.on('close', (code) => done(code === 0 ? resolve : reject)(code === 0 ? parseGdbusString(out) : new Error(err.trim() || `GNOME Shell helper exited ${code}`)));
+    });
+  }
+
+  async connect() {
+    if (this.proxy || this.listWindowsImpl) return this;
+    await this._call('ListWindows');
+    return this;
+  }
+
+  async windows() {
+    if (this.proxy) return { available: true, windows: JSON.parse(await this.proxy.ListWindows()) };
+    if (this.listWindowsImpl) return { available: true, windows: await this.listWindowsImpl() };
+    try { return { available: true, windows: JSON.parse(await this._call('ListWindows')) }; } catch (error) { return { available: false, reason: error.message, windows: [] }; }
+  }
+
+  async focused() {
+    if (this.proxy) return JSON.parse(await this.proxy.FocusedWindow());
+    if (this.focusedWindowImpl) return this.focusedWindowImpl();
+    try { return JSON.parse(await this._call('FocusedWindow')); } catch { return null; }
+  }
+
+  async focus(id) {
+    if (this.proxy) return this.proxy.FocusWindow(BigInt(id));
+    if (this.focusWindowImpl) return this.focusWindowImpl(id);
+    await this._call('FocusWindow', [String(id)]);
+  }
 }