Fixes

apps/gnome-extension/jarvis@qvac.local/shell-dbus.js

@@ -1,9 +1,10 @@
 import Gio from 'gi://Gio';
 import Meta from 'gi://Meta';
+import Shell from 'gi://Shell';
 
 const BUS = 'io.qvac.Jarvis.Shell';
 const PATH = '/io/qvac/Jarvis/Shell';
-const XML = `<node><interface name="io.qvac.Jarvis.Shell"><method name="ListWindows"><arg type="s" direction="out"/></method><method name="FocusedWindow"><arg type="s" direction="out"/></method><method name="FocusWindow"><arg type="t" direction="in"/></method></interface></node>`;
+const XML = `<node><interface name="io.qvac.Jarvis.Shell"><method name="ListWindows"><arg type="s" direction="out"/></method><method name="FocusedWindow"><arg type="s" direction="out"/></method><method name="FocusWindow"><arg type="t" direction="in"/></method><method name="Screenshot"><arg type="s" direction="in"/><arg type="s" direction="out"/></method></interface></node>`;
 const describe = (window) => ({ id: Number(window.get_id?.() || 0), title: window.get_title?.() || '', wm_class: window.get_wm_class?.() || '', pid: Number(window.get_pid?.() || 0), rect: (() => { const r = window.get_frame_rect?.(); return r ? [r.x, r.y, r.width, r.height] : null; })(), focused: window === global.display.get_focus_window?.() });
 
 export function installShellService() {
@@ -11,6 +12,17 @@ export function installShellService() {
     ListWindows() { const windows = global.display.get_tab_list(Meta.TabList.NORMAL_ALL, null).map(describe); return JSON.stringify(windows); },
     FocusedWindow() { const window = global.display.get_focus_window?.(); return JSON.stringify(window ? describe(window) : null); },
     FocusWindow(id) { const window = global.display.get_tab_list(Meta.TabList.NORMAL_ALL, null).find((candidate) => Number(candidate.get_id?.()) === Number(id)); if (!window) throw new Error('window not found'); window.activate(global.get_current_time()); },
+    async Screenshot(filename) {
+      const file = Gio.File.new_for_path(filename);
+      const stream = file.replace(null, false, Gio.FileCreateFlags.REPLACE_DESTINATION, null);
+      try {
+        const screenshot = new Shell.Screenshot();
+        await screenshot.screenshot(false, stream);
+      } finally {
+        stream.close(null);
+      }
+      return filename;
+    },
   };
   const ownerId = Gio.bus_own_name(Gio.BusType.SESSION, BUS, Gio.BusNameOwnerFlags.NONE, (connection) => { const exported = Gio.DBusExportedObject.wrapJSObject(XML, implementation); exported.export(connection, PATH); implementation._exported = exported; }, null, () => {});
   return () => { implementation._exported?.unexport(); Gio.bus_unown_name(ownerId); };

computer-use/atspi.js

@@ -1,9 +1,29 @@
 import { spawn } from 'node:child_process';
 
+function runHelper({ spawnImpl, python, helper, args, timeoutMs, label }) {
+  return new Promise((resolve, reject) => {
+    const child = spawnImpl(python, [helper, ...args], { stdio: ['ignore', 'pipe', 'pipe'] });
+    let out = ''; let err = ''; let settled = false;
+    const timer = timeoutMs > 0 ? setTimeout(() => { if (settled) return; settled = true; child.kill('SIGTERM'); setTimeout(() => child.kill('SIGKILL'), 500); reject(new Error(`${label} timed out`)); }, timeoutMs) : null;
+    const done = (fn) => (value) => { if (settled) return; settled = true; if (timer) clearTimeout(timer); fn(value); };
+    child.stdout.on('data', (d) => { out += d; });
+    child.stderr.on('data', (d) => { err += d; });
+    child.on('error', done(reject));
+    child.on('close', (code) => done(code === 0 ? resolve : reject)(code === 0 ? out : new Error(err.trim() || `${label} exited ${code}`)));
+  });
+}
+
 export class AtspiProvider {
-  constructor({ python = 'python3', helper = new URL('./py/atspi_snapshot.py', import.meta.url).pathname, spawnImpl = spawn } = {}) { this.python = python; this.helper = helper; this.spawnImpl = spawnImpl; }
+  constructor({ python = 'python3', helper = new URL('./py/atspi_snapshot.py', import.meta.url).pathname, spawnImpl = spawn, timeoutMs = 5000 } = {}) {
+    this.python = python; this.helper = helper; this.spawnImpl = spawnImpl; this.timeoutMs = timeoutMs;
+  }
   async tree({ focusedOnly = true, maxNodes = 400 } = {}) {
-    return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper, focusedOnly ? 'focused' : 'desktop', String(Math.min(400, maxNodes))], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => { if (code !== 0) return reject(new Error(err || `AT-SPI exited ${code}`)); try { resolve(JSON.parse(out)); } catch (error) { reject(error); } }); });
+    const out = await runHelper({ spawnImpl: this.spawnImpl, python: this.python, helper: this.helper, args: [focusedOnly ? 'focused' : 'desktop', String(Math.min(400, maxNodes))], timeoutMs: this.timeoutMs, label: 'AT-SPI' });
+    return JSON.parse(String(out));
+  }
+  async action(target, action) {
+    const helper = this.helper.replace('atspi_snapshot.py', 'atspi_action.py');
+    const out = await runHelper({ spawnImpl: this.spawnImpl, python: this.python, helper, args: [JSON.stringify(target), String(action)], timeoutMs: this.timeoutMs, label: 'AT-SPI action' });
+    return JSON.parse(String(out));
   }
-  async action(target, action) { return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper.replace('atspi_snapshot.py', 'atspi_action.py'), JSON.stringify(target), String(action)], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => { if (code !== 0) return reject(new Error(err || `AT-SPI action exited ${code}`)); try { resolve(JSON.parse(out)); } catch (error) { reject(error); } }); }); }
 }

computer-use/observer.js

@@ -24,22 +24,64 @@ export class DesktopObserver {
   constructor({ screenshot = new PortalScreenshot(), normalizer = new FrameNormalizer(), atspi = new AtspiProvider(), shell = new ShellProvider(), ocr, vision, tmpDir = '/tmp/jarvis-cu', timeouts = {} } = {}) {
     this.screenshot = screenshot; this.normalizer = normalizer; this.atspi = atspi; this.shell = shell; this.ocr = ocr; this.vision = vision; this.tmpDir = tmpDir;
     this.lastTree = [];
-    this.timeouts = { screenshot: timeouts.screenshot ?? 8000, tree: timeouts.tree ?? 5000, ocr: timeouts.ocr ?? 8000, vision: timeouts.vision ?? 8000 };
+    this.timeouts = { screenshot: timeouts.screenshot ?? 8000, tree: timeouts.tree ?? 5000, shell: timeouts.shell ?? 2000, ocr: timeouts.ocr ?? 8000, vision: timeouts.vision ?? 8000 };
   }
-  async tree({ focusedOnly = true, maxNodes = 400 } = {}) { const nodes = await timed(this.atspi.tree({ focusedOnly, maxNodes }), this.timeouts.tree, 'AT-SPI'); this.lastTree = nodes.map((node, index) => ({ ...node, ref: `r${index + 1}` })); return this.lastTree; }
+
+  async tree({ focusedOnly = true, maxNodes = 400 } = {}) {
+    const nodes = await timed(this.atspi.tree({ focusedOnly, maxNodes }), this.timeouts.tree, 'AT-SPI');
+    this.lastTree = nodes.map((node, index) => ({ ...node, ref: `r${index + 1}` }));
+    return this.lastTree;
+  }
+
+  async _captureFrame() {
+    const rawPath = await this.screenshot.capture(path.join(this.tmpDir, `observe-${Date.now()}.png`));
+    return this.normalizer.normalize(rawPath);
+  }
+
+  async _shellSnapshot() {
+    if (this.shell.connect) await this.shell.connect();
+    const [windows, focused] = await Promise.all([this.shell.windows(), this.shell.focused()]);
+    return { windows, focused };
+  }
+
   async observe({ includeTree = true, includeOcr = false, includeVision = false } = {}) {
     await mkdir(this.tmpDir, { recursive: true });
     const unavailable = [];
-    let rawPath; let frame = { path: null };
+    const note = (label) => (error) => { unavailable.push(`${label}: ${error.message}`); return null; };
-    try { rawPath = await timed(this.screenshot.capture(path.join(this.tmpDir, `observe-${Date.now()}.png`)), this.timeouts.screenshot, 'screenshot'); frame = await this.normalizer.normalize(rawPath); } catch (error) { unavailable.push(`screenshot: ${error.message}`); }
+    const [frame, shell, tree] = await Promise.all([
-    await this.shell.connect?.().catch((error) => unavailable.push(`Shell helper: ${error.message}`));
+      timed(this._captureFrame(), this.timeouts.screenshot, 'screenshot').catch(note('screenshot')),
-    const [windows, focused, tree] = await Promise.all([this.shell.windows(), this.shell.focused(), includeTree ? this.tree().catch((error) => { unavailable.push(`AT-SPI: ${error.message}`); return []; }) : Promise.resolve([])]);
+      timed(this._shellSnapshot(), this.timeouts.shell, 'Shell helper').catch(note('Shell helper')),
-    const result = { monitor: null, focused, windows: windows.windows || [], tree, screenshot_path: frame.path, ocr_blocks: [], vision_hint: null, unavailable };
+      includeTree ? this.tree().catch((error) => { unavailable.push(`AT-SPI: ${error.message}`); return []; }) : Promise.resolve([]),
-    if (!windows.available) result.unavailable.push(windows.reason);
+    ]);
-    if (frame.path && includeOcr && this.ocr) result.ocr_blocks = await timed(this.ocr(frame.path), this.timeouts.ocr, 'OCR').catch((error) => { result.unavailable.push(`OCR: ${error.message}`); return []; });
+    const windows = shell?.windows || { available: false, windows: [] };
-    if (frame.path && includeVision && this.vision) result.vision_hint = await timed(this.vision(frame.path), this.timeouts.vision, 'vision').catch((error) => { result.unavailable.push(`vision: ${error.message}`); return null; });
+    const result = {
+      monitor: null,
+      focused: shell?.focused || null,
+      windows: windows.windows || [],
+      tree,
+      screenshot_path: frame?.path || null,
+      ocr_blocks: [],
+      vision_hint: null,
+      unavailable,
+    };
+    if (shell && !windows.available && windows.reason) result.unavailable.push(windows.reason);
+    if (result.screenshot_path && includeOcr && this.ocr) result.ocr_blocks = await timed(this.ocr(result.screenshot_path), this.timeouts.ocr, 'OCR').catch((error) => { result.unavailable.push(`OCR: ${error.message}`); return []; });
+    if (result.screenshot_path && includeVision && this.vision) result.vision_hint = await timed(this.vision(result.screenshot_path), this.timeouts.vision, 'vision').catch((error) => { result.unavailable.push(`vision: ${error.message}`); return null; });
     return result;
   }
-  async zoom({ rect, ref } = {}) { await mkdir(this.tmpDir, { recursive: true }); const node = ref ? this.lastTree.find((item) => item.ref === ref) : null; const target = rect || node?.rect; if (!target) throw new Error('rect or current tree ref is required'); const raw = await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`)); const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), target); return { rect: target, screenshot_path: frame.path }; }
+
-  async find({ query, role } = {}) { const nodes = this.lastTree.length ? this.lastTree : await this.tree(); return nodes.map((node) => ({ ...node, score: score(query, node) })).filter((node) => node.score && (!role || normalize(node.role) === normalize(role))).sort((a, b) => b.score - a.score).slice(0, 20); }
+  async zoom({ rect, ref } = {}) {
+    await mkdir(this.tmpDir, { recursive: true });
+    const node = ref ? this.lastTree.find((item) => item.ref === ref) : null;
+    const target = rect || node?.rect;
+    if (!target) throw new Error('rect or current tree ref is required');
+    const raw = await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`));
+    const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), target);
+    return { rect: target, screenshot_path: frame.path };
+  }
+
+  async find({ query, role } = {}) {
+    const nodes = this.lastTree.length ? this.lastTree : await this.tree();
+    return nodes.map((node) => ({ ...node, score: score(query, node) })).filter((node) => node.score && (!role || normalize(node.role) === normalize(role))).sort((a, b) => b.score - a.score).slice(0, 20);
+  }
 }

computer-use/py/atspi_snapshot.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 import json
 import sys
+import gi
+gi.require_version('Atspi', '2.0')
 from gi.repository import Atspi
 
 mode, limit = sys.argv[1], int(sys.argv[2])

computer-use/py/portal_screenshot.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Capture one frame through GNOME Shell or the Screenshot portal and print its path."""
+"""Capture one frame through the Jarvis GNOME Shell helper, then the Screenshot portal."""
 import sys
 from gi.repository import Gio, GLib
 
@@ -7,22 +7,22 @@ target = sys.argv[1]
 bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
 
 
-def gnome_shell_screenshot():
+def jarvis_shell_screenshot():
     proxy = Gio.DBusProxy.new_sync(
         bus, Gio.DBusProxyFlags.NONE, None,
-        'org.gnome.Shell.Screenshot', '/org/gnome/Shell/Screenshot',
+        'io.qvac.Jarvis.Shell', '/io/qvac/Jarvis/Shell',
-        'org.gnome.Shell.Screenshot', None,
+        'io.qvac.Jarvis.Shell', None,
     )
-    ok, path = proxy.call_sync(
+    path = proxy.call_sync(
         'Screenshot',
-        GLib.Variant('(bbs)', (False, False, target)),
+        GLib.Variant('(s)', (target,)),
         Gio.DBusCallFlags.NONE,
         5000,
         None,
-    ).unpack()
+    ).unpack()[0]
-    if not ok:
+    if not path:
-        raise RuntimeError('GNOME Shell screenshot failed')
+        raise RuntimeError('Jarvis Shell screenshot returned no path')
-    return path or target
+    return path
 
 
 def portal_screenshot():
@@ -46,7 +46,7 @@ def portal_screenshot():
         loop.quit()
 
     sub = bus.signal_subscribe(None, 'org.freedesktop.portal.Request', 'Response', request, None, Gio.DBusSignalFlags.NONE, response)
-    GLib.timeout_add(7000, loop.quit)
+    GLib.timeout_add(4000, loop.quit)
     loop.run()
     bus.signal_unsubscribe(sub)
     if not result['uri']:
@@ -59,7 +59,7 @@ def portal_screenshot():
 
 
 errors = []
-for capture in (gnome_shell_screenshot, portal_screenshot):
+for capture in (jarvis_shell_screenshot, portal_screenshot):
     try:
         print(capture())
         raise SystemExit(0)

computer-use/shell-provider.js

@@ -1,7 +1,55 @@
-export class ShellProvider {
+import { spawn } from 'node:child_process';
-  constructor({ listWindows, focusedWindow, focusWindow, proxy } = {}) { this.listWindowsImpl = listWindows; this.focusedWindowImpl = focusedWindow; this.focusWindowImpl = focusWindow; this.proxy = proxy; }
+
-  async connect() { if (this.proxy) return this; const dbus = await import('dbus-next'); const bus = dbus.sessionBus(); const object = await bus.getProxyObject('io.qvac.Jarvis.Shell', '/io/qvac/Jarvis/Shell'); this.proxy = object.getInterface('io.qvac.Jarvis.Shell'); return this; }
+const DEST = 'io.qvac.Jarvis.Shell';
-  async windows() { if (this.proxy) return { available: true, windows: JSON.parse(await this.proxy.ListWindows()) }; if (!this.listWindowsImpl) return { available: false, reason: 'GNOME Shell helper is not connected', windows: [] }; return { available: true, windows: await this.listWindowsImpl() }; }
+const PATH = '/io/qvac/Jarvis/Shell';
-  async focused() { if (this.proxy) return JSON.parse(await this.proxy.FocusedWindow()); if (!this.focusedWindowImpl) return null; return this.focusedWindowImpl(); }
+const IFACE = 'io.qvac.Jarvis.Shell';
-  async focus(id) { if (this.proxy) return this.proxy.FocusWindow(BigInt(id)); if (!this.focusWindowImpl) throw new Error('GNOME Shell focus helper is unavailable'); return this.focusWindowImpl(id); }
+
+function parseGdbusString(out) {
+  const match = String(out).match(/'((?:\\.|[^'\\])*)'/);
+  if (!match) throw new Error('GNOME Shell helper returned no string');
+  return match[1].replace(/\\(.)/g, '$1');
+}
+
+export class ShellProvider {
+  constructor({ listWindows, focusedWindow, focusWindow, proxy, spawnImpl = spawn, timeoutMs = 2000 } = {}) {
+    this.listWindowsImpl = listWindows; this.focusedWindowImpl = focusedWindow; this.focusWindowImpl = focusWindow; this.proxy = proxy;
+    this.spawnImpl = spawnImpl; this.timeoutMs = timeoutMs;
+  }
+
+  _call(method, args = []) {
+    return new Promise((resolve, reject) => {
+      const child = this.spawnImpl('gdbus', ['call', '--session', '--dest', DEST, '--object-path', PATH, '--method', `${IFACE}.${method}`, ...args], { stdio: ['ignore', 'pipe', 'pipe'] });
+      let out = ''; let err = ''; let settled = false;
+      const timer = setTimeout(() => { if (settled) return; settled = true; child.kill('SIGTERM'); reject(new Error('GNOME Shell helper timed out')); }, this.timeoutMs);
+      const done = (fn) => (value) => { if (settled) return; settled = true; clearTimeout(timer); fn(value); };
+      child.stdout.on('data', (d) => { out += d; });
+      child.stderr.on('data', (d) => { err += d; });
+      child.on('error', done(reject));
+      child.on('close', (code) => done(code === 0 ? resolve : reject)(code === 0 ? parseGdbusString(out) : new Error(err.trim() || `GNOME Shell helper exited ${code}`)));
+    });
+  }
+
+  async connect() {
+    if (this.proxy || this.listWindowsImpl) return this;
+    await this._call('ListWindows');
+    return this;
+  }
+
+  async windows() {
+    if (this.proxy) return { available: true, windows: JSON.parse(await this.proxy.ListWindows()) };
+    if (this.listWindowsImpl) return { available: true, windows: await this.listWindowsImpl() };
+    try { return { available: true, windows: JSON.parse(await this._call('ListWindows')) }; } catch (error) { return { available: false, reason: error.message, windows: [] }; }
+  }
+
+  async focused() {
+    if (this.proxy) return JSON.parse(await this.proxy.FocusedWindow());
+    if (this.focusedWindowImpl) return this.focusedWindowImpl();
+    try { return JSON.parse(await this._call('FocusedWindow')); } catch { return null; }
+  }
+
+  async focus(id) {
+    if (this.proxy) return this.proxy.FocusWindow(BigInt(id));
+    if (this.focusWindowImpl) return this.focusWindowImpl(id);
+    await this._call('FocusWindow', [String(id)]);
+  }
 }

test/gnome-extension.test.js

@@ -506,3 +506,10 @@ test('both settings entry points use the shared preferences window', () => {
   assert.match(prefs, /fillSettingsWindow/);
   assert.match(control, /fillSettingsWindow/);
 });
+
+test('the Shell helper can capture a screenshot from inside GNOME', () => {
+  const source = readFileSync(new URL('../apps/gnome-extension/[email protected]/shell-dbus.js', import.meta.url), 'utf8');
+  assert.match(source, /method name="Screenshot"/);
+  assert.match(source, /new Shell\.Screenshot/);
+  assert.match(source, /screenshot\(false, stream\)/);
+});

test/observer.test.js

@@ -15,6 +15,22 @@ test('Phase 6 observer returns stable refs and ranked semantic matches', async (
   const zoom = await observer.zoom({ ref: 'r1' }); assert.deepEqual(zoom.rect, [10, 20, 30, 30]);
 });
 
+test('observer returns when the Shell helper never connects', async () => {
+  const observer = new DesktopObserver({
+    screenshot: { capture: async () => '/tmp/frame.png' },
+    normalizer: { normalize: async () => ({ path: '/tmp/frame.webp' }) },
+    shell: { connect: () => new Promise(() => {}), windows: async () => assert.fail('shell windows'), focused: async () => assert.fail('shell focused') },
+    atspi: { tree: async () => [{ role: 'label', name: 'Discord', rect: [0, 0, 10, 10] }] },
+    timeouts: { shell: 40 },
+  });
+  const started = Date.now();
+  const bundle = await observer.observe({ includeOcr: false });
+  assert.ok(Date.now() - started < 1000);
+  assert.equal(bundle.screenshot_path, '/tmp/frame.webp');
+  assert.equal(bundle.tree[0].name, 'Discord');
+  assert.match(bundle.unavailable.join(' '), /Shell helper/);
+});
+
 test('observer returns after a hung screenshot instead of staying on THINKING', async () => {
   const observer = new DesktopObserver({
     screenshot: { capture: () => new Promise(() => {}) },