diff --git a/apps/gnome-extension/jarvis@qvac.local/shell-dbus.js b/apps/gnome-extension/jarvis@qvac.local/shell-dbus.js index df9b05f..284f99c 100644 --- a/apps/gnome-extension/jarvis@qvac.local/shell-dbus.js +++ b/apps/gnome-extension/jarvis@qvac.local/shell-dbus.js @@ -1,9 +1,10 @@ import Gio from 'gi://Gio'; import Meta from 'gi://Meta'; +import Shell from 'gi://Shell'; const BUS = 'io.qvac.Jarvis.Shell'; const PATH = '/io/qvac/Jarvis/Shell'; -const XML = ``; +const XML = ``; const describe = (window) => ({ id: Number(window.get_id?.() || 0), title: window.get_title?.() || '', wm_class: window.get_wm_class?.() || '', pid: Number(window.get_pid?.() || 0), rect: (() => { const r = window.get_frame_rect?.(); return r ? [r.x, r.y, r.width, r.height] : null; })(), focused: window === global.display.get_focus_window?.() }); export function installShellService() { @@ -11,6 +12,17 @@ export function installShellService() { ListWindows() { const windows = global.display.get_tab_list(Meta.TabList.NORMAL_ALL, null).map(describe); return JSON.stringify(windows); }, FocusedWindow() { const window = global.display.get_focus_window?.(); return JSON.stringify(window ? describe(window) : null); }, FocusWindow(id) { const window = global.display.get_tab_list(Meta.TabList.NORMAL_ALL, null).find((candidate) => Number(candidate.get_id?.()) === Number(id)); if (!window) throw new Error('window not found'); window.activate(global.get_current_time()); }, + async Screenshot(filename) { + const file = Gio.File.new_for_path(filename); + const stream = file.replace(null, false, Gio.FileCreateFlags.REPLACE_DESTINATION, null); + try { + const screenshot = new Shell.Screenshot(); + await screenshot.screenshot(false, stream); + } finally { + stream.close(null); + } + return filename; + }, }; const ownerId = Gio.bus_own_name(Gio.BusType.SESSION, BUS, Gio.BusNameOwnerFlags.NONE, (connection) => { const exported = Gio.DBusExportedObject.wrapJSObject(XML, implementation); exported.export(connection, PATH); implementation._exported = exported; }, null, () => {}); return () => { implementation._exported?.unexport(); Gio.bus_unown_name(ownerId); }; diff --git a/computer-use/atspi.js b/computer-use/atspi.js index 3c139ef..81666f0 100644 --- a/computer-use/atspi.js +++ b/computer-use/atspi.js @@ -1,9 +1,29 @@ import { spawn } from 'node:child_process'; -export class AtspiProvider { - constructor({ python = 'python3', helper = new URL('./py/atspi_snapshot.py', import.meta.url).pathname, spawnImpl = spawn } = {}) { this.python = python; this.helper = helper; this.spawnImpl = spawnImpl; } - async tree({ focusedOnly = true, maxNodes = 400 } = {}) { - return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper, focusedOnly ? 'focused' : 'desktop', String(Math.min(400, maxNodes))], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => { if (code !== 0) return reject(new Error(err || `AT-SPI exited ${code}`)); try { resolve(JSON.parse(out)); } catch (error) { reject(error); } }); }); - } - async action(target, action) { return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper.replace('atspi_snapshot.py', 'atspi_action.py'), JSON.stringify(target), String(action)], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => { if (code !== 0) return reject(new Error(err || `AT-SPI action exited ${code}`)); try { resolve(JSON.parse(out)); } catch (error) { reject(error); } }); }); } +function runHelper({ spawnImpl, python, helper, args, timeoutMs, label }) { + return new Promise((resolve, reject) => { + const child = spawnImpl(python, [helper, ...args], { stdio: ['ignore', 'pipe', 'pipe'] }); + let out = ''; let err = ''; let settled = false; + const timer = timeoutMs > 0 ? setTimeout(() => { if (settled) return; settled = true; child.kill('SIGTERM'); setTimeout(() => child.kill('SIGKILL'), 500); reject(new Error(`${label} timed out`)); }, timeoutMs) : null; + const done = (fn) => (value) => { if (settled) return; settled = true; if (timer) clearTimeout(timer); fn(value); }; + child.stdout.on('data', (d) => { out += d; }); + child.stderr.on('data', (d) => { err += d; }); + child.on('error', done(reject)); + child.on('close', (code) => done(code === 0 ? resolve : reject)(code === 0 ? out : new Error(err.trim() || `${label} exited ${code}`))); + }); +} + +export class AtspiProvider { + constructor({ python = 'python3', helper = new URL('./py/atspi_snapshot.py', import.meta.url).pathname, spawnImpl = spawn, timeoutMs = 5000 } = {}) { + this.python = python; this.helper = helper; this.spawnImpl = spawnImpl; this.timeoutMs = timeoutMs; + } + async tree({ focusedOnly = true, maxNodes = 400 } = {}) { + const out = await runHelper({ spawnImpl: this.spawnImpl, python: this.python, helper: this.helper, args: [focusedOnly ? 'focused' : 'desktop', String(Math.min(400, maxNodes))], timeoutMs: this.timeoutMs, label: 'AT-SPI' }); + return JSON.parse(String(out)); + } + async action(target, action) { + const helper = this.helper.replace('atspi_snapshot.py', 'atspi_action.py'); + const out = await runHelper({ spawnImpl: this.spawnImpl, python: this.python, helper, args: [JSON.stringify(target), String(action)], timeoutMs: this.timeoutMs, label: 'AT-SPI action' }); + return JSON.parse(String(out)); + } } diff --git a/computer-use/observer.js b/computer-use/observer.js index d6623a9..dd8d196 100644 --- a/computer-use/observer.js +++ b/computer-use/observer.js @@ -24,22 +24,64 @@ export class DesktopObserver { constructor({ screenshot = new PortalScreenshot(), normalizer = new FrameNormalizer(), atspi = new AtspiProvider(), shell = new ShellProvider(), ocr, vision, tmpDir = '/tmp/jarvis-cu', timeouts = {} } = {}) { this.screenshot = screenshot; this.normalizer = normalizer; this.atspi = atspi; this.shell = shell; this.ocr = ocr; this.vision = vision; this.tmpDir = tmpDir; this.lastTree = []; - this.timeouts = { screenshot: timeouts.screenshot ?? 8000, tree: timeouts.tree ?? 5000, ocr: timeouts.ocr ?? 8000, vision: timeouts.vision ?? 8000 }; + this.timeouts = { screenshot: timeouts.screenshot ?? 8000, tree: timeouts.tree ?? 5000, shell: timeouts.shell ?? 2000, ocr: timeouts.ocr ?? 8000, vision: timeouts.vision ?? 8000 }; } - async tree({ focusedOnly = true, maxNodes = 400 } = {}) { const nodes = await timed(this.atspi.tree({ focusedOnly, maxNodes }), this.timeouts.tree, 'AT-SPI'); this.lastTree = nodes.map((node, index) => ({ ...node, ref: `r${index + 1}` })); return this.lastTree; } + + async tree({ focusedOnly = true, maxNodes = 400 } = {}) { + const nodes = await timed(this.atspi.tree({ focusedOnly, maxNodes }), this.timeouts.tree, 'AT-SPI'); + this.lastTree = nodes.map((node, index) => ({ ...node, ref: `r${index + 1}` })); + return this.lastTree; + } + + async _captureFrame() { + const rawPath = await this.screenshot.capture(path.join(this.tmpDir, `observe-${Date.now()}.png`)); + return this.normalizer.normalize(rawPath); + } + + async _shellSnapshot() { + if (this.shell.connect) await this.shell.connect(); + const [windows, focused] = await Promise.all([this.shell.windows(), this.shell.focused()]); + return { windows, focused }; + } + async observe({ includeTree = true, includeOcr = false, includeVision = false } = {}) { await mkdir(this.tmpDir, { recursive: true }); const unavailable = []; - let rawPath; let frame = { path: null }; - try { rawPath = await timed(this.screenshot.capture(path.join(this.tmpDir, `observe-${Date.now()}.png`)), this.timeouts.screenshot, 'screenshot'); frame = await this.normalizer.normalize(rawPath); } catch (error) { unavailable.push(`screenshot: ${error.message}`); } - await this.shell.connect?.().catch((error) => unavailable.push(`Shell helper: ${error.message}`)); - const [windows, focused, tree] = await Promise.all([this.shell.windows(), this.shell.focused(), includeTree ? this.tree().catch((error) => { unavailable.push(`AT-SPI: ${error.message}`); return []; }) : Promise.resolve([])]); - const result = { monitor: null, focused, windows: windows.windows || [], tree, screenshot_path: frame.path, ocr_blocks: [], vision_hint: null, unavailable }; - if (!windows.available) result.unavailable.push(windows.reason); - if (frame.path && includeOcr && this.ocr) result.ocr_blocks = await timed(this.ocr(frame.path), this.timeouts.ocr, 'OCR').catch((error) => { result.unavailable.push(`OCR: ${error.message}`); return []; }); - if (frame.path && includeVision && this.vision) result.vision_hint = await timed(this.vision(frame.path), this.timeouts.vision, 'vision').catch((error) => { result.unavailable.push(`vision: ${error.message}`); return null; }); + const note = (label) => (error) => { unavailable.push(`${label}: ${error.message}`); return null; }; + const [frame, shell, tree] = await Promise.all([ + timed(this._captureFrame(), this.timeouts.screenshot, 'screenshot').catch(note('screenshot')), + timed(this._shellSnapshot(), this.timeouts.shell, 'Shell helper').catch(note('Shell helper')), + includeTree ? this.tree().catch((error) => { unavailable.push(`AT-SPI: ${error.message}`); return []; }) : Promise.resolve([]), + ]); + const windows = shell?.windows || { available: false, windows: [] }; + const result = { + monitor: null, + focused: shell?.focused || null, + windows: windows.windows || [], + tree, + screenshot_path: frame?.path || null, + ocr_blocks: [], + vision_hint: null, + unavailable, + }; + if (shell && !windows.available && windows.reason) result.unavailable.push(windows.reason); + if (result.screenshot_path && includeOcr && this.ocr) result.ocr_blocks = await timed(this.ocr(result.screenshot_path), this.timeouts.ocr, 'OCR').catch((error) => { result.unavailable.push(`OCR: ${error.message}`); return []; }); + if (result.screenshot_path && includeVision && this.vision) result.vision_hint = await timed(this.vision(result.screenshot_path), this.timeouts.vision, 'vision').catch((error) => { result.unavailable.push(`vision: ${error.message}`); return null; }); return result; } - async zoom({ rect, ref } = {}) { await mkdir(this.tmpDir, { recursive: true }); const node = ref ? this.lastTree.find((item) => item.ref === ref) : null; const target = rect || node?.rect; if (!target) throw new Error('rect or current tree ref is required'); const raw = await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`)); const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), target); return { rect: target, screenshot_path: frame.path }; } - async find({ query, role } = {}) { const nodes = this.lastTree.length ? this.lastTree : await this.tree(); return nodes.map((node) => ({ ...node, score: score(query, node) })).filter((node) => node.score && (!role || normalize(node.role) === normalize(role))).sort((a, b) => b.score - a.score).slice(0, 20); } + + async zoom({ rect, ref } = {}) { + await mkdir(this.tmpDir, { recursive: true }); + const node = ref ? this.lastTree.find((item) => item.ref === ref) : null; + const target = rect || node?.rect; + if (!target) throw new Error('rect or current tree ref is required'); + const raw = await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`)); + const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), target); + return { rect: target, screenshot_path: frame.path }; + } + + async find({ query, role } = {}) { + const nodes = this.lastTree.length ? this.lastTree : await this.tree(); + return nodes.map((node) => ({ ...node, score: score(query, node) })).filter((node) => node.score && (!role || normalize(node.role) === normalize(role))).sort((a, b) => b.score - a.score).slice(0, 20); + } } diff --git a/computer-use/py/__pycache__/portal_screenshot.cpython-314.pyc b/computer-use/py/__pycache__/portal_screenshot.cpython-314.pyc index 3b52c18..2652ace 100644 Binary files a/computer-use/py/__pycache__/portal_screenshot.cpython-314.pyc and b/computer-use/py/__pycache__/portal_screenshot.cpython-314.pyc differ diff --git a/computer-use/py/atspi_snapshot.py b/computer-use/py/atspi_snapshot.py index 9c8ac59..3f8b48e 100644 --- a/computer-use/py/atspi_snapshot.py +++ b/computer-use/py/atspi_snapshot.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import json import sys +import gi +gi.require_version('Atspi', '2.0') from gi.repository import Atspi mode, limit = sys.argv[1], int(sys.argv[2]) diff --git a/computer-use/py/portal_screenshot.py b/computer-use/py/portal_screenshot.py index 4624c18..c2aa6df 100644 --- a/computer-use/py/portal_screenshot.py +++ b/computer-use/py/portal_screenshot.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Capture one frame through GNOME Shell or the Screenshot portal and print its path.""" +"""Capture one frame through the Jarvis GNOME Shell helper, then the Screenshot portal.""" import sys from gi.repository import Gio, GLib @@ -7,22 +7,22 @@ target = sys.argv[1] bus = Gio.bus_get_sync(Gio.BusType.SESSION, None) -def gnome_shell_screenshot(): +def jarvis_shell_screenshot(): proxy = Gio.DBusProxy.new_sync( bus, Gio.DBusProxyFlags.NONE, None, - 'org.gnome.Shell.Screenshot', '/org/gnome/Shell/Screenshot', - 'org.gnome.Shell.Screenshot', None, + 'io.qvac.Jarvis.Shell', '/io/qvac/Jarvis/Shell', + 'io.qvac.Jarvis.Shell', None, ) - ok, path = proxy.call_sync( + path = proxy.call_sync( 'Screenshot', - GLib.Variant('(bbs)', (False, False, target)), + GLib.Variant('(s)', (target,)), Gio.DBusCallFlags.NONE, 5000, None, - ).unpack() - if not ok: - raise RuntimeError('GNOME Shell screenshot failed') - return path or target + ).unpack()[0] + if not path: + raise RuntimeError('Jarvis Shell screenshot returned no path') + return path def portal_screenshot(): @@ -46,7 +46,7 @@ def portal_screenshot(): loop.quit() sub = bus.signal_subscribe(None, 'org.freedesktop.portal.Request', 'Response', request, None, Gio.DBusSignalFlags.NONE, response) - GLib.timeout_add(7000, loop.quit) + GLib.timeout_add(4000, loop.quit) loop.run() bus.signal_unsubscribe(sub) if not result['uri']: @@ -59,7 +59,7 @@ def portal_screenshot(): errors = [] -for capture in (gnome_shell_screenshot, portal_screenshot): +for capture in (jarvis_shell_screenshot, portal_screenshot): try: print(capture()) raise SystemExit(0) diff --git a/computer-use/shell-provider.js b/computer-use/shell-provider.js index 5cba4b3..6e371eb 100644 --- a/computer-use/shell-provider.js +++ b/computer-use/shell-provider.js @@ -1,7 +1,55 @@ -export class ShellProvider { - constructor({ listWindows, focusedWindow, focusWindow, proxy } = {}) { this.listWindowsImpl = listWindows; this.focusedWindowImpl = focusedWindow; this.focusWindowImpl = focusWindow; this.proxy = proxy; } - async connect() { if (this.proxy) return this; const dbus = await import('dbus-next'); const bus = dbus.sessionBus(); const object = await bus.getProxyObject('io.qvac.Jarvis.Shell', '/io/qvac/Jarvis/Shell'); this.proxy = object.getInterface('io.qvac.Jarvis.Shell'); return this; } - async windows() { if (this.proxy) return { available: true, windows: JSON.parse(await this.proxy.ListWindows()) }; if (!this.listWindowsImpl) return { available: false, reason: 'GNOME Shell helper is not connected', windows: [] }; return { available: true, windows: await this.listWindowsImpl() }; } - async focused() { if (this.proxy) return JSON.parse(await this.proxy.FocusedWindow()); if (!this.focusedWindowImpl) return null; return this.focusedWindowImpl(); } - async focus(id) { if (this.proxy) return this.proxy.FocusWindow(BigInt(id)); if (!this.focusWindowImpl) throw new Error('GNOME Shell focus helper is unavailable'); return this.focusWindowImpl(id); } +import { spawn } from 'node:child_process'; + +const DEST = 'io.qvac.Jarvis.Shell'; +const PATH = '/io/qvac/Jarvis/Shell'; +const IFACE = 'io.qvac.Jarvis.Shell'; + +function parseGdbusString(out) { + const match = String(out).match(/'((?:\\.|[^'\\])*)'/); + if (!match) throw new Error('GNOME Shell helper returned no string'); + return match[1].replace(/\\(.)/g, '$1'); +} + +export class ShellProvider { + constructor({ listWindows, focusedWindow, focusWindow, proxy, spawnImpl = spawn, timeoutMs = 2000 } = {}) { + this.listWindowsImpl = listWindows; this.focusedWindowImpl = focusedWindow; this.focusWindowImpl = focusWindow; this.proxy = proxy; + this.spawnImpl = spawnImpl; this.timeoutMs = timeoutMs; + } + + _call(method, args = []) { + return new Promise((resolve, reject) => { + const child = this.spawnImpl('gdbus', ['call', '--session', '--dest', DEST, '--object-path', PATH, '--method', `${IFACE}.${method}`, ...args], { stdio: ['ignore', 'pipe', 'pipe'] }); + let out = ''; let err = ''; let settled = false; + const timer = setTimeout(() => { if (settled) return; settled = true; child.kill('SIGTERM'); reject(new Error('GNOME Shell helper timed out')); }, this.timeoutMs); + const done = (fn) => (value) => { if (settled) return; settled = true; clearTimeout(timer); fn(value); }; + child.stdout.on('data', (d) => { out += d; }); + child.stderr.on('data', (d) => { err += d; }); + child.on('error', done(reject)); + child.on('close', (code) => done(code === 0 ? resolve : reject)(code === 0 ? parseGdbusString(out) : new Error(err.trim() || `GNOME Shell helper exited ${code}`))); + }); + } + + async connect() { + if (this.proxy || this.listWindowsImpl) return this; + await this._call('ListWindows'); + return this; + } + + async windows() { + if (this.proxy) return { available: true, windows: JSON.parse(await this.proxy.ListWindows()) }; + if (this.listWindowsImpl) return { available: true, windows: await this.listWindowsImpl() }; + try { return { available: true, windows: JSON.parse(await this._call('ListWindows')) }; } catch (error) { return { available: false, reason: error.message, windows: [] }; } + } + + async focused() { + if (this.proxy) return JSON.parse(await this.proxy.FocusedWindow()); + if (this.focusedWindowImpl) return this.focusedWindowImpl(); + try { return JSON.parse(await this._call('FocusedWindow')); } catch { return null; } + } + + async focus(id) { + if (this.proxy) return this.proxy.FocusWindow(BigInt(id)); + if (this.focusWindowImpl) return this.focusWindowImpl(id); + await this._call('FocusWindow', [String(id)]); + } } diff --git a/test/gnome-extension.test.js b/test/gnome-extension.test.js index 54597b4..b17e59c 100644 --- a/test/gnome-extension.test.js +++ b/test/gnome-extension.test.js @@ -506,3 +506,10 @@ test('both settings entry points use the shared preferences window', () => { assert.match(prefs, /fillSettingsWindow/); assert.match(control, /fillSettingsWindow/); }); + +test('the Shell helper can capture a screenshot from inside GNOME', () => { + const source = readFileSync(new URL('../apps/gnome-extension/jarvis@qvac.local/shell-dbus.js', import.meta.url), 'utf8'); + assert.match(source, /method name="Screenshot"/); + assert.match(source, /new Shell\.Screenshot/); + assert.match(source, /screenshot\(false, stream\)/); +}); diff --git a/test/observer.test.js b/test/observer.test.js index b8600ed..a9429c2 100644 --- a/test/observer.test.js +++ b/test/observer.test.js @@ -15,6 +15,22 @@ test('Phase 6 observer returns stable refs and ranked semantic matches', async ( const zoom = await observer.zoom({ ref: 'r1' }); assert.deepEqual(zoom.rect, [10, 20, 30, 30]); }); +test('observer returns when the Shell helper never connects', async () => { + const observer = new DesktopObserver({ + screenshot: { capture: async () => '/tmp/frame.png' }, + normalizer: { normalize: async () => ({ path: '/tmp/frame.webp' }) }, + shell: { connect: () => new Promise(() => {}), windows: async () => assert.fail('shell windows'), focused: async () => assert.fail('shell focused') }, + atspi: { tree: async () => [{ role: 'label', name: 'Discord', rect: [0, 0, 10, 10] }] }, + timeouts: { shell: 40 }, + }); + const started = Date.now(); + const bundle = await observer.observe({ includeOcr: false }); + assert.ok(Date.now() - started < 1000); + assert.equal(bundle.screenshot_path, '/tmp/frame.webp'); + assert.equal(bundle.tree[0].name, 'Discord'); + assert.match(bundle.unavailable.join(' '), /Shell helper/); +}); + test('observer returns after a hung screenshot instead of staying on THINKING', async () => { const observer = new DesktopObserver({ screenshot: { capture: () => new Promise(() => {}) },