P6

computer-use/atspi.js

@@ -0,0 +1,8 @@
+import { spawn } from 'node:child_process';
+
+export class AtspiProvider {
+  constructor({ python = 'python3', helper = new URL('./py/atspi_snapshot.py', import.meta.url).pathname, spawnImpl = spawn } = {}) { this.python = python; this.helper = helper; this.spawnImpl = spawnImpl; }
+  async tree({ focusedOnly = true, maxNodes = 400 } = {}) {
+    return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper, focusedOnly ? 'focused' : 'desktop', String(Math.min(400, maxNodes))], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => { if (code !== 0) return reject(new Error(err || `AT-SPI exited ${code}`)); try { resolve(JSON.parse(out)); } catch (error) { reject(error); } }); });
+  }
+}

computer-use/doctor.js

@@ -5,18 +5,18 @@ import { promisify } from 'node:util';
 
 const run = promisify(execFile);
 const checks = [
-  ['session', async () => process.env.XDG_SESSION_TYPE || 'unknown'],
-  ['portal', async () => { await access('/usr/share/dbus-1/services/org.freedesktop.portal.Desktop.service', constants.F_OK); return 'installed'; }],
-  ['pipewire', async () => { await run('sh', ['-lc', 'command -v pw-cat']); return 'installed'; }],
-  ['at-spi', async () => { await run('sh', ['-lc', 'command -v gsettings']); return 'desktop tools present'; }],
-  ['libei', async () => { await run('sh', ['-lc', 'ldconfig -p 2>/dev/null | grep -q libei']); return 'installed'; }],
-  ['ydotool', async () => { await run('sh', ['-lc', 'command -v ydotool']); return 'optional fallback present'; }],
+  ['session', false, async () => process.env.XDG_SESSION_TYPE || 'unknown'],
+  ['portal', false, async () => { await access('/usr/share/dbus-1/services/org.freedesktop.portal.Desktop.service', constants.F_OK); return 'installed'; }],
+  ['pipewire', false, async () => { await run('which', ['pw-cat']); return 'installed'; }],
+  ['at-spi', false, async () => { await run('python3', ['-c', 'import gi; gi.require_version("Atspi", "2.0"); from gi.repository import Atspi']); return 'PyGObject Atspi available'; }],
+  ['libei', false, async () => { const result = await run('ldconfig', ['-p']); if (!/libei|libeis/.test(result.stdout)) throw new Error('libei/libeis not found'); return 'libei/libeis installed'; }],
+  ['ydotool', true, async () => { await run('which', ['ydotool']); return 'optional fallback present'; }],
 ];
 
 let failed = 0;
-for (const [name, check] of checks) {
-  try { console.log(`ok   ${name}: ${await check()}`); }
-  catch { failed += 1; console.log(`---- ${name}: unavailable`); }
+for (const [name, optional, check] of checks) {
+  try { console.log(`ok   ${name}: ${await check()}${optional ? ' (optional)' : ''}`); }
+  catch { if (!optional) failed += 1; console.log(`---- ${name}: unavailable${optional ? ' (optional)' : ''}`); }
 }
-console.log(failed ? `cu-doctor: ${failed} optional/required checks unavailable` : 'cu-doctor: all checks passed');
+console.log(failed ? `cu-doctor: ${failed} required checks unavailable` : 'cu-doctor: required checks passed');
 process.exitCode = failed ? 1 : 0;

computer-use/frame.js

@@ -0,0 +1,15 @@
+import { mkdir, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { spawn } from 'node:child_process';
+
+export const MAX_LONG_EDGE = 1280;
+
+export class FrameNormalizer {
+  constructor({ helper = path.resolve(new URL('./py/normalize_frame.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn, tmpDir = '/tmp/jarvis-cu' } = {}) { this.helper = helper; this.python = python; this.spawnImpl = spawnImpl; this.tmpDir = tmpDir; }
+  async normalize(input, output = path.join(this.tmpDir, `frame-${Date.now()}.webp`), rect) {
+    await mkdir(path.dirname(output), { recursive: true });
+    const crop = rect ? rect.map(Number).map((value) => String(Math.round(value))) : [];
+    await new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper, input, output, String(MAX_LONG_EDGE), '70', ...crop], { stdio: ['ignore', 'pipe', 'pipe'] }); let error = ''; child.stderr?.on('data', (d) => { error += d; }); child.on('error', reject); child.on('close', (code) => code === 0 ? resolve() : reject(new Error(error || `frame normalization exited ${code}`))); });
+    const info = await stat(output); return { path: output, bytes: info.size, maxLongEdge: MAX_LONG_EDGE, mime: 'image/webp' };
+  }
+}

computer-use/observer.js

@@ -0,0 +1,29 @@
+import { mkdir } from 'node:fs/promises';
+import path from 'node:path';
+import { AtspiProvider } from './atspi.js';
+import { FrameNormalizer } from './frame.js';
+import { PortalScreenshot } from './portal-screenshot.js';
+import { ShellProvider } from './shell-provider.js';
+
+const normalize = (value) => String(value || '').toLowerCase().trim();
+const score = (query, node) => { const q = normalize(query); const text = `${normalize(node.name)} ${normalize(node.role)}`; if (!q || !text) return 0; if (text === q) return 100; if (text.includes(q)) return 75; const words = q.split(/\s+/).filter((w) => text.includes(w)); return words.length ? 40 + words.length * 10 : 0; };
+
+export class DesktopObserver {
+  constructor({ screenshot = new PortalScreenshot(), normalizer = new FrameNormalizer(), atspi = new AtspiProvider(), shell = new ShellProvider(), ocr, vision, tmpDir = '/tmp/jarvis-cu' } = {}) { this.screenshot = screenshot; this.normalizer = normalizer; this.atspi = atspi; this.shell = shell; this.ocr = ocr; this.vision = vision; this.tmpDir = tmpDir; this.lastTree = []; }
+  async tree({ focusedOnly = true, maxNodes = 400 } = {}) { const nodes = await this.atspi.tree({ focusedOnly, maxNodes }); this.lastTree = nodes.map((node, index) => ({ ...node, ref: `r${index + 1}` })); return this.lastTree; }
+  async observe({ includeTree = true, includeOcr = true, includeVision = false } = {}) {
+    await mkdir(this.tmpDir, { recursive: true });
+    const unavailable = [];
+    let rawPath; let frame = { path: null };
+    try { rawPath = await this.screenshot.capture(path.join(this.tmpDir, `observe-${Date.now()}.png`)); frame = await this.normalizer.normalize(rawPath); } catch (error) { unavailable.push(`screenshot: ${error.message}`); }
+    await this.shell.connect?.().catch((error) => unavailable.push(`Shell helper: ${error.message}`));
+    const [windows, focused, tree] = await Promise.all([this.shell.windows(), this.shell.focused(), includeTree ? this.tree().catch((error) => { unavailable.push(`AT-SPI: ${error.message}`); return []; }) : Promise.resolve([])]);
+    const result = { monitor: null, focused, windows: windows.windows || [], tree, screenshot_path: frame.path, ocr_blocks: [], vision_hint: null, unavailable };
+    if (!windows.available) result.unavailable.push(windows.reason);
+    if (includeOcr && this.ocr) result.ocr_blocks = await this.ocr(frame.path).catch((error) => { result.unavailable.push(`OCR: ${error.message}`); return []; });
+    if (includeVision && this.vision) result.vision_hint = await this.vision(frame.path).catch((error) => { result.unavailable.push(`vision: ${error.message}`); return null; });
+    return result;
+  }
+  async zoom({ rect, ref } = {}) { const node = ref ? this.lastTree.find((item) => item.ref === ref) : null; const target = rect || node?.rect; if (!target) throw new Error('rect or current tree ref is required'); const raw = await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`)); const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), target); return { rect: target, screenshot_path: frame.path }; }
+  async find({ query, role } = {}) { const nodes = this.lastTree.length ? this.lastTree : await this.tree(); return nodes.map((node) => ({ ...node, score: score(query, node) })).filter((node) => node.score && (!role || normalize(node.role) === normalize(role))).sort((a, b) => b.score - a.score).slice(0, 20); }
+}

computer-use/portal-screenshot.js

@@ -0,0 +1,9 @@
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+
+export class PortalScreenshot {
+  constructor({ helper = path.resolve(new URL('./py/portal_screenshot.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn, tmpDir = '/tmp/jarvis-cu' } = {}) { this.helper = helper; this.python = python; this.spawnImpl = spawnImpl; this.tmpDir = tmpDir; }
+  async capture(output = path.join(this.tmpDir, `portal-${Date.now()}.png`)) {
+    return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper, output], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => code === 0 ? resolve(out.trim() || output) : reject(new Error(err || `Screenshot portal exited ${code}`))); });
+  }
+}

computer-use/py/atspi_snapshot.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+import json
+import sys
+from gi.repository import Atspi
+
+mode, limit = sys.argv[1], int(sys.argv[2])
+Atspi.init()
+desktop = Atspi.get_desktop(0)
+nodes = []
+def walk(node, depth=0):
+    if len(nodes) >= limit or node is None or depth > 30:
+        return
+    try:
+        role = node.get_role_name() or ''
+        name = node.get_name() or ''
+        component = node.get_component()
+        rect = component.get_extents(Atspi.CoordType.SCREEN) if component else None
+        if role and (name or rect):
+            nodes.append({'role': role, 'name': name, 'rect': [rect.x, rect.y, rect.width, rect.height] if rect else None, 'state': [str(s) for s in node.get_state_set().get_states()]})
+        for i in range(node.get_child_count()):
+            walk(node.get_child_at_index(i), depth + 1)
+    except Exception:
+        return
+if mode == 'focused':
+    for i in range(desktop.get_child_count()):
+        app = desktop.get_child_at_index(i)
+        try:
+            for j in range(app.get_child_count()):
+                window = app.get_child_at_index(j)
+                if window.get_state_set().contains(Atspi.StateType.ACTIVE):
+                    walk(window)
+        except Exception:
+            continue
+else:
+    walk(desktop)
+print(json.dumps(nodes, ensure_ascii=False))

computer-use/py/normalize_frame.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+import sys
+from PIL import Image
+
+source, target, cap, quality = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4])
+with Image.open(source) as image:
+    image = image.convert('RGB')
+    if len(sys.argv) == 9:
+        x, y, width, height = [int(v) for v in sys.argv[5:9]]
+        image = image.crop((x, y, x + width, y + height))
+    scale = min(1.0, cap / max(image.width, image.height))
+    if scale < 1.0:
+        image = image.resize((round(image.width * scale), round(image.height * scale)), Image.Resampling.LANCZOS)
+    image.save(target, 'WEBP', quality=quality, method=4)

computer-use/py/portal_screenshot.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""Capture one frame through the GNOME Screenshot portal and print its path."""
+import sys
+from gi.repository import Gio, GLib
+
+target = sys.argv[1]
+bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
+proxy = Gio.DBusProxy.new_sync(bus, Gio.DBusProxyFlags.NONE, None, 'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop', 'org.freedesktop.portal.Screenshot', None)
+request = proxy.call_sync('Screenshot', GLib.Variant('(a{sv})', ({'interactive': GLib.Variant('b', False)},)), Gio.DBusCallFlags.NONE, -1, None).unpack()[0]
+loop = GLib.MainLoop()
+result = {'uri': None, 'code': 1}
+def response(_conn, _sender, _path, _interface, _member, params):
+    code, values = params.unpack()
+    if code == 0:
+        result['uri'] = values.get('uri')
+        result['code'] = 0
+    loop.quit()
+subscription = bus.signal_subscribe(None, 'org.freedesktop.portal.Request', 'Response', request, None, Gio.DBusSignalFlags.NONE, response)
+loop.run()
+bus.signal_unsubscribe(subscription)
+if not result['uri']:
+    raise SystemExit('Screenshot portal returned no image URI')
+ok, contents, _etag = Gio.File.new_for_uri(result['uri']).load_contents(None)
+if not ok:
+    raise SystemExit('could not read screenshot portal URI')
+Gio.File.new_for_path(target).replace_contents(contents, None, False, Gio.FileCreateFlags.REPLACE_DESTINATION, None)
+print(target)

computer-use/session.js

@@ -16,9 +16,11 @@ export class ComputerUseSession {
     this.stepsUsed = 0;
     this.sessionId = `cu_${this.clock().toString(36)}`;
     this.expiresAt = this.clock() + 3 * 60 * 1000;
-    return { session_id: this.sessionId, restore_token_present: Boolean(persist), monitors };
+    return { session_id: this.sessionId, restore_token_present: Boolean(persist), monitors, backend: this.backend };
   }
 
+  setBackend(backend) { this.backend = ['portal-ei', 'ydotool', 'none'].includes(backend) ? backend : 'none'; return this.backend; }
+
   revoke() {
     this.active = false;
     this.sessionId = null;

computer-use/shell-provider.js

@@ -0,0 +1,7 @@
+export class ShellProvider {
+  constructor({ listWindows, focusedWindow, focusWindow, proxy } = {}) { this.listWindowsImpl = listWindows; this.focusedWindowImpl = focusedWindow; this.focusWindowImpl = focusWindow; this.proxy = proxy; }
+  async connect() { if (this.proxy) return this; const dbus = await import('dbus-next'); const bus = dbus.sessionBus(); const object = await bus.getProxyObject('io.qvac.Jarvis.Shell', '/io/qvac/Jarvis/Shell'); this.proxy = object.getInterface('io.qvac.Jarvis.Shell'); return this; }
+  async windows() { if (this.proxy) return { available: true, windows: JSON.parse(await this.proxy.ListWindows()) }; if (!this.listWindowsImpl) return { available: false, reason: 'GNOME Shell helper is not connected', windows: [] }; return { available: true, windows: await this.listWindowsImpl() }; }
+  async focused() { if (this.proxy) return JSON.parse(await this.proxy.FocusedWindow()); if (!this.focusedWindowImpl) return null; return this.focusedWindowImpl(); }
+  async focus(id) { if (this.proxy) return this.proxy.FocusWindow(BigInt(id)); if (!this.focusWindowImpl) throw new Error('GNOME Shell focus helper is unavailable'); return this.focusWindowImpl(id); }
+}