Computer Use Updates

computer-use/actuator.js

@@ -2,11 +2,11 @@ import { setTimeout as delay } from 'node:timers/promises';
 import { assertSafeTarget, requiresConfirmation } from './safety.js';
 
 export function pointFor(target, args = {}) {
-  if (args.x != null && args.y != null) return { x: Number(args.x), y: Number(args.y) };
+  if (args.x != null && args.y != null) { const x = Number(args.x), y = Number(args.y); if (!Number.isFinite(x) || !Number.isFinite(y)) throw new Error('coordinates must be finite'); return { x, y }; }
   const rect = target?.rect;
   if (Array.isArray(rect) && rect.length >= 4) {
     const [x, y, w, h] = rect.map(Number);
-    if ([x, y, w, h].every(Number.isFinite)) return { x: x + w / 2, y: y + h / 2 };
+    if ([x, y, w, h].every(Number.isFinite) && w > 0 && h > 0 && x > -100000 && y > -100000) return { x: x + w / 2, y: y + h / 2 };
   }
   return null;
 }
@@ -54,7 +54,8 @@ export class ComputerActuator {
   async send(action) {
     await this.readyInput();
     if (!this.input?.send) throw new Error('portal EIS input backend is unavailable');
-    this.input.send(action);
+    this.session.assertActive();
+    return await this.input.send(action);
   }
 
   async semantic(target, action) {
@@ -63,14 +64,17 @@ export class ComputerActuator {
       const result = await this.atspiAction(target, action);
       if (!result || result.ok === false) return null;
       return result;
-    } catch {
+    } catch (error) {
+      if (/target not found|stale/i.test(error.message)) throw error;
       return null;
     }
   }
 
   async run(action, args, fn) {
+    this.session.assertActive();
     const target = await this.target(args);
     if (requiresConfirmation(action, target) && !(await this.confirm(action, target))) throw new Error('explicit confirmation required');
+    await this.readyInput();
     this.session.beginStep();
     await this.highlight?.(target, action);
     this.session.assertActive();
@@ -96,7 +100,7 @@ export class ComputerActuator {
 
   async click(args = {}) {
     return this.run('click', args, async (target) => {
-      const semantic = args.ref ? await this.semantic(target, args.action || 'click') : null;
+      const semantic = args.ref && (!args.button || args.button === 'left') ? await this.semantic(target, args.action || 'click') : null;
       if (semantic) return semantic;
       const point = pointFor(target, args);
       if (!point) throw new Error('click requires a semantic ref or coordinates');
@@ -134,7 +138,8 @@ export class ComputerActuator {
 
   async scroll(args = {}) {
     return this.run('scroll', args, async (target) => {
-      const point = pointFor(target, args) || { x: 0, y: 0 };
+      const point = pointFor(target, args);
+      if (!point) throw new Error('scroll requires a ref or coordinates');
       await this.send({ type: 'pointer', action: 'scroll', x: point.x, y: point.y, dx: args.dx || 0, dy: args.dy || 0 });
       return point;
     });

computer-use/doctor.js

@@ -1,28 +1,31 @@
-import { access } from 'node:fs/promises';
-import { constants } from 'node:fs';
 import { spawn } from 'node:child_process';
 
-const run = (file, args) => new Promise((resolve, reject) => {
+const run = (file, args, timeoutMs = 6000) => new Promise((resolve, reject) => {
   const child = spawn(file, args, { stdio: ['ignore', 'pipe', 'pipe'] });
-  let stdout = ''; let stderr = '';
-  child.stdout?.on('data', (chunk) => { stdout += chunk; });
-  child.stderr?.on('data', (chunk) => { stderr += chunk; });
-  child.once('error', reject);
-  child.once('close', (code) => code === 0 ? resolve({ stdout, stderr }) : reject(new Error(stderr || `${file} exited ${code}`)));
+  let stdout = '', stderr = '', settled = false;
+  const finish = (error) => { if (settled) return; settled = true; clearTimeout(timer); error ? reject(error) : resolve(stdout.trim()); };
+  const timer = setTimeout(() => { child.kill('SIGTERM'); finish(new Error(`${file} timed out`)); }, timeoutMs);
+  child.stdout?.on('data', chunk => { stdout = (stdout + chunk).slice(-64000); });
+  child.stderr?.on('data', chunk => { stderr = (stderr + chunk).slice(-2000); });
+  child.once('error', finish);
+  child.once('close', code => finish(code === 0 ? null : new Error(stderr.trim() || `${file} exited ${code}`)));
 });
+const portal = iface => run('gdbus', ['call', '--session', '--dest', 'org.freedesktop.portal.Desktop', '--object-path', '/org/freedesktop/portal/desktop', '--method', 'org.freedesktop.DBus.Properties.Get', `org.freedesktop.portal.${iface}`, 'version']);
 const checks = [
-  ['session', false, async () => process.env.XDG_SESSION_TYPE || 'unknown'],
-  ['portal', false, async () => { await access('/usr/share/dbus-1/services/org.freedesktop.portal.Desktop.service', constants.F_OK); return 'installed'; }],
-  ['pipewire', false, async () => { await run('which', ['pw-cat']); return 'installed'; }],
-  ['at-spi', false, async () => { await run('python3', ['-c', 'import gi; gi.require_version("Atspi", "2.0"); from gi.repository import Atspi']); return 'PyGObject Atspi available'; }],
-  ['libei', false, async () => { const result = await run('ldconfig', ['-p']); if (!/libei|libeis/.test(result.stdout)) throw new Error('libei/libeis not found'); return 'libei/libeis installed'; }],
-  ['ydotool', true, async () => { await run('which', ['ydotool']); return 'optional fallback present'; }],
+  ['session', false, async () => { if (process.env.XDG_SESSION_TYPE !== 'wayland') throw new Error('a GNOME Wayland session is required for the primary backend'); return 'Wayland'; }],
+  ['remote-desktop', false, () => portal('RemoteDesktop')],
+  ['screencast', false, () => portal('ScreenCast')],
+  ['pipewire', false, async () => { await run('pw-cli', ['info', '0']); return 'live server responds'; }],
+  ['frame-pipeline', false, async () => { await run('python3', ['-c', 'import gi; gi.require_version("Gst", "1.0"); from gi.repository import Gst; from PIL import Image; Gst.init(None); assert all(Gst.ElementFactory.find(n) for n in ["pipewiresrc", "videoconvert", "appsink"]), "missing GStreamer capture elements"; assert Image.registered_extensions().get(".webp") == "WEBP", "Pillow WebP missing"']); return 'GStreamer capture elements and Pillow WebP available'; }],
+  ['at-spi', false, async () => { await run('python3', ['-c', 'import gi; gi.require_version("Atspi", "2.0"); from gi.repository import Atspi; Atspi.init(); assert Atspi.get_desktop(0) is not None, "accessibility desktop unavailable"']); return 'live accessibility desktop responds'; }],
+  ['libei', false, async () => { await run('python3', ['-c', 'import ctypes; lib=ctypes.CDLL("libei.so.1"); assert lib.ei_new_sender and lib.ei_setup_backend_fd']); return 'sender library loads'; }],
+  ['shell-helper', false, async () => { await run('gdbus', ['call', '--session', '--dest', 'io.qvac.Jarvis.Shell', '--object-path', '/io/qvac/Jarvis/Shell', '--method', 'io.qvac.Jarvis.Shell.ListWindows']); return 'GNOME extension window service responds'; }],
+  ['daemon', true, async () => { await run('gdbus', ['call', '--session', '--dest', 'io.qvac.Jarvis', '--object-path', '/io/qvac/Jarvis', '--method', 'io.qvac.Jarvis.Session.ComputerStatus']); return 'computer-use status responds'; }],
 ];
-
 let failed = 0;
 for (const [name, optional, check] of checks) {
   try { console.log(`ok   ${name}: ${await check()}${optional ? ' (optional)' : ''}`); }
-  catch { if (!optional) failed += 1; console.log(`---- ${name}: unavailable${optional ? ' (optional)' : ''}`); }
+  catch (error) { if (!optional) failed++; console.log(`FAIL ${name}: ${error.message}${optional ? ' (optional)' : ''}`); }
 }
-console.log(failed ? `cu-doctor: ${failed} required checks unavailable` : 'cu-doctor: required checks passed');
+console.log(failed ? `cu-doctor: ${failed} required checks failed` : 'cu-doctor: live prerequisites passed; input and capture still require a consented cu-smoke run');
 process.exitCode = failed ? 1 : 0;

computer-use/frame.js

@@ -5,11 +5,27 @@ import { spawn } from 'node:child_process';
 export const MAX_LONG_EDGE = 1280;
 
 export class FrameNormalizer {
-  constructor({ helper = path.resolve(new URL('./py/normalize_frame.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn, tmpDir = '/tmp/jarvis-cu', maxLongEdge = MAX_LONG_EDGE, quality = 70 } = {}) { this.helper = helper; this.python = python; this.spawnImpl = spawnImpl; this.tmpDir = tmpDir; this.maxLongEdge = maxLongEdge; this.quality = quality; }
+  constructor({ helper = path.resolve(new URL('./py/normalize_frame.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn, tmpDir = '/tmp/jarvis-cu', maxLongEdge = MAX_LONG_EDGE, quality = 70 } = {}) {
+    Object.assign(this, { helper, python, spawnImpl, tmpDir, maxLongEdge, quality });
+  }
   async normalize(input, output = path.join(this.tmpDir, `frame-${Date.now()}.webp`), rect) {
     await mkdir(path.dirname(output), { recursive: true });
-    const crop = rect ? rect.map(Number).map((value) => String(Math.round(value))) : [];
-    await new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper, input, output, String(this.maxLongEdge), String(this.quality), ...crop], { stdio: ['ignore', 'pipe', 'pipe'] }); let error = ''; child.stderr?.on('data', (d) => { error += d; }); child.on('error', reject); child.on('close', (code) => code === 0 ? resolve() : reject(new Error(error || `frame normalization exited ${code}`))); });
-    const info = await stat(output); return { path: output, bytes: info.size, maxLongEdge: this.maxLongEdge, mime: 'image/webp' };
+    const crop = rect ? rect.map(Number).map(value => String(Math.round(value))) : [];
+    let metadata = {};
+    await new Promise((resolve, reject) => {
+      const child = this.spawnImpl(this.python, [this.helper, input, output, String(this.maxLongEdge), String(this.quality), ...crop], { stdio: ['ignore', 'pipe', 'pipe'] });
+      let stderr = '', stdout = '';
+      const timer = setTimeout(() => { child.kill('SIGTERM'); reject(new Error('frame normalization timed out')); }, 8000);
+      child.stdout?.on('data', data => { stdout += data; });
+      child.stderr?.on('data', data => { stderr += data; });
+      child.on('error', error => { clearTimeout(timer); reject(error); });
+      child.on('close', code => {
+        clearTimeout(timer);
+        try { metadata = JSON.parse(stdout || '{}'); } catch {}
+        code === 0 ? resolve() : reject(new Error(stderr || `frame normalization exited ${code}`));
+      });
+    });
+    const info = await stat(output);
+    return { ...metadata, path: output, bytes: info.size, maxLongEdge: this.maxLongEdge, mime: 'image/webp' };
   }
 }

computer-use/observer.js

@@ -24,12 +24,24 @@ export class DesktopObserver {
   constructor({ screenshot = new PortalScreenshot(), normalizer = new FrameNormalizer(), atspi = new AtspiProvider(), shell = new ShellProvider(), framebuffer, ocr, vision, tmpDir = '/tmp/jarvis-cu', timeouts = {} } = {}) {
     this.screenshot = screenshot; this.normalizer = normalizer; this.atspi = atspi; this.shell = shell; this.framebuffer = framebuffer; this.ocr = ocr; this.vision = vision; this.tmpDir = tmpDir;
     this.lastTree = [];
+    this._refSequence = 0;
     this.timeouts = { screenshot: timeouts.screenshot ?? 8000, tree: timeouts.tree ?? 5000, shell: timeouts.shell ?? 2000, ocr: timeouts.ocr ?? 8000, vision: timeouts.vision ?? 8000 };
   }
 
   async tree({ focusedOnly = true, maxNodes = 400 } = {}) {
+    this.lastTree = [];
     const nodes = await timed(this.atspi.tree({ focusedOnly, maxNodes }), this.timeouts.tree, 'AT-SPI');
-    this.lastTree = nodes.map((node, index) => ({ ...node, ref: `r${index + 1}` }));
+    const windows = (await timed(this.shell.windows(), this.timeouts.shell, 'Shell helper').catch(() => ({ windows: [] }))).windows || [];
+    this.lastTree = nodes.map((node) => {
+      const win = windows.find(w => w.pid === node.pid && (w.focused || windows.filter(x => x.pid === node.pid).length === 1));
+      const rect = node.rect?.slice();
+      if (win?.rect && node.window_rect && rect && rect[0] > -100000 && rect[1] > -100000) {
+        const bounds = win.buffer_rect || [win.rect[0] - Math.max(0, node.window_rect[2] - win.rect[2]) / 2, win.rect[1] - Math.max(0, node.window_rect[3] - win.rect[3]) / 2];
+        rect[0] += bounds[0] - node.window_rect[0];
+        rect[1] += bounds[1] - node.window_rect[1];
+      }
+      return { ...node, rect, ref: `r${++this._refSequence}` };
+    });
     return this.lastTree;
   }
 
@@ -68,6 +80,9 @@ export class DesktopObserver {
       tree,
       screenshot_path: frame?.path || null,
       frame_source: frame?.source || null,
+      frame: frame ? { width: frame.width, height: frame.height, source_width: frame.source_width, source_height: frame.source_height, scale: frame.scale } : null,
+      streams: this.framebuffer?.streams?.() || [],
+      coordinate_space: 'desktop logical pixels; tree rects use desktop coordinates. Scale screenshot coordinates to the stream size and add its position before clicking.',
       ocr_blocks: [],
       vision_hint: null,
       unavailable,
@@ -86,7 +101,15 @@ export class DesktopObserver {
     const raw = this.framebuffer?.capture
       ? await this.framebuffer.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`))
       : await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`));
-    const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), target);
+    const stream = this.framebuffer?.streams?.()?.[0];
+    let crop = target;
+    if (stream?.position && stream?.size) {
+      const dimensions = await this.normalizer.normalize(raw);
+      const sx = dimensions.source_width / stream.size[0], sy = dimensions.source_height / stream.size[1];
+      crop = [(target[0] - stream.position[0]) * sx, (target[1] - stream.position[1]) * sy, target[2] * sx, target[3] * sy];
+      if (crop[0] < 0 || crop[1] < 0 || crop[0] + crop[2] > dimensions.source_width || crop[1] + crop[3] > dimensions.source_height) throw new Error('zoom target lies outside the shared monitor');
+    }
+    const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), crop);
     return { rect: target, screenshot_path: frame.path };
   }
 

computer-use/portal-input.js

@@ -4,33 +4,44 @@ import path from 'node:path';
 /** Wayland input and ScreenCast boundary. The helper owns portal consent, the
  * EIS fd, and the PipeWire remote; Node sends only bounded JSON actions. */
 export class PortalInputBackend {
-  constructor({ command = process.env.JARVIS_EI_HELPER || path.resolve(new URL('./py/portal_remote_desktop.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn, timeoutMs = 120_000 } = {}) {
-    this.command = command; this.python = python; this.spawnImpl = spawnImpl; this.timeoutMs = timeoutMs;
+  constructor({ command = process.env.JARVIS_EI_HELPER || path.resolve(new URL('./py/portal_remote_desktop.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn, timeoutMs = 120_000, actionTimeoutMs = 10000, onClose = () => {} } = {}) {
+    this.command = command; this.python = python; this.spawnImpl = spawnImpl; this.timeoutMs = timeoutMs; this.actionTimeoutMs = actionTimeoutMs; this.onClose = onClose; this._pending = new Map(); this._sequence = 0; this.streams = [];
     this.process = null; this.available = false; this.screen = false; this._grant = null; this._frameWait = null;
   }
   grant({ persist = false, monitors = 'focused', mode = 'act' } = {}) {
     if (this._grant) return this._grant;
-    const child = this.process = this.spawnImpl(this.python, [this.command], { stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env, JARVIS_CU_MODE: mode } });
+    const child = this.process = this.spawnImpl(this.python, [this.command], {
+      stdio: ['pipe', 'pipe', 'pipe'],
+      env: { ...process.env, JARVIS_CU_MODE: mode, JARVIS_CU_PERSIST: persist ? '1' : '0' },
+    });
     this._grant = new Promise((resolve, reject) => {
       let buffer = ''; let settled = false;
       const timer = setTimeout(() => fail(new Error('portal input consent timed out')), this.timeoutMs);
       const fail = (error) => {
+        if (this.process !== child) return;
         clearTimeout(timer);
         if (this.process === child) { this.available = false; this.screen = false; this.process = null; this._grant = null; }
+        this._rejectPending(error);
         this._rejectFrame(error);
+        const wasReady = settled;
         if (!settled) { settled = true; reject(error); }
         child.kill('SIGTERM');
+        if (wasReady) this.onClose(error.message);
       };
       this._cancelGrant = () => fail(new Error('portal input grant revoked'));
       child.on('error', fail);
       child.once('close', () => {
         clearTimeout(timer);
-        if (this.process === child) { this.available = false; this.screen = false; this.process = null; this._grant = null; }
+        if (this.process !== child) return;
+        this.available = false; this.screen = false; this.process = null; this._grant = null;
+        this._rejectPending(new Error('portal helper exited'));
         this._rejectFrame(new Error('portal helper exited'));
-        if (!settled) { settled = true; reject(new Error('portal input helper exited before readiness')); }
+        if (settled) this.onClose('portal helper exited');
+        if (!settled) { settled = true; reject(new Error('portal input helper exited before readiness' + (diagnostics ? ': ' + diagnostics : ''))); }
       });
       child.stdin?.on('error', fail);
-      child.stderr?.on('data', () => {});
+      let diagnostics = '';
+      child.stderr?.on('data', (data) => { diagnostics = (diagnostics + String(data)).slice(-2000); });
       child.stdout.on('data', (data) => {
         buffer += String(data);
         if (buffer.length > 64 * 1024) { fail(new Error('portal helper output too large')); return; }
@@ -39,15 +50,20 @@ export class PortalInputBackend {
           const line = buffer.slice(0, index); buffer = buffer.slice(index + 1);
           let event;
           try { event = JSON.parse(line); } catch { continue; }
+          if (this.process !== child) continue;
+          if (event.type === 'ack') { this._settle(event.id, null, event); continue; }
           if (event.type === 'error') {
             if (!settled) { fail(new Error(event.reason || 'portal input unavailable')); return; }
-            this._rejectFrame(new Error(event.reason || 'portal helper error'));
+            const error = new Error(event.reason || 'portal helper error');
+            if (event.id != null && this._pending.has(event.id)) this._settle(event.id, error);
+            else if (event.id == null || event.id === this._frameWait?.id) this._rejectFrame(error);
             continue;
           }
-          if (event.type === 'frame') { this._resolveFrame(event.path || event); continue; }
+          if (event.type === 'frame') { if (!event.id || event.id === this._frameWait?.id) this._resolveFrame(event.path || event); continue; }
           if (event.type === 'ready' && this.process === child && !settled) {
             clearTimeout(timer); settled = true; this.available = true; this.screen = Boolean(event.screen);
-            resolve({ restore_token_present: Boolean(event.restore_token_present), monitors, backend: event.backend || 'portal-ei', screen: this.screen });
+            this.streams = event.streams || [];
+            resolve({ streams: this.streams, eis_error: event.eis_error || null, restore_token_present: Boolean(event.restore_token_present), monitors, backend: event.backend || 'portal-ei', screen: this.screen });
           }
         }
       });
@@ -56,24 +72,56 @@ export class PortalInputBackend {
   }
   async ready() {
     if (this._grant) {
-      try { await this._grant; } catch (error) { throw new Error(error?.message || 'Grant desktop from the tray, then try again'); }
+      try { await this._grant; } catch (error) { throw new Error(error?.message || 'Allow desktop access in Settings, then try again'); }
     }
-    if (!this.available || !this.process?.stdin?.writable) throw new Error('Grant desktop from the tray, then try again');
+    if (!this.available || !this.process?.stdin?.writable) throw new Error('Allow desktop access in Settings, then try again');
   }
-  send(action) { if (!this.available || !this.process?.stdin?.writable) throw new Error('portal EIS input backend is unavailable'); this.process.stdin.write(`${JSON.stringify(action)}\n`); }
+  send(action) {
+    if (!this.available || !this.process?.stdin?.writable) throw new Error('portal EIS input backend is unavailable');
+    const id = ++this._sequence;
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this._settle(id, new Error('desktop action acknowledgment timed out; grant revoked'));
+        this.revoke();
+        this.onClose('desktop action timed out');
+      }, this.actionTimeoutMs);
+      this._pending.set(id, { resolve, reject, timer });
+      try { this.process.stdin.write(JSON.stringify({ ...action, id }) + '\n'); }
+      catch (error) { this._settle(id, error); }
+    });
+  }
+  _settle(id, error, result) {
+    const wait = this._pending.get(id);
+    if (!wait) return;
+    this._pending.delete(id); clearTimeout(wait.timer);
+    if (error) wait.reject(error); else wait.resolve(result);
+  }
+  _rejectPending(error) { for (const id of this._pending.keys()) this._settle(id, error); }
   captureFrame(output) {
     if (!this.available || !this.process?.stdin?.writable) throw new Error('PipeWire ScreenCast is unavailable until desktop access is granted');
     if (this._frameWait) throw new Error('a PipeWire frame grab is already in flight');
     return new Promise((resolve, reject) => {
       const timer = setTimeout(() => this._rejectFrame(new Error('PipeWire frame timed out')), 8000);
-      this._frameWait = {
+      const id = ++this._sequence;
+      this._frameWait = { id,
         resolve: (value) => { clearTimeout(timer); resolve(value); },
         reject: (error) => { clearTimeout(timer); reject(error); },
       };
-      try { this.send({ type: 'frame', path: output }); } catch (error) { this._rejectFrame(error); }
+      try { this.process.stdin.write(JSON.stringify({ type: 'frame', path: output, id }) + '\n'); } catch (error) { this._rejectFrame(error); }
     });
   }
   _resolveFrame(path) { const wait = this._frameWait; this._frameWait = null; wait?.resolve(path); }
   _rejectFrame(error) { const wait = this._frameWait; this._frameWait = null; wait?.reject(error); }
-  revoke() { this._cancelGrant?.(); this._cancelGrant = null; this.available = false; this.screen = false; this.process = null; this._grant = null; }
+  revoke() {
+    const child = this.process;
+    this._rejectPending(new Error('portal input grant revoked'));
+    this.streams = [];
+    this._cancelGrant?.();
+    this._cancelGrant = null;
+    this.available = false;
+    this.screen = false;
+    this.process = null;
+    this._grant = null;
+    try { child?.kill?.('SIGTERM'); } catch {}
+  }
 }

computer-use/py/atspi_action.py

@@ -11,7 +11,7 @@ desktop = Atspi.get_desktop(0)
 found = None
 wanted_name = target.get('name') or ''
 wanted_role = target.get('role') or ''
-wanted_rect = target.get('rect') or None
+wanted_rect = target.get('raw_rect') or target.get('rect') or None
 
 def rect_close(node):
     if not wanted_rect or len(wanted_rect) < 4:
@@ -38,7 +38,21 @@ def walk(node, depth=0):
     except Exception:
         return
 
-walk(desktop)
+if target.get('pid') is not None and target.get('atspi_path') is not None:
+    for i in range(desktop.get_child_count()):
+        app = desktop.get_child_at_index(i)
+        if app.get_process_id() != target['pid']:
+            continue
+        candidate = app
+        for index in target['atspi_path']:
+            candidate = candidate.get_child_at_index(index)
+            if candidate is None:
+                break
+        if candidate and (candidate.get_name() or '') == wanted_name and (candidate.get_role_name() or '') == wanted_role and rect_close(candidate):
+            found = candidate
+            break
+else:
+    walk(desktop)
 if not found:
     raise SystemExit('AT-SPI target not found')
 actions = found.get_action()

computer-use/py/atspi_snapshot.py

@@ -9,7 +9,8 @@ mode, limit = sys.argv[1], int(sys.argv[2])
 Atspi.init()
 desktop = Atspi.get_desktop(0)
 nodes = []
-def walk(node, depth=0):
+def walk(node, depth=0, route=None, pid=None, window_rect=None):
+    route = route or []
     if len(nodes) >= limit or node is None or depth > 30:
         return
     try:
@@ -17,10 +18,13 @@ def walk(node, depth=0):
         name = node.get_name() or ''
         component = node.get_component()
         rect = component.get_extents(Atspi.CoordType.SCREEN) if component else None
+        bounds = [rect.x, rect.y, rect.width, rect.height] if rect else None
+        if window_rect is None and depth == 0:
+            window_rect = bounds
         if role and (name or rect):
-            nodes.append({'role': role, 'name': name, 'rect': [rect.x, rect.y, rect.width, rect.height] if rect else None, 'state': [str(s) for s in node.get_state_set().get_states()]})
+            nodes.append({'role': role, 'name': name, 'rect': [rect.x, rect.y, rect.width, rect.height] if rect else None, 'state': [str(s) for s in node.get_state_set().get_states()], 'pid': pid, 'atspi_path': route, 'window_rect': window_rect, 'raw_rect': bounds})
         for i in range(node.get_child_count()):
-            walk(node.get_child_at_index(i), depth + 1)
+            walk(node.get_child_at_index(i), depth + 1, route + [i], pid, window_rect)
     except Exception:
         return
 if mode == 'focused':
@@ -30,9 +34,12 @@ if mode == 'focused':
             for j in range(app.get_child_count()):
                 window = app.get_child_at_index(j)
                 if window.get_state_set().contains(Atspi.StateType.ACTIVE):
-                    walk(window)
+                    walk(window, route=[j], pid=app.get_process_id())
         except Exception:
             continue
 else:
-    walk(desktop)
+    for i in range(desktop.get_child_count()):
+        app = desktop.get_child_at_index(i)
+        for j in range(app.get_child_count()):
+            walk(app.get_child_at_index(j), route=[j], pid=app.get_process_id())
 print(json.dumps(nodes, ensure_ascii=False))

computer-use/py/libei_sender.py

@@ -45,6 +45,8 @@ COMBO_KEYS = {
     'backspace': KEY_BACKSPACE, 'tab': KEY_TAB, 'space': KEY_SPACE,
     'ctrl': KEY_LEFTCTRL, 'control': KEY_LEFTCTRL, 'alt': KEY_LEFTALT,
     'shift': KEY_LEFTSHIFT, 'super': KEY_LEFTMETA, 'meta': KEY_LEFTMETA,
+    'left': 105, 'right': 106, 'up': 103, 'down': 108,
+    'home': 102, 'end': 107, 'pageup': 104, 'pagedown': 109, 'delete': 111, 'insert': 110,
     **F_KEYS,
 }
 LETTER_KEYS = {ch: code for ch, code in zip('qwertyuiopasdfghjklzxcvbnm', [
@@ -52,6 +54,7 @@ LETTER_KEYS = {ch: code for ch, code in zip('qwertyuiopasdfghjklzxcvbnm', [
 ])}
 DIGIT_KEYS = {str(d): code for d, code in enumerate([11, 2, 3, 4, 5, 6, 7, 8, 9, 10])}
 PUNCT_KEYS = {
+    **{ch: (DIGIT_KEYS[d], True) for ch, d in zip('!@#$%^&*()', '1234567890')},
     '-': (12, False), '_': (12, True), '=': (13, False), '+': (13, True),
     '[': (26, False), '{': (26, True), ']': (27, False), '}': (27, True),
     ';': (39, False), ':': (39, True), "'": (40, False), '"': (40, True),
@@ -84,6 +87,7 @@ def _lib():
     lib.ei_event_get_device.argtypes = [ctypes.c_void_p]
     lib.ei_event_get_device.restype = ctypes.c_void_p
     lib.ei_event_unref.argtypes = [ctypes.c_void_p]
+    lib.ei_seat_bind_capabilities.argtypes = [ctypes.c_void_p]
     lib.ei_seat_bind_capabilities.restype = None
     lib.ei_seat_has_capability.argtypes = [ctypes.c_void_p, ctypes.c_int]
     lib.ei_seat_has_capability.restype = ctypes.c_int
@@ -164,12 +168,12 @@ class LibeiSender:
             readable, _, _ = select.select([self.fd], [], [], remaining)
             if readable:
                 self.dispatch()
-            if self._emulating and (self.pointer or self.keyboard):
+            if self.pointer_abs and self.keyboard and all(_ptr(d) in self._emulating for d in (self.pointer_abs, self.keyboard)):
                 if collected is None:
                     collected = time.time()
                 elif time.time() - collected >= 0.35 or self.pointer_abs:
                     return True
-        return bool(self._emulating and (self.pointer or self.keyboard))
+        return bool(self.pointer_abs and self.keyboard and all(_ptr(d) in self._emulating for d in (self.pointer_abs, self.keyboard)))
 
     def dispatch(self):
         self.lib.ei_dispatch(self.ei)
@@ -212,9 +216,9 @@ class LibeiSender:
         return BTN_LEFT
 
     def move(self, x, y):
-        device = self.pointer
-        if not device:
-            raise RuntimeError('no EIS pointer device')
+        device = self.pointer_abs
+        if not device or _ptr(device) not in self._emulating:
+            raise RuntimeError('no active absolute EIS pointer device')
         if self._has(device, CAP_POINTER_ABSOLUTE):
             self.lib.ei_device_pointer_motion_absolute(device, float(x), float(y))
         else:
@@ -233,19 +237,23 @@ class LibeiSender:
 
     def scroll(self, x, y, dx=0, dy=0):
         self.move(x, y)
-        self.lib.ei_device_scroll_discrete(self.pointer, int(dx), int(dy))
+        self.lib.ei_device_scroll_discrete(self.pointer, int(dx) * 120, int(dy) * 120)
         self._frame(self.pointer)
 
     def drag(self, start, end):
         self.move(start[0], start[1])
         self.lib.ei_device_button_button(self.pointer, BTN_LEFT, True)
         self._frame(self.pointer)
-        self.move(end[0], end[1])
-        self.lib.ei_device_button_button(self.pointer, BTN_LEFT, False)
-        self._frame(self.pointer)
+        try:
+            for step in range(1, 13):
+                self.move(start[0] + (end[0] - start[0]) * step / 12, start[1] + (end[1] - start[1]) * step / 12)
+                time.sleep(0.012)
+        finally:
+            self.lib.ei_device_button_button(self.pointer, BTN_LEFT, False)
+            self._frame(self.pointer)
 
     def _tap(self, code, mods=()):
-        if not self.keyboard:
+        if not self.keyboard or _ptr(self.keyboard) not in self._emulating:
             raise RuntimeError('no EIS keyboard device')
         for mod in mods:
             self.lib.ei_device_keyboard_key(self.keyboard, mod, True)
@@ -278,18 +286,30 @@ class LibeiSender:
             if punct:
                 code, shifted = punct
                 self._tap(code, (KEY_LEFTSHIFT,) if shifted else ())
+                continue
+            # GNOME/GTK Unicode input method; never silently drop characters.
+            self._tap(LETTER_KEYS['u'], (KEY_LEFTCTRL, KEY_LEFTSHIFT))
+            time.sleep(0.08)
+            for digit in format(ord(ch), 'x'):
+                self._tap(LETTER_KEYS.get(digit) or DIGIT_KEYS[digit])
+                time.sleep(0.01)
+            self._tap(KEY_ENTER)
+            time.sleep(0.1)
         if submit:
             self._tap(KEY_ENTER)
 
     def key_combo(self, combo):
         parts = [p.strip().lower() for p in str(combo).replace('-', '+').split('+') if p.strip()]
         if not parts:
-            return
-        mods = [COMBO_KEYS[part] for part in parts[:-1] if part in COMBO_KEYS]
+            raise ValueError('keyboard combination is required')
+        modifiers = {'ctrl', 'control', 'alt', 'shift', 'super', 'meta'}
+        if any(part not in modifiers for part in parts[:-1]):
+            raise ValueError('unsupported keyboard modifier')
+        mods = list(dict.fromkeys(COMBO_KEYS[part] for part in parts[:-1]))
         key = parts[-1]
         code = COMBO_KEYS.get(key) or LETTER_KEYS.get(key) or DIGIT_KEYS.get(key)
         if code is None:
-            return
+            raise ValueError(f'unsupported key: {key}')
         self._tap(code, tuple(mods))
 
     def handle(self, action):

computer-use/py/normalize_frame.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 import sys
+import json
 from PIL import Image
 
 source, target, cap, quality = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4])
 with Image.open(source) as image:
     image = image.convert('RGB')
+    source_width, source_height = image.size
     if len(sys.argv) == 9:
         x, y, width, height = [int(v) for v in sys.argv[5:9]]
         image = image.crop((x, y, x + width, y + height))
@@ -12,3 +14,5 @@ with Image.open(source) as image:
     if scale < 1.0:
         image = image.resize((round(image.width * scale), round(image.height * scale)), Image.Resampling.LANCZOS)
     image.save(target, 'WEBP', quality=quality, method=4)
+
+    print(json.dumps({'source_width': source_width, 'source_height': source_height, 'width': image.width, 'height': image.height, 'scale': scale}))

computer-use/py/portal_remote_desktop.py

@@ -3,8 +3,9 @@
 import json
 import os
 import select
-import subprocess
 import sys
+import time
+import signal
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
@@ -21,13 +22,16 @@ from pw_framebuffer import PipeWireFrameBuffer
 
 BTN = {'left': 0x110, 'right': 0x111, 'middle': 0x112}
 MODE = os.environ.get('JARVIS_CU_MODE', 'act')
+# persist_mode 2 made GNOME restore the previous share and skip the screen picker.
+PERSIST = os.environ.get('JARVIS_CU_PERSIST', '0') == '1'
+PERSIST_MODE = 2 if PERSIST else 0
 
 
 def log_error(reason):
     print(json.dumps({'type': 'error', 'reason': str(reason)}), flush=True)
 
 
-bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
+bus = None
 
 
 def portal_proxy(iface):
@@ -38,22 +42,35 @@ def portal_proxy(iface):
     )
 
 
-remote = portal_proxy('org.freedesktop.portal.RemoteDesktop')
-screencast = portal_proxy('org.freedesktop.portal.ScreenCast')
+remote = screencast = None
 
 
 def request(proxy, method, signature, values, timeout_ms=120000):
-    request_path = proxy.call_sync(method, GLib.Variant(signature, values), Gio.DBusCallFlags.NONE, timeout_ms, None).unpack()[0]
+    # Subscribe before sending the request: fast responses can precede the reply.
+    token = f'jarvis{GLib.get_real_time()}'
+    sender_name = bus.get_unique_name()[1:].replace('.', '_')
+    request_path = f'/org/freedesktop/portal/desktop/request/{sender_name}/{token}'
+    values = list(values)
+    values[-1] = dict(values[-1], handle_token=GLib.Variant('s', token))
     loop = GLib.MainLoop()
-    result = {'code': 1, 'results': {}}
-
+    result = {'code': None, 'results': {}}
     def response(_conn, _sender, _path, _interface, _member, params):
         result['code'], result['results'] = params.unpack()
         loop.quit()
-
+    def expired():
+        result['code'] = 'timeout'
+        loop.quit()
+        return False
     sub = bus.signal_subscribe(None, 'org.freedesktop.portal.Request', 'Response', request_path, None, Gio.DBusSignalFlags.NONE, response)
-    loop.run()
-    bus.signal_unsubscribe(sub)
+    timer = GLib.timeout_add(timeout_ms, expired)
+    try:
+        proxy.call_sync(method, GLib.Variant(signature, tuple(values)), Gio.DBusCallFlags.NONE, timeout_ms, None)
+        if result['code'] is None:
+            loop.run()
+    finally:
+        if result['code'] != 'timeout':
+            GLib.source_remove(timer)
+        bus.signal_unsubscribe(sub)
     if result['code'] != 0:
         raise RuntimeError(f'{method} portal response {result["code"]}')
     return result['results']
@@ -98,8 +115,10 @@ def stream_node_id(streams):
 
 
 class PortalNotify:
-    def __init__(self, session):
+    def __init__(self, session, node_id, origin=(0, 0)):
         self.session = session
+        self.node_id = node_id
+        self.origin = origin
 
     def handle(self, action):
         kind = action.get('type')
@@ -109,7 +128,7 @@ class PortalNotify:
         if kind == 'pointer' and name in ('click', 'double_click', 'move'):
             x = float(action.get('x') or 0)
             y = float(action.get('y') or 0)
-            notify('NotifyPointerMotionAbsolute', '(oa{sv}udd)', (session, opts, 0, x, y))
+            notify('NotifyPointerMotionAbsolute', '(oa{sv}udd)', (session, opts, self.node_id, x - self.origin[0], y - self.origin[1]))
             if name != 'move':
                 button = BTN.get(str(action.get('button') or 'left'), BTN['left'])
                 repeats = 2 if name == 'double_click' else 1
@@ -117,27 +136,56 @@ class PortalNotify:
                     notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, button, 1))
                     notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, button, 0))
         elif kind == 'pointer' and name == 'scroll':
-            notify('NotifyPointerAxisDiscrete', '(oa{sv}ui)', (session, opts, 0, int(action.get('dy') or 0)))
+            self.handle(dict(action, action='move'))
+            for axis, key in ((0, 'dy'), (1, 'dx')):
+                amount = int(action.get(key) or 0)
+                if amount:
+                    notify('NotifyPointerAxisDiscrete', '(oa{sv}ui)', (session, opts, axis, amount))
         elif kind == 'pointer' and name == 'drag':
             start = action.get('from') or [0, 0]
             end = action.get('to') or [0, 0]
-            notify('NotifyPointerMotionAbsolute', '(oa{sv}udd)', (session, opts, 0, float(start[0]), float(start[1])))
+            self.handle({'type': 'pointer', 'action': 'move', 'x': start[0], 'y': start[1]})
             notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, BTN['left'], 1))
-            notify('NotifyPointerMotionAbsolute', '(oa{sv}udd)', (session, opts, 0, float(end[0]), float(end[1])))
-            notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, BTN['left'], 0))
+            try:
+                for step in range(1, 13):
+                    self.handle({'type': 'pointer', 'action': 'move', 'x': start[0] + (end[0] - start[0]) * step / 12, 'y': start[1] + (end[1] - start[1]) * step / 12})
+                    time.sleep(0.012)
+            finally:
+                notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, BTN['left'], 0))
         elif kind == 'keyboard' and name == 'type':
             for ch in str(action.get('text') or ''):
-                keysym = 0xff0d if ch == '\n' else (0x020 if ch == ' ' else ord(ch))
+                if ord(ch) > 127:
+                    self.handle({'type': 'keyboard', 'action': 'key', 'combo': 'ctrl+shift+u'})
+                    time.sleep(0.08)
+                    for digit in format(ord(ch), 'x'):
+                        self.handle({'type': 'keyboard', 'action': 'type', 'text': digit})
+                        time.sleep(0.01)
+                    self.handle({'type': 'keyboard', 'action': 'key', 'combo': 'enter'})
+                    time.sleep(0.1)
+                    continue
+                keysym = {'\n': 0xff0d, '\t': 0xff09}.get(ch, ord(ch) if ord(ch) <= 255 else 0x01000000 | ord(ch))
                 notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, keysym, 1))
                 notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, keysym, 0))
             if action.get('submit'):
                 notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, 0xff0d, 1))
                 notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, 0xff0d, 0))
         elif kind == 'keyboard' and name == 'key':
-            combo = str(action.get('combo') or '').lower()
-            keysym = 0xff1b if 'esc' in combo else 0xff0d
-            notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, keysym, 1))
-            notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, keysym, 0))
+            from libei_sender import COMBO_KEYS, LETTER_KEYS, DIGIT_KEYS
+            parts = str(action.get('combo') or '').lower().replace('-', '+').split('+')
+            modifiers = {'ctrl', 'control', 'alt', 'shift', 'super', 'meta'}
+            if not parts or any(p not in modifiers for p in parts[:-1]):
+                raise ValueError('unsupported keyboard combination')
+            codes = [COMBO_KEYS.get(p) or LETTER_KEYS.get(p) or DIGIT_KEYS.get(p) for p in parts]
+            if any(code is None for code in codes):
+                raise ValueError('unsupported keyboard combination')
+            held = []
+            try:
+                for code in codes:
+                    notify('NotifyKeyboardKeycode', '(oa{sv}iu)', (session, opts, code, 1))
+                    held.append(code)
+            finally:
+                for code in reversed(held):
+                    notify('NotifyKeyboardKeycode', '(oa{sv}iu)', (session, opts, code, 0))
         else:
             raise RuntimeError(f'unsupported portal notify action {kind} {name}')
 
@@ -159,7 +207,7 @@ class SessionHandler:
             if not path:
                 raise RuntimeError('frame path is required')
             self.framebuffer.capture(path)
-            print(json.dumps({'type': 'frame', 'path': path, 'source': 'pipewire'}), flush=True)
+            print(json.dumps({'type': 'frame', 'path': path, 'source': 'pipewire', 'id': action.get('id')}), flush=True)
             return
         if self.input_handler is None:
             raise RuntimeError('desktop input is observe-only')
@@ -168,27 +216,35 @@ class SessionHandler:
 
 def run_loop(handler, extra_fd=None):
     stdin_fd = sys.stdin.fileno()
+    buffer = b''
     while True:
-        fds = [stdin_fd]
-        if extra_fd is not None:
-            fds.append(extra_fd)
-        readable, _, _ = select.select(fds, [], [], 0.25)
-        if extra_fd is not None and extra_fd in readable and hasattr(handler, 'dispatch'):
+        context = GLib.MainContext.default()
+        while context.pending():
+            context.iteration(False)
+        fds = [stdin_fd] + ([extra_fd] if extra_fd is not None else [])
+        readable, _, _ = select.select(fds, [], [], 0.1)
+        if extra_fd is not None and extra_fd in readable:
             handler.dispatch()
-        if stdin_fd in readable:
-            line = sys.stdin.readline()
-            if not line:
-                break
+        if stdin_fd not in readable:
+            continue
+        chunk = os.read(stdin_fd, 65536)
+        if not chunk:
+            break
+        buffer += chunk
+        if len(buffer) > 1024 * 1024:
+            raise RuntimeError('input message exceeds limit')
+        while b'\n' in buffer:
+            line, buffer = buffer.split(b'\n', 1)
+            action = {}
             try:
                 action = json.loads(line)
-            except Exception:
-                continue
-            if action.get('type') == 'close':
-                break
-            try:
+                if action.get('type') == 'close':
+                    return
                 handler.handle(action)
+                if action.get('type') != 'frame':
+                    print(json.dumps({'type': 'ack', 'id': action.get('id'), 'ok': True}), flush=True)
             except Exception as exc:
-                log_error(exc)
+                print(json.dumps({'type': 'error', 'id': action.get('id'), 'reason': str(exc)}), flush=True)
 
 
 def attach_screencast():
@@ -198,90 +254,74 @@ def attach_screencast():
         'types': GLib.Variant('u', 1),
         'multiple': GLib.Variant('b', False),
         'cursor_mode': GLib.Variant('u', 2),
-        'persist_mode': GLib.Variant('u', 2),
+        'persist_mode': GLib.Variant('u', PERSIST_MODE),
     }))
     return session
 
 
-def start_framebuffer(sc_session):
-    results = request(screencast, 'Start', '(osa{sv})', (sc_session, '', {}))
-    node_id = stream_node_id(results.get('streams') or [])
-    fd = unix_fd(screencast, 'OpenPipeWireRemote', sc_session)
-    return PipeWireFrameBuffer(fd, node_id)
-
-
-injector = None
-sender = None
-framebuffer = None
-session = None
-try:
-    sc_session = attach_screencast()
-    input_handler = None
-    extra_fd = None
-    backend = 'none'
-    eis_error = None
-    frame_error = None
-    if MODE != 'observe':
-        token = f'jarvisrd{GLib.get_real_time()}'
-        session = request(remote, 'CreateSession', '(a{sv})', ({'session_handle_token': GLib.Variant('s', token)},))['session_handle']
-        request(remote, 'SelectDevices', '(oa{sv})', (session, {'types': GLib.Variant('u', 7), 'persist_mode': GLib.Variant('u', 2)}))
-        request(remote, 'Start', '(osa{sv})', (session, '', {}))
-        try:
-            eis_fd = unix_fd(remote, 'ConnectToEIS', session)
-            sender = LibeiSender(eis_fd)
-            if not sender.wait_ready():
-                raise RuntimeError('libei connected but no pointer or keyboard device appeared')
-            input_handler = sender
-            extra_fd = sender.fd
-            backend = 'portal-ei'
-        except Exception as exc:
-            eis_error = str(exc)
-            if sender:
-                try:
-                    sender.close()
-                except Exception:
-                    pass
-            sender = None
-        if input_handler is None and os.environ.get('JARVIS_LIBEI_BRIDGE'):
-            injector = subprocess.Popen(os.environ['JARVIS_LIBEI_BRIDGE'], shell=True, stdin=subprocess.PIPE, text=True)
-
-            class Bridge:
-                def handle(self, action):
-                    injector.stdin.write(json.dumps(action) + '\n')
-                    injector.stdin.flush()
-
-            input_handler = Bridge()
-            backend = 'portal-ei'
-        if input_handler is None:
-            input_handler = PortalNotify(session)
-            backend = 'portal-notify'
+def main():
+    global bus, remote, screencast
+    bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
+    remote = portal_proxy('org.freedesktop.portal.RemoteDesktop')
+    screencast = portal_proxy('org.freedesktop.portal.ScreenCast')
+    sender = framebuffer = None
+    session = None
+    signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
     try:
-        framebuffer = start_framebuffer(sc_session)
+        if MODE == 'observe':
+            session = attach_screencast()
+            results = request(screencast, 'Start', '(osa{sv})', (session, '', {}))
+        else:
+            token = f'jarvisrd{GLib.get_real_time()}'
+            session = request(remote, 'CreateSession', '(a{sv})', ({'session_handle_token': GLib.Variant('s', token)},))['session_handle']
+            request(remote, 'SelectDevices', '(oa{sv})', (session, {'types': GLib.Variant('u', 3), 'persist_mode': GLib.Variant('u', PERSIST_MODE)}))
+            request(screencast, 'SelectSources', '(oa{sv})', (session, {
+                'types': GLib.Variant('u', 1), 'multiple': GLib.Variant('b', False), 'cursor_mode': GLib.Variant('u', 2),
+            }))
+            results = request(remote, 'Start', '(osa{sv})', (session, '', {}))
+        streams = results.get('streams') or []
+        node_id = stream_node_id(streams)
+        props = streams[0][1]
+        origin = props.get('position', (0, 0))
+        framebuffer = PipeWireFrameBuffer(unix_fd(screencast, 'OpenPipeWireRemote', session), node_id)
+        backend, extra_fd, input_handler, eis_error = 'none', None, None, None
+        if MODE != 'observe':
+            # Notify remains a tested fallback, selectable for diagnostics.
+            if os.environ.get('JARVIS_CU_INPUT_BACKEND') != 'notify':
+                try:
+                    sender = LibeiSender(unix_fd(remote, 'ConnectToEIS', session))
+                    if not sender.wait_ready():
+                        raise RuntimeError('EIS did not provide active absolute pointer and keyboard devices')
+                    input_handler, extra_fd, backend = sender, sender.fd, 'portal-ei'
+                except Exception as exc:
+                    eis_error = str(exc)
+                    if sender:
+                        sender.close()
+                        sender = None
+            if input_handler is None:
+                input_handler, backend = PortalNotify(session, node_id, origin), 'portal-notify'
+        def closed(*_):
+            raise SystemExit(0)
+        bus.signal_subscribe(None, 'org.freedesktop.portal.Session', 'Closed', session, None, Gio.DBusSignalFlags.NONE, closed)
+        print(json.dumps({
+            'type': 'ready', 'session': session, 'restore_token_present': bool(results.get('restore_token')),
+            'backend': backend, 'screen': True, 'eis_error': eis_error,
+            'streams': [{'node_id': int(n), **p} for n, p in streams],
+        }), flush=True)
+        run_loop(SessionHandler(input_handler, framebuffer), extra_fd)
     except Exception as exc:
-        frame_error = str(exc)
-        framebuffer = None
-    print(json.dumps({
-        'type': 'ready',
-        'session': session or sc_session,
-        'restore_token_present': True,
-        'backend': backend,
-        'screen': framebuffer is not None,
-        'eis_error': eis_error,
-        'frame_error': frame_error,
-    }), flush=True)
-    run_loop(SessionHandler(input_handler, framebuffer), extra_fd)
-except Exception as exc:
-    log_error(exc)
-finally:
-    if framebuffer:
-        try:
+        log_error(exc)
+    finally:
+        if framebuffer:
             framebuffer.close()
-        except Exception:
-            pass
-    if sender:
-        try:
+        if sender:
             sender.close()
-        except Exception:
-            pass
-    if injector:
-        injector.terminate()
+        if session:
+            try:
+                bus.call_sync('org.freedesktop.portal.Desktop', session, 'org.freedesktop.portal.Session', 'Close', None, None, Gio.DBusCallFlags.NONE, 2000, None)
+            except Exception:
+                pass
+
+
+if __name__ == '__main__':
+    main()

computer-use/shell-provider.js

@@ -25,7 +25,12 @@ export class ShellProvider {
       child.stdout.on('data', (d) => { out += d; });
       child.stderr.on('data', (d) => { err += d; });
       child.on('error', done(reject));
-      child.on('close', (code) => done(code === 0 ? resolve : reject)(code === 0 ? parseGdbusString(out) : new Error(err.trim() || `GNOME Shell helper exited ${code}`)));
+      child.on('close', (code) => {
+        try {
+          if (code !== 0) throw new Error(err.trim() || `GNOME Shell helper exited ${code}`);
+          done(resolve)(method === 'FocusWindow' ? true : parseGdbusString(out));
+        } catch (error) { done(reject)(error); }
+      });
     });
   }