P7

computer-use/actuator.js

@@ -0,0 +1,17 @@
+import { setTimeout as delay } from 'node:timers/promises';
+import { assertSafeTarget, requiresConfirmation } from './safety.js';
+
+export class ComputerActuator {
+  constructor({ session, input, atspiAction, find, highlight, audit, confirm = async () => false, sleep = delay, verify = async () => true } = {}) { this.session = session; this.input = input; this.atspiAction = atspiAction; this.find = find; this.highlight = highlight; this.audit = audit; this.confirm = confirm; this.sleep = sleep; this.verify = verify; }
+  async target(args = {}) { const target = args.ref && this.find ? (await this.find({ ref: args.ref }))[0] : args; if (target) assertSafeTarget(target); return target; }
+  async run(action, args, fn) { const target = await this.target(args); if (requiresConfirmation(action, target) && !(await this.confirm(action, target))) throw new Error('explicit confirmation required'); this.session.beginStep(); await this.highlight?.(target, action); try { const result = await fn(target); await this.sleep(150); if (!(await this.verify(target, action, result))) throw new Error('computer-use state did not change after action'); await this.audit?.record(action, target, { ok: true }); return { ok: true, action, target: target || null, result }; } catch (error) { await this.audit?.record(action, target, { ok: false, reason: error.message }); throw error; } }
+  async act({ ref, action }) { return this.run(`act:${action}`, { ref }, async (target) => { if (!this.atspiAction) throw new Error('AT-SPI action backend is unavailable'); return this.atspiAction(target, action); }); }
+  async click(args = {}) { return this.run('click', args, async (target) => { if (target && this.atspiAction) return this.atspiAction(target, 'click'); if (args.x == null || args.y == null) throw new Error('click requires a semantic ref or coordinates'); this.input.send({ type: 'pointer', action: 'click', x: args.x, y: args.y, button: args.button || 'left' }); }); }
+  async doubleClick(args = {}) { return this.run('double_click', args, async (target) => { if (args.x == null || args.y == null) throw new Error('double-click requires coordinates'); this.input.send({ type: 'pointer', action: 'double_click', x: args.x, y: args.y }); return target; }); }
+  async rightClick(args = {}) { return this.run('right_click', args, async () => { this.input.send({ type: 'pointer', action: 'click', x: args.x, y: args.y, button: 'right' }); }); }
+  async hover(args = {}) { return this.run('hover', args, async () => { this.input.send({ type: 'pointer', action: 'move', x: args.x, y: args.y }); }); }
+  async scroll(args = {}) { return this.run('scroll', args, async () => { this.input.send({ type: 'pointer', action: 'scroll', x: args.x, y: args.y, dx: args.dx || 0, dy: args.dy || 0 }); }); }
+  async drag({ from, to }) { return this.run('drag', { from, to }, async () => { this.input.send({ type: 'pointer', action: 'drag', from, to }); }); }
+  async type({ text, ref, submit = false }) { return this.run('type', { ref }, async (target) => { assertSafeTarget(target); this.input.send({ type: 'keyboard', action: 'type', text: String(text), submit: Boolean(submit) }); }); }
+  async key({ combo }) { return this.run(`key:${combo}`, {}, async () => { this.input.send({ type: 'keyboard', action: 'key', combo: String(combo).toLowerCase() }); }); }
+}

computer-use/atspi.js

@@ -5,4 +5,5 @@ export class AtspiProvider {
   async tree({ focusedOnly = true, maxNodes = 400 } = {}) {
     return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper, focusedOnly ? 'focused' : 'desktop', String(Math.min(400, maxNodes))], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => { if (code !== 0) return reject(new Error(err || `AT-SPI exited ${code}`)); try { resolve(JSON.parse(out)); } catch (error) { reject(error); } }); });
   }
+  async action(target, action) { return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper.replace('atspi_snapshot.py', 'atspi_action.py'), JSON.stringify(target), String(action)], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => { if (code !== 0) return reject(new Error(err || `AT-SPI action exited ${code}`)); try { resolve(JSON.parse(out)); } catch (error) { reject(error); } }); }); }
 }

computer-use/audit.js

@@ -0,0 +1,9 @@
+import { appendFile, mkdir, rm } from 'node:fs/promises';
+import crypto from 'node:crypto';
+import path from 'node:path';
+
+export class ComputerAudit {
+  constructor({ dir = path.join(process.env.XDG_DATA_HOME || path.join(process.env.HOME || '/tmp', '.local/share'), 'jarvis/audit') } = {}) { this.dir = dir; }
+  async record(action, target = {}, result = {}) { const safeTarget = typeof target === 'string' ? target : { ref: target.ref, role: target.role, name: target.name, rect: target.rect }; const targetHash = crypto.createHash('sha256').update(JSON.stringify(safeTarget)).digest('hex'); const row = { ts: new Date().toISOString(), action, target_hash: targetHash, ok: Boolean(result.ok), reason: result.reason || undefined }; await mkdir(this.dir, { recursive: true }); await appendFile(path.join(this.dir, `cu-${row.ts.slice(0, 10).replaceAll('-', '')}.jsonl`), `${JSON.stringify(row)}\n`); return row; }
+  async wipeTemp(dir = '/tmp/jarvis-cu') { await rm(dir, { recursive: true, force: true }); }
+}

computer-use/portal-input.js

@@ -0,0 +1,11 @@
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+
+/** Wayland input boundary. The helper owns portal consent and the EIS fd;
+ * Node sends only bounded JSON actions and never uses hidden uinput. */
+export class PortalInputBackend {
+  constructor({ command = process.env.JARVIS_EI_HELPER || path.resolve(new URL('./py/portal_remote_desktop.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn } = {}) { this.command = command; this.python = python; this.spawnImpl = spawnImpl; this.process = null; this.available = false; }
+  async grant({ persist = false, monitors = 'focused' } = {}) { if (this.process) return { restore_token_present: Boolean(persist) }; this.process = this.spawnImpl(this.python, [this.command], { stdio: ['pipe', 'pipe', 'pipe'] }); let ready = ''; this.process.stdout.on('data', (data) => { ready += String(data); for (const line of ready.split(/\r?\n/).slice(0, -1)) { try { const event = JSON.parse(line); if (event.type === 'ready') { this.available = true; this._ready = event; } } catch {} } ready = ready.split(/\r?\n/).pop() || ''; }); await new Promise((resolve, reject) => { this.process.once('error', reject); const timer = setTimeout(resolve, 1500); this.process.stdout.once('data', () => { clearTimeout(timer); resolve(); }); }); return { restore_token_present: Boolean(persist), monitors, backend: this.available ? 'portal-ei' : 'none' }; }
+  send(action) { if (!this.available || !this.process?.stdin?.writable) throw new Error('portal EIS input backend is unavailable'); this.process.stdin.write(`${JSON.stringify(action)}\n`); }
+  revoke() { this.available = false; this.process?.kill('SIGTERM'); this.process = null; }
+}

computer-use/py/atspi_action.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import json
+import sys
+from gi.repository import Atspi
+
+target, action = json.loads(sys.argv[1]), sys.argv[2]
+Atspi.init()
+desktop = Atspi.get_desktop(0)
+found = None
+def walk(node, depth=0):
+    global found
+    if found or node is None or depth > 30:
+        return
+    try:
+        if (node.get_name() or '') == target.get('name') and (node.get_role_name() or '') == target.get('role'):
+            found = node
+            return
+        for i in range(node.get_child_count()):
+            walk(node.get_child_at_index(i), depth + 1)
+    except Exception:
+        return
+walk(desktop)
+if not found:
+    raise SystemExit('AT-SPI target not found')
+actions = found.get_action()
+for i in range(actions.get_n_actions()):
+    if actions.get_action_name(i).lower() == action.lower():
+        if actions.do_action(i):
+            print(json.dumps({'ok': True, 'action': action}))
+            raise SystemExit(0)
+raise SystemExit(f'AT-SPI action unavailable: {action}')

computer-use/py/portal_remote_desktop.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""RemoteDesktop consent broker for the local EIS input worker."""
+import json
+import sys
+try:
+    import gi
+    gi.require_version('Gio', '2.0')
+    from gi.repository import Gio, GLib
+except Exception as exc:
+    print(json.dumps({'type': 'error', 'reason': f'PyGObject unavailable: {exc}'}), flush=True)
+    raise SystemExit(1)
+
+bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
+proxy = Gio.DBusProxy.new_sync(bus, Gio.DBusProxyFlags.NONE, None, 'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop', 'org.freedesktop.portal.RemoteDesktop', None)
+def request(method, signature, values):
+    request_path = proxy.call_sync(method, GLib.Variant(signature, values), Gio.DBusCallFlags.NONE, -1, None).unpack()[0]
+    loop = GLib.MainLoop(); result = {'code': 1, 'results': {}}
+    def response(_conn, _sender, _path, _interface, _member, params):
+        result['code'], result['results'] = params.unpack(); loop.quit()
+    sub = bus.signal_subscribe(None, 'org.freedesktop.portal.Request', 'Response', request_path, None, Gio.DBusSignalFlags.NONE, response)
+    loop.run(); bus.signal_unsubscribe(sub)
+    if result['code'] != 0: raise RuntimeError(f'{method} portal response {result["code"]}')
+    return result['results']
+try:
+    token = f'jarvis{GLib.get_real_time()}'
+    session = request('CreateSession', '(a{sv})', ({'session_handle_token': GLib.Variant('s', token)},))['session_handle']
+    request('SelectDevices', '(oa{sv})', (session, {'types': GLib.Variant('u', 3), 'persist_mode': GLib.Variant('u', 2)}))
+    request('Start', '(osa{sv})', (session, '', {}))
+    print(json.dumps({'type': 'ready', 'session': session, 'restore_token_present': True}), flush=True)
+except Exception as exc:
+    print(json.dumps({'type': 'error', 'reason': str(exc)}), flush=True)
+for line in sys.stdin:
+    try:
+        action = json.loads(line)
+        if action.get('type') == 'close':
+            break
+    except Exception:
+        continue

computer-use/safety.js

@@ -0,0 +1,5 @@
+const DANGEROUS_KEYS = new Set(['alt+f4', 'ctrl+q', 'ctrl+w', 'poweroff', 'reboot', 'shutdown']);
+export const isPasswordNode = (node) => /password|pam|credential/i.test(`${node?.role || ''} ${node?.name || ''}`);
+export const isDangerousKey = (combo) => DANGEROUS_KEYS.has(String(combo || '').toLowerCase().replace(/^key:/, ''));
+export function requiresConfirmation(action, target = {}) { return isDangerousKey(action) || /delete|format|purchase|send|install|power off|password|credential/i.test(`${action} ${target.name || ''} ${target.role || ''}`); }
+export function assertSafeTarget(target) { if (isPasswordNode(target)) throw new Error('computer use refuses password or PAM controls'); }

computer-use/session.js

@@ -1,7 +1,7 @@
 const MAX_STEPS = 100;
 
 export class ComputerUseSession {
-  constructor({ stepsMax = 20, clock = () => Date.now() } = {}) {
+  constructor({ stepsMax = 20, clock = () => Date.now(), audit } = {}) {
     this.clock = clock;
     this.stepsMax = Math.min(MAX_STEPS, Math.max(1, stepsMax));
     this.active = false;
@@ -9,6 +9,7 @@ export class ComputerUseSession {
     this.sessionId = null;
     this.backend = 'none';
     this.expiresAt = null;
+    this.audit = audit;
   }
 
   grant({ persist = false, monitors = 'focused' } = {}) {
@@ -26,6 +27,7 @@ export class ComputerUseSession {
     this.sessionId = null;
     this.expiresAt = null;
     this.backend = 'none';
+    void this.audit?.wipeTemp?.();
   }
 
   status() {