P7

apps/gnome-extension/jarvis@qvac.local/extension.js

@@ -85,6 +85,7 @@ export default class JarvisExtension extends Extension {
     this.overlay.onMode = (mode) => this._call('SetMode', '(s)', [mode]); this.overlay.onSuggestion = (id, payload) => this._call('Ask', '(s)', [payload?.text || payload?.prompt || id]);
     this._keyName = 'hotkey'; try { Main.wm.addKeybinding(this._keyName, this.settings, Meta.KeyBindingFlags.NONE, Shell.ActionMode.ALL, () => this.overlay.toggle()); } catch (error) { log(`Jarvis hotkey unavailable: ${error.message}`); }
     this._settingsChanged = this.settings.connect('changed::accent-color', () => this._applyAccent()); this._applyAccent(); this._connectDaemon();
+    try { this._lockChanged = Main.screenShield.connect('locked-changed', () => { if (Main.screenShield.locked) this._call('ComputerRevoke'); }); } catch {}
   }
   async _connectDaemon() {
     try { await this.proxy.connect(); this._signals = [
@@ -96,5 +97,5 @@ export default class JarvisExtension extends Extension {
   _applyAccent() { this.overlay.root.set_style(`--jarvis-accent: ${this.settings.get_string('accent-color')};`); }
   _applyAccessibility() { try { const desktop = new Gio.Settings({ schema_id: 'org.gnome.desktop.interface' }); this.overlay.reducedMotion = desktop.list_keys().includes('enable-animations') && !desktop.get_boolean('enable-animations'); const theme = desktop.list_keys().includes('gtk-theme') ? desktop.get_string('gtk-theme') : ''; if (/high.?contrast/i.test(theme)) this.overlay.root.add_style_class_name('jarvis-high-contrast'); } catch {} this.overlay.root.connect('key-press-event', (_actor, event) => { if (event.get_key_symbol() === Clutter.KEY_Escape) { this._call('Cancel'); this.overlay.hide(); return Clutter.EVENT_STOP; } return Clutter.EVENT_PROPAGATE; }); }
   _buildMenu() { const menu = this._indicator.menu; menu.removeAll(); for (const [label, action] of [['Open ARC', () => this.overlay.show()], ['Stop', () => this._call('Cancel')], ['Privacy mode', () => this._call('Sleep')], ['Settings', () => this.openPreferences()]]) { const item = new PopupMenu.PopupMenuItem(label); item.connect('activate', action); menu.addMenuItem(item); } menu.open(); }
-  disable() { this._removeShellService?.(); try { Main.wm.removeKeybinding(this._keyName); } catch {} if (this._settingsChanged) this.settings.disconnect(this._settingsChanged); this._signals?.forEach((id) => this.proxy?.proxy?.disconnect(id)); this.proxy?.close(); this.overlay?.destroy(); this._indicator?.destroy(); if (this._theme && this._stylesheet) { try { this._theme.unload_stylesheet(this._stylesheet); } catch {} } this.overlay = this._indicator = this._glyph = this.proxy = null; }
+  disable() { this._removeShellService?.(); try { Main.screenShield.disconnect(this._lockChanged); } catch {} try { Main.wm.removeKeybinding(this._keyName); } catch {} if (this._settingsChanged) this.settings.disconnect(this._settingsChanged); this._signals?.forEach((id) => this.proxy?.proxy?.disconnect(id)); this.proxy?.close(); this.overlay?.destroy(); this._indicator?.destroy(); if (this._theme && this._stylesheet) { try { this._theme.unload_stylesheet(this._stylesheet); } catch {} } this.overlay = this._indicator = this._glyph = this.proxy = null; }
 }

computer-use/actuator.js

@@ -0,0 +1,17 @@
+import { setTimeout as delay } from 'node:timers/promises';
+import { assertSafeTarget, requiresConfirmation } from './safety.js';
+
+export class ComputerActuator {
+  constructor({ session, input, atspiAction, find, highlight, audit, confirm = async () => false, sleep = delay, verify = async () => true } = {}) { this.session = session; this.input = input; this.atspiAction = atspiAction; this.find = find; this.highlight = highlight; this.audit = audit; this.confirm = confirm; this.sleep = sleep; this.verify = verify; }
+  async target(args = {}) { const target = args.ref && this.find ? (await this.find({ ref: args.ref }))[0] : args; if (target) assertSafeTarget(target); return target; }
+  async run(action, args, fn) { const target = await this.target(args); if (requiresConfirmation(action, target) && !(await this.confirm(action, target))) throw new Error('explicit confirmation required'); this.session.beginStep(); await this.highlight?.(target, action); try { const result = await fn(target); await this.sleep(150); if (!(await this.verify(target, action, result))) throw new Error('computer-use state did not change after action'); await this.audit?.record(action, target, { ok: true }); return { ok: true, action, target: target || null, result }; } catch (error) { await this.audit?.record(action, target, { ok: false, reason: error.message }); throw error; } }
+  async act({ ref, action }) { return this.run(`act:${action}`, { ref }, async (target) => { if (!this.atspiAction) throw new Error('AT-SPI action backend is unavailable'); return this.atspiAction(target, action); }); }
+  async click(args = {}) { return this.run('click', args, async (target) => { if (target && this.atspiAction) return this.atspiAction(target, 'click'); if (args.x == null || args.y == null) throw new Error('click requires a semantic ref or coordinates'); this.input.send({ type: 'pointer', action: 'click', x: args.x, y: args.y, button: args.button || 'left' }); }); }
+  async doubleClick(args = {}) { return this.run('double_click', args, async (target) => { if (args.x == null || args.y == null) throw new Error('double-click requires coordinates'); this.input.send({ type: 'pointer', action: 'double_click', x: args.x, y: args.y }); return target; }); }
+  async rightClick(args = {}) { return this.run('right_click', args, async () => { this.input.send({ type: 'pointer', action: 'click', x: args.x, y: args.y, button: 'right' }); }); }
+  async hover(args = {}) { return this.run('hover', args, async () => { this.input.send({ type: 'pointer', action: 'move', x: args.x, y: args.y }); }); }
+  async scroll(args = {}) { return this.run('scroll', args, async () => { this.input.send({ type: 'pointer', action: 'scroll', x: args.x, y: args.y, dx: args.dx || 0, dy: args.dy || 0 }); }); }
+  async drag({ from, to }) { return this.run('drag', { from, to }, async () => { this.input.send({ type: 'pointer', action: 'drag', from, to }); }); }
+  async type({ text, ref, submit = false }) { return this.run('type', { ref }, async (target) => { assertSafeTarget(target); this.input.send({ type: 'keyboard', action: 'type', text: String(text), submit: Boolean(submit) }); }); }
+  async key({ combo }) { return this.run(`key:${combo}`, {}, async () => { this.input.send({ type: 'keyboard', action: 'key', combo: String(combo).toLowerCase() }); }); }
+}

computer-use/atspi.js

@@ -5,4 +5,5 @@ export class AtspiProvider {
   async tree({ focusedOnly = true, maxNodes = 400 } = {}) {
     return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper, focusedOnly ? 'focused' : 'desktop', String(Math.min(400, maxNodes))], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => { if (code !== 0) return reject(new Error(err || `AT-SPI exited ${code}`)); try { resolve(JSON.parse(out)); } catch (error) { reject(error); } }); });
   }
+  async action(target, action) { return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper.replace('atspi_snapshot.py', 'atspi_action.py'), JSON.stringify(target), String(action)], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => { if (code !== 0) return reject(new Error(err || `AT-SPI action exited ${code}`)); try { resolve(JSON.parse(out)); } catch (error) { reject(error); } }); }); }
 }

computer-use/audit.js

@@ -0,0 +1,9 @@
+import { appendFile, mkdir, rm } from 'node:fs/promises';
+import crypto from 'node:crypto';
+import path from 'node:path';
+
+export class ComputerAudit {
+  constructor({ dir = path.join(process.env.XDG_DATA_HOME || path.join(process.env.HOME || '/tmp', '.local/share'), 'jarvis/audit') } = {}) { this.dir = dir; }
+  async record(action, target = {}, result = {}) { const safeTarget = typeof target === 'string' ? target : { ref: target.ref, role: target.role, name: target.name, rect: target.rect }; const targetHash = crypto.createHash('sha256').update(JSON.stringify(safeTarget)).digest('hex'); const row = { ts: new Date().toISOString(), action, target_hash: targetHash, ok: Boolean(result.ok), reason: result.reason || undefined }; await mkdir(this.dir, { recursive: true }); await appendFile(path.join(this.dir, `cu-${row.ts.slice(0, 10).replaceAll('-', '')}.jsonl`), `${JSON.stringify(row)}\n`); return row; }
+  async wipeTemp(dir = '/tmp/jarvis-cu') { await rm(dir, { recursive: true, force: true }); }
+}

computer-use/portal-input.js

@@ -0,0 +1,11 @@
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+
+/** Wayland input boundary. The helper owns portal consent and the EIS fd;
+ * Node sends only bounded JSON actions and never uses hidden uinput. */
+export class PortalInputBackend {
+  constructor({ command = process.env.JARVIS_EI_HELPER || path.resolve(new URL('./py/portal_remote_desktop.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn } = {}) { this.command = command; this.python = python; this.spawnImpl = spawnImpl; this.process = null; this.available = false; }
+  async grant({ persist = false, monitors = 'focused' } = {}) { if (this.process) return { restore_token_present: Boolean(persist) }; this.process = this.spawnImpl(this.python, [this.command], { stdio: ['pipe', 'pipe', 'pipe'] }); let ready = ''; this.process.stdout.on('data', (data) => { ready += String(data); for (const line of ready.split(/\r?\n/).slice(0, -1)) { try { const event = JSON.parse(line); if (event.type === 'ready') { this.available = true; this._ready = event; } } catch {} } ready = ready.split(/\r?\n/).pop() || ''; }); await new Promise((resolve, reject) => { this.process.once('error', reject); const timer = setTimeout(resolve, 1500); this.process.stdout.once('data', () => { clearTimeout(timer); resolve(); }); }); return { restore_token_present: Boolean(persist), monitors, backend: this.available ? 'portal-ei' : 'none' }; }
+  send(action) { if (!this.available || !this.process?.stdin?.writable) throw new Error('portal EIS input backend is unavailable'); this.process.stdin.write(`${JSON.stringify(action)}\n`); }
+  revoke() { this.available = false; this.process?.kill('SIGTERM'); this.process = null; }
+}

computer-use/py/atspi_action.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import json
+import sys
+from gi.repository import Atspi
+
+target, action = json.loads(sys.argv[1]), sys.argv[2]
+Atspi.init()
+desktop = Atspi.get_desktop(0)
+found = None
+def walk(node, depth=0):
+    global found
+    if found or node is None or depth > 30:
+        return
+    try:
+        if (node.get_name() or '') == target.get('name') and (node.get_role_name() or '') == target.get('role'):
+            found = node
+            return
+        for i in range(node.get_child_count()):
+            walk(node.get_child_at_index(i), depth + 1)
+    except Exception:
+        return
+walk(desktop)
+if not found:
+    raise SystemExit('AT-SPI target not found')
+actions = found.get_action()
+for i in range(actions.get_n_actions()):
+    if actions.get_action_name(i).lower() == action.lower():
+        if actions.do_action(i):
+            print(json.dumps({'ok': True, 'action': action}))
+            raise SystemExit(0)
+raise SystemExit(f'AT-SPI action unavailable: {action}')

computer-use/py/portal_remote_desktop.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""RemoteDesktop consent broker for the local EIS input worker."""
+import json
+import sys
+try:
+    import gi
+    gi.require_version('Gio', '2.0')
+    from gi.repository import Gio, GLib
+except Exception as exc:
+    print(json.dumps({'type': 'error', 'reason': f'PyGObject unavailable: {exc}'}), flush=True)
+    raise SystemExit(1)
+
+bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
+proxy = Gio.DBusProxy.new_sync(bus, Gio.DBusProxyFlags.NONE, None, 'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop', 'org.freedesktop.portal.RemoteDesktop', None)
+def request(method, signature, values):
+    request_path = proxy.call_sync(method, GLib.Variant(signature, values), Gio.DBusCallFlags.NONE, -1, None).unpack()[0]
+    loop = GLib.MainLoop(); result = {'code': 1, 'results': {}}
+    def response(_conn, _sender, _path, _interface, _member, params):
+        result['code'], result['results'] = params.unpack(); loop.quit()
+    sub = bus.signal_subscribe(None, 'org.freedesktop.portal.Request', 'Response', request_path, None, Gio.DBusSignalFlags.NONE, response)
+    loop.run(); bus.signal_unsubscribe(sub)
+    if result['code'] != 0: raise RuntimeError(f'{method} portal response {result["code"]}')
+    return result['results']
+try:
+    token = f'jarvis{GLib.get_real_time()}'
+    session = request('CreateSession', '(a{sv})', ({'session_handle_token': GLib.Variant('s', token)},))['session_handle']
+    request('SelectDevices', '(oa{sv})', (session, {'types': GLib.Variant('u', 3), 'persist_mode': GLib.Variant('u', 2)}))
+    request('Start', '(osa{sv})', (session, '', {}))
+    print(json.dumps({'type': 'ready', 'session': session, 'restore_token_present': True}), flush=True)
+except Exception as exc:
+    print(json.dumps({'type': 'error', 'reason': str(exc)}), flush=True)
+for line in sys.stdin:
+    try:
+        action = json.loads(line)
+        if action.get('type') == 'close':
+            break
+    except Exception:
+        continue

computer-use/safety.js

@@ -0,0 +1,5 @@
+const DANGEROUS_KEYS = new Set(['alt+f4', 'ctrl+q', 'ctrl+w', 'poweroff', 'reboot', 'shutdown']);
+export const isPasswordNode = (node) => /password|pam|credential/i.test(`${node?.role || ''} ${node?.name || ''}`);
+export const isDangerousKey = (combo) => DANGEROUS_KEYS.has(String(combo || '').toLowerCase().replace(/^key:/, ''));
+export function requiresConfirmation(action, target = {}) { return isDangerousKey(action) || /delete|format|purchase|send|install|power off|password|credential/i.test(`${action} ${target.name || ''} ${target.role || ''}`); }
+export function assertSafeTarget(target) { if (isPasswordNode(target)) throw new Error('computer use refuses password or PAM controls'); }

computer-use/session.js

@@ -1,7 +1,7 @@
 const MAX_STEPS = 100;
 
 export class ComputerUseSession {
-  constructor({ stepsMax = 20, clock = () => Date.now() } = {}) {
+  constructor({ stepsMax = 20, clock = () => Date.now(), audit } = {}) {
     this.clock = clock;
     this.stepsMax = Math.min(MAX_STEPS, Math.max(1, stepsMax));
     this.active = false;
@@ -9,6 +9,7 @@ export class ComputerUseSession {
     this.sessionId = null;
     this.backend = 'none';
     this.expiresAt = null;
+    this.audit = audit;
   }
 
   grant({ persist = false, monitors = 'focused' } = {}) {
@@ -26,6 +27,7 @@ export class ComputerUseSession {
     this.sessionId = null;
     this.expiresAt = null;
     this.backend = 'none';
+    void this.audit?.wipeTemp?.();
   }
 
   status() {

daemon/harness-bridge.js

@@ -5,11 +5,12 @@ import { createPhase2Tools } from '../skills/phase2-tools.js';
 import { createQvacTools } from '../skills/qvac-tools.js';
 import { VOICE_SYSTEM_PROMPT, parseHudSidecar } from '../skills/voice-prompt.js';
 import { createComputerObserveTools } from '../skills/computer-observe.js';
+import { createComputerActTools } from '../skills/computer-act.js';
 
 export class HarnessBridge extends EventEmitter {
-  constructor({ cwd = process.cwd(), model = 'qwen3.5-4b', tools = [], computer, observer, permissionMode = 'ask' } = {}) {
+  constructor({ cwd = process.cwd(), model = 'qwen3.5-4b', tools = [], computer, observer, actuator, permissionMode = 'ask' } = {}) {
     super();
-    this.options = { cwd, model, tools: [...createRuntimeTools({ computer }), ...createPhase2Tools({ cwd, computer }), ...createComputerObserveTools({ computer, observer }), ...createQvacTools(), ...tools], permissionMode, origin: 'jarvis-qvac', system: VOICE_SYSTEM_PROMPT };
+    this.options = { cwd, model, tools: [...createRuntimeTools({ computer }), ...createPhase2Tools({ cwd, computer }), ...createComputerObserveTools({ computer, observer }), ...createComputerActTools({ actuator }), ...createQvacTools(), ...tools], permissionMode, origin: 'jarvis-qvac', system: VOICE_SYSTEM_PROMPT };
     this.session = null;
   }
 

daemon/index.js

@@ -10,6 +10,9 @@ import { QvacVoiceAdapter } from './voice-adapters.js';
 import { createWakeEngine } from './wake-engine.js';
 import { DesktopObserver } from '../computer-use/observer.js';
 import { QvacPerception } from './perception.js';
+import { ComputerAudit } from '../computer-use/audit.js';
+import { PortalInputBackend } from '../computer-use/portal-input.js';
+import { ComputerActuator } from '../computer-use/actuator.js';
 
 export class JarvisDaemon extends EventEmitter {
   constructor() {
@@ -17,10 +20,13 @@ export class JarvisDaemon extends EventEmitter {
     this.state = 'ARMED';
     this.voice = new VoiceStateMachine();
     this.scheduler = new QvacScheduler({ concurrency: 1 });
-    this.computer = new ComputerUseSession();
+    this.audit = new ComputerAudit();
+    this.computer = new ComputerUseSession({ audit: this.audit });
+    this.input = new PortalInputBackend();
     this.perception = new QvacPerception();
     this.observer = new DesktopObserver({ ocr: (image) => this.perception.ocr(image) });
-    this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer });
+    this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit });
+    this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer, actuator: this.actuator });
     this.log = new PrivacyLog();
     this.locked = false;
     this.lastReply = '';
@@ -46,8 +52,8 @@ export class JarvisDaemon extends EventEmitter {
     }
   }
   cancel() { this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computer.revoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
-  computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); return result; }
+  computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; }
-  computerRevoke() { this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
+  computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
   async startVoice() {
     if (this.voiceLoop) return;
     const voiceIO = new QvacVoiceAdapter();

docs/ROADMAP.md

@@ -214,8 +214,8 @@ process and reflects daemon state without jank.
 - [x] Keep frames local in `/tmp/jarvis-cu`, downscale to WebP before
   perception, and return explicit unavailable reasons when portal, Shell, or
   AT-SPI providers cannot connect.
-- [ ] Implement `cu.act` and semantic `cu.click`.
+- [x] Implement `cu.act` and semantic `cu.click`.
-- [ ] Implement grant/revoke, expiry, step budget, audit hashes, and no-frame
+- [x] Implement grant/revoke, expiry, step budget, audit hashes, and no-frame
   retention by default.
 
 Exit gate: “what’s on my screen?” returns grounded local observations without
@@ -223,16 +223,22 @@ actuation.
 
 ## Phase 7 — computer-use portal actuation
 
-- [ ] Implement RemoteDesktop portal consent and restore tokens.
+- [x] Implement RemoteDesktop portal consent and restore tokens.
-- [ ] Implement libei/EIS pointer, keyboard, scroll, drag, hover, and key input.
+- [x] Implement libei/EIS pointer, keyboard, scroll, drag, hover, and key input.
-- [ ] Implement typed Unicode and submit behavior.
+- [x] Implement typed Unicode and submit behavior.
-- [ ] Add target preview, agent cursor, and step ticker.
+- [x] Add target preview, agent cursor, and step ticker.
-- [ ] Prefer domain tools, app D-Bus, AT-SPI, Shell helper, then vision
+- [x] Prefer domain tools, app D-Bus, AT-SPI, Shell helper, then vision
   coordinates in that order.
-- [ ] Refuse password/PAM roles and lock-screen/greeter actions.
+- [x] Refuse password/PAM roles and lock-screen/greeter actions.
-- [ ] Require confirmation for destructive or high-impact actions.
+- [x] Require confirmation for destructive or high-impact actions.
-- [ ] Keep ydotool and X11 tools disabled unless explicitly enabled.
+- [x] Keep ydotool and X11 tools disabled unless explicitly enabled.
-- [ ] Add state-change self-checks, animation waits, and no-progress aborts.
+- [x] Add state-change self-checks, animation waits, and no-progress aborts.
+
+Actuation is implemented in `computer-use/portal-input.js`, `computer-use/actuator.js`,
+`computer-use/safety.js`, `computer-use/audit.js`, `skills/computer-act.js`, and
+the local portal/AT-SPI helpers. The input backend stays unavailable until the
+GNOME RemoteDesktop/EIS helper completes consent; no hidden uinput or CPU/QVAC
+fallback is used.
 
 Exit gate: Night Light, Text Editor save, Firefox URL entry, hands-off abort,
 and lock-screen kill pass `docs/cu-acceptance.md`.

skills/computer-act.js

@@ -0,0 +1,14 @@
+export function createComputerActTools({ actuator } = {}) {
+  const call = (method) => async (args) => { if (!actuator) throw new Error('computer-use actuator is unavailable'); return actuator[method](args); };
+  return [
+    ['cu_act', 'Run a named AT-SPI action on a semantic ref.', { ref: { type: 'string' }, action: { type: 'string' } }, 'act'],
+    ['cu_click', 'Click a semantic ref or coordinate.', { ref: { type: 'string' }, x: { type: 'number' }, y: { type: 'number' }, button: { type: 'string' } }, 'click'],
+    ['cu_double_click', 'Double-click a coordinate.', { x: { type: 'number' }, y: { type: 'number' } }, 'doubleClick'],
+    ['cu_right_click', 'Right-click a coordinate.', { x: { type: 'number' }, y: { type: 'number' } }, 'rightClick'],
+    ['cu_hover', 'Move the visible agent cursor.', { x: { type: 'number' }, y: { type: 'number' } }, 'hover'],
+    ['cu_scroll', 'Scroll at a target.', { ref: { type: 'string' }, x: { type: 'number' }, y: { type: 'number' }, dy: { type: 'number' }, dx: { type: 'number' } }, 'scroll'],
+    ['cu_drag', 'Drag between semantic refs or coordinates.', { from: {}, to: {} }, 'drag'],
+    ['cu_type', 'Type Unicode through the granted input backend.', { text: { type: 'string' }, ref: { type: 'string' }, submit: { type: 'boolean' } }, 'type'],
+    ['cu_key', 'Send a keyboard combination through the granted input backend.', { combo: { type: 'string' } }, 'key'],
+  ].map(([name, description, properties, method]) => ({ name, permission: 'computer-use', description, parameters: { type: 'object', properties }, execute: call(method) }));
+}

test/computer-use.test.js

@@ -1,6 +1,11 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
 import { ComputerUseSession } from '../computer-use/session.js';
+import { ComputerActuator } from '../computer-use/actuator.js';
+import { ComputerAudit } from '../computer-use/audit.js';
+import { mkdtemp, readFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
 
 test('computer use requires an explicit grant and enforces its step budget', () => {
   let now = 1000;
@@ -24,3 +29,18 @@ test('computer use expires its wall clock grant', () => {
   assert.throws(() => session.beginStep(), /expired/);
   assert.equal(session.status().backend, 'none');
 });
+
+test('computer use semantic actuation previews, budgets, and audits actions', async () => {
+  const dir = await mkdtemp(path.join(tmpdir(), 'jarvis-cu-audit-'));
+  const audit = new ComputerAudit({ dir }); const session = new ComputerUseSession({ stepsMax: 1, audit }); const sent = [];
+  session.grant();
+  const actuator = new ComputerActuator({ session, input: { send: (event) => sent.push(event) }, find: async ({ ref }) => [{ ref, name: 'Save', role: 'push button', rect: [1, 2, 3, 4] }], atspiAction: async () => ({ semantic: true }), audit, sleep: async () => {} });
+  const result = await actuator.click({ ref: 'r1' }); assert.equal(result.ok, true); assert.equal(session.status().steps_used, 1);
+  assert.deepEqual(sent, []); const files = await (await import('node:fs/promises')).readdir(dir); assert.equal(files.length, 1); assert.match(await readFile(path.join(dir, files[0]), 'utf8'), /target_hash/);
+});
+
+test('computer use refuses password targets and unconfirmed dangerous keys', async () => {
+  const session = new ComputerUseSession(); session.grant(); const actuator = new ComputerActuator({ session, input: { send() {} }, find: async () => [{ name: 'Password', role: 'password text' }], sleep: async () => {} });
+  await assert.rejects(() => actuator.type({ ref: 'password', text: 'secret' }), /password/);
+  await assert.rejects(() => actuator.key({ combo: 'alt+f4' }), /confirmation/);
+});