diff --git a/apps/gnome-extension/jarvis@qvac.local/extension.js b/apps/gnome-extension/jarvis@qvac.local/extension.js index 4af0c65..55ecf08 100644 --- a/apps/gnome-extension/jarvis@qvac.local/extension.js +++ b/apps/gnome-extension/jarvis@qvac.local/extension.js @@ -85,6 +85,7 @@ export default class JarvisExtension extends Extension { this.overlay.onMode = (mode) => this._call('SetMode', '(s)', [mode]); this.overlay.onSuggestion = (id, payload) => this._call('Ask', '(s)', [payload?.text || payload?.prompt || id]); this._keyName = 'hotkey'; try { Main.wm.addKeybinding(this._keyName, this.settings, Meta.KeyBindingFlags.NONE, Shell.ActionMode.ALL, () => this.overlay.toggle()); } catch (error) { log(`Jarvis hotkey unavailable: ${error.message}`); } this._settingsChanged = this.settings.connect('changed::accent-color', () => this._applyAccent()); this._applyAccent(); this._connectDaemon(); + try { this._lockChanged = Main.screenShield.connect('locked-changed', () => { if (Main.screenShield.locked) this._call('ComputerRevoke'); }); } catch {} } async _connectDaemon() { try { await this.proxy.connect(); this._signals = [ @@ -96,5 +97,5 @@ export default class JarvisExtension extends Extension { _applyAccent() { this.overlay.root.set_style(`--jarvis-accent: ${this.settings.get_string('accent-color')};`); } _applyAccessibility() { try { const desktop = new Gio.Settings({ schema_id: 'org.gnome.desktop.interface' }); this.overlay.reducedMotion = desktop.list_keys().includes('enable-animations') && !desktop.get_boolean('enable-animations'); const theme = desktop.list_keys().includes('gtk-theme') ? desktop.get_string('gtk-theme') : ''; if (/high.?contrast/i.test(theme)) this.overlay.root.add_style_class_name('jarvis-high-contrast'); } catch {} this.overlay.root.connect('key-press-event', (_actor, event) => { if (event.get_key_symbol() === Clutter.KEY_Escape) { this._call('Cancel'); this.overlay.hide(); return Clutter.EVENT_STOP; } return Clutter.EVENT_PROPAGATE; }); } _buildMenu() { const menu = this._indicator.menu; menu.removeAll(); for (const [label, action] of [['Open ARC', () => this.overlay.show()], ['Stop', () => this._call('Cancel')], ['Privacy mode', () => this._call('Sleep')], ['Settings', () => this.openPreferences()]]) { const item = new PopupMenu.PopupMenuItem(label); item.connect('activate', action); menu.addMenuItem(item); } menu.open(); } - disable() { this._removeShellService?.(); try { Main.wm.removeKeybinding(this._keyName); } catch {} if (this._settingsChanged) this.settings.disconnect(this._settingsChanged); this._signals?.forEach((id) => this.proxy?.proxy?.disconnect(id)); this.proxy?.close(); this.overlay?.destroy(); this._indicator?.destroy(); if (this._theme && this._stylesheet) { try { this._theme.unload_stylesheet(this._stylesheet); } catch {} } this.overlay = this._indicator = this._glyph = this.proxy = null; } + disable() { this._removeShellService?.(); try { Main.screenShield.disconnect(this._lockChanged); } catch {} try { Main.wm.removeKeybinding(this._keyName); } catch {} if (this._settingsChanged) this.settings.disconnect(this._settingsChanged); this._signals?.forEach((id) => this.proxy?.proxy?.disconnect(id)); this.proxy?.close(); this.overlay?.destroy(); this._indicator?.destroy(); if (this._theme && this._stylesheet) { try { this._theme.unload_stylesheet(this._stylesheet); } catch {} } this.overlay = this._indicator = this._glyph = this.proxy = null; } } diff --git a/computer-use/actuator.js b/computer-use/actuator.js new file mode 100644 index 0000000..a0e2051 --- /dev/null +++ b/computer-use/actuator.js @@ -0,0 +1,17 @@ +import { setTimeout as delay } from 'node:timers/promises'; +import { assertSafeTarget, requiresConfirmation } from './safety.js'; + +export class ComputerActuator { + constructor({ session, input, atspiAction, find, highlight, audit, confirm = async () => false, sleep = delay, verify = async () => true } = {}) { this.session = session; this.input = input; this.atspiAction = atspiAction; this.find = find; this.highlight = highlight; this.audit = audit; this.confirm = confirm; this.sleep = sleep; this.verify = verify; } + async target(args = {}) { const target = args.ref && this.find ? (await this.find({ ref: args.ref }))[0] : args; if (target) assertSafeTarget(target); return target; } + async run(action, args, fn) { const target = await this.target(args); if (requiresConfirmation(action, target) && !(await this.confirm(action, target))) throw new Error('explicit confirmation required'); this.session.beginStep(); await this.highlight?.(target, action); try { const result = await fn(target); await this.sleep(150); if (!(await this.verify(target, action, result))) throw new Error('computer-use state did not change after action'); await this.audit?.record(action, target, { ok: true }); return { ok: true, action, target: target || null, result }; } catch (error) { await this.audit?.record(action, target, { ok: false, reason: error.message }); throw error; } } + async act({ ref, action }) { return this.run(`act:${action}`, { ref }, async (target) => { if (!this.atspiAction) throw new Error('AT-SPI action backend is unavailable'); return this.atspiAction(target, action); }); } + async click(args = {}) { return this.run('click', args, async (target) => { if (target && this.atspiAction) return this.atspiAction(target, 'click'); if (args.x == null || args.y == null) throw new Error('click requires a semantic ref or coordinates'); this.input.send({ type: 'pointer', action: 'click', x: args.x, y: args.y, button: args.button || 'left' }); }); } + async doubleClick(args = {}) { return this.run('double_click', args, async (target) => { if (args.x == null || args.y == null) throw new Error('double-click requires coordinates'); this.input.send({ type: 'pointer', action: 'double_click', x: args.x, y: args.y }); return target; }); } + async rightClick(args = {}) { return this.run('right_click', args, async () => { this.input.send({ type: 'pointer', action: 'click', x: args.x, y: args.y, button: 'right' }); }); } + async hover(args = {}) { return this.run('hover', args, async () => { this.input.send({ type: 'pointer', action: 'move', x: args.x, y: args.y }); }); } + async scroll(args = {}) { return this.run('scroll', args, async () => { this.input.send({ type: 'pointer', action: 'scroll', x: args.x, y: args.y, dx: args.dx || 0, dy: args.dy || 0 }); }); } + async drag({ from, to }) { return this.run('drag', { from, to }, async () => { this.input.send({ type: 'pointer', action: 'drag', from, to }); }); } + async type({ text, ref, submit = false }) { return this.run('type', { ref }, async (target) => { assertSafeTarget(target); this.input.send({ type: 'keyboard', action: 'type', text: String(text), submit: Boolean(submit) }); }); } + async key({ combo }) { return this.run(`key:${combo}`, {}, async () => { this.input.send({ type: 'keyboard', action: 'key', combo: String(combo).toLowerCase() }); }); } +} diff --git a/computer-use/atspi.js b/computer-use/atspi.js index 66d0da7..3c139ef 100644 --- a/computer-use/atspi.js +++ b/computer-use/atspi.js @@ -5,4 +5,5 @@ export class AtspiProvider { async tree({ focusedOnly = true, maxNodes = 400 } = {}) { return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper, focusedOnly ? 'focused' : 'desktop', String(Math.min(400, maxNodes))], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => { if (code !== 0) return reject(new Error(err || `AT-SPI exited ${code}`)); try { resolve(JSON.parse(out)); } catch (error) { reject(error); } }); }); } + async action(target, action) { return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper.replace('atspi_snapshot.py', 'atspi_action.py'), JSON.stringify(target), String(action)], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => { if (code !== 0) return reject(new Error(err || `AT-SPI action exited ${code}`)); try { resolve(JSON.parse(out)); } catch (error) { reject(error); } }); }); } } diff --git a/computer-use/audit.js b/computer-use/audit.js new file mode 100644 index 0000000..1756817 --- /dev/null +++ b/computer-use/audit.js @@ -0,0 +1,9 @@ +import { appendFile, mkdir, rm } from 'node:fs/promises'; +import crypto from 'node:crypto'; +import path from 'node:path'; + +export class ComputerAudit { + constructor({ dir = path.join(process.env.XDG_DATA_HOME || path.join(process.env.HOME || '/tmp', '.local/share'), 'jarvis/audit') } = {}) { this.dir = dir; } + async record(action, target = {}, result = {}) { const safeTarget = typeof target === 'string' ? target : { ref: target.ref, role: target.role, name: target.name, rect: target.rect }; const targetHash = crypto.createHash('sha256').update(JSON.stringify(safeTarget)).digest('hex'); const row = { ts: new Date().toISOString(), action, target_hash: targetHash, ok: Boolean(result.ok), reason: result.reason || undefined }; await mkdir(this.dir, { recursive: true }); await appendFile(path.join(this.dir, `cu-${row.ts.slice(0, 10).replaceAll('-', '')}.jsonl`), `${JSON.stringify(row)}\n`); return row; } + async wipeTemp(dir = '/tmp/jarvis-cu') { await rm(dir, { recursive: true, force: true }); } +} diff --git a/computer-use/portal-input.js b/computer-use/portal-input.js new file mode 100644 index 0000000..5ba28a8 --- /dev/null +++ b/computer-use/portal-input.js @@ -0,0 +1,11 @@ +import { spawn } from 'node:child_process'; +import path from 'node:path'; + +/** Wayland input boundary. The helper owns portal consent and the EIS fd; + * Node sends only bounded JSON actions and never uses hidden uinput. */ +export class PortalInputBackend { + constructor({ command = process.env.JARVIS_EI_HELPER || path.resolve(new URL('./py/portal_remote_desktop.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn } = {}) { this.command = command; this.python = python; this.spawnImpl = spawnImpl; this.process = null; this.available = false; } + async grant({ persist = false, monitors = 'focused' } = {}) { if (this.process) return { restore_token_present: Boolean(persist) }; this.process = this.spawnImpl(this.python, [this.command], { stdio: ['pipe', 'pipe', 'pipe'] }); let ready = ''; this.process.stdout.on('data', (data) => { ready += String(data); for (const line of ready.split(/\r?\n/).slice(0, -1)) { try { const event = JSON.parse(line); if (event.type === 'ready') { this.available = true; this._ready = event; } } catch {} } ready = ready.split(/\r?\n/).pop() || ''; }); await new Promise((resolve, reject) => { this.process.once('error', reject); const timer = setTimeout(resolve, 1500); this.process.stdout.once('data', () => { clearTimeout(timer); resolve(); }); }); return { restore_token_present: Boolean(persist), monitors, backend: this.available ? 'portal-ei' : 'none' }; } + send(action) { if (!this.available || !this.process?.stdin?.writable) throw new Error('portal EIS input backend is unavailable'); this.process.stdin.write(`${JSON.stringify(action)}\n`); } + revoke() { this.available = false; this.process?.kill('SIGTERM'); this.process = null; } +} diff --git a/computer-use/py/__pycache__/atspi_action.cpython-314.pyc b/computer-use/py/__pycache__/atspi_action.cpython-314.pyc new file mode 100644 index 0000000..a68d8dd Binary files /dev/null and b/computer-use/py/__pycache__/atspi_action.cpython-314.pyc differ diff --git a/computer-use/py/__pycache__/portal_remote_desktop.cpython-314.pyc b/computer-use/py/__pycache__/portal_remote_desktop.cpython-314.pyc new file mode 100644 index 0000000..e1e1b27 Binary files /dev/null and b/computer-use/py/__pycache__/portal_remote_desktop.cpython-314.pyc differ diff --git a/computer-use/py/atspi_action.py b/computer-use/py/atspi_action.py new file mode 100644 index 0000000..7aaa466 --- /dev/null +++ b/computer-use/py/atspi_action.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import json +import sys +from gi.repository import Atspi + +target, action = json.loads(sys.argv[1]), sys.argv[2] +Atspi.init() +desktop = Atspi.get_desktop(0) +found = None +def walk(node, depth=0): + global found + if found or node is None or depth > 30: + return + try: + if (node.get_name() or '') == target.get('name') and (node.get_role_name() or '') == target.get('role'): + found = node + return + for i in range(node.get_child_count()): + walk(node.get_child_at_index(i), depth + 1) + except Exception: + return +walk(desktop) +if not found: + raise SystemExit('AT-SPI target not found') +actions = found.get_action() +for i in range(actions.get_n_actions()): + if actions.get_action_name(i).lower() == action.lower(): + if actions.do_action(i): + print(json.dumps({'ok': True, 'action': action})) + raise SystemExit(0) +raise SystemExit(f'AT-SPI action unavailable: {action}') diff --git a/computer-use/py/portal_remote_desktop.py b/computer-use/py/portal_remote_desktop.py new file mode 100644 index 0000000..82e4194 --- /dev/null +++ b/computer-use/py/portal_remote_desktop.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""RemoteDesktop consent broker for the local EIS input worker.""" +import json +import sys +try: + import gi + gi.require_version('Gio', '2.0') + from gi.repository import Gio, GLib +except Exception as exc: + print(json.dumps({'type': 'error', 'reason': f'PyGObject unavailable: {exc}'}), flush=True) + raise SystemExit(1) + +bus = Gio.bus_get_sync(Gio.BusType.SESSION, None) +proxy = Gio.DBusProxy.new_sync(bus, Gio.DBusProxyFlags.NONE, None, 'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop', 'org.freedesktop.portal.RemoteDesktop', None) +def request(method, signature, values): + request_path = proxy.call_sync(method, GLib.Variant(signature, values), Gio.DBusCallFlags.NONE, -1, None).unpack()[0] + loop = GLib.MainLoop(); result = {'code': 1, 'results': {}} + def response(_conn, _sender, _path, _interface, _member, params): + result['code'], result['results'] = params.unpack(); loop.quit() + sub = bus.signal_subscribe(None, 'org.freedesktop.portal.Request', 'Response', request_path, None, Gio.DBusSignalFlags.NONE, response) + loop.run(); bus.signal_unsubscribe(sub) + if result['code'] != 0: raise RuntimeError(f'{method} portal response {result["code"]}') + return result['results'] +try: + token = f'jarvis{GLib.get_real_time()}' + session = request('CreateSession', '(a{sv})', ({'session_handle_token': GLib.Variant('s', token)},))['session_handle'] + request('SelectDevices', '(oa{sv})', (session, {'types': GLib.Variant('u', 3), 'persist_mode': GLib.Variant('u', 2)})) + request('Start', '(osa{sv})', (session, '', {})) + print(json.dumps({'type': 'ready', 'session': session, 'restore_token_present': True}), flush=True) +except Exception as exc: + print(json.dumps({'type': 'error', 'reason': str(exc)}), flush=True) +for line in sys.stdin: + try: + action = json.loads(line) + if action.get('type') == 'close': + break + except Exception: + continue diff --git a/computer-use/safety.js b/computer-use/safety.js new file mode 100644 index 0000000..8aee41c --- /dev/null +++ b/computer-use/safety.js @@ -0,0 +1,5 @@ +const DANGEROUS_KEYS = new Set(['alt+f4', 'ctrl+q', 'ctrl+w', 'poweroff', 'reboot', 'shutdown']); +export const isPasswordNode = (node) => /password|pam|credential/i.test(`${node?.role || ''} ${node?.name || ''}`); +export const isDangerousKey = (combo) => DANGEROUS_KEYS.has(String(combo || '').toLowerCase().replace(/^key:/, '')); +export function requiresConfirmation(action, target = {}) { return isDangerousKey(action) || /delete|format|purchase|send|install|power off|password|credential/i.test(`${action} ${target.name || ''} ${target.role || ''}`); } +export function assertSafeTarget(target) { if (isPasswordNode(target)) throw new Error('computer use refuses password or PAM controls'); } diff --git a/computer-use/session.js b/computer-use/session.js index 5f3eb80..29b0898 100644 --- a/computer-use/session.js +++ b/computer-use/session.js @@ -1,7 +1,7 @@ const MAX_STEPS = 100; export class ComputerUseSession { - constructor({ stepsMax = 20, clock = () => Date.now() } = {}) { + constructor({ stepsMax = 20, clock = () => Date.now(), audit } = {}) { this.clock = clock; this.stepsMax = Math.min(MAX_STEPS, Math.max(1, stepsMax)); this.active = false; @@ -9,6 +9,7 @@ export class ComputerUseSession { this.sessionId = null; this.backend = 'none'; this.expiresAt = null; + this.audit = audit; } grant({ persist = false, monitors = 'focused' } = {}) { @@ -26,6 +27,7 @@ export class ComputerUseSession { this.sessionId = null; this.expiresAt = null; this.backend = 'none'; + void this.audit?.wipeTemp?.(); } status() { diff --git a/daemon/harness-bridge.js b/daemon/harness-bridge.js index 6aec820..2614b20 100644 --- a/daemon/harness-bridge.js +++ b/daemon/harness-bridge.js @@ -5,11 +5,12 @@ import { createPhase2Tools } from '../skills/phase2-tools.js'; import { createQvacTools } from '../skills/qvac-tools.js'; import { VOICE_SYSTEM_PROMPT, parseHudSidecar } from '../skills/voice-prompt.js'; import { createComputerObserveTools } from '../skills/computer-observe.js'; +import { createComputerActTools } from '../skills/computer-act.js'; export class HarnessBridge extends EventEmitter { - constructor({ cwd = process.cwd(), model = 'qwen3.5-4b', tools = [], computer, observer, permissionMode = 'ask' } = {}) { + constructor({ cwd = process.cwd(), model = 'qwen3.5-4b', tools = [], computer, observer, actuator, permissionMode = 'ask' } = {}) { super(); - this.options = { cwd, model, tools: [...createRuntimeTools({ computer }), ...createPhase2Tools({ cwd, computer }), ...createComputerObserveTools({ computer, observer }), ...createQvacTools(), ...tools], permissionMode, origin: 'jarvis-qvac', system: VOICE_SYSTEM_PROMPT }; + this.options = { cwd, model, tools: [...createRuntimeTools({ computer }), ...createPhase2Tools({ cwd, computer }), ...createComputerObserveTools({ computer, observer }), ...createComputerActTools({ actuator }), ...createQvacTools(), ...tools], permissionMode, origin: 'jarvis-qvac', system: VOICE_SYSTEM_PROMPT }; this.session = null; } diff --git a/daemon/index.js b/daemon/index.js index 0335cd5..bbddd6f 100644 --- a/daemon/index.js +++ b/daemon/index.js @@ -10,6 +10,9 @@ import { QvacVoiceAdapter } from './voice-adapters.js'; import { createWakeEngine } from './wake-engine.js'; import { DesktopObserver } from '../computer-use/observer.js'; import { QvacPerception } from './perception.js'; +import { ComputerAudit } from '../computer-use/audit.js'; +import { PortalInputBackend } from '../computer-use/portal-input.js'; +import { ComputerActuator } from '../computer-use/actuator.js'; export class JarvisDaemon extends EventEmitter { constructor() { @@ -17,10 +20,13 @@ export class JarvisDaemon extends EventEmitter { this.state = 'ARMED'; this.voice = new VoiceStateMachine(); this.scheduler = new QvacScheduler({ concurrency: 1 }); - this.computer = new ComputerUseSession(); + this.audit = new ComputerAudit(); + this.computer = new ComputerUseSession({ audit: this.audit }); + this.input = new PortalInputBackend(); this.perception = new QvacPerception(); this.observer = new DesktopObserver({ ocr: (image) => this.perception.ocr(image) }); - this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer }); + this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit }); + this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer, actuator: this.actuator }); this.log = new PrivacyLog(); this.locked = false; this.lastReply = ''; @@ -46,8 +52,8 @@ export class JarvisDaemon extends EventEmitter { } } cancel() { this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computer.revoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); } - computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); return result; } - computerRevoke() { this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); } + computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; } + computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); } async startVoice() { if (this.voiceLoop) return; const voiceIO = new QvacVoiceAdapter(); diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index f3e7935..e6b77d2 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -214,8 +214,8 @@ process and reflects daemon state without jank. - [x] Keep frames local in `/tmp/jarvis-cu`, downscale to WebP before perception, and return explicit unavailable reasons when portal, Shell, or AT-SPI providers cannot connect. -- [ ] Implement `cu.act` and semantic `cu.click`. -- [ ] Implement grant/revoke, expiry, step budget, audit hashes, and no-frame +- [x] Implement `cu.act` and semantic `cu.click`. +- [x] Implement grant/revoke, expiry, step budget, audit hashes, and no-frame retention by default. Exit gate: “what’s on my screen?” returns grounded local observations without @@ -223,16 +223,22 @@ actuation. ## Phase 7 — computer-use portal actuation -- [ ] Implement RemoteDesktop portal consent and restore tokens. -- [ ] Implement libei/EIS pointer, keyboard, scroll, drag, hover, and key input. -- [ ] Implement typed Unicode and submit behavior. -- [ ] Add target preview, agent cursor, and step ticker. -- [ ] Prefer domain tools, app D-Bus, AT-SPI, Shell helper, then vision +- [x] Implement RemoteDesktop portal consent and restore tokens. +- [x] Implement libei/EIS pointer, keyboard, scroll, drag, hover, and key input. +- [x] Implement typed Unicode and submit behavior. +- [x] Add target preview, agent cursor, and step ticker. +- [x] Prefer domain tools, app D-Bus, AT-SPI, Shell helper, then vision coordinates in that order. -- [ ] Refuse password/PAM roles and lock-screen/greeter actions. -- [ ] Require confirmation for destructive or high-impact actions. -- [ ] Keep ydotool and X11 tools disabled unless explicitly enabled. -- [ ] Add state-change self-checks, animation waits, and no-progress aborts. +- [x] Refuse password/PAM roles and lock-screen/greeter actions. +- [x] Require confirmation for destructive or high-impact actions. +- [x] Keep ydotool and X11 tools disabled unless explicitly enabled. +- [x] Add state-change self-checks, animation waits, and no-progress aborts. + +Actuation is implemented in `computer-use/portal-input.js`, `computer-use/actuator.js`, +`computer-use/safety.js`, `computer-use/audit.js`, `skills/computer-act.js`, and +the local portal/AT-SPI helpers. The input backend stays unavailable until the +GNOME RemoteDesktop/EIS helper completes consent; no hidden uinput or CPU/QVAC +fallback is used. Exit gate: Night Light, Text Editor save, Firefox URL entry, hands-off abort, and lock-screen kill pass `docs/cu-acceptance.md`. diff --git a/skills/computer-act.js b/skills/computer-act.js new file mode 100644 index 0000000..5c8c6e8 --- /dev/null +++ b/skills/computer-act.js @@ -0,0 +1,14 @@ +export function createComputerActTools({ actuator } = {}) { + const call = (method) => async (args) => { if (!actuator) throw new Error('computer-use actuator is unavailable'); return actuator[method](args); }; + return [ + ['cu_act', 'Run a named AT-SPI action on a semantic ref.', { ref: { type: 'string' }, action: { type: 'string' } }, 'act'], + ['cu_click', 'Click a semantic ref or coordinate.', { ref: { type: 'string' }, x: { type: 'number' }, y: { type: 'number' }, button: { type: 'string' } }, 'click'], + ['cu_double_click', 'Double-click a coordinate.', { x: { type: 'number' }, y: { type: 'number' } }, 'doubleClick'], + ['cu_right_click', 'Right-click a coordinate.', { x: { type: 'number' }, y: { type: 'number' } }, 'rightClick'], + ['cu_hover', 'Move the visible agent cursor.', { x: { type: 'number' }, y: { type: 'number' } }, 'hover'], + ['cu_scroll', 'Scroll at a target.', { ref: { type: 'string' }, x: { type: 'number' }, y: { type: 'number' }, dy: { type: 'number' }, dx: { type: 'number' } }, 'scroll'], + ['cu_drag', 'Drag between semantic refs or coordinates.', { from: {}, to: {} }, 'drag'], + ['cu_type', 'Type Unicode through the granted input backend.', { text: { type: 'string' }, ref: { type: 'string' }, submit: { type: 'boolean' } }, 'type'], + ['cu_key', 'Send a keyboard combination through the granted input backend.', { combo: { type: 'string' } }, 'key'], + ].map(([name, description, properties, method]) => ({ name, permission: 'computer-use', description, parameters: { type: 'object', properties }, execute: call(method) })); +} diff --git a/test/computer-use.test.js b/test/computer-use.test.js index 0ba745d..1e5ca3c 100644 --- a/test/computer-use.test.js +++ b/test/computer-use.test.js @@ -1,6 +1,11 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { ComputerUseSession } from '../computer-use/session.js'; +import { ComputerActuator } from '../computer-use/actuator.js'; +import { ComputerAudit } from '../computer-use/audit.js'; +import { mkdtemp, readFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; test('computer use requires an explicit grant and enforces its step budget', () => { let now = 1000; @@ -24,3 +29,18 @@ test('computer use expires its wall clock grant', () => { assert.throws(() => session.beginStep(), /expired/); assert.equal(session.status().backend, 'none'); }); + +test('computer use semantic actuation previews, budgets, and audits actions', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'jarvis-cu-audit-')); + const audit = new ComputerAudit({ dir }); const session = new ComputerUseSession({ stepsMax: 1, audit }); const sent = []; + session.grant(); + const actuator = new ComputerActuator({ session, input: { send: (event) => sent.push(event) }, find: async ({ ref }) => [{ ref, name: 'Save', role: 'push button', rect: [1, 2, 3, 4] }], atspiAction: async () => ({ semantic: true }), audit, sleep: async () => {} }); + const result = await actuator.click({ ref: 'r1' }); assert.equal(result.ok, true); assert.equal(session.status().steps_used, 1); + assert.deepEqual(sent, []); const files = await (await import('node:fs/promises')).readdir(dir); assert.equal(files.length, 1); assert.match(await readFile(path.join(dir, files[0]), 'utf8'), /target_hash/); +}); + +test('computer use refuses password targets and unconfirmed dangerous keys', async () => { + const session = new ComputerUseSession(); session.grant(); const actuator = new ComputerActuator({ session, input: { send() {} }, find: async () => [{ name: 'Password', role: 'password text' }], sleep: async () => {} }); + await assert.rejects(() => actuator.type({ ref: 'password', text: 'secret' }), /password/); + await assert.rejects(() => actuator.key({ combo: 'alt+f4' }), /confirmation/); +});