import test from 'node:test'; import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; import { EventEmitter } from 'node:events'; import { ComputerUseSession } from '../computer-use/session.js'; import { CameraSession } from '../computer-use/camera-session.js'; import { PortalCamera } from '../computer-use/portal-camera.js'; import { ComputerActuator } from '../computer-use/actuator.js'; import { ComputerAudit } from '../computer-use/audit.js'; import { mkdtemp, readFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; test('computer use requires an explicit grant and enforces its step budget', () => { let now = 1000; const session = new ComputerUseSession({ stepsMax: 2, clock: () => now }); assert.throws(() => session.beginStep(), /inactive/); assert.deepEqual(session.grant({ persist: true }).restore_token_present, true); session.beginStep(); session.beginStep(); assert.throws(() => session.beginStep(), /budget/); assert.equal(session.status().steps_used, 2); session.revoke(); assert.equal(session.status().active, false); now += 1; }); test('computer use expires its wall clock grant', () => { let now = 0; const session = new ComputerUseSession({ clock: () => now }); session.grant(); now = 180001; assert.throws(() => session.beginStep(), /expired/); assert.equal(session.status().backend, 'none'); }); test('computer use records portal-notify as a live backend', () => { const session = new ComputerUseSession(); session.grant(); assert.equal(session.setBackend('portal-notify'), 'portal-notify'); assert.equal(session.status().backend, 'portal-notify'); }); test('computer use semantic actuation previews, budgets, and audits actions', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'jarvis-cu-audit-')); const audit = new ComputerAudit({ dir }); const session = new ComputerUseSession({ stepsMax: 1, audit }); const sent = []; session.grant(); const actuator = new ComputerActuator({ session, input: { send: (event) => sent.push(event) }, find: async ({ ref }) => [{ ref, name: 'Save', role: 'push button', rect: [1, 2, 3, 4] }], atspiAction: async () => ({ semantic: true }), audit, sleep: async () => {} }); const result = await actuator.click({ ref: 'r1' }); assert.equal(result.ok, true); assert.equal(result.name, 'Save'); assert.equal(session.status().steps_used, 1); assert.equal(result.target, undefined); assert.deepEqual(sent, []); const files = await (await import('node:fs/promises')).readdir(dir); assert.equal(files.length, 1); assert.match(await readFile(path.join(dir, files[0]), 'utf8'), /target_hash/); }); test('semantic click falls back to the control center when AT-SPI has no click action', async () => { const session = new ComputerUseSession(); session.grant(); const sent = []; const actuator = new ComputerActuator({ session, input: { send: (event) => sent.push(event) }, find: async ({ ref }) => [{ ref, name: 'Discord', role: 'frame', rect: [10, 20, 100, 40] }], atspiAction: async () => { throw new Error('AT-SPI action unavailable: click'); }, sleep: async () => {}, }); const result = await actuator.click({ ref: 'r1' }); assert.equal(result.ok, true); assert.equal(sent[0].type, 'pointer'); assert.equal(sent[0].x, 60); assert.equal(sent[0].y, 40); }); test('typing focuses the target then injects keys and does not stringify missing text', async () => { const session = new ComputerUseSession(); session.grant(); const sent = []; const actuator = new ComputerActuator({ session, input: { send: (event) => sent.push(event) }, find: async () => [{ ref: 'r2', name: 'Message', role: 'entry', rect: [0, 0, 20, 10] }], sleep: async () => {}, }); const result = await actuator.type({ ref: 'r2', text: 'hello' }); assert.equal(result.ok, true); assert.equal(result.result.typed, 5); assert.equal(sent[0].action, 'click'); assert.equal(sent[1].action, 'type'); assert.equal(sent[1].text, 'hello'); const empty = await actuator.type({ ref: 'r2' }); assert.equal(empty.result.typed, 0); assert.equal(sent[3].text, ''); }); test('computer use refuses password targets and unconfirmed dangerous keys', async () => { const session = new ComputerUseSession(); session.grant(); const actuator = new ComputerActuator({ session, input: { send() {} }, find: async () => [{ name: 'Password', role: 'password text' }], sleep: async () => {} }); await assert.rejects(() => actuator.type({ ref: 'password', text: 'secret' }), /password/); await assert.rejects(() => actuator.key({ combo: 'alt+f4' }), /confirmation/); }); test('Send buttons do not require extra confirmation after a desktop grant', async () => { const session = new ComputerUseSession(); session.grant(); const sent = []; const actuator = new ComputerActuator({ session, input: { send: (event) => sent.push(event) }, find: async () => [{ ref: 'r3', name: 'Send', role: 'push button', rect: [0, 0, 40, 20] }], atspiAction: async () => { throw new Error('AT-SPI action unavailable: click'); }, sleep: async () => {}, }); const result = await actuator.click({ ref: 'r3' }); assert.equal(result.ok, true); assert.equal(sent[0].action, 'click'); assert.equal(sent[0].x, 20); }); test('libei sender binds bitmask capabilities from libei.h', () => { const pyDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../computer-use/py'); const compiled = spawnSync('python3', ['-m', 'py_compile', 'libei_sender.py', 'portal_remote_desktop.py', 'portal_screenshot.py', 'portal_camera.py', 'pw_framebuffer.py'], { cwd: pyDir, encoding: 'utf8' }); assert.equal(compiled.status, 0, compiled.stderr); const caps = spawnSync('python3', ['-c', 'from libei_sender import CAP_POINTER, CAP_KEYBOARD, CAP_BUTTON, F_KEYS; assert CAP_POINTER == 1 and CAP_KEYBOARD == 4 and CAP_BUTTON == 32 and F_KEYS["f11"] == 87'], { cwd: pyDir, encoding: 'utf8' }); assert.equal(caps.status, 0, caps.stderr); }); test('camera session expires its wall clock grant', () => { let now = 0; const session = new CameraSession({ enabled: false, clock: () => now }); session.grant(); assert.equal(session.status().active, true); now = 180001; assert.throws(() => session.assertActive(), /expired/); assert.equal(session.status().active, false); assert.equal(session.status().backend, 'none'); }); test('portal camera parses helper JSON and rejects failures', async () => { const spawnImpl = (_cmd, args) => { const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); child.kill = () => {}; setImmediate(() => { if (args.includes('fail')) { child.stdout.emit('data', '{"ok":false,"error":"denied"}\n'); child.emit('close', 1); return; } if (args.includes('access')) { child.stdout.emit('data', '{"ok":true,"via":"portal"}\n'); } else { child.stdout.emit('data', '{"ok":true,"path":"/tmp/x.png","via":"portal"}\n'); } child.emit('close', 0); }); return child; }; const helper = new PortalCamera({ spawnImpl, timeoutMs: 1000, accessTimeoutMs: 1000, helper: 'portal_camera.py' }); assert.equal((await helper.access()).via, 'portal'); assert.equal((await helper.capture('/tmp/x.png')).path, '/tmp/x.png'); const failing = new PortalCamera({ spawnImpl: () => { const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); child.kill = () => {}; setImmediate(() => { child.stdout.emit('data', '{"ok":false,"error":"denied"}\n'); child.emit('close', 1); }); return child; }, timeoutMs: 1000, accessTimeoutMs: 1000, }); await assert.rejects(() => failing.access(), /denied/); });