gnome-jarvis/test/computer-use.test.js

import test from 'node:test';
import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import { ComputerUseSession } from '../computer-use/session.js';
import { ComputerActuator } from '../computer-use/actuator.js';
import { ComputerAudit } from '../computer-use/audit.js';
import { mkdtemp, readFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

test('computer use requires an explicit grant and enforces its step budget', () => {
  let now = 1000;
  const session = new ComputerUseSession({ stepsMax: 2, clock: () => now });
  assert.throws(() => session.beginStep(), /inactive/);
  assert.deepEqual(session.grant({ persist: true }).restore_token_present, true);
  session.beginStep();
  session.beginStep();
  assert.throws(() => session.beginStep(), /budget/);
  assert.equal(session.status().steps_used, 2);
  session.revoke();
  assert.equal(session.status().active, false);
  now += 1;
});

test('computer use expires its wall clock grant', () => {
  let now = 0;
  const session = new ComputerUseSession({ clock: () => now });
  session.grant();
  now = 180001;
  assert.throws(() => session.beginStep(), /expired/);
  assert.equal(session.status().backend, 'none');
});

test('computer use records portal-notify as a live backend', () => {
  const session = new ComputerUseSession();
  session.grant();
  assert.equal(session.setBackend('portal-notify'), 'portal-notify');
  assert.equal(session.status().backend, 'portal-notify');
});

test('computer use semantic actuation previews, budgets, and audits actions', async () => {
  const dir = await mkdtemp(path.join(tmpdir(), 'jarvis-cu-audit-'));
  const audit = new ComputerAudit({ dir }); const session = new ComputerUseSession({ stepsMax: 1, audit }); const sent = [];
  session.grant();
  const actuator = new ComputerActuator({ session, input: { send: (event) => sent.push(event) }, find: async ({ ref }) => [{ ref, name: 'Save', role: 'push button', rect: [1, 2, 3, 4] }], atspiAction: async () => ({ semantic: true }), audit, sleep: async () => {} });
  const result = await actuator.click({ ref: 'r1' }); assert.equal(result.ok, true); assert.equal(result.name, 'Save'); assert.equal(session.status().steps_used, 1);
  assert.equal(result.target, undefined);
  assert.deepEqual(sent, []); const files = await (await import('node:fs/promises')).readdir(dir); assert.equal(files.length, 1); assert.match(await readFile(path.join(dir, files[0]), 'utf8'), /target_hash/);
});

test('semantic click falls back to the control center when AT-SPI has no click action', async () => {
  const session = new ComputerUseSession(); session.grant(); const sent = [];
  const actuator = new ComputerActuator({
    session,
    input: { send: (event) => sent.push(event) },
    find: async ({ ref }) => [{ ref, name: 'Discord', role: 'frame', rect: [10, 20, 100, 40] }],
    atspiAction: async () => { throw new Error('AT-SPI action unavailable: click'); },
    sleep: async () => {},
  });
  const result = await actuator.click({ ref: 'r1' });
  assert.equal(result.ok, true);
  assert.equal(sent[0].type, 'pointer');
  assert.equal(sent[0].x, 60);
  assert.equal(sent[0].y, 40);
});

test('typing focuses the target then injects keys and does not stringify missing text', async () => {
  const session = new ComputerUseSession(); session.grant(); const sent = [];
  const actuator = new ComputerActuator({
    session,
    input: { send: (event) => sent.push(event) },
    find: async () => [{ ref: 'r2', name: 'Message', role: 'entry', rect: [0, 0, 20, 10] }],
    sleep: async () => {},
  });
  const result = await actuator.type({ ref: 'r2', text: 'hello' });
  assert.equal(result.ok, true);
  assert.equal(result.result.typed, 5);
  assert.equal(sent[0].action, 'click');
  assert.equal(sent[1].action, 'type');
  assert.equal(sent[1].text, 'hello');
  const empty = await actuator.type({ ref: 'r2' });
  assert.equal(empty.result.typed, 0);
  assert.equal(sent[3].text, '');
});

test('computer use refuses password targets and unconfirmed dangerous keys', async () => {
  const session = new ComputerUseSession(); session.grant(); const actuator = new ComputerActuator({ session, input: { send() {} }, find: async () => [{ name: 'Password', role: 'password text' }], sleep: async () => {} });
  await assert.rejects(() => actuator.type({ ref: 'password', text: 'secret' }), /password/);
  await assert.rejects(() => actuator.key({ combo: 'alt+f4' }), /confirmation/);
});

test('Send buttons do not require extra confirmation after a desktop grant', async () => {
  const session = new ComputerUseSession(); session.grant(); const sent = [];
  const actuator = new ComputerActuator({
    session,
    input: { send: (event) => sent.push(event) },
    find: async () => [{ ref: 'r3', name: 'Send', role: 'push button', rect: [0, 0, 40, 20] }],
    atspiAction: async () => { throw new Error('AT-SPI action unavailable: click'); },
    sleep: async () => {},
  });
  const result = await actuator.click({ ref: 'r3' });
  assert.equal(result.ok, true);
  assert.equal(sent[0].action, 'click');
  assert.equal(sent[0].x, 20);
});

test('libei sender binds bitmask capabilities from libei.h', () => {
  const pyDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../computer-use/py');
  const compiled = spawnSync('python3', ['-m', 'py_compile', 'libei_sender.py', 'portal_remote_desktop.py', 'portal_screenshot.py', 'pw_framebuffer.py'], { cwd: pyDir, encoding: 'utf8' });
  assert.equal(compiled.status, 0, compiled.stderr);
  const caps = spawnSync('python3', ['-c', 'from libei_sender import CAP_POINTER, CAP_KEYBOARD, CAP_BUTTON, F_KEYS; assert CAP_POINTER == 1 and CAP_KEYBOARD == 4 and CAP_BUTTON == 32 and F_KEYS["f11"] == 87'], { cwd: pyDir, encoding: 'utf8' });
  assert.equal(caps.status, 0, caps.stderr);
});