gnome-jarvis/computer-use/actuator.js

import { setTimeout as delay } from 'node:timers/promises';
import { assertSafeTarget, requiresConfirmation } from './safety.js';

export function pointFor(target, args = {}) {
  if (args.x != null && args.y != null) { const x = Number(args.x), y = Number(args.y); if (!Number.isFinite(x) || !Number.isFinite(y)) throw new Error('coordinates must be finite'); return { x, y }; }
  const rect = target?.rect;
  if (Array.isArray(rect) && rect.length >= 4) {
    const [x, y, w, h] = rect.map(Number);
    if ([x, y, w, h].every(Number.isFinite) && w > 0 && h > 0 && x > -100000 && y > -100000) return { x: x + w / 2, y: y + h / 2 };
  }
  return null;
}

export function compactTarget(target) {
  if (!target || typeof target !== 'object') return null;
  const out = {};
  if (target.ref) out.ref = target.ref;
  if (target.role) out.role = target.role;
  if (target.name) out.name = String(target.name).slice(0, 80);
  return Object.keys(out).length ? out : null;
}

function asPoint(value) {
  if (!value) return null;
  if (Array.isArray(value) && value.length >= 2) return { x: Number(value[0]), y: Number(value[1]) };
  if (typeof value === 'object') return pointFor(value, value);
  return null;
}

export class ComputerActuator {
  constructor({ session, input, atspiAction, find, highlight, audit, confirm = async () => false, sleep = delay, verify = async () => true } = {}) {
    this.session = session;
    this.input = input;
    this.atspiAction = atspiAction;
    this.find = find;
    this.highlight = highlight;
    this.audit = audit;
    this.confirm = confirm;
    this.sleep = sleep;
    this.verify = verify;
  }

  async target(args = {}) {
    const target = args.ref && this.find ? (await this.find({ ref: args.ref }))[0] : args;
    if (args.ref && !target) throw new Error('unknown or stale computer-use ref');
    if (target) assertSafeTarget(target);
    return target;
  }

  async readyInput() {
    if (typeof this.input?.ready === 'function') await this.input.ready();
  }

  async send(action) {
    await this.readyInput();
    if (!this.input?.send) throw new Error('portal EIS input backend is unavailable');
    this.session.assertActive();
    return await this.input.send(action);
  }

  async semantic(target, action) {
    if (!this.atspiAction || !target) return null;
    try {
      const result = await this.atspiAction(target, action);
      if (!result || result.ok === false) return null;
      return result;
    } catch (error) {
      if (/target not found|stale/i.test(error.message)) throw error;
      return null;
    }
  }

  async run(action, args, fn) {
    this.session.assertActive();
    const target = await this.target(args);
    if (requiresConfirmation(action, target) && !(await this.confirm(action, target))) throw new Error('explicit confirmation required');
    await this.readyInput();
    this.session.beginStep();
    await this.highlight?.(target, action);
    this.session.assertActive();
    try {
      const result = await fn(target);
      await this.sleep(150);
      if (!(await this.verify(target, action, result))) throw new Error('computer-use state did not change after action');
      await this.audit?.record(action, target, { ok: true });
      return { ok: true, action, ...compactTarget(target), result: result ?? null };
    } catch (error) {
      await this.audit?.record(action, target, { ok: false, reason: error.message });
      throw error;
    }
  }

  async act({ ref, action }) {
    return this.run(`act:${action}`, { ref }, async (target) => {
      const semantic = await this.semantic(target, action);
      if (semantic) return semantic;
      throw new Error('AT-SPI action backend is unavailable');
    });
  }

  async click(args = {}) {
    return this.run('click', args, async (target) => {
      const semantic = args.ref && (!args.button || args.button === 'left') ? await this.semantic(target, args.action || 'click') : null;
      if (semantic) return semantic;
      const point = pointFor(target, args);
      if (!point) throw new Error('click requires a semantic ref or coordinates');
      await this.send({ type: 'pointer', action: 'click', x: point.x, y: point.y, button: args.button || 'left' });
      return { via: 'pointer', ...point };
    });
  }

  async doubleClick(args = {}) {
    return this.run('double_click', args, async (target) => {
      const point = pointFor(target, args);
      if (!point) throw new Error('double-click requires a semantic ref or coordinates');
      await this.send({ type: 'pointer', action: 'double_click', x: point.x, y: point.y });
      return point;
    });
  }

  async rightClick(args = {}) {
    return this.run('right_click', args, async (target) => {
      const point = pointFor(target, args);
      if (!point) throw new Error('right-click requires a semantic ref or coordinates');
      await this.send({ type: 'pointer', action: 'click', x: point.x, y: point.y, button: 'right' });
      return point;
    });
  }

  async hover(args = {}) {
    return this.run('hover', args, async (target) => {
      const point = pointFor(target, args);
      if (!point) throw new Error('hover requires a semantic ref or coordinates');
      await this.send({ type: 'pointer', action: 'move', x: point.x, y: point.y });
      return point;
    });
  }

  async scroll(args = {}) {
    return this.run('scroll', args, async (target) => {
      const point = pointFor(target, args);
      if (!point) throw new Error('scroll requires a ref or coordinates');
      await this.send({ type: 'pointer', action: 'scroll', x: point.x, y: point.y, dx: args.dx || 0, dy: args.dy || 0 });
      return point;
    });
  }

  async drag({ from, to } = {}) {
    return this.run('drag', { from, to }, async () => {
      const start = asPoint(from) || (from?.ref ? pointFor(await this.target(from), from) : null);
      const end = asPoint(to) || (to?.ref ? pointFor(await this.target(to), to) : null);
      if (!start || !end) throw new Error('drag requires from and to refs or coordinates');
      await this.send({ type: 'pointer', action: 'drag', from: [start.x, start.y], to: [end.x, end.y] });
      return { from: start, to: end };
    });
  }

  async type({ text, ref, submit = false } = {}) {
    return this.run('type', { ref }, async (target) => {
      assertSafeTarget(target);
      const body = text == null ? '' : String(text);
      const point = pointFor(target, {});
      if (point) {
        await this.send({ type: 'pointer', action: 'click', x: point.x, y: point.y, button: 'left' });
        await this.sleep(80);
      }
      await this.send({ type: 'keyboard', action: 'type', text: body, submit: Boolean(submit) });
      return { typed: body.length, submit: Boolean(submit) };
    });
  }

  async key({ combo } = {}) {
    return this.run(`key:${combo}`, {}, async () => {
      await this.send({ type: 'keyboard', action: 'key', combo: String(combo || '').toLowerCase() });
      return { combo: String(combo || '').toLowerCase() };
    });
  }
}