import { setTimeout as delay } from 'node:timers/promises'; import { assertSafeTarget, requiresConfirmation } from './safety.js'; export function pointFor(target, args = {}) { if (args.x != null && args.y != null) { const x = Number(args.x), y = Number(args.y); if (!Number.isFinite(x) || !Number.isFinite(y)) throw new Error('coordinates must be finite'); return { x, y }; } const rect = target?.rect; if (Array.isArray(rect) && rect.length >= 4) { const [x, y, w, h] = rect.map(Number); if ([x, y, w, h].every(Number.isFinite) && w > 0 && h > 0 && x > -100000 && y > -100000) return { x: x + w / 2, y: y + h / 2 }; } return null; } export function compactTarget(target) { if (!target || typeof target !== 'object') return null; const out = {}; if (target.ref) out.ref = target.ref; if (target.role) out.role = target.role; if (target.name) out.name = String(target.name).slice(0, 80); return Object.keys(out).length ? out : null; } function asPoint(value) { if (!value) return null; if (Array.isArray(value) && value.length >= 2) return { x: Number(value[0]), y: Number(value[1]) }; if (typeof value === 'object') return pointFor(value, value); return null; } export class ComputerActuator { constructor({ session, input, atspiAction, find, highlight, audit, confirm = async () => false, sleep = delay, verify = async () => true } = {}) { this.session = session; this.input = input; this.atspiAction = atspiAction; this.find = find; this.highlight = highlight; this.audit = audit; this.confirm = confirm; this.sleep = sleep; this.verify = verify; } async target(args = {}) { const target = args.ref && this.find ? (await this.find({ ref: args.ref }))[0] : args; if (args.ref && !target) throw new Error('unknown or stale computer-use ref'); if (target) assertSafeTarget(target); return target; } async readyInput() { if (typeof this.input?.ready === 'function') await this.input.ready(); } async send(action) { await this.readyInput(); if (!this.input?.send) throw new Error('portal EIS input backend is unavailable'); this.session.assertActive(); return await this.input.send(action); } async semantic(target, action) { if (!this.atspiAction || !target) return null; try { const result = await this.atspiAction(target, action); if (!result || result.ok === false) return null; return result; } catch (error) { if (/target not found|stale/i.test(error.message)) throw error; return null; } } async run(action, args, fn) { this.session.assertActive(); const target = await this.target(args); if (requiresConfirmation(action, target) && !(await this.confirm(action, target))) throw new Error('explicit confirmation required'); await this.readyInput(); this.session.beginStep(); await this.highlight?.(target, action); this.session.assertActive(); try { const result = await fn(target); await this.sleep(150); if (!(await this.verify(target, action, result))) throw new Error('computer-use state did not change after action'); await this.audit?.record(action, target, { ok: true }); return { ok: true, action, ...compactTarget(target), result: result ?? null }; } catch (error) { await this.audit?.record(action, target, { ok: false, reason: error.message }); throw error; } } async act({ ref, action }) { return this.run(`act:${action}`, { ref }, async (target) => { const semantic = await this.semantic(target, action); if (semantic) return semantic; throw new Error('AT-SPI action backend is unavailable'); }); } async click(args = {}) { return this.run('click', args, async (target) => { const semantic = args.ref && (!args.button || args.button === 'left') ? await this.semantic(target, args.action || 'click') : null; if (semantic) return semantic; const point = pointFor(target, args); if (!point) throw new Error('click requires a semantic ref or coordinates'); await this.send({ type: 'pointer', action: 'click', x: point.x, y: point.y, button: args.button || 'left' }); return { via: 'pointer', ...point }; }); } async doubleClick(args = {}) { return this.run('double_click', args, async (target) => { const point = pointFor(target, args); if (!point) throw new Error('double-click requires a semantic ref or coordinates'); await this.send({ type: 'pointer', action: 'double_click', x: point.x, y: point.y }); return point; }); } async rightClick(args = {}) { return this.run('right_click', args, async (target) => { const point = pointFor(target, args); if (!point) throw new Error('right-click requires a semantic ref or coordinates'); await this.send({ type: 'pointer', action: 'click', x: point.x, y: point.y, button: 'right' }); return point; }); } async hover(args = {}) { return this.run('hover', args, async (target) => { const point = pointFor(target, args); if (!point) throw new Error('hover requires a semantic ref or coordinates'); await this.send({ type: 'pointer', action: 'move', x: point.x, y: point.y }); return point; }); } async scroll(args = {}) { return this.run('scroll', args, async (target) => { const point = pointFor(target, args); if (!point) throw new Error('scroll requires a ref or coordinates'); await this.send({ type: 'pointer', action: 'scroll', x: point.x, y: point.y, dx: args.dx || 0, dy: args.dy || 0 }); return point; }); } async drag({ from, to } = {}) { return this.run('drag', { from, to }, async () => { const start = asPoint(from) || (from?.ref ? pointFor(await this.target(from), from) : null); const end = asPoint(to) || (to?.ref ? pointFor(await this.target(to), to) : null); if (!start || !end) throw new Error('drag requires from and to refs or coordinates'); await this.send({ type: 'pointer', action: 'drag', from: [start.x, start.y], to: [end.x, end.y] }); return { from: start, to: end }; }); } async type({ text, ref, submit = false } = {}) { return this.run('type', { ref }, async (target) => { assertSafeTarget(target); const body = text == null ? '' : String(text); const point = pointFor(target, {}); if (point) { await this.send({ type: 'pointer', action: 'click', x: point.x, y: point.y, button: 'left' }); await this.sleep(80); } await this.send({ type: 'keyboard', action: 'type', text: body, submit: Boolean(submit) }); return { typed: body.length, submit: Boolean(submit) }; }); } async key({ combo } = {}) { return this.run(`key:${combo}`, {}, async () => { await this.send({ type: 'keyboard', action: 'key', combo: String(combo || '').toLowerCase() }); return { combo: String(combo || '').toLowerCase() }; }); } }