+169
-13
@@ -1,17 +1,173 @@
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { assertSafeTarget, requiresConfirmation } from './safety.js';
|
||||
|
||||
export class ComputerActuator {
|
||||
constructor({ session, input, atspiAction, find, highlight, audit, confirm = async () => false, sleep = delay, verify = async () => true } = {}) { this.session = session; this.input = input; this.atspiAction = atspiAction; this.find = find; this.highlight = highlight; this.audit = audit; this.confirm = confirm; this.sleep = sleep; this.verify = verify; }
|
||||
async target(args = {}) { const target = args.ref && this.find ? (await this.find({ ref: args.ref }))[0] : args; if (args.ref && !target) throw new Error('unknown or stale computer-use ref'); if (target) assertSafeTarget(target); return target; }
|
||||
async run(action, args, fn) { const target = await this.target(args); if (requiresConfirmation(action, target) && !(await this.confirm(action, target))) throw new Error('explicit confirmation required'); this.session.beginStep(); await this.highlight?.(target, action); this.session.assertActive(); try { const result = await fn(target); await this.sleep(150); if (!(await this.verify(target, action, result))) throw new Error('computer-use state did not change after action'); await this.audit?.record(action, target, { ok: true }); return { ok: true, action, target: target || null, result }; } catch (error) { await this.audit?.record(action, target, { ok: false, reason: error.message }); throw error; } }
|
||||
async act({ ref, action }) { return this.run(`act:${action}`, { ref }, async (target) => { if (!this.atspiAction) throw new Error('AT-SPI action backend is unavailable'); return this.atspiAction(target, action); }); }
|
||||
async click(args = {}) { return this.run('click', args, async (target) => { if (args.ref && target && this.atspiAction) return this.atspiAction(target, 'click'); if (args.x == null || args.y == null) throw new Error('click requires a semantic ref or coordinates'); this.input.send({ type: 'pointer', action: 'click', x: args.x, y: args.y, button: args.button || 'left' }); }); }
|
||||
async doubleClick(args = {}) { return this.run('double_click', args, async (target) => { if (args.x == null || args.y == null) throw new Error('double-click requires coordinates'); this.input.send({ type: 'pointer', action: 'double_click', x: args.x, y: args.y }); return target; }); }
|
||||
async rightClick(args = {}) { return this.run('right_click', args, async () => { this.input.send({ type: 'pointer', action: 'click', x: args.x, y: args.y, button: 'right' }); }); }
|
||||
async hover(args = {}) { return this.run('hover', args, async () => { this.input.send({ type: 'pointer', action: 'move', x: args.x, y: args.y }); }); }
|
||||
async scroll(args = {}) { return this.run('scroll', args, async () => { this.input.send({ type: 'pointer', action: 'scroll', x: args.x, y: args.y, dx: args.dx || 0, dy: args.dy || 0 }); }); }
|
||||
async drag({ from, to }) { return this.run('drag', { from, to }, async () => { this.input.send({ type: 'pointer', action: 'drag', from, to }); }); }
|
||||
async type({ text, ref, submit = false }) { return this.run('type', { ref }, async (target) => { assertSafeTarget(target); this.input.send({ type: 'keyboard', action: 'type', text: String(text), submit: Boolean(submit) }); }); }
|
||||
async key({ combo }) { return this.run(`key:${combo}`, {}, async () => { this.input.send({ type: 'keyboard', action: 'key', combo: String(combo).toLowerCase() }); }); }
|
||||
export function pointFor(target, args = {}) {
|
||||
if (args.x != null && args.y != null) return { x: Number(args.x), y: Number(args.y) };
|
||||
const rect = target?.rect;
|
||||
if (Array.isArray(rect) && rect.length >= 4) {
|
||||
const [x, y, w, h] = rect.map(Number);
|
||||
if ([x, y, w, h].every(Number.isFinite)) return { x: x + w / 2, y: y + h / 2 };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function compactTarget(target) {
|
||||
if (!target || typeof target !== 'object') return null;
|
||||
const out = {};
|
||||
if (target.ref) out.ref = target.ref;
|
||||
if (target.role) out.role = target.role;
|
||||
if (target.name) out.name = String(target.name).slice(0, 80);
|
||||
return Object.keys(out).length ? out : null;
|
||||
}
|
||||
|
||||
function asPoint(value) {
|
||||
if (!value) return null;
|
||||
if (Array.isArray(value) && value.length >= 2) return { x: Number(value[0]), y: Number(value[1]) };
|
||||
if (typeof value === 'object') return pointFor(value, value);
|
||||
return null;
|
||||
}
|
||||
|
||||
export class ComputerActuator {
|
||||
constructor({ session, input, atspiAction, find, highlight, audit, confirm = async () => false, sleep = delay, verify = async () => true } = {}) {
|
||||
this.session = session;
|
||||
this.input = input;
|
||||
this.atspiAction = atspiAction;
|
||||
this.find = find;
|
||||
this.highlight = highlight;
|
||||
this.audit = audit;
|
||||
this.confirm = confirm;
|
||||
this.sleep = sleep;
|
||||
this.verify = verify;
|
||||
}
|
||||
|
||||
async target(args = {}) {
|
||||
const target = args.ref && this.find ? (await this.find({ ref: args.ref }))[0] : args;
|
||||
if (args.ref && !target) throw new Error('unknown or stale computer-use ref');
|
||||
if (target) assertSafeTarget(target);
|
||||
return target;
|
||||
}
|
||||
|
||||
async readyInput() {
|
||||
if (typeof this.input?.ready === 'function') await this.input.ready();
|
||||
}
|
||||
|
||||
async send(action) {
|
||||
await this.readyInput();
|
||||
if (!this.input?.send) throw new Error('portal EIS input backend is unavailable');
|
||||
this.input.send(action);
|
||||
}
|
||||
|
||||
async semantic(target, action) {
|
||||
if (!this.atspiAction || !target) return null;
|
||||
try {
|
||||
const result = await this.atspiAction(target, action);
|
||||
if (!result || result.ok === false) return null;
|
||||
return result;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async run(action, args, fn) {
|
||||
const target = await this.target(args);
|
||||
if (requiresConfirmation(action, target) && !(await this.confirm(action, target))) throw new Error('explicit confirmation required');
|
||||
this.session.beginStep();
|
||||
await this.highlight?.(target, action);
|
||||
this.session.assertActive();
|
||||
try {
|
||||
const result = await fn(target);
|
||||
await this.sleep(150);
|
||||
if (!(await this.verify(target, action, result))) throw new Error('computer-use state did not change after action');
|
||||
await this.audit?.record(action, target, { ok: true });
|
||||
return { ok: true, action, ...compactTarget(target), result: result ?? null };
|
||||
} catch (error) {
|
||||
await this.audit?.record(action, target, { ok: false, reason: error.message });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async act({ ref, action }) {
|
||||
return this.run(`act:${action}`, { ref }, async (target) => {
|
||||
const semantic = await this.semantic(target, action);
|
||||
if (semantic) return semantic;
|
||||
throw new Error('AT-SPI action backend is unavailable');
|
||||
});
|
||||
}
|
||||
|
||||
async click(args = {}) {
|
||||
return this.run('click', args, async (target) => {
|
||||
const semantic = args.ref ? await this.semantic(target, args.action || 'click') : null;
|
||||
if (semantic) return semantic;
|
||||
const point = pointFor(target, args);
|
||||
if (!point) throw new Error('click requires a semantic ref or coordinates');
|
||||
await this.send({ type: 'pointer', action: 'click', x: point.x, y: point.y, button: args.button || 'left' });
|
||||
return { via: 'pointer', ...point };
|
||||
});
|
||||
}
|
||||
|
||||
async doubleClick(args = {}) {
|
||||
return this.run('double_click', args, async (target) => {
|
||||
const point = pointFor(target, args);
|
||||
if (!point) throw new Error('double-click requires a semantic ref or coordinates');
|
||||
await this.send({ type: 'pointer', action: 'double_click', x: point.x, y: point.y });
|
||||
return point;
|
||||
});
|
||||
}
|
||||
|
||||
async rightClick(args = {}) {
|
||||
return this.run('right_click', args, async (target) => {
|
||||
const point = pointFor(target, args);
|
||||
if (!point) throw new Error('right-click requires a semantic ref or coordinates');
|
||||
await this.send({ type: 'pointer', action: 'click', x: point.x, y: point.y, button: 'right' });
|
||||
return point;
|
||||
});
|
||||
}
|
||||
|
||||
async hover(args = {}) {
|
||||
return this.run('hover', args, async (target) => {
|
||||
const point = pointFor(target, args);
|
||||
if (!point) throw new Error('hover requires a semantic ref or coordinates');
|
||||
await this.send({ type: 'pointer', action: 'move', x: point.x, y: point.y });
|
||||
return point;
|
||||
});
|
||||
}
|
||||
|
||||
async scroll(args = {}) {
|
||||
return this.run('scroll', args, async (target) => {
|
||||
const point = pointFor(target, args) || { x: 0, y: 0 };
|
||||
await this.send({ type: 'pointer', action: 'scroll', x: point.x, y: point.y, dx: args.dx || 0, dy: args.dy || 0 });
|
||||
return point;
|
||||
});
|
||||
}
|
||||
|
||||
async drag({ from, to } = {}) {
|
||||
return this.run('drag', { from, to }, async () => {
|
||||
const start = asPoint(from) || (from?.ref ? pointFor(await this.target(from), from) : null);
|
||||
const end = asPoint(to) || (to?.ref ? pointFor(await this.target(to), to) : null);
|
||||
if (!start || !end) throw new Error('drag requires from and to refs or coordinates');
|
||||
await this.send({ type: 'pointer', action: 'drag', from: [start.x, start.y], to: [end.x, end.y] });
|
||||
return { from: start, to: end };
|
||||
});
|
||||
}
|
||||
|
||||
async type({ text, ref, submit = false } = {}) {
|
||||
return this.run('type', { ref }, async (target) => {
|
||||
assertSafeTarget(target);
|
||||
const body = text == null ? '' : String(text);
|
||||
const point = pointFor(target, {});
|
||||
if (point) {
|
||||
await this.send({ type: 'pointer', action: 'click', x: point.x, y: point.y, button: 'left' });
|
||||
await this.sleep(80);
|
||||
}
|
||||
await this.send({ type: 'keyboard', action: 'type', text: body, submit: Boolean(submit) });
|
||||
return { typed: body.length, submit: Boolean(submit) };
|
||||
});
|
||||
}
|
||||
|
||||
async key({ combo } = {}) {
|
||||
return this.run(`key:${combo}`, {}, async () => {
|
||||
await this.send({ type: 'keyboard', action: 'key', combo: String(combo || '').toLowerCase() });
|
||||
return { combo: String(combo || '').toLowerCase() };
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user