@@ -3,10 +3,10 @@ import { assertSafeTarget, requiresConfirmation } from './safety.js';
|
||||
|
||||
export class ComputerActuator {
|
||||
constructor({ session, input, atspiAction, find, highlight, audit, confirm = async () => false, sleep = delay, verify = async () => true } = {}) { this.session = session; this.input = input; this.atspiAction = atspiAction; this.find = find; this.highlight = highlight; this.audit = audit; this.confirm = confirm; this.sleep = sleep; this.verify = verify; }
|
||||
async target(args = {}) { const target = args.ref && this.find ? (await this.find({ ref: args.ref }))[0] : args; if (target) assertSafeTarget(target); return target; }
|
||||
async run(action, args, fn) { const target = await this.target(args); if (requiresConfirmation(action, target) && !(await this.confirm(action, target))) throw new Error('explicit confirmation required'); this.session.beginStep(); await this.highlight?.(target, action); try { const result = await fn(target); await this.sleep(150); if (!(await this.verify(target, action, result))) throw new Error('computer-use state did not change after action'); await this.audit?.record(action, target, { ok: true }); return { ok: true, action, target: target || null, result }; } catch (error) { await this.audit?.record(action, target, { ok: false, reason: error.message }); throw error; } }
|
||||
async target(args = {}) { const target = args.ref && this.find ? (await this.find({ ref: args.ref }))[0] : args; if (args.ref && !target) throw new Error('unknown or stale computer-use ref'); if (target) assertSafeTarget(target); return target; }
|
||||
async run(action, args, fn) { const target = await this.target(args); if (requiresConfirmation(action, target) && !(await this.confirm(action, target))) throw new Error('explicit confirmation required'); this.session.beginStep(); await this.highlight?.(target, action); this.session.assertActive(); try { const result = await fn(target); await this.sleep(150); if (!(await this.verify(target, action, result))) throw new Error('computer-use state did not change after action'); await this.audit?.record(action, target, { ok: true }); return { ok: true, action, target: target || null, result }; } catch (error) { await this.audit?.record(action, target, { ok: false, reason: error.message }); throw error; } }
|
||||
async act({ ref, action }) { return this.run(`act:${action}`, { ref }, async (target) => { if (!this.atspiAction) throw new Error('AT-SPI action backend is unavailable'); return this.atspiAction(target, action); }); }
|
||||
async click(args = {}) { return this.run('click', args, async (target) => { if (target && this.atspiAction) return this.atspiAction(target, 'click'); if (args.x == null || args.y == null) throw new Error('click requires a semantic ref or coordinates'); this.input.send({ type: 'pointer', action: 'click', x: args.x, y: args.y, button: args.button || 'left' }); }); }
|
||||
async click(args = {}) { return this.run('click', args, async (target) => { if (args.ref && target && this.atspiAction) return this.atspiAction(target, 'click'); if (args.x == null || args.y == null) throw new Error('click requires a semantic ref or coordinates'); this.input.send({ type: 'pointer', action: 'click', x: args.x, y: args.y, button: args.button || 'left' }); }); }
|
||||
async doubleClick(args = {}) { return this.run('double_click', args, async (target) => { if (args.x == null || args.y == null) throw new Error('double-click requires coordinates'); this.input.send({ type: 'pointer', action: 'double_click', x: args.x, y: args.y }); return target; }); }
|
||||
async rightClick(args = {}) { return this.run('right_click', args, async () => { this.input.send({ type: 'pointer', action: 'click', x: args.x, y: args.y, button: 'right' }); }); }
|
||||
async hover(args = {}) { return this.run('hover', args, async () => { this.input.send({ type: 'pointer', action: 'move', x: args.x, y: args.y }); }); }
|
||||
|
||||
@@ -5,11 +5,11 @@ import { spawn } from 'node:child_process';
|
||||
export const MAX_LONG_EDGE = 1280;
|
||||
|
||||
export class FrameNormalizer {
|
||||
constructor({ helper = path.resolve(new URL('./py/normalize_frame.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn, tmpDir = '/tmp/jarvis-cu' } = {}) { this.helper = helper; this.python = python; this.spawnImpl = spawnImpl; this.tmpDir = tmpDir; }
|
||||
constructor({ helper = path.resolve(new URL('./py/normalize_frame.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn, tmpDir = '/tmp/jarvis-cu', maxLongEdge = MAX_LONG_EDGE, quality = 70 } = {}) { this.helper = helper; this.python = python; this.spawnImpl = spawnImpl; this.tmpDir = tmpDir; this.maxLongEdge = maxLongEdge; this.quality = quality; }
|
||||
async normalize(input, output = path.join(this.tmpDir, `frame-${Date.now()}.webp`), rect) {
|
||||
await mkdir(path.dirname(output), { recursive: true });
|
||||
const crop = rect ? rect.map(Number).map((value) => String(Math.round(value))) : [];
|
||||
await new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper, input, output, String(MAX_LONG_EDGE), '70', ...crop], { stdio: ['ignore', 'pipe', 'pipe'] }); let error = ''; child.stderr?.on('data', (d) => { error += d; }); child.on('error', reject); child.on('close', (code) => code === 0 ? resolve() : reject(new Error(error || `frame normalization exited ${code}`))); });
|
||||
const info = await stat(output); return { path: output, bytes: info.size, maxLongEdge: MAX_LONG_EDGE, mime: 'image/webp' };
|
||||
await new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper, input, output, String(this.maxLongEdge), String(this.quality), ...crop], { stdio: ['ignore', 'pipe', 'pipe'] }); let error = ''; child.stderr?.on('data', (d) => { error += d; }); child.on('error', reject); child.on('close', (code) => code === 0 ? resolve() : reject(new Error(error || `frame normalization exited ${code}`))); });
|
||||
const info = await stat(output); return { path: output, bytes: info.size, maxLongEdge: this.maxLongEdge, mime: 'image/webp' };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,10 +20,10 @@ export class DesktopObserver {
|
||||
const [windows, focused, tree] = await Promise.all([this.shell.windows(), this.shell.focused(), includeTree ? this.tree().catch((error) => { unavailable.push(`AT-SPI: ${error.message}`); return []; }) : Promise.resolve([])]);
|
||||
const result = { monitor: null, focused, windows: windows.windows || [], tree, screenshot_path: frame.path, ocr_blocks: [], vision_hint: null, unavailable };
|
||||
if (!windows.available) result.unavailable.push(windows.reason);
|
||||
if (includeOcr && this.ocr) result.ocr_blocks = await this.ocr(frame.path).catch((error) => { result.unavailable.push(`OCR: ${error.message}`); return []; });
|
||||
if (includeVision && this.vision) result.vision_hint = await this.vision(frame.path).catch((error) => { result.unavailable.push(`vision: ${error.message}`); return null; });
|
||||
if (frame.path && includeOcr && this.ocr) result.ocr_blocks = await this.ocr(frame.path).catch((error) => { result.unavailable.push(`OCR: ${error.message}`); return []; });
|
||||
if (frame.path && includeVision && this.vision) result.vision_hint = await this.vision(frame.path).catch((error) => { result.unavailable.push(`vision: ${error.message}`); return null; });
|
||||
return result;
|
||||
}
|
||||
async zoom({ rect, ref } = {}) { const node = ref ? this.lastTree.find((item) => item.ref === ref) : null; const target = rect || node?.rect; if (!target) throw new Error('rect or current tree ref is required'); const raw = await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`)); const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), target); return { rect: target, screenshot_path: frame.path }; }
|
||||
async zoom({ rect, ref } = {}) { await mkdir(this.tmpDir, { recursive: true }); const node = ref ? this.lastTree.find((item) => item.ref === ref) : null; const target = rect || node?.rect; if (!target) throw new Error('rect or current tree ref is required'); const raw = await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`)); const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), target); return { rect: target, screenshot_path: frame.path }; }
|
||||
async find({ query, role } = {}) { const nodes = this.lastTree.length ? this.lastTree : await this.tree(); return nodes.map((node) => ({ ...node, score: score(query, node) })).filter((node) => node.score && (!role || normalize(node.role) === normalize(role))).sort((a, b) => b.score - a.score).slice(0, 20); }
|
||||
}
|
||||
|
||||
@@ -4,8 +4,49 @@ import path from 'node:path';
|
||||
/** Wayland input boundary. The helper owns portal consent and the EIS fd;
|
||||
* Node sends only bounded JSON actions and never uses hidden uinput. */
|
||||
export class PortalInputBackend {
|
||||
constructor({ command = process.env.JARVIS_EI_HELPER || path.resolve(new URL('./py/portal_remote_desktop.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn } = {}) { this.command = command; this.python = python; this.spawnImpl = spawnImpl; this.process = null; this.available = false; }
|
||||
async grant({ persist = false, monitors = 'focused' } = {}) { if (this.process) return { restore_token_present: Boolean(persist) }; this.process = this.spawnImpl(this.python, [this.command], { stdio: ['pipe', 'pipe', 'pipe'] }); let ready = ''; this.process.stdout.on('data', (data) => { ready += String(data); for (const line of ready.split(/\r?\n/).slice(0, -1)) { try { const event = JSON.parse(line); if (event.type === 'ready') { this.available = true; this._ready = event; } } catch {} } ready = ready.split(/\r?\n/).pop() || ''; }); await new Promise((resolve, reject) => { this.process.once('error', reject); const timer = setTimeout(resolve, 1500); this.process.stdout.once('data', () => { clearTimeout(timer); resolve(); }); }); return { restore_token_present: Boolean(persist), monitors, backend: this.available ? 'portal-ei' : 'none' }; }
|
||||
constructor({ command = process.env.JARVIS_EI_HELPER || path.resolve(new URL('./py/portal_remote_desktop.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn, timeoutMs = 120_000 } = {}) {
|
||||
this.command = command; this.python = python; this.spawnImpl = spawnImpl; this.timeoutMs = timeoutMs;
|
||||
this.process = null; this.available = false; this._grant = null;
|
||||
}
|
||||
grant({ persist = false, monitors = 'focused' } = {}) {
|
||||
if (this._grant) return this._grant;
|
||||
const child = this.process = this.spawnImpl(this.python, [this.command], { stdio: ['pipe', 'pipe', 'pipe'] });
|
||||
this._grant = new Promise((resolve, reject) => {
|
||||
let buffer = ''; let settled = false;
|
||||
const timer = setTimeout(() => fail(new Error('portal input consent timed out')), this.timeoutMs);
|
||||
const fail = (error) => {
|
||||
clearTimeout(timer);
|
||||
if (this.process === child) { this.available = false; this.process = null; this._grant = null; }
|
||||
if (!settled) { settled = true; reject(error); }
|
||||
child.kill('SIGTERM');
|
||||
};
|
||||
this._cancelGrant = () => fail(new Error('portal input grant revoked'));
|
||||
child.on('error', fail);
|
||||
child.once('close', () => {
|
||||
clearTimeout(timer);
|
||||
if (this.process === child) { this.available = false; this.process = null; this._grant = null; }
|
||||
if (!settled) { settled = true; reject(new Error('portal input helper exited before readiness')); }
|
||||
});
|
||||
child.stdin?.on('error', fail);
|
||||
child.stderr?.on('data', () => {});
|
||||
child.stdout.on('data', (data) => {
|
||||
buffer += String(data);
|
||||
if (buffer.length > 64 * 1024) { fail(new Error('portal helper output too large')); return; }
|
||||
let index;
|
||||
while ((index = buffer.indexOf('\n')) >= 0) {
|
||||
const line = buffer.slice(0, index); buffer = buffer.slice(index + 1);
|
||||
let event;
|
||||
try { event = JSON.parse(line); } catch { continue; }
|
||||
if (event.type === 'error') { fail(new Error(event.reason || 'portal input unavailable')); return; }
|
||||
if (event.type === 'ready' && this.process === child && !settled) {
|
||||
clearTimeout(timer); settled = true; this.available = true;
|
||||
resolve({ restore_token_present: Boolean(event.restore_token_present), monitors, backend: 'portal-ei' });
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
return this._grant;
|
||||
}
|
||||
send(action) { if (!this.available || !this.process?.stdin?.writable) throw new Error('portal EIS input backend is unavailable'); this.process.stdin.write(`${JSON.stringify(action)}\n`); }
|
||||
revoke() { this.available = false; this.process?.kill('SIGTERM'); this.process = null; }
|
||||
revoke() { this._cancelGrant?.(); this._cancelGrant = null; this.available = false; this.process = null; this._grant = null; }
|
||||
}
|
||||
|
||||
+14
-4
@@ -1,8 +1,10 @@
|
||||
const MAX_STEPS = 100;
|
||||
|
||||
export class ComputerUseSession {
|
||||
constructor({ stepsMax = 20, clock = () => Date.now(), audit } = {}) {
|
||||
constructor({ stepsMax = 20, grantMinutes = 3, mode = 'act', clock = () => Date.now(), audit } = {}) {
|
||||
this.clock = clock;
|
||||
this.mode = mode;
|
||||
this.grantMinutes = grantMinutes;
|
||||
this.stepsMax = Math.min(MAX_STEPS, Math.max(1, stepsMax));
|
||||
this.active = false;
|
||||
this.stepsUsed = 0;
|
||||
@@ -13,10 +15,11 @@ export class ComputerUseSession {
|
||||
}
|
||||
|
||||
grant({ persist = false, monitors = 'focused' } = {}) {
|
||||
if (this.mode === 'off') throw new Error('Desktop access is disabled in Settings');
|
||||
this.active = true;
|
||||
this.stepsUsed = 0;
|
||||
this.sessionId = `cu_${this.clock().toString(36)}`;
|
||||
this.expiresAt = this.clock() + 3 * 60 * 1000;
|
||||
this.expiresAt = this.clock() + this.grantMinutes * 60 * 1000;
|
||||
return { session_id: this.sessionId, restore_token_present: Boolean(persist), monitors, backend: this.backend };
|
||||
}
|
||||
|
||||
@@ -27,12 +30,14 @@ export class ComputerUseSession {
|
||||
this.sessionId = null;
|
||||
this.expiresAt = null;
|
||||
this.backend = 'none';
|
||||
void this.audit?.wipeTemp?.();
|
||||
Promise.resolve(this.audit?.wipeTemp?.()).catch(() => {});
|
||||
}
|
||||
|
||||
status() {
|
||||
if (this.active && this.expiresAt != null && this.clock() >= this.expiresAt) this.revoke();
|
||||
return {
|
||||
active: this.active,
|
||||
mode: this.mode,
|
||||
steps_used: this.stepsUsed,
|
||||
steps_max: this.stepsMax,
|
||||
grant_expires_at: this.expiresAt,
|
||||
@@ -40,12 +45,17 @@ export class ComputerUseSession {
|
||||
};
|
||||
}
|
||||
|
||||
beginStep() {
|
||||
assertActive() {
|
||||
if (!this.active) throw new Error('computer-use grant is inactive');
|
||||
if (this.expiresAt && this.clock() >= this.expiresAt) {
|
||||
this.revoke();
|
||||
throw new Error('computer-use grant expired');
|
||||
}
|
||||
}
|
||||
|
||||
beginStep() {
|
||||
this.assertActive();
|
||||
if (this.mode !== 'act') throw new Error('Desktop access is observe-only');
|
||||
if (this.stepsUsed >= this.stepsMax) throw new Error('computer-use step budget exhausted');
|
||||
this.stepsUsed += 1;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user