Files
gnome-jarvis/computer-use/portal-input.js
T
snxraven 599bfe440d
Rolling release / release (push) Failing after 1m46s
Computer Use Updates
2026-09-13 19:30:17 -04:00

128 lines
6.8 KiB
JavaScript

import { spawn } from 'node:child_process';
import path from 'node:path';
/** Wayland input and ScreenCast boundary. The helper owns portal consent, the
* EIS fd, and the PipeWire remote; Node sends only bounded JSON actions. */
export class PortalInputBackend {
constructor({ command = process.env.JARVIS_EI_HELPER || path.resolve(new URL('./py/portal_remote_desktop.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn, timeoutMs = 120_000, actionTimeoutMs = 10000, onClose = () => {} } = {}) {
this.command = command; this.python = python; this.spawnImpl = spawnImpl; this.timeoutMs = timeoutMs; this.actionTimeoutMs = actionTimeoutMs; this.onClose = onClose; this._pending = new Map(); this._sequence = 0; this.streams = [];
this.process = null; this.available = false; this.screen = false; this._grant = null; this._frameWait = null;
}
grant({ persist = false, monitors = 'focused', mode = 'act' } = {}) {
if (this._grant) return this._grant;
const child = this.process = this.spawnImpl(this.python, [this.command], {
stdio: ['pipe', 'pipe', 'pipe'],
env: { ...process.env, JARVIS_CU_MODE: mode, JARVIS_CU_PERSIST: persist ? '1' : '0' },
});
this._grant = new Promise((resolve, reject) => {
let buffer = ''; let settled = false;
const timer = setTimeout(() => fail(new Error('portal input consent timed out')), this.timeoutMs);
const fail = (error) => {
if (this.process !== child) return;
clearTimeout(timer);
if (this.process === child) { this.available = false; this.screen = false; this.process = null; this._grant = null; }
this._rejectPending(error);
this._rejectFrame(error);
const wasReady = settled;
if (!settled) { settled = true; reject(error); }
child.kill('SIGTERM');
if (wasReady) this.onClose(error.message);
};
this._cancelGrant = () => fail(new Error('portal input grant revoked'));
child.on('error', fail);
child.once('close', () => {
clearTimeout(timer);
if (this.process !== child) return;
this.available = false; this.screen = false; this.process = null; this._grant = null;
this._rejectPending(new Error('portal helper exited'));
this._rejectFrame(new Error('portal helper exited'));
if (settled) this.onClose('portal helper exited');
if (!settled) { settled = true; reject(new Error('portal input helper exited before readiness' + (diagnostics ? ': ' + diagnostics : ''))); }
});
child.stdin?.on('error', fail);
let diagnostics = '';
child.stderr?.on('data', (data) => { diagnostics = (diagnostics + String(data)).slice(-2000); });
child.stdout.on('data', (data) => {
buffer += String(data);
if (buffer.length > 64 * 1024) { fail(new Error('portal helper output too large')); return; }
let index;
while ((index = buffer.indexOf('\n')) >= 0) {
const line = buffer.slice(0, index); buffer = buffer.slice(index + 1);
let event;
try { event = JSON.parse(line); } catch { continue; }
if (this.process !== child) continue;
if (event.type === 'ack') { this._settle(event.id, null, event); continue; }
if (event.type === 'error') {
if (!settled) { fail(new Error(event.reason || 'portal input unavailable')); return; }
const error = new Error(event.reason || 'portal helper error');
if (event.id != null && this._pending.has(event.id)) this._settle(event.id, error);
else if (event.id == null || event.id === this._frameWait?.id) this._rejectFrame(error);
continue;
}
if (event.type === 'frame') { if (!event.id || event.id === this._frameWait?.id) this._resolveFrame(event.path || event); continue; }
if (event.type === 'ready' && this.process === child && !settled) {
clearTimeout(timer); settled = true; this.available = true; this.screen = Boolean(event.screen);
this.streams = event.streams || [];
resolve({ streams: this.streams, eis_error: event.eis_error || null, restore_token_present: Boolean(event.restore_token_present), monitors, backend: event.backend || 'portal-ei', screen: this.screen });
}
}
});
});
return this._grant;
}
async ready() {
if (this._grant) {
try { await this._grant; } catch (error) { throw new Error(error?.message || 'Allow desktop access in Settings, then try again'); }
}
if (!this.available || !this.process?.stdin?.writable) throw new Error('Allow desktop access in Settings, then try again');
}
send(action) {
if (!this.available || !this.process?.stdin?.writable) throw new Error('portal EIS input backend is unavailable');
const id = ++this._sequence;
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
this._settle(id, new Error('desktop action acknowledgment timed out; grant revoked'));
this.revoke();
this.onClose('desktop action timed out');
}, this.actionTimeoutMs);
this._pending.set(id, { resolve, reject, timer });
try { this.process.stdin.write(JSON.stringify({ ...action, id }) + '\n'); }
catch (error) { this._settle(id, error); }
});
}
_settle(id, error, result) {
const wait = this._pending.get(id);
if (!wait) return;
this._pending.delete(id); clearTimeout(wait.timer);
if (error) wait.reject(error); else wait.resolve(result);
}
_rejectPending(error) { for (const id of this._pending.keys()) this._settle(id, error); }
captureFrame(output) {
if (!this.available || !this.process?.stdin?.writable) throw new Error('PipeWire ScreenCast is unavailable until desktop access is granted');
if (this._frameWait) throw new Error('a PipeWire frame grab is already in flight');
return new Promise((resolve, reject) => {
const timer = setTimeout(() => this._rejectFrame(new Error('PipeWire frame timed out')), 8000);
const id = ++this._sequence;
this._frameWait = { id,
resolve: (value) => { clearTimeout(timer); resolve(value); },
reject: (error) => { clearTimeout(timer); reject(error); },
};
try { this.process.stdin.write(JSON.stringify({ type: 'frame', path: output, id }) + '\n'); } catch (error) { this._rejectFrame(error); }
});
}
_resolveFrame(path) { const wait = this._frameWait; this._frameWait = null; wait?.resolve(path); }
_rejectFrame(error) { const wait = this._frameWait; this._frameWait = null; wait?.reject(error); }
revoke() {
const child = this.process;
this._rejectPending(new Error('portal input grant revoked'));
this.streams = [];
this._cancelGrant?.();
this._cancelGrant = null;
this.available = false;
this.screen = false;
this.process = null;
this._grant = null;
try { child?.kill?.('SIGTERM'); } catch {}
}
}