123 lines
8.2 KiB
JavaScript
123 lines
8.2 KiB
JavaScript
import { EventEmitter } from 'node:events';
|
|
import { HarnessBridge } from './harness-bridge.js';
|
|
import { ComputerUseSession } from '../computer-use/session.js';
|
|
import { VoiceStateMachine } from './voice-state.js';
|
|
import { QvacScheduler } from './qvac-scheduler.js';
|
|
import { cancelQvac, resumeQvac, suspendQvac, callQvac, cancelQvacRequest, qvacStatus } from './qvac-master.js';
|
|
import { PrivacyLog } from './privacy-log.js';
|
|
import { VoiceLoop } from './voice-loop.js';
|
|
import { QvacVoiceAdapter } from './voice-adapters.js';
|
|
import { createWakeEngine } from './wake-engine.js';
|
|
import { DesktopObserver } from '../computer-use/observer.js';
|
|
import { QvacPerception } from './perception.js';
|
|
import { ComputerAudit } from '../computer-use/audit.js';
|
|
import { PortalInputBackend } from '../computer-use/portal-input.js';
|
|
import { ComputerActuator } from '../computer-use/actuator.js';
|
|
import { RuntimeTelemetry } from './telemetry.js';
|
|
import { StateRecovery } from './recovery.js';
|
|
import { spokenReply } from '../skills/voice-prompt.js';
|
|
|
|
export class JarvisDaemon extends EventEmitter {
|
|
constructor() {
|
|
super();
|
|
this.recovery = new StateRecovery(); const restored = this.recovery.load(); this.state = restored.state; this.mode = restored.mode;
|
|
this.voice = new VoiceStateMachine();
|
|
this.scheduler = new QvacScheduler({ concurrency: 1 });
|
|
this.audit = new ComputerAudit();
|
|
this.computer = new ComputerUseSession({ audit: this.audit });
|
|
this.input = new PortalInputBackend();
|
|
this.perception = new QvacPerception();
|
|
this.observer = new DesktopObserver({ ocr: (image) => this.perception.ocr(image) });
|
|
this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit });
|
|
this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer, actuator: this.actuator });
|
|
this.log = new PrivacyLog();
|
|
this.locked = false;
|
|
this.lastReply = '';
|
|
this.voiceLoop = null;
|
|
this._idleTimer = setInterval(() => this.tickIdle(), 30_000);
|
|
this._idleTimer.unref?.();
|
|
this.telemetry = new RuntimeTelemetry();
|
|
this._telemetryTimer = setInterval(() => this.telemetry.sample(), 60_000); this._telemetryTimer.unref?.();
|
|
this.harness.on('agent_message_chunk', (ev) => this.emit('Token', ev?.text || ev?.delta || ''));
|
|
this.harness.on('permission', (ev) => this.emit('ConfirmationRequired', ev));
|
|
this.harness.on('hud_sidecar', (ev) => this.emit('ChipOffered', 'sidecar', ev?.title || 'Suggested action', JSON.stringify(ev || {})));
|
|
}
|
|
|
|
setState(state) { this.state = state; try { this.recovery.save({ state, mode: this.mode }); } catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); } this.emit('StateChanged', state); this.log.record('state', { state }).catch(() => {}); }
|
|
async arm() { if (this.locked) return; await resumeQvac().catch(() => {}); this.voice.wake(); this.setState('LISTENING'); }
|
|
async sleep() { this.voice.sleep(); this.setState('SLEEPING'); await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); }
|
|
say(text) {
|
|
const spoken = spokenReply(text) || spokenReply(this.lastReply) || 'There is nothing to repeat.';
|
|
this.lastReply = spoken;
|
|
this.emit('Reply', spoken);
|
|
this._beginSpeech();
|
|
this._speakReply(spoken);
|
|
}
|
|
async ask(text) {
|
|
this.voiceLoop?.interrupt?.();
|
|
try {
|
|
this.voice.typedUtterance(); this.setState('THINKING');
|
|
const startedAt = Date.now(); const reply = await this.scheduler.run(() => this.harness.ask(text), { lane: 'voice' }); this.telemetry.record('llm', startedAt, { success: true });
|
|
const spoken = spokenReply(reply);
|
|
this._beginSpeech(); this.lastReply = spoken; this.emit('Reply', spoken); this._speakReply(spoken); return spoken;
|
|
} catch (error) {
|
|
this.telemetry.record('llm', Date.now(), { success: false }); this.emit('Error', 'QVAC', error.message); this.voice.cancel(); this.setState('ARMED'); throw error;
|
|
}
|
|
}
|
|
_beginSpeech() {
|
|
try {
|
|
if (this.voice.state === 'ARMED' || this.voice.state === 'SLEEPING') this.voice.wake();
|
|
if (this.voice.state !== 'SPEAKING') this.voice.speak();
|
|
} catch {}
|
|
this.setState('SPEAKING');
|
|
}
|
|
_finishSpeech() {
|
|
try { this.voice.finishSpeaking(); } catch {}
|
|
if (this.state === 'SPEAKING') this.setState('LISTENING');
|
|
}
|
|
_speakReply(spoken) {
|
|
const playing = this.voiceLoop?.speak?.(spoken);
|
|
if (!playing) { this._finishSpeech(); return; }
|
|
Promise.resolve(playing).catch((error) => {
|
|
this.emit('Error', 'TTS', error.message);
|
|
if (this.state === 'SPEAKING') this._finishSpeech();
|
|
});
|
|
}
|
|
cancel() { this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computer.revoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
|
|
computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; }
|
|
computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
|
|
async startVoice() {
|
|
if (this.voiceLoop) return;
|
|
const voiceIO = new QvacVoiceAdapter();
|
|
const loop = new VoiceLoop({ daemon: this, wake: createWakeEngine(), asr: voiceIO, tts: voiceIO });
|
|
this.voiceLoop = loop;
|
|
try { await loop.start(); this.emit('StateChanged', this.state); } catch (error) {
|
|
this.voiceLoop = null; await loop.stop?.().catch(() => {}); this.emit('Error', 'VOICE_UNAVAILABLE', error.message); throw error;
|
|
}
|
|
}
|
|
setPushToTalk(pressed) { this.voiceLoop?.setPushToTalk(pressed); this.emit('PushToTalk', Boolean(pressed)); }
|
|
runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), computer: this.computer.status(), voice: this.voiceLoop?.metrics?.snapshot?.() || null, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }
|
|
async assessModelFit(model) { return JSON.stringify(await callQvac('assessModelFit', { modelSrc: String(model) })); }
|
|
async downloadModel(model) { return JSON.stringify(await callQvac('downloadAsset', { modelSrc: String(model) })); }
|
|
async cancelModel(model) { return JSON.stringify(await cancelQvacRequest({ modelId: String(model) })); }
|
|
async wipeComputerTraces() { await this.audit.wipeTemp(); return true; }
|
|
handleLockScreen(locked) { this.locked = Boolean(locked); if (this.locked) { this.cancel(); this.setState('ARMED'); } this.emit('LockScreenChanged', this.locked); }
|
|
tickIdle() { if (!this.locked && this.voice.expireIdle() === 'SLEEPING' && this.state !== 'SLEEPING') this.sleep(); }
|
|
async close() { clearInterval(this._idleTimer); clearInterval(this._telemetryTimer); this.computerRevoke(); this.recovery.save({ state: 'ARMED', mode: this.mode }); await this.voiceLoop?.stop?.(); await this.harness.close(); }
|
|
}
|
|
|
|
export async function startDaemon() {
|
|
const daemon = new JarvisDaemon();
|
|
daemon.startVoice().catch((error) => console.error(`jarvisd: voice unavailable: ${error.message}`));
|
|
import('./dbus-service.js')
|
|
.then(({ serveOnSessionBus }) => serveOnSessionBus(daemon))
|
|
.then(() => console.log('jarvisd: D-Bus name io.qvac.Jarvis ready'))
|
|
.catch((error) => {
|
|
daemon.emit('Error', 'DBUS_UNAVAILABLE', error.message);
|
|
console.error(`jarvisd: D-Bus unavailable: ${error.message}`);
|
|
});
|
|
process.on('SIGINT', () => daemon.close().finally(() => process.exit(0)));
|
|
console.log('jarvisd ready; QVAC inference remains GPU-gated');
|
|
return daemon;
|
|
}
|