+89
-20
@@ -1,3 +1,8 @@
|
||||
import { PipeWireCapture } from './audio-pipewire.js';
|
||||
import { PipeWirePlayback } from './audio-playback.js';
|
||||
import { VadSegmenter } from './vad.js';
|
||||
import { FrameNormalizer } from '../computer-use/frame.js';
|
||||
import { ttsConfiguration } from './tts-config.js';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { HarnessBridge } from './harness-bridge.js';
|
||||
import { ComputerUseSession } from '../computer-use/session.js';
|
||||
@@ -22,13 +27,15 @@ export class JarvisDaemon extends EventEmitter {
|
||||
constructor() {
|
||||
super();
|
||||
this.recovery = new StateRecovery(); const restored = this.recovery.load(); this.state = restored.state; this.mode = restored.mode;
|
||||
this.voice = new VoiceStateMachine();
|
||||
this.settings = voiceSettings();
|
||||
this.startupSettings = this.settings;
|
||||
this.voice = new VoiceStateMachine({ idleMs: this.settings.idleMinutes * 60_000 });
|
||||
this.scheduler = new QvacScheduler({ concurrency: 1 });
|
||||
this.audit = new ComputerAudit();
|
||||
this.computer = new ComputerUseSession({ audit: this.audit });
|
||||
this.computer = new ComputerUseSession({ audit: this.audit, stepsMax: this.settings.computerSteps, grantMinutes: this.settings.computerGrantMinutes, mode: this.settings.computerMode });
|
||||
this.input = new PortalInputBackend();
|
||||
this.perception = new QvacPerception();
|
||||
this.observer = new DesktopObserver({ ocr: (image) => this.perception.ocr(image) });
|
||||
this.observer = new DesktopObserver({ normalizer: new FrameNormalizer({ maxLongEdge: this.settings.screenshotMaxEdge, quality: this.settings.screenshotQuality }), ocr: (image) => this.perception.ocr(image) });
|
||||
this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit });
|
||||
this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer, actuator: this.actuator });
|
||||
this.log = new PrivacyLog();
|
||||
@@ -36,6 +43,7 @@ export class JarvisDaemon extends EventEmitter {
|
||||
this.lastReply = '';
|
||||
this.voiceLoop = null;
|
||||
this._activeAsk = null;
|
||||
this._askGeneration = 0;
|
||||
this._idleTimer = setInterval(() => this.tickIdle(), 30_000);
|
||||
this._idleTimer.unref?.();
|
||||
this.telemetry = new RuntimeTelemetry();
|
||||
@@ -50,7 +58,7 @@ export class JarvisDaemon extends EventEmitter {
|
||||
|
||||
setState(state) { this.state = state; try { this.recovery.save({ state, mode: this.mode }); } catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); } this.emit('StateChanged', state); this.log.record('state', { state }).catch(() => {}); }
|
||||
async arm() { if (this.locked) return; await resumeQvac().catch(() => {}); this.voice.wake(); this.setState('LISTENING'); }
|
||||
async sleep() { this.voice.sleep(); this.setState('SLEEPING'); await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); }
|
||||
async sleep() { this.cancel(); this.voice.sleep(); this.setState('SLEEPING'); await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); }
|
||||
say(text) {
|
||||
const spoken = spokenReply(text) || spokenReply(this.lastReply) || 'There is nothing to repeat.';
|
||||
this.lastReply = spoken;
|
||||
@@ -59,6 +67,9 @@ export class JarvisDaemon extends EventEmitter {
|
||||
this._speakReply(spoken);
|
||||
}
|
||||
async ask(text) {
|
||||
if (this.locked) throw new Error('Jarvis is locked');
|
||||
if (this._settingsReload) throw new Error('Settings are being applied; try again shortly');
|
||||
const generation = ++this._askGeneration;
|
||||
this.voiceLoop?.interrupt?.();
|
||||
try {
|
||||
this.voice.typedUtterance(); this.setState('THINKING');
|
||||
@@ -66,17 +77,20 @@ export class JarvisDaemon extends EventEmitter {
|
||||
const job = this.scheduler.run(() => this.harness.ask(text), { lane: 'voice' });
|
||||
this._activeAsk = job;
|
||||
const reply = await job;
|
||||
if (generation !== this._askGeneration || this.locked) return '';
|
||||
this.telemetry.record('llm', startedAt, { success: true });
|
||||
const spoken = spokenReply(reply);
|
||||
if (!spoken) { this.setState('LISTENING'); return ''; }
|
||||
if (!spoken) { this._finishSpeech(); return ''; }
|
||||
this._beginSpeech(); this.lastReply = spoken; this.emit('Reply', spoken); this._speakReply(spoken); return spoken;
|
||||
} catch (error) {
|
||||
if (generation !== this._askGeneration) return '';
|
||||
this.telemetry.record('llm', Date.now(), { success: false }); this.emit('Error', 'QVAC', error.message); this.voice.cancel(); this.setState('ARMED'); throw error;
|
||||
} finally {
|
||||
this._activeAsk = null;
|
||||
if (generation === this._askGeneration) this._activeAsk = null;
|
||||
}
|
||||
}
|
||||
async resetContext() {
|
||||
this._askGeneration += 1;
|
||||
this.harness.cancel();
|
||||
this.scheduler.cancelQueued((job) => job.lane === 'voice');
|
||||
await this._activeAsk?.catch?.(() => {});
|
||||
@@ -99,15 +113,18 @@ export class JarvisDaemon extends EventEmitter {
|
||||
}
|
||||
_finishSpeech() {
|
||||
try { this.voice.finishSpeaking(); } catch {}
|
||||
if (this.state === 'SPEAKING') this.setState('LISTENING');
|
||||
if (this.settings.listeningMode !== 'conversation') { this.voice.cancel(); this.setState('ARMED'); }
|
||||
else this.setState('LISTENING');
|
||||
}
|
||||
_speakReply(spoken) {
|
||||
const generation = this._askGeneration;
|
||||
if (!this.voiceLoop) {
|
||||
this._finishSpeech();
|
||||
return;
|
||||
}
|
||||
const play = async () => {
|
||||
await this.ensureTts();
|
||||
if (generation !== this._askGeneration || this.locked) return;
|
||||
if (!this.voiceLoop?.status?.tts) {
|
||||
const reason = this.voiceLoop?.status?.errors?.tts;
|
||||
if (reason && reason !== 'Spoken replies are disabled') this.emit('Error', 'TTS', reason);
|
||||
@@ -119,14 +136,15 @@ export class JarvisDaemon extends EventEmitter {
|
||||
.catch((error) => {
|
||||
this.emit('Error', 'TTS', error.message);
|
||||
})
|
||||
.finally(() => this._finishSpeech());
|
||||
.finally(() => { if (generation === this._askGeneration) this._finishSpeech(); });
|
||||
}
|
||||
async ensureTts() {
|
||||
await this._voiceStarting;
|
||||
if (!this.voiceLoop) return;
|
||||
const settings = voiceSettings();
|
||||
if (!settings.ttsEnabled) return;
|
||||
const settings = this.settings;
|
||||
if (!settings.ttsEnabled) { this.voiceLoop.status.tts = false; return; }
|
||||
if (this.voiceLoop.status.tts) return;
|
||||
if (!this.voiceLoop.tts) this.voiceLoop.tts = new QvacVoiceAdapter({ role: 'tts' });
|
||||
if (!this.voiceLoop.tts) this.voiceLoop.tts = new QvacVoiceAdapter({ role: 'tts', settings });
|
||||
try {
|
||||
await this.voiceLoop.tts.start();
|
||||
this.voiceLoop.status.tts = true;
|
||||
@@ -139,16 +157,26 @@ export class JarvisDaemon extends EventEmitter {
|
||||
this.emit('Error', 'TTS', message);
|
||||
}
|
||||
}
|
||||
cancel() { this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computer.revoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
|
||||
computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; }
|
||||
cancel() { this._askGeneration += 1; this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computerRevoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
|
||||
computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); if (this.settings.computerMode === 'observe') return result; this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; }
|
||||
computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
|
||||
async startVoice() {
|
||||
startVoice() {
|
||||
if (this._voiceStarting) return this._voiceStarting;
|
||||
this._voiceStarting = this._startVoice().finally(() => { this._voiceStarting = null; });
|
||||
return this._voiceStarting;
|
||||
}
|
||||
async _startVoice() {
|
||||
if (this.voiceLoop) return;
|
||||
const settings = voiceSettings();
|
||||
const asr = new QvacVoiceAdapter({ role: 'asr' });
|
||||
const tts = settings.ttsEnabled ? new QvacVoiceAdapter({ role: 'tts' }) : null;
|
||||
const settings = this.settings;
|
||||
const asr = settings.microphoneEnabled ? new QvacVoiceAdapter({ role: 'asr', settings }) : null;
|
||||
const tts = settings.ttsEnabled ? new QvacVoiceAdapter({ role: 'tts', settings }) : null;
|
||||
if (!settings.ttsEnabled) console.log('jarvisd: voice: TTS disabled in config.json');
|
||||
const loop = new VoiceLoop({ daemon: this, wake: createWakeEngine(settings), asr, tts });
|
||||
const loop = new VoiceLoop({ daemon: this, wake: createWakeEngine(settings), asr, tts,
|
||||
capture: new PipeWireCapture({ target: settings.inputTarget }),
|
||||
playback: new PipeWirePlayback({ target: settings.outputTarget }),
|
||||
vad: new VadSegmenter({ params: { threshold: settings.vadThreshold, minSpeechDurationMs: settings.vadMinSpeechMs, minSilenceDurationMs: settings.vadSilenceMs, maxSpeechDurationMs: settings.vadMaxSpeechSeconds * 1000 } }),
|
||||
cooldownMs: settings.playbackCooldownMs, listeningMode: settings.listeningMode,
|
||||
});
|
||||
loop.on('error', (error) => { console.error(`jarvisd: voice: ${error.message}`); this.emit('Error', 'VOICE', error.message); });
|
||||
this.voiceLoop = loop;
|
||||
try { await loop.start(); this.emit('StateChanged', this.state); } catch (error) {
|
||||
@@ -157,15 +185,56 @@ export class JarvisDaemon extends EventEmitter {
|
||||
if (loop.status.tts) console.log('jarvisd: voice: TTS ready');
|
||||
else if (loop.status.errors.tts) console.error(`jarvisd: voice: TTS: ${loop.status.errors.tts}`);
|
||||
}
|
||||
async reloadSettings() {
|
||||
if (this._settingsReload) return this._settingsReload;
|
||||
this._settingsReload = this._reloadSettings().finally(() => { this._settingsReload = null; });
|
||||
return this._settingsReload;
|
||||
}
|
||||
async _reloadSettings() {
|
||||
if (this._activeAsk) throw new Error('Wait for the current request to finish before applying settings');
|
||||
const next = voiceSettings(null, { strict: true });
|
||||
if (next.ttsEnabled) ttsConfiguration(next);
|
||||
await this._voiceStarting;
|
||||
const previous = this.settings;
|
||||
this._askGeneration += 1;
|
||||
await this.voiceLoop?.stop?.();
|
||||
this.voiceLoop = null;
|
||||
this.settings = next;
|
||||
this.voice.idleMs = next.idleMinutes * 60_000;
|
||||
if (['computerMode', 'computerSteps', 'computerGrantMinutes'].some(key => next[key] !== previous[key])) this.computerRevoke();
|
||||
Object.assign(this.computer, { mode: next.computerMode, stepsMax: next.computerSteps, grantMinutes: next.computerGrantMinutes });
|
||||
Object.assign(this.observer.normalizer, { maxLongEdge: next.screenshotMaxEdge, quality: next.screenshotQuality });
|
||||
await this.startVoice();
|
||||
this.voice.cancel(); this.setState('ARMED');
|
||||
return JSON.stringify({ applied: true, voice: this.voiceLoop.status, environmentOverrides: ['JARVIS_TTS_MODEL', 'JARVIS_ASR_MODEL', 'JARVIS_WAKE_COMMAND', 'JARVIS_QVAC_MODEL'].filter(key => process.env[key]), restartRequired: ['chatModel', 'modelProfile', 'maxTurns', 'maxShellCalls', 'maxToolRounds'].filter(key => next[key] !== this.startupSettings[key]) });
|
||||
}
|
||||
async previewVoice(text) {
|
||||
if (this.locked) throw new Error('Unlock the desktop to preview a voice');
|
||||
if (this._activeAsk) throw new Error('Wait for the current request to finish before previewing');
|
||||
await this._settingsReload;
|
||||
await this.startVoice(); await this.ensureTts();
|
||||
if (!this.voiceLoop?.status.tts) throw new Error(this.voiceLoop?.status.errors.tts || 'Enable spoken replies to preview a voice');
|
||||
const preview = String(text || this.settings.previewText).slice(0, 1000);
|
||||
const generation = this._askGeneration;
|
||||
this._beginSpeech();
|
||||
try { await this.voiceLoop.speak(preview); } finally { if (generation === this._askGeneration) this._finishSpeech(); }
|
||||
}
|
||||
stopSpeech() { this.voiceLoop?.interrupt?.(); this._finishSpeech(); }
|
||||
setPushToTalk(pressed) { this.voiceLoop?.setPushToTalk(pressed); this.emit('PushToTalk', Boolean(pressed)); }
|
||||
runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), computer: this.computer.status(), voice: this.voiceLoop ? { ...this.voiceLoop.metrics.snapshot(), ...this.voiceLoop.status } : null, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }
|
||||
runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), settings: this.settings, computer: this.computer.status(), voice: this.voiceLoop ? { ...this.voiceLoop.metrics.snapshot(), ...this.voiceLoop.status } : null, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }
|
||||
async assessModelFit(model) { return JSON.stringify(await callQvac('assessModelFit', { modelSrc: String(model) })); }
|
||||
async downloadModel(model) { return JSON.stringify(await callQvac('downloadAsset', { modelSrc: String(model) })); }
|
||||
async cancelModel(model) { return JSON.stringify(await cancelQvacRequest({ modelId: String(model) })); }
|
||||
async wipeComputerTraces() { await this.audit.wipeTemp(); return true; }
|
||||
handleLockScreen(locked) { this.locked = Boolean(locked); if (this.locked) { this.cancel(); this.setState('ARMED'); } this.emit('LockScreenChanged', this.locked); }
|
||||
tickIdle() { if (!this.locked && this.voice.expireIdle() === 'SLEEPING' && this.state !== 'SLEEPING') this.sleep(); }
|
||||
async close() { clearInterval(this._idleTimer); clearInterval(this._telemetryTimer); this.computerRevoke(); this.recovery.save({ state: 'ARMED', mode: this.mode }); await this.voiceLoop?.stop?.(); await this.harness.close(); }
|
||||
async close() {
|
||||
clearInterval(this._idleTimer); clearInterval(this._telemetryTimer);
|
||||
this.cancel();
|
||||
try { this.recovery.save({ state: 'ARMED', mode: this.mode }); }
|
||||
catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); }
|
||||
try { await this.voiceLoop?.stop?.(); } finally { await this.harness.close(); }
|
||||
}
|
||||
}
|
||||
|
||||
export async function startDaemon() {
|
||||
|
||||
Reference in New Issue
Block a user