import { PipeWireCapture } from './audio-pipewire.js'; import { PipeWirePlayback } from './audio-playback.js'; import { VadSegmenter } from './vad.js'; import { FrameNormalizer } from '../computer-use/frame.js'; import { ttsConfiguration } from './tts-config.js'; import { EventEmitter } from 'node:events'; import { HarnessBridge } from './harness-bridge.js'; import { ComputerUseSession } from '../computer-use/session.js'; import { VoiceStateMachine } from './voice-state.js'; import { QvacScheduler } from './qvac-scheduler.js'; import { cancelQvac, closeQvac, resumeQvac, suspendQvac, callQvac, cancelQvacRequest, qvacStatus } from './qvac-master.js'; import { PrivacyLog } from './privacy-log.js'; import { VoiceLoop } from './voice-loop.js'; import { QvacVoiceAdapter } from './voice-adapters.js'; import { voiceSettings } from './voice-settings.js'; import { createWakeEngine } from './wake-engine.js'; import { DesktopObserver } from '../computer-use/observer.js'; import { QvacPerception } from './perception.js'; import { ComputerAudit } from '../computer-use/audit.js'; import { PortalInputBackend } from '../computer-use/portal-input.js'; import { ComputerActuator } from '../computer-use/actuator.js'; import { RuntimeTelemetry } from './telemetry.js'; import { StateRecovery } from './recovery.js'; import { spokenReply } from '../skills/voice-prompt.js'; export class JarvisDaemon extends EventEmitter { constructor() { super(); this.recovery = new StateRecovery(); const restored = this.recovery.load(); this.state = restored.state; this.mode = restored.mode; this.settings = voiceSettings(); this.startupSettings = this.settings; this.voice = new VoiceStateMachine({ idleMs: this.settings.idleMinutes * 60_000 }); this.scheduler = new QvacScheduler({ concurrency: 1 }); this.audit = new ComputerAudit(); this.computer = new ComputerUseSession({ audit: this.audit, stepsMax: this.settings.computerSteps, grantMinutes: this.settings.computerGrantMinutes, mode: this.settings.computerMode }); this.input = new PortalInputBackend(); this.perception = new QvacPerception(); this.observer = new DesktopObserver({ normalizer: new FrameNormalizer({ maxLongEdge: this.settings.screenshotMaxEdge, quality: this.settings.screenshotQuality }), ocr: (image) => this.perception.ocr(image), framebuffer: { capture: (output) => this.input.captureFrame(output) }, }); this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit }); this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer, actuator: this.actuator, fsAccess: this.settings.fsAccess }); this.log = new PrivacyLog(); this.locked = false; this.lastReply = ''; this.voiceLoop = null; this._activeAsk = null; this._askGeneration = 0; this._idleTimer = setInterval(() => this.tickIdle(), 30_000); this._idleTimer.unref?.(); this.telemetry = new RuntimeTelemetry(); this._telemetryTimer = setInterval(() => this.telemetry.sample(), 60_000); this._telemetryTimer.unref?.(); this.harness.on('agent_message_chunk', (ev) => this.emit('Token', ev?.text || ev?.delta || '')); this.harness.on('agent_thought_chunk', (ev) => this.emit('Thinking', String(ev?.text || ev?.delta || ''))); this.harness.on('tool_call', (ev) => this.emit('ToolCall', JSON.stringify({ name: ev?.call?.name || 'tool', arguments: ev?.call?.arguments || {} }))); this.harness.on('tool_result', (ev) => this.emit('ToolResult', JSON.stringify({ name: ev?.name || 'tool', result: ev?.result || '', toolCallId: ev?.toolCallId || '' }))); this.harness.on('permission', (ev) => this.emit('ConfirmationRequired', ev)); this.harness.on('hud_sidecar', (ev) => this.emit('ChipOffered', 'sidecar', ev?.title || 'Suggested action', JSON.stringify(ev || {}))); } setState(state) { this.state = state; try { this.recovery.save({ state, mode: this.mode }); } catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); } this.emit('StateChanged', state); this.log.record('state', { state }).catch(() => {}); } async arm() { if (this.locked) return; await resumeQvac().catch(() => {}); await this.ensureAsr(); this.voice.wake(); this.setState('LISTENING'); } async sleep() { if (this._activeAsk) { await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); return; } this.cancel(); this.voice.sleep(); this.setState('SLEEPING'); if (this.settings.freeVramOnIdle !== false) await this.parkModels(); else await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); } async parkModels() { try { await this.voiceLoop?.parkModels?.(); } catch (error) { this.emit('Error', 'VOICE_UNLOAD', error.message); } try { await this.harness.resetContext(); } catch (error) { this.emit('Error', 'QVAC_UNLOAD', error.message); } try { await closeQvac(); } catch (error) { this.emit('Error', 'QVAC_CLOSE', error.message); } } say(text) { const spoken = spokenReply(text) || spokenReply(this.lastReply) || 'There is nothing to repeat.'; this.lastReply = spoken; this.emit('Reply', spoken); this._beginSpeech(); this._speakReply(spoken); } async ask(text) { if (this.locked) throw new Error('Jarvis is locked'); if (this._settingsReload) throw new Error('Settings are being applied; try again shortly'); const generation = ++this._askGeneration; this.voiceLoop?.interrupt?.(); try { this.voice.typedUtterance(); this.setState('THINKING'); const startedAt = Date.now(); const job = this.scheduler.run(() => this.harness.ask(text), { lane: 'voice' }); this._activeAsk = job; const reply = await job; if (generation !== this._askGeneration || this.locked) return ''; this.telemetry.record('llm', startedAt, { success: true }); const spoken = spokenReply(reply); if (!spoken) { this._finishSpeech(); return ''; } this._beginSpeech(); this.lastReply = spoken; this.emit('Reply', spoken); this._speakReply(spoken); return spoken; } catch (error) { if (generation !== this._askGeneration) return ''; this.telemetry.record('llm', Date.now(), { success: false }); this.emit('Error', 'QVAC', error.message); this.voice.cancel(); this.setState('ARMED'); throw error; } finally { if (generation === this._askGeneration) this._activeAsk = null; } } async resetContext() { this._askGeneration += 1; this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); await this._activeAsk?.catch?.(() => {}); await this.harness.resetContext(); this.lastReply = ''; this.voiceLoop?.interrupt?.(); this.voice.cancel(); this.setState('ARMED'); this.emit('ContextReset'); } confirmPermission(jobId, toolCallId, decision) { this.harness.session?.permit?.(String(jobId || ''), String(toolCallId || ''), String(decision || 'deny')); } _beginSpeech() { try { if (this.voice.state === 'ARMED' || this.voice.state === 'SLEEPING') this.voice.wake(); if (this.voice.state !== 'SPEAKING') this.voice.speak(); } catch {} this.setState('SPEAKING'); } _finishSpeech() { try { this.voice.finishSpeaking(); } catch {} if (this.settings.listeningMode !== 'conversation') { this.voice.cancel(); this.setState('ARMED'); } else this.setState('LISTENING'); } _speakReply(spoken) { const generation = this._askGeneration; if (!this.voiceLoop) { this._finishSpeech(); return; } const play = async () => { await this.ensureTts(); if (generation !== this._askGeneration || this.locked) return; if (!this.voiceLoop?.status?.tts) { const reason = this.voiceLoop?.status?.errors?.tts; if (reason && reason !== 'Spoken replies are disabled') this.emit('Error', 'TTS', reason); return; } await this.voiceLoop.speak(spoken); }; Promise.resolve(play()) .catch((error) => { this.emit('Error', 'TTS', error.message); }) .finally(() => { if (generation === this._askGeneration) this._finishSpeech(); }); } async ensureTts() { await this._voiceStarting; if (!this.voiceLoop) return; const settings = this.settings; if (!settings.ttsEnabled) { this.voiceLoop.status.tts = false; return; } if (this.voiceLoop.status.tts) return; if (!this.voiceLoop.tts) this.voiceLoop.tts = new QvacVoiceAdapter({ role: 'tts', settings }); const ready = await this.voiceLoop.ensureTts(); if (ready) console.log('jarvisd: voice: TTS ready'); else if (this.voiceLoop.status.errors.tts) { const message = this.voiceLoop.status.errors.tts; console.error(`jarvisd: voice: TTS: ${message}`); this.emit('Error', 'TTS', message); } } async ensureAsr() { await this._voiceStarting; if (!this.voiceLoop || !this.settings.microphoneEnabled) return; if (this.voiceLoop.status?.asr) return; if (!this.voiceLoop.asr) this.voiceLoop.asr = new QvacVoiceAdapter({ role: 'asr', settings: this.settings }); await this.voiceLoop.ensureAsr?.(); } cancel() { this._askGeneration += 1; this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computerRevoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); } computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); this.input.grant({ persist, mode: this.settings.computerMode }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; } computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); } startVoice() { if (this._voiceStarting) return this._voiceStarting; this._voiceStarting = this._startVoice().finally(() => { this._voiceStarting = null; }); return this._voiceStarting; } async _startVoice() { if (this.voiceLoop) return; const settings = this.settings; const asr = settings.microphoneEnabled ? new QvacVoiceAdapter({ role: 'asr', settings }) : null; const tts = settings.ttsEnabled ? new QvacVoiceAdapter({ role: 'tts', settings }) : null; if (!settings.ttsEnabled) console.log('jarvisd: voice: TTS disabled in config.json'); const loop = new VoiceLoop({ daemon: this, wake: createWakeEngine(settings), asr, tts, capture: new PipeWireCapture({ target: settings.inputTarget }), playback: new PipeWirePlayback({ target: settings.outputTarget }), vad: new VadSegmenter({ params: { threshold: settings.vadThreshold, minSpeechDurationMs: settings.vadMinSpeechMs, minSilenceDurationMs: settings.vadSilenceMs, maxSpeechDurationMs: settings.vadMaxSpeechSeconds * 1000 } }), cooldownMs: settings.playbackCooldownMs, listeningMode: settings.listeningMode, }); loop.on('error', (error) => { console.error(`jarvisd: voice: ${error.message}`); this.emit('Error', 'VOICE', error.message); }); this.voiceLoop = loop; try { await loop.start(); this.emit('StateChanged', this.state); } catch (error) { this.voiceLoop = null; await loop.stop?.().catch(() => {}); this.emit('Error', 'VOICE_UNAVAILABLE', error.message); throw error; } if (loop.status.wake) console.log('jarvisd: voice: wake detector ready'); else if (loop.status.errors.wake) console.error(`jarvisd: voice: wake: ${loop.status.errors.wake}`); if (loop.status.capture) console.log('jarvisd: voice: microphone capture ready'); else if (loop.status.errors.capture) console.error(`jarvisd: voice: capture: ${loop.status.errors.capture}`); } async reloadSettings() { if (this._settingsReload) return this._settingsReload; this._settingsReload = this._reloadSettings().finally(() => { this._settingsReload = null; }); return this._settingsReload; } async _reloadSettings() { if (this._activeAsk) throw new Error('Wait for the current request to finish before applying settings'); const next = voiceSettings(null, { strict: true }); if (next.ttsEnabled) ttsConfiguration(next); await this._voiceStarting; const previous = this.settings; this._askGeneration += 1; await this.voiceLoop?.stop?.(); this.voiceLoop = null; this.settings = next; this.voice.idleMs = next.idleMinutes * 60_000; if (['computerMode', 'computerSteps', 'computerGrantMinutes'].some(key => next[key] !== previous[key])) this.computerRevoke(); Object.assign(this.computer, { mode: next.computerMode, stepsMax: next.computerSteps, grantMinutes: next.computerGrantMinutes }); Object.assign(this.observer.normalizer, { maxLongEdge: next.screenshotMaxEdge, quality: next.screenshotQuality }); await this.startVoice(); this.voice.cancel(); this.setState('ARMED'); return JSON.stringify({ applied: true, voice: this.voiceLoop.status, environmentOverrides: ['JARVIS_TTS_MODEL', 'JARVIS_ASR_MODEL', 'JARVIS_WAKE_COMMAND', 'JARVIS_QVAC_MODEL'].filter(key => process.env[key]), restartRequired: ['chatModel', 'modelProfile', 'maxTurns', 'maxShellCalls', 'maxToolRounds', 'fsAccess'].filter(key => next[key] !== this.startupSettings[key]) }); } async previewVoice(text) { if (this.locked) throw new Error('Unlock the desktop to preview a voice'); if (this._activeAsk) throw new Error('Wait for the current request to finish before previewing'); await this._settingsReload; await this.startVoice(); await this.ensureTts(); if (!this.voiceLoop?.status.tts) throw new Error(this.voiceLoop?.status.errors.tts || 'Enable spoken replies to preview a voice'); const preview = String(text || this.settings.previewText).slice(0, 1000); const generation = this._askGeneration; this._beginSpeech(); try { await this.voiceLoop.speak(preview); } finally { if (generation === this._askGeneration) this._finishSpeech(); } } stopSpeech() { this.voiceLoop?.interrupt?.(); this._finishSpeech(); } setPushToTalk(pressed) { this.voiceLoop?.setPushToTalk(pressed); this.emit('PushToTalk', Boolean(pressed)); } runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), settings: this.settings, computer: this.computer.status(), voice: this.voiceLoop ? { ...this.voiceLoop.metrics.snapshot(), ...this.voiceLoop.status } : null, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); } async assessModelFit(model) { return JSON.stringify(await callQvac('assessModelFit', { modelSrc: String(model) })); } async downloadModel(model) { return JSON.stringify(await callQvac('downloadAsset', { modelSrc: String(model) })); } async cancelModel(model) { return JSON.stringify(await cancelQvacRequest({ modelId: String(model) })); } async wipeComputerTraces() { await this.audit.wipeTemp(); return true; } handleLockScreen(locked) { this.locked = Boolean(locked); if (this.locked) { this.cancel(); this.setState('ARMED'); } this.emit('LockScreenChanged', this.locked); } tickIdle() { if (!this.locked && this.voice.expireIdle() === 'SLEEPING' && this.state !== 'SLEEPING') this.sleep(); } async close() { clearInterval(this._idleTimer); clearInterval(this._telemetryTimer); this.cancel(); try { this.recovery.save({ state: 'ARMED', mode: this.mode }); } catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); } try { await this.voiceLoop?.stop?.(); } finally { await this.harness.close(); } } } export async function startDaemon() { const daemon = new JarvisDaemon(); daemon.startVoice().catch((error) => console.error(`jarvisd: voice unavailable: ${error.message}`)); import('./dbus-service.js') .then(({ serveOnSessionBus }) => serveOnSessionBus(daemon)) .then(() => console.log('jarvisd: D-Bus name io.qvac.Jarvis ready')) .catch((error) => { daemon.emit('Error', 'DBUS_UNAVAILABLE', error.message); console.error(`jarvisd: D-Bus unavailable: ${error.message}`); }); process.on('SIGINT', () => daemon.close().finally(() => process.exit(0))); console.log('jarvisd ready; QVAC inference remains GPU-gated'); return daemon; }