gnome-jarvis/daemon/index.js

import { PipeWireCapture } from './audio-pipewire.js';
import { PipeWirePlayback } from './audio-playback.js';
import { VadSegmenter } from './vad.js';
import { FrameNormalizer } from '../computer-use/frame.js';
import { ttsConfiguration } from './tts-config.js';
import { EventEmitter } from 'node:events';
import { HarnessBridge } from './harness-bridge.js';
import { ComputerUseSession } from '../computer-use/session.js';
import { VoiceStateMachine } from './voice-state.js';
import { QvacScheduler } from './qvac-scheduler.js';
import { cancelQvac, closeQvac, resumeQvac, suspendQvac, callQvac, cancelQvacRequest, qvacStatus } from './qvac-master.js';
import { PrivacyLog } from './privacy-log.js';
import { VoiceLoop } from './voice-loop.js';
import { QvacVoiceAdapter } from './voice-adapters.js';
import { voiceSettings } from './voice-settings.js';
import { createWakeEngine } from './wake-engine.js';
import { DesktopObserver } from '../computer-use/observer.js';
import { QvacPerception } from './perception.js';
import { ComputerAudit } from '../computer-use/audit.js';
import { PortalInputBackend } from '../computer-use/portal-input.js';
import { ComputerActuator } from '../computer-use/actuator.js';
import { RuntimeTelemetry } from './telemetry.js';
import { StateRecovery } from './recovery.js';
import { spokenReply, voiceSystemPrompt } from '../skills/voice-prompt.js';
import { ensureAgentWorkspace, applyAssistantName, applyAssistantPrompt, normalizeAssistantName } from './agent-workspace.js';

export class JarvisDaemon extends EventEmitter {
  constructor() {
    super();
    this.recovery = new StateRecovery(); const restored = this.recovery.load(); this.state = restored.state; this.mode = restored.mode;
    this.settings = voiceSettings();
    this.startupSettings = this.settings;
    this.workspace = ensureAgentWorkspace({ name: this.settings.assistantName, prompt: this.settings.assistantPrompt });
    this.voice = new VoiceStateMachine({ idleMs: this.settings.idleMinutes * 60_000 });
    this.scheduler = new QvacScheduler({ concurrency: 1 });
    this.audit = new ComputerAudit();
    this.computer = new ComputerUseSession({ audit: this.audit, stepsMax: this.settings.computerSteps, grantMinutes: this.settings.computerGrantMinutes, mode: this.settings.computerMode });
    this.input = new PortalInputBackend();
    this.perception = new QvacPerception();
    this.observer = new DesktopObserver({
      normalizer: new FrameNormalizer({ maxLongEdge: this.settings.screenshotMaxEdge, quality: this.settings.screenshotQuality }),
      ocr: (image) => this.perception.ocr(image),
      framebuffer: { capture: (output) => this.input.captureFrame(output) },
    });
    this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit });
    this.harness = new HarnessBridge({ cwd: this.workspace, computer: this.computer, observer: this.observer, actuator: this.actuator, fsAccess: this.settings.fsAccess });
    this.log = new PrivacyLog();
    this.locked = false;
    this.lastReply = '';
    this.voiceLoop = null;
    this._activeAsk = null;
    this._askGeneration = 0;
    this._idleTimer = setInterval(() => this.tickIdle(), 30_000);
    this._idleTimer.unref?.();
    this.telemetry = new RuntimeTelemetry();
    this._telemetryTimer = setInterval(() => this.telemetry.sample(), 60_000); this._telemetryTimer.unref?.();
    this.harness.on('agent_message_chunk', (ev) => this.emit('Token', ev?.text || ev?.delta || ''));
    this.harness.on('agent_thought_chunk', (ev) => this.emit('Thinking', String(ev?.text || ev?.delta || '')));
    this.harness.on('tool_call', (ev) => this.emit('ToolCall', JSON.stringify({ name: ev?.call?.name || 'tool', arguments: ev?.call?.arguments || {} })));
    this.harness.on('tool_result', (ev) => this.emit('ToolResult', JSON.stringify({ name: ev?.name || 'tool', result: ev?.result || '', toolCallId: ev?.toolCallId || '' })));
    this.harness.on('permission', (ev) => this.emit('ConfirmationRequired', ev));
    this.harness.on('hud_sidecar', (ev) => this.emit('ChipOffered', 'sidecar', ev?.title || 'Suggested action', JSON.stringify(ev || {})));
  }

  setState(state) { this.state = state; try { this.recovery.save({ state, mode: this.mode }); } catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); } this.emit('StateChanged', state); this.log.record('state', { state }).catch(() => {}); }
  async arm() {
    if (this.locked) return;
    await resumeQvac().catch(() => {});
    await this.ensureAsr();
    this.voice.wake();
    this.setState('LISTENING');
  }
  async sleep() {
    if (this._activeAsk) {
      await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message));
      return;
    }
    this.cancel();
    this.voice.sleep();
    this.setState('SLEEPING');
    if (this.settings.freeVramOnIdle !== false) await this.parkModels();
    else await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message));
  }
  async parkModels() {
    try { await this.voiceLoop?.parkModels?.(); }
    catch (error) { this.emit('Error', 'VOICE_UNLOAD', error.message); }
    try { await this.harness.resetContext(); }
    catch (error) { this.emit('Error', 'QVAC_UNLOAD', error.message); }
    try { await closeQvac(); }
    catch (error) { this.emit('Error', 'QVAC_CLOSE', error.message); }
  }
  say(text) {
    const spoken = spokenReply(text) || spokenReply(this.lastReply) || 'There is nothing to repeat.';
    this.lastReply = spoken;
    this.emit('Reply', spoken);
    this._beginSpeech();
    this._speakReply(spoken);
  }
  async ask(text) {
    if (this.locked) throw new Error('Jarvis is locked');
    if (this._settingsReload) throw new Error('Settings are being applied; try again shortly');
    const generation = ++this._askGeneration;
    this.voiceLoop?.interrupt?.();
    try {
      this.voice.typedUtterance(); this.setState('THINKING');
      const startedAt = Date.now();
      const job = this.scheduler.run(() => this.harness.ask(text), { lane: 'voice' });
      this._activeAsk = job;
      const reply = await job;
      if (generation !== this._askGeneration || this.locked) return '';
      this.telemetry.record('llm', startedAt, { success: true });
      const spoken = spokenReply(reply);
      if (!spoken) { this._finishSpeech(); return ''; }
      this._beginSpeech(); this.lastReply = spoken; this.emit('Reply', spoken); this._speakReply(spoken); return spoken;
    } catch (error) {
      if (generation !== this._askGeneration) return '';
      this.telemetry.record('llm', Date.now(), { success: false }); this.emit('Error', 'QVAC', error.message); this.voice.cancel(); this.setState('ARMED'); throw error;
    } finally {
      if (generation === this._askGeneration) this._activeAsk = null;
    }
  }
  async resetContext() {
    this._askGeneration += 1;
    this.harness.cancel();
    this.scheduler.cancelQueued((job) => job.lane === 'voice');
    await this._activeAsk?.catch?.(() => {});
    await this.harness.resetContext();
    this.lastReply = '';
    this.voiceLoop?.interrupt?.();
    this.voice.cancel();
    this.setState('ARMED');
    this.emit('ContextReset');
  }
  confirmPermission(jobId, toolCallId, decision) {
    this.harness.session?.permit?.(String(jobId || ''), String(toolCallId || ''), String(decision || 'deny'));
  }
  _beginSpeech() {
    try {
      if (this.voice.state === 'ARMED' || this.voice.state === 'SLEEPING') this.voice.wake();
      if (this.voice.state !== 'SPEAKING') this.voice.speak();
    } catch {}
    this.setState('SPEAKING');
  }
  _finishSpeech() {
    try { this.voice.finishSpeaking(); } catch {}
    if (this.settings.listeningMode !== 'conversation') { this.voice.cancel(); this.setState('ARMED'); }
    else this.setState('LISTENING');
  }
  _speakReply(spoken) {
    const generation = this._askGeneration;
    if (!this.voiceLoop) {
      this._finishSpeech();
      return;
    }
    const play = async () => {
      await this.ensureTts();
      if (generation !== this._askGeneration || this.locked) return;
      if (!this.voiceLoop?.status?.tts) {
        const reason = this.voiceLoop?.status?.errors?.tts;
        if (reason && reason !== 'Spoken replies are disabled') this.emit('Error', 'TTS', reason);
        return;
      }
      await this.voiceLoop.speak(spoken);
    };
    Promise.resolve(play())
      .catch((error) => {
        this.emit('Error', 'TTS', error.message);
      })
      .finally(() => { if (generation === this._askGeneration) this._finishSpeech(); });
  }
  async ensureTts() {
    await this._voiceStarting;
    if (!this.voiceLoop) return;
    const settings = this.settings;
    if (!settings.ttsEnabled) { this.voiceLoop.status.tts = false; return; }
    if (this.voiceLoop.status.tts) return;
    if (!this.voiceLoop.tts) this.voiceLoop.tts = new QvacVoiceAdapter({ role: 'tts', settings });
    const ready = await this.voiceLoop.ensureTts();
    if (ready) console.log('jarvisd: voice: TTS ready');
    else if (this.voiceLoop.status.errors.tts) {
      const message = this.voiceLoop.status.errors.tts;
      console.error(`jarvisd: voice: TTS: ${message}`);
      this.emit('Error', 'TTS', message);
    }
  }
  async ensureAsr() {
    await this._voiceStarting;
    if (!this.voiceLoop || !this.settings.microphoneEnabled) return;
    if (this.voiceLoop.status?.asr) return;
    if (!this.voiceLoop.asr) this.voiceLoop.asr = new QvacVoiceAdapter({ role: 'asr', settings: this.settings });
    await this.voiceLoop.ensureAsr?.();
  }
  cancel() { this._askGeneration += 1; this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computerRevoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
  computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); this.input.grant({ persist, mode: this.settings.computerMode }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; }
  computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
  startVoice() {
    if (this._voiceStarting) return this._voiceStarting;
    this._voiceStarting = this._startVoice().finally(() => { this._voiceStarting = null; });
    return this._voiceStarting;
  }
  async _startVoice() {
    if (this.voiceLoop) return;
    const settings = this.settings;
    const asr = settings.microphoneEnabled ? new QvacVoiceAdapter({ role: 'asr', settings }) : null;
    const tts = settings.ttsEnabled ? new QvacVoiceAdapter({ role: 'tts', settings }) : null;
    if (!settings.ttsEnabled) console.log('jarvisd: voice: TTS disabled in config.json');
    const loop = new VoiceLoop({ daemon: this, wake: createWakeEngine(settings), asr, tts,
      capture: new PipeWireCapture({ target: settings.inputTarget }),
      playback: new PipeWirePlayback({ target: settings.outputTarget }),
      vad: new VadSegmenter({ params: { threshold: settings.vadThreshold, minSpeechDurationMs: settings.vadMinSpeechMs, minSilenceDurationMs: settings.vadSilenceMs, maxSpeechDurationMs: settings.vadMaxSpeechSeconds * 1000 } }),
      cooldownMs: settings.playbackCooldownMs, listeningMode: settings.listeningMode,
    });
    loop.on('error', (error) => { console.error(`jarvisd: voice: ${error.message}`); this.emit('Error', 'VOICE', error.message); });
    this.voiceLoop = loop;
    try { await loop.start(); this.emit('StateChanged', this.state); } catch (error) {
      this.voiceLoop = null; await loop.stop?.().catch(() => {}); this.emit('Error', 'VOICE_UNAVAILABLE', error.message); throw error;
    }
    if (loop.status.wake) console.log('jarvisd: voice: wake detector ready');
    else if (loop.status.errors.wake) console.error(`jarvisd: voice: wake: ${loop.status.errors.wake}`);
    if (loop.status.capture) console.log('jarvisd: voice: microphone capture ready');
    else if (loop.status.errors.capture) console.error(`jarvisd: voice: capture: ${loop.status.errors.capture}`);
  }
  async reloadSettings() {
    if (this._settingsReload) return this._settingsReload;
    this._settingsReload = this._reloadSettings().finally(() => { this._settingsReload = null; });
    return this._settingsReload;
  }
  async _reloadSettings() {
    if (this._activeAsk) throw new Error('Wait for the current request to finish before applying settings');
    const next = voiceSettings(null, { strict: true });
    if (next.ttsEnabled) ttsConfiguration(next);
    await this._voiceStarting;
    const previous = this.settings;
    this._askGeneration += 1;
    await this.voiceLoop?.stop?.();
    this.voiceLoop = null;
    this.settings = next;
    this.voice.idleMs = next.idleMinutes * 60_000;
    const who = applyAssistantName(this.workspace, next.assistantName);
    applyAssistantPrompt(this.workspace, next.assistantPrompt);
    if (who !== normalizeAssistantName(previous.assistantName) || String(next.assistantPrompt || '') !== String(previous.assistantPrompt || '')) {
      this.harness.options.system = voiceSystemPrompt(who, next.assistantPrompt);
      await this.harness.resetContext();
    }
    if (['computerMode', 'computerSteps', 'computerGrantMinutes'].some(key => next[key] !== previous[key])) this.computerRevoke();
    Object.assign(this.computer, { mode: next.computerMode, stepsMax: next.computerSteps, grantMinutes: next.computerGrantMinutes });
    Object.assign(this.observer.normalizer, { maxLongEdge: next.screenshotMaxEdge, quality: next.screenshotQuality });
    await this.startVoice();
    this.voice.cancel(); this.setState('ARMED');
    return JSON.stringify({ applied: true, voice: this.voiceLoop.status, environmentOverrides: ['JARVIS_TTS_MODEL', 'JARVIS_ASR_MODEL', 'JARVIS_WAKE_COMMAND', 'JARVIS_QVAC_MODEL'].filter(key => process.env[key]), restartRequired: ['chatModel', 'modelProfile', 'maxTurns', 'maxShellCalls', 'maxToolRounds', 'fsAccess'].filter(key => next[key] !== this.startupSettings[key]) });
  }
  async previewVoice(text) {
    if (this.locked) throw new Error('Unlock the desktop to preview a voice');
    if (this._activeAsk) throw new Error('Wait for the current request to finish before previewing');
    await this._settingsReload;
    await this.startVoice(); await this.ensureTts();
    if (!this.voiceLoop?.status.tts) throw new Error(this.voiceLoop?.status.errors.tts || 'Enable spoken replies to preview a voice');
    const preview = String(text || this.settings.previewText).slice(0, 1000);
    const generation = this._askGeneration;
    this._beginSpeech();
    try { await this.voiceLoop.speak(preview); } finally { if (generation === this._askGeneration) this._finishSpeech(); }
  }
  stopSpeech() { this.voiceLoop?.interrupt?.(); this._finishSpeech(); }
  setPushToTalk(pressed) { this.voiceLoop?.setPushToTalk(pressed); this.emit('PushToTalk', Boolean(pressed)); }
  runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), settings: this.settings, computer: this.computer.status(), voice: this.voiceLoop ? { ...this.voiceLoop.metrics.snapshot(), ...this.voiceLoop.status } : null, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }
  async assessModelFit(model) { return JSON.stringify(await callQvac('assessModelFit', { modelSrc: String(model) })); }
  async downloadModel(model) { return JSON.stringify(await callQvac('downloadAsset', { modelSrc: String(model) })); }
  async cancelModel(model) { return JSON.stringify(await cancelQvacRequest({ modelId: String(model) })); }
  async wipeComputerTraces() { await this.audit.wipeTemp(); return true; }
  handleLockScreen(locked) { this.locked = Boolean(locked); if (this.locked) { this.cancel(); this.setState('ARMED'); } this.emit('LockScreenChanged', this.locked); }
  tickIdle() { if (!this.locked && this.voice.expireIdle() === 'SLEEPING' && this.state !== 'SLEEPING') this.sleep(); }
  async close() {
    clearInterval(this._idleTimer); clearInterval(this._telemetryTimer);
    this.cancel();
    try { this.recovery.save({ state: 'ARMED', mode: this.mode }); }
    catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); }
    try { await this.voiceLoop?.stop?.(); } finally { await this.harness.close(); }
  }
}

export async function startDaemon() {
  const daemon = new JarvisDaemon();
  daemon.startVoice().catch((error) => console.error(`jarvisd: voice unavailable: ${error.message}`));
  import('./dbus-service.js')
    .then(({ serveOnSessionBus }) => serveOnSessionBus(daemon))
    .then(() => console.log('jarvisd: D-Bus name io.qvac.Jarvis ready'))
    .catch((error) => {
      daemon.emit('Error', 'DBUS_UNAVAILABLE', error.message);
      console.error(`jarvisd: D-Bus unavailable: ${error.message}`);
    });
  process.on('SIGINT', () => daemon.close().finally(() => process.exit(0)));
  console.log('jarvisd ready; QVAC inference remains GPU-gated');
  return daemon;
}