gnome-jarvis/daemon/voice-loop.js

import { EventEmitter } from 'node:events';
import { PipeWireCapture } from './audio-pipewire.js';
import { PipeWirePlayback } from './audio-playback.js';
import { WakeEngine } from './wake-engine.js';
import { VadSegmenter } from './vad.js';
import { isMeaningfulTranscript, isSpeakable, SentenceBuffer } from './transcript.js';
import { VoiceMetrics } from './voice-metrics.js';

export const POST_PLAYBACK_COOLDOWN_MS = 400;

const FAST_COMMANDS = new Map([
  ['cancel', 'cancel'], ['stop', 'cancel'], ['never mind', 'cancel'], ['hands off', 'cancel'],
  ['stop clicking', 'cancel'], ["that's enough", 'cancel'], ['go to sleep', 'sleep'],
  ['privacy mode', 'sleep'], ['repeat that', 'repeat'], ['dictate this', 'dictate'],
  ['look at my screen', 'screen'], ['use the computer', 'computer'], ['take the wheel', 'computer'],
  ['switch to compose', 'compose'], ['switch to imagine', 'imagine'], ['switch to files', 'files'],
  ['switch to computer', 'computer'],
]);

export function fastCommand(text) { return FAST_COMMANDS.get(String(text || '').trim().toLowerCase()) || null; }

export class VoiceLoop extends EventEmitter {
  constructor({ daemon, capture = new PipeWireCapture(), playback = new PipeWirePlayback(), wake = new WakeEngine(), vad = new VadSegmenter(), asr, tts, cooldownMs = POST_PLAYBACK_COOLDOWN_MS, now = () => Date.now() } = {}) {
    super(); this.daemon = daemon; this.capture = capture; this.playback = playback; this.wake = wake; this.vad = vad; this.asr = asr; this.tts = tts; this.cooldownMs = cooldownMs; this.now = now;
    this.isSpeaking = false; this.cooldownUntil = 0; this.running = false; this.ptt = false; this._generation = 0; this._speechQueue = Promise.resolve(); this.metrics = new VoiceMetrics();
    capture.on('audio', (chunk) => this.pushAudio(chunk));
    capture.on('error', (error) => { this.status.capture = false; this.status.errors.capture = error.message; this.emit('error', error); });
    capture.on('close', () => { this.status.capture = false; if (this.running) { this.status.errors.capture = 'Microphone stream closed'; this.emit('error', new Error('Microphone stream closed')); } });
    wake.on('unavailable', () => { this.status.wake = false; });
    wake.on('error', (error) => { this.status.wake = false; this.emit('error', error); });
    this.status = { asr: false, tts: false, capture: false, wake: false, errors: {} };
    wake.on('wake', (phrase) => this.wakeHeard(phrase));
    vad.on('level', (rms) => daemon?.emit('ListeningLevel', rms));
    vad.on('utterance', (audio) => this.transcribe(audio).catch((error) => this.emit('error', error)));
  }

  async start() {
    if (this.running) return;
    for (const [name, adapter] of [['tts', this.tts], ['asr', this.asr]]) {
      try {
        if (adapter) { await adapter.start?.(); this.status[name] = true; }
        else if (name === 'tts') this.status.errors.tts = 'Spoken replies are disabled';
      } catch (error) {
        const message = String(error?.message || error);
        this.status.errors[name] = message;
        this.emit('error', new Error(`${name.toUpperCase()}: ${message}`));
      }
    }
    this.running = true;
    if (this.status.asr) {
      try { this.capture.start(); this.status.capture = true; }
      catch (error) { this.status.errors.capture = error.message; this.emit('error', error); }
      try { this.wake.start?.(); this.wake.resume(); this.status.wake = Boolean(this.wake.command || this.wake.detect); }
      catch (error) { this.status.errors.wake = error.message; this.emit('error', error); }
    }
  }
  async stop() { this.running = false; this.capture.stop(); this.wake.close(); this.vad.reset(); this.playback.stop(); await this.asr?.stop?.(); if (this.tts !== this.asr) await this.tts?.stop?.(); }
  setPushToTalk(pressed) { this.ptt = Boolean(pressed) && this.status.asr && this.status.capture; if (this.ptt) this.wakeHeard('push-to-talk'); else this.vad.end(); }
  pushAudio(chunk) {
    if (!this.running || this.daemon?.locked || this.daemon?.state === 'SLEEPING') return;
    if (this.isSpeaking || this.now() < this.cooldownUntil) { this.metrics.feedbackDrop(); return; }
    try { this.daemon?.emit('ListeningLevel', Math.min(1, Math.max(0, chunk?.length ? 0.01 : 0))); } catch {}
    if (!this.ptt) this.wake.push(chunk);
    if (this.ptt || this.daemon?.state === 'LISTENING') this.vad.push(chunk);
  }
  wakeHeard(phrase) {
    if (!this.running || this.daemon?.locked) return;
    this.metrics.wakeAccepted(); this.daemon?.emit('WakeHeard', phrase); this.daemon?.arm?.(); this.emit('wake', phrase);
  }
  async transcribe(audio) {
    if (!this.status.asr || !this.asr?.transcribeAudio) return;
    const text = await this.asr.transcribeAudio(audio).catch((error) => { this.emit('error', error); return ''; });
    if (!this.running || this.daemon?.locked) return;
    if (!isMeaningfulTranscript(text)) { this.metrics.wakeRejected(); return; }
    this.metrics.utterance();
    this.daemon?.emit('PartialTranscript', text); this.daemon?.emit('FinalTranscript', text);
    const command = fastCommand(text);
    if (command === 'cancel' || command === 'sleep') { this.daemon?.cancel?.(); if (command === 'sleep') await this.daemon?.sleep?.(); return; }
    if (command === 'repeat') { this.daemon?.say?.(this.daemon?.lastReply || 'There is nothing to repeat.'); return; }
    if (command === 'computer') { this.daemon?.computerGrant?.(false); return; }
    if (command === 'dictate') { this.emit('dictate'); return; }
    if (command === 'screen') { await this.daemon?.ask?.('What is on my screen?'); return; }
    await this.daemon?.ask?.(text);
  }
  interrupt() {
    this._generation += 1;
    this.playback.stop();
    this.isSpeaking = false;
    this.wake.resume();
  }
  _releaseSpeaking() {
    this.isSpeaking = false;
    this.wake.resume();
    if (this.daemon?.state === 'SPEAKING') { this.daemon.voice?.finishSpeaking?.(); this.daemon.setState?.('LISTENING'); }
  }
  async speak(text) {
    const generation = this._generation || 0;
    if (!isSpeakable(text) || !this.tts?.speak || !this.status.tts) { this._releaseSpeaking(); return; }
    this.metrics.reply();
    const sentences = String(text).match(/[^.!?]+[.!?]+|[^.!?]+$/g)?.map((part) => part.trim()).filter(isSpeakable) || [];
    if (!sentences.length) { this._releaseSpeaking(); return; }
    this._speechQueue = this._speechQueue.catch(() => {}).then(async () => {
      if (generation !== this._generation) return;
      for (const sentence of sentences) {
        if (generation !== this._generation) return;
        await this.speakSentence(sentence);
      }
    });
    return this._speechQueue;
  }
  async speakSentence(text) {
    this.isSpeaking = true; this.wake.pause(); this.daemon?.emit('StateChanged', 'SPEAKING');
    try { const audio = await this.tts.speak(text); await this.playback.play(audio.samples); try { this.daemon?.emit('SpeakingLevel', 0); } catch {} }
    finally {
      this.isSpeaking = false; this.cooldownUntil = this.now() + this.cooldownMs; this.wake.resume();
      if (this.daemon?.state === 'SPEAKING') { this.daemon.voice?.finishSpeaking?.(); this.daemon.setState?.('LISTENING'); }
    }
  }
}

export { SentenceBuffer };