81 lines
4.9 KiB
JavaScript
81 lines
4.9 KiB
JavaScript
import { EventEmitter } from 'node:events';
|
|
import { PipeWireCapture } from './audio-pipewire.js';
|
|
import { PipeWirePlayback } from './audio-playback.js';
|
|
import { WakeEngine } from './wake-engine.js';
|
|
import { VadSegmenter } from './vad.js';
|
|
import { isMeaningfulTranscript, SentenceBuffer } from './transcript.js';
|
|
import { VoiceMetrics } from './voice-metrics.js';
|
|
|
|
export const POST_PLAYBACK_COOLDOWN_MS = 400;
|
|
|
|
const FAST_COMMANDS = new Map([
|
|
['cancel', 'cancel'], ['stop', 'cancel'], ['never mind', 'cancel'], ['hands off', 'cancel'],
|
|
['stop clicking', 'cancel'], ["that's enough", 'cancel'], ['go to sleep', 'sleep'],
|
|
['privacy mode', 'sleep'], ['repeat that', 'repeat'], ['dictate this', 'dictate'],
|
|
['look at my screen', 'screen'], ['use the computer', 'computer'], ['take the wheel', 'computer'],
|
|
['switch to compose', 'compose'], ['switch to imagine', 'imagine'], ['switch to files', 'files'],
|
|
['switch to computer', 'computer'],
|
|
]);
|
|
|
|
export function fastCommand(text) { return FAST_COMMANDS.get(String(text || '').trim().toLowerCase()) || null; }
|
|
|
|
export class VoiceLoop extends EventEmitter {
|
|
constructor({ daemon, capture = new PipeWireCapture(), playback = new PipeWirePlayback(), wake = new WakeEngine(), vad = new VadSegmenter(), asr, tts, cooldownMs = POST_PLAYBACK_COOLDOWN_MS, now = () => Date.now() } = {}) {
|
|
super(); this.daemon = daemon; this.capture = capture; this.playback = playback; this.wake = wake; this.vad = vad; this.asr = asr; this.tts = tts; this.cooldownMs = cooldownMs; this.now = now;
|
|
this.isSpeaking = false; this.cooldownUntil = 0; this.running = false; this.ptt = false; this._speechQueue = Promise.resolve(); this.metrics = new VoiceMetrics();
|
|
capture.on('audio', (chunk) => this.pushAudio(chunk));
|
|
capture.on('error', (error) => this.emit('error', error));
|
|
wake.on('wake', (phrase) => this.wakeHeard(phrase));
|
|
vad.on('level', (rms) => daemon?.emit('ListeningLevel', rms));
|
|
vad.on('utterance', (audio) => this.transcribe(audio));
|
|
}
|
|
|
|
async start() { if (this.running) return; await this.asr?.start?.(); await this.tts?.start?.(); this.running = true; this.capture.start(); this.wake.start?.(); this.wake.resume(); }
|
|
async stop() { this.running = false; this.capture.stop(); this.wake.close(); this.vad.reset(); this.playback.stop(); await this.asr?.stop?.(); await this.tts?.stop?.(); }
|
|
setPushToTalk(pressed) { this.ptt = Boolean(pressed); if (this.ptt) this.wakeHeard('push-to-talk'); else this.vad.end(); }
|
|
pushAudio(chunk) {
|
|
if (!this.running) return;
|
|
if (this.isSpeaking || this.now() < this.cooldownUntil) { this.metrics.feedbackDrop(); return; }
|
|
this.daemon?.emit('ListeningLevel', Math.min(1, Math.max(0, chunk.length ? 0.01 : 0)));
|
|
if (!this.ptt) this.wake.push(chunk);
|
|
if (this.ptt || this.daemon?.state === 'LISTENING') this.vad.push(chunk);
|
|
}
|
|
wakeHeard(phrase) {
|
|
if (!this.running || this.daemon?.locked) return;
|
|
this.metrics.wakeAccepted(); this.daemon?.emit('WakeHeard', phrase); this.daemon?.arm?.(); this.emit('wake', phrase);
|
|
}
|
|
async transcribe(audio) {
|
|
if (!this.asr?.transcribeAudio) return;
|
|
const text = await this.asr.transcribeAudio(audio).catch((error) => { this.emit('error', error); return ''; });
|
|
if (!isMeaningfulTranscript(text)) { this.metrics.wakeRejected(); return; }
|
|
this.metrics.utterance();
|
|
this.daemon?.emit('PartialTranscript', text); this.daemon?.emit('FinalTranscript', text);
|
|
const command = fastCommand(text);
|
|
if (command === 'cancel' || command === 'sleep') { this.daemon?.cancel?.(); if (command === 'sleep') await this.daemon?.sleep?.(); return; }
|
|
if (command === 'repeat') { this.daemon?.say?.(this.daemon?.lastReply || 'There is nothing to repeat.'); return; }
|
|
if (command === 'computer') { this.daemon?.computerGrant?.(false); return; }
|
|
if (command === 'dictate') { this.emit('dictate'); return; }
|
|
if (command === 'screen') { await this.daemon?.ask?.('What is on my screen?'); return; }
|
|
await this.daemon?.ask?.(text);
|
|
}
|
|
async speak(text) {
|
|
if (!isMeaningfulTranscript(text) || !this.tts?.speak) return;
|
|
this.metrics.reply();
|
|
const sentences = String(text).match(/[^.!?]+[.!?]+|[^.!?]+$/g)?.map((part) => part.trim()).filter(isMeaningfulTranscript) || [];
|
|
this._speechQueue = this._speechQueue.then(async () => {
|
|
for (const sentence of sentences) await this.speakSentence(sentence);
|
|
});
|
|
return this._speechQueue;
|
|
}
|
|
async speakSentence(text) {
|
|
this.isSpeaking = true; this.wake.pause(); this.daemon?.emit('StateChanged', 'SPEAKING');
|
|
try { const audio = await this.tts.speak(text); await this.playback.play(audio.samples); this.daemon?.emit('SpeakingLevel', 0); }
|
|
finally {
|
|
this.isSpeaking = false; this.cooldownUntil = this.now() + this.cooldownMs; this.wake.resume();
|
|
if (this.daemon?.state === 'SPEAKING') { this.daemon.voice?.finishSpeaking?.(); this.daemon.setState?.('LISTENING'); }
|
|
}
|
|
}
|
|
}
|
|
|
|
export { SentenceBuffer };
|