import test from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { EventEmitter } from 'node:events'; import { VoiceLoop, fastCommand } from '../daemon/voice-loop.js'; import { pcmS16le } from '../daemon/voice-adapters.js'; import { WakeEngine } from '../daemon/wake-engine.js'; import { VadSegmenter } from '../daemon/vad.js'; import { PipeWireCapture, pcmRms } from '../daemon/audio-pipewire.js'; import { SentenceBuffer, isMeaningfulTranscript, isSpeakable, speakableForTts, stripMarkdownForSpeech, forChatDisplay } from '../daemon/transcript.js'; test('Phase 4 transcript filtering and sentence buffering are deterministic', () => { assert.equal(isMeaningfulTranscript('[BLANK_AUDIO]'), false); assert.equal(isMeaningfulTranscript('hi'), false); assert.equal(isSpeakable('OK.'), true); assert.equal(isSpeakable('[object Object]'), false); assert.equal(speakableForTts('Using `run_terminal_cmd`'), 'Using run terminal cmd'); assert.equal(speakableForTts('Your IP is 192.168.0.1'), 'Your I P is 192 168 0 1'); assert.equal( speakableForTts('QVAC fetched https://example.com/ip'), 'Quantum Verse Automatic Computer fetched example dot com slash I P' ); assert.equal( speakableForTts('## Hello\n- First item\n- **Second** item\nSee [docs](https://example.com).'), 'Hello First item Second item See docs.' ); assert.equal(forChatDisplay('Greetings, **young rebel**.\nHow may I serve you?'), 'Greetings, young rebel.\nHow may I serve you?'); assert.equal(forChatDisplay('Greetings, **Raven**. Next line.'), 'Greetings, Raven. Next line.'); assert.equal(forChatDisplay('Use 2 * 3, not **four**.'), 'Use 2 * 3, not four.'); assert.equal(stripMarkdownForSpeech('Greetings, **young rebel**.\nHow may I serve you?'), 'Greetings, young rebel. How may I serve you?'); assert.equal(isMeaningfulTranscript('what time is it'), true); const out = []; const buffer = new SentenceBuffer({ onSentence: (s) => out.push(s) }); buffer.push('First sentence. Second'); buffer.push(' sentence!'); buffer.flush(); assert.deepEqual(out, ['First sentence.', 'Second sentence!']); }); test('Phase 4 wake engine emits only configured local detections', () => { const wake = new WakeEngine({ phrases: ['hey jarvis'], detect: () => ({ phrase: 'hey jarvis' }) }); const heard = []; wake.on('wake', (phrase) => heard.push(phrase)); wake.push(Buffer.from([0])); assert.deepEqual(heard, ['hey jarvis']); wake.pause(); wake.push(Buffer.from([0])); assert.equal(heard.length, 1); }); test('Phase 4 VAD emits a bounded utterance after silence', () => { const vad = new VadSegmenter({ frameMs: 100, params: { threshold: 0.6, minSpeechDurationMs: 100, minSilenceDurationMs: 200 } }); const utterances = []; vad.on('utterance', (audio) => utterances.push(audio)); const loud = Buffer.alloc(3200); for (let i = 0; i < loud.length; i += 2) loud.writeInt16LE(20_000, i); const quiet = Buffer.alloc(3200); vad.push(loud); vad.push(quiet); vad.push(quiet); assert.equal(utterances.length, 1); assert.ok(utterances[0].length > 0); assert.ok(pcmRms(loud) > 0); }); test('Phase 4 fast commands are handled before the harness', () => { assert.equal(fastCommand('Hands Off'), 'cancel'); assert.equal(fastCommand('take the wheel'), 'computer'); assert.equal(fastCommand('use the camera'), 'camera'); assert.equal(fastCommand('use my webcam'), 'camera'); assert.equal(fastCommand('ordinary question'), null); }); test('Phase 4 feedback gate drops capture while speaking and during cooldown', () => { const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {}; const wake = new WakeEngine({ detect: () => ({ phrase: 'hey jarvis' }) }); const vad = new VadSegmenter({ frameMs: 20 }); let pushed = 0; const original = vad.push.bind(vad); vad.push = (x) => { pushed++; original(x); }; const daemon = new EventEmitter(); daemon.state = 'LISTENING'; const loop = new VoiceLoop({ daemon, capture, wake, vad, now: () => 1000 }); loop.running = true; loop.isSpeaking = true; loop.pushAudio(Buffer.alloc(3200)); assert.equal(pushed, 0); loop.isSpeaking = false; loop.cooldownUntil = 1200; loop.pushAudio(Buffer.alloc(3200)); assert.equal(pushed, 0); }); test('Phase 4 PipeWire capture uses a named 16 kHz mono node', () => { let args; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); child.kill = () => {}; const capture = new PipeWireCapture({ spawnImpl: (_cmd, received) => { args = received; return child; } }); capture.start(); assert.deepEqual(args, ['--record', '--raw', '--format', 's16', '--rate', '16000', '--channels', '1', '--properties', 'node.name=Jarvis', '-']); capture.stop(); }); test('empty TTS still leaves the daemon listening', async () => { const daemon = new EventEmitter(); daemon.state = 'SPEAKING'; daemon.voice = { finishSpeaking() { daemon.finished = true; } }; daemon.setState = (state) => { daemon.state = state; }; const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {}; const wake = new WakeEngine({ detect: () => null }); const loop = new VoiceLoop({ daemon, capture, wake, vad: new VadSegmenter(), tts: { speak: async () => ({ samples: new Int16Array(0) }) } }); await loop.speak('[object Object]'); assert.equal(daemon.state, 'LISTENING'); assert.equal(daemon.finished, true); }); test('multi-sentence speech returns to listening once', async () => { const daemon = new EventEmitter(); const states = []; daemon.state = 'SPEAKING'; daemon.voice = { finishSpeaking() { daemon.finished = (daemon.finished || 0) + 1; }, }; daemon.setState = (state) => { daemon.state = state; states.push(state); }; const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {}; const wake = new WakeEngine({ detect: () => null }); const loop = new VoiceLoop({ daemon, capture, wake, vad: new VadSegmenter(), tts: { speak: async () => ({ samples: new Int16Array(2) }) }, playback: { play: async () => {}, stop() {} }, }); loop.status.tts = true; await loop.speak('Hello there. How are you today?'); assert.equal(daemon.state, 'LISTENING'); assert.equal(daemon.finished, 1); assert.deepEqual(states, ['LISTENING']); }); test('PCM packing uses even s16le sample pairs', () => { const buf = Buffer.alloc(4); buf.writeInt16LE(256, 0); buf.writeInt16LE(-2, 2); const packed = pcmS16le(buf); assert.equal(packed.samples.length, 2); assert.equal(packed.samples[0], 256); assert.equal(packed.samples[1], -2); const fromBytes = pcmS16le(Uint8Array.from([0, 1, 0xfe, 0xff])); assert.equal(fromBytes.samples.length, 2); assert.equal(fromBytes.samples[0], 256); }); test('ASR failure keeps the microphone hot and still allows speech output', async () => { const capture = new EventEmitter(); let captured = false; capture.start = () => { captured = true; }; capture.stop = () => {}; let spoken = ''; const loop = new VoiceLoop({ capture, wake: new WakeEngine(), asr: { start: async () => { throw new Error('Whisper plugin missing'); } }, tts: { start: async () => {}, speak: async (text) => { spoken = text; return { samples: new Int16Array(8) }; } }, playback: { play: async () => {}, stop() {} }, }); loop.on('error', () => {}); await loop.start(); assert.equal(captured, true); assert.equal(loop.status.asr, false); assert.equal(loop.status.tts, false); await loop.speak('Speech still works.'); assert.equal(spoken, 'Speech still works.'); assert.equal(loop.status.tts, true); assert.equal(await loop.ensureAsr(), false); assert.match(loop.status.errors.asr, /Whisper/); await loop.stop(); }); test('TTS failure preserves ASR and push to talk without wake command', async () => { const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {}; const loop = new VoiceLoop({ capture, wake: new WakeEngine(), asr: { start: async () => {} }, tts: { start: async () => { throw new Error('TTS failed'); } }, playback: { stop() {} }, }); loop.on('error', () => {}); await loop.start(); assert.equal(loop.status.asr, false); await loop.setPushToTalk(true); assert.equal(loop.status.asr, true); assert.equal(loop.ptt, true); assert.equal(await loop.ensureTts(), false); assert.equal(loop.status.tts, false); assert.equal(loop.status.wake, false); await loop.stop(); }); test('QVAC numeric PCM arrays preserve signed 16-bit samples', () => { assert.deepEqual([...pcmS16le([256, -32768, 32767]).samples], [256, -32768, 32767]); }); test('voice loop retries microphone capture after PipeWire comes up', async () => { let starts = 0; const capture = new EventEmitter(); capture.start = () => { starts += 1; if (starts === 1) queueMicrotask(() => capture.emit('close', { code: 1 })); }; capture.stop = () => {}; const loop = new VoiceLoop({ capture, asr: { start: async () => {} }, tts: { start: async () => {} }, wake: new WakeEngine({ detect: () => null }), vad: new VadSegmenter(), captureRetryMs: 20, }); loop.on('error', () => {}); await loop.start(); assert.equal(loop.status.asr, false); await new Promise((resolve) => setTimeout(resolve, 5)); assert.equal(loop.status.capture, false); await new Promise((resolve) => setTimeout(resolve, 40)); assert.ok(starts >= 2); assert.equal(loop.status.capture, true); await loop.stop(); }); test('voice loop retries ASR after a first-login GPU miss', async () => { let attempts = 0; const capture = new EventEmitter(); capture.start = () => { capture.started = true; }; capture.stop = () => {}; const loop = new VoiceLoop({ capture, asr: { start: async () => { attempts += 1; if (attempts === 1) throw new Error('GPU not ready'); } }, tts: { start: async () => {} }, wake: new WakeEngine({ detect: () => null }), vad: new VadSegmenter(), asrRetryMs: 20, }); loop.on('error', () => {}); await loop.start(); assert.equal(loop.status.asr, false); assert.equal(capture.started, true); assert.equal(attempts, 0); assert.equal(await loop.ensureAsr(), false); assert.equal(attempts, 1); await new Promise((resolve) => setTimeout(resolve, 50)); assert.equal(attempts, 2); assert.equal(loop.status.asr, true); assert.equal(capture.started, true); await loop.stop(); }); test('voice adapters load canonical ASR and TTS plugins', () => { const source = readFileSync(new URL('../daemon/voice-adapters.js', import.meta.url), 'utf8'); assert.match(source, /whispercpp-transcription/); assert.match(source, /tts-ggml/); }); function tonePcm(ms, amplitude = 12_000, hz = 220) { const samples = Math.floor(16_000 * ms / 1000); const buf = Buffer.alloc(samples * 2); for (let i = 0; i < samples; i++) buf.writeInt16LE(Math.floor(amplitude * Math.sin(2 * Math.PI * hz * i / 16_000)), i * 2); return buf; } test('built-in CPU wake detector fires on a two-peak hey-jarvis cadence', async () => { const { KeywordWakeEngine, createWakeEngine } = await import('../daemon/wake-engine.js'); const engine = createWakeEngine({ command: 'jarvis-wake-bridge', phrases: ['hey jarvis'] }); assert.equal(engine instanceof KeywordWakeEngine, true); const heard = []; engine.on('wake', (phrase) => heard.push(phrase)); engine.push(Buffer.concat([tonePcm(80, 0), tonePcm(220), tonePcm(90, 0), tonePcm(280), tonePcm(200, 0)])); assert.deepEqual(heard, ['hey jarvis']); engine.push(tonePcm(800)); assert.equal(heard.length, 1); }); test('sleeping voice loop still feeds the wake detector and skips VAD', () => { const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {}; let wakeFrames = 0; let vadFrames = 0; const wake = new WakeEngine({ detect: () => null }); wake.push = () => { wakeFrames += 1; }; const vad = new VadSegmenter(); vad.push = () => { vadFrames += 1; }; const daemon = new EventEmitter(); daemon.state = 'SLEEPING'; const loop = new VoiceLoop({ daemon, capture, wake, vad }); loop.running = true; loop.pushAudio(Buffer.alloc(640)); assert.equal(wakeFrames, 1); assert.equal(vadFrames, 0); }); test('voice start keeps the microphone hot without loading ASR or TTS', async () => { const capture = new EventEmitter(); let captured = false; capture.start = () => { captured = true; }; capture.stop = () => {}; let asrStarts = 0; let ttsStarts = 0; const loop = new VoiceLoop({ capture, asr: { start: async () => { asrStarts += 1; } }, tts: { start: async () => { ttsStarts += 1; } }, wake: new WakeEngine({ detect: () => null }), }); await loop.start(); assert.equal(captured, true); assert.equal(loop.status.capture, true); assert.equal(loop.status.asr, false); assert.equal(loop.status.tts, false); assert.equal(asrStarts, 0); assert.equal(ttsStarts, 0); assert.equal(loop.status.wake, true); await loop.parkModels(); assert.equal(loop.status.asr, false); assert.equal(loop.status.tts, false); await loop.stop(); }); function deferred() { let resolve, reject; const promise = new Promise((yes, no) => { resolve = yes; reject = no; }); return { promise, resolve, reject }; } function pipelineLoop(tts, playback) { const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {}; const loop = new VoiceLoop({ capture, wake: new WakeEngine({ detect: () => null }), tts, playback }); loop.status.tts = true; return loop; } const drainMicrotasks = async () => { for (let i = 0; i < 20; i++) await Promise.resolve(); }; test('speech prepares only the next sentence during playback, preserving order and feedback gating', async () => { const spoken = [], played = []; const gates = [deferred(), deferred(), deferred()]; const loop = pipelineLoop({ speak: async (text) => { spoken.push(text); return { samples: Int16Array.of(spoken.length), sampleRate: 16000 }; } }, { play: async (samples) => { played.push(samples[0]); await gates[played.length - 1].promise; }, stop() {} }); const task = loop.speak('First sentence. Second sentence. Third sentence.'); await drainMicrotasks(); assert.equal(spoken.length, 2); assert.deepEqual(played, [1]); assert.equal(loop.isSpeaking, true); gates[0].resolve(); await drainMicrotasks(); assert.equal(spoken.length, 3); assert.deepEqual(played, [1, 2]); assert.equal(loop.isSpeaking, true); gates[1].resolve(); await drainMicrotasks(); assert.deepEqual(played, [1, 2, 3]); gates[2].resolve(); await task; assert.equal(loop.isSpeaking, false); assert.equal(loop.metrics.snapshot().synthesisCount, 3); }); test('interrupt discards prefetched audio and drains synthesis before speech completes', async () => { const prefetch = deferred(), playing = deferred(); let synthesized = 0, played = 0, finished = false; const loop = pipelineLoop({ speak: async () => { if (++synthesized === 2) await prefetch.promise; return { samples: Int16Array.of(synthesized), sampleRate: 16000 }; } }, { play: async () => { played++; await playing.promise; }, stop() { playing.resolve(); } }); const task = loop.speak('First sentence. Second sentence. Third sentence.').then(() => { finished = true; }); await drainMicrotasks(); loop.interrupt(); await drainMicrotasks(); assert.equal(finished, false); prefetch.resolve(); await task; assert.equal(played, 1); assert.equal(synthesized, 2); assert.equal(loop.isSpeaking, false); }); for (const failure of ['synthesis', 'playback']) { test(`speech pipeline handles ${failure} failure and accepts the next reply`, async () => { let synthesized = 0; const loop = pipelineLoop({ speak: async () => { if (++synthesized === 2 && failure === 'synthesis') throw new Error('synthesis failed'); return { samples: Int16Array.of(synthesized), sampleRate: 16000 }; } }, { play: async () => { if (failure === 'playback') throw new Error('playback failed'); }, stop() {} }); await assert.rejects(loop.speak('First sentence. Second sentence.'), /failed/); assert.equal(loop.isSpeaking, false); loop.playback.play = async () => {}; await loop.speak('Recovery sentence.'); assert.equal(loop.isSpeaking, false); }); }