364 lines
16 KiB
JavaScript
364 lines
16 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { readFileSync } from 'node:fs';
|
|
import { EventEmitter } from 'node:events';
|
|
import { VoiceLoop, fastCommand } from '../daemon/voice-loop.js';
|
|
import { pcmS16le } from '../daemon/voice-adapters.js';
|
|
import { WakeEngine } from '../daemon/wake-engine.js';
|
|
import { VadSegmenter } from '../daemon/vad.js';
|
|
import { PipeWireCapture, pcmRms } from '../daemon/audio-pipewire.js';
|
|
import { SentenceBuffer, isMeaningfulTranscript, isSpeakable, speakableForTts, stripMarkdownForSpeech, forChatDisplay } from '../daemon/transcript.js';
|
|
|
|
test('Phase 4 transcript filtering and sentence buffering are deterministic', () => {
|
|
assert.equal(isMeaningfulTranscript('[BLANK_AUDIO]'), false);
|
|
assert.equal(isMeaningfulTranscript('hi'), false);
|
|
assert.equal(isSpeakable('OK.'), true);
|
|
assert.equal(isSpeakable('[object Object]'), false);
|
|
assert.equal(speakableForTts('Using `run_terminal_cmd`'), 'Using run terminal cmd');
|
|
assert.equal(speakableForTts('Your IP is 192.168.0.1'), 'Your I P is 192 168 0 1');
|
|
assert.equal(
|
|
speakableForTts('QVAC fetched https://example.com/ip'),
|
|
'Quantum Verse Automatic Computer fetched example dot com slash I P'
|
|
);
|
|
assert.equal(
|
|
speakableForTts('## Hello\n- First item\n- **Second** item\nSee [docs](https://example.com).'),
|
|
'Hello First item Second item See docs.'
|
|
);
|
|
assert.equal(forChatDisplay('Greetings, **young rebel**.\nHow may I serve you?'), 'Greetings, young rebel.\nHow may I serve you?');
|
|
assert.equal(forChatDisplay('Greetings, **Raven**. Next line.'), 'Greetings, Raven. Next line.');
|
|
assert.equal(forChatDisplay('Use 2 * 3, not **four**.'), 'Use 2 * 3, not four.');
|
|
assert.equal(stripMarkdownForSpeech('Greetings, **young rebel**.\nHow may I serve you?'), 'Greetings, young rebel. How may I serve you?');
|
|
assert.equal(isMeaningfulTranscript('what time is it'), true);
|
|
const out = []; const buffer = new SentenceBuffer({ onSentence: (s) => out.push(s) });
|
|
buffer.push('First sentence. Second'); buffer.push(' sentence!'); buffer.flush();
|
|
assert.deepEqual(out, ['First sentence.', 'Second sentence!']);
|
|
});
|
|
|
|
test('Phase 4 wake engine emits only configured local detections', () => {
|
|
const wake = new WakeEngine({ phrases: ['hey jarvis'], detect: () => ({ phrase: 'hey jarvis' }) });
|
|
const heard = []; wake.on('wake', (phrase) => heard.push(phrase)); wake.push(Buffer.from([0]));
|
|
assert.deepEqual(heard, ['hey jarvis']);
|
|
wake.pause(); wake.push(Buffer.from([0])); assert.equal(heard.length, 1);
|
|
});
|
|
|
|
test('Phase 4 VAD emits a bounded utterance after silence', () => {
|
|
const vad = new VadSegmenter({ frameMs: 100, params: { threshold: 0.6, minSpeechDurationMs: 100, minSilenceDurationMs: 200 } });
|
|
const utterances = []; vad.on('utterance', (audio) => utterances.push(audio));
|
|
const loud = Buffer.alloc(3200); for (let i = 0; i < loud.length; i += 2) loud.writeInt16LE(20_000, i); const quiet = Buffer.alloc(3200);
|
|
vad.push(loud); vad.push(quiet); vad.push(quiet);
|
|
assert.equal(utterances.length, 1); assert.ok(utterances[0].length > 0); assert.ok(pcmRms(loud) > 0);
|
|
});
|
|
|
|
test('Phase 4 fast commands are handled before the harness', () => {
|
|
assert.equal(fastCommand('Hands Off'), 'cancel');
|
|
assert.equal(fastCommand('take the wheel'), 'computer');
|
|
assert.equal(fastCommand('use the camera'), 'camera');
|
|
assert.equal(fastCommand('use my webcam'), 'camera');
|
|
assert.equal(fastCommand('ordinary question'), null);
|
|
});
|
|
|
|
test('Phase 4 feedback gate drops capture while speaking and during cooldown', () => {
|
|
const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {};
|
|
const wake = new WakeEngine({ detect: () => ({ phrase: 'hey jarvis' }) });
|
|
const vad = new VadSegmenter({ frameMs: 20 }); let pushed = 0; const original = vad.push.bind(vad); vad.push = (x) => { pushed++; original(x); };
|
|
const daemon = new EventEmitter(); daemon.state = 'LISTENING';
|
|
const loop = new VoiceLoop({ daemon, capture, wake, vad, now: () => 1000 }); loop.running = true;
|
|
loop.isSpeaking = true; loop.pushAudio(Buffer.alloc(3200)); assert.equal(pushed, 0);
|
|
loop.isSpeaking = false; loop.cooldownUntil = 1200; loop.pushAudio(Buffer.alloc(3200)); assert.equal(pushed, 0);
|
|
});
|
|
|
|
test('Phase 4 PipeWire capture uses a named 16 kHz mono node', () => {
|
|
let args; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); child.kill = () => {};
|
|
const capture = new PipeWireCapture({ spawnImpl: (_cmd, received) => { args = received; return child; } }); capture.start();
|
|
assert.deepEqual(args, ['--record', '--raw', '--format', 's16', '--rate', '16000', '--channels', '1', '--properties', 'node.name=Jarvis', '-']);
|
|
capture.stop();
|
|
});
|
|
|
|
test('empty TTS still leaves the daemon listening', async () => {
|
|
const daemon = new EventEmitter();
|
|
daemon.state = 'SPEAKING';
|
|
daemon.voice = { finishSpeaking() { daemon.finished = true; } };
|
|
daemon.setState = (state) => { daemon.state = state; };
|
|
const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {};
|
|
const wake = new WakeEngine({ detect: () => null });
|
|
const loop = new VoiceLoop({ daemon, capture, wake, vad: new VadSegmenter(), tts: { speak: async () => ({ samples: new Int16Array(0) }) } });
|
|
await loop.speak('[object Object]');
|
|
assert.equal(daemon.state, 'LISTENING');
|
|
assert.equal(daemon.finished, true);
|
|
});
|
|
|
|
test('multi-sentence speech returns to listening once', async () => {
|
|
const daemon = new EventEmitter();
|
|
const states = [];
|
|
daemon.state = 'SPEAKING';
|
|
daemon.voice = {
|
|
finishSpeaking() {
|
|
daemon.finished = (daemon.finished || 0) + 1;
|
|
},
|
|
};
|
|
daemon.setState = (state) => { daemon.state = state; states.push(state); };
|
|
const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {};
|
|
const wake = new WakeEngine({ detect: () => null });
|
|
const loop = new VoiceLoop({
|
|
daemon,
|
|
capture,
|
|
wake,
|
|
vad: new VadSegmenter(),
|
|
tts: { speak: async () => ({ samples: new Int16Array(2) }) },
|
|
playback: { play: async () => {}, stop() {} },
|
|
});
|
|
loop.status.tts = true;
|
|
await loop.speak('Hello there. How are you today?');
|
|
assert.equal(daemon.state, 'LISTENING');
|
|
assert.equal(daemon.finished, 1);
|
|
assert.deepEqual(states, ['LISTENING']);
|
|
});
|
|
|
|
test('PCM packing uses even s16le sample pairs', () => {
|
|
const buf = Buffer.alloc(4);
|
|
buf.writeInt16LE(256, 0);
|
|
buf.writeInt16LE(-2, 2);
|
|
const packed = pcmS16le(buf);
|
|
assert.equal(packed.samples.length, 2);
|
|
assert.equal(packed.samples[0], 256);
|
|
assert.equal(packed.samples[1], -2);
|
|
const fromBytes = pcmS16le(Uint8Array.from([0, 1, 0xfe, 0xff]));
|
|
assert.equal(fromBytes.samples.length, 2);
|
|
assert.equal(fromBytes.samples[0], 256);
|
|
});
|
|
|
|
test('ASR failure keeps the microphone hot and still allows speech output', async () => {
|
|
const capture = new EventEmitter(); let captured = false;
|
|
capture.start = () => { captured = true; }; capture.stop = () => {};
|
|
let spoken = '';
|
|
const loop = new VoiceLoop({ capture, wake: new WakeEngine(),
|
|
asr: { start: async () => { throw new Error('Whisper plugin missing'); } },
|
|
tts: { start: async () => {}, speak: async (text) => { spoken = text; return { samples: new Int16Array(8) }; } },
|
|
playback: { play: async () => {}, stop() {} },
|
|
});
|
|
loop.on('error', () => {});
|
|
await loop.start();
|
|
assert.equal(captured, true);
|
|
assert.equal(loop.status.asr, false);
|
|
assert.equal(loop.status.tts, false);
|
|
await loop.speak('Speech still works.');
|
|
assert.equal(spoken, 'Speech still works.');
|
|
assert.equal(loop.status.tts, true);
|
|
assert.equal(await loop.ensureAsr(), false);
|
|
assert.match(loop.status.errors.asr, /Whisper/); await loop.stop();
|
|
});
|
|
|
|
test('TTS failure preserves ASR and push to talk without wake command', async () => {
|
|
const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {};
|
|
const loop = new VoiceLoop({ capture, wake: new WakeEngine(),
|
|
asr: { start: async () => {} }, tts: { start: async () => { throw new Error('TTS failed'); } },
|
|
playback: { stop() {} },
|
|
});
|
|
loop.on('error', () => {}); await loop.start();
|
|
assert.equal(loop.status.asr, false);
|
|
await loop.setPushToTalk(true);
|
|
assert.equal(loop.status.asr, true); assert.equal(loop.ptt, true);
|
|
assert.equal(await loop.ensureTts(), false);
|
|
assert.equal(loop.status.tts, false); assert.equal(loop.status.wake, false); await loop.stop();
|
|
});
|
|
|
|
test('QVAC numeric PCM arrays preserve signed 16-bit samples', () => {
|
|
assert.deepEqual([...pcmS16le([256, -32768, 32767]).samples], [256, -32768, 32767]);
|
|
});
|
|
|
|
test('voice loop retries microphone capture after PipeWire comes up', async () => {
|
|
let starts = 0;
|
|
const capture = new EventEmitter();
|
|
capture.start = () => {
|
|
starts += 1;
|
|
if (starts === 1) queueMicrotask(() => capture.emit('close', { code: 1 }));
|
|
};
|
|
capture.stop = () => {};
|
|
const loop = new VoiceLoop({
|
|
capture,
|
|
asr: { start: async () => {} },
|
|
tts: { start: async () => {} },
|
|
wake: new WakeEngine({ detect: () => null }),
|
|
vad: new VadSegmenter(),
|
|
captureRetryMs: 20,
|
|
});
|
|
loop.on('error', () => {});
|
|
await loop.start();
|
|
assert.equal(loop.status.asr, false);
|
|
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
assert.equal(loop.status.capture, false);
|
|
await new Promise((resolve) => setTimeout(resolve, 40));
|
|
assert.ok(starts >= 2);
|
|
assert.equal(loop.status.capture, true);
|
|
await loop.stop();
|
|
});
|
|
|
|
test('voice loop retries ASR after a first-login GPU miss', async () => {
|
|
let attempts = 0;
|
|
const capture = new EventEmitter();
|
|
capture.start = () => { capture.started = true; };
|
|
capture.stop = () => {};
|
|
const loop = new VoiceLoop({
|
|
capture,
|
|
asr: { start: async () => { attempts += 1; if (attempts === 1) throw new Error('GPU not ready'); } },
|
|
tts: { start: async () => {} },
|
|
wake: new WakeEngine({ detect: () => null }),
|
|
vad: new VadSegmenter(),
|
|
asrRetryMs: 20,
|
|
});
|
|
loop.on('error', () => {});
|
|
await loop.start();
|
|
assert.equal(loop.status.asr, false);
|
|
assert.equal(capture.started, true);
|
|
assert.equal(attempts, 0);
|
|
assert.equal(await loop.ensureAsr(), false);
|
|
assert.equal(attempts, 1);
|
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
assert.equal(attempts, 2);
|
|
assert.equal(loop.status.asr, true);
|
|
assert.equal(capture.started, true);
|
|
await loop.stop();
|
|
});
|
|
|
|
test('voice adapters load canonical ASR and TTS plugins', () => {
|
|
const source = readFileSync(new URL('../daemon/voice-adapters.js', import.meta.url), 'utf8');
|
|
assert.match(source, /whispercpp-transcription/);
|
|
assert.match(source, /tts-ggml/);
|
|
});
|
|
|
|
function tonePcm(ms, amplitude = 12_000, hz = 220) {
|
|
const samples = Math.floor(16_000 * ms / 1000);
|
|
const buf = Buffer.alloc(samples * 2);
|
|
for (let i = 0; i < samples; i++) buf.writeInt16LE(Math.floor(amplitude * Math.sin(2 * Math.PI * hz * i / 16_000)), i * 2);
|
|
return buf;
|
|
}
|
|
|
|
test('built-in CPU wake detector fires on a two-peak hey-jarvis cadence', async () => {
|
|
const { KeywordWakeEngine, createWakeEngine } = await import('../daemon/wake-engine.js');
|
|
const engine = createWakeEngine({ command: 'jarvis-wake-bridge', phrases: ['hey jarvis'] });
|
|
assert.equal(engine instanceof KeywordWakeEngine, true);
|
|
const heard = [];
|
|
engine.on('wake', (phrase) => heard.push(phrase));
|
|
engine.push(Buffer.concat([tonePcm(80, 0), tonePcm(220), tonePcm(90, 0), tonePcm(280), tonePcm(200, 0)]));
|
|
assert.deepEqual(heard, ['hey jarvis']);
|
|
engine.push(tonePcm(800));
|
|
assert.equal(heard.length, 1);
|
|
});
|
|
|
|
test('sleeping voice loop still feeds the wake detector and skips VAD', () => {
|
|
const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {};
|
|
let wakeFrames = 0; let vadFrames = 0;
|
|
const wake = new WakeEngine({ detect: () => null });
|
|
wake.push = () => { wakeFrames += 1; };
|
|
const vad = new VadSegmenter();
|
|
vad.push = () => { vadFrames += 1; };
|
|
const daemon = new EventEmitter(); daemon.state = 'SLEEPING';
|
|
const loop = new VoiceLoop({ daemon, capture, wake, vad });
|
|
loop.running = true;
|
|
loop.pushAudio(Buffer.alloc(640));
|
|
assert.equal(wakeFrames, 1);
|
|
assert.equal(vadFrames, 0);
|
|
});
|
|
|
|
test('voice start keeps the microphone hot without loading ASR or TTS', async () => {
|
|
const capture = new EventEmitter(); let captured = false;
|
|
capture.start = () => { captured = true; }; capture.stop = () => {};
|
|
let asrStarts = 0; let ttsStarts = 0;
|
|
const loop = new VoiceLoop({
|
|
capture,
|
|
asr: { start: async () => { asrStarts += 1; } },
|
|
tts: { start: async () => { ttsStarts += 1; } },
|
|
wake: new WakeEngine({ detect: () => null }),
|
|
});
|
|
await loop.start();
|
|
assert.equal(captured, true);
|
|
assert.equal(loop.status.capture, true);
|
|
assert.equal(loop.status.asr, false);
|
|
assert.equal(loop.status.tts, false);
|
|
assert.equal(asrStarts, 0);
|
|
assert.equal(ttsStarts, 0);
|
|
assert.equal(loop.status.wake, true);
|
|
await loop.parkModels();
|
|
assert.equal(loop.status.asr, false);
|
|
assert.equal(loop.status.tts, false);
|
|
await loop.stop();
|
|
});
|
|
|
|
function deferred() {
|
|
let resolve, reject;
|
|
const promise = new Promise((yes, no) => { resolve = yes; reject = no; });
|
|
return { promise, resolve, reject };
|
|
}
|
|
|
|
function pipelineLoop(tts, playback) {
|
|
const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {};
|
|
const loop = new VoiceLoop({ capture, wake: new WakeEngine({ detect: () => null }), tts, playback });
|
|
loop.status.tts = true;
|
|
return loop;
|
|
}
|
|
|
|
const drainMicrotasks = async () => { for (let i = 0; i < 20; i++) await Promise.resolve(); };
|
|
|
|
test('speech prepares only the next sentence during playback, preserving order and feedback gating', async () => {
|
|
const spoken = [], played = [];
|
|
const gates = [deferred(), deferred(), deferred()];
|
|
const loop = pipelineLoop({ speak: async (text) => {
|
|
spoken.push(text);
|
|
return { samples: Int16Array.of(spoken.length), sampleRate: 16000 };
|
|
} }, { play: async (samples) => {
|
|
played.push(samples[0]);
|
|
await gates[played.length - 1].promise;
|
|
}, stop() {} });
|
|
const task = loop.speak('First sentence. Second sentence. Third sentence.');
|
|
await drainMicrotasks();
|
|
assert.equal(spoken.length, 2);
|
|
assert.deepEqual(played, [1]);
|
|
assert.equal(loop.isSpeaking, true);
|
|
gates[0].resolve();
|
|
await drainMicrotasks();
|
|
assert.equal(spoken.length, 3);
|
|
assert.deepEqual(played, [1, 2]);
|
|
assert.equal(loop.isSpeaking, true);
|
|
gates[1].resolve();
|
|
await drainMicrotasks();
|
|
assert.deepEqual(played, [1, 2, 3]);
|
|
gates[2].resolve();
|
|
await task;
|
|
assert.equal(loop.isSpeaking, false);
|
|
assert.equal(loop.metrics.snapshot().synthesisCount, 3);
|
|
});
|
|
|
|
test('interrupt discards prefetched audio and drains synthesis before speech completes', async () => {
|
|
const prefetch = deferred(), playing = deferred();
|
|
let synthesized = 0, played = 0, finished = false;
|
|
const loop = pipelineLoop({ speak: async () => {
|
|
if (++synthesized === 2) await prefetch.promise;
|
|
return { samples: Int16Array.of(synthesized), sampleRate: 16000 };
|
|
} }, { play: async () => { played++; await playing.promise; }, stop() { playing.resolve(); } });
|
|
const task = loop.speak('First sentence. Second sentence. Third sentence.').then(() => { finished = true; });
|
|
await drainMicrotasks();
|
|
loop.interrupt();
|
|
await drainMicrotasks();
|
|
assert.equal(finished, false);
|
|
prefetch.resolve();
|
|
await task;
|
|
assert.equal(played, 1);
|
|
assert.equal(synthesized, 2);
|
|
assert.equal(loop.isSpeaking, false);
|
|
});
|
|
|
|
for (const failure of ['synthesis', 'playback']) {
|
|
test(`speech pipeline handles ${failure} failure and accepts the next reply`, async () => {
|
|
let synthesized = 0;
|
|
const loop = pipelineLoop({ speak: async () => {
|
|
if (++synthesized === 2 && failure === 'synthesis') throw new Error('synthesis failed');
|
|
return { samples: Int16Array.of(synthesized), sampleRate: 16000 };
|
|
} }, { play: async () => { if (failure === 'playback') throw new Error('playback failed'); }, stop() {} });
|
|
await assert.rejects(loop.speak('First sentence. Second sentence.'), /failed/);
|
|
assert.equal(loop.isSpeaking, false);
|
|
loop.playback.play = async () => {};
|
|
await loop.speak('Recovery sentence.');
|
|
assert.equal(loop.isSpeaking, false);
|
|
});
|
|
}
|