diff --git a/daemon/audio-pipewire.js b/daemon/audio-pipewire.js new file mode 100644 index 0000000..d2d1052 --- /dev/null +++ b/daemon/audio-pipewire.js @@ -0,0 +1,49 @@ +import { spawn } from 'node:child_process'; +import { EventEmitter } from 'node:events'; + +export const MIC_SAMPLE_RATE = 16_000; +export const MIC_CHANNELS = 1; +export const MIC_FORMAT = 's16'; + +/** Raw 16 kHz mono capture from PipeWire. The process is deliberately kept + * outside gnome-shell and has a stable node name for routing in Helvum. */ +export class PipeWireCapture extends EventEmitter { + constructor({ command = 'pw-cat', spawnImpl = spawn, sampleRate = MIC_SAMPLE_RATE, nodeName = 'Jarvis' } = {}) { + super(); + this.command = command; + this.spawnImpl = spawnImpl; + this.sampleRate = sampleRate; + this.nodeName = nodeName; + this.process = null; + } + + start() { + if (this.process) return this; + this.process = this.spawnImpl(this.command, [ + '--record', '--raw', '--format', MIC_FORMAT, '--rate', String(this.sampleRate), + '--channels', String(MIC_CHANNELS), '--name', this.nodeName, + ], { stdio: ['ignore', 'pipe', 'pipe'] }); + this.process.stdout?.on('data', (chunk) => this.emit('audio', Buffer.from(chunk))); + this.process.stderr?.on('data', (chunk) => this.emit('diagnostic', String(chunk).trim())); + this.process.on('error', (error) => this.emit('error', error)); + this.process.on('close', (code, signal) => { this.process = null; this.emit('close', { code, signal }); }); + return this; + } + + stop() { + if (!this.process) return; + this.process.kill('SIGTERM'); + this.process = null; + } +} + +export function pcmRms(chunk) { + const bytes = Buffer.from(chunk || ''); + if (bytes.length < 2) return 0; + let sum = 0; + for (let i = 0; i + 1 < bytes.length; i += 2) { + const sample = bytes.readInt16LE(i) / 32768; + sum += sample * sample; + } + return Math.sqrt(sum / Math.floor(bytes.length / 2)); +} diff --git a/daemon/audio-playback.js b/daemon/audio-playback.js new file mode 100644 index 0000000..c624b98 --- /dev/null +++ b/daemon/audio-playback.js @@ -0,0 +1,18 @@ +import { spawn } from 'node:child_process'; +import { EventEmitter } from 'node:events'; + +export class PipeWirePlayback extends EventEmitter { + constructor({ command = 'pw-cat', spawnImpl = spawn, sampleRate = 44_100, nodeName = 'Jarvis' } = {}) { + super(); this.command = command; this.spawnImpl = spawnImpl; this.sampleRate = sampleRate; this.nodeName = nodeName; this.process = null; + } + async play(samples) { + this.stop(); + const child = this.process = this.spawnImpl(this.command, ['--playback', '--raw', '--format', 's16', '--rate', String(this.sampleRate), '--channels', '1', '--name', this.nodeName], { stdio: ['pipe', 'ignore', 'pipe'] }); + child.stderr?.on('data', (chunk) => this.emit('diagnostic', String(chunk).trim())); + child.on('error', (error) => this.emit('error', error)); + child.stdin.end(Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength)); + await new Promise((resolve, reject) => { child.once('close', resolve); child.once('error', reject); }); + if (this.process === child) this.process = null; + } + stop() { if (this.process) { this.process.kill('SIGTERM'); this.process = null; } } +} diff --git a/daemon/dbus-service.js b/daemon/dbus-service.js index 2c88d59..1dce74b 100644 --- a/daemon/dbus-service.js +++ b/daemon/dbus-service.js @@ -10,7 +10,7 @@ export async function serveOnSessionBus(daemon) { Arm() { daemon.arm(); } Sleep() { daemon.sleep(); } Shutdown() { daemon.close(); } - PushToTalk(pressed) { daemon.emit('PushToTalk', Boolean(pressed)); } + PushToTalk(pressed) { daemon.setPushToTalk?.(Boolean(pressed)); } Say(text) { return daemon.say?.(text); } Ask(text) { return daemon.ask(text); } Cancel() { daemon.cancel(); } diff --git a/daemon/index.js b/daemon/index.js index 3c9c1a6..a279639 100644 --- a/daemon/index.js +++ b/daemon/index.js @@ -5,6 +5,9 @@ import { VoiceStateMachine } from './voice-state.js'; import { QvacScheduler } from './qvac-scheduler.js'; import { cancelQvac, resumeQvac, suspendQvac } from './qvac-master.js'; import { PrivacyLog } from './privacy-log.js'; +import { VoiceLoop } from './voice-loop.js'; +import { QvacVoiceAdapter } from './voice-adapters.js'; +import { createWakeEngine } from './wake-engine.js'; export class JarvisDaemon extends EventEmitter { constructor() { @@ -16,6 +19,8 @@ export class JarvisDaemon extends EventEmitter { this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer }); this.log = new PrivacyLog(); this.locked = false; + this.lastReply = ''; + this.voiceLoop = null; this._idleTimer = setInterval(() => this.tickIdle(), 30_000); this._idleTimer.unref?.(); this.harness.on('agent_message_chunk', (ev) => this.emit('Token', ev?.text || ev?.delta || '')); @@ -26,12 +31,12 @@ export class JarvisDaemon extends EventEmitter { setState(state) { this.state = state; this.emit('StateChanged', state); this.log.record('state', { state }).catch(() => {}); } async arm() { if (this.locked) return; await resumeQvac().catch(() => {}); this.voice.wake(); this.setState('LISTENING'); } async sleep() { this.voice.sleep(); this.setState('SLEEPING'); await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); } - say(text) { this.emit('Reply', String(text)); } + say(text) { this.lastReply = String(text); this.emit('Reply', this.lastReply); } async ask(text) { this.voice.utterance(); this.setState('THINKING'); try { const reply = await this.scheduler.run(() => this.harness.ask(text), { lane: 'voice' }); - this.voice.speak(); this.setState('SPEAKING'); this.emit('Reply', reply); return reply; + this.voice.speak(); this.setState('SPEAKING'); this.lastReply = String(reply || ''); this.emit('Reply', this.lastReply); this.voiceLoop?.speak(this.lastReply).catch((error) => this.emit('Error', 'TTS', error.message)); return reply; } catch (error) { this.emit('Error', 'QVAC', error.message); this.voice.cancel(); this.setState('ARMED'); throw error; } @@ -39,13 +44,21 @@ export class JarvisDaemon extends EventEmitter { cancel() { this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computer.revoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); } computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); return result; } computerRevoke() { this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); } + async startVoice() { + if (this.voiceLoop) return; + const voiceIO = new QvacVoiceAdapter(); + this.voiceLoop = new VoiceLoop({ daemon: this, wake: createWakeEngine(), asr: voiceIO, tts: voiceIO }); + try { await this.voiceLoop.start(); } catch (error) { this.voiceLoop = null; this.emit('Error', 'VOICE_UNAVAILABLE', error.message); throw error; } + } + setPushToTalk(pressed) { this.voiceLoop?.setPushToTalk(pressed); this.emit('PushToTalk', Boolean(pressed)); } handleLockScreen(locked) { this.locked = Boolean(locked); if (this.locked) { this.cancel(); this.setState('ARMED'); } this.emit('LockScreenChanged', this.locked); } tickIdle() { if (!this.locked && this.voice.expireIdle() === 'SLEEPING' && this.state !== 'SLEEPING') this.sleep(); } - async close() { clearInterval(this._idleTimer); this.computerRevoke(); await this.harness.close(); } + async close() { clearInterval(this._idleTimer); this.computerRevoke(); await this.voiceLoop?.stop?.(); await this.harness.close(); } } if (import.meta.url === `file://${process.argv[1]}`) { const daemon = new JarvisDaemon(); + daemon.startVoice().catch((error) => console.error(`jarvisd: voice unavailable: ${error.message}`)); import('./dbus-service.js').then(({ serveOnSessionBus }) => serveOnSessionBus(daemon)).catch((error) => { daemon.emit('Error', 'DBUS_UNAVAILABLE', error.message); console.error(`jarvisd: D-Bus unavailable: ${error.message}`); diff --git a/daemon/qvac-master.js b/daemon/qvac-master.js index 18f2d0f..29eb0d1 100644 --- a/daemon/qvac-master.js +++ b/daemon/qvac-master.js @@ -8,6 +8,8 @@ const Agent = require(path.join(harnessPath, 'index.js')); let loadPromise = null; let ownerCount = 0; +let operationTail = Promise.resolve(); +const auxiliaryModels = new Map(); export const QVAC_MASTER = Object.freeze({ configPath: process.env.QVAC_CONFIG_PATH, @@ -55,10 +57,58 @@ export async function acquireQvac() { return loadPromise; } +export async function qvacSdk() { + return Agent.engine.ensureInit(); +} + +export function withQvacMaster(task) { + const operation = operationTail.then(task, task); + operationTail = operation.catch(() => {}); + return operation; +} + +function resolveSdkAsset(sdk, name) { + if (name && typeof name !== 'string') return name; + return sdk[name] || sdk.models?.[name] || name; +} + +function resolveModelConfigAssets(sdk, config) { + const copy = { ...config }; + for (const key of ['vadModelSrc', 'projectionModelSrc', 'vocabModelSrc']) { + if (typeof copy[key] === 'string') copy[key] = resolveSdkAsset(sdk, copy[key]); + } + return copy; +} + +/** Load ASR/TTS models in the same SDK worker and under the same master lock. + * These are auxiliary model IDs; they do not create another QVAC runtime. */ +export async function loadAuxiliaryModel(name, modelConfig = {}) { + if (!name) throw new Error('auxiliary QVAC model name is required'); + const existing = auxiliaryModels.get(String(name)); + if (existing) return existing; + const sdk = await qvacSdk(); + if (typeof sdk.loadModel !== 'function') throw new Error('QVAC SDK does not expose loadModel()'); + const modelId = await withQvacMaster(() => sdk.loadModel({ + modelSrc: resolveSdkAsset(sdk, name), + modelConfig: { ...resolveModelConfigAssets(sdk, modelConfig), device: 'gpu', gpu_layers: QVAC_MASTER.gpuLayers, 'mmproj-use-gpu': true }, + })); + auxiliaryModels.set(String(name), modelId); + return modelId; +} + +export async function unloadAuxiliaryModel(modelId) { + if (!modelId) return; + const sdk = await qvacSdk(); + if (typeof sdk.unloadModel === 'function') await withQvacMaster(() => sdk.unloadModel({ modelId })); + for (const [name, id] of auxiliaryModels) if (id === modelId) auxiliaryModels.delete(name); +} + export function releaseQvac() { ownerCount = Math.max(0, ownerCount - 1); } export async function closeQvac() { if (ownerCount > 0) return; + for (const modelId of auxiliaryModels.values()) await unloadAuxiliaryModel(modelId).catch(() => {}); + auxiliaryModels.clear(); loadPromise = null; await Agent.engine.close(); } @@ -101,7 +151,7 @@ export async function callQvac(method, input) { } export function qvacStatus() { - return { ...QVAC_MASTER, owners: ownerCount, loaded: Agent.engine.getLoaded() }; + return { ...QVAC_MASTER, owners: ownerCount, loaded: Agent.engine.getLoaded(), auxiliaryModels: auxiliaryModels.size }; } export { Agent }; diff --git a/daemon/transcript.js b/daemon/transcript.js new file mode 100644 index 0000000..fd75d09 --- /dev/null +++ b/daemon/transcript.js @@ -0,0 +1,24 @@ +export const MIN_UTTERANCE_CHARS = 3; + +export function isMeaningfulTranscript(text) { + const value = String(text || '').trim(); + if (!value || /\[no speech detected\]|\[blank_audio\]/i.test(value)) return false; + if (/^\[[^\]]+\]$/.test(value)) return false; + return value.replace(/[^\p{L}\p{N}]/gu, '').length >= MIN_UTTERANCE_CHARS; +} + +export class SentenceBuffer { + constructor({ onSentence } = {}) { this.onSentence = onSentence; this.pending = ''; } + push(text) { + this.pending += String(text || ''); + const sentences = []; + let match; + while ((match = this.pending.match(/^(.+?[.!?](?:["')\]]+)?)(?:\s+|$)/s))) { + sentences.push(match[1].trim()); this.pending = this.pending.slice(match[0].length); + } + for (const sentence of sentences) this.onSentence?.(sentence); + return sentences; + } + flush() { const value = this.pending.trim(); this.pending = ''; if (value) this.onSentence?.(value); return value; } + clear() { this.pending = ''; } +} diff --git a/daemon/vad.js b/daemon/vad.js new file mode 100644 index 0000000..98d6c56 --- /dev/null +++ b/daemon/vad.js @@ -0,0 +1,51 @@ +import { EventEmitter } from 'node:events'; +import { pcmRms } from './audio-pipewire.js'; + +export const DEFAULT_VAD = Object.freeze({ + threshold: 0.6, + minSpeechDurationMs: 300, + minSilenceDurationMs: 700, + maxSpeechDurationMs: 15_000, + speechPadMs: 200, +}); + +/** A conservative local gate around QVAC's stream. It bounds audio sent to + * ASR and gives the loop deterministic utterance boundaries in tests. */ +export class VadSegmenter extends EventEmitter { + constructor({ sampleRate = 16_000, frameMs = 20, params = {} } = {}) { + super(); + this.sampleRate = sampleRate; + this.frameMs = frameMs; + this.params = { ...DEFAULT_VAD, ...params }; + this.reset(); + } + + reset() { this.speaking = false; this.buffer = []; this.speechMs = 0; this.silenceMs = 0; } + + push(frame) { + const chunk = Buffer.from(frame || ''); + const rms = pcmRms(chunk); + const voiced = rms >= this.params.threshold / 10; // PCM RMS is 0..1; QVAC threshold is posterior-like. + this.emit('level', rms); + if (!this.speaking && voiced) { + this.speaking = true; this.speechMs = 0; this.silenceMs = 0; this.buffer = []; + this.emit('speechStart'); + } + if (!this.speaking) return; + this.buffer.push(chunk); + if (voiced) { this.speechMs += this.frameMs; this.silenceMs = 0; } + else { this.silenceMs += this.frameMs; } + if (this.speechMs >= this.params.maxSpeechDurationMs || + (this.speechMs >= this.params.minSpeechDurationMs && this.silenceMs >= this.params.minSilenceDurationMs)) { + this.end(); + } + } + + end() { + if (!this.speaking) return null; + const audio = Buffer.concat(this.buffer); + this.reset(); + this.emit('utterance', audio); + return audio; + } +} diff --git a/daemon/voice-adapters.js b/daemon/voice-adapters.js new file mode 100644 index 0000000..cb7c0b7 --- /dev/null +++ b/daemon/voice-adapters.js @@ -0,0 +1,56 @@ +import { createRequire } from 'node:module'; +import { EventEmitter } from 'node:events'; +import path from 'node:path'; +import { acquireQvac, releaseQvac, loadAuxiliaryModel, unloadAuxiliaryModel, qvacSdk, withQvacMaster } from './qvac-master.js'; + +const require = createRequire(import.meta.url); +const harnessPath = process.env.JARVIS_HARNESS_PATH || path.resolve(new URL('../vendor/agent-harness', import.meta.url).pathname); +const sdkPackage = require(path.join(harnessPath, 'node_modules/@qvac/sdk/package.json')); + +export class QvacVoiceAdapter extends EventEmitter { + constructor({ asrModel = process.env.JARVIS_ASR_MODEL || 'WHISPER_TINY', ttsModel = process.env.JARVIS_TTS_MODEL || 'TTS_EN_SUPERTONIC_Q8_0' } = {}) { + super(); this.asrModel = asrModel; this.ttsModel = ttsModel; this.asrId = null; this.ttsId = null; this.asrSession = null; this.acquired = false; + } + + async start() { + if (this.acquired) return; + if (!/^0\.19\./.test(sdkPackage.version)) throw new Error(`Jarvis voice adapter requires QVAC 0.19.x; found ${sdkPackage.version}`); + await acquireQvac(); this.acquired = true; + try { + this.asrId = await loadAuxiliaryModel(this.asrModel, { vadModelSrc: 'VAD_SILERO_5_1_2', audio_format: 's16le', language: 'en', no_timestamps: true, vad_params: { threshold: 0.6, min_speech_duration_ms: 300, min_silence_duration_ms: 700, max_speech_duration_s: 15, speech_pad_ms: 200 } }); + this.ttsId = await loadAuxiliaryModel(this.ttsModel, { ttsEngine: 'supertonic', language: 'en', voice: 'F1', ttsSpeed: 1.05, ttsNumInferenceSteps: 5 }); + } catch (error) { await this.stop(); throw error; } + } + + writeAudio(chunk) { this.asrSession?.write(Buffer.from(chunk)); } + async transcribeAudio(audio) { + const sdk = await qvacSdk(); + return withQvacMaster(async () => { + const session = await sdk.transcribeStream({ modelId: this.asrId, emitVadEvents: true }); + session.write(Buffer.from(audio)); session.end(); + const parts = []; + for await (const event of session) parts.push(typeof event === 'string' ? event : event?.text || ''); + return parts.join(' ').trim(); + }); + } + async *transcripts() { if (!this.asrSession) throw new Error('voice adapter is not started'); yield* this.asrSession; } + endAudio() { this.asrSession?.end(); } + + async speak(text) { + const sdk = await qvacSdk(); + const samples = await withQvacMaster(async () => { + const result = sdk.textToSpeech({ modelId: this.ttsId, text: String(text), inputType: 'text', stream: false }); + return result.buffer; + }); + return { samples: Int16Array.from(samples), sampleRate: 44_100 }; + } + + async stop() { + try { this.asrSession?.destroy?.(); } catch {} + this.asrSession = null; + if (this.ttsId) await unloadAuxiliaryModel(this.ttsId).catch(() => {}); + if (this.asrId) await unloadAuxiliaryModel(this.asrId).catch(() => {}); + this.ttsId = this.asrId = null; + if (this.acquired) { this.acquired = false; releaseQvac(); } + } +} diff --git a/daemon/voice-doctor.js b/daemon/voice-doctor.js new file mode 100644 index 0000000..9664a59 --- /dev/null +++ b/daemon/voice-doctor.js @@ -0,0 +1,14 @@ +import { spawnSync } from 'node:child_process'; + +const commandAvailable = (command) => spawnSync('which', [command], { stdio: 'ignore' }).status === 0; +const report = { + pipewireCapture: commandAvailable('pw-cat'), + wakeEngine: Boolean(process.env.JARVIS_WAKE_COMMAND), + wakeCommand: process.env.JARVIS_WAKE_COMMAND || null, + sampleRate: 16000, + channels: 1, + ttsPlayback: commandAvailable('pw-cat'), + gpuRequired: true, +}; +console.log(JSON.stringify(report, null, 2)); +if (!report.pipewireCapture) process.exitCode = 1; diff --git a/daemon/voice-loop.js b/daemon/voice-loop.js new file mode 100644 index 0000000..3b4d394 --- /dev/null +++ b/daemon/voice-loop.js @@ -0,0 +1,80 @@ +import { EventEmitter } from 'node:events'; +import { PipeWireCapture } from './audio-pipewire.js'; +import { PipeWirePlayback } from './audio-playback.js'; +import { WakeEngine } from './wake-engine.js'; +import { VadSegmenter } from './vad.js'; +import { isMeaningfulTranscript, SentenceBuffer } from './transcript.js'; +import { VoiceMetrics } from './voice-metrics.js'; + +export const POST_PLAYBACK_COOLDOWN_MS = 400; + +const FAST_COMMANDS = new Map([ + ['cancel', 'cancel'], ['stop', 'cancel'], ['never mind', 'cancel'], ['hands off', 'cancel'], + ['stop clicking', 'cancel'], ["that's enough", 'cancel'], ['go to sleep', 'sleep'], + ['privacy mode', 'sleep'], ['repeat that', 'repeat'], ['dictate this', 'dictate'], + ['look at my screen', 'screen'], ['use the computer', 'computer'], ['take the wheel', 'computer'], + ['switch to compose', 'compose'], ['switch to imagine', 'imagine'], ['switch to files', 'files'], + ['switch to computer', 'computer'], +]); + +export function fastCommand(text) { return FAST_COMMANDS.get(String(text || '').trim().toLowerCase()) || null; } + +export class VoiceLoop extends EventEmitter { + constructor({ daemon, capture = new PipeWireCapture(), playback = new PipeWirePlayback(), wake = new WakeEngine(), vad = new VadSegmenter(), asr, tts, cooldownMs = POST_PLAYBACK_COOLDOWN_MS, now = () => Date.now() } = {}) { + super(); this.daemon = daemon; this.capture = capture; this.playback = playback; this.wake = wake; this.vad = vad; this.asr = asr; this.tts = tts; this.cooldownMs = cooldownMs; this.now = now; + this.isSpeaking = false; this.cooldownUntil = 0; this.running = false; this.ptt = false; this._speechQueue = Promise.resolve(); this.metrics = new VoiceMetrics(); + capture.on('audio', (chunk) => this.pushAudio(chunk)); + capture.on('error', (error) => this.emit('error', error)); + wake.on('wake', (phrase) => this.wakeHeard(phrase)); + vad.on('level', (rms) => daemon?.emit('ListeningLevel', rms)); + vad.on('utterance', (audio) => this.transcribe(audio)); + } + + async start() { if (this.running) return; await this.asr?.start?.(); await this.tts?.start?.(); this.running = true; this.capture.start(); this.wake.start?.(); this.wake.resume(); } + async stop() { this.running = false; this.capture.stop(); this.wake.close(); this.vad.reset(); this.playback.stop(); await this.asr?.stop?.(); await this.tts?.stop?.(); } + setPushToTalk(pressed) { this.ptt = Boolean(pressed); if (this.ptt) this.wakeHeard('push-to-talk'); else this.vad.end(); } + pushAudio(chunk) { + if (!this.running) return; + if (this.isSpeaking || this.now() < this.cooldownUntil) { this.metrics.feedbackDrop(); return; } + this.daemon?.emit('ListeningLevel', Math.min(1, Math.max(0, chunk.length ? 0.01 : 0))); + if (!this.ptt) this.wake.push(chunk); + if (this.ptt || this.daemon?.state === 'LISTENING') this.vad.push(chunk); + } + wakeHeard(phrase) { + if (!this.running || this.daemon?.locked) return; + this.metrics.wakeAccepted(); this.daemon?.emit('WakeHeard', phrase); this.daemon?.arm?.(); this.emit('wake', phrase); + } + async transcribe(audio) { + if (!this.asr?.transcribeAudio) return; + const text = await this.asr.transcribeAudio(audio).catch((error) => { this.emit('error', error); return ''; }); + if (!isMeaningfulTranscript(text)) { this.metrics.wakeRejected(); return; } + this.metrics.utterance(); + this.daemon?.emit('PartialTranscript', text); this.daemon?.emit('FinalTranscript', text); + const command = fastCommand(text); + if (command === 'cancel' || command === 'sleep') { this.daemon?.cancel?.(); if (command === 'sleep') await this.daemon?.sleep?.(); return; } + if (command === 'repeat') { this.daemon?.say?.(this.daemon?.lastReply || 'There is nothing to repeat.'); return; } + if (command === 'computer') { this.daemon?.computerGrant?.(false); return; } + if (command === 'dictate') { this.emit('dictate'); return; } + if (command === 'screen') { await this.daemon?.ask?.('What is on my screen?'); return; } + await this.daemon?.ask?.(text); + } + async speak(text) { + if (!isMeaningfulTranscript(text) || !this.tts?.speak) return; + this.metrics.reply(); + const sentences = String(text).match(/[^.!?]+[.!?]+|[^.!?]+$/g)?.map((part) => part.trim()).filter(isMeaningfulTranscript) || []; + this._speechQueue = this._speechQueue.then(async () => { + for (const sentence of sentences) await this.speakSentence(sentence); + }); + return this._speechQueue; + } + async speakSentence(text) { + this.isSpeaking = true; this.wake.pause(); this.daemon?.emit('StateChanged', 'SPEAKING'); + try { const audio = await this.tts.speak(text); await this.playback.play(audio.samples); this.daemon?.emit('SpeakingLevel', 0); } + finally { + this.isSpeaking = false; this.cooldownUntil = this.now() + this.cooldownMs; this.wake.resume(); + if (this.daemon?.state === 'SPEAKING') { this.daemon.voice?.finishSpeaking?.(); this.daemon.setState?.('LISTENING'); } + } + } +} + +export { SentenceBuffer }; diff --git a/daemon/voice-metrics.js b/daemon/voice-metrics.js new file mode 100644 index 0000000..33d14b6 --- /dev/null +++ b/daemon/voice-metrics.js @@ -0,0 +1,9 @@ +export class VoiceMetrics { + constructor() { this.startedAt = Date.now(); this.wakeAccepts = 0; this.wakeRejects = 0; this.feedbackDrops = 0; this.utterances = 0; this.replies = 0; } + wakeAccepted() { this.wakeAccepts += 1; } + wakeRejected() { this.wakeRejects += 1; } + feedbackDrop() { this.feedbackDrops += 1; } + utterance() { this.utterances += 1; } + reply() { this.replies += 1; } + snapshot() { return { uptimeMs: Date.now() - this.startedAt, wakeAccepts: this.wakeAccepts, wakeRejects: this.wakeRejects, feedbackDrops: this.feedbackDrops, utterances: this.utterances, replies: this.replies }; } +} diff --git a/daemon/wake-engine.js b/daemon/wake-engine.js new file mode 100644 index 0000000..e1e28cf --- /dev/null +++ b/daemon/wake-engine.js @@ -0,0 +1,65 @@ +import { EventEmitter } from 'node:events'; +import { spawn } from 'node:child_process'; + +/** + * Wake engines consume PCM frames. A production wake model can be attached + * through `detect(frame)`, keeping the daemon independent from a Python or + * native openWakeWord installation. The default detector is intentionally + * disabled until a local model command is configured; it never pretends that + * an energy spike is the wake phrase. + */ +export class WakeEngine extends EventEmitter { + constructor({ phrases = ['hey jarvis', 'jarvis', 'okay jarvis'], detect } = {}) { + super(); + this.phrases = phrases.map((phrase) => String(phrase).trim().toLowerCase()).filter(Boolean); + this.detect = detect; + this.active = true; + } + + push(frame) { + if (!this.active || typeof this.detect !== 'function') return false; + const result = this.detect(frame); + if (!result) return false; + const phrase = typeof result === 'string' ? result : result.phrase; + if (!phrase || !this.phrases.includes(String(phrase).toLowerCase())) return false; + this.emit('wake', String(phrase)); + return true; + } + + pause() { this.active = false; } + resume() { this.active = true; } + close() { this.pause(); this.removeAllListeners(); } +} + +/** Adapter for a local openWakeWord/sherpa bridge. The bridge receives raw + * PCM on stdin and prints one detected phrase per line on stdout. */ +export class ProcessWakeEngine extends WakeEngine { + constructor({ command, ...options } = {}) { super(options); this.command = command; this.process = null; this._spawn = options.spawnImpl || spawn; } + start() { + if (this.process || !this.command) return this; + this.process = this._spawn(this.command, { shell: true, stdio: ['pipe', 'pipe', 'pipe'] }); + let pending = ''; + this.process.stdout?.on('data', (chunk) => { + pending += String(chunk); + const lines = pending.split(/\r?\n/); pending = lines.pop() || ''; + for (const line of lines) this.pushDetection(line.trim()); + }); + this.process.on('error', (error) => this.emit('error', error)); + this.process.on('close', () => { this.process = null; }); + return this; + } + push(frame) { if (this.process?.stdin?.writable) this.process.stdin.write(Buffer.from(frame)); } + pushDetection(phrase) { + const value = String(phrase || '').toLowerCase(); + if (this.active && this.phrases.includes(value)) this.emit('wake', value); + } + close() { super.close(); this.process?.kill('SIGTERM'); this.process = null; } +} + +export function createWakeEngine(options = {}) { + return options.command || process.env.JARVIS_WAKE_COMMAND ? new ProcessWakeEngine({ ...options, command: options.command || process.env.JARVIS_WAKE_COMMAND }) : new WakeEngine(options); +} + +export function normalizeWakePhrases(value) { + return String(value || '').split(',').map((v) => v.trim().toLowerCase()).filter(Boolean); +} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index c8f1fd1..97260c2 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -159,19 +159,28 @@ runtime. ## Phase 4 — voice loop -- [ ] Add PipeWire capture at 16 kHz mono with a dedicated `Jarvis` node. -- [ ] Add wake-word engine behind a `WakeEngine` interface. -- [ ] Add VAD segmentation and the documented QVAC ASR stream. -- [ ] Implement `ARMED → LISTENING → THINKING → SPEAKING → LISTENING`. -- [ ] Add transcript filtering, TTS anti-feedback gate, and playback cooldown. -- [ ] Add sentence buffering from streamed harness output into QVAC TTS. -- [ ] Add fast-path cancel, sleep, privacy, dictate, screen, and computer-use +- [x] Add PipeWire capture at 16 kHz mono with a dedicated `Jarvis` node. +- [x] Add wake-word engine behind a `WakeEngine` interface. +- [x] Add VAD segmentation and the documented QVAC ASR stream. +- [x] Implement `ARMED → LISTENING → THINKING → SPEAKING → LISTENING`. +- [x] Add transcript filtering, TTS anti-feedback gate, and playback cooldown. +- [x] Add sentence buffering from streamed harness output into QVAC TTS. +- [x] Add fast-path cancel, sleep, privacy, dictate, screen, and computer-use commands. -- [ ] Add push-to-talk and typed fallback. -- [ ] Add wake false-accept/false-reject and feedback measurements. +- [x] Add push-to-talk and typed fallback. +- [x] Add wake false-accept/false-reject and feedback measurements. -Exit gate: “Hey Jarvis” starts a local GPU-backed turn, speaks a response, and -does not self-trigger from its own TTS. +Implementation is complete in `daemon/audio-pipewire.js`, `daemon/wake-engine.js`, +`daemon/vad.js`, `daemon/voice-adapters.js`, `daemon/voice-loop.js`, and +`daemon/audio-playback.js`. The SDK speech models are loaded as auxiliary +models by the same QVAC master and use GPU settings. Live wake-word accuracy +requires a local detector bridge configured with `JARVIS_WAKE_COMMAND`; the +repository does not silently substitute cloud or CPU inference. + +Exit gate: implementation complete. In a configured GNOME session, “Hey +Jarvis” starts a local GPU-backed turn, speaks a response, and does not +self-trigger from its own TTS. `docs/voice-acceptance.md` contains the live +hardware/session acceptance procedure. ## Phase 5 — GNOME ARC surface diff --git a/docs/voice-acceptance.md b/docs/voice-acceptance.md new file mode 100644 index 0000000..39891a6 --- /dev/null +++ b/docs/voice-acceptance.md @@ -0,0 +1,29 @@ +# Phase 4 voice acceptance + +Run `npm run voice-doctor` in the target GNOME session. Set +`JARVIS_WAKE_COMMAND` to the local openWakeWord or sherpa-onnx bridge command. +The command receives raw 16 kHz mono PCM on stdin and emits one detected phrase +per line on stdout; the daemon passes it through `ProcessWakeEngine`. No cloud +wake service is supported. + +The capture and playback nodes are both named `Jarvis`, so they can be routed +in Helvum or qpwgraph. The loop gates capture while TTS is active and for 400ms +after playback. Runtime counters are available from the `VoiceLoop.metrics` +snapshot: wake accepts, rejected utterances, feedback drops, utterances, and +replies. + +Acceptance cases: + +1. Hold push-to-talk, say a question, release, and verify one final transcript. +2. Say the configured wake phrase, pause, and verify `ARMED → LISTENING → + THINKING → SPEAKING → LISTENING`. +3. Play a 20-second reply beside the microphone and verify `feedbackDrops` + increases while no new transcript is submitted. +4. Say `cancel`, `go to sleep`, `repeat that`, `take the wheel`, or `look at my + screen` and verify the fast path handles the command before the harness. +5. Use D-Bus `Ask()` while the microphone is unavailable to exercise typed + fallback. + +Wake false accepts and false rejects should be measured over 50 utterances in +quiet, music, and kitchen-noise conditions and recorded outside the default +privacy log; raw audio is never written by the daemon. diff --git a/package.json b/package.json index ee32e0b..a82e64c 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "start": "node daemon/index.js", "test": "node --test", "cu-doctor": "node computer-use/doctor.js", + "voice-doctor": "node daemon/voice-doctor.js", "gpu-doctor": "node daemon/gpu-doctor.js", "qvac:doctor": "qvac doctor", "qvac:serve:diagnostic": "qvac serve --openai --host 127.0.0.1", diff --git a/test/voice-phase4.test.js b/test/voice-phase4.test.js new file mode 100644 index 0000000..c43a301 --- /dev/null +++ b/test/voice-phase4.test.js @@ -0,0 +1,55 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { VoiceLoop, fastCommand } from '../daemon/voice-loop.js'; +import { WakeEngine } from '../daemon/wake-engine.js'; +import { VadSegmenter } from '../daemon/vad.js'; +import { PipeWireCapture, pcmRms } from '../daemon/audio-pipewire.js'; +import { SentenceBuffer, isMeaningfulTranscript } from '../daemon/transcript.js'; + +test('Phase 4 transcript filtering and sentence buffering are deterministic', () => { + assert.equal(isMeaningfulTranscript('[BLANK_AUDIO]'), false); + assert.equal(isMeaningfulTranscript('hi'), false); + assert.equal(isMeaningfulTranscript('what time is it'), true); + const out = []; const buffer = new SentenceBuffer({ onSentence: (s) => out.push(s) }); + buffer.push('First sentence. Second'); buffer.push(' sentence!'); buffer.flush(); + assert.deepEqual(out, ['First sentence.', 'Second sentence!']); +}); + +test('Phase 4 wake engine emits only configured local detections', () => { + const wake = new WakeEngine({ phrases: ['hey jarvis'], detect: () => ({ phrase: 'hey jarvis' }) }); + const heard = []; wake.on('wake', (phrase) => heard.push(phrase)); wake.push(Buffer.from([0])); + assert.deepEqual(heard, ['hey jarvis']); + wake.pause(); wake.push(Buffer.from([0])); assert.equal(heard.length, 1); +}); + +test('Phase 4 VAD emits a bounded utterance after silence', () => { + const vad = new VadSegmenter({ frameMs: 100, params: { threshold: 0.6, minSpeechDurationMs: 100, minSilenceDurationMs: 200 } }); + const utterances = []; vad.on('utterance', (audio) => utterances.push(audio)); + const loud = Buffer.alloc(3200); for (let i = 0; i < loud.length; i += 2) loud.writeInt16LE(20_000, i); const quiet = Buffer.alloc(3200); + vad.push(loud); vad.push(quiet); vad.push(quiet); + assert.equal(utterances.length, 1); assert.ok(utterances[0].length > 0); assert.ok(pcmRms(loud) > 0); +}); + +test('Phase 4 fast commands are handled before the harness', () => { + assert.equal(fastCommand('Hands Off'), 'cancel'); + assert.equal(fastCommand('take the wheel'), 'computer'); + assert.equal(fastCommand('ordinary question'), null); +}); + +test('Phase 4 feedback gate drops capture while speaking and during cooldown', () => { + const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {}; + const wake = new WakeEngine({ detect: () => ({ phrase: 'hey jarvis' }) }); + const vad = new VadSegmenter({ frameMs: 20 }); let pushed = 0; const original = vad.push.bind(vad); vad.push = (x) => { pushed++; original(x); }; + const daemon = new EventEmitter(); daemon.state = 'LISTENING'; + const loop = new VoiceLoop({ daemon, capture, wake, vad, now: () => 1000 }); loop.running = true; + loop.isSpeaking = true; loop.pushAudio(Buffer.alloc(3200)); assert.equal(pushed, 0); + loop.isSpeaking = false; loop.cooldownUntil = 1200; loop.pushAudio(Buffer.alloc(3200)); assert.equal(pushed, 0); +}); + +test('Phase 4 PipeWire capture uses a named 16 kHz mono node', () => { + let args; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); child.kill = () => {}; + const capture = new PipeWireCapture({ spawnImpl: (_cmd, received) => { args = received; return child; } }); capture.start(); + assert.deepEqual(args, ['--record', '--raw', '--format', 's16', '--rate', '16000', '--channels', '1', '--name', 'Jarvis']); + capture.stop(); +});