Updates

daemon/index.js

@@ -15,6 +15,7 @@ import { PortalInputBackend } from '../computer-use/portal-input.js';
 import { ComputerActuator } from '../computer-use/actuator.js';
 import { RuntimeTelemetry } from './telemetry.js';
 import { StateRecovery } from './recovery.js';
+import { spokenReply } from '../skills/voice-prompt.js';
 
 export class JarvisDaemon extends EventEmitter {
   constructor() {
@@ -45,24 +46,54 @@ export class JarvisDaemon extends EventEmitter {
   setState(state) { this.state = state; try { this.recovery.save({ state, mode: this.mode }); } catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); } this.emit('StateChanged', state); this.log.record('state', { state }).catch(() => {}); }
   async arm() { if (this.locked) return; await resumeQvac().catch(() => {}); this.voice.wake(); this.setState('LISTENING'); }
   async sleep() { this.voice.sleep(); this.setState('SLEEPING'); await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); }
-  say(text) { this.lastReply = String(text); this.emit('Reply', this.lastReply); }
+  say(text) {
+    const spoken = spokenReply(text) || spokenReply(this.lastReply) || 'There is nothing to repeat.';
+    this.lastReply = spoken;
+    this.emit('Reply', spoken);
+    this._beginSpeech();
+    this._speakReply(spoken);
+  }
   async ask(text) {
-    this.voice.typedUtterance(); this.setState('THINKING');
+    this.voiceLoop?.interrupt?.();
     try {
+      this.voice.typedUtterance(); this.setState('THINKING');
       const startedAt = Date.now(); const reply = await this.scheduler.run(() => this.harness.ask(text), { lane: 'voice' }); this.telemetry.record('llm', startedAt, { success: true });
-      this.voice.speak(); this.setState('SPEAKING'); this.lastReply = String(reply || ''); this.emit('Reply', this.lastReply); this.voiceLoop?.speak(this.lastReply).catch((error) => this.emit('Error', 'TTS', error.message)); return reply;
+      const spoken = spokenReply(reply);
+      this._beginSpeech(); this.lastReply = spoken; this.emit('Reply', spoken); this._speakReply(spoken); return spoken;
     } catch (error) {
       this.telemetry.record('llm', Date.now(), { success: false }); this.emit('Error', 'QVAC', error.message); this.voice.cancel(); this.setState('ARMED'); throw error;
     }
   }
-  cancel() { this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computer.revoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
+  _beginSpeech() {
+    try {
+      if (this.voice.state === 'ARMED' || this.voice.state === 'SLEEPING') this.voice.wake();
+      if (this.voice.state !== 'SPEAKING') this.voice.speak();
+    } catch {}
+    this.setState('SPEAKING');
+  }
+  _finishSpeech() {
+    try { this.voice.finishSpeaking(); } catch {}
+    if (this.state === 'SPEAKING') this.setState('LISTENING');
+  }
+  _speakReply(spoken) {
+    const playing = this.voiceLoop?.speak?.(spoken);
+    if (!playing) { this._finishSpeech(); return; }
+    Promise.resolve(playing).catch((error) => {
+      this.emit('Error', 'TTS', error.message);
+      if (this.state === 'SPEAKING') this._finishSpeech();
+    });
+  }
+  cancel() { this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computer.revoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
   computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; }
   computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
   async startVoice() {
     if (this.voiceLoop) return;
     const voiceIO = new QvacVoiceAdapter();
-    this.voiceLoop = new VoiceLoop({ daemon: this, wake: createWakeEngine(), asr: voiceIO, tts: voiceIO });
-    try { await this.voiceLoop.start(); } catch (error) { this.voiceLoop = null; this.emit('Error', 'VOICE_UNAVAILABLE', error.message); throw error; }
+    const loop = new VoiceLoop({ daemon: this, wake: createWakeEngine(), asr: voiceIO, tts: voiceIO });
+    this.voiceLoop = loop;
+    try { await loop.start(); this.emit('StateChanged', this.state); } catch (error) {
+      this.voiceLoop = null; await loop.stop?.().catch(() => {}); this.emit('Error', 'VOICE_UNAVAILABLE', error.message); throw error;
+    }
   }
   setPushToTalk(pressed) { this.voiceLoop?.setPushToTalk(pressed); this.emit('PushToTalk', Boolean(pressed)); }
   runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), computer: this.computer.status(), voice: this.voiceLoop?.metrics?.snapshot?.() || null, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }

daemon/voice-adapters.js

@@ -1,6 +1,23 @@
 import { EventEmitter } from 'node:events';
 import { acquireQvac, releaseQvac, loadAuxiliaryModel, unloadAuxiliaryModel, qvacSdk, withQvacMaster, assertSdkVersion } from './qvac-master.js';
 
+export function pcmS16le(samples, sampleRate = 44_100) {
+  if (samples instanceof Int16Array) return { samples, sampleRate };
+  const bytes = toUint8(samples);
+  const even = bytes.byteLength - (bytes.byteLength % 2);
+  const copy = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + even);
+  return { samples: new Int16Array(copy), sampleRate };
+}
+
+function toUint8(samples) {
+  if (!samples) return new Uint8Array(0);
+  if (samples instanceof ArrayBuffer) return new Uint8Array(samples);
+  if (ArrayBuffer.isView(samples)) return new Uint8Array(samples.buffer, samples.byteOffset, samples.byteLength);
+  if (typeof samples === 'string') return Uint8Array.from(Buffer.from(samples));
+  if (Array.isArray(samples) || samples.length != null) return Uint8Array.from(Buffer.from(samples));
+  return new Uint8Array(0);
+}
+
 export class QvacVoiceAdapter extends EventEmitter {
   constructor({ asrModel = process.env.JARVIS_ASR_MODEL || 'WHISPER_TINY', ttsModel = process.env.JARVIS_TTS_MODEL || 'TTS_EN_SUPERTONIC_Q8_0' } = {}) {
     super(); this.asrModel = asrModel; this.ttsModel = ttsModel; this.asrId = null; this.ttsId = null; this.asrSession = null; this.acquired = false;
@@ -33,10 +50,16 @@ export class QvacVoiceAdapter extends EventEmitter {
   async speak(text) {
     const sdk = await qvacSdk();
     const samples = await withQvacMaster(async () => {
-      const result = sdk.textToSpeech({ modelId: this.ttsId, text: String(text), inputType: 'text', stream: false });
-      return result.buffer;
+      const result = await sdk.textToSpeech({ modelId: this.ttsId, text: String(text), inputType: 'text', stream: false });
+      if (result?.buffer != null) return await result.buffer;
+      if (result?.bufferStream) {
+        const chunks = [];
+        for await (const chunk of result.bufferStream) chunks.push(Buffer.from(chunk));
+        return Buffer.concat(chunks);
+      }
+      return result;
     });
-    return { samples: Int16Array.from(samples), sampleRate: 44_100 };
+    return pcmS16le(samples, 44_100);
   }
 
   async stop() {

daemon/voice-loop.js

@@ -22,7 +22,7 @@ export function fastCommand(text) { return FAST_COMMANDS.get(String(text || '').
 export class VoiceLoop extends EventEmitter {
   constructor({ daemon, capture = new PipeWireCapture(), playback = new PipeWirePlayback(), wake = new WakeEngine(), vad = new VadSegmenter(), asr, tts, cooldownMs = POST_PLAYBACK_COOLDOWN_MS, now = () => Date.now() } = {}) {
     super(); this.daemon = daemon; this.capture = capture; this.playback = playback; this.wake = wake; this.vad = vad; this.asr = asr; this.tts = tts; this.cooldownMs = cooldownMs; this.now = now;
-    this.isSpeaking = false; this.cooldownUntil = 0; this.running = false; this.ptt = false; this._speechQueue = Promise.resolve(); this.metrics = new VoiceMetrics();
+    this.isSpeaking = false; this.cooldownUntil = 0; this.running = false; this.ptt = false; this._generation = 0; this._speechQueue = Promise.resolve(); this.metrics = new VoiceMetrics();
     capture.on('audio', (chunk) => this.pushAudio(chunk));
     capture.on('error', (error) => this.emit('error', error));
     wake.on('wake', (phrase) => this.wakeHeard(phrase));
@@ -58,12 +58,29 @@ export class VoiceLoop extends EventEmitter {
     if (command === 'screen') { await this.daemon?.ask?.('What is on my screen?'); return; }
     await this.daemon?.ask?.(text);
   }
+  interrupt() {
+    this._generation += 1;
+    this.playback.stop();
+    this.isSpeaking = false;
+    this.wake.resume();
+  }
+  _releaseSpeaking() {
+    this.isSpeaking = false;
+    this.wake.resume();
+    if (this.daemon?.state === 'SPEAKING') { this.daemon.voice?.finishSpeaking?.(); this.daemon.setState?.('LISTENING'); }
+  }
   async speak(text) {
-    if (!isMeaningfulTranscript(text) || !this.tts?.speak) return;
+    const generation = this._generation || 0;
+    if (!isMeaningfulTranscript(text) || !this.tts?.speak) { this._releaseSpeaking(); return; }
     this.metrics.reply();
     const sentences = String(text).match(/[^.!?]+[.!?]+|[^.!?]+$/g)?.map((part) => part.trim()).filter(isMeaningfulTranscript) || [];
-    this._speechQueue = this._speechQueue.then(async () => {
-      for (const sentence of sentences) await this.speakSentence(sentence);
+    if (!sentences.length) { this._releaseSpeaking(); return; }
+    this._speechQueue = this._speechQueue.catch(() => {}).then(async () => {
+      if (generation !== this._generation) return;
+      for (const sentence of sentences) {
+        if (generation !== this._generation) return;
+        await this.speakSentence(sentence);
+      }
     });
     return this._speechQueue;
   }

daemon/voice-state.js

@@ -16,8 +16,8 @@ export class VoiceStateMachine {
   transition(next, reason = 'unspecified') {
     if (!STATES.includes(next)) throw new Error(`unknown Jarvis state: ${next}`);
     const allowed = {
-      ARMED: ['LISTENING', 'SLEEPING'], LISTENING: ['THINKING', 'ARMED', 'SLEEPING'],
-      THINKING: ['SPEAKING', 'ARMED', 'LISTENING'], SPEAKING: ['LISTENING', 'ARMED'],
+      ARMED: ['LISTENING', 'SLEEPING'], LISTENING: ['THINKING', 'ARMED', 'SLEEPING', 'SPEAKING'],
+      THINKING: ['SPEAKING', 'ARMED', 'LISTENING'], SPEAKING: ['LISTENING', 'ARMED', 'THINKING'],
       SLEEPING: ['LISTENING', 'ARMED'],
     };
     if (next !== this.state && !allowed[this.state].includes(next)) {