P4
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import { EventEmitter } from 'node:events';
|
||||
|
||||
export const MIC_SAMPLE_RATE = 16_000;
|
||||
export const MIC_CHANNELS = 1;
|
||||
export const MIC_FORMAT = 's16';
|
||||
|
||||
/** Raw 16 kHz mono capture from PipeWire. The process is deliberately kept
|
||||
* outside gnome-shell and has a stable node name for routing in Helvum. */
|
||||
export class PipeWireCapture extends EventEmitter {
|
||||
constructor({ command = 'pw-cat', spawnImpl = spawn, sampleRate = MIC_SAMPLE_RATE, nodeName = 'Jarvis' } = {}) {
|
||||
super();
|
||||
this.command = command;
|
||||
this.spawnImpl = spawnImpl;
|
||||
this.sampleRate = sampleRate;
|
||||
this.nodeName = nodeName;
|
||||
this.process = null;
|
||||
}
|
||||
|
||||
start() {
|
||||
if (this.process) return this;
|
||||
this.process = this.spawnImpl(this.command, [
|
||||
'--record', '--raw', '--format', MIC_FORMAT, '--rate', String(this.sampleRate),
|
||||
'--channels', String(MIC_CHANNELS), '--name', this.nodeName,
|
||||
], { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
this.process.stdout?.on('data', (chunk) => this.emit('audio', Buffer.from(chunk)));
|
||||
this.process.stderr?.on('data', (chunk) => this.emit('diagnostic', String(chunk).trim()));
|
||||
this.process.on('error', (error) => this.emit('error', error));
|
||||
this.process.on('close', (code, signal) => { this.process = null; this.emit('close', { code, signal }); });
|
||||
return this;
|
||||
}
|
||||
|
||||
stop() {
|
||||
if (!this.process) return;
|
||||
this.process.kill('SIGTERM');
|
||||
this.process = null;
|
||||
}
|
||||
}
|
||||
|
||||
export function pcmRms(chunk) {
|
||||
const bytes = Buffer.from(chunk || '');
|
||||
if (bytes.length < 2) return 0;
|
||||
let sum = 0;
|
||||
for (let i = 0; i + 1 < bytes.length; i += 2) {
|
||||
const sample = bytes.readInt16LE(i) / 32768;
|
||||
sum += sample * sample;
|
||||
}
|
||||
return Math.sqrt(sum / Math.floor(bytes.length / 2));
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import { EventEmitter } from 'node:events';
|
||||
|
||||
export class PipeWirePlayback extends EventEmitter {
|
||||
constructor({ command = 'pw-cat', spawnImpl = spawn, sampleRate = 44_100, nodeName = 'Jarvis' } = {}) {
|
||||
super(); this.command = command; this.spawnImpl = spawnImpl; this.sampleRate = sampleRate; this.nodeName = nodeName; this.process = null;
|
||||
}
|
||||
async play(samples) {
|
||||
this.stop();
|
||||
const child = this.process = this.spawnImpl(this.command, ['--playback', '--raw', '--format', 's16', '--rate', String(this.sampleRate), '--channels', '1', '--name', this.nodeName], { stdio: ['pipe', 'ignore', 'pipe'] });
|
||||
child.stderr?.on('data', (chunk) => this.emit('diagnostic', String(chunk).trim()));
|
||||
child.on('error', (error) => this.emit('error', error));
|
||||
child.stdin.end(Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength));
|
||||
await new Promise((resolve, reject) => { child.once('close', resolve); child.once('error', reject); });
|
||||
if (this.process === child) this.process = null;
|
||||
}
|
||||
stop() { if (this.process) { this.process.kill('SIGTERM'); this.process = null; } }
|
||||
}
|
||||
@@ -10,7 +10,7 @@ export async function serveOnSessionBus(daemon) {
|
||||
Arm() { daemon.arm(); }
|
||||
Sleep() { daemon.sleep(); }
|
||||
Shutdown() { daemon.close(); }
|
||||
PushToTalk(pressed) { daemon.emit('PushToTalk', Boolean(pressed)); }
|
||||
PushToTalk(pressed) { daemon.setPushToTalk?.(Boolean(pressed)); }
|
||||
Say(text) { return daemon.say?.(text); }
|
||||
Ask(text) { return daemon.ask(text); }
|
||||
Cancel() { daemon.cancel(); }
|
||||
|
||||
+16
-3
@@ -5,6 +5,9 @@ import { VoiceStateMachine } from './voice-state.js';
|
||||
import { QvacScheduler } from './qvac-scheduler.js';
|
||||
import { cancelQvac, resumeQvac, suspendQvac } from './qvac-master.js';
|
||||
import { PrivacyLog } from './privacy-log.js';
|
||||
import { VoiceLoop } from './voice-loop.js';
|
||||
import { QvacVoiceAdapter } from './voice-adapters.js';
|
||||
import { createWakeEngine } from './wake-engine.js';
|
||||
|
||||
export class JarvisDaemon extends EventEmitter {
|
||||
constructor() {
|
||||
@@ -16,6 +19,8 @@ export class JarvisDaemon extends EventEmitter {
|
||||
this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer });
|
||||
this.log = new PrivacyLog();
|
||||
this.locked = false;
|
||||
this.lastReply = '';
|
||||
this.voiceLoop = null;
|
||||
this._idleTimer = setInterval(() => this.tickIdle(), 30_000);
|
||||
this._idleTimer.unref?.();
|
||||
this.harness.on('agent_message_chunk', (ev) => this.emit('Token', ev?.text || ev?.delta || ''));
|
||||
@@ -26,12 +31,12 @@ export class JarvisDaemon extends EventEmitter {
|
||||
setState(state) { this.state = state; this.emit('StateChanged', state); this.log.record('state', { state }).catch(() => {}); }
|
||||
async arm() { if (this.locked) return; await resumeQvac().catch(() => {}); this.voice.wake(); this.setState('LISTENING'); }
|
||||
async sleep() { this.voice.sleep(); this.setState('SLEEPING'); await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); }
|
||||
say(text) { this.emit('Reply', String(text)); }
|
||||
say(text) { this.lastReply = String(text); this.emit('Reply', this.lastReply); }
|
||||
async ask(text) {
|
||||
this.voice.utterance(); this.setState('THINKING');
|
||||
try {
|
||||
const reply = await this.scheduler.run(() => this.harness.ask(text), { lane: 'voice' });
|
||||
this.voice.speak(); this.setState('SPEAKING'); this.emit('Reply', reply); return reply;
|
||||
this.voice.speak(); this.setState('SPEAKING'); this.lastReply = String(reply || ''); this.emit('Reply', this.lastReply); this.voiceLoop?.speak(this.lastReply).catch((error) => this.emit('Error', 'TTS', error.message)); return reply;
|
||||
} catch (error) {
|
||||
this.emit('Error', 'QVAC', error.message); this.voice.cancel(); this.setState('ARMED'); throw error;
|
||||
}
|
||||
@@ -39,13 +44,21 @@ export class JarvisDaemon extends EventEmitter {
|
||||
cancel() { this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computer.revoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
|
||||
computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); return result; }
|
||||
computerRevoke() { this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
|
||||
async startVoice() {
|
||||
if (this.voiceLoop) return;
|
||||
const voiceIO = new QvacVoiceAdapter();
|
||||
this.voiceLoop = new VoiceLoop({ daemon: this, wake: createWakeEngine(), asr: voiceIO, tts: voiceIO });
|
||||
try { await this.voiceLoop.start(); } catch (error) { this.voiceLoop = null; this.emit('Error', 'VOICE_UNAVAILABLE', error.message); throw error; }
|
||||
}
|
||||
setPushToTalk(pressed) { this.voiceLoop?.setPushToTalk(pressed); this.emit('PushToTalk', Boolean(pressed)); }
|
||||
handleLockScreen(locked) { this.locked = Boolean(locked); if (this.locked) { this.cancel(); this.setState('ARMED'); } this.emit('LockScreenChanged', this.locked); }
|
||||
tickIdle() { if (!this.locked && this.voice.expireIdle() === 'SLEEPING' && this.state !== 'SLEEPING') this.sleep(); }
|
||||
async close() { clearInterval(this._idleTimer); this.computerRevoke(); await this.harness.close(); }
|
||||
async close() { clearInterval(this._idleTimer); this.computerRevoke(); await this.voiceLoop?.stop?.(); await this.harness.close(); }
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const daemon = new JarvisDaemon();
|
||||
daemon.startVoice().catch((error) => console.error(`jarvisd: voice unavailable: ${error.message}`));
|
||||
import('./dbus-service.js').then(({ serveOnSessionBus }) => serveOnSessionBus(daemon)).catch((error) => {
|
||||
daemon.emit('Error', 'DBUS_UNAVAILABLE', error.message);
|
||||
console.error(`jarvisd: D-Bus unavailable: ${error.message}`);
|
||||
|
||||
+51
-1
@@ -8,6 +8,8 @@ const Agent = require(path.join(harnessPath, 'index.js'));
|
||||
|
||||
let loadPromise = null;
|
||||
let ownerCount = 0;
|
||||
let operationTail = Promise.resolve();
|
||||
const auxiliaryModels = new Map();
|
||||
|
||||
export const QVAC_MASTER = Object.freeze({
|
||||
configPath: process.env.QVAC_CONFIG_PATH,
|
||||
@@ -55,10 +57,58 @@ export async function acquireQvac() {
|
||||
return loadPromise;
|
||||
}
|
||||
|
||||
export async function qvacSdk() {
|
||||
return Agent.engine.ensureInit();
|
||||
}
|
||||
|
||||
export function withQvacMaster(task) {
|
||||
const operation = operationTail.then(task, task);
|
||||
operationTail = operation.catch(() => {});
|
||||
return operation;
|
||||
}
|
||||
|
||||
function resolveSdkAsset(sdk, name) {
|
||||
if (name && typeof name !== 'string') return name;
|
||||
return sdk[name] || sdk.models?.[name] || name;
|
||||
}
|
||||
|
||||
function resolveModelConfigAssets(sdk, config) {
|
||||
const copy = { ...config };
|
||||
for (const key of ['vadModelSrc', 'projectionModelSrc', 'vocabModelSrc']) {
|
||||
if (typeof copy[key] === 'string') copy[key] = resolveSdkAsset(sdk, copy[key]);
|
||||
}
|
||||
return copy;
|
||||
}
|
||||
|
||||
/** Load ASR/TTS models in the same SDK worker and under the same master lock.
|
||||
* These are auxiliary model IDs; they do not create another QVAC runtime. */
|
||||
export async function loadAuxiliaryModel(name, modelConfig = {}) {
|
||||
if (!name) throw new Error('auxiliary QVAC model name is required');
|
||||
const existing = auxiliaryModels.get(String(name));
|
||||
if (existing) return existing;
|
||||
const sdk = await qvacSdk();
|
||||
if (typeof sdk.loadModel !== 'function') throw new Error('QVAC SDK does not expose loadModel()');
|
||||
const modelId = await withQvacMaster(() => sdk.loadModel({
|
||||
modelSrc: resolveSdkAsset(sdk, name),
|
||||
modelConfig: { ...resolveModelConfigAssets(sdk, modelConfig), device: 'gpu', gpu_layers: QVAC_MASTER.gpuLayers, 'mmproj-use-gpu': true },
|
||||
}));
|
||||
auxiliaryModels.set(String(name), modelId);
|
||||
return modelId;
|
||||
}
|
||||
|
||||
export async function unloadAuxiliaryModel(modelId) {
|
||||
if (!modelId) return;
|
||||
const sdk = await qvacSdk();
|
||||
if (typeof sdk.unloadModel === 'function') await withQvacMaster(() => sdk.unloadModel({ modelId }));
|
||||
for (const [name, id] of auxiliaryModels) if (id === modelId) auxiliaryModels.delete(name);
|
||||
}
|
||||
|
||||
export function releaseQvac() { ownerCount = Math.max(0, ownerCount - 1); }
|
||||
|
||||
export async function closeQvac() {
|
||||
if (ownerCount > 0) return;
|
||||
for (const modelId of auxiliaryModels.values()) await unloadAuxiliaryModel(modelId).catch(() => {});
|
||||
auxiliaryModels.clear();
|
||||
loadPromise = null;
|
||||
await Agent.engine.close();
|
||||
}
|
||||
@@ -101,7 +151,7 @@ export async function callQvac(method, input) {
|
||||
}
|
||||
|
||||
export function qvacStatus() {
|
||||
return { ...QVAC_MASTER, owners: ownerCount, loaded: Agent.engine.getLoaded() };
|
||||
return { ...QVAC_MASTER, owners: ownerCount, loaded: Agent.engine.getLoaded(), auxiliaryModels: auxiliaryModels.size };
|
||||
}
|
||||
|
||||
export { Agent };
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
export const MIN_UTTERANCE_CHARS = 3;
|
||||
|
||||
export function isMeaningfulTranscript(text) {
|
||||
const value = String(text || '').trim();
|
||||
if (!value || /\[no speech detected\]|\[blank_audio\]/i.test(value)) return false;
|
||||
if (/^\[[^\]]+\]$/.test(value)) return false;
|
||||
return value.replace(/[^\p{L}\p{N}]/gu, '').length >= MIN_UTTERANCE_CHARS;
|
||||
}
|
||||
|
||||
export class SentenceBuffer {
|
||||
constructor({ onSentence } = {}) { this.onSentence = onSentence; this.pending = ''; }
|
||||
push(text) {
|
||||
this.pending += String(text || '');
|
||||
const sentences = [];
|
||||
let match;
|
||||
while ((match = this.pending.match(/^(.+?[.!?](?:["')\]]+)?)(?:\s+|$)/s))) {
|
||||
sentences.push(match[1].trim()); this.pending = this.pending.slice(match[0].length);
|
||||
}
|
||||
for (const sentence of sentences) this.onSentence?.(sentence);
|
||||
return sentences;
|
||||
}
|
||||
flush() { const value = this.pending.trim(); this.pending = ''; if (value) this.onSentence?.(value); return value; }
|
||||
clear() { this.pending = ''; }
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { pcmRms } from './audio-pipewire.js';
|
||||
|
||||
export const DEFAULT_VAD = Object.freeze({
|
||||
threshold: 0.6,
|
||||
minSpeechDurationMs: 300,
|
||||
minSilenceDurationMs: 700,
|
||||
maxSpeechDurationMs: 15_000,
|
||||
speechPadMs: 200,
|
||||
});
|
||||
|
||||
/** A conservative local gate around QVAC's stream. It bounds audio sent to
|
||||
* ASR and gives the loop deterministic utterance boundaries in tests. */
|
||||
export class VadSegmenter extends EventEmitter {
|
||||
constructor({ sampleRate = 16_000, frameMs = 20, params = {} } = {}) {
|
||||
super();
|
||||
this.sampleRate = sampleRate;
|
||||
this.frameMs = frameMs;
|
||||
this.params = { ...DEFAULT_VAD, ...params };
|
||||
this.reset();
|
||||
}
|
||||
|
||||
reset() { this.speaking = false; this.buffer = []; this.speechMs = 0; this.silenceMs = 0; }
|
||||
|
||||
push(frame) {
|
||||
const chunk = Buffer.from(frame || '');
|
||||
const rms = pcmRms(chunk);
|
||||
const voiced = rms >= this.params.threshold / 10; // PCM RMS is 0..1; QVAC threshold is posterior-like.
|
||||
this.emit('level', rms);
|
||||
if (!this.speaking && voiced) {
|
||||
this.speaking = true; this.speechMs = 0; this.silenceMs = 0; this.buffer = [];
|
||||
this.emit('speechStart');
|
||||
}
|
||||
if (!this.speaking) return;
|
||||
this.buffer.push(chunk);
|
||||
if (voiced) { this.speechMs += this.frameMs; this.silenceMs = 0; }
|
||||
else { this.silenceMs += this.frameMs; }
|
||||
if (this.speechMs >= this.params.maxSpeechDurationMs ||
|
||||
(this.speechMs >= this.params.minSpeechDurationMs && this.silenceMs >= this.params.minSilenceDurationMs)) {
|
||||
this.end();
|
||||
}
|
||||
}
|
||||
|
||||
end() {
|
||||
if (!this.speaking) return null;
|
||||
const audio = Buffer.concat(this.buffer);
|
||||
this.reset();
|
||||
this.emit('utterance', audio);
|
||||
return audio;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import { createRequire } from 'node:module';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import path from 'node:path';
|
||||
import { acquireQvac, releaseQvac, loadAuxiliaryModel, unloadAuxiliaryModel, qvacSdk, withQvacMaster } from './qvac-master.js';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const harnessPath = process.env.JARVIS_HARNESS_PATH || path.resolve(new URL('../vendor/agent-harness', import.meta.url).pathname);
|
||||
const sdkPackage = require(path.join(harnessPath, 'node_modules/@qvac/sdk/package.json'));
|
||||
|
||||
export class QvacVoiceAdapter extends EventEmitter {
|
||||
constructor({ asrModel = process.env.JARVIS_ASR_MODEL || 'WHISPER_TINY', ttsModel = process.env.JARVIS_TTS_MODEL || 'TTS_EN_SUPERTONIC_Q8_0' } = {}) {
|
||||
super(); this.asrModel = asrModel; this.ttsModel = ttsModel; this.asrId = null; this.ttsId = null; this.asrSession = null; this.acquired = false;
|
||||
}
|
||||
|
||||
async start() {
|
||||
if (this.acquired) return;
|
||||
if (!/^0\.19\./.test(sdkPackage.version)) throw new Error(`Jarvis voice adapter requires QVAC 0.19.x; found ${sdkPackage.version}`);
|
||||
await acquireQvac(); this.acquired = true;
|
||||
try {
|
||||
this.asrId = await loadAuxiliaryModel(this.asrModel, { vadModelSrc: 'VAD_SILERO_5_1_2', audio_format: 's16le', language: 'en', no_timestamps: true, vad_params: { threshold: 0.6, min_speech_duration_ms: 300, min_silence_duration_ms: 700, max_speech_duration_s: 15, speech_pad_ms: 200 } });
|
||||
this.ttsId = await loadAuxiliaryModel(this.ttsModel, { ttsEngine: 'supertonic', language: 'en', voice: 'F1', ttsSpeed: 1.05, ttsNumInferenceSteps: 5 });
|
||||
} catch (error) { await this.stop(); throw error; }
|
||||
}
|
||||
|
||||
writeAudio(chunk) { this.asrSession?.write(Buffer.from(chunk)); }
|
||||
async transcribeAudio(audio) {
|
||||
const sdk = await qvacSdk();
|
||||
return withQvacMaster(async () => {
|
||||
const session = await sdk.transcribeStream({ modelId: this.asrId, emitVadEvents: true });
|
||||
session.write(Buffer.from(audio)); session.end();
|
||||
const parts = [];
|
||||
for await (const event of session) parts.push(typeof event === 'string' ? event : event?.text || '');
|
||||
return parts.join(' ').trim();
|
||||
});
|
||||
}
|
||||
async *transcripts() { if (!this.asrSession) throw new Error('voice adapter is not started'); yield* this.asrSession; }
|
||||
endAudio() { this.asrSession?.end(); }
|
||||
|
||||
async speak(text) {
|
||||
const sdk = await qvacSdk();
|
||||
const samples = await withQvacMaster(async () => {
|
||||
const result = sdk.textToSpeech({ modelId: this.ttsId, text: String(text), inputType: 'text', stream: false });
|
||||
return result.buffer;
|
||||
});
|
||||
return { samples: Int16Array.from(samples), sampleRate: 44_100 };
|
||||
}
|
||||
|
||||
async stop() {
|
||||
try { this.asrSession?.destroy?.(); } catch {}
|
||||
this.asrSession = null;
|
||||
if (this.ttsId) await unloadAuxiliaryModel(this.ttsId).catch(() => {});
|
||||
if (this.asrId) await unloadAuxiliaryModel(this.asrId).catch(() => {});
|
||||
this.ttsId = this.asrId = null;
|
||||
if (this.acquired) { this.acquired = false; releaseQvac(); }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
const commandAvailable = (command) => spawnSync('which', [command], { stdio: 'ignore' }).status === 0;
|
||||
const report = {
|
||||
pipewireCapture: commandAvailable('pw-cat'),
|
||||
wakeEngine: Boolean(process.env.JARVIS_WAKE_COMMAND),
|
||||
wakeCommand: process.env.JARVIS_WAKE_COMMAND || null,
|
||||
sampleRate: 16000,
|
||||
channels: 1,
|
||||
ttsPlayback: commandAvailable('pw-cat'),
|
||||
gpuRequired: true,
|
||||
};
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
if (!report.pipewireCapture) process.exitCode = 1;
|
||||
@@ -0,0 +1,80 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { PipeWireCapture } from './audio-pipewire.js';
|
||||
import { PipeWirePlayback } from './audio-playback.js';
|
||||
import { WakeEngine } from './wake-engine.js';
|
||||
import { VadSegmenter } from './vad.js';
|
||||
import { isMeaningfulTranscript, SentenceBuffer } from './transcript.js';
|
||||
import { VoiceMetrics } from './voice-metrics.js';
|
||||
|
||||
export const POST_PLAYBACK_COOLDOWN_MS = 400;
|
||||
|
||||
const FAST_COMMANDS = new Map([
|
||||
['cancel', 'cancel'], ['stop', 'cancel'], ['never mind', 'cancel'], ['hands off', 'cancel'],
|
||||
['stop clicking', 'cancel'], ["that's enough", 'cancel'], ['go to sleep', 'sleep'],
|
||||
['privacy mode', 'sleep'], ['repeat that', 'repeat'], ['dictate this', 'dictate'],
|
||||
['look at my screen', 'screen'], ['use the computer', 'computer'], ['take the wheel', 'computer'],
|
||||
['switch to compose', 'compose'], ['switch to imagine', 'imagine'], ['switch to files', 'files'],
|
||||
['switch to computer', 'computer'],
|
||||
]);
|
||||
|
||||
export function fastCommand(text) { return FAST_COMMANDS.get(String(text || '').trim().toLowerCase()) || null; }
|
||||
|
||||
export class VoiceLoop extends EventEmitter {
|
||||
constructor({ daemon, capture = new PipeWireCapture(), playback = new PipeWirePlayback(), wake = new WakeEngine(), vad = new VadSegmenter(), asr, tts, cooldownMs = POST_PLAYBACK_COOLDOWN_MS, now = () => Date.now() } = {}) {
|
||||
super(); this.daemon = daemon; this.capture = capture; this.playback = playback; this.wake = wake; this.vad = vad; this.asr = asr; this.tts = tts; this.cooldownMs = cooldownMs; this.now = now;
|
||||
this.isSpeaking = false; this.cooldownUntil = 0; this.running = false; this.ptt = false; this._speechQueue = Promise.resolve(); this.metrics = new VoiceMetrics();
|
||||
capture.on('audio', (chunk) => this.pushAudio(chunk));
|
||||
capture.on('error', (error) => this.emit('error', error));
|
||||
wake.on('wake', (phrase) => this.wakeHeard(phrase));
|
||||
vad.on('level', (rms) => daemon?.emit('ListeningLevel', rms));
|
||||
vad.on('utterance', (audio) => this.transcribe(audio));
|
||||
}
|
||||
|
||||
async start() { if (this.running) return; await this.asr?.start?.(); await this.tts?.start?.(); this.running = true; this.capture.start(); this.wake.start?.(); this.wake.resume(); }
|
||||
async stop() { this.running = false; this.capture.stop(); this.wake.close(); this.vad.reset(); this.playback.stop(); await this.asr?.stop?.(); await this.tts?.stop?.(); }
|
||||
setPushToTalk(pressed) { this.ptt = Boolean(pressed); if (this.ptt) this.wakeHeard('push-to-talk'); else this.vad.end(); }
|
||||
pushAudio(chunk) {
|
||||
if (!this.running) return;
|
||||
if (this.isSpeaking || this.now() < this.cooldownUntil) { this.metrics.feedbackDrop(); return; }
|
||||
this.daemon?.emit('ListeningLevel', Math.min(1, Math.max(0, chunk.length ? 0.01 : 0)));
|
||||
if (!this.ptt) this.wake.push(chunk);
|
||||
if (this.ptt || this.daemon?.state === 'LISTENING') this.vad.push(chunk);
|
||||
}
|
||||
wakeHeard(phrase) {
|
||||
if (!this.running || this.daemon?.locked) return;
|
||||
this.metrics.wakeAccepted(); this.daemon?.emit('WakeHeard', phrase); this.daemon?.arm?.(); this.emit('wake', phrase);
|
||||
}
|
||||
async transcribe(audio) {
|
||||
if (!this.asr?.transcribeAudio) return;
|
||||
const text = await this.asr.transcribeAudio(audio).catch((error) => { this.emit('error', error); return ''; });
|
||||
if (!isMeaningfulTranscript(text)) { this.metrics.wakeRejected(); return; }
|
||||
this.metrics.utterance();
|
||||
this.daemon?.emit('PartialTranscript', text); this.daemon?.emit('FinalTranscript', text);
|
||||
const command = fastCommand(text);
|
||||
if (command === 'cancel' || command === 'sleep') { this.daemon?.cancel?.(); if (command === 'sleep') await this.daemon?.sleep?.(); return; }
|
||||
if (command === 'repeat') { this.daemon?.say?.(this.daemon?.lastReply || 'There is nothing to repeat.'); return; }
|
||||
if (command === 'computer') { this.daemon?.computerGrant?.(false); return; }
|
||||
if (command === 'dictate') { this.emit('dictate'); return; }
|
||||
if (command === 'screen') { await this.daemon?.ask?.('What is on my screen?'); return; }
|
||||
await this.daemon?.ask?.(text);
|
||||
}
|
||||
async speak(text) {
|
||||
if (!isMeaningfulTranscript(text) || !this.tts?.speak) return;
|
||||
this.metrics.reply();
|
||||
const sentences = String(text).match(/[^.!?]+[.!?]+|[^.!?]+$/g)?.map((part) => part.trim()).filter(isMeaningfulTranscript) || [];
|
||||
this._speechQueue = this._speechQueue.then(async () => {
|
||||
for (const sentence of sentences) await this.speakSentence(sentence);
|
||||
});
|
||||
return this._speechQueue;
|
||||
}
|
||||
async speakSentence(text) {
|
||||
this.isSpeaking = true; this.wake.pause(); this.daemon?.emit('StateChanged', 'SPEAKING');
|
||||
try { const audio = await this.tts.speak(text); await this.playback.play(audio.samples); this.daemon?.emit('SpeakingLevel', 0); }
|
||||
finally {
|
||||
this.isSpeaking = false; this.cooldownUntil = this.now() + this.cooldownMs; this.wake.resume();
|
||||
if (this.daemon?.state === 'SPEAKING') { this.daemon.voice?.finishSpeaking?.(); this.daemon.setState?.('LISTENING'); }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export { SentenceBuffer };
|
||||
@@ -0,0 +1,9 @@
|
||||
export class VoiceMetrics {
|
||||
constructor() { this.startedAt = Date.now(); this.wakeAccepts = 0; this.wakeRejects = 0; this.feedbackDrops = 0; this.utterances = 0; this.replies = 0; }
|
||||
wakeAccepted() { this.wakeAccepts += 1; }
|
||||
wakeRejected() { this.wakeRejects += 1; }
|
||||
feedbackDrop() { this.feedbackDrops += 1; }
|
||||
utterance() { this.utterances += 1; }
|
||||
reply() { this.replies += 1; }
|
||||
snapshot() { return { uptimeMs: Date.now() - this.startedAt, wakeAccepts: this.wakeAccepts, wakeRejects: this.wakeRejects, feedbackDrops: this.feedbackDrops, utterances: this.utterances, replies: this.replies }; }
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { spawn } from 'node:child_process';
|
||||
|
||||
/**
|
||||
* Wake engines consume PCM frames. A production wake model can be attached
|
||||
* through `detect(frame)`, keeping the daemon independent from a Python or
|
||||
* native openWakeWord installation. The default detector is intentionally
|
||||
* disabled until a local model command is configured; it never pretends that
|
||||
* an energy spike is the wake phrase.
|
||||
*/
|
||||
export class WakeEngine extends EventEmitter {
|
||||
constructor({ phrases = ['hey jarvis', 'jarvis', 'okay jarvis'], detect } = {}) {
|
||||
super();
|
||||
this.phrases = phrases.map((phrase) => String(phrase).trim().toLowerCase()).filter(Boolean);
|
||||
this.detect = detect;
|
||||
this.active = true;
|
||||
}
|
||||
|
||||
push(frame) {
|
||||
if (!this.active || typeof this.detect !== 'function') return false;
|
||||
const result = this.detect(frame);
|
||||
if (!result) return false;
|
||||
const phrase = typeof result === 'string' ? result : result.phrase;
|
||||
if (!phrase || !this.phrases.includes(String(phrase).toLowerCase())) return false;
|
||||
this.emit('wake', String(phrase));
|
||||
return true;
|
||||
}
|
||||
|
||||
pause() { this.active = false; }
|
||||
resume() { this.active = true; }
|
||||
close() { this.pause(); this.removeAllListeners(); }
|
||||
}
|
||||
|
||||
/** Adapter for a local openWakeWord/sherpa bridge. The bridge receives raw
|
||||
* PCM on stdin and prints one detected phrase per line on stdout. */
|
||||
export class ProcessWakeEngine extends WakeEngine {
|
||||
constructor({ command, ...options } = {}) { super(options); this.command = command; this.process = null; this._spawn = options.spawnImpl || spawn; }
|
||||
start() {
|
||||
if (this.process || !this.command) return this;
|
||||
this.process = this._spawn(this.command, { shell: true, stdio: ['pipe', 'pipe', 'pipe'] });
|
||||
let pending = '';
|
||||
this.process.stdout?.on('data', (chunk) => {
|
||||
pending += String(chunk);
|
||||
const lines = pending.split(/\r?\n/); pending = lines.pop() || '';
|
||||
for (const line of lines) this.pushDetection(line.trim());
|
||||
});
|
||||
this.process.on('error', (error) => this.emit('error', error));
|
||||
this.process.on('close', () => { this.process = null; });
|
||||
return this;
|
||||
}
|
||||
push(frame) { if (this.process?.stdin?.writable) this.process.stdin.write(Buffer.from(frame)); }
|
||||
pushDetection(phrase) {
|
||||
const value = String(phrase || '').toLowerCase();
|
||||
if (this.active && this.phrases.includes(value)) this.emit('wake', value);
|
||||
}
|
||||
close() { super.close(); this.process?.kill('SIGTERM'); this.process = null; }
|
||||
}
|
||||
|
||||
export function createWakeEngine(options = {}) {
|
||||
return options.command || process.env.JARVIS_WAKE_COMMAND ? new ProcessWakeEngine({ ...options, command: options.command || process.env.JARVIS_WAKE_COMMAND }) : new WakeEngine(options);
|
||||
}
|
||||
|
||||
export function normalizeWakePhrases(value) {
|
||||
return String(value || '').split(',').map((v) => v.trim().toLowerCase()).filter(Boolean);
|
||||
}
|
||||
Reference in New Issue
Block a user