@@ -8,12 +8,13 @@ export const MIC_FORMAT = 's16';
|
||||
/** Raw 16 kHz mono capture from PipeWire. The process is deliberately kept
|
||||
* outside gnome-shell and has a stable node name for routing in Helvum. */
|
||||
export class PipeWireCapture extends EventEmitter {
|
||||
constructor({ command = 'pw-cat', spawnImpl = spawn, sampleRate = MIC_SAMPLE_RATE, nodeName = 'Jarvis' } = {}) {
|
||||
constructor({ command = 'pw-cat', spawnImpl = spawn, sampleRate = MIC_SAMPLE_RATE, nodeName = 'Jarvis', target = '' } = {}) {
|
||||
super();
|
||||
this.command = command;
|
||||
this.spawnImpl = spawnImpl;
|
||||
this.sampleRate = sampleRate;
|
||||
this.nodeName = nodeName;
|
||||
this.target = target;
|
||||
this.process = null;
|
||||
}
|
||||
|
||||
@@ -21,7 +22,7 @@ export class PipeWireCapture extends EventEmitter {
|
||||
if (this.process) return this;
|
||||
this.process = this.spawnImpl(this.command, [
|
||||
'--record', '--raw', '--format', MIC_FORMAT, '--rate', String(this.sampleRate),
|
||||
'--channels', String(MIC_CHANNELS), '--properties', `node.name=${this.nodeName}`, '-',
|
||||
'--channels', String(MIC_CHANNELS), ...(this.target ? ['--target', this.target] : []), '--properties', `node.name=${this.nodeName}`, '-',
|
||||
], { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
this.process.stdout?.on('data', (chunk) => this.emit('audio', Buffer.from(chunk)));
|
||||
this.process.stderr?.on('data', (chunk) => this.emit('diagnostic', String(chunk).trim()));
|
||||
|
||||
@@ -2,12 +2,12 @@ import { spawn } from 'node:child_process';
|
||||
import { EventEmitter } from 'node:events';
|
||||
|
||||
export class PipeWirePlayback extends EventEmitter {
|
||||
constructor({ command = 'pw-cat', spawnImpl = spawn, sampleRate = 44_100, nodeName = 'Jarvis' } = {}) {
|
||||
super(); this.command = command; this.spawnImpl = spawnImpl; this.sampleRate = sampleRate; this.nodeName = nodeName; this.process = null;
|
||||
constructor({ command = 'pw-cat', spawnImpl = spawn, sampleRate = 44_100, nodeName = 'Jarvis', target = '' } = {}) {
|
||||
super(); this.command = command; this.spawnImpl = spawnImpl; this.sampleRate = sampleRate; this.nodeName = nodeName; this.target = target; this.process = null;
|
||||
}
|
||||
async play(samples) {
|
||||
async play(samples, sampleRate = this.sampleRate) {
|
||||
this.stop();
|
||||
const child = this.process = this.spawnImpl(this.command, ['--playback', '--raw', '--format', 's16', '--rate', String(this.sampleRate), '--channels', '1', '--properties', `node.name=${this.nodeName}`, '-'], { stdio: ['pipe', 'ignore', 'pipe'] });
|
||||
const child = this.process = this.spawnImpl(this.command, ['--playback', '--raw', '--format', 's16', '--rate', String(sampleRate), '--channels', '1', ...(this.target ? ['--target', this.target] : []), '--properties', `node.name=${this.nodeName}`, '-'], { stdio: ['pipe', 'ignore', 'pipe'] });
|
||||
let diagnostic = '';
|
||||
child.stderr?.on('data', (chunk) => { diagnostic = (diagnostic + String(chunk)).slice(-2048); this.emit('diagnostic', String(chunk).trim()); });
|
||||
await new Promise((resolve, reject) => {
|
||||
|
||||
@@ -24,6 +24,9 @@ export async function serveOnSessionBus(daemon) {
|
||||
PushToTalk(pressed) { daemon.setPushToTalk?.(Boolean(pressed)); }
|
||||
async Say(text) { await daemon.say?.(text); }
|
||||
async Ask(text) { await daemon.ask(text); }
|
||||
ReloadSettings() { return daemon.reloadSettings(); }
|
||||
PreviewVoice(text) { return daemon.previewVoice(text); }
|
||||
StopSpeech() { daemon.stopSpeech(); }
|
||||
async ResetContext() { await daemon.resetContext(); }
|
||||
Confirm(jobId, toolCallId, decision) { daemon.confirmPermission?.(jobId, toolCallId, decision); }
|
||||
Cancel() { daemon.cancel(); }
|
||||
@@ -64,6 +67,9 @@ export async function serveOnSessionBus(daemon) {
|
||||
Session.configureMembers({
|
||||
methods: {
|
||||
Arm: { inSignature: '', outSignature: '' },
|
||||
ReloadSettings: { inSignature: '', outSignature: 's' },
|
||||
PreviewVoice: { inSignature: 's', outSignature: '' },
|
||||
StopSpeech: { inSignature: '', outSignature: '' },
|
||||
Sleep: { inSignature: '', outSignature: '', method: 'Sleep' },
|
||||
Shutdown: { inSignature: '', outSignature: '', method: 'Shutdown' },
|
||||
PushToTalk: { inSignature: 'b', outSignature: '', method: 'PushToTalk' },
|
||||
|
||||
@@ -23,8 +23,8 @@ try {
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
// Resource discovery initializes QVAC handles even without loading a model.
|
||||
// This one-shot command owns them and must close them before returning to
|
||||
// first-run. Bound cleanup in case a native handle never settles.
|
||||
// This one-shot command owns them and must close them before returning.
|
||||
// Bound cleanup in case a native handle never settles.
|
||||
const watchdog = setTimeout(() => {
|
||||
console.error('gpu-doctor: QVAC cleanup timed out');
|
||||
process.exit(1);
|
||||
|
||||
+15
-11
@@ -1,5 +1,6 @@
|
||||
import { voiceSettings } from './voice-settings.js';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { acquireQvac, closeQvac, releaseQvac, Agent } from './qvac-master.js';
|
||||
import { acquireQvac, closeQvac, releaseQvac, Agent, QVAC_MASTER } from './qvac-master.js';
|
||||
import { createRuntimeTools } from '../skills/runtime-tools.js';
|
||||
import { createPhase2Tools } from '../skills/phase2-tools.js';
|
||||
import { createQvacTools } from '../skills/qvac-tools.js';
|
||||
@@ -9,8 +10,9 @@ import { createComputerActTools } from '../skills/computer-act.js';
|
||||
import { createPhase9GatewayTool } from '../skills/phase9-tools.js';
|
||||
|
||||
export class HarnessBridge extends EventEmitter {
|
||||
constructor({ cwd = process.cwd(), model = 'qwen3.5-4b', tools = [], computer, observer, actuator, permissionMode = 'ask' } = {}) {
|
||||
constructor({ cwd = process.cwd(), model = QVAC_MASTER.model, tools = [], computer, observer, actuator, permissionMode = 'ask' } = {}) {
|
||||
super();
|
||||
const settings = voiceSettings();
|
||||
this.options = {
|
||||
cwd,
|
||||
model,
|
||||
@@ -29,9 +31,9 @@ export class HarnessBridge extends EventEmitter {
|
||||
origin: 'jarvis-qvac',
|
||||
system: VOICE_SYSTEM_PROMPT,
|
||||
voice: true,
|
||||
maxTurns: 6,
|
||||
maxShellCalls: 1,
|
||||
maxToolRounds: 4,
|
||||
maxTurns: settings.maxTurns,
|
||||
maxShellCalls: settings.maxShellCalls,
|
||||
maxToolRounds: settings.maxToolRounds,
|
||||
};
|
||||
this.session = null;
|
||||
}
|
||||
@@ -42,10 +44,12 @@ export class HarnessBridge extends EventEmitter {
|
||||
throw new Error(`Jarvis uses one QVAC master model (${process.env.JARVIS_QVAC_MODEL}); requested ${this.options.model}`);
|
||||
}
|
||||
await acquireQvac();
|
||||
this._acquired = true;
|
||||
try {
|
||||
this.session = await Agent.create(this.options);
|
||||
} catch (error) {
|
||||
releaseQvac();
|
||||
this._acquired = false;
|
||||
await closeQvac();
|
||||
throw error;
|
||||
}
|
||||
@@ -93,14 +97,14 @@ export class HarnessBridge extends EventEmitter {
|
||||
cancel() { this.session?.cancel(); }
|
||||
|
||||
async resetContext() {
|
||||
this.session?.cancel?.();
|
||||
await this.session?.dispose?.();
|
||||
this.session = null;
|
||||
try { this.session?.cancel?.(); await this.session?.dispose?.(); }
|
||||
finally {
|
||||
this.session = null;
|
||||
if (this._acquired) { releaseQvac(); this._acquired = false; }
|
||||
}
|
||||
}
|
||||
|
||||
async close() {
|
||||
await this.session?.dispose();
|
||||
releaseQvac();
|
||||
await closeQvac();
|
||||
try { await this.resetContext(); } finally { await closeQvac(); }
|
||||
}
|
||||
}
|
||||
|
||||
+89
-20
@@ -1,3 +1,8 @@
|
||||
import { PipeWireCapture } from './audio-pipewire.js';
|
||||
import { PipeWirePlayback } from './audio-playback.js';
|
||||
import { VadSegmenter } from './vad.js';
|
||||
import { FrameNormalizer } from '../computer-use/frame.js';
|
||||
import { ttsConfiguration } from './tts-config.js';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { HarnessBridge } from './harness-bridge.js';
|
||||
import { ComputerUseSession } from '../computer-use/session.js';
|
||||
@@ -22,13 +27,15 @@ export class JarvisDaemon extends EventEmitter {
|
||||
constructor() {
|
||||
super();
|
||||
this.recovery = new StateRecovery(); const restored = this.recovery.load(); this.state = restored.state; this.mode = restored.mode;
|
||||
this.voice = new VoiceStateMachine();
|
||||
this.settings = voiceSettings();
|
||||
this.startupSettings = this.settings;
|
||||
this.voice = new VoiceStateMachine({ idleMs: this.settings.idleMinutes * 60_000 });
|
||||
this.scheduler = new QvacScheduler({ concurrency: 1 });
|
||||
this.audit = new ComputerAudit();
|
||||
this.computer = new ComputerUseSession({ audit: this.audit });
|
||||
this.computer = new ComputerUseSession({ audit: this.audit, stepsMax: this.settings.computerSteps, grantMinutes: this.settings.computerGrantMinutes, mode: this.settings.computerMode });
|
||||
this.input = new PortalInputBackend();
|
||||
this.perception = new QvacPerception();
|
||||
this.observer = new DesktopObserver({ ocr: (image) => this.perception.ocr(image) });
|
||||
this.observer = new DesktopObserver({ normalizer: new FrameNormalizer({ maxLongEdge: this.settings.screenshotMaxEdge, quality: this.settings.screenshotQuality }), ocr: (image) => this.perception.ocr(image) });
|
||||
this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit });
|
||||
this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer, actuator: this.actuator });
|
||||
this.log = new PrivacyLog();
|
||||
@@ -36,6 +43,7 @@ export class JarvisDaemon extends EventEmitter {
|
||||
this.lastReply = '';
|
||||
this.voiceLoop = null;
|
||||
this._activeAsk = null;
|
||||
this._askGeneration = 0;
|
||||
this._idleTimer = setInterval(() => this.tickIdle(), 30_000);
|
||||
this._idleTimer.unref?.();
|
||||
this.telemetry = new RuntimeTelemetry();
|
||||
@@ -50,7 +58,7 @@ export class JarvisDaemon extends EventEmitter {
|
||||
|
||||
setState(state) { this.state = state; try { this.recovery.save({ state, mode: this.mode }); } catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); } this.emit('StateChanged', state); this.log.record('state', { state }).catch(() => {}); }
|
||||
async arm() { if (this.locked) return; await resumeQvac().catch(() => {}); this.voice.wake(); this.setState('LISTENING'); }
|
||||
async sleep() { this.voice.sleep(); this.setState('SLEEPING'); await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); }
|
||||
async sleep() { this.cancel(); this.voice.sleep(); this.setState('SLEEPING'); await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); }
|
||||
say(text) {
|
||||
const spoken = spokenReply(text) || spokenReply(this.lastReply) || 'There is nothing to repeat.';
|
||||
this.lastReply = spoken;
|
||||
@@ -59,6 +67,9 @@ export class JarvisDaemon extends EventEmitter {
|
||||
this._speakReply(spoken);
|
||||
}
|
||||
async ask(text) {
|
||||
if (this.locked) throw new Error('Jarvis is locked');
|
||||
if (this._settingsReload) throw new Error('Settings are being applied; try again shortly');
|
||||
const generation = ++this._askGeneration;
|
||||
this.voiceLoop?.interrupt?.();
|
||||
try {
|
||||
this.voice.typedUtterance(); this.setState('THINKING');
|
||||
@@ -66,17 +77,20 @@ export class JarvisDaemon extends EventEmitter {
|
||||
const job = this.scheduler.run(() => this.harness.ask(text), { lane: 'voice' });
|
||||
this._activeAsk = job;
|
||||
const reply = await job;
|
||||
if (generation !== this._askGeneration || this.locked) return '';
|
||||
this.telemetry.record('llm', startedAt, { success: true });
|
||||
const spoken = spokenReply(reply);
|
||||
if (!spoken) { this.setState('LISTENING'); return ''; }
|
||||
if (!spoken) { this._finishSpeech(); return ''; }
|
||||
this._beginSpeech(); this.lastReply = spoken; this.emit('Reply', spoken); this._speakReply(spoken); return spoken;
|
||||
} catch (error) {
|
||||
if (generation !== this._askGeneration) return '';
|
||||
this.telemetry.record('llm', Date.now(), { success: false }); this.emit('Error', 'QVAC', error.message); this.voice.cancel(); this.setState('ARMED'); throw error;
|
||||
} finally {
|
||||
this._activeAsk = null;
|
||||
if (generation === this._askGeneration) this._activeAsk = null;
|
||||
}
|
||||
}
|
||||
async resetContext() {
|
||||
this._askGeneration += 1;
|
||||
this.harness.cancel();
|
||||
this.scheduler.cancelQueued((job) => job.lane === 'voice');
|
||||
await this._activeAsk?.catch?.(() => {});
|
||||
@@ -99,15 +113,18 @@ export class JarvisDaemon extends EventEmitter {
|
||||
}
|
||||
_finishSpeech() {
|
||||
try { this.voice.finishSpeaking(); } catch {}
|
||||
if (this.state === 'SPEAKING') this.setState('LISTENING');
|
||||
if (this.settings.listeningMode !== 'conversation') { this.voice.cancel(); this.setState('ARMED'); }
|
||||
else this.setState('LISTENING');
|
||||
}
|
||||
_speakReply(spoken) {
|
||||
const generation = this._askGeneration;
|
||||
if (!this.voiceLoop) {
|
||||
this._finishSpeech();
|
||||
return;
|
||||
}
|
||||
const play = async () => {
|
||||
await this.ensureTts();
|
||||
if (generation !== this._askGeneration || this.locked) return;
|
||||
if (!this.voiceLoop?.status?.tts) {
|
||||
const reason = this.voiceLoop?.status?.errors?.tts;
|
||||
if (reason && reason !== 'Spoken replies are disabled') this.emit('Error', 'TTS', reason);
|
||||
@@ -119,14 +136,15 @@ export class JarvisDaemon extends EventEmitter {
|
||||
.catch((error) => {
|
||||
this.emit('Error', 'TTS', error.message);
|
||||
})
|
||||
.finally(() => this._finishSpeech());
|
||||
.finally(() => { if (generation === this._askGeneration) this._finishSpeech(); });
|
||||
}
|
||||
async ensureTts() {
|
||||
await this._voiceStarting;
|
||||
if (!this.voiceLoop) return;
|
||||
const settings = voiceSettings();
|
||||
if (!settings.ttsEnabled) return;
|
||||
const settings = this.settings;
|
||||
if (!settings.ttsEnabled) { this.voiceLoop.status.tts = false; return; }
|
||||
if (this.voiceLoop.status.tts) return;
|
||||
if (!this.voiceLoop.tts) this.voiceLoop.tts = new QvacVoiceAdapter({ role: 'tts' });
|
||||
if (!this.voiceLoop.tts) this.voiceLoop.tts = new QvacVoiceAdapter({ role: 'tts', settings });
|
||||
try {
|
||||
await this.voiceLoop.tts.start();
|
||||
this.voiceLoop.status.tts = true;
|
||||
@@ -139,16 +157,26 @@ export class JarvisDaemon extends EventEmitter {
|
||||
this.emit('Error', 'TTS', message);
|
||||
}
|
||||
}
|
||||
cancel() { this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computer.revoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
|
||||
computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; }
|
||||
cancel() { this._askGeneration += 1; this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computerRevoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
|
||||
computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); if (this.settings.computerMode === 'observe') return result; this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; }
|
||||
computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
|
||||
async startVoice() {
|
||||
startVoice() {
|
||||
if (this._voiceStarting) return this._voiceStarting;
|
||||
this._voiceStarting = this._startVoice().finally(() => { this._voiceStarting = null; });
|
||||
return this._voiceStarting;
|
||||
}
|
||||
async _startVoice() {
|
||||
if (this.voiceLoop) return;
|
||||
const settings = voiceSettings();
|
||||
const asr = new QvacVoiceAdapter({ role: 'asr' });
|
||||
const tts = settings.ttsEnabled ? new QvacVoiceAdapter({ role: 'tts' }) : null;
|
||||
const settings = this.settings;
|
||||
const asr = settings.microphoneEnabled ? new QvacVoiceAdapter({ role: 'asr', settings }) : null;
|
||||
const tts = settings.ttsEnabled ? new QvacVoiceAdapter({ role: 'tts', settings }) : null;
|
||||
if (!settings.ttsEnabled) console.log('jarvisd: voice: TTS disabled in config.json');
|
||||
const loop = new VoiceLoop({ daemon: this, wake: createWakeEngine(settings), asr, tts });
|
||||
const loop = new VoiceLoop({ daemon: this, wake: createWakeEngine(settings), asr, tts,
|
||||
capture: new PipeWireCapture({ target: settings.inputTarget }),
|
||||
playback: new PipeWirePlayback({ target: settings.outputTarget }),
|
||||
vad: new VadSegmenter({ params: { threshold: settings.vadThreshold, minSpeechDurationMs: settings.vadMinSpeechMs, minSilenceDurationMs: settings.vadSilenceMs, maxSpeechDurationMs: settings.vadMaxSpeechSeconds * 1000 } }),
|
||||
cooldownMs: settings.playbackCooldownMs, listeningMode: settings.listeningMode,
|
||||
});
|
||||
loop.on('error', (error) => { console.error(`jarvisd: voice: ${error.message}`); this.emit('Error', 'VOICE', error.message); });
|
||||
this.voiceLoop = loop;
|
||||
try { await loop.start(); this.emit('StateChanged', this.state); } catch (error) {
|
||||
@@ -157,15 +185,56 @@ export class JarvisDaemon extends EventEmitter {
|
||||
if (loop.status.tts) console.log('jarvisd: voice: TTS ready');
|
||||
else if (loop.status.errors.tts) console.error(`jarvisd: voice: TTS: ${loop.status.errors.tts}`);
|
||||
}
|
||||
async reloadSettings() {
|
||||
if (this._settingsReload) return this._settingsReload;
|
||||
this._settingsReload = this._reloadSettings().finally(() => { this._settingsReload = null; });
|
||||
return this._settingsReload;
|
||||
}
|
||||
async _reloadSettings() {
|
||||
if (this._activeAsk) throw new Error('Wait for the current request to finish before applying settings');
|
||||
const next = voiceSettings(null, { strict: true });
|
||||
if (next.ttsEnabled) ttsConfiguration(next);
|
||||
await this._voiceStarting;
|
||||
const previous = this.settings;
|
||||
this._askGeneration += 1;
|
||||
await this.voiceLoop?.stop?.();
|
||||
this.voiceLoop = null;
|
||||
this.settings = next;
|
||||
this.voice.idleMs = next.idleMinutes * 60_000;
|
||||
if (['computerMode', 'computerSteps', 'computerGrantMinutes'].some(key => next[key] !== previous[key])) this.computerRevoke();
|
||||
Object.assign(this.computer, { mode: next.computerMode, stepsMax: next.computerSteps, grantMinutes: next.computerGrantMinutes });
|
||||
Object.assign(this.observer.normalizer, { maxLongEdge: next.screenshotMaxEdge, quality: next.screenshotQuality });
|
||||
await this.startVoice();
|
||||
this.voice.cancel(); this.setState('ARMED');
|
||||
return JSON.stringify({ applied: true, voice: this.voiceLoop.status, environmentOverrides: ['JARVIS_TTS_MODEL', 'JARVIS_ASR_MODEL', 'JARVIS_WAKE_COMMAND', 'JARVIS_QVAC_MODEL'].filter(key => process.env[key]), restartRequired: ['chatModel', 'modelProfile', 'maxTurns', 'maxShellCalls', 'maxToolRounds'].filter(key => next[key] !== this.startupSettings[key]) });
|
||||
}
|
||||
async previewVoice(text) {
|
||||
if (this.locked) throw new Error('Unlock the desktop to preview a voice');
|
||||
if (this._activeAsk) throw new Error('Wait for the current request to finish before previewing');
|
||||
await this._settingsReload;
|
||||
await this.startVoice(); await this.ensureTts();
|
||||
if (!this.voiceLoop?.status.tts) throw new Error(this.voiceLoop?.status.errors.tts || 'Enable spoken replies to preview a voice');
|
||||
const preview = String(text || this.settings.previewText).slice(0, 1000);
|
||||
const generation = this._askGeneration;
|
||||
this._beginSpeech();
|
||||
try { await this.voiceLoop.speak(preview); } finally { if (generation === this._askGeneration) this._finishSpeech(); }
|
||||
}
|
||||
stopSpeech() { this.voiceLoop?.interrupt?.(); this._finishSpeech(); }
|
||||
setPushToTalk(pressed) { this.voiceLoop?.setPushToTalk(pressed); this.emit('PushToTalk', Boolean(pressed)); }
|
||||
runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), computer: this.computer.status(), voice: this.voiceLoop ? { ...this.voiceLoop.metrics.snapshot(), ...this.voiceLoop.status } : null, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }
|
||||
runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), settings: this.settings, computer: this.computer.status(), voice: this.voiceLoop ? { ...this.voiceLoop.metrics.snapshot(), ...this.voiceLoop.status } : null, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }
|
||||
async assessModelFit(model) { return JSON.stringify(await callQvac('assessModelFit', { modelSrc: String(model) })); }
|
||||
async downloadModel(model) { return JSON.stringify(await callQvac('downloadAsset', { modelSrc: String(model) })); }
|
||||
async cancelModel(model) { return JSON.stringify(await cancelQvacRequest({ modelId: String(model) })); }
|
||||
async wipeComputerTraces() { await this.audit.wipeTemp(); return true; }
|
||||
handleLockScreen(locked) { this.locked = Boolean(locked); if (this.locked) { this.cancel(); this.setState('ARMED'); } this.emit('LockScreenChanged', this.locked); }
|
||||
tickIdle() { if (!this.locked && this.voice.expireIdle() === 'SLEEPING' && this.state !== 'SLEEPING') this.sleep(); }
|
||||
async close() { clearInterval(this._idleTimer); clearInterval(this._telemetryTimer); this.computerRevoke(); this.recovery.save({ state: 'ARMED', mode: this.mode }); await this.voiceLoop?.stop?.(); await this.harness.close(); }
|
||||
async close() {
|
||||
clearInterval(this._idleTimer); clearInterval(this._telemetryTimer);
|
||||
this.cancel();
|
||||
try { this.recovery.save({ state: 'ARMED', mode: this.mode }); }
|
||||
catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); }
|
||||
try { await this.voiceLoop?.stop?.(); } finally { await this.harness.close(); }
|
||||
}
|
||||
}
|
||||
|
||||
export async function startDaemon() {
|
||||
|
||||
+9
-4
@@ -8,22 +8,27 @@ const MAX_LINE = 64 * 1024;
|
||||
export async function createEventSocket({ socketPath, onMessage }) {
|
||||
await mkdir(path.dirname(socketPath), { recursive: true });
|
||||
try { await rm(socketPath, { force: true }); } catch {}
|
||||
const clients = new Set();
|
||||
const server = net.createServer((socket) => {
|
||||
clients.add(socket);
|
||||
socket.on('error', () => {});
|
||||
socket.once('close', () => clients.delete(socket));
|
||||
const invalid = () => { if (!socket.destroyed) socket.write(JSON.stringify({ error: 'invalid IPC message' }) + '\n'); };
|
||||
let buffer = '';
|
||||
socket.setEncoding('utf8');
|
||||
socket.on('data', (chunk) => {
|
||||
buffer += chunk;
|
||||
if (buffer.length > MAX_LINE * 2) { socket.destroy(new Error('IPC buffer too large')); return; }
|
||||
let index;
|
||||
while ((index = buffer.indexOf('\n')) >= 0) {
|
||||
const line = buffer.slice(0, index); buffer = buffer.slice(index + 1);
|
||||
if (line.length > MAX_LINE) { socket.destroy(new Error('IPC message too large')); return; }
|
||||
try { onMessage?.(JSON.parse(line), socket); } catch { socket.write(JSON.stringify({ error: 'invalid IPC message' }) + '\n'); }
|
||||
if (Buffer.byteLength(line) > MAX_LINE) { socket.destroy(new Error('IPC message too large')); return; }
|
||||
try { Promise.resolve(onMessage?.(JSON.parse(line), socket)).catch(invalid); } catch { invalid(); }
|
||||
}
|
||||
if (Buffer.byteLength(buffer) > MAX_LINE) socket.destroy(new Error('IPC message too large'));
|
||||
});
|
||||
});
|
||||
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(socketPath, resolve); });
|
||||
return { server, socketPath, close: () => new Promise((resolve) => server.close(() => fs.unlink(socketPath, () => resolve()))) };
|
||||
return { server, socketPath, close: () => new Promise((resolve) => { for (const client of clients) client.destroy(); server.close(() => fs.unlink(socketPath, () => resolve())); }) };
|
||||
}
|
||||
|
||||
export function sendEvent(socket, event, payload = {}) {
|
||||
|
||||
@@ -1 +1 @@
|
||||
export function assertLocalEndpoint(url) { const parsed = new URL(url); if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') throw new Error('only HTTP(S) model endpoints are supported'); if (!['127.0.0.1', 'localhost', '::1'].includes(parsed.hostname) && process.env.JARVIS_P2P_ENABLE !== '1') throw new Error('network access is disabled unless optional P2P/model fetch is explicitly enabled'); return parsed; }
|
||||
export function assertLocalEndpoint(url) { const parsed = new URL(url); if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') throw new Error('only HTTP(S) model endpoints are supported'); if (!['127.0.0.1', 'localhost', '[::1]'].includes(parsed.hostname) && process.env.JARVIS_P2P_ENABLE !== '1') throw new Error('network access is disabled unless optional P2P/model fetch is explicitly enabled'); return parsed; }
|
||||
|
||||
+21
-27
@@ -1,3 +1,5 @@
|
||||
import { voiceSettings } from './voice-settings.js';
|
||||
import { profile } from './model-profiles.js';
|
||||
import { createRequire } from 'node:module';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
@@ -17,7 +19,7 @@ const auxiliaryModels = new Map();
|
||||
|
||||
export const QVAC_MASTER = Object.freeze({
|
||||
configPath: process.env.QVAC_CONFIG_PATH,
|
||||
model: process.env.JARVIS_QVAC_MODEL || 'qwen3.5-4b',
|
||||
model: process.env.JARVIS_QVAC_MODEL || voiceSettings().chatModel || profile(voiceSettings().modelProfile).model,
|
||||
device: 'gpu',
|
||||
gpuLayers: 99,
|
||||
});
|
||||
@@ -45,34 +47,26 @@ export function assertSdkVersion() {
|
||||
|
||||
export async function acquireQvac({ auxiliaryOnly = false } = {}) {
|
||||
if (auxiliaryOnly) { await qvacSdk(); ownerCount += 1; return; }
|
||||
ownerCount += 1;
|
||||
if (!loadPromise) {
|
||||
assertSdkVersion();
|
||||
const resources = await Agent.engine.resources();
|
||||
const gpuVisible = (resources.gpus?.length || 0) > 0 ||
|
||||
Boolean(resources.drivers?.vulkan || resources.drivers?.cuda || resources.drivers?.opencl || resources.gpu);
|
||||
if (!gpuVisible) {
|
||||
ownerCount = Math.max(0, ownerCount - 1);
|
||||
throw new Error('Jarvis requires a QVAC-visible GPU backend; run npm run gpu-doctor');
|
||||
}
|
||||
loadPromise = Agent.engine.load({
|
||||
model: QVAC_MASTER.model,
|
||||
tools: true,
|
||||
device: 'gpu',
|
||||
gpu_layers: QVAC_MASTER.gpuLayers,
|
||||
mmprojUseGpu: true,
|
||||
}).then((loaded) => {
|
||||
if (loaded.device !== 'gpu') {
|
||||
throw new Error(`Jarvis requires GPU QVAC inference; loaded device was ${loaded.device}`);
|
||||
}
|
||||
// Publish the promise before the first await so concurrent callers share
|
||||
// resource discovery and loading. Count only successful acquisitions.
|
||||
loadPromise = Promise.resolve().then(async () => {
|
||||
assertSdkVersion();
|
||||
const resources = await Agent.engine.resources();
|
||||
const gpuVisible = (resources.gpus?.length || 0) > 0 ||
|
||||
Boolean(resources.drivers?.vulkan || resources.drivers?.cuda || resources.drivers?.opencl || resources.gpu);
|
||||
if (!gpuVisible) throw new Error('Jarvis requires a QVAC-visible GPU backend; run npm run gpu-doctor');
|
||||
const loaded = await Agent.engine.load({
|
||||
model: QVAC_MASTER.model, tools: true, device: 'gpu',
|
||||
gpu_layers: QVAC_MASTER.gpuLayers, mmprojUseGpu: true,
|
||||
});
|
||||
if (loaded.device !== 'gpu') throw new Error(`Jarvis requires GPU QVAC inference; loaded device was ${loaded.device}`);
|
||||
return loaded;
|
||||
}).catch((error) => {
|
||||
loadPromise = null;
|
||||
ownerCount = Math.max(0, ownerCount - 1);
|
||||
throw error;
|
||||
});
|
||||
}).catch((error) => { loadPromise = null; throw error; });
|
||||
}
|
||||
return loadPromise;
|
||||
const loaded = await loadPromise;
|
||||
ownerCount += 1;
|
||||
return loaded;
|
||||
}
|
||||
|
||||
export async function qvacSdk() {
|
||||
@@ -92,7 +86,7 @@ function resolveSdkAsset(sdk, name) {
|
||||
|
||||
function resolveModelConfigAssets(sdk, config) {
|
||||
const copy = { ...config };
|
||||
for (const key of ['vadModelSrc', 'projectionModelSrc', 'vocabModelSrc']) {
|
||||
for (const key of ['vadModelSrc', 'projectionModelSrc', 'vocabModelSrc', 's3genModelSrc']) {
|
||||
if (typeof copy[key] === 'string') copy[key] = resolveSdkAsset(sdk, copy[key]);
|
||||
}
|
||||
return copy;
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
// Model configuration fields are checked against the installed QVAC 0.19 SDK.
|
||||
export const TTS_PRESETS = Object.freeze({
|
||||
'supertonic-en': { model: 'TTS_EN_SUPERTONIC_Q8_0', engine: 'supertonic', sampleRate: 44100 },
|
||||
supertonic3: { model: 'TTS_MULTILINGUAL_SUPERTONIC3_Q8_0', engine: 'supertonic', sampleRate: 44100 },
|
||||
chatterbox: { model: 'TTS_T3_TURBO_EN_CHATTERBOX_Q8_0', engine: 'chatterbox', sampleRate: 24000 },
|
||||
parler: { model: 'TTS_MINI_V1_EN_PARLER_TTS_Q8_0', engine: 'parler', sampleRate: 44100 },
|
||||
});
|
||||
|
||||
export function ttsConfiguration(settings) {
|
||||
const preset = TTS_PRESETS[settings.ttsPreset];
|
||||
if (!preset) throw new Error('Unknown speech model preset');
|
||||
const config = { ttsEngine: preset.engine, useGPU: settings.ttsUseGpu };
|
||||
if (preset.engine === 'supertonic') {
|
||||
Object.assign(config, { language: settings.ttsPreset === 'supertonic3' ? settings.ttsLanguage : 'en', voice: settings.voiceId, ttsSpeed: settings.ttsSpeed, ttsNumInferenceSteps: settings.ttsSteps });
|
||||
} else {
|
||||
Object.assign(config, { threads: settings.ttsThreads, seed: settings.ttsSeed });
|
||||
if (preset.engine === 'chatterbox') {
|
||||
Object.assign(config, { language: 'en', s3genModelSrc: 'TTS_S3GEN_EN_CHATTERBOX', cfmSteps: settings.ttsCfmSteps });
|
||||
if (settings.ttsReferenceAudio) config.referenceAudioSrc = settings.ttsReferenceAudio;
|
||||
} else {
|
||||
if (!settings.ttsDescription) throw new Error('Describe the Parler voice before applying settings');
|
||||
Object.assign(config, { description: settings.ttsDescription, temperature: settings.ttsTemperature });
|
||||
}
|
||||
}
|
||||
return { model: settings.ttsModel || preset.model, sampleRate: preset.sampleRate, config };
|
||||
}
|
||||
|
||||
export function scaleSpeech(samples, volume = 100) {
|
||||
if (volume === 100) return samples;
|
||||
const output = new Int16Array(samples.length);
|
||||
const gain = Math.max(0, Math.min(100, volume)) / 100;
|
||||
for (let i = 0; i < samples.length; i++) output[i] = Math.round(samples[i] * gain);
|
||||
return output;
|
||||
}
|
||||
+5
-4
@@ -20,7 +20,7 @@ export class VadSegmenter extends EventEmitter {
|
||||
this.reset();
|
||||
}
|
||||
|
||||
reset() { this.speaking = false; this.buffer = []; this.speechMs = 0; this.silenceMs = 0; }
|
||||
reset() { this.speaking = false; this.buffer = []; this.speechMs = 0; this.silenceMs = 0; this.recordingMs = 0; }
|
||||
|
||||
push(frame) {
|
||||
const chunk = Buffer.from(frame || '');
|
||||
@@ -34,11 +34,12 @@ export class VadSegmenter extends EventEmitter {
|
||||
}
|
||||
if (!this.speaking) return;
|
||||
this.buffer.push(chunk);
|
||||
this.recordingMs += durationMs;
|
||||
if (voiced) { this.speechMs += durationMs; this.silenceMs = 0; }
|
||||
else { this.silenceMs += durationMs; }
|
||||
if (this.speechMs >= this.params.maxSpeechDurationMs ||
|
||||
(this.speechMs >= this.params.minSpeechDurationMs && this.silenceMs >= this.params.minSilenceDurationMs)) {
|
||||
this.end();
|
||||
if (this.recordingMs >= this.params.maxSpeechDurationMs || this.silenceMs >= this.params.minSilenceDurationMs) {
|
||||
if (this.speechMs >= this.params.minSpeechDurationMs) this.end();
|
||||
else this.reset();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { voiceSettings } from './voice-settings.js';
|
||||
import { ttsConfiguration, scaleSpeech } from './tts-config.js';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { acquireQvac, releaseQvac, loadAuxiliaryModel, unloadAuxiliaryModel, qvacSdk, withQvacMaster, assertSdkVersion } from './qvac-master.js';
|
||||
|
||||
@@ -20,8 +22,11 @@ function toUint8(samples) {
|
||||
}
|
||||
|
||||
export class QvacVoiceAdapter extends EventEmitter {
|
||||
constructor({ role = 'both', asrModel = process.env.JARVIS_ASR_MODEL || 'WHISPER_TINY', ttsModel = process.env.JARVIS_TTS_MODEL || 'TTS_EN_SUPERTONIC_Q8_0' } = {}) {
|
||||
super(); this.role = role; this.asrModel = asrModel; this.ttsModel = ttsModel; this.asrId = null; this.ttsId = null; this.asrSession = null; this.acquired = false;
|
||||
constructor({ role = 'both', settings = voiceSettings(), asrModel = settings.asrModel, ttsModel } = {}) {
|
||||
super(); this.role = role; this.settings = settings;
|
||||
this.ttsConfig = role === 'asr' ? null : ttsConfiguration(settings);
|
||||
this.asrModel = asrModel; this.ttsModel = ttsModel || this.ttsConfig?.model;
|
||||
this.asrId = null; this.ttsId = null; this.asrSession = null; this.acquired = false;
|
||||
}
|
||||
|
||||
async start() {
|
||||
@@ -29,8 +34,8 @@ export class QvacVoiceAdapter extends EventEmitter {
|
||||
assertSdkVersion();
|
||||
await acquireQvac({ auxiliaryOnly: true }); this.acquired = true;
|
||||
try {
|
||||
if (this.role !== 'tts') this.asrId = await loadAuxiliaryModel(this.asrModel, { vadModelSrc: 'VAD_SILERO_5_1_2', audio_format: 's16le', language: 'en', no_timestamps: true, vad_params: { threshold: 0.6, min_speech_duration_ms: 300, min_silence_duration_ms: 700, max_speech_duration_s: 15, speech_pad_ms: 200 } }, 'whispercpp-transcription');
|
||||
if (this.role !== 'asr') this.ttsId = await loadAuxiliaryModel(this.ttsModel, { ttsEngine: 'supertonic', language: 'en', voice: 'F1', ttsSpeed: 1.05, ttsNumInferenceSteps: 5 }, 'tts-ggml');
|
||||
if (this.role !== 'tts') this.asrId = await loadAuxiliaryModel(this.asrModel, { vadModelSrc: 'VAD_SILERO_5_1_2', audio_format: 's16le', language: this.settings.asrLanguage, no_timestamps: true, vad_params: { threshold: this.settings.vadThreshold, min_speech_duration_ms: this.settings.vadMinSpeechMs, min_silence_duration_ms: this.settings.vadSilenceMs, max_speech_duration_s: this.settings.vadMaxSpeechSeconds, speech_pad_ms: 200 } }, 'whispercpp-transcription');
|
||||
if (this.role !== 'asr') this.ttsId = await loadAuxiliaryModel(this.ttsModel, this.ttsConfig.config, 'tts-ggml');
|
||||
} catch (error) { await this.stop(); throw error; }
|
||||
}
|
||||
|
||||
@@ -58,7 +63,9 @@ export class QvacVoiceAdapter extends EventEmitter {
|
||||
}
|
||||
return result;
|
||||
});
|
||||
return pcmS16le(samples, 44_100);
|
||||
const audio = pcmS16le(samples, this.ttsConfig.sampleRate);
|
||||
audio.samples = scaleSpeech(audio.samples, this.settings.ttsVolume);
|
||||
return audio;
|
||||
}
|
||||
|
||||
async stop() {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { voiceSettings } from './voice-settings.js';
|
||||
import { ttsConfiguration } from './tts-config.js';
|
||||
import { access } from 'node:fs/promises';
|
||||
|
||||
const commandAvailable = async (command) => {
|
||||
@@ -7,13 +9,18 @@ const commandAvailable = async (command) => {
|
||||
return false;
|
||||
};
|
||||
const pipewire = await commandAvailable('pw-cat');
|
||||
const settings = voiceSettings();
|
||||
const report = {
|
||||
pipewireCapture: pipewire,
|
||||
wakeEngine: Boolean(process.env.JARVIS_WAKE_COMMAND),
|
||||
wakeCommand: process.env.JARVIS_WAKE_COMMAND || null,
|
||||
wakeEngine: settings.listeningMode !== 'ptt' && Boolean(settings.command),
|
||||
wakeCommand: settings.command || null,
|
||||
sampleRate: 16000,
|
||||
channels: 1,
|
||||
ttsPlayback: pipewire,
|
||||
microphoneEnabled: settings.microphoneEnabled,
|
||||
inputTarget: settings.inputTarget || 'system default',
|
||||
outputTarget: settings.outputTarget || 'system default',
|
||||
speech: settings.ttsEnabled ? ttsConfiguration(settings) : 'disabled',
|
||||
gpuRequired: true,
|
||||
};
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
|
||||
+65
-16
@@ -20,18 +20,23 @@ const FAST_COMMANDS = new Map([
|
||||
export function fastCommand(text) { return FAST_COMMANDS.get(String(text || '').trim().toLowerCase()) || null; }
|
||||
|
||||
export class VoiceLoop extends EventEmitter {
|
||||
constructor({ daemon, capture = new PipeWireCapture(), playback = new PipeWirePlayback(), wake = new WakeEngine(), vad = new VadSegmenter(), asr, tts, cooldownMs = POST_PLAYBACK_COOLDOWN_MS, now = () => Date.now() } = {}) {
|
||||
super(); this.daemon = daemon; this.capture = capture; this.playback = playback; this.wake = wake; this.vad = vad; this.asr = asr; this.tts = tts; this.cooldownMs = cooldownMs; this.now = now;
|
||||
this.isSpeaking = false; this.cooldownUntil = 0; this.running = false; this.ptt = false; this._generation = 0; this._speechQueue = Promise.resolve(); this.metrics = new VoiceMetrics();
|
||||
constructor({ daemon, capture = new PipeWireCapture(), playback = new PipeWirePlayback(), wake = new WakeEngine(), vad = new VadSegmenter(), asr, tts, cooldownMs = POST_PLAYBACK_COOLDOWN_MS, listeningMode = 'conversation', now = () => Date.now(), captureRetryMs = 2000, asrRetryMs = 4000 } = {}) {
|
||||
super(); this.daemon = daemon; this.capture = capture; this.playback = playback; this.wake = wake; this.vad = vad; this.asr = asr; this.tts = tts; this.cooldownMs = cooldownMs; this.listeningMode = listeningMode; this.now = now;
|
||||
this.captureRetryMs = captureRetryMs; this.asrRetryMs = asrRetryMs;
|
||||
this.isSpeaking = false; this.cooldownUntil = 0; this.running = false; this.ptt = false; this._generation = 0; this._speechQueue = Promise.resolve(); this.metrics = new VoiceMetrics(); this._transcriptions = new Set();
|
||||
capture.on('audio', (chunk) => this.pushAudio(chunk));
|
||||
capture.on('error', (error) => { this.status.capture = false; this.status.errors.capture = error.message; this.emit('error', error); });
|
||||
capture.on('close', () => { this.status.capture = false; if (this.running) { this.status.errors.capture = 'Microphone stream closed'; this.emit('error', new Error('Microphone stream closed')); } });
|
||||
capture.on('error', (error) => { this.status.capture = false; this.status.errors.capture = error.message; this.emit('error', error); this._scheduleCaptureRetry(); });
|
||||
capture.on('close', () => { this.status.capture = false; if (this.running) { this.status.errors.capture = 'Microphone stream closed'; this._scheduleCaptureRetry(); } });
|
||||
wake.on('unavailable', () => { this.status.wake = false; });
|
||||
wake.on('error', (error) => { this.status.wake = false; this.emit('error', error); });
|
||||
this.status = { asr: false, tts: false, capture: false, wake: false, errors: {} };
|
||||
wake.on('wake', (phrase) => this.wakeHeard(phrase));
|
||||
vad.on('level', (rms) => daemon?.emit('ListeningLevel', rms));
|
||||
vad.on('utterance', (audio) => this.transcribe(audio).catch((error) => this.emit('error', error)));
|
||||
vad.on('utterance', (audio) => {
|
||||
const task = this.transcribe(audio).catch((error) => this.emit('error', error));
|
||||
this._transcriptions.add(task);
|
||||
task.finally(() => this._transcriptions.delete(task)).catch(() => {});
|
||||
});
|
||||
}
|
||||
|
||||
async start() {
|
||||
@@ -48,20 +53,62 @@ export class VoiceLoop extends EventEmitter {
|
||||
}
|
||||
this.running = true;
|
||||
if (this.status.asr) {
|
||||
try { this.capture.start(); this.status.capture = true; }
|
||||
catch (error) { this.status.errors.capture = error.message; this.emit('error', error); }
|
||||
try { this.wake.start?.(); this.wake.resume(); this.status.wake = Boolean(this.wake.command || this.wake.detect); }
|
||||
catch (error) { this.status.errors.wake = error.message; this.emit('error', error); }
|
||||
this._armCapture();
|
||||
this._armWake();
|
||||
} else if (this.asr) {
|
||||
this._scheduleAsrRetry();
|
||||
}
|
||||
}
|
||||
async stop() { this.running = false; this.capture.stop(); this.wake.close(); this.vad.reset(); this.playback.stop(); await this.asr?.stop?.(); if (this.tts !== this.asr) await this.tts?.stop?.(); }
|
||||
_notifyStatus() { try { this.daemon?.emit?.('StateChanged', this.daemon.state || 'ARMED'); } catch {} }
|
||||
_armWake() {
|
||||
try { if (this.listeningMode !== 'ptt') this.wake.start?.(); this.wake.resume(); this.status.wake = this.listeningMode !== 'ptt' && Boolean(this.wake.command || this.wake.detect); }
|
||||
catch (error) { this.status.errors.wake = error.message; this.emit('error', error); }
|
||||
}
|
||||
_armCapture() {
|
||||
if (!this.running || !this.status.asr || this.status.capture) return;
|
||||
try { this.capture.start(); this.status.capture = true; delete this.status.errors.capture; this._notifyStatus(); }
|
||||
catch (error) { this.status.errors.capture = error.message; this.emit('error', error); this._scheduleCaptureRetry(); }
|
||||
}
|
||||
_scheduleCaptureRetry() {
|
||||
if (!this.running || this._captureRetry || this.status.capture || !this.status.asr) return;
|
||||
this._captureRetry = setTimeout(() => {
|
||||
this._captureRetry = 0;
|
||||
this._armCapture();
|
||||
}, this.captureRetryMs);
|
||||
this._captureRetry.unref?.();
|
||||
}
|
||||
_scheduleAsrRetry() {
|
||||
if (!this.running || this._asrRetry || this.status.asr || !this.asr) return;
|
||||
this._asrRetry = setTimeout(() => {
|
||||
this._asrRetry = 0;
|
||||
if (!this.running || this.status.asr || !this.asr) return;
|
||||
this.asr.start().then(() => {
|
||||
if (!this.running || this.status.asr) return;
|
||||
this.status.asr = true;
|
||||
delete this.status.errors.asr;
|
||||
this._armCapture();
|
||||
this._armWake();
|
||||
this._notifyStatus();
|
||||
}).catch((error) => {
|
||||
this.status.errors.asr = String(error?.message || error);
|
||||
this._scheduleAsrRetry();
|
||||
});
|
||||
}, this.asrRetryMs);
|
||||
this._asrRetry.unref?.();
|
||||
}
|
||||
async stop() {
|
||||
this.running = false;
|
||||
if (this._captureRetry) { clearTimeout(this._captureRetry); this._captureRetry = 0; }
|
||||
if (this._asrRetry) { clearTimeout(this._asrRetry); this._asrRetry = 0; }
|
||||
this.interrupt(); this.capture.stop(); this.wake.close(); this.vad.reset(); this.playback.stop(); await Promise.allSettled([this._speechQueue, ...this._transcriptions]); await this.asr?.stop?.(); if (this.tts !== this.asr) await this.tts?.stop?.();
|
||||
}
|
||||
setPushToTalk(pressed) { this.ptt = Boolean(pressed) && this.status.asr && this.status.capture; if (this.ptt) this.wakeHeard('push-to-talk'); else this.vad.end(); }
|
||||
pushAudio(chunk) {
|
||||
if (!this.running || this.daemon?.locked || this.daemon?.state === 'SLEEPING') return;
|
||||
if (this.isSpeaking || this.now() < this.cooldownUntil) { this.metrics.feedbackDrop(); return; }
|
||||
try { this.daemon?.emit('ListeningLevel', Math.min(1, Math.max(0, chunk?.length ? 0.01 : 0))); } catch {}
|
||||
if (!this.ptt) this.wake.push(chunk);
|
||||
if (this.ptt || this.daemon?.state === 'LISTENING') this.vad.push(chunk);
|
||||
if (!this.ptt && this.listeningMode !== 'ptt') this.wake.push(chunk);
|
||||
if (this.ptt || (this.listeningMode !== 'ptt' && this.daemon?.state === 'LISTENING')) this.vad.push(chunk);
|
||||
}
|
||||
wakeHeard(phrase) {
|
||||
if (!this.running || this.daemon?.locked) return;
|
||||
@@ -69,8 +116,9 @@ export class VoiceLoop extends EventEmitter {
|
||||
}
|
||||
async transcribe(audio) {
|
||||
if (!this.status.asr || !this.asr?.transcribeAudio) return;
|
||||
const generation = this._generation;
|
||||
const text = await this.asr.transcribeAudio(audio).catch((error) => { this.emit('error', error); return ''; });
|
||||
if (!this.running || this.daemon?.locked) return;
|
||||
if (!this.running || this.daemon?.locked || generation !== this._generation) return;
|
||||
if (!isMeaningfulTranscript(text)) { this.metrics.wakeRejected(); return; }
|
||||
this.metrics.utterance();
|
||||
this.daemon?.emit('PartialTranscript', text); this.daemon?.emit('FinalTranscript', text);
|
||||
@@ -93,7 +141,8 @@ export class VoiceLoop extends EventEmitter {
|
||||
this.wake.resume();
|
||||
if (this.daemon?.state === 'SPEAKING') {
|
||||
try { this.daemon.voice?.finishSpeaking?.(); } catch {}
|
||||
this.daemon.setState?.('LISTENING');
|
||||
if (this.daemon._finishSpeech) this.daemon._finishSpeech();
|
||||
else this.daemon.setState?.('LISTENING');
|
||||
}
|
||||
}
|
||||
async speak(text) {
|
||||
@@ -127,7 +176,7 @@ export class VoiceLoop extends EventEmitter {
|
||||
try {
|
||||
const audio = await this.tts.speak(speakableForTts(text));
|
||||
if (generation !== this._generation) return;
|
||||
await this.playback.play(audio.samples);
|
||||
await this.playback.play(audio.samples, audio.sampleRate);
|
||||
try { this.daemon?.emit('SpeakingLevel', 0); } catch {}
|
||||
} finally {
|
||||
this.isSpeaking = false;
|
||||
|
||||
+16
-17
@@ -1,25 +1,24 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { normalizeSettings } from '../apps/gnome-extension/[email protected]/settings-values.js';
|
||||
|
||||
function flag(value, fallback = true) {
|
||||
if (value === undefined || value === null) return fallback;
|
||||
if (typeof value === 'string') return !['false', '0', 'off', 'no'].includes(value.trim().toLowerCase());
|
||||
return value !== false;
|
||||
export const SETTINGS_FIELDS = JSON.parse(fs.readFileSync(new URL('../apps/gnome-extension/[email protected]/settings-catalog.json', import.meta.url).pathname, 'utf8')).fields;
|
||||
export const configPath = () => path.join(process.env.XDG_CONFIG_HOME || path.join(os.homedir(), '.config'), 'jarvis/config.json');
|
||||
export function readSettings({ strict = false } = {}) {
|
||||
try {
|
||||
const config = JSON.parse(fs.readFileSync(configPath(), 'utf8'));
|
||||
if (!config || typeof config !== 'object' || Array.isArray(config)) throw new Error('Settings must be a JSON object');
|
||||
return config;
|
||||
} catch (error) { if (strict && error.code !== 'ENOENT') throw error; return {}; }
|
||||
}
|
||||
|
||||
export function voiceSettings(source = null) {
|
||||
let config = source;
|
||||
if (!config) {
|
||||
config = {};
|
||||
try { config = JSON.parse(fs.readFileSync(path.join(process.env.XDG_CONFIG_HOME || path.join(os.homedir(), '.config'), 'jarvis/config.json'), 'utf8')); } catch {}
|
||||
}
|
||||
const phrase = config.wakePhrase || config.wake_phrase || 'hey jarvis';
|
||||
const aliases = Array.isArray(config.aliases) ? config.aliases : ['jarvis', 'okay jarvis'];
|
||||
export function voiceSettings(source = null, { strict = false } = {}) {
|
||||
const settings = normalizeSettings(source ?? readSettings({ strict }), SETTINGS_FIELDS, { strict });
|
||||
return {
|
||||
command: process.env.JARVIS_WAKE_COMMAND || config.wakeCommand || config.wake_command || '',
|
||||
phrases: [phrase, ...aliases].map((item) => String(item || '').trim()).filter(Boolean),
|
||||
ttsEnabled: flag(config.ttsEnabled ?? config.tts_enabled, true),
|
||||
modelProfile: config.modelProfile || config.model_profile || 'laptop-16gb',
|
||||
...settings,
|
||||
command: process.env.JARVIS_WAKE_COMMAND || settings.wakeCommand,
|
||||
phrases: [settings.wakePhrase, ...settings.aliases].map(item => item.trim()).filter(Boolean),
|
||||
asrModel: process.env.JARVIS_ASR_MODEL || settings.asrModel,
|
||||
ttsModel: process.env.JARVIS_TTS_MODEL || settings.ttsModel,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ export class VoiceStateMachine {
|
||||
}
|
||||
|
||||
expireIdle() {
|
||||
if (this.state !== 'SLEEPING' && this.now() - this.lastActivity >= this.idleMs) this.sleep();
|
||||
if (['ARMED', 'LISTENING'].includes(this.state) && this.now() - this.lastActivity >= this.idleMs) this.sleep();
|
||||
return this.state;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user