Check Point
Rolling release / release (push) Successful in 6m36s

This commit is contained in:
2026-09-12 09:07:09 -04:00
parent e9040d110a
commit a4073b9020
63 changed files with 2459 additions and 632 deletions
+3 -2
View File
@@ -8,12 +8,13 @@ export const MIC_FORMAT = 's16';
/** Raw 16 kHz mono capture from PipeWire. The process is deliberately kept
* outside gnome-shell and has a stable node name for routing in Helvum. */
export class PipeWireCapture extends EventEmitter {
constructor({ command = 'pw-cat', spawnImpl = spawn, sampleRate = MIC_SAMPLE_RATE, nodeName = 'Jarvis' } = {}) {
constructor({ command = 'pw-cat', spawnImpl = spawn, sampleRate = MIC_SAMPLE_RATE, nodeName = 'Jarvis', target = '' } = {}) {
super();
this.command = command;
this.spawnImpl = spawnImpl;
this.sampleRate = sampleRate;
this.nodeName = nodeName;
this.target = target;
this.process = null;
}
@@ -21,7 +22,7 @@ export class PipeWireCapture extends EventEmitter {
if (this.process) return this;
this.process = this.spawnImpl(this.command, [
'--record', '--raw', '--format', MIC_FORMAT, '--rate', String(this.sampleRate),
'--channels', String(MIC_CHANNELS), '--properties', `node.name=${this.nodeName}`, '-',
'--channels', String(MIC_CHANNELS), ...(this.target ? ['--target', this.target] : []), '--properties', `node.name=${this.nodeName}`, '-',
], { stdio: ['ignore', 'pipe', 'pipe'] });
this.process.stdout?.on('data', (chunk) => this.emit('audio', Buffer.from(chunk)));
this.process.stderr?.on('data', (chunk) => this.emit('diagnostic', String(chunk).trim()));
+4 -4
View File
@@ -2,12 +2,12 @@ import { spawn } from 'node:child_process';
import { EventEmitter } from 'node:events';
export class PipeWirePlayback extends EventEmitter {
constructor({ command = 'pw-cat', spawnImpl = spawn, sampleRate = 44_100, nodeName = 'Jarvis' } = {}) {
super(); this.command = command; this.spawnImpl = spawnImpl; this.sampleRate = sampleRate; this.nodeName = nodeName; this.process = null;
constructor({ command = 'pw-cat', spawnImpl = spawn, sampleRate = 44_100, nodeName = 'Jarvis', target = '' } = {}) {
super(); this.command = command; this.spawnImpl = spawnImpl; this.sampleRate = sampleRate; this.nodeName = nodeName; this.target = target; this.process = null;
}
async play(samples) {
async play(samples, sampleRate = this.sampleRate) {
this.stop();
const child = this.process = this.spawnImpl(this.command, ['--playback', '--raw', '--format', 's16', '--rate', String(this.sampleRate), '--channels', '1', '--properties', `node.name=${this.nodeName}`, '-'], { stdio: ['pipe', 'ignore', 'pipe'] });
const child = this.process = this.spawnImpl(this.command, ['--playback', '--raw', '--format', 's16', '--rate', String(sampleRate), '--channels', '1', ...(this.target ? ['--target', this.target] : []), '--properties', `node.name=${this.nodeName}`, '-'], { stdio: ['pipe', 'ignore', 'pipe'] });
let diagnostic = '';
child.stderr?.on('data', (chunk) => { diagnostic = (diagnostic + String(chunk)).slice(-2048); this.emit('diagnostic', String(chunk).trim()); });
await new Promise((resolve, reject) => {
+6
View File
@@ -24,6 +24,9 @@ export async function serveOnSessionBus(daemon) {
PushToTalk(pressed) { daemon.setPushToTalk?.(Boolean(pressed)); }
async Say(text) { await daemon.say?.(text); }
async Ask(text) { await daemon.ask(text); }
ReloadSettings() { return daemon.reloadSettings(); }
PreviewVoice(text) { return daemon.previewVoice(text); }
StopSpeech() { daemon.stopSpeech(); }
async ResetContext() { await daemon.resetContext(); }
Confirm(jobId, toolCallId, decision) { daemon.confirmPermission?.(jobId, toolCallId, decision); }
Cancel() { daemon.cancel(); }
@@ -64,6 +67,9 @@ export async function serveOnSessionBus(daemon) {
Session.configureMembers({
methods: {
Arm: { inSignature: '', outSignature: '' },
ReloadSettings: { inSignature: '', outSignature: 's' },
PreviewVoice: { inSignature: 's', outSignature: '' },
StopSpeech: { inSignature: '', outSignature: '' },
Sleep: { inSignature: '', outSignature: '', method: 'Sleep' },
Shutdown: { inSignature: '', outSignature: '', method: 'Shutdown' },
PushToTalk: { inSignature: 'b', outSignature: '', method: 'PushToTalk' },
+2 -2
View File
@@ -23,8 +23,8 @@ try {
process.exitCode = 1;
} finally {
// Resource discovery initializes QVAC handles even without loading a model.
// This one-shot command owns them and must close them before returning to
// first-run. Bound cleanup in case a native handle never settles.
// This one-shot command owns them and must close them before returning.
// Bound cleanup in case a native handle never settles.
const watchdog = setTimeout(() => {
console.error('gpu-doctor: QVAC cleanup timed out');
process.exit(1);
+15 -11
View File
@@ -1,5 +1,6 @@
import { voiceSettings } from './voice-settings.js';
import { EventEmitter } from 'node:events';
import { acquireQvac, closeQvac, releaseQvac, Agent } from './qvac-master.js';
import { acquireQvac, closeQvac, releaseQvac, Agent, QVAC_MASTER } from './qvac-master.js';
import { createRuntimeTools } from '../skills/runtime-tools.js';
import { createPhase2Tools } from '../skills/phase2-tools.js';
import { createQvacTools } from '../skills/qvac-tools.js';
@@ -9,8 +10,9 @@ import { createComputerActTools } from '../skills/computer-act.js';
import { createPhase9GatewayTool } from '../skills/phase9-tools.js';
export class HarnessBridge extends EventEmitter {
constructor({ cwd = process.cwd(), model = 'qwen3.5-4b', tools = [], computer, observer, actuator, permissionMode = 'ask' } = {}) {
constructor({ cwd = process.cwd(), model = QVAC_MASTER.model, tools = [], computer, observer, actuator, permissionMode = 'ask' } = {}) {
super();
const settings = voiceSettings();
this.options = {
cwd,
model,
@@ -29,9 +31,9 @@ export class HarnessBridge extends EventEmitter {
origin: 'jarvis-qvac',
system: VOICE_SYSTEM_PROMPT,
voice: true,
maxTurns: 6,
maxShellCalls: 1,
maxToolRounds: 4,
maxTurns: settings.maxTurns,
maxShellCalls: settings.maxShellCalls,
maxToolRounds: settings.maxToolRounds,
};
this.session = null;
}
@@ -42,10 +44,12 @@ export class HarnessBridge extends EventEmitter {
throw new Error(`Jarvis uses one QVAC master model (${process.env.JARVIS_QVAC_MODEL}); requested ${this.options.model}`);
}
await acquireQvac();
this._acquired = true;
try {
this.session = await Agent.create(this.options);
} catch (error) {
releaseQvac();
this._acquired = false;
await closeQvac();
throw error;
}
@@ -93,14 +97,14 @@ export class HarnessBridge extends EventEmitter {
cancel() { this.session?.cancel(); }
async resetContext() {
this.session?.cancel?.();
await this.session?.dispose?.();
this.session = null;
try { this.session?.cancel?.(); await this.session?.dispose?.(); }
finally {
this.session = null;
if (this._acquired) { releaseQvac(); this._acquired = false; }
}
}
async close() {
await this.session?.dispose();
releaseQvac();
await closeQvac();
try { await this.resetContext(); } finally { await closeQvac(); }
}
}
+89 -20
View File
@@ -1,3 +1,8 @@
import { PipeWireCapture } from './audio-pipewire.js';
import { PipeWirePlayback } from './audio-playback.js';
import { VadSegmenter } from './vad.js';
import { FrameNormalizer } from '../computer-use/frame.js';
import { ttsConfiguration } from './tts-config.js';
import { EventEmitter } from 'node:events';
import { HarnessBridge } from './harness-bridge.js';
import { ComputerUseSession } from '../computer-use/session.js';
@@ -22,13 +27,15 @@ export class JarvisDaemon extends EventEmitter {
constructor() {
super();
this.recovery = new StateRecovery(); const restored = this.recovery.load(); this.state = restored.state; this.mode = restored.mode;
this.voice = new VoiceStateMachine();
this.settings = voiceSettings();
this.startupSettings = this.settings;
this.voice = new VoiceStateMachine({ idleMs: this.settings.idleMinutes * 60_000 });
this.scheduler = new QvacScheduler({ concurrency: 1 });
this.audit = new ComputerAudit();
this.computer = new ComputerUseSession({ audit: this.audit });
this.computer = new ComputerUseSession({ audit: this.audit, stepsMax: this.settings.computerSteps, grantMinutes: this.settings.computerGrantMinutes, mode: this.settings.computerMode });
this.input = new PortalInputBackend();
this.perception = new QvacPerception();
this.observer = new DesktopObserver({ ocr: (image) => this.perception.ocr(image) });
this.observer = new DesktopObserver({ normalizer: new FrameNormalizer({ maxLongEdge: this.settings.screenshotMaxEdge, quality: this.settings.screenshotQuality }), ocr: (image) => this.perception.ocr(image) });
this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit });
this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer, actuator: this.actuator });
this.log = new PrivacyLog();
@@ -36,6 +43,7 @@ export class JarvisDaemon extends EventEmitter {
this.lastReply = '';
this.voiceLoop = null;
this._activeAsk = null;
this._askGeneration = 0;
this._idleTimer = setInterval(() => this.tickIdle(), 30_000);
this._idleTimer.unref?.();
this.telemetry = new RuntimeTelemetry();
@@ -50,7 +58,7 @@ export class JarvisDaemon extends EventEmitter {
setState(state) { this.state = state; try { this.recovery.save({ state, mode: this.mode }); } catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); } this.emit('StateChanged', state); this.log.record('state', { state }).catch(() => {}); }
async arm() { if (this.locked) return; await resumeQvac().catch(() => {}); this.voice.wake(); this.setState('LISTENING'); }
async sleep() { this.voice.sleep(); this.setState('SLEEPING'); await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); }
async sleep() { this.cancel(); this.voice.sleep(); this.setState('SLEEPING'); await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); }
say(text) {
const spoken = spokenReply(text) || spokenReply(this.lastReply) || 'There is nothing to repeat.';
this.lastReply = spoken;
@@ -59,6 +67,9 @@ export class JarvisDaemon extends EventEmitter {
this._speakReply(spoken);
}
async ask(text) {
if (this.locked) throw new Error('Jarvis is locked');
if (this._settingsReload) throw new Error('Settings are being applied; try again shortly');
const generation = ++this._askGeneration;
this.voiceLoop?.interrupt?.();
try {
this.voice.typedUtterance(); this.setState('THINKING');
@@ -66,17 +77,20 @@ export class JarvisDaemon extends EventEmitter {
const job = this.scheduler.run(() => this.harness.ask(text), { lane: 'voice' });
this._activeAsk = job;
const reply = await job;
if (generation !== this._askGeneration || this.locked) return '';
this.telemetry.record('llm', startedAt, { success: true });
const spoken = spokenReply(reply);
if (!spoken) { this.setState('LISTENING'); return ''; }
if (!spoken) { this._finishSpeech(); return ''; }
this._beginSpeech(); this.lastReply = spoken; this.emit('Reply', spoken); this._speakReply(spoken); return spoken;
} catch (error) {
if (generation !== this._askGeneration) return '';
this.telemetry.record('llm', Date.now(), { success: false }); this.emit('Error', 'QVAC', error.message); this.voice.cancel(); this.setState('ARMED'); throw error;
} finally {
this._activeAsk = null;
if (generation === this._askGeneration) this._activeAsk = null;
}
}
async resetContext() {
this._askGeneration += 1;
this.harness.cancel();
this.scheduler.cancelQueued((job) => job.lane === 'voice');
await this._activeAsk?.catch?.(() => {});
@@ -99,15 +113,18 @@ export class JarvisDaemon extends EventEmitter {
}
_finishSpeech() {
try { this.voice.finishSpeaking(); } catch {}
if (this.state === 'SPEAKING') this.setState('LISTENING');
if (this.settings.listeningMode !== 'conversation') { this.voice.cancel(); this.setState('ARMED'); }
else this.setState('LISTENING');
}
_speakReply(spoken) {
const generation = this._askGeneration;
if (!this.voiceLoop) {
this._finishSpeech();
return;
}
const play = async () => {
await this.ensureTts();
if (generation !== this._askGeneration || this.locked) return;
if (!this.voiceLoop?.status?.tts) {
const reason = this.voiceLoop?.status?.errors?.tts;
if (reason && reason !== 'Spoken replies are disabled') this.emit('Error', 'TTS', reason);
@@ -119,14 +136,15 @@ export class JarvisDaemon extends EventEmitter {
.catch((error) => {
this.emit('Error', 'TTS', error.message);
})
.finally(() => this._finishSpeech());
.finally(() => { if (generation === this._askGeneration) this._finishSpeech(); });
}
async ensureTts() {
await this._voiceStarting;
if (!this.voiceLoop) return;
const settings = voiceSettings();
if (!settings.ttsEnabled) return;
const settings = this.settings;
if (!settings.ttsEnabled) { this.voiceLoop.status.tts = false; return; }
if (this.voiceLoop.status.tts) return;
if (!this.voiceLoop.tts) this.voiceLoop.tts = new QvacVoiceAdapter({ role: 'tts' });
if (!this.voiceLoop.tts) this.voiceLoop.tts = new QvacVoiceAdapter({ role: 'tts', settings });
try {
await this.voiceLoop.tts.start();
this.voiceLoop.status.tts = true;
@@ -139,16 +157,26 @@ export class JarvisDaemon extends EventEmitter {
this.emit('Error', 'TTS', message);
}
}
cancel() { this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computer.revoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; }
cancel() { this._askGeneration += 1; this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computerRevoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); if (this.settings.computerMode === 'observe') return result; this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; }
computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
async startVoice() {
startVoice() {
if (this._voiceStarting) return this._voiceStarting;
this._voiceStarting = this._startVoice().finally(() => { this._voiceStarting = null; });
return this._voiceStarting;
}
async _startVoice() {
if (this.voiceLoop) return;
const settings = voiceSettings();
const asr = new QvacVoiceAdapter({ role: 'asr' });
const tts = settings.ttsEnabled ? new QvacVoiceAdapter({ role: 'tts' }) : null;
const settings = this.settings;
const asr = settings.microphoneEnabled ? new QvacVoiceAdapter({ role: 'asr', settings }) : null;
const tts = settings.ttsEnabled ? new QvacVoiceAdapter({ role: 'tts', settings }) : null;
if (!settings.ttsEnabled) console.log('jarvisd: voice: TTS disabled in config.json');
const loop = new VoiceLoop({ daemon: this, wake: createWakeEngine(settings), asr, tts });
const loop = new VoiceLoop({ daemon: this, wake: createWakeEngine(settings), asr, tts,
capture: new PipeWireCapture({ target: settings.inputTarget }),
playback: new PipeWirePlayback({ target: settings.outputTarget }),
vad: new VadSegmenter({ params: { threshold: settings.vadThreshold, minSpeechDurationMs: settings.vadMinSpeechMs, minSilenceDurationMs: settings.vadSilenceMs, maxSpeechDurationMs: settings.vadMaxSpeechSeconds * 1000 } }),
cooldownMs: settings.playbackCooldownMs, listeningMode: settings.listeningMode,
});
loop.on('error', (error) => { console.error(`jarvisd: voice: ${error.message}`); this.emit('Error', 'VOICE', error.message); });
this.voiceLoop = loop;
try { await loop.start(); this.emit('StateChanged', this.state); } catch (error) {
@@ -157,15 +185,56 @@ export class JarvisDaemon extends EventEmitter {
if (loop.status.tts) console.log('jarvisd: voice: TTS ready');
else if (loop.status.errors.tts) console.error(`jarvisd: voice: TTS: ${loop.status.errors.tts}`);
}
async reloadSettings() {
if (this._settingsReload) return this._settingsReload;
this._settingsReload = this._reloadSettings().finally(() => { this._settingsReload = null; });
return this._settingsReload;
}
async _reloadSettings() {
if (this._activeAsk) throw new Error('Wait for the current request to finish before applying settings');
const next = voiceSettings(null, { strict: true });
if (next.ttsEnabled) ttsConfiguration(next);
await this._voiceStarting;
const previous = this.settings;
this._askGeneration += 1;
await this.voiceLoop?.stop?.();
this.voiceLoop = null;
this.settings = next;
this.voice.idleMs = next.idleMinutes * 60_000;
if (['computerMode', 'computerSteps', 'computerGrantMinutes'].some(key => next[key] !== previous[key])) this.computerRevoke();
Object.assign(this.computer, { mode: next.computerMode, stepsMax: next.computerSteps, grantMinutes: next.computerGrantMinutes });
Object.assign(this.observer.normalizer, { maxLongEdge: next.screenshotMaxEdge, quality: next.screenshotQuality });
await this.startVoice();
this.voice.cancel(); this.setState('ARMED');
return JSON.stringify({ applied: true, voice: this.voiceLoop.status, environmentOverrides: ['JARVIS_TTS_MODEL', 'JARVIS_ASR_MODEL', 'JARVIS_WAKE_COMMAND', 'JARVIS_QVAC_MODEL'].filter(key => process.env[key]), restartRequired: ['chatModel', 'modelProfile', 'maxTurns', 'maxShellCalls', 'maxToolRounds'].filter(key => next[key] !== this.startupSettings[key]) });
}
async previewVoice(text) {
if (this.locked) throw new Error('Unlock the desktop to preview a voice');
if (this._activeAsk) throw new Error('Wait for the current request to finish before previewing');
await this._settingsReload;
await this.startVoice(); await this.ensureTts();
if (!this.voiceLoop?.status.tts) throw new Error(this.voiceLoop?.status.errors.tts || 'Enable spoken replies to preview a voice');
const preview = String(text || this.settings.previewText).slice(0, 1000);
const generation = this._askGeneration;
this._beginSpeech();
try { await this.voiceLoop.speak(preview); } finally { if (generation === this._askGeneration) this._finishSpeech(); }
}
stopSpeech() { this.voiceLoop?.interrupt?.(); this._finishSpeech(); }
setPushToTalk(pressed) { this.voiceLoop?.setPushToTalk(pressed); this.emit('PushToTalk', Boolean(pressed)); }
runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), computer: this.computer.status(), voice: this.voiceLoop ? { ...this.voiceLoop.metrics.snapshot(), ...this.voiceLoop.status } : null, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }
runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), settings: this.settings, computer: this.computer.status(), voice: this.voiceLoop ? { ...this.voiceLoop.metrics.snapshot(), ...this.voiceLoop.status } : null, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }
async assessModelFit(model) { return JSON.stringify(await callQvac('assessModelFit', { modelSrc: String(model) })); }
async downloadModel(model) { return JSON.stringify(await callQvac('downloadAsset', { modelSrc: String(model) })); }
async cancelModel(model) { return JSON.stringify(await cancelQvacRequest({ modelId: String(model) })); }
async wipeComputerTraces() { await this.audit.wipeTemp(); return true; }
handleLockScreen(locked) { this.locked = Boolean(locked); if (this.locked) { this.cancel(); this.setState('ARMED'); } this.emit('LockScreenChanged', this.locked); }
tickIdle() { if (!this.locked && this.voice.expireIdle() === 'SLEEPING' && this.state !== 'SLEEPING') this.sleep(); }
async close() { clearInterval(this._idleTimer); clearInterval(this._telemetryTimer); this.computerRevoke(); this.recovery.save({ state: 'ARMED', mode: this.mode }); await this.voiceLoop?.stop?.(); await this.harness.close(); }
async close() {
clearInterval(this._idleTimer); clearInterval(this._telemetryTimer);
this.cancel();
try { this.recovery.save({ state: 'ARMED', mode: this.mode }); }
catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); }
try { await this.voiceLoop?.stop?.(); } finally { await this.harness.close(); }
}
}
export async function startDaemon() {
+9 -4
View File
@@ -8,22 +8,27 @@ const MAX_LINE = 64 * 1024;
export async function createEventSocket({ socketPath, onMessage }) {
await mkdir(path.dirname(socketPath), { recursive: true });
try { await rm(socketPath, { force: true }); } catch {}
const clients = new Set();
const server = net.createServer((socket) => {
clients.add(socket);
socket.on('error', () => {});
socket.once('close', () => clients.delete(socket));
const invalid = () => { if (!socket.destroyed) socket.write(JSON.stringify({ error: 'invalid IPC message' }) + '\n'); };
let buffer = '';
socket.setEncoding('utf8');
socket.on('data', (chunk) => {
buffer += chunk;
if (buffer.length > MAX_LINE * 2) { socket.destroy(new Error('IPC buffer too large')); return; }
let index;
while ((index = buffer.indexOf('\n')) >= 0) {
const line = buffer.slice(0, index); buffer = buffer.slice(index + 1);
if (line.length > MAX_LINE) { socket.destroy(new Error('IPC message too large')); return; }
try { onMessage?.(JSON.parse(line), socket); } catch { socket.write(JSON.stringify({ error: 'invalid IPC message' }) + '\n'); }
if (Buffer.byteLength(line) > MAX_LINE) { socket.destroy(new Error('IPC message too large')); return; }
try { Promise.resolve(onMessage?.(JSON.parse(line), socket)).catch(invalid); } catch { invalid(); }
}
if (Buffer.byteLength(buffer) > MAX_LINE) socket.destroy(new Error('IPC message too large'));
});
});
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(socketPath, resolve); });
return { server, socketPath, close: () => new Promise((resolve) => server.close(() => fs.unlink(socketPath, () => resolve()))) };
return { server, socketPath, close: () => new Promise((resolve) => { for (const client of clients) client.destroy(); server.close(() => fs.unlink(socketPath, () => resolve())); }) };
}
export function sendEvent(socket, event, payload = {}) {
+1 -1
View File
@@ -1 +1 @@
export function assertLocalEndpoint(url) { const parsed = new URL(url); if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') throw new Error('only HTTP(S) model endpoints are supported'); if (!['127.0.0.1', 'localhost', '::1'].includes(parsed.hostname) && process.env.JARVIS_P2P_ENABLE !== '1') throw new Error('network access is disabled unless optional P2P/model fetch is explicitly enabled'); return parsed; }
export function assertLocalEndpoint(url) { const parsed = new URL(url); if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') throw new Error('only HTTP(S) model endpoints are supported'); if (!['127.0.0.1', 'localhost', '[::1]'].includes(parsed.hostname) && process.env.JARVIS_P2P_ENABLE !== '1') throw new Error('network access is disabled unless optional P2P/model fetch is explicitly enabled'); return parsed; }
+21 -27
View File
@@ -1,3 +1,5 @@
import { voiceSettings } from './voice-settings.js';
import { profile } from './model-profiles.js';
import { createRequire } from 'node:module';
import fs from 'node:fs';
import path from 'node:path';
@@ -17,7 +19,7 @@ const auxiliaryModels = new Map();
export const QVAC_MASTER = Object.freeze({
configPath: process.env.QVAC_CONFIG_PATH,
model: process.env.JARVIS_QVAC_MODEL || 'qwen3.5-4b',
model: process.env.JARVIS_QVAC_MODEL || voiceSettings().chatModel || profile(voiceSettings().modelProfile).model,
device: 'gpu',
gpuLayers: 99,
});
@@ -45,34 +47,26 @@ export function assertSdkVersion() {
export async function acquireQvac({ auxiliaryOnly = false } = {}) {
if (auxiliaryOnly) { await qvacSdk(); ownerCount += 1; return; }
ownerCount += 1;
if (!loadPromise) {
assertSdkVersion();
const resources = await Agent.engine.resources();
const gpuVisible = (resources.gpus?.length || 0) > 0 ||
Boolean(resources.drivers?.vulkan || resources.drivers?.cuda || resources.drivers?.opencl || resources.gpu);
if (!gpuVisible) {
ownerCount = Math.max(0, ownerCount - 1);
throw new Error('Jarvis requires a QVAC-visible GPU backend; run npm run gpu-doctor');
}
loadPromise = Agent.engine.load({
model: QVAC_MASTER.model,
tools: true,
device: 'gpu',
gpu_layers: QVAC_MASTER.gpuLayers,
mmprojUseGpu: true,
}).then((loaded) => {
if (loaded.device !== 'gpu') {
throw new Error(`Jarvis requires GPU QVAC inference; loaded device was ${loaded.device}`);
}
// Publish the promise before the first await so concurrent callers share
// resource discovery and loading. Count only successful acquisitions.
loadPromise = Promise.resolve().then(async () => {
assertSdkVersion();
const resources = await Agent.engine.resources();
const gpuVisible = (resources.gpus?.length || 0) > 0 ||
Boolean(resources.drivers?.vulkan || resources.drivers?.cuda || resources.drivers?.opencl || resources.gpu);
if (!gpuVisible) throw new Error('Jarvis requires a QVAC-visible GPU backend; run npm run gpu-doctor');
const loaded = await Agent.engine.load({
model: QVAC_MASTER.model, tools: true, device: 'gpu',
gpu_layers: QVAC_MASTER.gpuLayers, mmprojUseGpu: true,
});
if (loaded.device !== 'gpu') throw new Error(`Jarvis requires GPU QVAC inference; loaded device was ${loaded.device}`);
return loaded;
}).catch((error) => {
loadPromise = null;
ownerCount = Math.max(0, ownerCount - 1);
throw error;
});
}).catch((error) => { loadPromise = null; throw error; });
}
return loadPromise;
const loaded = await loadPromise;
ownerCount += 1;
return loaded;
}
export async function qvacSdk() {
@@ -92,7 +86,7 @@ function resolveSdkAsset(sdk, name) {
function resolveModelConfigAssets(sdk, config) {
const copy = { ...config };
for (const key of ['vadModelSrc', 'projectionModelSrc', 'vocabModelSrc']) {
for (const key of ['vadModelSrc', 'projectionModelSrc', 'vocabModelSrc', 's3genModelSrc']) {
if (typeof copy[key] === 'string') copy[key] = resolveSdkAsset(sdk, copy[key]);
}
return copy;
+34
View File
@@ -0,0 +1,34 @@
// Model configuration fields are checked against the installed QVAC 0.19 SDK.
export const TTS_PRESETS = Object.freeze({
'supertonic-en': { model: 'TTS_EN_SUPERTONIC_Q8_0', engine: 'supertonic', sampleRate: 44100 },
supertonic3: { model: 'TTS_MULTILINGUAL_SUPERTONIC3_Q8_0', engine: 'supertonic', sampleRate: 44100 },
chatterbox: { model: 'TTS_T3_TURBO_EN_CHATTERBOX_Q8_0', engine: 'chatterbox', sampleRate: 24000 },
parler: { model: 'TTS_MINI_V1_EN_PARLER_TTS_Q8_0', engine: 'parler', sampleRate: 44100 },
});
export function ttsConfiguration(settings) {
const preset = TTS_PRESETS[settings.ttsPreset];
if (!preset) throw new Error('Unknown speech model preset');
const config = { ttsEngine: preset.engine, useGPU: settings.ttsUseGpu };
if (preset.engine === 'supertonic') {
Object.assign(config, { language: settings.ttsPreset === 'supertonic3' ? settings.ttsLanguage : 'en', voice: settings.voiceId, ttsSpeed: settings.ttsSpeed, ttsNumInferenceSteps: settings.ttsSteps });
} else {
Object.assign(config, { threads: settings.ttsThreads, seed: settings.ttsSeed });
if (preset.engine === 'chatterbox') {
Object.assign(config, { language: 'en', s3genModelSrc: 'TTS_S3GEN_EN_CHATTERBOX', cfmSteps: settings.ttsCfmSteps });
if (settings.ttsReferenceAudio) config.referenceAudioSrc = settings.ttsReferenceAudio;
} else {
if (!settings.ttsDescription) throw new Error('Describe the Parler voice before applying settings');
Object.assign(config, { description: settings.ttsDescription, temperature: settings.ttsTemperature });
}
}
return { model: settings.ttsModel || preset.model, sampleRate: preset.sampleRate, config };
}
export function scaleSpeech(samples, volume = 100) {
if (volume === 100) return samples;
const output = new Int16Array(samples.length);
const gain = Math.max(0, Math.min(100, volume)) / 100;
for (let i = 0; i < samples.length; i++) output[i] = Math.round(samples[i] * gain);
return output;
}
+5 -4
View File
@@ -20,7 +20,7 @@ export class VadSegmenter extends EventEmitter {
this.reset();
}
reset() { this.speaking = false; this.buffer = []; this.speechMs = 0; this.silenceMs = 0; }
reset() { this.speaking = false; this.buffer = []; this.speechMs = 0; this.silenceMs = 0; this.recordingMs = 0; }
push(frame) {
const chunk = Buffer.from(frame || '');
@@ -34,11 +34,12 @@ export class VadSegmenter extends EventEmitter {
}
if (!this.speaking) return;
this.buffer.push(chunk);
this.recordingMs += durationMs;
if (voiced) { this.speechMs += durationMs; this.silenceMs = 0; }
else { this.silenceMs += durationMs; }
if (this.speechMs >= this.params.maxSpeechDurationMs ||
(this.speechMs >= this.params.minSpeechDurationMs && this.silenceMs >= this.params.minSilenceDurationMs)) {
this.end();
if (this.recordingMs >= this.params.maxSpeechDurationMs || this.silenceMs >= this.params.minSilenceDurationMs) {
if (this.speechMs >= this.params.minSpeechDurationMs) this.end();
else this.reset();
}
}
+12 -5
View File
@@ -1,3 +1,5 @@
import { voiceSettings } from './voice-settings.js';
import { ttsConfiguration, scaleSpeech } from './tts-config.js';
import { EventEmitter } from 'node:events';
import { acquireQvac, releaseQvac, loadAuxiliaryModel, unloadAuxiliaryModel, qvacSdk, withQvacMaster, assertSdkVersion } from './qvac-master.js';
@@ -20,8 +22,11 @@ function toUint8(samples) {
}
export class QvacVoiceAdapter extends EventEmitter {
constructor({ role = 'both', asrModel = process.env.JARVIS_ASR_MODEL || 'WHISPER_TINY', ttsModel = process.env.JARVIS_TTS_MODEL || 'TTS_EN_SUPERTONIC_Q8_0' } = {}) {
super(); this.role = role; this.asrModel = asrModel; this.ttsModel = ttsModel; this.asrId = null; this.ttsId = null; this.asrSession = null; this.acquired = false;
constructor({ role = 'both', settings = voiceSettings(), asrModel = settings.asrModel, ttsModel } = {}) {
super(); this.role = role; this.settings = settings;
this.ttsConfig = role === 'asr' ? null : ttsConfiguration(settings);
this.asrModel = asrModel; this.ttsModel = ttsModel || this.ttsConfig?.model;
this.asrId = null; this.ttsId = null; this.asrSession = null; this.acquired = false;
}
async start() {
@@ -29,8 +34,8 @@ export class QvacVoiceAdapter extends EventEmitter {
assertSdkVersion();
await acquireQvac({ auxiliaryOnly: true }); this.acquired = true;
try {
if (this.role !== 'tts') this.asrId = await loadAuxiliaryModel(this.asrModel, { vadModelSrc: 'VAD_SILERO_5_1_2', audio_format: 's16le', language: 'en', no_timestamps: true, vad_params: { threshold: 0.6, min_speech_duration_ms: 300, min_silence_duration_ms: 700, max_speech_duration_s: 15, speech_pad_ms: 200 } }, 'whispercpp-transcription');
if (this.role !== 'asr') this.ttsId = await loadAuxiliaryModel(this.ttsModel, { ttsEngine: 'supertonic', language: 'en', voice: 'F1', ttsSpeed: 1.05, ttsNumInferenceSteps: 5 }, 'tts-ggml');
if (this.role !== 'tts') this.asrId = await loadAuxiliaryModel(this.asrModel, { vadModelSrc: 'VAD_SILERO_5_1_2', audio_format: 's16le', language: this.settings.asrLanguage, no_timestamps: true, vad_params: { threshold: this.settings.vadThreshold, min_speech_duration_ms: this.settings.vadMinSpeechMs, min_silence_duration_ms: this.settings.vadSilenceMs, max_speech_duration_s: this.settings.vadMaxSpeechSeconds, speech_pad_ms: 200 } }, 'whispercpp-transcription');
if (this.role !== 'asr') this.ttsId = await loadAuxiliaryModel(this.ttsModel, this.ttsConfig.config, 'tts-ggml');
} catch (error) { await this.stop(); throw error; }
}
@@ -58,7 +63,9 @@ export class QvacVoiceAdapter extends EventEmitter {
}
return result;
});
return pcmS16le(samples, 44_100);
const audio = pcmS16le(samples, this.ttsConfig.sampleRate);
audio.samples = scaleSpeech(audio.samples, this.settings.ttsVolume);
return audio;
}
async stop() {
+9 -2
View File
@@ -1,3 +1,5 @@
import { voiceSettings } from './voice-settings.js';
import { ttsConfiguration } from './tts-config.js';
import { access } from 'node:fs/promises';
const commandAvailable = async (command) => {
@@ -7,13 +9,18 @@ const commandAvailable = async (command) => {
return false;
};
const pipewire = await commandAvailable('pw-cat');
const settings = voiceSettings();
const report = {
pipewireCapture: pipewire,
wakeEngine: Boolean(process.env.JARVIS_WAKE_COMMAND),
wakeCommand: process.env.JARVIS_WAKE_COMMAND || null,
wakeEngine: settings.listeningMode !== 'ptt' && Boolean(settings.command),
wakeCommand: settings.command || null,
sampleRate: 16000,
channels: 1,
ttsPlayback: pipewire,
microphoneEnabled: settings.microphoneEnabled,
inputTarget: settings.inputTarget || 'system default',
outputTarget: settings.outputTarget || 'system default',
speech: settings.ttsEnabled ? ttsConfiguration(settings) : 'disabled',
gpuRequired: true,
};
console.log(JSON.stringify(report, null, 2));
+65 -16
View File
@@ -20,18 +20,23 @@ const FAST_COMMANDS = new Map([
export function fastCommand(text) { return FAST_COMMANDS.get(String(text || '').trim().toLowerCase()) || null; }
export class VoiceLoop extends EventEmitter {
constructor({ daemon, capture = new PipeWireCapture(), playback = new PipeWirePlayback(), wake = new WakeEngine(), vad = new VadSegmenter(), asr, tts, cooldownMs = POST_PLAYBACK_COOLDOWN_MS, now = () => Date.now() } = {}) {
super(); this.daemon = daemon; this.capture = capture; this.playback = playback; this.wake = wake; this.vad = vad; this.asr = asr; this.tts = tts; this.cooldownMs = cooldownMs; this.now = now;
this.isSpeaking = false; this.cooldownUntil = 0; this.running = false; this.ptt = false; this._generation = 0; this._speechQueue = Promise.resolve(); this.metrics = new VoiceMetrics();
constructor({ daemon, capture = new PipeWireCapture(), playback = new PipeWirePlayback(), wake = new WakeEngine(), vad = new VadSegmenter(), asr, tts, cooldownMs = POST_PLAYBACK_COOLDOWN_MS, listeningMode = 'conversation', now = () => Date.now(), captureRetryMs = 2000, asrRetryMs = 4000 } = {}) {
super(); this.daemon = daemon; this.capture = capture; this.playback = playback; this.wake = wake; this.vad = vad; this.asr = asr; this.tts = tts; this.cooldownMs = cooldownMs; this.listeningMode = listeningMode; this.now = now;
this.captureRetryMs = captureRetryMs; this.asrRetryMs = asrRetryMs;
this.isSpeaking = false; this.cooldownUntil = 0; this.running = false; this.ptt = false; this._generation = 0; this._speechQueue = Promise.resolve(); this.metrics = new VoiceMetrics(); this._transcriptions = new Set();
capture.on('audio', (chunk) => this.pushAudio(chunk));
capture.on('error', (error) => { this.status.capture = false; this.status.errors.capture = error.message; this.emit('error', error); });
capture.on('close', () => { this.status.capture = false; if (this.running) { this.status.errors.capture = 'Microphone stream closed'; this.emit('error', new Error('Microphone stream closed')); } });
capture.on('error', (error) => { this.status.capture = false; this.status.errors.capture = error.message; this.emit('error', error); this._scheduleCaptureRetry(); });
capture.on('close', () => { this.status.capture = false; if (this.running) { this.status.errors.capture = 'Microphone stream closed'; this._scheduleCaptureRetry(); } });
wake.on('unavailable', () => { this.status.wake = false; });
wake.on('error', (error) => { this.status.wake = false; this.emit('error', error); });
this.status = { asr: false, tts: false, capture: false, wake: false, errors: {} };
wake.on('wake', (phrase) => this.wakeHeard(phrase));
vad.on('level', (rms) => daemon?.emit('ListeningLevel', rms));
vad.on('utterance', (audio) => this.transcribe(audio).catch((error) => this.emit('error', error)));
vad.on('utterance', (audio) => {
const task = this.transcribe(audio).catch((error) => this.emit('error', error));
this._transcriptions.add(task);
task.finally(() => this._transcriptions.delete(task)).catch(() => {});
});
}
async start() {
@@ -48,20 +53,62 @@ export class VoiceLoop extends EventEmitter {
}
this.running = true;
if (this.status.asr) {
try { this.capture.start(); this.status.capture = true; }
catch (error) { this.status.errors.capture = error.message; this.emit('error', error); }
try { this.wake.start?.(); this.wake.resume(); this.status.wake = Boolean(this.wake.command || this.wake.detect); }
catch (error) { this.status.errors.wake = error.message; this.emit('error', error); }
this._armCapture();
this._armWake();
} else if (this.asr) {
this._scheduleAsrRetry();
}
}
async stop() { this.running = false; this.capture.stop(); this.wake.close(); this.vad.reset(); this.playback.stop(); await this.asr?.stop?.(); if (this.tts !== this.asr) await this.tts?.stop?.(); }
_notifyStatus() { try { this.daemon?.emit?.('StateChanged', this.daemon.state || 'ARMED'); } catch {} }
_armWake() {
try { if (this.listeningMode !== 'ptt') this.wake.start?.(); this.wake.resume(); this.status.wake = this.listeningMode !== 'ptt' && Boolean(this.wake.command || this.wake.detect); }
catch (error) { this.status.errors.wake = error.message; this.emit('error', error); }
}
_armCapture() {
if (!this.running || !this.status.asr || this.status.capture) return;
try { this.capture.start(); this.status.capture = true; delete this.status.errors.capture; this._notifyStatus(); }
catch (error) { this.status.errors.capture = error.message; this.emit('error', error); this._scheduleCaptureRetry(); }
}
_scheduleCaptureRetry() {
if (!this.running || this._captureRetry || this.status.capture || !this.status.asr) return;
this._captureRetry = setTimeout(() => {
this._captureRetry = 0;
this._armCapture();
}, this.captureRetryMs);
this._captureRetry.unref?.();
}
_scheduleAsrRetry() {
if (!this.running || this._asrRetry || this.status.asr || !this.asr) return;
this._asrRetry = setTimeout(() => {
this._asrRetry = 0;
if (!this.running || this.status.asr || !this.asr) return;
this.asr.start().then(() => {
if (!this.running || this.status.asr) return;
this.status.asr = true;
delete this.status.errors.asr;
this._armCapture();
this._armWake();
this._notifyStatus();
}).catch((error) => {
this.status.errors.asr = String(error?.message || error);
this._scheduleAsrRetry();
});
}, this.asrRetryMs);
this._asrRetry.unref?.();
}
async stop() {
this.running = false;
if (this._captureRetry) { clearTimeout(this._captureRetry); this._captureRetry = 0; }
if (this._asrRetry) { clearTimeout(this._asrRetry); this._asrRetry = 0; }
this.interrupt(); this.capture.stop(); this.wake.close(); this.vad.reset(); this.playback.stop(); await Promise.allSettled([this._speechQueue, ...this._transcriptions]); await this.asr?.stop?.(); if (this.tts !== this.asr) await this.tts?.stop?.();
}
setPushToTalk(pressed) { this.ptt = Boolean(pressed) && this.status.asr && this.status.capture; if (this.ptt) this.wakeHeard('push-to-talk'); else this.vad.end(); }
pushAudio(chunk) {
if (!this.running || this.daemon?.locked || this.daemon?.state === 'SLEEPING') return;
if (this.isSpeaking || this.now() < this.cooldownUntil) { this.metrics.feedbackDrop(); return; }
try { this.daemon?.emit('ListeningLevel', Math.min(1, Math.max(0, chunk?.length ? 0.01 : 0))); } catch {}
if (!this.ptt) this.wake.push(chunk);
if (this.ptt || this.daemon?.state === 'LISTENING') this.vad.push(chunk);
if (!this.ptt && this.listeningMode !== 'ptt') this.wake.push(chunk);
if (this.ptt || (this.listeningMode !== 'ptt' && this.daemon?.state === 'LISTENING')) this.vad.push(chunk);
}
wakeHeard(phrase) {
if (!this.running || this.daemon?.locked) return;
@@ -69,8 +116,9 @@ export class VoiceLoop extends EventEmitter {
}
async transcribe(audio) {
if (!this.status.asr || !this.asr?.transcribeAudio) return;
const generation = this._generation;
const text = await this.asr.transcribeAudio(audio).catch((error) => { this.emit('error', error); return ''; });
if (!this.running || this.daemon?.locked) return;
if (!this.running || this.daemon?.locked || generation !== this._generation) return;
if (!isMeaningfulTranscript(text)) { this.metrics.wakeRejected(); return; }
this.metrics.utterance();
this.daemon?.emit('PartialTranscript', text); this.daemon?.emit('FinalTranscript', text);
@@ -93,7 +141,8 @@ export class VoiceLoop extends EventEmitter {
this.wake.resume();
if (this.daemon?.state === 'SPEAKING') {
try { this.daemon.voice?.finishSpeaking?.(); } catch {}
this.daemon.setState?.('LISTENING');
if (this.daemon._finishSpeech) this.daemon._finishSpeech();
else this.daemon.setState?.('LISTENING');
}
}
async speak(text) {
@@ -127,7 +176,7 @@ export class VoiceLoop extends EventEmitter {
try {
const audio = await this.tts.speak(speakableForTts(text));
if (generation !== this._generation) return;
await this.playback.play(audio.samples);
await this.playback.play(audio.samples, audio.sampleRate);
try { this.daemon?.emit('SpeakingLevel', 0); } catch {}
} finally {
this.isSpeaking = false;
+16 -17
View File
@@ -1,25 +1,24 @@
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import { normalizeSettings } from '../apps/gnome-extension/[email protected]/settings-values.js';
function flag(value, fallback = true) {
if (value === undefined || value === null) return fallback;
if (typeof value === 'string') return !['false', '0', 'off', 'no'].includes(value.trim().toLowerCase());
return value !== false;
export const SETTINGS_FIELDS = JSON.parse(fs.readFileSync(new URL('../apps/gnome-extension/[email protected]/settings-catalog.json', import.meta.url).pathname, 'utf8')).fields;
export const configPath = () => path.join(process.env.XDG_CONFIG_HOME || path.join(os.homedir(), '.config'), 'jarvis/config.json');
export function readSettings({ strict = false } = {}) {
try {
const config = JSON.parse(fs.readFileSync(configPath(), 'utf8'));
if (!config || typeof config !== 'object' || Array.isArray(config)) throw new Error('Settings must be a JSON object');
return config;
} catch (error) { if (strict && error.code !== 'ENOENT') throw error; return {}; }
}
export function voiceSettings(source = null) {
let config = source;
if (!config) {
config = {};
try { config = JSON.parse(fs.readFileSync(path.join(process.env.XDG_CONFIG_HOME || path.join(os.homedir(), '.config'), 'jarvis/config.json'), 'utf8')); } catch {}
}
const phrase = config.wakePhrase || config.wake_phrase || 'hey jarvis';
const aliases = Array.isArray(config.aliases) ? config.aliases : ['jarvis', 'okay jarvis'];
export function voiceSettings(source = null, { strict = false } = {}) {
const settings = normalizeSettings(source ?? readSettings({ strict }), SETTINGS_FIELDS, { strict });
return {
command: process.env.JARVIS_WAKE_COMMAND || config.wakeCommand || config.wake_command || '',
phrases: [phrase, ...aliases].map((item) => String(item || '').trim()).filter(Boolean),
ttsEnabled: flag(config.ttsEnabled ?? config.tts_enabled, true),
modelProfile: config.modelProfile || config.model_profile || 'laptop-16gb',
...settings,
command: process.env.JARVIS_WAKE_COMMAND || settings.wakeCommand,
phrases: [settings.wakePhrase, ...settings.aliases].map(item => item.trim()).filter(Boolean),
asrModel: process.env.JARVIS_ASR_MODEL || settings.asrModel,
ttsModel: process.env.JARVIS_TTS_MODEL || settings.ttsModel,
};
}
+1 -1
View File
@@ -48,7 +48,7 @@ export class VoiceStateMachine {
}
expireIdle() {
if (this.state !== 'SLEEPING' && this.now() - this.lastActivity >= this.idleMs) this.sleep();
if (['ARMED', 'LISTENING'].includes(this.state) && this.now() - this.lastActivity >= this.idleMs) this.sleep();
return this.state;
}
}