Updates
Rolling release / release (push) Successful in 7m21s

This commit is contained in:
2026-09-12 09:40:42 -04:00
parent a4073b9020
commit f26204505e
23 changed files with 531 additions and 114 deletions
+13 -3
View File
@@ -1,24 +1,34 @@
import { voiceSettings } from './voice-settings.js';
import { EventEmitter } from 'node:events';
import os from 'node:os';
import { acquireQvac, closeQvac, releaseQvac, Agent, QVAC_MASTER } from './qvac-master.js';
import { createRuntimeTools } from '../skills/runtime-tools.js';
import { createPhase2Tools } from '../skills/phase2-tools.js';
import { createPhase2Tools, filesystemRoots } from '../skills/phase2-tools.js';
import { createQvacTools } from '../skills/qvac-tools.js';
import { VOICE_SYSTEM_PROMPT, parseHudSidecar } from '../skills/voice-prompt.js';
import { createComputerObserveTools } from '../skills/computer-observe.js';
import { createComputerActTools } from '../skills/computer-act.js';
import { createPhase9GatewayTool } from '../skills/phase9-tools.js';
export function harnessRoots(fsAccess) {
if (fsAccess === 'filesystem') return ['/'];
if (fsAccess === 'home') return [os.homedir()];
return [];
}
export class HarnessBridge extends EventEmitter {
constructor({ cwd = process.cwd(), model = QVAC_MASTER.model, tools = [], computer, observer, actuator, permissionMode = 'ask' } = {}) {
constructor({ cwd = process.cwd(), model = QVAC_MASTER.model, tools = [], computer, observer, actuator, permissionMode = 'ask', fsAccess } = {}) {
super();
const settings = voiceSettings();
const access = fsAccess ?? settings.fsAccess;
const roots = harnessRoots(access);
this.options = {
cwd,
roots,
model,
tools: [
...createRuntimeTools({ computer }),
...createPhase2Tools({ cwd, computer }),
...createPhase2Tools({ cwd, computer, roots: filesystemRoots(access, cwd) }),
...createComputerObserveTools({ computer, observer }),
...createComputerActTools({ actuator }),
...createQvacTools(),
+44 -15
View File
@@ -8,7 +8,7 @@ import { HarnessBridge } from './harness-bridge.js';
import { ComputerUseSession } from '../computer-use/session.js';
import { VoiceStateMachine } from './voice-state.js';
import { QvacScheduler } from './qvac-scheduler.js';
import { cancelQvac, resumeQvac, suspendQvac, callQvac, cancelQvacRequest, qvacStatus } from './qvac-master.js';
import { cancelQvac, closeQvac, resumeQvac, suspendQvac, callQvac, cancelQvacRequest, qvacStatus } from './qvac-master.js';
import { PrivacyLog } from './privacy-log.js';
import { VoiceLoop } from './voice-loop.js';
import { QvacVoiceAdapter } from './voice-adapters.js';
@@ -37,7 +37,7 @@ export class JarvisDaemon extends EventEmitter {
this.perception = new QvacPerception();
this.observer = new DesktopObserver({ normalizer: new FrameNormalizer({ maxLongEdge: this.settings.screenshotMaxEdge, quality: this.settings.screenshotQuality }), ocr: (image) => this.perception.ocr(image) });
this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit });
this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer, actuator: this.actuator });
this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer, actuator: this.actuator, fsAccess: this.settings.fsAccess });
this.log = new PrivacyLog();
this.locked = false;
this.lastReply = '';
@@ -57,8 +57,32 @@ export class JarvisDaemon extends EventEmitter {
}
setState(state) { this.state = state; try { this.recovery.save({ state, mode: this.mode }); } catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); } this.emit('StateChanged', state); this.log.record('state', { state }).catch(() => {}); }
async arm() { if (this.locked) return; await resumeQvac().catch(() => {}); this.voice.wake(); this.setState('LISTENING'); }
async sleep() { this.cancel(); this.voice.sleep(); this.setState('SLEEPING'); await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); }
async arm() {
if (this.locked) return;
await resumeQvac().catch(() => {});
await this.ensureAsr();
this.voice.wake();
this.setState('LISTENING');
}
async sleep() {
if (this._activeAsk) {
await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message));
return;
}
this.cancel();
this.voice.sleep();
this.setState('SLEEPING');
if (this.settings.freeVramOnIdle !== false) await this.parkModels();
else await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message));
}
async parkModels() {
try { await this.voiceLoop?.parkModels?.(); }
catch (error) { this.emit('Error', 'VOICE_UNLOAD', error.message); }
try { await this.harness.resetContext(); }
catch (error) { this.emit('Error', 'QVAC_UNLOAD', error.message); }
try { await closeQvac(); }
catch (error) { this.emit('Error', 'QVAC_CLOSE', error.message); }
}
say(text) {
const spoken = spokenReply(text) || spokenReply(this.lastReply) || 'There is nothing to repeat.';
this.lastReply = spoken;
@@ -145,18 +169,21 @@ export class JarvisDaemon extends EventEmitter {
if (!settings.ttsEnabled) { this.voiceLoop.status.tts = false; return; }
if (this.voiceLoop.status.tts) return;
if (!this.voiceLoop.tts) this.voiceLoop.tts = new QvacVoiceAdapter({ role: 'tts', settings });
try {
await this.voiceLoop.tts.start();
this.voiceLoop.status.tts = true;
delete this.voiceLoop.status.errors.tts;
console.log('jarvisd: voice: TTS ready');
} catch (error) {
const message = String(error?.message || error);
this.voiceLoop.status.errors.tts = message;
const ready = await this.voiceLoop.ensureTts();
if (ready) console.log('jarvisd: voice: TTS ready');
else if (this.voiceLoop.status.errors.tts) {
const message = this.voiceLoop.status.errors.tts;
console.error(`jarvisd: voice: TTS: ${message}`);
this.emit('Error', 'TTS', message);
}
}
async ensureAsr() {
await this._voiceStarting;
if (!this.voiceLoop || !this.settings.microphoneEnabled) return;
if (this.voiceLoop.status?.asr) return;
if (!this.voiceLoop.asr) this.voiceLoop.asr = new QvacVoiceAdapter({ role: 'asr', settings: this.settings });
await this.voiceLoop.ensureAsr?.();
}
cancel() { this._askGeneration += 1; this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computerRevoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); if (this.settings.computerMode === 'observe') return result; this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; }
computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
@@ -182,8 +209,10 @@ export class JarvisDaemon extends EventEmitter {
try { await loop.start(); this.emit('StateChanged', this.state); } catch (error) {
this.voiceLoop = null; await loop.stop?.().catch(() => {}); this.emit('Error', 'VOICE_UNAVAILABLE', error.message); throw error;
}
if (loop.status.tts) console.log('jarvisd: voice: TTS ready');
else if (loop.status.errors.tts) console.error(`jarvisd: voice: TTS: ${loop.status.errors.tts}`);
if (loop.status.wake) console.log('jarvisd: voice: wake detector ready');
else if (loop.status.errors.wake) console.error(`jarvisd: voice: wake: ${loop.status.errors.wake}`);
if (loop.status.capture) console.log('jarvisd: voice: microphone capture ready');
else if (loop.status.errors.capture) console.error(`jarvisd: voice: capture: ${loop.status.errors.capture}`);
}
async reloadSettings() {
if (this._settingsReload) return this._settingsReload;
@@ -206,7 +235,7 @@ export class JarvisDaemon extends EventEmitter {
Object.assign(this.observer.normalizer, { maxLongEdge: next.screenshotMaxEdge, quality: next.screenshotQuality });
await this.startVoice();
this.voice.cancel(); this.setState('ARMED');
return JSON.stringify({ applied: true, voice: this.voiceLoop.status, environmentOverrides: ['JARVIS_TTS_MODEL', 'JARVIS_ASR_MODEL', 'JARVIS_WAKE_COMMAND', 'JARVIS_QVAC_MODEL'].filter(key => process.env[key]), restartRequired: ['chatModel', 'modelProfile', 'maxTurns', 'maxShellCalls', 'maxToolRounds'].filter(key => next[key] !== this.startupSettings[key]) });
return JSON.stringify({ applied: true, voice: this.voiceLoop.status, environmentOverrides: ['JARVIS_TTS_MODEL', 'JARVIS_ASR_MODEL', 'JARVIS_WAKE_COMMAND', 'JARVIS_QVAC_MODEL'].filter(key => process.env[key]), restartRequired: ['chatModel', 'modelProfile', 'maxTurns', 'maxShellCalls', 'maxToolRounds', 'fsAccess'].filter(key => next[key] !== this.startupSettings[key]) });
}
async previewVoice(text) {
if (this.locked) throw new Error('Unlock the desktop to preview a voice');
+2
View File
@@ -1,5 +1,6 @@
import { voiceSettings } from './voice-settings.js';
import { ttsConfiguration } from './tts-config.js';
import { isBuiltinWakeCommand } from './wake-engine.js';
import { access } from 'node:fs/promises';
const commandAvailable = async (command) => {
@@ -14,6 +15,7 @@ const report = {
pipewireCapture: pipewire,
wakeEngine: settings.listeningMode !== 'ptt' && Boolean(settings.command),
wakeCommand: settings.command || null,
wakeBuiltin: settings.listeningMode !== 'ptt' && isBuiltinWakeCommand(settings.command),
sampleRate: 16000,
channels: 1,
ttsPlayback: pipewire,
+82 -31
View File
@@ -29,7 +29,7 @@ export class VoiceLoop extends EventEmitter {
capture.on('close', () => { this.status.capture = false; if (this.running) { this.status.errors.capture = 'Microphone stream closed'; this._scheduleCaptureRetry(); } });
wake.on('unavailable', () => { this.status.wake = false; });
wake.on('error', (error) => { this.status.wake = false; this.emit('error', error); });
this.status = { asr: false, tts: false, capture: false, wake: false, errors: {} };
this.status = { asr: false, tts: false, capture: false, wake: false, microphone: this.asr != null, speech: this.tts != null, errors: {} };
wake.on('wake', (phrase) => this.wakeHeard(phrase));
vad.on('level', (rms) => daemon?.emit('ListeningLevel', rms));
vad.on('utterance', (audio) => {
@@ -41,23 +41,10 @@ export class VoiceLoop extends EventEmitter {
async start() {
if (this.running) return;
for (const [name, adapter] of [['tts', this.tts], ['asr', this.asr]]) {
try {
if (adapter) { await adapter.start?.(); this.status[name] = true; }
else if (name === 'tts') this.status.errors.tts = 'Spoken replies are disabled';
} catch (error) {
const message = String(error?.message || error);
this.status.errors[name] = message;
this.emit('error', new Error(`${name.toUpperCase()}: ${message}`));
}
}
if (!this.tts) this.status.errors.tts = 'Spoken replies are disabled';
this.running = true;
if (this.status.asr) {
this._armCapture();
this._armWake();
} else if (this.asr) {
this._scheduleAsrRetry();
}
this._armWake();
this._armCapture();
}
_notifyStatus() { try { this.daemon?.emit?.('StateChanged', this.daemon.state || 'ARMED'); } catch {} }
_armWake() {
@@ -65,12 +52,12 @@ export class VoiceLoop extends EventEmitter {
catch (error) { this.status.errors.wake = error.message; this.emit('error', error); }
}
_armCapture() {
if (!this.running || !this.status.asr || this.status.capture) return;
if (!this.running || this.status.capture || this.asr == null) return;
try { this.capture.start(); this.status.capture = true; delete this.status.errors.capture; this._notifyStatus(); }
catch (error) { this.status.errors.capture = error.message; this.emit('error', error); this._scheduleCaptureRetry(); }
}
_scheduleCaptureRetry() {
if (!this.running || this._captureRetry || this.status.capture || !this.status.asr) return;
if (!this.running || this._captureRetry || this.status.capture || this.asr == null) return;
this._captureRetry = setTimeout(() => {
this._captureRetry = 0;
this._armCapture();
@@ -82,19 +69,65 @@ export class VoiceLoop extends EventEmitter {
this._asrRetry = setTimeout(() => {
this._asrRetry = 0;
if (!this.running || this.status.asr || !this.asr) return;
this.asr.start().then(() => {
if (!this.running || this.status.asr) return;
this.ensureAsr().catch(() => {});
}, this.asrRetryMs);
this._asrRetry.unref?.();
}
async ensureAsr() {
if (!this.asr) return false;
if (this.status.asr) return true;
if (this._asrStarting) return this._asrStarting;
this._asrStarting = (async () => {
try {
await this.asr.start();
if (!this.running) return false;
this.status.asr = true;
delete this.status.errors.asr;
this._armCapture();
this._armWake();
this._notifyStatus();
}).catch((error) => {
this.status.errors.asr = String(error?.message || error);
return true;
} catch (error) {
const message = String(error?.message || error);
this.status.errors.asr = message;
this.emit('error', new Error(`ASR: ${message}`));
this._scheduleAsrRetry();
});
}, this.asrRetryMs);
this._asrRetry.unref?.();
return false;
} finally {
this._asrStarting = null;
}
})();
return this._asrStarting;
}
async ensureTts() {
if (!this.tts) { this.status.errors.tts = 'Spoken replies are disabled'; return false; }
if (this.status.tts) return true;
if (this._ttsStarting) return this._ttsStarting;
this._ttsStarting = (async () => {
try {
await this.tts.start();
this.status.tts = true;
delete this.status.errors.tts;
return true;
} catch (error) {
const message = String(error?.message || error);
this.status.errors.tts = message;
this.emit('error', new Error(`TTS: ${message}`));
return false;
} finally {
this._ttsStarting = null;
}
})();
return this._ttsStarting;
}
async parkModels() {
if (this._asrRetry) { clearTimeout(this._asrRetry); this._asrRetry = 0; }
this._asrStarting = null;
this._ttsStarting = null;
await this.asr?.stop?.();
if (this.tts && this.tts !== this.asr) await this.tts.stop?.();
this.status.asr = false;
this.status.tts = false;
}
async stop() {
this.running = false;
@@ -102,17 +135,34 @@ export class VoiceLoop extends EventEmitter {
if (this._asrRetry) { clearTimeout(this._asrRetry); this._asrRetry = 0; }
this.interrupt(); this.capture.stop(); this.wake.close(); this.vad.reset(); this.playback.stop(); await Promise.allSettled([this._speechQueue, ...this._transcriptions]); await this.asr?.stop?.(); if (this.tts !== this.asr) await this.tts?.stop?.();
}
setPushToTalk(pressed) { this.ptt = Boolean(pressed) && this.status.asr && this.status.capture; if (this.ptt) this.wakeHeard('push-to-talk'); else this.vad.end(); }
setPushToTalk(pressed) {
if (!pressed) {
this._pttHeld = false;
this.ptt = false;
this.vad.end();
return Promise.resolve();
}
this._pttHeld = true;
return this.ensureAsr().then((ok) => {
if (!this._pttHeld) return;
this.ptt = Boolean(ok && this.status.capture);
if (this.ptt) this.wakeHeard('push-to-talk');
}).catch((error) => this.emit('error', error));
}
pushAudio(chunk) {
if (!this.running || this.daemon?.locked || this.daemon?.state === 'SLEEPING') return;
if (this.isSpeaking || this.now() < this.cooldownUntil) { this.metrics.feedbackDrop(); return; }
try { this.daemon?.emit('ListeningLevel', Math.min(1, Math.max(0, chunk?.length ? 0.01 : 0))); } catch {}
if (!this.running || this.daemon?.locked) return;
const sleeping = this.daemon?.state === 'SLEEPING';
if (!sleeping && (this.isSpeaking || this.now() < this.cooldownUntil)) { this.metrics.feedbackDrop(); return; }
if (!sleeping) {
try { this.daemon?.emit('ListeningLevel', Math.min(1, Math.max(0, chunk?.length ? 0.01 : 0))); } catch {}
}
if (!this.ptt && this.listeningMode !== 'ptt') this.wake.push(chunk);
if (sleeping) return;
if (this.ptt || (this.listeningMode !== 'ptt' && this.daemon?.state === 'LISTENING')) this.vad.push(chunk);
}
wakeHeard(phrase) {
if (!this.running || this.daemon?.locked) return;
this.metrics.wakeAccepted(); this.daemon?.emit('WakeHeard', phrase); this.daemon?.arm?.(); this.emit('wake', phrase);
this.metrics.wakeAccepted(); this.daemon?.emit('WakeHeard', phrase); this.ensureAsr().catch((error) => this.emit('error', error)); this.daemon?.arm?.(); this.emit('wake', phrase);
}
async transcribe(audio) {
if (!this.status.asr || !this.asr?.transcribeAudio) return;
@@ -147,6 +197,7 @@ export class VoiceLoop extends EventEmitter {
}
async speak(text) {
const generation = this._generation || 0;
if (this.tts && !this.status.tts && typeof this.tts.start === 'function') await this.ensureTts();
if (!isSpeakable(text) || !this.tts?.speak || !this.status.tts) {
this._releaseSpeaking();
return;
+118 -5
View File
@@ -1,12 +1,20 @@
import { EventEmitter } from 'node:events';
import { spawn } from 'node:child_process';
import path from 'node:path';
export const BUILTIN_WAKE_COMMAND = 'jarvis-wake-bridge';
const FRAME_BYTES = 640;
const SPEECH_RMS = 0.018;
const SILENCE_FRAMES = 8;
const MIN_SPEECH_FRAMES = 16;
const MAX_SPEECH_FRAMES = 90;
const WAKE_COOLDOWN_MS = 1500;
/**
* Wake engines consume PCM frames. A production wake model can be attached
* through `detect(frame)`, keeping the daemon independent from a Python or
* native openWakeWord installation. The default detector is intentionally
* disabled until a local model command is configured; it never pretends that
* an energy spike is the wake phrase.
* through `detect(frame)` or a stdin bridge. The built-in CPU detector looks
* for a short two-to-four-peak cadence matching “hey jarvis” without loading
* Whisper or any GPU model.
*/
export class WakeEngine extends EventEmitter {
constructor({ phrases = ['hey jarvis', 'jarvis', 'okay jarvis'], detect } = {}) {
@@ -31,6 +39,100 @@ export class WakeEngine extends EventEmitter {
close() { this.pause(); this.removeAllListeners(); }
}
function frameRms(buf) {
const samples = buf.length / 2;
if (!samples) return 0;
let sum = 0;
for (let i = 0; i < buf.length; i += 2) {
const sample = buf.readInt16LE(i) / 32768;
sum += sample * sample;
}
return Math.sqrt(sum / samples);
}
function countPeaks(series) {
const smooth = series.map((value, index, items) => {
const prev = items[index - 1] ?? value;
const next = items[index + 1] ?? value;
return (prev + value + next) / 3;
});
const max = Math.max(0, ...smooth);
if (max < SPEECH_RMS) return 0;
const floor = Math.max(SPEECH_RMS * 1.2, max * 0.45);
let peaks = 0;
let raised = false;
for (let i = 1; i < smooth.length - 1; i++) {
if (smooth[i] >= floor && smooth[i] >= smooth[i - 1] && smooth[i] >= smooth[i + 1]) {
if (!raised) { peaks += 1; raised = true; }
} else if (smooth[i] < floor * 0.65) {
raised = false;
}
}
return peaks;
}
/** CPU keyword spotter. Keeps GPU ASR unloaded until a real wake. */
export class KeywordWakeEngine extends WakeEngine {
constructor(options = {}) {
super(options);
this.command = options.command || BUILTIN_WAKE_COMMAND;
this.detect = (frame) => this._ingest(frame);
this._pending = Buffer.alloc(0);
this._rms = [];
this._silence = 0;
this._lastWake = 0;
}
start() { this.active = true; return this; }
push(frame) {
if (!this.active) return false;
return this._ingest(frame);
}
_ingest(frame) {
this._pending = Buffer.concat([this._pending, Buffer.from(frame)]);
let woke = false;
while (this._pending.length >= FRAME_BYTES) {
const next = this._pending.subarray(0, FRAME_BYTES);
this._pending = this._pending.subarray(FRAME_BYTES);
if (this._acceptFrame(next)) woke = true;
}
return woke;
}
_acceptFrame(frame) {
const rms = frameRms(frame);
if (rms >= SPEECH_RMS) {
this._rms.push(rms);
this._silence = 0;
if (this._rms.length > MAX_SPEECH_FRAMES + SILENCE_FRAMES) this._rms.shift();
return false;
}
if (!this._rms.length) return false;
this._rms.push(rms);
this._silence += 1;
if (this._silence < SILENCE_FRAMES) return false;
const series = this._rms;
this._rms = [];
this._silence = 0;
return this._maybeWake(series);
}
_maybeWake(series) {
const voiced = series.filter((value) => value >= SPEECH_RMS).length;
if (voiced < MIN_SPEECH_FRAMES || voiced > MAX_SPEECH_FRAMES) return false;
const peaks = countPeaks(series);
if (peaks < 2 || peaks > 4) return false;
const now = Date.now();
if (now - this._lastWake < WAKE_COOLDOWN_MS) return false;
const phrase = this.phrases[0] || 'hey jarvis';
this._lastWake = now;
this.emit('wake', phrase);
return true;
}
}
/** Adapter for a local openWakeWord/sherpa bridge. The bridge receives raw
* PCM on stdin and prints one detected phrase per line on stdout. */
export class ProcessWakeEngine extends WakeEngine {
@@ -58,8 +160,19 @@ export class ProcessWakeEngine extends WakeEngine {
close() { super.close(); this.process?.kill('SIGTERM'); this.process = null; }
}
export function isBuiltinWakeCommand(command) {
const value = String(command || '').trim();
if (!value) return false;
const executable = value.split(/\s+/)[0];
const base = path.basename(executable);
return value === BUILTIN_WAKE_COMMAND || base === BUILTIN_WAKE_COMMAND || base === 'wake-bridge.js';
}
export function createWakeEngine(options = {}) {
return options.command || process.env.JARVIS_WAKE_COMMAND ? new ProcessWakeEngine({ ...options, command: options.command || process.env.JARVIS_WAKE_COMMAND }) : new WakeEngine(options);
const command = options.command || process.env.JARVIS_WAKE_COMMAND || '';
if (isBuiltinWakeCommand(command)) return new KeywordWakeEngine({ ...options, command: BUILTIN_WAKE_COMMAND });
if (command) return new ProcessWakeEngine({ ...options, command });
return new WakeEngine(options);
}
export function normalizeWakePhrases(value) {