From f26204505e6f42bf15613c4db9d05331b516bddc Mon Sep 17 00:00:00 2001 From: Raven Scott Date: Sat, 12 Sep 2026 09:40:42 -0400 Subject: [PATCH] Updates --- README.md | 9 +- .../jarvis@qvac.local/extension.js | 5 +- .../jarvis@qvac.local/settings-catalog.json | 41 +++++- daemon/harness-bridge.js | 16 ++- daemon/index.js | 59 ++++++--- daemon/voice-doctor.js | 2 + daemon/voice-loop.js | 113 +++++++++++----- daemon/wake-engine.js | 123 +++++++++++++++++- docs/ROADMAP.md | 7 +- docs/security-privacy.md | 7 + docs/settings.md | 4 +- docs/troubleshooting.md | 9 +- docs/voice-acceptance.md | 10 +- docs/voice-pipeline.md | 25 ++-- packaging/install.sh | 7 +- packaging/wake-bridge.js | 8 ++ skills/phase2-tools.js | 37 ++++-- test/daemon.test.js | 20 ++- test/gnome-extension.test.js | 15 +++ test/install.test.js | 3 + test/review-regressions.test.js | 29 ++++- test/settings.test.js | 10 +- test/voice-phase4.test.js | 86 ++++++++++-- 23 files changed, 531 insertions(+), 114 deletions(-) create mode 100644 packaging/wake-bridge.js diff --git a/README.md b/README.md index 4d4200d..2f1bb5b 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,8 @@ curl -fsSL https://git.ssh.surf/snxraven/gnome-jarvis/raw/branch/main/packaging/ Omit `--enable` to install files without starting the service. With `--enable`, the installer writes default settings if needed, starts `jarvisd`, and enables the GNOME extension. The default wake phrase is -**hey jarvis**; change it in Settings. Spoken replies are on. +**hey jarvis** with a built-in CPU detector; change it in Settings. Spoken +replies are on. If GNOME says the extension does not exist immediately after installation, log out and back in once. GNOME Shell only refreshes its user extension catalogue @@ -124,9 +125,9 @@ bash packaging/install.sh --enable ~~~ The installer writes `~/.config/jarvis/config.json` when it is missing, with -wake phrase `hey jarvis`, model profile `laptop-16gb`, and TTS enabled. Change -those later in Settings. Diagnostics stay available as `npm run gpu-doctor`, -`npm run voice-doctor`, and `npm run cu-doctor`. +wake phrase `hey jarvis`, CPU wake detector `jarvis-wake-bridge`, model profile +`laptop-16gb`, and TTS enabled. Change those later in Settings. Diagnostics stay +available as `npm run gpu-doctor`, `npm run voice-doctor`, and `npm run cu-doctor`. Inspect the installed daemon with: diff --git a/apps/gnome-extension/jarvis@qvac.local/extension.js b/apps/gnome-extension/jarvis@qvac.local/extension.js index 0257975..6d11d75 100644 --- a/apps/gnome-extension/jarvis@qvac.local/extension.js +++ b/apps/gnome-extension/jarvis@qvac.local/extension.js @@ -243,8 +243,9 @@ export default class JarvisExtension extends Extension { const voice = status.voice; this._eachView((view) => view.setConnectionStatus(voice ? 'local' : 'voice-unavailable')); if (voice) { - const input = voice.asr && voice.capture; - this._eachView((view) => view.setVoiceStatus({ tts: voice.tts, input, wake: voice.wake })); + const input = Boolean(voice.capture); + const tts = Boolean(voice.tts || voice.speech); + this._eachView((view) => view.setVoiceStatus({ tts, input, wake: voice.wake })); } } catch { this._eachView((view) => view.setConnectionStatus('voice-unavailable')); } } diff --git a/apps/gnome-extension/jarvis@qvac.local/settings-catalog.json b/apps/gnome-extension/jarvis@qvac.local/settings-catalog.json index 3525987..17a55b1 100644 --- a/apps/gnome-extension/jarvis@qvac.local/settings-catalog.json +++ b/apps/gnome-extension/jarvis@qvac.local/settings-catalog.json @@ -481,9 +481,9 @@ "key": "wakeCommand", "title": "Wake detector command", "group": "Wake and privacy", - "default": "", + "default": "jarvis-wake-bridge", "type": "string", - "description": "Advanced: local program that receives microphone audio. Leave empty to use Hold Talk.", + "description": "Local program that receives microphone audio. jarvis-wake-bridge is the built-in CPU detector. Leave it unless you have openWakeWord or sherpa.", "aliases": [ "wake_command" ] @@ -524,6 +524,17 @@ "max": 240, "step": 1 }, + { + "key": "freeVramOnIdle", + "title": "Free GPU memory when idle", + "group": "Wake and privacy", + "default": true, + "type": "boolean", + "description": "Unload speech and chat models after the idle delay. Hey Jarvis stays ready on the CPU.", + "aliases": [ + "free_vram_on_idle" + ] + }, { "key": "vadThreshold", "title": "Speech detection threshold", @@ -740,6 +751,32 @@ "max": 20, "step": 1, "restart": true + }, + { + "key": "fsAccess", + "title": "File access", + "group": "Agent limits", + "default": "workspace", + "type": "choice", + "description": "Requires restart. The agent still runs as your user account and asks before writing. Shell commands can already reach other paths.", + "options": [ + { + "value": "workspace", + "label": "Jarvis workspace only" + }, + { + "value": "home", + "label": "Home directory" + }, + { + "value": "filesystem", + "label": "Entire filesystem" + } + ], + "aliases": [ + "fs_access" + ], + "restart": true } ] } diff --git a/daemon/harness-bridge.js b/daemon/harness-bridge.js index 15bbc53..231aa4c 100644 --- a/daemon/harness-bridge.js +++ b/daemon/harness-bridge.js @@ -1,24 +1,34 @@ import { voiceSettings } from './voice-settings.js'; import { EventEmitter } from 'node:events'; +import os from 'node:os'; import { acquireQvac, closeQvac, releaseQvac, Agent, QVAC_MASTER } from './qvac-master.js'; import { createRuntimeTools } from '../skills/runtime-tools.js'; -import { createPhase2Tools } from '../skills/phase2-tools.js'; +import { createPhase2Tools, filesystemRoots } from '../skills/phase2-tools.js'; import { createQvacTools } from '../skills/qvac-tools.js'; import { VOICE_SYSTEM_PROMPT, parseHudSidecar } from '../skills/voice-prompt.js'; import { createComputerObserveTools } from '../skills/computer-observe.js'; import { createComputerActTools } from '../skills/computer-act.js'; import { createPhase9GatewayTool } from '../skills/phase9-tools.js'; +export function harnessRoots(fsAccess) { + if (fsAccess === 'filesystem') return ['/']; + if (fsAccess === 'home') return [os.homedir()]; + return []; +} + export class HarnessBridge extends EventEmitter { - constructor({ cwd = process.cwd(), model = QVAC_MASTER.model, tools = [], computer, observer, actuator, permissionMode = 'ask' } = {}) { + constructor({ cwd = process.cwd(), model = QVAC_MASTER.model, tools = [], computer, observer, actuator, permissionMode = 'ask', fsAccess } = {}) { super(); const settings = voiceSettings(); + const access = fsAccess ?? settings.fsAccess; + const roots = harnessRoots(access); this.options = { cwd, + roots, model, tools: [ ...createRuntimeTools({ computer }), - ...createPhase2Tools({ cwd, computer }), + ...createPhase2Tools({ cwd, computer, roots: filesystemRoots(access, cwd) }), ...createComputerObserveTools({ computer, observer }), ...createComputerActTools({ actuator }), ...createQvacTools(), diff --git a/daemon/index.js b/daemon/index.js index c04be15..163e6b4 100644 --- a/daemon/index.js +++ b/daemon/index.js @@ -8,7 +8,7 @@ import { HarnessBridge } from './harness-bridge.js'; import { ComputerUseSession } from '../computer-use/session.js'; import { VoiceStateMachine } from './voice-state.js'; import { QvacScheduler } from './qvac-scheduler.js'; -import { cancelQvac, resumeQvac, suspendQvac, callQvac, cancelQvacRequest, qvacStatus } from './qvac-master.js'; +import { cancelQvac, closeQvac, resumeQvac, suspendQvac, callQvac, cancelQvacRequest, qvacStatus } from './qvac-master.js'; import { PrivacyLog } from './privacy-log.js'; import { VoiceLoop } from './voice-loop.js'; import { QvacVoiceAdapter } from './voice-adapters.js'; @@ -37,7 +37,7 @@ export class JarvisDaemon extends EventEmitter { this.perception = new QvacPerception(); this.observer = new DesktopObserver({ normalizer: new FrameNormalizer({ maxLongEdge: this.settings.screenshotMaxEdge, quality: this.settings.screenshotQuality }), ocr: (image) => this.perception.ocr(image) }); this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit }); - this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer, actuator: this.actuator }); + this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer, actuator: this.actuator, fsAccess: this.settings.fsAccess }); this.log = new PrivacyLog(); this.locked = false; this.lastReply = ''; @@ -57,8 +57,32 @@ export class JarvisDaemon extends EventEmitter { } setState(state) { this.state = state; try { this.recovery.save({ state, mode: this.mode }); } catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); } this.emit('StateChanged', state); this.log.record('state', { state }).catch(() => {}); } - async arm() { if (this.locked) return; await resumeQvac().catch(() => {}); this.voice.wake(); this.setState('LISTENING'); } - async sleep() { this.cancel(); this.voice.sleep(); this.setState('SLEEPING'); await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); } + async arm() { + if (this.locked) return; + await resumeQvac().catch(() => {}); + await this.ensureAsr(); + this.voice.wake(); + this.setState('LISTENING'); + } + async sleep() { + if (this._activeAsk) { + await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); + return; + } + this.cancel(); + this.voice.sleep(); + this.setState('SLEEPING'); + if (this.settings.freeVramOnIdle !== false) await this.parkModels(); + else await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); + } + async parkModels() { + try { await this.voiceLoop?.parkModels?.(); } + catch (error) { this.emit('Error', 'VOICE_UNLOAD', error.message); } + try { await this.harness.resetContext(); } + catch (error) { this.emit('Error', 'QVAC_UNLOAD', error.message); } + try { await closeQvac(); } + catch (error) { this.emit('Error', 'QVAC_CLOSE', error.message); } + } say(text) { const spoken = spokenReply(text) || spokenReply(this.lastReply) || 'There is nothing to repeat.'; this.lastReply = spoken; @@ -145,18 +169,21 @@ export class JarvisDaemon extends EventEmitter { if (!settings.ttsEnabled) { this.voiceLoop.status.tts = false; return; } if (this.voiceLoop.status.tts) return; if (!this.voiceLoop.tts) this.voiceLoop.tts = new QvacVoiceAdapter({ role: 'tts', settings }); - try { - await this.voiceLoop.tts.start(); - this.voiceLoop.status.tts = true; - delete this.voiceLoop.status.errors.tts; - console.log('jarvisd: voice: TTS ready'); - } catch (error) { - const message = String(error?.message || error); - this.voiceLoop.status.errors.tts = message; + const ready = await this.voiceLoop.ensureTts(); + if (ready) console.log('jarvisd: voice: TTS ready'); + else if (this.voiceLoop.status.errors.tts) { + const message = this.voiceLoop.status.errors.tts; console.error(`jarvisd: voice: TTS: ${message}`); this.emit('Error', 'TTS', message); } } + async ensureAsr() { + await this._voiceStarting; + if (!this.voiceLoop || !this.settings.microphoneEnabled) return; + if (this.voiceLoop.status?.asr) return; + if (!this.voiceLoop.asr) this.voiceLoop.asr = new QvacVoiceAdapter({ role: 'asr', settings: this.settings }); + await this.voiceLoop.ensureAsr?.(); + } cancel() { this._askGeneration += 1; this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computerRevoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); } computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); if (this.settings.computerMode === 'observe') return result; this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; } computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); } @@ -182,8 +209,10 @@ export class JarvisDaemon extends EventEmitter { try { await loop.start(); this.emit('StateChanged', this.state); } catch (error) { this.voiceLoop = null; await loop.stop?.().catch(() => {}); this.emit('Error', 'VOICE_UNAVAILABLE', error.message); throw error; } - if (loop.status.tts) console.log('jarvisd: voice: TTS ready'); - else if (loop.status.errors.tts) console.error(`jarvisd: voice: TTS: ${loop.status.errors.tts}`); + if (loop.status.wake) console.log('jarvisd: voice: wake detector ready'); + else if (loop.status.errors.wake) console.error(`jarvisd: voice: wake: ${loop.status.errors.wake}`); + if (loop.status.capture) console.log('jarvisd: voice: microphone capture ready'); + else if (loop.status.errors.capture) console.error(`jarvisd: voice: capture: ${loop.status.errors.capture}`); } async reloadSettings() { if (this._settingsReload) return this._settingsReload; @@ -206,7 +235,7 @@ export class JarvisDaemon extends EventEmitter { Object.assign(this.observer.normalizer, { maxLongEdge: next.screenshotMaxEdge, quality: next.screenshotQuality }); await this.startVoice(); this.voice.cancel(); this.setState('ARMED'); - return JSON.stringify({ applied: true, voice: this.voiceLoop.status, environmentOverrides: ['JARVIS_TTS_MODEL', 'JARVIS_ASR_MODEL', 'JARVIS_WAKE_COMMAND', 'JARVIS_QVAC_MODEL'].filter(key => process.env[key]), restartRequired: ['chatModel', 'modelProfile', 'maxTurns', 'maxShellCalls', 'maxToolRounds'].filter(key => next[key] !== this.startupSettings[key]) }); + return JSON.stringify({ applied: true, voice: this.voiceLoop.status, environmentOverrides: ['JARVIS_TTS_MODEL', 'JARVIS_ASR_MODEL', 'JARVIS_WAKE_COMMAND', 'JARVIS_QVAC_MODEL'].filter(key => process.env[key]), restartRequired: ['chatModel', 'modelProfile', 'maxTurns', 'maxShellCalls', 'maxToolRounds', 'fsAccess'].filter(key => next[key] !== this.startupSettings[key]) }); } async previewVoice(text) { if (this.locked) throw new Error('Unlock the desktop to preview a voice'); diff --git a/daemon/voice-doctor.js b/daemon/voice-doctor.js index cf98889..92d0fa8 100644 --- a/daemon/voice-doctor.js +++ b/daemon/voice-doctor.js @@ -1,5 +1,6 @@ import { voiceSettings } from './voice-settings.js'; import { ttsConfiguration } from './tts-config.js'; +import { isBuiltinWakeCommand } from './wake-engine.js'; import { access } from 'node:fs/promises'; const commandAvailable = async (command) => { @@ -14,6 +15,7 @@ const report = { pipewireCapture: pipewire, wakeEngine: settings.listeningMode !== 'ptt' && Boolean(settings.command), wakeCommand: settings.command || null, + wakeBuiltin: settings.listeningMode !== 'ptt' && isBuiltinWakeCommand(settings.command), sampleRate: 16000, channels: 1, ttsPlayback: pipewire, diff --git a/daemon/voice-loop.js b/daemon/voice-loop.js index 15fdca8..ca7b56f 100644 --- a/daemon/voice-loop.js +++ b/daemon/voice-loop.js @@ -29,7 +29,7 @@ export class VoiceLoop extends EventEmitter { capture.on('close', () => { this.status.capture = false; if (this.running) { this.status.errors.capture = 'Microphone stream closed'; this._scheduleCaptureRetry(); } }); wake.on('unavailable', () => { this.status.wake = false; }); wake.on('error', (error) => { this.status.wake = false; this.emit('error', error); }); - this.status = { asr: false, tts: false, capture: false, wake: false, errors: {} }; + this.status = { asr: false, tts: false, capture: false, wake: false, microphone: this.asr != null, speech: this.tts != null, errors: {} }; wake.on('wake', (phrase) => this.wakeHeard(phrase)); vad.on('level', (rms) => daemon?.emit('ListeningLevel', rms)); vad.on('utterance', (audio) => { @@ -41,23 +41,10 @@ export class VoiceLoop extends EventEmitter { async start() { if (this.running) return; - for (const [name, adapter] of [['tts', this.tts], ['asr', this.asr]]) { - try { - if (adapter) { await adapter.start?.(); this.status[name] = true; } - else if (name === 'tts') this.status.errors.tts = 'Spoken replies are disabled'; - } catch (error) { - const message = String(error?.message || error); - this.status.errors[name] = message; - this.emit('error', new Error(`${name.toUpperCase()}: ${message}`)); - } - } + if (!this.tts) this.status.errors.tts = 'Spoken replies are disabled'; this.running = true; - if (this.status.asr) { - this._armCapture(); - this._armWake(); - } else if (this.asr) { - this._scheduleAsrRetry(); - } + this._armWake(); + this._armCapture(); } _notifyStatus() { try { this.daemon?.emit?.('StateChanged', this.daemon.state || 'ARMED'); } catch {} } _armWake() { @@ -65,12 +52,12 @@ export class VoiceLoop extends EventEmitter { catch (error) { this.status.errors.wake = error.message; this.emit('error', error); } } _armCapture() { - if (!this.running || !this.status.asr || this.status.capture) return; + if (!this.running || this.status.capture || this.asr == null) return; try { this.capture.start(); this.status.capture = true; delete this.status.errors.capture; this._notifyStatus(); } catch (error) { this.status.errors.capture = error.message; this.emit('error', error); this._scheduleCaptureRetry(); } } _scheduleCaptureRetry() { - if (!this.running || this._captureRetry || this.status.capture || !this.status.asr) return; + if (!this.running || this._captureRetry || this.status.capture || this.asr == null) return; this._captureRetry = setTimeout(() => { this._captureRetry = 0; this._armCapture(); @@ -82,19 +69,65 @@ export class VoiceLoop extends EventEmitter { this._asrRetry = setTimeout(() => { this._asrRetry = 0; if (!this.running || this.status.asr || !this.asr) return; - this.asr.start().then(() => { - if (!this.running || this.status.asr) return; + this.ensureAsr().catch(() => {}); + }, this.asrRetryMs); + this._asrRetry.unref?.(); + } + async ensureAsr() { + if (!this.asr) return false; + if (this.status.asr) return true; + if (this._asrStarting) return this._asrStarting; + this._asrStarting = (async () => { + try { + await this.asr.start(); + if (!this.running) return false; this.status.asr = true; delete this.status.errors.asr; this._armCapture(); this._armWake(); this._notifyStatus(); - }).catch((error) => { - this.status.errors.asr = String(error?.message || error); + return true; + } catch (error) { + const message = String(error?.message || error); + this.status.errors.asr = message; + this.emit('error', new Error(`ASR: ${message}`)); this._scheduleAsrRetry(); - }); - }, this.asrRetryMs); - this._asrRetry.unref?.(); + return false; + } finally { + this._asrStarting = null; + } + })(); + return this._asrStarting; + } + async ensureTts() { + if (!this.tts) { this.status.errors.tts = 'Spoken replies are disabled'; return false; } + if (this.status.tts) return true; + if (this._ttsStarting) return this._ttsStarting; + this._ttsStarting = (async () => { + try { + await this.tts.start(); + this.status.tts = true; + delete this.status.errors.tts; + return true; + } catch (error) { + const message = String(error?.message || error); + this.status.errors.tts = message; + this.emit('error', new Error(`TTS: ${message}`)); + return false; + } finally { + this._ttsStarting = null; + } + })(); + return this._ttsStarting; + } + async parkModels() { + if (this._asrRetry) { clearTimeout(this._asrRetry); this._asrRetry = 0; } + this._asrStarting = null; + this._ttsStarting = null; + await this.asr?.stop?.(); + if (this.tts && this.tts !== this.asr) await this.tts.stop?.(); + this.status.asr = false; + this.status.tts = false; } async stop() { this.running = false; @@ -102,17 +135,34 @@ export class VoiceLoop extends EventEmitter { if (this._asrRetry) { clearTimeout(this._asrRetry); this._asrRetry = 0; } this.interrupt(); this.capture.stop(); this.wake.close(); this.vad.reset(); this.playback.stop(); await Promise.allSettled([this._speechQueue, ...this._transcriptions]); await this.asr?.stop?.(); if (this.tts !== this.asr) await this.tts?.stop?.(); } - setPushToTalk(pressed) { this.ptt = Boolean(pressed) && this.status.asr && this.status.capture; if (this.ptt) this.wakeHeard('push-to-talk'); else this.vad.end(); } + setPushToTalk(pressed) { + if (!pressed) { + this._pttHeld = false; + this.ptt = false; + this.vad.end(); + return Promise.resolve(); + } + this._pttHeld = true; + return this.ensureAsr().then((ok) => { + if (!this._pttHeld) return; + this.ptt = Boolean(ok && this.status.capture); + if (this.ptt) this.wakeHeard('push-to-talk'); + }).catch((error) => this.emit('error', error)); + } pushAudio(chunk) { - if (!this.running || this.daemon?.locked || this.daemon?.state === 'SLEEPING') return; - if (this.isSpeaking || this.now() < this.cooldownUntil) { this.metrics.feedbackDrop(); return; } - try { this.daemon?.emit('ListeningLevel', Math.min(1, Math.max(0, chunk?.length ? 0.01 : 0))); } catch {} + if (!this.running || this.daemon?.locked) return; + const sleeping = this.daemon?.state === 'SLEEPING'; + if (!sleeping && (this.isSpeaking || this.now() < this.cooldownUntil)) { this.metrics.feedbackDrop(); return; } + if (!sleeping) { + try { this.daemon?.emit('ListeningLevel', Math.min(1, Math.max(0, chunk?.length ? 0.01 : 0))); } catch {} + } if (!this.ptt && this.listeningMode !== 'ptt') this.wake.push(chunk); + if (sleeping) return; if (this.ptt || (this.listeningMode !== 'ptt' && this.daemon?.state === 'LISTENING')) this.vad.push(chunk); } wakeHeard(phrase) { if (!this.running || this.daemon?.locked) return; - this.metrics.wakeAccepted(); this.daemon?.emit('WakeHeard', phrase); this.daemon?.arm?.(); this.emit('wake', phrase); + this.metrics.wakeAccepted(); this.daemon?.emit('WakeHeard', phrase); this.ensureAsr().catch((error) => this.emit('error', error)); this.daemon?.arm?.(); this.emit('wake', phrase); } async transcribe(audio) { if (!this.status.asr || !this.asr?.transcribeAudio) return; @@ -147,6 +197,7 @@ export class VoiceLoop extends EventEmitter { } async speak(text) { const generation = this._generation || 0; + if (this.tts && !this.status.tts && typeof this.tts.start === 'function') await this.ensureTts(); if (!isSpeakable(text) || !this.tts?.speak || !this.status.tts) { this._releaseSpeaking(); return; diff --git a/daemon/wake-engine.js b/daemon/wake-engine.js index 9c4080d..1760476 100644 --- a/daemon/wake-engine.js +++ b/daemon/wake-engine.js @@ -1,12 +1,20 @@ import { EventEmitter } from 'node:events'; import { spawn } from 'node:child_process'; +import path from 'node:path'; + +export const BUILTIN_WAKE_COMMAND = 'jarvis-wake-bridge'; +const FRAME_BYTES = 640; +const SPEECH_RMS = 0.018; +const SILENCE_FRAMES = 8; +const MIN_SPEECH_FRAMES = 16; +const MAX_SPEECH_FRAMES = 90; +const WAKE_COOLDOWN_MS = 1500; /** * Wake engines consume PCM frames. A production wake model can be attached - * through `detect(frame)`, keeping the daemon independent from a Python or - * native openWakeWord installation. The default detector is intentionally - * disabled until a local model command is configured; it never pretends that - * an energy spike is the wake phrase. + * through `detect(frame)` or a stdin bridge. The built-in CPU detector looks + * for a short two-to-four-peak cadence matching “hey jarvis” without loading + * Whisper or any GPU model. */ export class WakeEngine extends EventEmitter { constructor({ phrases = ['hey jarvis', 'jarvis', 'okay jarvis'], detect } = {}) { @@ -31,6 +39,100 @@ export class WakeEngine extends EventEmitter { close() { this.pause(); this.removeAllListeners(); } } +function frameRms(buf) { + const samples = buf.length / 2; + if (!samples) return 0; + let sum = 0; + for (let i = 0; i < buf.length; i += 2) { + const sample = buf.readInt16LE(i) / 32768; + sum += sample * sample; + } + return Math.sqrt(sum / samples); +} + +function countPeaks(series) { + const smooth = series.map((value, index, items) => { + const prev = items[index - 1] ?? value; + const next = items[index + 1] ?? value; + return (prev + value + next) / 3; + }); + const max = Math.max(0, ...smooth); + if (max < SPEECH_RMS) return 0; + const floor = Math.max(SPEECH_RMS * 1.2, max * 0.45); + let peaks = 0; + let raised = false; + for (let i = 1; i < smooth.length - 1; i++) { + if (smooth[i] >= floor && smooth[i] >= smooth[i - 1] && smooth[i] >= smooth[i + 1]) { + if (!raised) { peaks += 1; raised = true; } + } else if (smooth[i] < floor * 0.65) { + raised = false; + } + } + return peaks; +} + +/** CPU keyword spotter. Keeps GPU ASR unloaded until a real wake. */ +export class KeywordWakeEngine extends WakeEngine { + constructor(options = {}) { + super(options); + this.command = options.command || BUILTIN_WAKE_COMMAND; + this.detect = (frame) => this._ingest(frame); + this._pending = Buffer.alloc(0); + this._rms = []; + this._silence = 0; + this._lastWake = 0; + } + + start() { this.active = true; return this; } + + push(frame) { + if (!this.active) return false; + return this._ingest(frame); + } + + _ingest(frame) { + this._pending = Buffer.concat([this._pending, Buffer.from(frame)]); + let woke = false; + while (this._pending.length >= FRAME_BYTES) { + const next = this._pending.subarray(0, FRAME_BYTES); + this._pending = this._pending.subarray(FRAME_BYTES); + if (this._acceptFrame(next)) woke = true; + } + return woke; + } + + _acceptFrame(frame) { + const rms = frameRms(frame); + if (rms >= SPEECH_RMS) { + this._rms.push(rms); + this._silence = 0; + if (this._rms.length > MAX_SPEECH_FRAMES + SILENCE_FRAMES) this._rms.shift(); + return false; + } + if (!this._rms.length) return false; + this._rms.push(rms); + this._silence += 1; + if (this._silence < SILENCE_FRAMES) return false; + const series = this._rms; + this._rms = []; + this._silence = 0; + return this._maybeWake(series); + } + + _maybeWake(series) { + const voiced = series.filter((value) => value >= SPEECH_RMS).length; + if (voiced < MIN_SPEECH_FRAMES || voiced > MAX_SPEECH_FRAMES) return false; + const peaks = countPeaks(series); + if (peaks < 2 || peaks > 4) return false; + const now = Date.now(); + if (now - this._lastWake < WAKE_COOLDOWN_MS) return false; + const phrase = this.phrases[0] || 'hey jarvis'; + this._lastWake = now; + this.emit('wake', phrase); + return true; + } +} + /** Adapter for a local openWakeWord/sherpa bridge. The bridge receives raw * PCM on stdin and prints one detected phrase per line on stdout. */ export class ProcessWakeEngine extends WakeEngine { @@ -58,8 +160,19 @@ export class ProcessWakeEngine extends WakeEngine { close() { super.close(); this.process?.kill('SIGTERM'); this.process = null; } } +export function isBuiltinWakeCommand(command) { + const value = String(command || '').trim(); + if (!value) return false; + const executable = value.split(/\s+/)[0]; + const base = path.basename(executable); + return value === BUILTIN_WAKE_COMMAND || base === BUILTIN_WAKE_COMMAND || base === 'wake-bridge.js'; +} + export function createWakeEngine(options = {}) { - return options.command || process.env.JARVIS_WAKE_COMMAND ? new ProcessWakeEngine({ ...options, command: options.command || process.env.JARVIS_WAKE_COMMAND }) : new WakeEngine(options); + const command = options.command || process.env.JARVIS_WAKE_COMMAND || ''; + if (isBuiltinWakeCommand(command)) return new KeywordWakeEngine({ ...options, command: BUILTIN_WAKE_COMMAND }); + if (command) return new ProcessWakeEngine({ ...options, command }); + return new WakeEngine(options); } export function normalizeWakePhrases(value) { diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 1408ba4..ffa4c70 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -173,9 +173,10 @@ runtime. Implementation is complete in `daemon/audio-pipewire.js`, `daemon/wake-engine.js`, `daemon/vad.js`, `daemon/voice-adapters.js`, `daemon/voice-loop.js`, and `daemon/audio-playback.js`. The SDK speech models are loaded as auxiliary -models by the same QVAC master and use GPU settings. Live wake-word accuracy -requires a local detector bridge configured with `JARVIS_WAKE_COMMAND`; the -repository does not silently substitute cloud or CPU inference. +models by the same QVAC master and use GPU settings. Live wake-word detection +uses the built-in CPU keyword spotter by default (`jarvis-wake-bridge`). A +custom openWakeWord or sherpa-onnx command can replace it; cloud wake is not +supported. Exit gate: implementation complete. In a configured GNOME session, “Hey Jarvis” starts a local GPU-backed turn, speaks a response, and does not diff --git a/docs/security-privacy.md b/docs/security-privacy.md index fbd08d2..3f3160e 100644 --- a/docs/security-privacy.md +++ b/docs/security-privacy.md @@ -28,6 +28,13 @@ flowchart TD - QVAC binds to localhost; bearer tokens, when used, come from user-owned configuration and are not logged. +## Filesystem access + +Path tools default to the Jarvis workspace (`~/.local/share/jarvis-qvac` when +installed). Settings can widen that to your home directory or the entire +filesystem. This is still your user account: Jarvis does not gain root, and +writes plus shell commands still go through the confirmation gate. + ## Computer-use controls Computer use requires a spoken or HUD grant, shows a visible cursor/target, and diff --git a/docs/settings.md b/docs/settings.md index ab3ce1f..1d79053 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -112,9 +112,10 @@ and WebP quality tune observation detail and processing cost. - **Wake phrase** — `wakePhrase`, default `"hey jarvis"`. Must match a phrase supported by your local wake detector. - **Wake aliases** — `aliases`, default `["jarvis", "okay jarvis"]`. Additional detector phrases, separated by commas. -- **Wake detector command** — `wakeCommand`, default `""`. Advanced: local program that receives microphone audio. Leave empty to use Hold Talk. +- **Wake detector command** — `wakeCommand`, default `"jarvis-wake-bridge"`. Built-in CPU detector for Hey Jarvis. Replace it only if you have openWakeWord or sherpa. Empty disables the detector and uses Hold Talk. - **Listening behavior** — `listeningMode`, default `"conversation"`. Hold Talk only disables the wake detector and automatic follow-up listening. Choices: `conversation`, `single`, `ptt`. - **Idle sleep delay (minutes)** — `idleMinutes`, default `30`. Sleep after this much listening inactivity. Range: 1–240. +- **Free GPU memory when idle** — `freeVramOnIdle`, default `true`. Unload speech and chat models after the idle delay. The microphone and Hey Jarvis detector stay ready on the CPU. The first reply after a long idle reloads models. ### Detection tuning @@ -150,6 +151,7 @@ and WebP quality tune observation detail and processing cost. - **Maximum reasoning turns** — `maxTurns`, default `6`. Requires restart. Limits how long the assistant works on one request. Range: 1–30. - **Shell commands per request** — `maxShellCalls`, default `1`. Requires restart. Commands still require normal permissions. Range: 1–10. - **Tool rounds per request** — `maxToolRounds`, default `4`. Requires restart. Limits repeated tool use. Range: 1–20. +- **File access** — `fsAccess`, default `"workspace"`. Requires restart. Limits `read_file`, `list_dir`, `grep`, and the `fs_*` tools. The agent still runs as your user account, not root, and asks before writing. Shell commands can already reach other paths. Choices: `workspace`, `home`, `filesystem`. ## GNOME appearance and boundaries diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 1bcc801..06c8e56 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -10,10 +10,11 @@ GPU. ## Voice doctor finds no wake engine -Install a local openWakeWord or sherpa-onnx bridge and set -`JARVIS_WAKE_COMMAND`. The bridge must read raw 16 kHz mono PCM from stdin and -write detected phrases, one per line, to stdout. Push-to-talk remains useful -for typed or manually triggered testing. +Fresh installs use the built-in CPU detector (`jarvis-wake-bridge`). If +`listeningMode` is Hold Talk only, or `wakeCommand` is empty, the detector is +disabled. A custom openWakeWord or sherpa-onnx bridge must read raw 16 kHz mono +PCM from stdin and write detected phrases, one per line, to stdout. Push-to-talk +remains useful for typed or manually triggered testing. ## Portal consent succeeds but input is unavailable diff --git a/docs/voice-acceptance.md b/docs/voice-acceptance.md index 39891a6..ecf83c8 100644 --- a/docs/voice-acceptance.md +++ b/docs/voice-acceptance.md @@ -1,10 +1,10 @@ # Phase 4 voice acceptance -Run `npm run voice-doctor` in the target GNOME session. Set -`JARVIS_WAKE_COMMAND` to the local openWakeWord or sherpa-onnx bridge command. -The command receives raw 16 kHz mono PCM on stdin and emits one detected phrase -per line on stdout; the daemon passes it through `ProcessWakeEngine`. No cloud -wake service is supported. +Run `npm run voice-doctor` in the target GNOME session. Fresh installs use the +built-in CPU detector (`jarvis-wake-bridge`). To use openWakeWord or sherpa-onnx +instead, set `wakeCommand` or `JARVIS_WAKE_COMMAND` to a program that receives +raw 16 kHz mono PCM on stdin and emits one detected phrase per line on stdout. +No cloud wake service is supported. The capture and playback nodes are both named `Jarvis`, so they can be routed in Helvum or qpwgraph. The loop gates capture while TTS is active and for 400ms diff --git a/docs/voice-pipeline.md b/docs/voice-pipeline.md index 55b29f8..b8ed624 100644 --- a/docs/voice-pipeline.md +++ b/docs/voice-pipeline.md @@ -31,9 +31,12 @@ stateDiagram-v2 SLEEPING --> LISTENING: wake phrase ``` -The wake stage is intentionally separate from QVAC ASR. Configure a local -openWakeWord or sherpa-onnx bridge through `JARVIS_WAKE_COMMAND`; no cloud wake -service is supported. During TTS, capture submission is gated and a 300–500 ms +The wake stage is intentionally separate from QVAC ASR. A built-in CPU detector +listens for “hey jarvis” (and aliases) without loading Whisper. You can replace +it with a local openWakeWord or sherpa-onnx bridge through `wakeCommand` / +`JARVIS_WAKE_COMMAND`; no cloud wake service is supported. Microphone audio is +still sent to the wake detector while Jarvis is sleeping so the phrase can +restore listening. During TTS, capture submission is gated and a 300–500 ms cooldown prevents playback from becoming a new utterance. Trash transcripts and very short fragments are discarded. @@ -45,16 +48,18 @@ replies. Raw audio is not persisted by default. ## Release voice readiness -ASR and TTS initialize independently in the same QVAC runtime. Voice models do -not require the chat model to load first. The runtime status includes separate -`asr`, `tts`, `capture`, and `wake` flags plus per-component errors. Without a -wake bridge, use Hold Talk (mouse, Space, or Enter) in the GNOME overlay. +ASR, TTS, and the chat model stay unloaded until wake, Hold Talk, or a typed +question. Idle sleep unloads them again so GPU memory is freed; only PipeWire +capture and the CPU wake detector stay resident. Runtime status includes +separate `asr`, `tts`, `capture`, and `wake` flags plus per-component errors. +The tray shows **WAKE ON** when the detector is alive. Hold Talk (mouse, Space, +or Enter) still works in the GNOME overlay. The Control Center's wake command and spoken-reply setting are read from `~/.config/jarvis/config.json` on daemon startup. Restart `jarvisd.service` after -changing them. `JARVIS_WAKE_COMMAND` overrides the saved command. A wake command -must consume 16 kHz mono signed PCM and emit detected phrases on stdout; simply -setting a phrase does not install a wake detector. +changing them. `JARVIS_WAKE_COMMAND` overrides the saved command. A custom wake +command must consume 16 kHz mono signed PCM and emit detected phrases on stdout. +The default `jarvis-wake-bridge` value selects the built-in CPU detector. QVAC TTS numeric arrays contain signed 16-bit samples, not bytes. Supertonic plays at 44.1 kHz. Captured utterances use the non-streaming transcription API diff --git a/packaging/install.sh b/packaging/install.sh index 03b9960..42e30ed 100755 --- a/packaging/install.sh +++ b/packaging/install.sh @@ -200,11 +200,14 @@ else cat > "${CONFIG_DIR}/config.json" <<'EOF' { "wakePhrase": "hey jarvis", + "wakeCommand": "jarvis-wake-bridge", "modelProfile": "laptop-16gb", - "ttsEnabled": true + "ttsEnabled": true, + "fsAccess": "workspace", + "freeVramOnIdle": true } EOF - log "created ${CONFIG_DIR}/config.json with default wake phrase, laptop-16gb profile, and TTS enabled" + log "created ${CONFIG_DIR}/config.json with default wake phrase, CPU wake detector, laptop-16gb profile, and TTS enabled" fi if [[ ! -f "${CONFIG_DIR}/qvac.config.json" ]]; then sed "s#__JARVIS_CACHE__#${CACHE_DIR}#g" "${ROOT_DIR}/packaging/qvac.config.template.json" > "${CONFIG_DIR}/qvac.config.json" diff --git a/packaging/wake-bridge.js b/packaging/wake-bridge.js new file mode 100644 index 0000000..95add75 --- /dev/null +++ b/packaging/wake-bridge.js @@ -0,0 +1,8 @@ +#!/usr/bin/env node +import { KeywordWakeEngine, normalizeWakePhrases } from '../daemon/wake-engine.js'; + +const phrases = normalizeWakePhrases(process.env.JARVIS_WAKE_PHRASES || process.argv.slice(2).join(',') || 'hey jarvis,jarvis,okay jarvis'); +const engine = new KeywordWakeEngine({ phrases }); +engine.on('wake', (phrase) => process.stdout.write(`${phrase}\n`)); +process.stdin.on('data', (chunk) => engine.push(chunk)); +process.stdin.on('end', () => engine.close()); diff --git a/skills/phase2-tools.js b/skills/phase2-tools.js index 2467c52..5979991 100644 --- a/skills/phase2-tools.js +++ b/skills/phase2-tools.js @@ -6,10 +6,21 @@ import { qvacStatus } from '../daemon/qvac-master.js'; const PERMISSIONS = Object.freeze({ read: 'read', write: 'write', dangerous: 'dangerous', computerUse: 'computer-use' }); +export function filesystemRoots(fsAccess, cwd = process.cwd()) { + if (fsAccess === 'filesystem') return ['/']; + if (fsAccess === 'home') return [os.homedir()]; + return [path.resolve(cwd)]; +} + +function isInside(root, candidate) { + const relative = path.relative(root, candidate); + return relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative); +} + // Resolve symlinks as well as lexical paths before accessing workspace files. -async function workspacePath(cwd, file, { write = false } = {}) { - const root = await realpath(path.resolve(cwd)); - const target = path.resolve(cwd, file); +export async function workspacePath(cwd, file, { write = false, roots } = {}) { + const allow = (roots?.length ? roots : [cwd]).map((root) => path.resolve(root)); + const target = path.isAbsolute(file) ? path.resolve(file) : path.resolve(cwd, file); let resolved; try { resolved = await realpath(target); } catch (error) { @@ -18,8 +29,10 @@ async function workspacePath(cwd, file, { write = false } = {}) { if (entry?.isSymbolicLink()) throw new Error('path is outside the Jarvis workspace or is a dangling symlink'); resolved = path.join(await realpath(path.dirname(target)), path.basename(target)); } - const relative = path.relative(root, resolved); - if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) throw new Error('path is outside the Jarvis workspace'); + const resolvedRoots = await Promise.all(allow.map(async (root) => { + try { return await realpath(root); } catch { return root; } + })); + if (!resolvedRoots.some((root) => isInside(root, resolved))) throw new Error('path is outside the Jarvis workspace'); return resolved; } @@ -60,7 +73,9 @@ async function searchFiles(root, query, limit = 50) { return hits; } -export function createPhase2Tools({ cwd = process.cwd(), computer } = {}) { +export function createPhase2Tools({ cwd = process.cwd(), computer, roots } = {}) { + const allow = roots?.length ? roots : [cwd]; + const bound = (file, options) => workspacePath(cwd, file, { ...options, roots: allow }); return [ { name: 'app_list', permission: PERMISSIONS.read, @@ -69,21 +84,21 @@ export function createPhase2Tools({ cwd = process.cwd(), computer } = {}) { }, { name: 'fs_search', permission: PERMISSIONS.read, - description: 'Search local file and directory names below the Jarvis workspace.', + description: 'Search local file and directory names inside the allowed filesystem roots.', parameters: { type: 'object', properties: { query: { type: 'string' }, root: { type: 'string' }, limit: { type: 'number' } }, required: ['query'] }, - execute: async ({ query, root = cwd, limit = 50 }) => searchFiles(await workspacePath(cwd, root), query, Math.min(100, Number(limit) || 50)), + execute: async ({ query, root = cwd, limit = 50 }) => searchFiles(await bound(root), query, Math.min(100, Number(limit) || 50)), }, { name: 'fs_read', permission: PERMISSIONS.read, - description: 'Read a UTF-8 local text file below the Jarvis workspace, capped at 400 KiB.', + description: 'Read a UTF-8 local text file inside the allowed filesystem roots, capped at 400 KiB.', parameters: { type: 'object', properties: { file: { type: 'string' } }, required: ['file'] }, - execute: async ({ file }) => { const target = await workspacePath(cwd, file); return (await readFile(target, 'utf8')).slice(0, 400 * 1024); }, + execute: async ({ file }) => { const target = await bound(file); return (await readFile(target, 'utf8')).slice(0, 400 * 1024); }, }, { name: 'fs_write', permission: PERMISSIONS.write, description: 'Write a local text file only after an explicit confirmation flag is supplied.', parameters: { type: 'object', properties: { file: { type: 'string' }, contents: { type: 'string' }, confirmed: { type: 'boolean' } }, required: ['file', 'contents', 'confirmed'] }, - execute: async ({ file, contents, confirmed }) => { if (confirmed !== true) return { confirmation_required: true, action: 'write', file }; const target = await workspacePath(cwd, file, { write: true }); await writeFile(target, String(contents), 'utf8'); return { ok: true, file: target, bytes: Buffer.byteLength(String(contents)) }; }, + execute: async ({ file, contents, confirmed }) => { if (confirmed !== true) return { confirmation_required: true, action: 'write', file }; const target = await bound(file, { write: true }); await writeFile(target, String(contents), 'utf8'); return { ok: true, file: target, bytes: Buffer.byteLength(String(contents)) }; }, }, { name: 'memory_recall', permission: PERMISSIONS.read, diff --git a/test/daemon.test.js b/test/daemon.test.js index bbc4f2b..3ee9b4a 100644 --- a/test/daemon.test.js +++ b/test/daemon.test.js @@ -7,7 +7,7 @@ import assert from 'node:assert/strict'; import { VoiceStateMachine } from '../daemon/voice-state.js'; import { QvacScheduler } from '../daemon/qvac-scheduler.js'; import { JarvisDaemon } from '../daemon/index.js'; -import { HarnessBridge } from '../daemon/harness-bridge.js'; +import { HarnessBridge, harnessRoots } from '../daemon/harness-bridge.js'; import { spokenReply } from '../skills/voice-prompt.js'; import { voiceSettings } from '../daemon/voice-settings.js'; @@ -67,11 +67,13 @@ test('ask extracts harness reply text instead of stringifying the object', async }); test('harness bridge caps voice shell chaining', () => { - const bridge = new HarnessBridge(); + const bridge = new HarnessBridge({ fsAccess: 'workspace' }); assert.equal(bridge.options.origin, 'jarvis-qvac'); assert.equal(bridge.options.voice, true); assert.equal(bridge.options.maxShellCalls, 1); assert.equal(bridge.options.maxTurns, 6); + assert.deepEqual(bridge.options.roots, []); + assert.deepEqual(harnessRoots('filesystem'), ['/']); assert.deepEqual(bridge.options.builtinTools, [ 'read_file', 'list_dir', @@ -150,6 +152,20 @@ test('QVAC scheduler prioritizes voice and keeps one active job', async () => { assert.deepEqual(scheduler.status(), { running: 0, queued: [] }); }); +test('idle sleep unloads GPU models instead of only suspending', async () => { + const daemon = new JarvisDaemon(); + const calls = []; + daemon.settings = { ...daemon.settings, freeVramOnIdle: true }; + daemon.parkModels = async () => { calls.push('park'); }; + daemon.voiceLoop = { interrupt() {} }; + daemon.harness = { cancel() {}, resetContext: async () => {}, close: async () => {} }; + try { + await daemon.sleep(); + assert.equal(daemon.state, 'SLEEPING'); + assert.deepEqual(calls, ['park']); + } finally { await daemon.close(); } +}); + test('shutdown still closes the harness when recovery or voice cleanup fails', async () => { const daemon = new JarvisDaemon(); let closed = false; daemon.recovery.save = () => { throw new Error('disk full'); }; diff --git a/test/gnome-extension.test.js b/test/gnome-extension.test.js index bacc29d..fe1f95c 100644 --- a/test/gnome-extension.test.js +++ b/test/gnome-extension.test.js @@ -255,6 +255,21 @@ test('session panel stays closed while Jarvis speaks unless expanded', () => { assert.match(session.view.transcript.children[0].text, /still talking/); }); +test('lazy ASR still shows the microphone as available when capture is up', () => { + const { ConversationView } = harness(); + const popup = new ConversationView({ compact: true }); + popup.setVoiceStatus({ tts: true, input: true, wake: true }); + assert.match(popup.status.text, /SPEECH ON/); + assert.match(popup.status.text, /WAKE ON/); + assert.doesNotMatch(popup.status.text, /MIC UNAVAILABLE/); + assert.equal(popup.talk.label, 'Hold to talk'); + popup.setVoiceStatus({ tts: false, input: false, wake: false }); + assert.match(popup.status.text, /MIC UNAVAILABLE/); + assert.equal(popup.talk.label, 'Mic unavailable'); + assert.match(extensionSource, /Boolean\(voice\.capture\)/); + assert.doesNotMatch(extensionSource, /voice\.asr && voice\.capture/); +}); + test('voice status lives in the panel chip, not a floating overlay', () => { const { JarvisOsd, SessionPanel, timers, chrome } = harness(); const osd = new JarvisOsd(); diff --git a/test/install.test.js b/test/install.test.js index 045a051..4d28356 100644 --- a/test/install.test.js +++ b/test/install.test.js @@ -167,8 +167,11 @@ test('install.sh seeds default config when missing and does not require first-ru const config = JSON.parse(await readFile(path.join(home, '.config/jarvis/config.json'), 'utf8')); assert.equal(config.wakePhrase, 'hey jarvis'); + assert.equal(config.wakeCommand, 'jarvis-wake-bridge'); assert.equal(config.modelProfile, 'laptop-16gb'); assert.equal(config.ttsEnabled, true); + assert.equal(config.fsAccess, 'workspace'); + assert.equal(config.freeVramOnIdle, true); assert.match(result.stdout, /Jarvis is ready/); assert.doesNotMatch(result.stdout, /first-run/); const installSource = readFileSync(new URL('../packaging/install.sh', import.meta.url), 'utf8'); diff --git a/test/review-regressions.test.js b/test/review-regressions.test.js index 1720792..c749fae 100644 --- a/test/review-regressions.test.js +++ b/test/review-regressions.test.js @@ -1,19 +1,20 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { EventEmitter } from 'node:events'; -import { mkdtemp, mkdir, writeFile, readFile, symlink } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; +import { mkdtemp, mkdir, writeFile, readFile, symlink, rm } from 'node:fs/promises'; +import os, { tmpdir } from 'node:os'; import path from 'node:path'; import { createRequire } from 'node:module'; import { ComputerUseSession } from '../computer-use/session.js'; import { ComputerActuator } from '../computer-use/actuator.js'; import { PortalInputBackend } from '../computer-use/portal-input.js'; import { createComputerObserveTools } from '../skills/computer-observe.js'; -import { createPhase2Tools } from '../skills/phase2-tools.js'; +import { createPhase2Tools, filesystemRoots } from '../skills/phase2-tools.js'; import { VoiceStateMachine } from '../daemon/voice-state.js'; import { assertLocalEndpoint } from '../daemon/network-policy.js'; const require = createRequire(import.meta.url); const custom = require('../vendor/agent-harness/agent/custom-tools.js'); +const sandbox = require('../vendor/agent-harness/agent/sandbox.js'); test('custom tool permissions survive registration and default to confirmation', () => { const id = 'review-permissions'; @@ -29,6 +30,28 @@ test('custom tool permissions survive registration and default to confirmation', } finally { custom.clear(id); } }); +test('filesystem access lets path tools reach home while workspace stays jailed', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'jarvis-fs-access-')); + const workspace = path.join(dir, 'workspace'); await mkdir(workspace); + const homeProbe = await mkdtemp(path.join(os.homedir(), '.jarvis-fs-test-')); + try { + await writeFile(path.join(homeProbe, 'note.txt'), 'from-home'); + const jailed = Object.fromEntries(createPhase2Tools({ cwd: workspace, roots: filesystemRoots('workspace', workspace) }).map(t => [t.name, t])); + await assert.rejects(jailed.fs_read.execute({ file: path.join(homeProbe, 'note.txt') }), /outside/); + const opened = Object.fromEntries(createPhase2Tools({ cwd: workspace, roots: filesystemRoots('filesystem', workspace) }).map(t => [t.name, t])); + assert.equal(await opened.fs_read.execute({ file: path.join(homeProbe, 'note.txt') }), 'from-home'); + const home = Object.fromEntries(createPhase2Tools({ cwd: workspace, roots: filesystemRoots('home', workspace) }).map(t => [t.name, t])); + assert.equal(await home.fs_read.execute({ file: path.join(homeProbe, 'note.txt') }), 'from-home'); + sandbox.setGrants({ roots: [path.resolve(workspace)] }); + assert.equal(sandbox.isAllowed('jarvis-qvac', path.join(homeProbe, 'note.txt')), false); + sandbox.setGrants({ roots: [path.resolve(workspace), '/'] }); + assert.equal(sandbox.isAllowed('jarvis-qvac', path.join(homeProbe, 'note.txt')), true); + } finally { + await rm(homeProbe, { recursive: true, force: true }); + await rm(dir, { recursive: true, force: true }); + } +}); + test('workspace tools reject outside roots and symlink escapes and count UTF-8 bytes', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'jarvis-path-test-')); const root = path.join(dir, 'workspace'); await mkdir(root); diff --git a/test/settings.test.js b/test/settings.test.js index 74cebff..03ca435 100644 --- a/test/settings.test.js +++ b/test/settings.test.js @@ -171,5 +171,13 @@ test('a disabled microphone never starts capture while speech output remains ava const capture = new EventEmitter(); capture.start = () => assert.fail('microphone should be disabled'); const loop = new VoiceLoop({ capture, asr: null, tts: { start: async () => {} } }); await loop.start(); - assert.equal(loop.status.capture, false); assert.equal(loop.status.asr, false); assert.equal(loop.status.tts, true); + assert.equal(loop.status.capture, false); assert.equal(loop.status.asr, false); assert.equal(loop.status.tts, false); + assert.equal(await loop.ensureTts(), true); assert.equal(loop.status.tts, true); +}); + +test('catalog defaults enable CPU wake, workspace files, and idle VRAM unload', () => { + const settings = voiceSettings({}); + assert.equal(settings.fsAccess, 'workspace'); + assert.equal(settings.freeVramOnIdle, true); + assert.equal(settings.wakeCommand, 'jarvis-wake-bridge'); }); diff --git a/test/voice-phase4.test.js b/test/voice-phase4.test.js index 6eba14e..7048d08 100644 --- a/test/voice-phase4.test.js +++ b/test/voice-phase4.test.js @@ -117,7 +117,7 @@ test('PCM packing uses even s16le sample pairs', () => { assert.equal(fromBytes.samples[0], 256); }); -test('ASR failure preserves speech output and reports unavailable microphone', async () => { +test('ASR failure keeps the microphone hot and still allows speech output', async () => { const capture = new EventEmitter(); let captured = false; capture.start = () => { captured = true; }; capture.stop = () => {}; let spoken = ''; @@ -127,12 +127,14 @@ test('ASR failure preserves speech output and reports unavailable microphone', a playback: { play: async () => {}, stop() {} }, }); loop.on('error', () => {}); - await loop.start(); await loop.speak('Speech still works.'); - assert.equal(spoken, 'Speech still works.'); assert.equal(captured, false); - assert.equal(loop.status.tts, true); assert.equal(loop.status.asr, false); - spoken = ''; - await loop.speak('OK.'); - assert.equal(spoken, 'OK.'); + await loop.start(); + assert.equal(captured, true); + assert.equal(loop.status.asr, false); + assert.equal(loop.status.tts, false); + await loop.speak('Speech still works.'); + assert.equal(spoken, 'Speech still works.'); + assert.equal(loop.status.tts, true); + assert.equal(await loop.ensureAsr(), false); assert.match(loop.status.errors.asr, /Whisper/); await loop.stop(); }); @@ -142,8 +144,11 @@ test('TTS failure preserves ASR and push to talk without wake command', async () asr: { start: async () => {} }, tts: { start: async () => { throw new Error('TTS failed'); } }, playback: { stop() {} }, }); - loop.on('error', () => {}); await loop.start(); loop.setPushToTalk(true); + loop.on('error', () => {}); await loop.start(); + assert.equal(loop.status.asr, false); + await loop.setPushToTalk(true); assert.equal(loop.status.asr, true); assert.equal(loop.ptt, true); + assert.equal(await loop.ensureTts(), false); assert.equal(loop.status.tts, false); assert.equal(loop.status.wake, false); await loop.stop(); }); @@ -169,7 +174,7 @@ test('voice loop retries microphone capture after PipeWire comes up', async () = }); loop.on('error', () => {}); await loop.start(); - assert.equal(loop.status.asr, true); + assert.equal(loop.status.asr, false); await new Promise((resolve) => setTimeout(resolve, 5)); assert.equal(loop.status.capture, false); await new Promise((resolve) => setTimeout(resolve, 40)); @@ -194,7 +199,10 @@ test('voice loop retries ASR after a first-login GPU miss', async () => { loop.on('error', () => {}); await loop.start(); assert.equal(loop.status.asr, false); - assert.equal(capture.started, undefined); + assert.equal(capture.started, true); + assert.equal(attempts, 0); + assert.equal(await loop.ensureAsr(), false); + assert.equal(attempts, 1); await new Promise((resolve) => setTimeout(resolve, 50)); assert.equal(attempts, 2); assert.equal(loop.status.asr, true); @@ -207,3 +215,61 @@ test('voice adapters load canonical ASR and TTS plugins', () => { assert.match(source, /whispercpp-transcription/); assert.match(source, /tts-ggml/); }); + +function tonePcm(ms, amplitude = 12_000, hz = 220) { + const samples = Math.floor(16_000 * ms / 1000); + const buf = Buffer.alloc(samples * 2); + for (let i = 0; i < samples; i++) buf.writeInt16LE(Math.floor(amplitude * Math.sin(2 * Math.PI * hz * i / 16_000)), i * 2); + return buf; +} + +test('built-in CPU wake detector fires on a two-peak hey-jarvis cadence', async () => { + const { KeywordWakeEngine, createWakeEngine } = await import('../daemon/wake-engine.js'); + const engine = createWakeEngine({ command: 'jarvis-wake-bridge', phrases: ['hey jarvis'] }); + assert.equal(engine instanceof KeywordWakeEngine, true); + const heard = []; + engine.on('wake', (phrase) => heard.push(phrase)); + engine.push(Buffer.concat([tonePcm(80, 0), tonePcm(220), tonePcm(90, 0), tonePcm(280), tonePcm(200, 0)])); + assert.deepEqual(heard, ['hey jarvis']); + engine.push(tonePcm(800)); + assert.equal(heard.length, 1); +}); + +test('sleeping voice loop still feeds the wake detector and skips VAD', () => { + const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {}; + let wakeFrames = 0; let vadFrames = 0; + const wake = new WakeEngine({ detect: () => null }); + wake.push = () => { wakeFrames += 1; }; + const vad = new VadSegmenter(); + vad.push = () => { vadFrames += 1; }; + const daemon = new EventEmitter(); daemon.state = 'SLEEPING'; + const loop = new VoiceLoop({ daemon, capture, wake, vad }); + loop.running = true; + loop.pushAudio(Buffer.alloc(640)); + assert.equal(wakeFrames, 1); + assert.equal(vadFrames, 0); +}); + +test('voice start keeps the microphone hot without loading ASR or TTS', async () => { + const capture = new EventEmitter(); let captured = false; + capture.start = () => { captured = true; }; capture.stop = () => {}; + let asrStarts = 0; let ttsStarts = 0; + const loop = new VoiceLoop({ + capture, + asr: { start: async () => { asrStarts += 1; } }, + tts: { start: async () => { ttsStarts += 1; } }, + wake: new WakeEngine({ detect: () => null }), + }); + await loop.start(); + assert.equal(captured, true); + assert.equal(loop.status.capture, true); + assert.equal(loop.status.asr, false); + assert.equal(loop.status.tts, false); + assert.equal(asrStarts, 0); + assert.equal(ttsStarts, 0); + assert.equal(loop.status.wake, true); + await loop.parkModels(); + assert.equal(loop.status.asr, false); + assert.equal(loop.status.tts, false); + await loop.stop(); +});