snxraven/gnome-jarvis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Updates
Rolling release / release (push) Successful in 8m31s
Details

Browse Source
This commit is contained in:
snxraven
2026-09-13 22:24:14 -04:00
parent 1ca4224377
commit a097adf4eb
62 changed files with 2707 additions and 957 deletions
Download Patch File Download Diff File Expand all files Collapse all files
daemon/index.js
+51 -7
View File
@@ -6,6 +6,8 @@ import { ttsConfiguration } from './tts-config.js';
import { EventEmitter } from 'node:events';
import { HarnessBridge } from './harness-bridge.js';
import { ComputerUseSession } from '../computer-use/session.js';
import { CameraSession } from '../computer-use/camera-session.js';
import { PortalCamera } from '../computer-use/portal-camera.js';
import { VoiceStateMachine } from './voice-state.js';
import { QvacScheduler } from './qvac-scheduler.js';
import { cancelQvac, closeQvac, resumeQvac, suspendQvac, callQvac, cancelQvacRequest, qvacStatus } from './qvac-master.js';
@@ -19,9 +21,10 @@ import { QvacPerception } from './perception.js';
import { ComputerAudit } from '../computer-use/audit.js';
import { PortalInputBackend } from '../computer-use/portal-input.js';
import { ComputerActuator } from '../computer-use/actuator.js';
import { BrowserClient } from '../browser-use/client.js';
import { RuntimeTelemetry } from './telemetry.js';
import { StateRecovery } from './recovery.js';
import { spokenReply, voiceSystemPrompt } from '../skills/voice-prompt.js';
import { spokenReply } from '../skills/voice-prompt.js';
import { ensureAgentWorkspace, applyAssistantName, applyAssistantPrompt, normalizeAssistantName } from './agent-workspace.js';
function chunkText(ev) {
@@ -41,6 +44,18 @@ export class JarvisDaemon extends EventEmitter {
this.scheduler = new QvacScheduler({ concurrency: 1 });
this.audit = new ComputerAudit();
this.computer = new ComputerUseSession({ audit: this.audit, stepsMax: this.settings.computerSteps, grantMinutes: this.settings.computerGrantMinutes, mode: this.settings.computerMode });
this.camera = new CameraSession({
audit: this.audit,
enabled: this.settings.webcamEnabled,
grantMinutes: this.settings.webcamGrantMinutes,
device: this.settings.webcamDevice,
});
this.webcam = new PortalCamera();
this.webcamNormalizer = new FrameNormalizer({
tmpDir: '/tmp/jarvis-webcam',
maxLongEdge: this.settings.webcamMaxEdge,
quality: this.settings.screenshotQuality,
});
this.input = new PortalInputBackend({ onClose: () => this.computerRevoke() });
this.perception = new QvacPerception();
this.observer = new DesktopObserver({
@@ -49,7 +64,8 @@ export class JarvisDaemon extends EventEmitter {
framebuffer: { capture: (output) => this.input.captureFrame(output), streams: () => this.input.streams },
});
this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit });
this.harness = new HarnessBridge({ cwd: this.workspace, computer: this.computer, observer: this.observer, actuator: this.actuator, fsAccess: this.settings.fsAccess });
this.browser = new BrowserClient();
this.harness = new HarnessBridge({ cwd: this.workspace, computer: this.computer, observer: this.observer, actuator: this.actuator, browser: this.browser, camera: this.camera, webcam: this.webcam, webcamNormalizer: this.webcamNormalizer, fsAccess: this.settings.fsAccess });
this.log = new PrivacyLog();
this.locked = false;
this.muted = false;
@@ -202,7 +218,7 @@ export class JarvisDaemon extends EventEmitter {
if (!this.voiceLoop.asr) this.voiceLoop.asr = new QvacVoiceAdapter({ role: 'asr', settings: this.settings });
await this.voiceLoop.ensureAsr?.();
}
cancel() { this._askGeneration += 1; this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computerRevoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
cancel() { this._askGeneration += 1; this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computerRevoke(); this.webcamRevoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
computerGrant(persist = false) {
if (this.locked) throw new Error('Unlock the desktop before granting computer use');
this.computerRevoke();
@@ -220,6 +236,30 @@ export class JarvisDaemon extends EventEmitter {
return result;
}
computerRevoke() { this._computerGeneration = (this._computerGeneration || 0) + 1; clearTimeout(this._computerExpiry); if (this.observer) this.observer.lastTree = []; this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
webcamGrant() {
if (this.locked) throw new Error('Unlock the desktop before granting camera access');
this.webcamRevoke();
this.camera.enabled = true;
const result = this.camera.grant();
this.camera.expiresAt = Date.now() + this.camera.grantMinutes * 60000;
this._webcamExpiry = setTimeout(() => this.webcamRevoke(), Math.max(0, this.camera.expiresAt - Date.now()));
this._webcamExpiry.unref?.();
const generation = this._webcamGeneration;
this.webcam.access({ device: this.camera.device }).then((payload) => {
if (generation !== this._webcamGeneration || !this.camera.status().active) return;
this.camera.setBackend(payload.via || 'portal');
}).catch((error) => {
if (generation !== this._webcamGeneration) return;
this.emit('Error', 'WEBCAM_GRANT', error.message);
});
return result;
}
webcamRevoke() {
this._webcamGeneration = (this._webcamGeneration || 0) + 1;
clearTimeout(this._webcamExpiry);
this.camera?.revoke();
if (this.camera) this.camera.enabled = Boolean(this.settings?.webcamEnabled);
}
startVoice() {
if (this._voiceStarting) return this._voiceStarting;
this._voiceStarting = this._startVoice().finally(() => { this._voiceStarting = null; });
@@ -266,12 +306,16 @@ export class JarvisDaemon extends EventEmitter {
const who = applyAssistantName(this.workspace, next.assistantName);
applyAssistantPrompt(this.workspace, next.assistantPrompt);
if (who !== normalizeAssistantName(previous.assistantName) || String(next.assistantPrompt || '') !== String(previous.assistantPrompt || '')) {
this.harness.options.system = voiceSystemPrompt(who, next.assistantPrompt);
this.harness.setIdentity(who, next.assistantPrompt);
await this.harness.resetContext();
}
if (['computerMode', 'computerSteps', 'computerGrantMinutes'].some(key => next[key] !== previous[key])) this.computerRevoke();
Object.assign(this.computer, { mode: next.computerMode, stepsMax: next.computerSteps, grantMinutes: next.computerGrantMinutes });
Object.assign(this.observer.normalizer, { maxLongEdge: next.screenshotMaxEdge, quality: next.screenshotQuality });
if (!next.webcamEnabled && previous.webcamEnabled) this.webcamRevoke();
else if (['webcamGrantMinutes', 'webcamDevice'].some(key => next[key] !== previous[key])) this.webcamRevoke();
Object.assign(this.camera, { enabled: next.webcamEnabled, grantMinutes: next.webcamGrantMinutes, device: next.webcamDevice });
Object.assign(this.webcamNormalizer, { maxLongEdge: next.webcamMaxEdge, quality: next.screenshotQuality });
await this.startVoice();
this.voice.cancel(); this.setState('ARMED');
return JSON.stringify({ applied: true, voice: this.voiceLoop.status, environmentOverrides: ['JARVIS_TTS_MODEL', 'JARVIS_ASR_MODEL', 'JARVIS_WAKE_COMMAND', 'JARVIS_QVAC_MODEL'].filter(key => process.env[key]), restartRequired: ['chatModel', 'modelProfile', 'maxTurns', 'maxShellCalls', 'maxToolRounds', 'fsAccess'].filter(key => next[key] !== this.startupSettings[key]) });
@@ -321,11 +365,11 @@ export class JarvisDaemon extends EventEmitter {
this.setState('ARMED');
}
}
runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), settings: this.settings, computer: this.computer.status(), voice: this.voiceLoop ? { ...this.voiceLoop.metrics.snapshot(), ...this.voiceLoop.status, muted: this.muted } : { muted: this.muted }, muted: this.muted, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }
runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), settings: this.settings, computer: this.computer.status(), camera: this.camera.status(), voice: this.voiceLoop ? { ...this.voiceLoop.metrics.snapshot(), ...this.voiceLoop.status, muted: this.muted } : { muted: this.muted }, muted: this.muted, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }
async assessModelFit(model) { return JSON.stringify(await callQvac('assessModelFit', { modelSrc: String(model) })); }
async downloadModel(model) { return JSON.stringify(await callQvac('downloadAsset', { modelSrc: String(model) })); }
async cancelModel(model) { return JSON.stringify(await cancelQvacRequest({ modelId: String(model) })); }
async wipeComputerTraces() { await this.audit.wipeTemp(); return true; }
async wipeComputerTraces() { await this.audit.wipeTemp(); await this.audit.wipeTemp('/tmp/jarvis-webcam'); return true; }
handleLockScreen(locked) { this.locked = Boolean(locked); if (this.locked) { this.cancel(); this.setState('ARMED'); } this.emit('LockScreenChanged', this.locked); }
tickIdle() { if (!this.locked && this.voice.expireIdle() === 'SLEEPING' && this.state !== 'SLEEPING') this.sleep(); }
async close() {
@@ -333,7 +377,7 @@ export class JarvisDaemon extends EventEmitter {
this.cancel();
try { this.recovery.save({ state: 'ARMED', mode: this.mode }); }
catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); }
try { await this.voiceLoop?.stop?.(); } finally { await this.harness.close(); }
try { await this.voiceLoop?.stop?.(); } finally { this.browser?.close?.(); await this.harness.close(); }
}
}