P6

daemon/harness-bridge.js

@@ -4,11 +4,12 @@ import { createRuntimeTools } from '../skills/runtime-tools.js';
 import { createPhase2Tools } from '../skills/phase2-tools.js';
 import { createQvacTools } from '../skills/qvac-tools.js';
 import { VOICE_SYSTEM_PROMPT, parseHudSidecar } from '../skills/voice-prompt.js';
+import { createComputerObserveTools } from '../skills/computer-observe.js';
 
 export class HarnessBridge extends EventEmitter {
-  constructor({ cwd = process.cwd(), model = 'qwen3.5-4b', tools = [], computer, permissionMode = 'ask' } = {}) {
+  constructor({ cwd = process.cwd(), model = 'qwen3.5-4b', tools = [], computer, observer, permissionMode = 'ask' } = {}) {
     super();
-    this.options = { cwd, model, tools: [...createRuntimeTools({ computer }), ...createPhase2Tools({ cwd, computer }), ...createQvacTools(), ...tools], permissionMode, origin: 'jarvis-qvac', system: VOICE_SYSTEM_PROMPT };
+    this.options = { cwd, model, tools: [...createRuntimeTools({ computer }), ...createPhase2Tools({ cwd, computer }), ...createComputerObserveTools({ computer, observer }), ...createQvacTools(), ...tools], permissionMode, origin: 'jarvis-qvac', system: VOICE_SYSTEM_PROMPT };
     this.session = null;
   }
 

daemon/index.js

@@ -8,6 +8,8 @@ import { PrivacyLog } from './privacy-log.js';
 import { VoiceLoop } from './voice-loop.js';
 import { QvacVoiceAdapter } from './voice-adapters.js';
 import { createWakeEngine } from './wake-engine.js';
+import { DesktopObserver } from '../computer-use/observer.js';
+import { QvacPerception } from './perception.js';
 
 export class JarvisDaemon extends EventEmitter {
   constructor() {
@@ -16,7 +18,9 @@ export class JarvisDaemon extends EventEmitter {
     this.voice = new VoiceStateMachine();
     this.scheduler = new QvacScheduler({ concurrency: 1 });
     this.computer = new ComputerUseSession();
-    this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer });
+    this.perception = new QvacPerception();
+    this.observer = new DesktopObserver({ ocr: (image) => this.perception.ocr(image) });
+    this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer });
     this.log = new PrivacyLog();
     this.locked = false;
     this.lastReply = '';

daemon/perception.js

@@ -0,0 +1,14 @@
+import { acquireQvac, releaseQvac, loadAuxiliaryModel, qvacSdk, withQvacMaster } from './qvac-master.js';
+
+export class QvacPerception {
+  constructor({ model = process.env.JARVIS_OCR_MODEL || 'OCR_LATIN' } = {}) { this.model = model; this.modelId = null; }
+  async ocr(image) {
+    await acquireQvac();
+    try {
+      this.modelId ||= await loadAuxiliaryModel(this.model, {});
+      const sdk = await qvacSdk();
+      const result = await withQvacMaster(() => sdk.ocr({ modelId: this.modelId, image, stream: false }));
+      return result.blocks;
+    } finally { releaseQvac(); }
+  }
+}