Better Search

daemon/index.js

@@ -35,7 +35,11 @@ export class JarvisDaemon extends EventEmitter {
     this.computer = new ComputerUseSession({ audit: this.audit, stepsMax: this.settings.computerSteps, grantMinutes: this.settings.computerGrantMinutes, mode: this.settings.computerMode });
     this.input = new PortalInputBackend();
     this.perception = new QvacPerception();
-    this.observer = new DesktopObserver({ normalizer: new FrameNormalizer({ maxLongEdge: this.settings.screenshotMaxEdge, quality: this.settings.screenshotQuality }), ocr: (image) => this.perception.ocr(image) });
+    this.observer = new DesktopObserver({
+      normalizer: new FrameNormalizer({ maxLongEdge: this.settings.screenshotMaxEdge, quality: this.settings.screenshotQuality }),
+      ocr: (image) => this.perception.ocr(image),
+      framebuffer: { capture: (output) => this.input.captureFrame(output) },
+    });
     this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit });
     this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer, actuator: this.actuator, fsAccess: this.settings.fsAccess });
     this.log = new PrivacyLog();
@@ -185,7 +189,7 @@ export class JarvisDaemon extends EventEmitter {
     await this.voiceLoop.ensureAsr?.();
   }
   cancel() { this._askGeneration += 1; this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computerRevoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
-  computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); if (this.settings.computerMode === 'observe') return result; this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; }
+  computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); this.input.grant({ persist, mode: this.settings.computerMode }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; }
   computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
   startVoice() {
     if (this._voiceStarting) return this._voiceStarting;