Computer Use Updates

daemon/index.js

@@ -41,12 +41,12 @@ export class JarvisDaemon extends EventEmitter {
     this.scheduler = new QvacScheduler({ concurrency: 1 });
     this.audit = new ComputerAudit();
     this.computer = new ComputerUseSession({ audit: this.audit, stepsMax: this.settings.computerSteps, grantMinutes: this.settings.computerGrantMinutes, mode: this.settings.computerMode });
-    this.input = new PortalInputBackend();
+    this.input = new PortalInputBackend({ onClose: () => this.computerRevoke() });
     this.perception = new QvacPerception();
     this.observer = new DesktopObserver({
       normalizer: new FrameNormalizer({ maxLongEdge: this.settings.screenshotMaxEdge, quality: this.settings.screenshotQuality }),
       ocr: (image) => this.perception.ocr(image),
-      framebuffer: { capture: (output) => this.input.captureFrame(output) },
+      framebuffer: { capture: (output) => this.input.captureFrame(output), streams: () => this.input.streams },
     });
     this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit });
     this.harness = new HarnessBridge({ cwd: this.workspace, computer: this.computer, observer: this.observer, actuator: this.actuator, fsAccess: this.settings.fsAccess });
@@ -203,8 +203,23 @@ export class JarvisDaemon extends EventEmitter {
     await this.voiceLoop.ensureAsr?.();
   }
   cancel() { this._askGeneration += 1; this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computerRevoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
-  computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); this.input.grant({ persist, mode: this.settings.computerMode }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; }
-  computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
+  computerGrant(persist = false) {
+    if (this.locked) throw new Error('Unlock the desktop before granting computer use');
+    this.computerRevoke();
+    const result = this.computer.grant({ persist });
+    this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result }));
+    const generation = this._computerGeneration;
+    this.input.grant({ persist, mode: this.settings.computerMode }).then((backend) => {
+      if (generation !== this._computerGeneration || !this.computer.status().active) return;
+      this.computer.expiresAt = Date.now() + this.computer.grantMinutes * 60000;
+      this._computerExpiry = setTimeout(() => this.computerRevoke(), Math.max(0, this.computer.expiresAt - Date.now()));
+      this._computerExpiry.unref?.();
+      this.computer.setBackend(backend.backend);
+      this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend }));
+    }).catch((error) => { if (generation !== this._computerGeneration) return; this.computerRevoke(); this.emit('Error', 'CU_GRANT', error.message); });
+    return result;
+  }
+  computerRevoke() { this._computerGeneration = (this._computerGeneration || 0) + 1; clearTimeout(this._computerExpiry); if (this.observer) this.observer.lastTree = []; this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
   startVoice() {
     if (this._voiceStarting) return this._voiceStarting;
     this._voiceStarting = this._startVoice().finally(() => { this._voiceStarting = null; });