Updates

daemon/agent-workspace.js

@@ -14,6 +14,7 @@ const SEED_FILES = [
   'TOOLS.md',
   'PERSONA.md',
   'skills/skill-creator/SKILL.md',
+  'skills/browser/SKILL.md',
 ];
 
 export function normalizeAssistantName(value) {
@@ -37,6 +38,32 @@ function copyIfMissing(from, to) {
   return true;
 }
 
+const OLD_AGENTS_BROWSER = '- `web_search` / `google_search` / `fetch_page` / `web_fetch` / `wiki_search` / `hn_search` / `code_search` — Playwright Chromium. For cookie walls or extra clicks, call `browser` with snapshot then click or type.';
+const NEW_AGENTS_BROWSER = `- Web tools share one headed Playwright Chromium window. \`web_search\` / \`google_search\` / \`wiki_search\` / \`hn_search\` / \`code_search\` find links. \`fetch_page\` / \`web_fetch\` read a public page. For cookie banners, forms, logins, or leftover challenges, call \`browser\`. Before a multi-step browse, \`read_file\` \`skills/browser/SKILL.md\`.
+- \`browser\` actions: \`navigate\` (needs \`url\`), \`snapshot\`, \`click\` (\`ref\` from the last snapshot), \`type\` (\`ref\` + \`text\`, optional \`submit\`), \`press\` (\`key\`), \`scroll\` (\`dy\`), \`wait\` (\`ms\`). Snapshot or navigate first. Refs change after every click. Do not use \`cu_observe\` or the shell for websites.`;
+const OLD_TOOLS_BROWSER = '- Public HTTP via curl or wget is blocked. Use `web_search` / `web_fetch` in the Jarvis Chromium window. For cookie walls, call `browser` with snapshot then click or type.';
+const NEW_TOOLS_BROWSER = `- Public HTTP via curl or wget is blocked. Use \`web_search\` / \`web_fetch\` in the Jarvis Chromium window.
+
+## Browser
+
+- Search, fetch, and \`browser\` share one headed Playwright Chromium window.
+- \`web_search\` finds links. \`fetch_page\` reads a public page.
+- Cookie walls, forms, leftover challenges: call \`browser\`. \`read_file\` \`skills/browser/SKILL.md\` for the playbook.
+- \`browser\` actions: \`navigate\` + \`url\`, \`snapshot\`, \`click\`/\`type\` with \`ref\` from the last snapshot, \`press\` + \`key\`, \`scroll\` + \`dy\`, \`wait\` + \`ms\`.
+- Snapshot or navigate before every click or type. Refs go stale after a click.
+- Do not use \`cu_observe\` or the shell for websites.`;
+
+function replaceOnce(file, from, to) {
+  try {
+    const text = fs.readFileSync(file, 'utf8');
+    if (!from || !text.includes(from)) return false;
+    fs.writeFileSync(file, text.replace(from, to));
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 function escapeRegExp(value) {
   return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
 }
@@ -90,6 +117,8 @@ export function ensureAgentWorkspace({ name = 'Jarvis', prompt = '' } = {}) {
   for (const rel of SEED_FILES) {
     copyIfMissing(path.join(TEMPLATE_DIR, rel), path.join(dest, rel));
   }
+  replaceOnce(path.join(dest, 'AGENTS.md'), OLD_AGENTS_BROWSER, NEW_AGENTS_BROWSER);
+  replaceOnce(path.join(dest, 'TOOLS.md'), OLD_TOOLS_BROWSER, NEW_TOOLS_BROWSER);
   applyAssistantName(dest, name);
   applyAssistantPrompt(dest, prompt);
   return dest;

daemon/dbus-service.js

@@ -39,6 +39,9 @@ export async function serveOnSessionBus(daemon) {
     ComputerGrant(persist) { daemon.computerGrant(persist); }
     ComputerRevoke() { daemon.computerRevoke(); }
     ComputerStatus() { return JSON.stringify(daemon.computer.status()); }
+    WebcamGrant() { daemon.webcamGrant(); }
+    WebcamRevoke() { daemon.webcamRevoke(); }
+    WebcamStatus() { return JSON.stringify(daemon.camera.status()); }
     GetRuntimeStatus() { return daemon.runtimeStatus(); }
     AssessModelFit(model) { return daemon.assessModelFit(model); }
     DownloadModel(model) { return daemon.downloadModel(model); }
@@ -89,6 +92,9 @@ export async function serveOnSessionBus(daemon) {
     ComputerGrant: { inSignature: 'b', outSignature: '', method: 'ComputerGrant' },
     ComputerRevoke: { inSignature: '', outSignature: '', method: 'ComputerRevoke' },
     ComputerStatus: { inSignature: '', outSignature: 's', method: 'ComputerStatus' },
+    WebcamGrant: { inSignature: '', outSignature: '', method: 'WebcamGrant' },
+    WebcamRevoke: { inSignature: '', outSignature: '', method: 'WebcamRevoke' },
+    WebcamStatus: { inSignature: '', outSignature: 's', method: 'WebcamStatus' },
     GetRuntimeStatus: { inSignature: '', outSignature: 's', method: 'GetRuntimeStatus' },
     AssessModelFit: { inSignature: 's', outSignature: 's', method: 'AssessModelFit' },
     DownloadModel: { inSignature: 's', outSignature: 's', method: 'DownloadModel' },

daemon/harness-bridge.js

@@ -9,6 +9,8 @@ import { voiceSystemPrompt, parseHudSidecar } from '../skills/voice-prompt.js';
 import { createComputerObserveTools } from '../skills/computer-observe.js';
 import { createComputerActTools } from '../skills/computer-act.js';
 import { createPhase9GatewayTool } from '../skills/phase9-tools.js';
+import { createBrowserTools } from '../skills/browser-tools.js';
+import { createWebcamTools } from '../skills/webcam-tools.js';
 import { ensureAgentWorkspace, normalizeAssistantName } from './agent-workspace.js';
 
 export function harnessRoots(fsAccess) {
@@ -18,28 +20,33 @@ export function harnessRoots(fsAccess) {
 }
 
 export class HarnessBridge extends EventEmitter {
-  constructor({ cwd, model = QVAC_MASTER.model, tools = [], computer, observer, actuator, permissionMode = 'ask', fsAccess } = {}) {
+  constructor({ cwd, model = QVAC_MASTER.model, tools = [], computer, observer, actuator, browser, camera, webcam, webcamNormalizer, permissionMode = 'ask', fsAccess } = {}) {
     super();
     const settings = voiceSettings();
     const access = fsAccess ?? settings.fsAccess;
     const assistantName = normalizeAssistantName(settings.assistantName);
     const workspace = cwd || ensureAgentWorkspace({ name: assistantName, prompt: settings.assistantPrompt });
     const roots = harnessRoots(access);
+    this.assistantName = assistantName;
+    this.assistantPrompt = settings.assistantPrompt;
     this.options = {
       cwd: workspace,
       roots,
       model,
       tools: [
-        ...createRuntimeTools({ computer }),
+        ...createRuntimeTools({ computer, camera }),
         ...createPhase2Tools({ cwd: workspace, computer, roots: filesystemRoots(access, workspace) }),
         ...createComputerObserveTools({ computer, observer }),
         ...createComputerActTools({ actuator }),
         ...createQvacTools(),
         ...createPhase9GatewayTool(),
+        ...createBrowserTools({ browser }),
+        ...createWebcamTools({ camera, capture: webcam, normalizer: webcamNormalizer }),
         ...tools,
       ],
       builtinTools: ['read_file', 'write_file', 'search_replace', 'list_dir', 'grep', 'run_terminal_cmd', 'web_fetch', 'fetch_page', 'google_search', 'web_search', 'wiki_search', 'hn_search', 'code_search', 'todo_write', 'task', 'update_goal', 'memory_search', 'memory_get', 'memory_write'],
       webFetch: true,
+      browser,
       permissionMode,
       origin: 'jarvis-qvac',
       system: voiceSystemPrompt(assistantName, settings.assistantPrompt),
@@ -78,7 +85,19 @@ export class HarnessBridge extends EventEmitter {
     return this.session;
   }
 
+  setIdentity(name, extra) {
+    this.assistantName = normalizeAssistantName(name);
+    this.assistantPrompt = extra || '';
+    this.refreshPrompt();
+  }
+
+  refreshPrompt() {
+    if (!this.options) return;
+    this.options.system = voiceSystemPrompt(this.assistantName, this.assistantPrompt);
+  }
+
   async ask(text) {
+    this.refreshPrompt();
     if (!this.session) await this.start();
     // Some QVAC runs deliver the final assistant text through the stream but
     // omit it from the final envelope after a tool call. Keep the current

daemon/index.js

@@ -6,6 +6,8 @@ import { ttsConfiguration } from './tts-config.js';
 import { EventEmitter } from 'node:events';
 import { HarnessBridge } from './harness-bridge.js';
 import { ComputerUseSession } from '../computer-use/session.js';
+import { CameraSession } from '../computer-use/camera-session.js';
+import { PortalCamera } from '../computer-use/portal-camera.js';
 import { VoiceStateMachine } from './voice-state.js';
 import { QvacScheduler } from './qvac-scheduler.js';
 import { cancelQvac, closeQvac, resumeQvac, suspendQvac, callQvac, cancelQvacRequest, qvacStatus } from './qvac-master.js';
@@ -19,9 +21,10 @@ import { QvacPerception } from './perception.js';
 import { ComputerAudit } from '../computer-use/audit.js';
 import { PortalInputBackend } from '../computer-use/portal-input.js';
 import { ComputerActuator } from '../computer-use/actuator.js';
+import { BrowserClient } from '../browser-use/client.js';
 import { RuntimeTelemetry } from './telemetry.js';
 import { StateRecovery } from './recovery.js';
-import { spokenReply, voiceSystemPrompt } from '../skills/voice-prompt.js';
+import { spokenReply } from '../skills/voice-prompt.js';
 import { ensureAgentWorkspace, applyAssistantName, applyAssistantPrompt, normalizeAssistantName } from './agent-workspace.js';
 
 function chunkText(ev) {
@@ -41,6 +44,18 @@ export class JarvisDaemon extends EventEmitter {
     this.scheduler = new QvacScheduler({ concurrency: 1 });
     this.audit = new ComputerAudit();
     this.computer = new ComputerUseSession({ audit: this.audit, stepsMax: this.settings.computerSteps, grantMinutes: this.settings.computerGrantMinutes, mode: this.settings.computerMode });
+    this.camera = new CameraSession({
+      audit: this.audit,
+      enabled: this.settings.webcamEnabled,
+      grantMinutes: this.settings.webcamGrantMinutes,
+      device: this.settings.webcamDevice,
+    });
+    this.webcam = new PortalCamera();
+    this.webcamNormalizer = new FrameNormalizer({
+      tmpDir: '/tmp/jarvis-webcam',
+      maxLongEdge: this.settings.webcamMaxEdge,
+      quality: this.settings.screenshotQuality,
+    });
     this.input = new PortalInputBackend({ onClose: () => this.computerRevoke() });
     this.perception = new QvacPerception();
     this.observer = new DesktopObserver({
@@ -49,7 +64,8 @@ export class JarvisDaemon extends EventEmitter {
       framebuffer: { capture: (output) => this.input.captureFrame(output), streams: () => this.input.streams },
     });
     this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit });
-    this.harness = new HarnessBridge({ cwd: this.workspace, computer: this.computer, observer: this.observer, actuator: this.actuator, fsAccess: this.settings.fsAccess });
+    this.browser = new BrowserClient();
+    this.harness = new HarnessBridge({ cwd: this.workspace, computer: this.computer, observer: this.observer, actuator: this.actuator, browser: this.browser, camera: this.camera, webcam: this.webcam, webcamNormalizer: this.webcamNormalizer, fsAccess: this.settings.fsAccess });
     this.log = new PrivacyLog();
     this.locked = false;
     this.muted = false;
@@ -202,7 +218,7 @@ export class JarvisDaemon extends EventEmitter {
     if (!this.voiceLoop.asr) this.voiceLoop.asr = new QvacVoiceAdapter({ role: 'asr', settings: this.settings });
     await this.voiceLoop.ensureAsr?.();
   }
-  cancel() { this._askGeneration += 1; this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computerRevoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
+  cancel() { this._askGeneration += 1; this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computerRevoke(); this.webcamRevoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); }
   computerGrant(persist = false) {
     if (this.locked) throw new Error('Unlock the desktop before granting computer use');
     this.computerRevoke();
@@ -220,6 +236,30 @@ export class JarvisDaemon extends EventEmitter {
     return result;
   }
   computerRevoke() { this._computerGeneration = (this._computerGeneration || 0) + 1; clearTimeout(this._computerExpiry); if (this.observer) this.observer.lastTree = []; this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); }
+  webcamGrant() {
+    if (this.locked) throw new Error('Unlock the desktop before granting camera access');
+    this.webcamRevoke();
+    this.camera.enabled = true;
+    const result = this.camera.grant();
+    this.camera.expiresAt = Date.now() + this.camera.grantMinutes * 60000;
+    this._webcamExpiry = setTimeout(() => this.webcamRevoke(), Math.max(0, this.camera.expiresAt - Date.now()));
+    this._webcamExpiry.unref?.();
+    const generation = this._webcamGeneration;
+    this.webcam.access({ device: this.camera.device }).then((payload) => {
+      if (generation !== this._webcamGeneration || !this.camera.status().active) return;
+      this.camera.setBackend(payload.via || 'portal');
+    }).catch((error) => {
+      if (generation !== this._webcamGeneration) return;
+      this.emit('Error', 'WEBCAM_GRANT', error.message);
+    });
+    return result;
+  }
+  webcamRevoke() {
+    this._webcamGeneration = (this._webcamGeneration || 0) + 1;
+    clearTimeout(this._webcamExpiry);
+    this.camera?.revoke();
+    if (this.camera) this.camera.enabled = Boolean(this.settings?.webcamEnabled);
+  }
   startVoice() {
     if (this._voiceStarting) return this._voiceStarting;
     this._voiceStarting = this._startVoice().finally(() => { this._voiceStarting = null; });
@@ -266,12 +306,16 @@ export class JarvisDaemon extends EventEmitter {
     const who = applyAssistantName(this.workspace, next.assistantName);
     applyAssistantPrompt(this.workspace, next.assistantPrompt);
     if (who !== normalizeAssistantName(previous.assistantName) || String(next.assistantPrompt || '') !== String(previous.assistantPrompt || '')) {
-      this.harness.options.system = voiceSystemPrompt(who, next.assistantPrompt);
+      this.harness.setIdentity(who, next.assistantPrompt);
       await this.harness.resetContext();
     }
     if (['computerMode', 'computerSteps', 'computerGrantMinutes'].some(key => next[key] !== previous[key])) this.computerRevoke();
     Object.assign(this.computer, { mode: next.computerMode, stepsMax: next.computerSteps, grantMinutes: next.computerGrantMinutes });
     Object.assign(this.observer.normalizer, { maxLongEdge: next.screenshotMaxEdge, quality: next.screenshotQuality });
+    if (!next.webcamEnabled && previous.webcamEnabled) this.webcamRevoke();
+    else if (['webcamGrantMinutes', 'webcamDevice'].some(key => next[key] !== previous[key])) this.webcamRevoke();
+    Object.assign(this.camera, { enabled: next.webcamEnabled, grantMinutes: next.webcamGrantMinutes, device: next.webcamDevice });
+    Object.assign(this.webcamNormalizer, { maxLongEdge: next.webcamMaxEdge, quality: next.screenshotQuality });
     await this.startVoice();
     this.voice.cancel(); this.setState('ARMED');
     return JSON.stringify({ applied: true, voice: this.voiceLoop.status, environmentOverrides: ['JARVIS_TTS_MODEL', 'JARVIS_ASR_MODEL', 'JARVIS_WAKE_COMMAND', 'JARVIS_QVAC_MODEL'].filter(key => process.env[key]), restartRequired: ['chatModel', 'modelProfile', 'maxTurns', 'maxShellCalls', 'maxToolRounds', 'fsAccess'].filter(key => next[key] !== this.startupSettings[key]) });
@@ -321,11 +365,11 @@ export class JarvisDaemon extends EventEmitter {
       this.setState('ARMED');
     }
   }
-  runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), settings: this.settings, computer: this.computer.status(), voice: this.voiceLoop ? { ...this.voiceLoop.metrics.snapshot(), ...this.voiceLoop.status, muted: this.muted } : { muted: this.muted }, muted: this.muted, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }
+  runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), settings: this.settings, computer: this.computer.status(), camera: this.camera.status(), voice: this.voiceLoop ? { ...this.voiceLoop.metrics.snapshot(), ...this.voiceLoop.status, muted: this.muted } : { muted: this.muted }, muted: this.muted, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }
   async assessModelFit(model) { return JSON.stringify(await callQvac('assessModelFit', { modelSrc: String(model) })); }
   async downloadModel(model) { return JSON.stringify(await callQvac('downloadAsset', { modelSrc: String(model) })); }
   async cancelModel(model) { return JSON.stringify(await cancelQvacRequest({ modelId: String(model) })); }
-  async wipeComputerTraces() { await this.audit.wipeTemp(); return true; }
+  async wipeComputerTraces() { await this.audit.wipeTemp(); await this.audit.wipeTemp('/tmp/jarvis-webcam'); return true; }
   handleLockScreen(locked) { this.locked = Boolean(locked); if (this.locked) { this.cancel(); this.setState('ARMED'); } this.emit('LockScreenChanged', this.locked); }
   tickIdle() { if (!this.locked && this.voice.expireIdle() === 'SLEEPING' && this.state !== 'SLEEPING') this.sleep(); }
   async close() {
@@ -333,7 +377,7 @@ export class JarvisDaemon extends EventEmitter {
     this.cancel();
     try { this.recovery.save({ state: 'ARMED', mode: this.mode }); }
     catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); }
-    try { await this.voiceLoop?.stop?.(); } finally { await this.harness.close(); }
+    try { await this.voiceLoop?.stop?.(); } finally { this.browser?.close?.(); await this.harness.close(); }
   }
 }
 

daemon/voice-loop.js

@@ -14,7 +14,8 @@ const FAST_COMMANDS = new Map([
   ['privacy mode', 'sleep'], ['repeat that', 'repeat'], ['dictate this', 'dictate'],
   ['look at my screen', 'screen'], ['use the computer', 'computer'], ['take the wheel', 'computer'],
   ['switch to compose', 'compose'], ['switch to imagine', 'imagine'], ['switch to files', 'files'],
-  ['switch to computer', 'computer'],
+  ['switch to computer', 'computer'], ['use the camera', 'camera'], ['use my webcam', 'camera'],
+  ['allow the camera', 'camera'],
 ]);
 
 export function fastCommand(text) { return FAST_COMMANDS.get(String(text || '').trim().toLowerCase()) || null; }
@@ -206,6 +207,7 @@ export class VoiceLoop extends EventEmitter {
     if (command === 'cancel' || command === 'sleep') { this.daemon?.cancel?.(); if (command === 'sleep') await this.daemon?.sleep?.(); return; }
     if (command === 'repeat') { this.daemon?.say?.(this.daemon?.lastReply || 'There is nothing to repeat.'); return; }
     if (command === 'computer') { this.daemon?.computerGrant?.(false); return; }
+    if (command === 'camera') { this.daemon?.webcamGrant?.(); return; }
     if (command === 'dictate') { this.emit('dictate'); return; }
     if (command === 'screen') { await this.daemon?.ask?.('What is on my screen?'); return; }
     await this.daemon?.ask?.(text);