Updates

apps/gnome-extension/jarvis@qvac.local/extension.js

@@ -66,6 +66,8 @@ export default class JarvisExtension extends Extension {
     this._applyAccessibility();
     this._removeShellService = installShellService();
     this._assistantName = 'Jarvis';
+    this._muted = false;
+    this._muteTouched = false;
     this._indicator = new PanelMenu.Button(0.0, 'Jarvis QVAC', false); this._indicator.accessible_name = 'Jarvis voice assistant';
     this._panelBox = new St.BoxLayout({ style_class: 'jarvis-panel-box', y_align: Clutter.ActorAlign.CENTER });
     this._mark = new St.Icon({ icon_name: 'audio-input-microphone-symbolic', icon_size: 16, style_class: 'jarvis-panel-mark', y_align: Clutter.ActorAlign.CENTER });
@@ -84,14 +86,14 @@ export default class JarvisExtension extends Extension {
     this._mountPopup();
     this._indicator.connect('button-press-event', (_actor, event) => {
       const button = event.get_button();
-      if (button === 2) { this._call('Arm'); return Clutter.EVENT_STOP; }
+      if (button === 2) { if (!this._muted) this._call('SetListening', '(b)', [true]); return Clutter.EVENT_STOP; }
       return Clutter.EVENT_PROPAGATE;
     });
     this._keyName = 'hotkey';
     try {
       Main.wm.addKeybinding(this._keyName, this.settings, Meta.KeyBindingFlags.NONE, Shell.ActionMode.NORMAL | Shell.ActionMode.OVERVIEW, () => {
         this._openPopup();
-        this._call('Arm');
+        if (!this._muted) this._call('SetListening', '(b)', [true]);
       });
     } catch (error) { log(`Jarvis hotkey unavailable: ${error.message}`); }
     this._settingsChanged = this.settings.connect('changed::accent-color', () => this._applyAccent());
@@ -102,7 +104,14 @@ export default class JarvisExtension extends Extension {
     try { this._lockChanged = Main.screenShield.connect('locked-changed', () => { if (Main.screenShield.locked) { this._closeShell(); this._call('ComputerRevoke'); } }); } catch {}
   }
   _bindSurface(view) {
-    view.onTalk = (pressed) => { if (pressed) { this._call('PushToTalk', '(b)', [true]); this._call('Arm'); } else { this._call('PushToTalk', '(b)', [false]); } };
+    view.onTalk = (pressed) => { if (this._muted) return; if (pressed) this._call('SetListening', '(b)', [true]); else this._call('SetListening', '(b)', [false]); };
+    view.onListen = () => { if (this._muted) return; this._call('SetListening', '(b)', [view._state !== 'LISTENING']); };
+    view.onMute = (muted) => {
+      this._muted = Boolean(muted);
+      this._muteTouched = true;
+      this._eachView((surface) => surface.setMuted(this._muted));
+      this._call('SetMuted', '(b)', [this._muted]);
+    };
     view.onStop = () => this._call('Cancel');
     view.onReset = () => { this._eachView((surface) => { surface.clear(); surface.setNotice('New conversation'); }); this._call('ResetContext'); };
     view.onAsk = (text) => { this._eachView((surface) => surface.addRow('U', text)); this._call('Ask', '(s)', [text]); };
@@ -257,9 +266,12 @@ export default class JarvisExtension extends Extension {
       const voice = status.voice;
       this._eachView((view) => view.setConnectionStatus(voice ? 'local' : 'voice-unavailable'));
       if (voice) {
-        const input = Boolean(voice.capture);
+        const remoteMuted = Boolean(voice.muted ?? status.muted);
+        if (!this._muteTouched) this._muted = remoteMuted;
+        const muted = Boolean(this._muted);
+        const input = Boolean(voice.capture) || muted;
         const tts = Boolean(voice.tts || voice.speech);
-        this._eachView((view) => view.setVoiceStatus({ tts, input, wake: voice.wake }));
+        this._eachView((view) => view.setVoiceStatus({ tts, input, wake: muted ? false : voice.wake, muted }));
       }
       const name = status.settings?.assistantName;
       if (name) this._applyAssistantName(name);
@@ -267,13 +279,13 @@ export default class JarvisExtension extends Extension {
   }
   _call(name, signature, value) {
     if (!this.proxy?.owned?.()) {
-      if (name !== 'PushToTalk') this._eachView((view) => view.setNotice(`${this._assistantName || 'Jarvis'} is starting…`));
+      if (name !== 'PushToTalk' && name !== 'SetMuted' && name !== 'SetListening') this._eachView((view) => view.setNotice(`${this._assistantName || 'Jarvis'} is starting…`));
       this._connectDaemon();
       return;
     }
     this.proxy.call(name, signature, value).catch((error) => {
       log(`Jarvis ${name}: ${error.message}`);
-      if (name === 'PushToTalk') return;
+      if (name === 'PushToTalk' || name === 'SetMuted' || name === 'SetListening') return;
       const fallback = name === 'ResetContext' ? 'Could not reset conversation' : `${name} failed: ${shortError(error.message)}`;
       this._eachView((view) => view.setNotice(fallback));
     });

apps/gnome-extension/jarvis@qvac.local/stylesheet.css

@@ -22,16 +22,19 @@
 .jarvis-tab { padding: 4px 10px; border-radius: 999px; background-color: transparent; color: #D7DDE8; font-size: 12px; }
 .jarvis-tab:hover, .jarvis-tab:focus { color: #F6F7FB; background-color: rgba(255, 255, 255, .10); }
 .jarvis-tab-active { background-color: #F4B942; color: #16191F; font-weight: bold; }
-.jarvis-thinking { color: #D7DDE8; font-size: 12px; padding: 8px 10px; border-left: 2px solid #4FD2FF; background-color: rgba(79, 210, 255, .08); border-radius: 8px; }
+.jarvis-thinking { color: #D7DDE8; font-size: 12px; padding: 8px 10px; border-left: 2px solid #4FD2FF; background-color: rgba(79, 210, 255, .08); border-radius: 8px; min-height: 40px; }
 .jarvis-notice { color: #F4B942; font-size: 12px; }
 .jarvis-confirm { spacing: 8px; padding: 4px 0; }
 .jarvis-confirm-actions { spacing: 6px; }
 .jarvis-confirm-label { color: #F6F7FB; font-size: 13px; }
 .jarvis-popup-scroll { height: 180px; }
-.jarvis-thinking-scroll { height: 160px; }
+.jarvis-thinking-pane { height: 160px; max-height: 160px; }
+.jarvis-thinking-scroll { height: 160px; max-height: 160px; }
 .jarvis-thinking-box { spacing: 0; }
 .jarvis-session-scroll { height: 220px; }
-.jarvis-session-thinking { height: 200px; }
+.jarvis-session-thinking-pane { height: 200px; max-height: 200px; }
+.jarvis-session-thinking { height: 200px; max-height: 200px; }
+.jarvis-thinking-scroll StScrollBar, .jarvis-session-thinking StScrollBar { min-width: 10px; padding: 0 1px; }
 .jarvis-transcript { spacing: 6px; }
 .jarvis-row { padding: 6px 10px; border-radius: 8px; font-size: 13px; color: #F6F7FB; }
 .jarvis-row-user { border-left: 2px solid #F4B942; background-color: rgba(244, 185, 66, .10); }
@@ -46,6 +49,11 @@
 .jarvis-talk { background-color: #F4B942; color: #16191F; font-weight: bold; padding: 8px 14px; }
 .jarvis-talk:hover, .jarvis-talk:focus { background-color: #FFE09A; color: #16191F; }
 .jarvis-talk:active { background-color: #FFE09A; color: #16191F; }
+.jarvis-talk-off { background-color: rgba(255, 255, 255, .10); color: #F6F7FB; font-weight: bold; }
+.jarvis-talk-off:hover, .jarvis-talk-off:focus { background-color: rgba(244, 185, 66, .28); color: #F6F7FB; }
+.jarvis-mute { background-color: rgba(255, 255, 255, .10); color: #F6F7FB; font-weight: bold; padding: 8px 14px; }
+.jarvis-mute:hover, .jarvis-mute:focus { background-color: rgba(174, 182, 200, .28); color: #F6F7FB; }
+.jarvis-mute-active { background-color: #AEB6C8; color: #16191F; }
 .jarvis-controls { spacing: 6px; }
 .jarvis-settings { font-size: 12px; padding: 4px 8px; color: #F6F7FB; }
 .jarvis-popup StEntry, .jarvis-session StEntry { border-radius: 10px; padding: 8px 10px; background-color: rgba(255, 255, 255, .08); color: #F6F7FB; border: 1px solid rgba(255, 255, 255, .22); }

apps/gnome-extension/jarvis@qvac.local/ui.js

@@ -52,6 +52,7 @@ export const previewToolResult = (value) => {
     }
     if (parsed && typeof parsed === 'object') {
       if (parsed.error) return safeText(parsed.error).slice(0, 160);
+      if (parsed.ok && parsed.action) return `${prettyToolName(parsed.action)}${parsed.name ? ` · ${parsed.name}` : ''}`.slice(0, 160);
       if (parsed.status && parsed.url) return `${parsed.status} ${parsed.url}`.slice(0, 160);
     }
   } catch {
@@ -94,7 +95,7 @@ export class ConversationView {
     this.settings.accessible_name = 'Open Jarvis settings';
     this._bindChip(this.settings, () => this.onSettings?.());
     this.header.add_child(this.settings);
-    this.statusLine = new St.Label({ text: 'armed · Hold Talk to speak', style_class: 'jarvis-status-line' });
+    this.statusLine = new St.Label({ text: 'armed · tap Listening to speak', style_class: 'jarvis-status-line' });
     this.tabs = new St.BoxLayout({ style_class: 'jarvis-tabs', x_expand: true });
     this.chatTab = new St.Button({ label: 'Chat', style_class: 'jarvis-tab jarvis-tab-active', reactive: true, can_focus: true });
     this.thinkTab = new St.Button({ label: 'Thinking', style_class: 'jarvis-tab', reactive: true, can_focus: true });
@@ -116,18 +117,44 @@ export class ConversationView {
       this.confirmButtons.add_child(button);
     }
     this.confirm.add_child(this.confirmButtons);
-    this.scroll = new St.ScrollView({ style_class: compact ? 'jarvis-popup-scroll' : 'jarvis-session-scroll', overlay_scrollbars: true, x_expand: true });
+    this.scroll = new St.ScrollView({ style_class: compact ? 'jarvis-popup-scroll' : 'jarvis-session-scroll', overlay_scrollbars: false, x_expand: true });
     try { this.scroll.hscrollbar_policy = St.PolicyType.NEVER; this.scroll.vscrollbar_policy = St.PolicyType.AUTOMATIC; } catch {}
     this.transcript = new St.BoxLayout({ style_class: 'jarvis-transcript', vertical: true, x_expand: true });
     this.transcript.accessible_name = 'Conversation transcript';
     if (typeof this.scroll.set_child === 'function') this.scroll.set_child(this.transcript); else this.scroll.add_child(this.transcript);
-    this.thinkingScroll = new St.ScrollView({ style_class: compact ? 'jarvis-thinking-scroll' : 'jarvis-session-thinking', overlay_scrollbars: true, x_expand: true, visible: false });
-    try { this.thinkingScroll.hscrollbar_policy = St.PolicyType.NEVER; this.thinkingScroll.vscrollbar_policy = St.PolicyType.AUTOMATIC; } catch {}
-    this.thinkingBox = new St.BoxLayout({ style_class: 'jarvis-thinking-box', vertical: true, x_expand: true });
-    this.thinking = wrapLabel(new St.Label({ text: '', style_class: 'jarvis-thinking', x_expand: true, can_focus: true }));
+    const thinkHeight = compact ? 160 : 200;
+    this.thinkingPane = new St.BoxLayout({
+      style_class: compact ? 'jarvis-thinking-pane' : 'jarvis-session-thinking-pane',
+      vertical: true,
+      x_expand: true,
+      y_expand: false,
+      visible: false,
+      reactive: true,
+      clip_to_allocation: true,
+      height: thinkHeight,
+    });
+    try { this.thinkingPane.set_height(thinkHeight); } catch {}
+    try { this.thinkingPane.set_style?.(`height: ${thinkHeight}px; max-height: ${thinkHeight}px;`); } catch {}
+    try { this.thinkingPane.clip_to_allocation = true; } catch {}
+    this.thinkingScroll = new St.ScrollView({
+      style_class: compact ? 'jarvis-thinking-scroll' : 'jarvis-session-thinking',
+      overlay_scrollbars: false,
+      x_expand: true,
+      y_expand: true,
+      visible: false,
+      reactive: true,
+      enable_mouse_scrolling: true,
+      clip_to_allocation: true,
+    });
+    try { this.thinkingScroll.hscrollbar_policy = St.PolicyType.NEVER; this.thinkingScroll.vscrollbar_policy = St.PolicyType.ALWAYS; } catch {}
+    try { this.thinkingScroll.overlay_scrollbars = false; } catch {}
+    try { this.thinkingScroll.clip_to_allocation = true; } catch {}
+    this.thinkingBox = new St.BoxLayout({ style_class: 'jarvis-thinking-box', vertical: true, x_expand: true, y_expand: false });
+    this.thinking = wrapLabel(new St.Label({ text: '', style_class: 'jarvis-thinking', x_expand: true, y_align: Clutter.ActorAlign.START, reactive: true, can_focus: true }));
     this.thinking.accessible_name = 'Jarvis thinking';
     this.thinkingBox.add_child(this.thinking);
     if (typeof this.thinkingScroll.set_child === 'function') this.thinkingScroll.set_child(this.thinkingBox); else this.thinkingScroll.add_child(this.thinkingBox);
+    this.thinkingPane.add_child(this.thinkingScroll);
     this.chipScroll = new St.ScrollView({ style_class: 'jarvis-chip-scroll', overlay_scrollbars: true, x_expand: true, visible: false });
     try { this.chipScroll.vscrollbar_policy = St.PolicyType.NEVER; this.chipScroll.hscrollbar_policy = St.PolicyType.AUTOMATIC; } catch {}
     this.chips = new St.BoxLayout({ style_class: 'jarvis-chips' });
@@ -136,14 +163,14 @@ export class ConversationView {
     this.entry = new St.Entry({ hint_text: 'Ask Jarvis…', can_focus: true, x_expand: true });
     this.entry.clutter_text.connect('activate', () => { const text = this.entry.get_text().trim(); if (text) { this.onAsk?.(text); this.entry.set_text(''); } });
     this.controls = new St.BoxLayout({ style_class: 'jarvis-controls' });
-    this.talk = new St.Button({ label: 'Hold to talk', style_class: 'jarvis-chip jarvis-talk', reactive: true, can_focus: true });
-    this.talk.accessible_name = 'Hold to talk';
-    this.talk.connect('notify::pressed', () => this.onTalk?.(Boolean(this.talk.pressed)));
-    this.talk.connect('button-press-event', () => { this.onTalk?.(true); return Clutter.EVENT_PROPAGATE; });
-    this.talk.connect('button-release-event', () => { this.onTalk?.(false); return Clutter.EVENT_PROPAGATE; });
-    this.talk.connect('leave-event', () => { this.onTalk?.(false); return Clutter.EVENT_PROPAGATE; });
-    this.talk.connect('key-focus-out', () => this.onTalk?.(false));
+    this.talk = new St.Button({ label: 'Listening', style_class: 'jarvis-chip jarvis-talk jarvis-talk-off', reactive: true, can_focus: true });
+    this.talk.accessible_name = 'Start listening';
+    this._bindChip(this.talk, () => this.onListen?.());
+    this.mute = new St.Button({ label: 'Mute', style_class: 'jarvis-chip jarvis-mute', reactive: true, can_focus: true });
+    this.mute.accessible_name = 'Mute microphone';
+    this._bindChip(this.mute, () => this.onMute?.(!this._muted));
     this.controls.add_child(this.talk);
+    this.controls.add_child(this.mute);
     this.stop = new St.Button({ label: 'Stop', style_class: 'jarvis-chip jarvis-chip-quiet', reactive: true, can_focus: true });
     this._bindChip(this.stop, () => this.onStop?.());
     this.reset = new St.Button({ label: 'Reset', style_class: 'jarvis-chip jarvis-chip-quiet', reactive: true, can_focus: true });
@@ -166,7 +193,7 @@ export class ConversationView {
     this.root.add_child(this.notice);
     this.root.add_child(this.confirm);
     this.root.add_child(this.scroll);
-    this.root.add_child(this.thinkingScroll);
+    this.root.add_child(this.thinkingPane);
     this.root.add_child(this.chipScroll);
     this.root.add_child(this.entry);
     this.root.add_child(this.controls);
@@ -174,12 +201,14 @@ export class ConversationView {
     this.replyFinalized = false;
     this._state = 'ARMED';
     this._tab = 'chat';
+    this._muted = false;
     this._activity = '';
     this.reducedMotion = false;
     this._chatLayoutTimers = [];
     this._thinkLayoutTimers = [];
     this._bindFollow(this.scroll);
     this._bindFollow(this.thinkingScroll);
+    this._fitThinking();
   }
   _bindChip(button, action) {
     button.connect('clicked', () => action?.());
@@ -208,12 +237,29 @@ export class ConversationView {
   setNotice(text) { const body = shortError(text); this.notice.text = body; this.notice.visible = Boolean(body); }
   _bindFollow(scroll) {
     const adjustment = scroll?.get_vadjustment?.() || scroll?.vadjustment;
-    // Layout can change the range after the token handler has returned.
+    scroll._jarvisFollow = true;
+    adjustment?.connect?.('notify::value', () => {
+      if (this._pinning || this._destroyed) return;
+      scroll._jarvisFollow = this._nearBottom(adjustment);
+    });
     for (const signal of ['notify::upper', 'notify::page-size'])
-      adjustment?.connect?.(signal, () => this._pinScroll(scroll));
+      adjustment?.connect?.(signal, () => {
+        if (scroll._jarvisFollow !== false) this._pinScroll(scroll);
+      });
     scroll.connect('notify::mapped', () => {
       if (scroll.mapped) this._followAfterLayout(scroll);
+      if (scroll === this.thinkingScroll && scroll.mapped) this._fitThinking();
     });
+    if (scroll === this.thinkingScroll) {
+      scroll.connect('notify::allocation', () => this._fitThinking());
+      scroll.connect('notify::width', () => this._fitThinking());
+    }
+  }
+  _nearBottom(adjustment) {
+    const upper = Number(adjustment?.upper) || 0;
+    const page = Number(adjustment?.page_size) || 0;
+    const value = Number(adjustment?.value) || 0;
+    return value >= Math.max(0, upper - page) - 32;
   }
   _clearLayoutTimers(slot) {
     for (const id of this[slot] || []) {
@@ -227,24 +273,58 @@ export class ConversationView {
     try { child?.queue_relayout?.(); } catch {}
     try { child?.get_first_child?.()?.queue_relayout?.(); } catch {}
   }
+  _viewportWidth(scroll) {
+    const read = (box) => {
+      if (!box) return 0;
+      if (typeof box.get_width === 'function') return Number(box.get_width()) || 0;
+      const x1 = Number(box.x1) || 0;
+      const x2 = Number(box.x2) || 0;
+      return x2 > x1 ? x2 - x1 : Number(box.width) || 0;
+    };
+    try {
+      const width = read(scroll?.get_allocation_box?.() || scroll?.allocation);
+      if (width >= 200) return width;
+    } catch {}
+    return Number(scroll?.width) || Number(this.root?.width) || (this.compact ? 292 : 388);
+  }
+  _fitThinking() {
+    if (this._destroyed) return;
+    const label = this.thinking;
+    const text = label?.clutter_text;
+    if (!label || !text) return;
+    const width = Math.max(200, this._viewportWidth(this.thinkingScroll) - 28);
+    try { text.line_wrap = true; } catch {}
+    try { text.line_wrap_mode = Pango.WrapMode.WORD_CHAR; } catch {}
+    try { text.ellipsize = Pango.EllipsizeMode.NONE; } catch {}
+    try { text.width = width; } catch {}
+    try { text.set_line_wrap?.(true); } catch {}
+    try { label.queue_relayout?.(); } catch {}
+    try { this.thinkingBox?.queue_relayout?.(); } catch {}
+  }
   _pinScroll(scroll) {
     if (this._destroyed) return;
     const adjustment = scroll?.get_vadjustment?.() || scroll?.vadjustment;
     if (!adjustment) return;
+    if (scroll._jarvisFollow === false) return;
     const upper = Number(adjustment.upper) || 0;
     const page = Number(adjustment.page_size) || 0;
+    this._pinning = true;
     try { adjustment.value = Math.max(0, upper - page); } catch {}
+    this._pinning = false;
   }
   _followAfterLayout(scroll) {
     if (this._destroyed) return;
     const slot = scroll === this.thinkingScroll ? '_thinkLayoutTimers' : '_chatLayoutTimers';
     this._relayoutPane(scroll);
+    if (scroll === this.thinkingScroll) this._fitThinking();
+    scroll._jarvisFollow = true;
     this._pinScroll(scroll);
     // Coalesce bursts without postponing the follow on every token. Track the
     // source until it runs so teardown never leaves callbacks on dead actors.
     if (this[slot].length) return;
     this[slot] = [GLib.timeout_add(GLib.PRIORITY_DEFAULT, 0, () => {
       this[slot] = [];
+      if (scroll === this.thinkingScroll) this._fitThinking();
       this._pinScroll(scroll);
       return GLib.SOURCE_REMOVE;
     })];
@@ -261,10 +341,12 @@ export class ConversationView {
     this.chatTab.style_class = thinking ? 'jarvis-tab' : 'jarvis-tab jarvis-tab-active';
     this.thinkTab.style_class = thinking ? 'jarvis-tab jarvis-tab-active' : 'jarvis-tab';
     this.scroll.visible = !thinking;
+    this.thinkingPane.visible = thinking;
     this.thinkingScroll.visible = thinking;
     this.chipScroll.visible = !thinking && this.chips.get_n_children() > 0;
     this.entry.visible = !thinking;
-    this.talk.visible = !thinking;
+    this._refreshListenButton();
+    if (thinking) this._fitThinking();
     this.followActive();
   }
   addRow(who, text) {
@@ -298,8 +380,15 @@ export class ConversationView {
   updateThinking(text) {
     const chunk = safeText(text);
     if (!chunk) return;
-    this.thinking.text = `${this.thinking.text || ''}${chunk}`;
+    // Map the pane before writing so the wrap width is the viewport, not 0.
     this.showTab('thinking');
+    const next = `${this.thinking.text || ''}${chunk}`;
+    this.thinking.text = next;
+    try { this.thinking.set_text?.(next); } catch {}
+    try { if (this.thinking.clutter_text) this.thinking.clutter_text.text = next; } catch {}
+    this.thinkingScroll._jarvisFollow = true;
+    this._fitThinking();
+    this._followAfterLayout(this.thinkingScroll);
   }
   toggleThinking() { this.showTab(this._tab === 'thinking' ? 'chat' : 'thinking'); }
   finishThinking() { this.showTab('chat'); }
@@ -401,16 +490,39 @@ export class ConversationView {
     this.status.accessible_name = this.status.text;
     this.status.style_class = 'jarvis-local';
   }
-  setVoiceStatus({ tts, input, wake } = {}) {
-    this._voice = { tts: Boolean(tts), input: Boolean(input), wake: Boolean(wake) };
-    this.status.text = input ? 'Local' : 'Mic off';
+  setVoiceStatus({ tts, input, wake, muted } = {}) {
+    if (muted != null) this._muted = Boolean(muted);
+    this._voice = { tts: Boolean(tts), input: Boolean(input), wake: Boolean(wake), muted: this._muted };
+    this.status.text = this._muted ? 'Muted' : input ? 'Local' : 'Mic off';
     this.status.accessible_name = this.status.text;
     this.status.style_class = 'jarvis-local';
-    this.talk.reactive = Boolean(input);
-    this.talk.can_focus = Boolean(input);
-    this.talk.label = input ? 'Hold to talk' : 'Mic unavailable';
+    this._refreshMuteButton();
+    this._refreshListenButton();
     this._refreshStatusLine();
   }
+  setMuted(muted) {
+    this._muted = Boolean(muted);
+    if (this._voice) this._voice.muted = this._muted;
+    this._refreshMuteButton();
+    this._refreshListenButton();
+    this._refreshStatusLine();
+  }
+  _refreshMuteButton() {
+    if (!this.mute) return;
+    this.mute.label = this._muted ? 'Muted' : 'Mute';
+    this.mute.style_class = this._muted ? 'jarvis-chip jarvis-mute jarvis-mute-active' : 'jarvis-chip jarvis-mute';
+    this.mute.accessible_name = this._muted ? 'Unmute microphone' : 'Mute microphone';
+  }
+  _refreshListenButton() {
+    if (!this.talk) return;
+    const listening = !this._muted && this._state === 'LISTENING';
+    const available = !this._muted && (this._voice?.input !== false);
+    this.talk.label = 'Listening';
+    this.talk.reactive = available;
+    this.talk.can_focus = available;
+    this.talk.style_class = listening ? 'jarvis-chip jarvis-talk' : 'jarvis-chip jarvis-talk jarvis-talk-off';
+    this.talk.accessible_name = this._muted ? 'Listening disabled while muted' : listening ? 'Stop listening' : 'Start listening';
+  }
   setState(state) {
     const value = STATES.has(state) ? state : 'ARMED';
     const changed = this._state !== value;
@@ -429,23 +541,28 @@ export class ConversationView {
     }
     this._refreshStatusLine();
     const name = this._assistantName || 'Jarvis';
-    this.title.text = value === 'SLEEPING' ? `${name} · privacy` : name;
+    this.title.text = this._muted ? `${name} · muted` : value === 'SLEEPING' ? `${name} · privacy` : name;
+    this._refreshListenButton();
   }
   _refreshStatusLine() {
     const value = this._state;
     const voice = this._voice || {};
     const bits = [];
+    if (this._muted) bits.push('muted');
+    else {
       if (voice.tts) bits.push('speech on');
       else if (voice.tts === false) bits.push('speech off');
       if (voice.input && voice.wake) bits.push('wake on');
-    else if (voice.input) bits.push('hold talk');
+      else if (voice.input) bits.push('listening ready');
       else if (voice.input === false) bits.push('mic off');
+    }
     const extra = bits.length ? ` · ${bits.join(' · ')}` : '';
-    if (value === 'LISTENING') this.statusLine.text = `Listening${extra}`;
+    if (this._muted) this.statusLine.text = `muted · microphone off`;
+    else if (value === 'LISTENING') this.statusLine.text = `Listening${extra}`;
     else if (value === 'SPEAKING') this.statusLine.text = `Speaking${extra}`;
     else if (value === 'SLEEPING') this.statusLine.text = `privacy · microphone off${extra}`;
     else if (value === 'THINKING') this.statusLine.text = `${this._activity || 'Thinking'}${extra}`;
-    else this.statusLine.text = `armed · Hold Talk to speak${extra}`;
+    else this.statusLine.text = `armed · tap Listening to speak${extra}`;
   }
   addChip(id, label, payload) {
     const chip = new St.Button({ label: safeText(label), style_class: 'jarvis-chip jarvis-chip-suggested', reactive: true, can_focus: true });
@@ -462,7 +579,7 @@ export class ConversationView {
       this.addRow('J', `${step.n ? `${step.n}. ` : ''}${action || 'computer step'}`);
     } catch { this.addRow('J', json); }
   }
-  endTalk() { this.onTalk?.(false); }
+  endTalk() {}
   destroy() {
     this._destroyed = true;
     this._clearLayoutTimers('_chatLayoutTimers');
@@ -587,7 +704,10 @@ export class SessionPanel {
       this.root.set_width(width);
       this.root.set_position(monitor.x + Math.max(24, monitor.width - width - 24), monitor.y + 40);
       this.view.scroll.set_height(Math.max(120, Math.min(280, monitor.height - 360)));
-      this.view.thinkingScroll.set_height(Math.max(120, Math.min(240, monitor.height - 400)));
+      const thinkHeight = Math.max(120, Math.min(240, monitor.height - 400));
+      this.view.thinkingPane?.set_height?.(thinkHeight);
+      this.view.thinkingPane?.set_style?.(`height: ${thinkHeight}px; max-height: ${thinkHeight}px;`);
+      this.view.thinkingScroll.set_height(thinkHeight);
     }
     this.root.visible = true;
     this.view.followActive();

computer-use/actuator.js

@@ -1,17 +1,173 @@
 import { setTimeout as delay } from 'node:timers/promises';
 import { assertSafeTarget, requiresConfirmation } from './safety.js';
 
-export class ComputerActuator {
-  constructor({ session, input, atspiAction, find, highlight, audit, confirm = async () => false, sleep = delay, verify = async () => true } = {}) { this.session = session; this.input = input; this.atspiAction = atspiAction; this.find = find; this.highlight = highlight; this.audit = audit; this.confirm = confirm; this.sleep = sleep; this.verify = verify; }
-  async target(args = {}) { const target = args.ref && this.find ? (await this.find({ ref: args.ref }))[0] : args; if (args.ref && !target) throw new Error('unknown or stale computer-use ref'); if (target) assertSafeTarget(target); return target; }
-  async run(action, args, fn) { const target = await this.target(args); if (requiresConfirmation(action, target) && !(await this.confirm(action, target))) throw new Error('explicit confirmation required'); this.session.beginStep(); await this.highlight?.(target, action); this.session.assertActive(); try { const result = await fn(target); await this.sleep(150); if (!(await this.verify(target, action, result))) throw new Error('computer-use state did not change after action'); await this.audit?.record(action, target, { ok: true }); return { ok: true, action, target: target || null, result }; } catch (error) { await this.audit?.record(action, target, { ok: false, reason: error.message }); throw error; } }
-  async act({ ref, action }) { return this.run(`act:${action}`, { ref }, async (target) => { if (!this.atspiAction) throw new Error('AT-SPI action backend is unavailable'); return this.atspiAction(target, action); }); }
-  async click(args = {}) { return this.run('click', args, async (target) => { if (args.ref && target && this.atspiAction) return this.atspiAction(target, 'click'); if (args.x == null || args.y == null) throw new Error('click requires a semantic ref or coordinates'); this.input.send({ type: 'pointer', action: 'click', x: args.x, y: args.y, button: args.button || 'left' }); }); }
-  async doubleClick(args = {}) { return this.run('double_click', args, async (target) => { if (args.x == null || args.y == null) throw new Error('double-click requires coordinates'); this.input.send({ type: 'pointer', action: 'double_click', x: args.x, y: args.y }); return target; }); }
-  async rightClick(args = {}) { return this.run('right_click', args, async () => { this.input.send({ type: 'pointer', action: 'click', x: args.x, y: args.y, button: 'right' }); }); }
-  async hover(args = {}) { return this.run('hover', args, async () => { this.input.send({ type: 'pointer', action: 'move', x: args.x, y: args.y }); }); }
-  async scroll(args = {}) { return this.run('scroll', args, async () => { this.input.send({ type: 'pointer', action: 'scroll', x: args.x, y: args.y, dx: args.dx || 0, dy: args.dy || 0 }); }); }
-  async drag({ from, to }) { return this.run('drag', { from, to }, async () => { this.input.send({ type: 'pointer', action: 'drag', from, to }); }); }
-  async type({ text, ref, submit = false }) { return this.run('type', { ref }, async (target) => { assertSafeTarget(target); this.input.send({ type: 'keyboard', action: 'type', text: String(text), submit: Boolean(submit) }); }); }
-  async key({ combo }) { return this.run(`key:${combo}`, {}, async () => { this.input.send({ type: 'keyboard', action: 'key', combo: String(combo).toLowerCase() }); }); }
+export function pointFor(target, args = {}) {
+  if (args.x != null && args.y != null) return { x: Number(args.x), y: Number(args.y) };
+  const rect = target?.rect;
+  if (Array.isArray(rect) && rect.length >= 4) {
+    const [x, y, w, h] = rect.map(Number);
+    if ([x, y, w, h].every(Number.isFinite)) return { x: x + w / 2, y: y + h / 2 };
+  }
+  return null;
+}
+
+export function compactTarget(target) {
+  if (!target || typeof target !== 'object') return null;
+  const out = {};
+  if (target.ref) out.ref = target.ref;
+  if (target.role) out.role = target.role;
+  if (target.name) out.name = String(target.name).slice(0, 80);
+  return Object.keys(out).length ? out : null;
+}
+
+function asPoint(value) {
+  if (!value) return null;
+  if (Array.isArray(value) && value.length >= 2) return { x: Number(value[0]), y: Number(value[1]) };
+  if (typeof value === 'object') return pointFor(value, value);
+  return null;
+}
+
+export class ComputerActuator {
+  constructor({ session, input, atspiAction, find, highlight, audit, confirm = async () => false, sleep = delay, verify = async () => true } = {}) {
+    this.session = session;
+    this.input = input;
+    this.atspiAction = atspiAction;
+    this.find = find;
+    this.highlight = highlight;
+    this.audit = audit;
+    this.confirm = confirm;
+    this.sleep = sleep;
+    this.verify = verify;
+  }
+
+  async target(args = {}) {
+    const target = args.ref && this.find ? (await this.find({ ref: args.ref }))[0] : args;
+    if (args.ref && !target) throw new Error('unknown or stale computer-use ref');
+    if (target) assertSafeTarget(target);
+    return target;
+  }
+
+  async readyInput() {
+    if (typeof this.input?.ready === 'function') await this.input.ready();
+  }
+
+  async send(action) {
+    await this.readyInput();
+    if (!this.input?.send) throw new Error('portal EIS input backend is unavailable');
+    this.input.send(action);
+  }
+
+  async semantic(target, action) {
+    if (!this.atspiAction || !target) return null;
+    try {
+      const result = await this.atspiAction(target, action);
+      if (!result || result.ok === false) return null;
+      return result;
+    } catch {
+      return null;
+    }
+  }
+
+  async run(action, args, fn) {
+    const target = await this.target(args);
+    if (requiresConfirmation(action, target) && !(await this.confirm(action, target))) throw new Error('explicit confirmation required');
+    this.session.beginStep();
+    await this.highlight?.(target, action);
+    this.session.assertActive();
+    try {
+      const result = await fn(target);
+      await this.sleep(150);
+      if (!(await this.verify(target, action, result))) throw new Error('computer-use state did not change after action');
+      await this.audit?.record(action, target, { ok: true });
+      return { ok: true, action, ...compactTarget(target), result: result ?? null };
+    } catch (error) {
+      await this.audit?.record(action, target, { ok: false, reason: error.message });
+      throw error;
+    }
+  }
+
+  async act({ ref, action }) {
+    return this.run(`act:${action}`, { ref }, async (target) => {
+      const semantic = await this.semantic(target, action);
+      if (semantic) return semantic;
+      throw new Error('AT-SPI action backend is unavailable');
+    });
+  }
+
+  async click(args = {}) {
+    return this.run('click', args, async (target) => {
+      const semantic = args.ref ? await this.semantic(target, args.action || 'click') : null;
+      if (semantic) return semantic;
+      const point = pointFor(target, args);
+      if (!point) throw new Error('click requires a semantic ref or coordinates');
+      await this.send({ type: 'pointer', action: 'click', x: point.x, y: point.y, button: args.button || 'left' });
+      return { via: 'pointer', ...point };
+    });
+  }
+
+  async doubleClick(args = {}) {
+    return this.run('double_click', args, async (target) => {
+      const point = pointFor(target, args);
+      if (!point) throw new Error('double-click requires a semantic ref or coordinates');
+      await this.send({ type: 'pointer', action: 'double_click', x: point.x, y: point.y });
+      return point;
+    });
+  }
+
+  async rightClick(args = {}) {
+    return this.run('right_click', args, async (target) => {
+      const point = pointFor(target, args);
+      if (!point) throw new Error('right-click requires a semantic ref or coordinates');
+      await this.send({ type: 'pointer', action: 'click', x: point.x, y: point.y, button: 'right' });
+      return point;
+    });
+  }
+
+  async hover(args = {}) {
+    return this.run('hover', args, async (target) => {
+      const point = pointFor(target, args);
+      if (!point) throw new Error('hover requires a semantic ref or coordinates');
+      await this.send({ type: 'pointer', action: 'move', x: point.x, y: point.y });
+      return point;
+    });
+  }
+
+  async scroll(args = {}) {
+    return this.run('scroll', args, async (target) => {
+      const point = pointFor(target, args) || { x: 0, y: 0 };
+      await this.send({ type: 'pointer', action: 'scroll', x: point.x, y: point.y, dx: args.dx || 0, dy: args.dy || 0 });
+      return point;
+    });
+  }
+
+  async drag({ from, to } = {}) {
+    return this.run('drag', { from, to }, async () => {
+      const start = asPoint(from) || (from?.ref ? pointFor(await this.target(from), from) : null);
+      const end = asPoint(to) || (to?.ref ? pointFor(await this.target(to), to) : null);
+      if (!start || !end) throw new Error('drag requires from and to refs or coordinates');
+      await this.send({ type: 'pointer', action: 'drag', from: [start.x, start.y], to: [end.x, end.y] });
+      return { from: start, to: end };
+    });
+  }
+
+  async type({ text, ref, submit = false } = {}) {
+    return this.run('type', { ref }, async (target) => {
+      assertSafeTarget(target);
+      const body = text == null ? '' : String(text);
+      const point = pointFor(target, {});
+      if (point) {
+        await this.send({ type: 'pointer', action: 'click', x: point.x, y: point.y, button: 'left' });
+        await this.sleep(80);
+      }
+      await this.send({ type: 'keyboard', action: 'type', text: body, submit: Boolean(submit) });
+      return { typed: body.length, submit: Boolean(submit) };
+    });
+  }
+
+  async key({ combo } = {}) {
+    return this.run(`key:${combo}`, {}, async () => {
+      await this.send({ type: 'keyboard', action: 'key', combo: String(combo || '').toLowerCase() });
+      return { combo: String(combo || '').toLowerCase() };
+    });
+  }
 }

computer-use/portal-input.js

@@ -54,6 +54,12 @@ export class PortalInputBackend {
     });
     return this._grant;
   }
+  async ready() {
+    if (this._grant) {
+      try { await this._grant; } catch (error) { throw new Error(error?.message || 'Grant desktop from the tray, then try again'); }
+    }
+    if (!this.available || !this.process?.stdin?.writable) throw new Error('Grant desktop from the tray, then try again');
+  }
   send(action) { if (!this.available || !this.process?.stdin?.writable) throw new Error('portal EIS input backend is unavailable'); this.process.stdin.write(`${JSON.stringify(action)}\n`); }
   captureFrame(output) {
     if (!this.available || !this.process?.stdin?.writable) throw new Error('PipeWire ScreenCast is unavailable until desktop access is granted');

computer-use/py/atspi_action.py

@@ -1,31 +1,60 @@
 #!/usr/bin/env python3
 import json
 import sys
+import gi
+gi.require_version('Atspi', '2.0')
 from gi.repository import Atspi
 
 target, action = json.loads(sys.argv[1]), sys.argv[2]
 Atspi.init()
 desktop = Atspi.get_desktop(0)
 found = None
+wanted_name = target.get('name') or ''
+wanted_role = target.get('role') or ''
+wanted_rect = target.get('rect') or None
+
+def rect_close(node):
+    if not wanted_rect or len(wanted_rect) < 4:
+        return True
+    try:
+        component = node.get_component()
+        if not component:
+            return True
+        extents = component.get_extents(Atspi.CoordType.SCREEN)
+        return abs(extents.x - wanted_rect[0]) < 8 and abs(extents.y - wanted_rect[1]) < 8
+    except Exception:
+        return True
+
 def walk(node, depth=0):
     global found
     if found or node is None or depth > 30:
         return
     try:
-        if (node.get_name() or '') == target.get('name') and (node.get_role_name() or '') == target.get('role'):
+        if (node.get_name() or '') == wanted_name and (node.get_role_name() or '') == wanted_role and rect_close(node):
             found = node
             return
         for i in range(node.get_child_count()):
             walk(node.get_child_at_index(i), depth + 1)
     except Exception:
         return
+
 walk(desktop)
 if not found:
     raise SystemExit('AT-SPI target not found')
 actions = found.get_action()
-for i in range(actions.get_n_actions()):
-    if actions.get_action_name(i).lower() == action.lower():
+if actions is None:
+    raise SystemExit(f'AT-SPI action unavailable: {action}')
+aliases = [action.lower()]
+if action.lower() in ('click', 'press', 'activate'):
+    aliases.extend(['press', 'click', 'activate'])
+seen = set()
+for name in aliases:
+    if name in seen:
+        continue
+    seen.add(name)
+    for i in range(actions.get_n_actions()):
+        if actions.get_action_name(i).lower() == name:
             if actions.do_action(i):
-            print(json.dumps({'ok': True, 'action': action}))
+                print(json.dumps({'ok': True, 'action': name}))
                 raise SystemExit(0)
 raise SystemExit(f'AT-SPI action unavailable: {action}')

computer-use/py/libei_sender.py

@@ -51,6 +51,14 @@ LETTER_KEYS = {ch: code for ch, code in zip('qwertyuiopasdfghjklzxcvbnm', [
     16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 30, 31, 32, 33, 34, 35, 36, 37, 38, 44, 45, 46, 47, 48, 49,
 ])}
 DIGIT_KEYS = {str(d): code for d, code in enumerate([11, 2, 3, 4, 5, 6, 7, 8, 9, 10])}
+PUNCT_KEYS = {
+    '-': (12, False), '_': (12, True), '=': (13, False), '+': (13, True),
+    '[': (26, False), '{': (26, True), ']': (27, False), '}': (27, True),
+    ';': (39, False), ':': (39, True), "'": (40, False), '"': (40, True),
+    '`': (41, False), '~': (41, True), '\\': (43, False), '|': (43, True),
+    ',': (51, False), '<': (51, True), '.': (52, False), '>': (52, True),
+    '/': (53, False), '?': (53, True),
+}
 
 
 def _ptr(value):
@@ -263,9 +271,13 @@ class LibeiSender:
                 continue
             lower = ch.lower()
             code = LETTER_KEYS.get(lower) or DIGIT_KEYS.get(ch)
-            if code is None:
-                continue
+            if code is not None:
                 self._tap(code, (KEY_LEFTSHIFT,) if ch.isupper() else ())
+                continue
+            punct = PUNCT_KEYS.get(ch)
+            if punct:
+                code, shifted = punct
+                self._tap(code, (KEY_LEFTSHIFT,) if shifted else ())
         if submit:
             self._tap(KEY_ENTER)
 

computer-use/py/portal_remote_desktop.py

@@ -118,6 +118,13 @@ class PortalNotify:
                     notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, button, 0))
         elif kind == 'pointer' and name == 'scroll':
             notify('NotifyPointerAxisDiscrete', '(oa{sv}ui)', (session, opts, 0, int(action.get('dy') or 0)))
+        elif kind == 'pointer' and name == 'drag':
+            start = action.get('from') or [0, 0]
+            end = action.get('to') or [0, 0]
+            notify('NotifyPointerMotionAbsolute', '(oa{sv}udd)', (session, opts, 0, float(start[0]), float(start[1])))
+            notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, BTN['left'], 1))
+            notify('NotifyPointerMotionAbsolute', '(oa{sv}udd)', (session, opts, 0, float(end[0]), float(end[1])))
+            notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, BTN['left'], 0))
         elif kind == 'keyboard' and name == 'type':
             for ch in str(action.get('text') or ''):
                 keysym = 0xff0d if ch == '\n' else (0x020 if ch == ' ' else ord(ch))

computer-use/safety.js

@@ -1,5 +1,8 @@
 const DANGEROUS_KEYS = new Set(['alt+f4', 'ctrl+q', 'ctrl+w', 'poweroff', 'reboot', 'shutdown']);
 export const isPasswordNode = (node) => /password|pam|credential/i.test(`${node?.role || ''} ${node?.name || ''}`);
 export const isDangerousKey = (combo) => DANGEROUS_KEYS.has(String(combo || '').toLowerCase().replace(/^key:/, ''));
-export function requiresConfirmation(action, target = {}) { return isDangerousKey(action) || /delete|format|purchase|send|install|power off|password|credential/i.test(`${action} ${target.name || ''} ${target.role || ''}`); }
+export function requiresConfirmation(action, target = {}) {
+  if (isDangerousKey(action)) return true;
+  return /\b(delete|format|purchase|install|power off|password|credential)\b/i.test(`${action} ${target.name || ''} ${target.role || ''}`);
+}
 export function assertSafeTarget(target) { if (isPasswordNode(target)) throw new Error('computer use refuses password or PAM controls'); }

daemon/dbus-service.js

@@ -21,6 +21,8 @@ export async function serveOnSessionBus(daemon) {
     Arm() { daemon.arm(); }
     Sleep() { daemon.sleep(); }
     Shutdown() { daemon.close(); }
+    SetMuted(muted) { daemon.setMuted?.(Boolean(muted)); }
+    SetListening(on) { daemon.setListening?.(Boolean(on)); }
     PushToTalk(pressed) { daemon.setPushToTalk?.(Boolean(pressed)); }
     async Say(text) { await daemon.say?.(text); }
     async Ask(text) { await daemon.ask(text); }
@@ -73,6 +75,8 @@ export async function serveOnSessionBus(daemon) {
     Sleep: { inSignature: '', outSignature: '', method: 'Sleep' },
     Shutdown: { inSignature: '', outSignature: '', method: 'Shutdown' },
     PushToTalk: { inSignature: 'b', outSignature: '', method: 'PushToTalk' },
+    SetMuted: { inSignature: 'b', outSignature: '', method: 'SetMuted' },
+    SetListening: { inSignature: 'b', outSignature: '', method: 'SetListening' },
     Say: { inSignature: 's', outSignature: '', method: 'Say' },
     Ask: { inSignature: 's', outSignature: '', method: 'Ask' },
     ResetContext: { inSignature: '', outSignature: '', method: 'ResetContext' },

daemon/index.js

@@ -24,6 +24,12 @@ import { StateRecovery } from './recovery.js';
 import { spokenReply, voiceSystemPrompt } from '../skills/voice-prompt.js';
 import { ensureAgentWorkspace, applyAssistantName, applyAssistantPrompt, normalizeAssistantName } from './agent-workspace.js';
 
+function chunkText(ev) {
+  if (ev == null) return '';
+  if (typeof ev === 'string' || typeof ev === 'number' || typeof ev === 'boolean') return String(ev);
+  return String(ev.text || ev.delta || ev.content || ev.chunk || '');
+}
+
 export class JarvisDaemon extends EventEmitter {
   constructor() {
     super();
@@ -46,6 +52,8 @@ export class JarvisDaemon extends EventEmitter {
     this.harness = new HarnessBridge({ cwd: this.workspace, computer: this.computer, observer: this.observer, actuator: this.actuator, fsAccess: this.settings.fsAccess });
     this.log = new PrivacyLog();
     this.locked = false;
+    this.muted = false;
+    this.listenEnabled = true;
     this.lastReply = '';
     this.voiceLoop = null;
     this._activeAsk = null;
@@ -54,8 +62,11 @@ export class JarvisDaemon extends EventEmitter {
     this._idleTimer.unref?.();
     this.telemetry = new RuntimeTelemetry();
     this._telemetryTimer = setInterval(() => this.telemetry.sample(), 60_000); this._telemetryTimer.unref?.();
-    this.harness.on('agent_message_chunk', (ev) => this.emit('Token', ev?.text || ev?.delta || ''));
-    this.harness.on('agent_thought_chunk', (ev) => this.emit('Thinking', String(ev?.text || ev?.delta || '')));
+    this.harness.on('agent_message_chunk', (ev) => this.emit('Token', chunkText(ev)));
+    this.harness.on('agent_thought_chunk', (ev) => {
+      const text = chunkText(ev);
+      if (text) this.emit('Thinking', text);
+    });
     this.harness.on('tool_call', (ev) => this.emit('ToolCall', JSON.stringify({ name: ev?.call?.name || 'tool', arguments: ev?.call?.arguments || {} })));
     this.harness.on('tool_result', (ev) => this.emit('ToolResult', JSON.stringify({ name: ev?.name || 'tool', result: ev?.result || '', toolCallId: ev?.toolCallId || '' })));
     this.harness.on('permission', (ev) => this.emit('ConfirmationRequired', ev));
@@ -64,7 +75,8 @@ export class JarvisDaemon extends EventEmitter {
 
   setState(state) { this.state = state; try { this.recovery.save({ state, mode: this.mode }); } catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); } this.emit('StateChanged', state); this.log.record('state', { state }).catch(() => {}); }
   async arm() {
-    if (this.locked) return;
+    if (this.locked || this.muted) return;
+    this.listenEnabled = true;
     await resumeQvac().catch(() => {});
     await this.ensureAsr();
     this.voice.wake();
@@ -143,7 +155,7 @@ export class JarvisDaemon extends EventEmitter {
   }
   _finishSpeech() {
     try { this.voice.finishSpeaking(); } catch {}
-    if (this.settings.listeningMode !== 'conversation') { this.voice.cancel(); this.setState('ARMED'); }
+    if (this.muted || !this.listenEnabled || this.settings.listeningMode !== 'conversation') { this.voice.cancel(); this.setState('ARMED'); }
     else this.setState('LISTENING');
   }
   _speakReply(spoken) {
@@ -212,7 +224,7 @@ export class JarvisDaemon extends EventEmitter {
     });
     loop.on('error', (error) => { console.error(`jarvisd: voice: ${error.message}`); this.emit('Error', 'VOICE', error.message); });
     this.voiceLoop = loop;
-    try { await loop.start(); this.emit('StateChanged', this.state); } catch (error) {
+    try { await loop.start(); if (this.muted) loop.setMuted(true); this.emit('StateChanged', this.state); } catch (error) {
       this.voiceLoop = null; await loop.stop?.().catch(() => {}); this.emit('Error', 'VOICE_UNAVAILABLE', error.message); throw error;
     }
     if (loop.status.wake) console.log('jarvisd: voice: wake detector ready');
@@ -261,8 +273,40 @@ export class JarvisDaemon extends EventEmitter {
     try { await this.voiceLoop.speak(preview); } finally { if (generation === this._askGeneration) this._finishSpeech(); }
   }
   stopSpeech() { this.voiceLoop?.interrupt?.(); this._finishSpeech(); }
-  setPushToTalk(pressed) { this.voiceLoop?.setPushToTalk(pressed); this.emit('PushToTalk', Boolean(pressed)); }
-  runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), settings: this.settings, computer: this.computer.status(), voice: this.voiceLoop ? { ...this.voiceLoop.metrics.snapshot(), ...this.voiceLoop.status } : null, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }
+  setPushToTalk(pressed) {
+    if (this.muted && pressed) return;
+    this.voiceLoop?.setPushToTalk(pressed);
+    this.emit('PushToTalk', Boolean(pressed));
+  }
+  setMuted(muted) {
+    const next = Boolean(muted);
+    this.muted = next;
+    if (next) this.listenEnabled = false;
+    this.voiceLoop?.setMuted?.(next);
+    if (next) {
+      this.voiceLoop?.setPushToTalk?.(false);
+      if (this.state === 'LISTENING') {
+        this.voice.cancel();
+        this.setState('ARMED');
+      } else {
+        this.emit('StateChanged', this.state);
+      }
+    } else {
+      this.emit('StateChanged', this.state);
+    }
+  }
+  setListening(on) {
+    if (this.muted) return;
+    if (on) return this.arm();
+    this.listenEnabled = false;
+    this.voiceLoop?.setPushToTalk?.(false);
+    try { this.voiceLoop?.vad?.reset?.(); } catch {}
+    if (this.state === 'LISTENING') {
+      this.voice.cancel();
+      this.setState('ARMED');
+    }
+  }
+  runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), settings: this.settings, computer: this.computer.status(), voice: this.voiceLoop ? { ...this.voiceLoop.metrics.snapshot(), ...this.voiceLoop.status, muted: this.muted } : { muted: this.muted }, muted: this.muted, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); }
   async assessModelFit(model) { return JSON.stringify(await callQvac('assessModelFit', { modelSrc: String(model) })); }
   async downloadModel(model) { return JSON.stringify(await callQvac('downloadAsset', { modelSrc: String(model) })); }
   async cancelModel(model) { return JSON.stringify(await cancelQvacRequest({ modelId: String(model) })); }

daemon/voice-loop.js

@@ -23,16 +23,17 @@ export class VoiceLoop extends EventEmitter {
   constructor({ daemon, capture = new PipeWireCapture(), playback = new PipeWirePlayback(), wake = new WakeEngine(), vad = new VadSegmenter(), asr, tts, cooldownMs = POST_PLAYBACK_COOLDOWN_MS, listeningMode = 'conversation', now = () => Date.now(), captureRetryMs = 2000, asrRetryMs = 4000 } = {}) {
     super(); this.daemon = daemon; this.capture = capture; this.playback = playback; this.wake = wake; this.vad = vad; this.asr = asr; this.tts = tts; this.cooldownMs = cooldownMs; this.listeningMode = listeningMode; this.now = now;
     this.captureRetryMs = captureRetryMs; this.asrRetryMs = asrRetryMs;
-    this.isSpeaking = false; this.cooldownUntil = 0; this.running = false; this.ptt = false; this._generation = 0; this._speechQueue = Promise.resolve(); this.metrics = new VoiceMetrics(); this._transcriptions = new Set();
+    this.isSpeaking = false; this.cooldownUntil = 0; this.running = false; this.ptt = false; this.muted = false; this._generation = 0; this._speechQueue = Promise.resolve(); this.metrics = new VoiceMetrics(); this._transcriptions = new Set();
     capture.on('audio', (chunk) => this.pushAudio(chunk));
     capture.on('error', (error) => { this.status.capture = false; this.status.errors.capture = error.message; this.emit('error', error); this._scheduleCaptureRetry(); });
     capture.on('close', () => { this.status.capture = false; if (this.running) { this.status.errors.capture = 'Microphone stream closed'; this._scheduleCaptureRetry(); } });
     wake.on('unavailable', () => { this.status.wake = false; });
     wake.on('error', (error) => { this.status.wake = false; this.emit('error', error); });
-    this.status = { asr: false, tts: false, capture: false, wake: false, microphone: this.asr != null, speech: this.tts != null, errors: {} };
+    this.status = { asr: false, tts: false, capture: false, wake: false, muted: false, microphone: this.asr != null, speech: this.tts != null, errors: {} };
     wake.on('wake', (phrase) => this.wakeHeard(phrase));
     vad.on('level', (rms) => daemon?.emit('ListeningLevel', rms));
     vad.on('utterance', (audio) => {
+      if (this.muted || this.daemon?.muted) return;
       const task = this.transcribe(audio).catch((error) => this.emit('error', error));
       this._transcriptions.add(task);
       task.finally(() => this._transcriptions.delete(task)).catch(() => {});
@@ -48,16 +49,21 @@ export class VoiceLoop extends EventEmitter {
   }
   _notifyStatus() { try { this.daemon?.emit?.('StateChanged', this.daemon.state || 'ARMED'); } catch {} }
   _armWake() {
+    if (this.muted) {
+      try { this.wake.pause?.(); } catch {}
+      this.status.wake = false;
+      return;
+    }
     try { if (this.listeningMode !== 'ptt') this.wake.start?.(); this.wake.resume(); this.status.wake = this.listeningMode !== 'ptt' && Boolean(this.wake.command || this.wake.detect); }
     catch (error) { this.status.errors.wake = error.message; this.emit('error', error); }
   }
   _armCapture() {
-    if (!this.running || this.status.capture || this.asr == null) return;
+    if (!this.running || this.muted || this.status.capture || this.asr == null) return;
     try { this.capture.start(); this.status.capture = true; delete this.status.errors.capture; this._notifyStatus(); }
     catch (error) { this.status.errors.capture = error.message; this.emit('error', error); this._scheduleCaptureRetry(); }
   }
   _scheduleCaptureRetry() {
-    if (!this.running || this._captureRetry || this.status.capture || this.asr == null) return;
+    if (!this.running || this.muted || this._captureRetry || this.status.capture || this.asr == null) return;
     this._captureRetry = setTimeout(() => {
       this._captureRetry = 0;
       this._armCapture();
@@ -135,7 +141,30 @@ export class VoiceLoop extends EventEmitter {
     if (this._asrRetry) { clearTimeout(this._asrRetry); this._asrRetry = 0; }
     this.interrupt(); this.capture.stop(); this.wake.close(); this.vad.reset(); this.playback.stop(); await Promise.allSettled([this._speechQueue, ...this._transcriptions]); await this.asr?.stop?.(); if (this.tts !== this.asr) await this.tts?.stop?.();
   }
+  setMuted(muted) {
+    const next = Boolean(muted);
+    this.muted = next;
+    this.status.muted = next;
+    if (next) {
+      this._pttHeld = false;
+      this.ptt = false;
+      try { this.vad.reset?.(); } catch {}
+      try { this.wake.pause?.(); } catch {}
+      this.status.wake = false;
+      if (this._captureRetry) { clearTimeout(this._captureRetry); this._captureRetry = 0; }
+      try { this.capture.stop(); } catch {}
+      this.status.capture = false;
+      this._notifyStatus();
+      return;
+    }
+    if (this.running) {
+      this._armCapture();
+      this._armWake();
+      this._notifyStatus();
+    }
+  }
   setPushToTalk(pressed) {
+    if (this.muted && pressed) return Promise.resolve();
     if (!pressed) {
       this._pttHeld = false;
       this.ptt = false;
@@ -150,7 +179,7 @@ export class VoiceLoop extends EventEmitter {
     }).catch((error) => this.emit('error', error));
   }
   pushAudio(chunk) {
-    if (!this.running || this.daemon?.locked) return;
+    if (!this.running || this.muted || this.daemon?.locked) return;
     const sleeping = this.daemon?.state === 'SLEEPING';
     if (!sleeping && (this.isSpeaking || this.now() < this.cooldownUntil)) { this.metrics.feedbackDrop(); return; }
     if (!sleeping) {
@@ -161,14 +190,15 @@ export class VoiceLoop extends EventEmitter {
     if (this.ptt || (this.listeningMode !== 'ptt' && this.daemon?.state === 'LISTENING')) this.vad.push(chunk);
   }
   wakeHeard(phrase) {
-    if (!this.running || this.daemon?.locked) return;
+    if (!this.running || this.muted || this.daemon?.muted || this.daemon?.locked) return;
     this.metrics.wakeAccepted(); this.daemon?.emit('WakeHeard', phrase); this.ensureAsr().catch((error) => this.emit('error', error)); this.daemon?.arm?.(); this.emit('wake', phrase);
   }
   async transcribe(audio) {
+    if (this.muted || this.daemon?.muted) return;
     if (!this.status.asr || !this.asr?.transcribeAudio) return;
     const generation = this._generation;
     const text = await this.asr.transcribeAudio(audio).catch((error) => { this.emit('error', error); return ''; });
-    if (!this.running || this.daemon?.locked || generation !== this._generation) return;
+    if (!this.running || this.muted || this.daemon?.muted || this.daemon?.locked || generation !== this._generation) return;
     if (!isMeaningfulTranscript(text)) { this.metrics.wakeRejected(); return; }
     this.metrics.utterance();
     this.daemon?.emit('PartialTranscript', text); this.daemon?.emit('FinalTranscript', text);
@@ -184,11 +214,11 @@ export class VoiceLoop extends EventEmitter {
     this._generation += 1;
     this.playback.stop();
     this.isSpeaking = false;
-    this.wake.resume();
+    if (!this.muted) this.wake.resume();
   }
   _releaseSpeaking() {
     this.isSpeaking = false;
-    this.wake.resume();
+    if (!this.muted) this.wake.resume();
     if (this.daemon?.state === 'SPEAKING') {
       try { this.daemon.voice?.finishSpeaking?.(); } catch {}
       if (this.daemon._finishSpeech) this.daemon._finishSpeech();

dbus/io.qvac.Jarvis.Session.xml

@@ -3,6 +3,8 @@
   <interface name="io.qvac.Jarvis.Session">
     <method name="Arm"/><method name="Sleep"/><method name="Shutdown"/>
     <method name="PushToTalk"><arg name="pressed" type="b" direction="in"/></method>
+    <method name="SetMuted"><arg name="muted" type="b" direction="in"/></method>
+    <method name="SetListening"><arg name="on" type="b" direction="in"/></method>
     <method name="Say"><arg name="text" type="s" direction="in"/></method>
     <method name="Ask"><arg name="text" type="s" direction="in"/></method>
     <method name="ReloadSettings"><arg type="s" direction="out"/></method>

skills/computer-act.js

@@ -1,14 +1,14 @@
 export function createComputerActTools({ actuator } = {}) {
   const call = (method) => async (args) => { if (!actuator) throw new Error('computer-use actuator is unavailable'); return actuator[method](args); };
   return [
-    ['cu_act', 'Run a named AT-SPI action on a semantic ref.', { ref: { type: 'string' }, action: { type: 'string' } }, 'act'],
-    ['cu_click', 'Click a semantic ref or coordinate.', { ref: { type: 'string' }, x: { type: 'number' }, y: { type: 'number' }, button: { type: 'string' } }, 'click'],
-    ['cu_double_click', 'Double-click a coordinate.', { x: { type: 'number' }, y: { type: 'number' } }, 'doubleClick'],
-    ['cu_right_click', 'Right-click a coordinate.', { x: { type: 'number' }, y: { type: 'number' } }, 'rightClick'],
-    ['cu_hover', 'Move the visible agent cursor.', { x: { type: 'number' }, y: { type: 'number' } }, 'hover'],
-    ['cu_scroll', 'Scroll at a target.', { ref: { type: 'string' }, x: { type: 'number' }, y: { type: 'number' }, dy: { type: 'number' }, dx: { type: 'number' } }, 'scroll'],
+    ['cu_act', 'Run a named AT-SPI action on a semantic ref. Desktop grant required; do not paste the result JSON to the user.', { ref: { type: 'string' }, action: { type: 'string' } }, 'act'],
+    ['cu_click', 'Click a control. Prefer a tree ref from cu_observe or cu_find; coordinates are the fallback. Desktop grant required.', { ref: { type: 'string' }, x: { type: 'number' }, y: { type: 'number' }, button: { type: 'string' } }, 'click'],
+    ['cu_double_click', 'Double-click a control or coordinate.', { ref: { type: 'string' }, x: { type: 'number' }, y: { type: 'number' } }, 'doubleClick'],
+    ['cu_right_click', 'Right-click a control or coordinate.', { ref: { type: 'string' }, x: { type: 'number' }, y: { type: 'number' } }, 'rightClick'],
+    ['cu_hover', 'Move the pointer to a control or coordinate.', { ref: { type: 'string' }, x: { type: 'number' }, y: { type: 'number' } }, 'hover'],
+    ['cu_scroll', 'Scroll at a control or coordinate.', { ref: { type: 'string' }, x: { type: 'number' }, y: { type: 'number' }, dy: { type: 'number' }, dx: { type: 'number' } }, 'scroll'],
     ['cu_drag', 'Drag between semantic refs or coordinates.', { from: { type: 'object', properties: { ref: { type: 'string' }, x: { type: 'number' }, y: { type: 'number' } } }, to: { type: 'object', properties: { ref: { type: 'string' }, x: { type: 'number' }, y: { type: 'number' } } } }, 'drag'],
-    ['cu_type', 'Type Unicode through the granted input backend.', { text: { type: 'string' }, ref: { type: 'string' }, submit: { type: 'boolean' } }, 'type'],
-    ['cu_key', 'Send a keyboard combination through the granted input backend.', { combo: { type: 'string' } }, 'key'],
-  ].map(([name, description, properties, method]) => ({ name, permission: 'computer-use', description, parameters: { type: 'object', properties }, execute: call(method) }));
+    ['cu_type', 'Click the target if given, then type Unicode. Use a text, entry, or document ref from cu_find. Desktop grant required.', { text: { type: 'string' }, ref: { type: 'string' }, submit: { type: 'boolean' } }, 'type'],
+    ['cu_key', 'Send a keyboard combination such as enter, ctrl+l, or alt+tab.', { combo: { type: 'string' } }, 'key'],
+  ].map(([name, description, properties, method]) => ({ name, permission: 'read', description, parameters: { type: 'object', properties }, execute: call(method) }));
 }

skills/computer-observe.js

@@ -1,12 +1,37 @@
 const PERMISSION = 'read';
 const inactive = (computer) => !computer?.status?.().active ? { unavailable: 'computer-use grant required', action: 'cu.grant' } : null;
 
+export function summarizeNode(node) {
+  if (!node || typeof node !== 'object') return node;
+  const out = { ref: node.ref, role: node.role, name: node.name };
+  if (Array.isArray(node.rect)) out.rect = node.rect;
+  if (node.score != null) out.score = node.score;
+  return out;
+}
+
+export function summarizeTree(nodes, limit = 80) {
+  const list = Array.isArray(nodes) ? nodes : [];
+  return list.slice(0, limit).map(summarizeNode);
+}
+
 export function createComputerObserveTools({ computer, observer } = {}) {
   const guard = () => { const result = inactive(computer); if (result) throw new Error(result.unavailable); };
   return [
-    { name: 'cu_observe', permission: PERMISSION, description: 'Read the live PipeWire ScreenCast frame buffer from the granted desktop session, plus AT-SPI. This is not the Screenshot portal. OCR is off unless include_ocr is true.', parameters: { type: 'object', properties: { include_tree: { type: 'boolean' }, include_ocr: { type: 'boolean' }, include_vision: { type: 'boolean' } } }, execute: async ({ include_tree = true, include_ocr = false, include_vision = false } = {}) => { guard(); return observer.observe({ includeTree: include_tree, includeOcr: include_ocr, includeVision: include_vision }); } },
+    { name: 'cu_observe', permission: PERMISSION, description: 'Look at the granted desktop: focused window, AT-SPI controls, and the live ScreenCast frame. Prefer this before clicking or typing. This is not the Screenshot portal.', parameters: { type: 'object', properties: { include_tree: { type: 'boolean' }, include_ocr: { type: 'boolean' }, include_vision: { type: 'boolean' } } }, execute: async ({ include_tree = true, include_ocr = false, include_vision = false } = {}) => {
+      guard();
+      const bundle = await observer.observe({ includeTree: include_tree, includeOcr: include_ocr, includeVision: include_vision });
+      return {
+        focused: bundle.focused || null,
+        windows: (bundle.windows || []).slice(0, 12),
+        tree: summarizeTree(bundle.tree),
+        frame_source: bundle.frame_source || null,
+        screenshot_path: bundle.screenshot_path || null,
+        unavailable: bundle.unavailable || [],
+        hint: 'Use cu_find or a tree ref with cu_click / cu_type. Do not paste this JSON to the user.',
+      };
+    } },
     { name: 'cu_zoom', permission: PERMISSION, description: 'Request a higher resolution local crop by rectangle or current AT-SPI ref.', parameters: { type: 'object', properties: { rect: { type: 'array', items: { type: 'number' } }, ref: { type: 'string' } } }, execute: async (args) => { guard(); return observer.zoom(args); } },
-    { name: 'cu_tree', permission: PERMISSION, description: 'Return visible AT-SPI roles, names, states, bounds, and per-step refs.', parameters: { type: 'object', properties: { app: { type: 'string' }, focused_only: { type: 'boolean' }, max_nodes: { type: 'number' } } }, execute: async ({ focused_only = true, max_nodes = 400 } = {}) => { guard(); return observer.tree({ focusedOnly: focused_only, maxNodes: max_nodes }); } },
-    { name: 'cu_find', permission: PERMISSION, description: 'Find a visible desktop element by accessible name/role.', parameters: { type: 'object', properties: { query: { type: 'string' }, role: { type: 'string' } }, required: ['query'] }, execute: async (args) => { guard(); return observer.find(args); } },
+    { name: 'cu_tree', permission: PERMISSION, description: 'Return visible AT-SPI roles, names, bounds, and per-step refs.', parameters: { type: 'object', properties: { app: { type: 'string' }, focused_only: { type: 'boolean' }, max_nodes: { type: 'number' } } }, execute: async ({ focused_only = true, max_nodes = 80 } = {}) => { guard(); return summarizeTree(await observer.tree({ focusedOnly: focused_only, maxNodes: Math.min(400, Number(max_nodes) || 80) }), Math.min(80, Number(max_nodes) || 80)); } },
+    { name: 'cu_find', permission: PERMISSION, description: 'Find a visible desktop element by accessible name/role.', parameters: { type: 'object', properties: { query: { type: 'string' }, role: { type: 'string' } }, required: ['query'] }, execute: async (args) => { guard(); return summarizeTree(await observer.find(args), 12); } },
   ];
 }

skills/voice-prompt.js

@@ -32,9 +32,9 @@ This computer can reach the internet. web_search, google_search, fetch_page, web
 
 File tools may read any path they accept. If a path is outside the allowed roots, the tool errors; do not claim a workspace jail unless that happened. Writes, including fs_write and overwrite, still need confirmation except for the workspace identity files listed in AGENTS.md.
 
-Computer use requires an explicit user grant from Settings, Computer use, Allow now, or Grant desktop in the tray. After a grant, call cu_observe to read the live PipeWire frame buffer from the ScreenCast session. Do not take a screenshot. Do not wait for a libei injector. Never click or type while it is inactive, locked, or revoked. Never ask for passwords or credentials.
+Computer use requires an explicit user grant from Settings, Computer use, Allow now, or Grant desktop in the tray. After a grant, call cu_observe, then cu_find or a tree ref, then cu_click and cu_type. Those tools move the real pointer and keyboard. Never paste tool JSON, AT-SPI trees, or {"ok":true} blobs into chat or speech. Speak a short status only when the desktop task is done or blocked.
+Do not take a screenshot. Do not wait for a libei injector. Never click or type while the grant is inactive, locked, or revoked. Never ask for passwords or credentials. Prefer a text, entry, or document ref when typing, not a whole window frame.
 Destructive actions require confirmation in both the heads-up display and spoken conversation.
-Prefer structured tools and accessibility references over coordinates.
 
 For questions about the computer, files, processes, or system state, call the most relevant registered tool before answering. Never say that computer use or terminal access is unavailable unless a tool result reports that limitation.
 When the user asks to use the command line or terminal, call run_terminal_cmd once, then speak the result. Do not chain extra commands (hostnamectl then uname then free) unless the previous result was an error.
@@ -67,5 +67,10 @@ export function spokenReply(reply) {
     .replace(/<tool_call>[\s\S]*?<\/tool_call>/gi, '')
     .replace(/<function\s*=[^>]*>[\s\S]*?<\/function>/gi, '')
     .replace(/<tool_call>[\s\S]*$/gi, '');
-  return forChatDisplay(parseHudSidecar(stripped).spoken);
+  const spoken = forChatDisplay(parseHudSidecar(stripped).spoken).trim();
+  if (!spoken) return '';
+  if (/^[{\[]/.test(spoken)) {
+    try { JSON.parse(spoken); return ''; } catch { /* keep non-JSON */ }
+  }
+  return spoken.replace(/\{[\s\S]{0,400}?"ok"\s*:\s*true[\s\S]{0,800}?\}/g, '').trim();
 }

test/computer-use.test.js

@@ -44,16 +44,67 @@ test('computer use semantic actuation previews, budgets, and audits actions', as
   const audit = new ComputerAudit({ dir }); const session = new ComputerUseSession({ stepsMax: 1, audit }); const sent = [];
   session.grant();
   const actuator = new ComputerActuator({ session, input: { send: (event) => sent.push(event) }, find: async ({ ref }) => [{ ref, name: 'Save', role: 'push button', rect: [1, 2, 3, 4] }], atspiAction: async () => ({ semantic: true }), audit, sleep: async () => {} });
-  const result = await actuator.click({ ref: 'r1' }); assert.equal(result.ok, true); assert.equal(session.status().steps_used, 1);
+  const result = await actuator.click({ ref: 'r1' }); assert.equal(result.ok, true); assert.equal(result.name, 'Save'); assert.equal(session.status().steps_used, 1);
+  assert.equal(result.target, undefined);
   assert.deepEqual(sent, []); const files = await (await import('node:fs/promises')).readdir(dir); assert.equal(files.length, 1); assert.match(await readFile(path.join(dir, files[0]), 'utf8'), /target_hash/);
 });
 
+test('semantic click falls back to the control center when AT-SPI has no click action', async () => {
+  const session = new ComputerUseSession(); session.grant(); const sent = [];
+  const actuator = new ComputerActuator({
+    session,
+    input: { send: (event) => sent.push(event) },
+    find: async ({ ref }) => [{ ref, name: 'Discord', role: 'frame', rect: [10, 20, 100, 40] }],
+    atspiAction: async () => { throw new Error('AT-SPI action unavailable: click'); },
+    sleep: async () => {},
+  });
+  const result = await actuator.click({ ref: 'r1' });
+  assert.equal(result.ok, true);
+  assert.equal(sent[0].type, 'pointer');
+  assert.equal(sent[0].x, 60);
+  assert.equal(sent[0].y, 40);
+});
+
+test('typing focuses the target then injects keys and does not stringify missing text', async () => {
+  const session = new ComputerUseSession(); session.grant(); const sent = [];
+  const actuator = new ComputerActuator({
+    session,
+    input: { send: (event) => sent.push(event) },
+    find: async () => [{ ref: 'r2', name: 'Message', role: 'entry', rect: [0, 0, 20, 10] }],
+    sleep: async () => {},
+  });
+  const result = await actuator.type({ ref: 'r2', text: 'hello' });
+  assert.equal(result.ok, true);
+  assert.equal(result.result.typed, 5);
+  assert.equal(sent[0].action, 'click');
+  assert.equal(sent[1].action, 'type');
+  assert.equal(sent[1].text, 'hello');
+  const empty = await actuator.type({ ref: 'r2' });
+  assert.equal(empty.result.typed, 0);
+  assert.equal(sent[3].text, '');
+});
+
 test('computer use refuses password targets and unconfirmed dangerous keys', async () => {
   const session = new ComputerUseSession(); session.grant(); const actuator = new ComputerActuator({ session, input: { send() {} }, find: async () => [{ name: 'Password', role: 'password text' }], sleep: async () => {} });
   await assert.rejects(() => actuator.type({ ref: 'password', text: 'secret' }), /password/);
   await assert.rejects(() => actuator.key({ combo: 'alt+f4' }), /confirmation/);
 });
 
+test('Send buttons do not require extra confirmation after a desktop grant', async () => {
+  const session = new ComputerUseSession(); session.grant(); const sent = [];
+  const actuator = new ComputerActuator({
+    session,
+    input: { send: (event) => sent.push(event) },
+    find: async () => [{ ref: 'r3', name: 'Send', role: 'push button', rect: [0, 0, 40, 20] }],
+    atspiAction: async () => { throw new Error('AT-SPI action unavailable: click'); },
+    sleep: async () => {},
+  });
+  const result = await actuator.click({ ref: 'r3' });
+  assert.equal(result.ok, true);
+  assert.equal(sent[0].action, 'click');
+  assert.equal(sent[0].x, 20);
+});
+
 test('libei sender binds bitmask capabilities from libei.h', () => {
   const pyDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../computer-use/py');
   const compiled = spawnSync('python3', ['-m', 'py_compile', 'libei_sender.py', 'portal_remote_desktop.py', 'portal_screenshot.py', 'pw_framebuffer.py'], { cwd: pyDir, encoding: 'utf8' });

test/daemon.test.js

@@ -48,6 +48,8 @@ test('spoken replies use harness text and strip HUD sidecars', () => {
   );
   assert.equal(spokenReply({}), '');
   assert.equal(spokenReply('[object Object]'), '');
+  assert.equal(spokenReply({ text: '{"ok":true,"action":"type","name":"Discord"}' }), '');
+  assert.equal(spokenReply({ text: 'Done. {"ok":true,"action":"click","ref":"r1"}' }), 'Done.');
   assert.equal(
     spokenReply({ text: '## Status\n- CPU is **fine**.\nUse `htop` if you want more.' }),
     'Status\n- CPU is fine.\nUse htop if you want more.',
@@ -61,6 +63,21 @@ test('voice settings default TTS on and honor an explicit disable', () => {
   assert.equal(voiceSettings({ tts_enabled: false }).ttsEnabled, false);
 });
 
+test('thought chunks forward SDK text onto Thinking', async () => {
+  const daemon = new JarvisDaemon();
+  const thoughts = [];
+  daemon.on('Thinking', (text) => thoughts.push(text));
+  try {
+    daemon.harness.emit('agent_thought_chunk', { type: 'agent_thought_chunk', text: 'Considering the lookup.' });
+    daemon.harness.emit('agent_thought_chunk', 'plain thought');
+    daemon.harness.emit('agent_thought_chunk', { delta: ' via delta' });
+    daemon.harness.emit('agent_thought_chunk', { text: '' });
+    assert.deepEqual(thoughts, ['Considering the lookup.', 'plain thought', ' via delta']);
+  } finally {
+    await daemon.close();
+  }
+});
+
 test('ask extracts harness reply text instead of stringifying the object', async () => {
   const daemon = new JarvisDaemon();
   daemon.harness = { ask: async () => ({ ok: true, text: 'Hello there.', reason: 'stop' }), cancel() {}, close: async () => {} };
@@ -216,3 +233,38 @@ test('cancel suppresses a late reply and revokes portal input', async () => {
     assert.deepEqual(replies, []); assert.equal(daemon.state, 'ARMED'); assert.equal(revoked, true);
   } finally { await daemon.close(); }
 });
+
+test('mute ignores arm and does not return to listening after speech', async () => {
+  const daemon = new JarvisDaemon();
+  const muted = [];
+  daemon.voiceLoop = {
+    setMuted(value) { muted.push(Boolean(value)); },
+    setPushToTalk(value) { this.ptt = value; },
+    interrupt() {},
+    status: { capture: false, wake: false, muted: false },
+    metrics: { snapshot() { return {}; } },
+  };
+  try {
+    daemon.setState('LISTENING');
+    daemon.setMuted(true);
+    assert.equal(daemon.muted, true);
+    assert.equal(daemon.listenEnabled, false);
+    assert.equal(daemon.state, 'ARMED');
+    assert.deepEqual(muted, [true]);
+    await daemon.arm();
+    assert.equal(daemon.state, 'ARMED');
+    daemon.setListening(true);
+    assert.equal(daemon.state, 'ARMED');
+    daemon.setState('SPEAKING');
+    daemon._finishSpeech();
+    assert.equal(daemon.state, 'ARMED');
+    const status = JSON.parse(daemon.runtimeStatus());
+    assert.equal(status.muted, true);
+    assert.equal(status.voice.muted, true);
+    daemon.setMuted(false);
+    daemon.setListening(false);
+    daemon.setState('SPEAKING');
+    daemon._finishSpeech();
+    assert.equal(daemon.state, 'ARMED');
+  } finally { await daemon.close(); }
+});

test/gnome-extension.test.js

@@ -27,7 +27,8 @@ function harness() {
       this.children = [];
       this.visible = props.visible !== false;
       this.text = props.text || props.hint_text || '';
-      this.clutter_text = { connect() {}, ellipsize: null };
+      this.clutter_text = { connect() {}, ellipsize: null, width: 0, text: this.text };
+      this.allocation = { x1: 0, x2: 320, get_width() { return this.x2 - this.x1; } };
       this.vadjustment = {
         value: 0, upper: 240, page_size: 80, handlers: {},
         connect(name, handler) { this.handlers[name] = handler; },
@@ -35,6 +36,7 @@ function harness() {
       };
     }
     get_vadjustment() { return this.vadjustment; }
+    get_allocation_box() { return this.allocation; }
     add_child(child) { this.children.push(child); child.get_parent = () => this; }
     remove_child(child) { this.children = this.children.filter((item) => item !== child); }
     destroy_all_children() { this.children = []; }
@@ -83,7 +85,7 @@ function harness() {
   const context = vm.createContext({
     Extension: class {},
     global: { stage: { get_key_focus() { return null; } } },
-    St: { BoxLayout, Label, Icon: Actor, Widget: Actor, Button: Actor, Entry: Actor, ScrollView, PolicyType: { NEVER: 0, AUTOMATIC: 1 } },
+    St: { BoxLayout, Label, Icon: Actor, Widget: Actor, Button: Actor, Entry: Actor, ScrollView, PolicyType: { NEVER: 0, AUTOMATIC: 1, ALWAYS: 2 } },
     Clutter: { ActorAlign: { CENTER: 0, START: 1 }, EVENT_STOP: 1, EVENT_PROPAGATE: 0, KEY_space: 32, KEY_Return: 65293, KEY_Escape: 65307 },
     Pango: { WrapMode: { WORD_CHAR: 2 }, EllipsizeMode: { NONE: 0, END: 3 } },
     PopupMenu: {
@@ -141,6 +143,7 @@ test('empty chrome widgets start hidden', () => {
   const cu = new ComputerUseChrome();
   assert.equal(popup.confirm.visible, false);
   assert.equal(popup.thinkingScroll.visible, false);
+  assert.equal(popup.thinkingPane.visible, false);
   assert.equal(popup.chipScroll.visible, false);
   assert.equal(cu.root.visible, false);
   assert.equal(cu.target.visible, false);
@@ -262,7 +265,11 @@ test('GNOME GI imports expose default namespaces and panel has a real menu', ()
   assert.match(extensionSource, /PopupMenuItem\('Settings'\)/);
   assert.match(extensionSource, /PopupMenuItem\('Grant desktop'\)/);
   assert.doesNotMatch(uiSource, /button-press-event', \(\) => Clutter\.EVENT_STOP/);
-  assert.match(uiSource, /notify::pressed/);
+  assert.match(uiSource, /onListen/);
+  assert.match(uiSource, /jarvis-mute/);
+  assert.match(extensionSource, /SetMuted/);
+  assert.match(extensionSource, /SetListening/);
+  assert.doesNotMatch(extensionSource, /this\._call\('Arm'\)/);
   assert.match(extensionSource, /ComputerGrant/);
   assert.match(extensionSource, /OpenExtensionPrefs/);
   assert.match(uiSource, /finalizeReply/);
@@ -301,11 +308,16 @@ test('lazy ASR still shows the microphone as available when capture is up', () =
   assert.match(popup.statusLine.text, /speech on/);
   assert.match(popup.statusLine.text, /wake on/);
   assert.doesNotMatch(popup.status.style_class, /jarvis-state-/);
-  assert.equal(popup.talk.label, 'Hold to talk');
+  assert.equal(popup.talk.label, 'Listening');
   popup.setVoiceStatus({ tts: false, input: false, wake: false });
   assert.match(popup.status.text, /Mic off/);
   assert.match(popup.statusLine.text, /mic off/);
-  assert.equal(popup.talk.label, 'Mic unavailable');
+  assert.equal(popup.talk.label, 'Listening');
+  assert.equal(popup.talk.reactive, false);
+  popup.setVoiceStatus({ tts: false, input: false, wake: false, muted: true });
+  assert.equal(popup.status.text, 'Muted');
+  assert.match(popup.statusLine.text, /muted/);
+  assert.equal(popup.talk.reactive, false);
   assert.match(extensionSource, /Boolean\(voice\.capture\)/);
   assert.doesNotMatch(extensionSource, /voice\.asr && voice\.capture/);
 });
@@ -450,7 +462,7 @@ test('Settings chip is in the header and opens preferences', () => {
   assert.deepEqual(calls, ['settings', 'session']);
 });
 
-test('HUD chips and hold-to-talk receive clicks instead of swallowing them', () => {
+test('HUD chips, listening, and mute receive clicks instead of swallowing them', () => {
   const { ConversationView } = harness();
   const popup = new ConversationView({ compact: true });
   const calls = [];
@@ -458,63 +470,87 @@ test('HUD chips and hold-to-talk receive clicks instead of swallowing them', ()
   popup.onReset = () => calls.push('reset');
   popup.onExpand = () => calls.push('open');
   popup.onSettings = () => calls.push('settings');
-  popup.onTalk = (pressed) => calls.push(pressed ? 'talk-down' : 'talk-up');
+  popup.onListen = () => calls.push('listen');
+  popup.onMute = (muted) => calls.push(muted ? 'mute' : 'unmute');
   popup.stop.handlers.clicked();
   popup.reset.handlers.clicked();
   popup.expand.handlers.clicked();
   popup.settings.handlers.clicked();
-  popup.talk.pressed = true;
-  popup.talk.handlers['notify::pressed']();
-  popup.talk.pressed = false;
-  popup.talk.handlers['notify::pressed']();
-  assert.deepEqual(calls, ['stop', 'reset', 'open', 'settings', 'talk-down', 'talk-up']);
+  popup.talk.handlers.clicked();
+  popup.mute.handlers.clicked();
+  assert.deepEqual(calls, ['stop', 'reset', 'open', 'settings', 'listen', 'mute']);
   assert.equal(popup.stop.handlers['button-press-event'], undefined);
   assert.equal(popup.settings.handlers['button-press-event'], undefined);
+  assert.equal(popup.talk.handlers['notify::pressed'], undefined);
+  assert.equal(popup.mute.label, 'Mute');
+  popup.setMuted(true);
+  assert.equal(popup.mute.label, 'Muted');
+  assert.match(popup.statusLine.text, /muted/);
+  popup.setState('LISTENING');
+  assert.equal(popup.talk.reactive, false);
+  assert.equal(popup.talk.visible, true);
+  popup.showTab('thinking');
+  assert.equal(popup.mute.visible, true);
+  assert.equal(popup.talk.visible, true);
 });
 
-test('closing the popup ends hold-to-talk', () => {
+test('closing the popup does not stop listening', () => {
   const { ConversationView } = harness();
   const popup = new ConversationView({ compact: true });
   const talks = [];
   popup.onTalk = (pressed) => talks.push(pressed);
+  popup.onListen = () => talks.push('listen');
   popup.endTalk();
-  assert.deepEqual(talks, [false]);
+  assert.deepEqual(talks, []);
 });
 
 test('Chat and Thinking tabs exist and thinking auto-follows', () => {
   const { ConversationView } = harness();
   const popup = new ConversationView({ compact: true });
   const session = new ConversationView({ compact: false });
+  assert.equal(popup.thinkingPane.clip_to_allocation, true);
+  assert.equal(popup.thinkingPane.height, 160);
+  assert.equal(popup.thinkingPane.children[0], popup.thinkingScroll);
+  assert.equal(popup.thinkingScroll.overlay_scrollbars, false);
+  assert.equal(popup.thinkingScroll.vscrollbar_policy, 2);
+  assert.equal(popup.thinkingScroll.clip_to_allocation, true);
+  assert.equal(popup.thinkingScroll.enable_mouse_scrolling, true);
   for (const view of [popup, session]) {
     assert.equal(view.chatTab.label, 'Chat');
     assert.equal(view.thinkTab.label, 'Thinking');
     assert.equal(view.thinkingBox.children[0], view.thinking);
     assert.equal(view.thinkingScroll.children[0], view.thinkingBox);
+    assert.equal(view.thinkingPane.visible, false);
     assert.equal(view.thinkingScroll.visible, false);
     assert.equal(view.scroll.visible, true);
     view.setState('THINKING');
     view.updateThinking('Considering the lookup. ');
     assert.equal(view._tab, 'thinking');
+    assert.equal(view.thinkingPane.visible, true);
     assert.equal(view.thinkingScroll.visible, true);
     assert.equal(view.scroll.visible, false);
     assert.match(view.thinking.text, /Considering the lookup/);
     assert.equal(view.thinking.clutter_text.ellipsize, 0);
+    assert.ok(view.thinking.clutter_text.width >= 80);
     assert.equal(view.thinkingScroll.vadjustment.value, 160);
     view.token('Latest reply token');
     view.scroll.vadjustment.value = 0;
     view.setState('SPEAKING');
     assert.equal(view._tab, 'chat');
     assert.equal(view.scroll.visible, true);
+    assert.equal(view.thinkingPane.visible, false);
     assert.equal(view.thinkingScroll.visible, false);
     assert.equal(view.scroll.vadjustment.value, 160);
     view.showTab('chat', { user: true });
     view.updateThinking('still thinking');
     assert.equal(view._tab, 'thinking');
+    assert.equal(view.thinkingPane.visible, true);
     assert.equal(view.thinkingScroll.visible, true);
     assert.match(view.thinking.text, /still thinking/);
     view.thinkingScroll.vadjustment.value = 0;
     view.showTab('thinking', { user: true });
     assert.equal(view._tab, 'thinking');
+    assert.equal(view.thinkingPane.visible, true);
     assert.equal(view.thinkingScroll.visible, true);
     assert.equal(view.thinkingScroll.vadjustment.value, 160);
   }

test/review-regressions.test.js

@@ -9,6 +9,7 @@ import { ComputerUseSession } from '../computer-use/session.js';
 import { ComputerActuator } from '../computer-use/actuator.js';
 import { PortalInputBackend } from '../computer-use/portal-input.js';
 import { createComputerObserveTools } from '../skills/computer-observe.js';
+import { createComputerActTools } from '../skills/computer-act.js';
 import { createPhase2Tools, filesystemRoots } from '../skills/phase2-tools.js';
 import { VoiceStateMachine } from '../daemon/voice-state.js';
 import { assertLocalEndpoint } from '../daemon/network-policy.js';
@@ -83,11 +84,16 @@ test('stale semantic refs fail before input and revocation during preview preven
   await assert.rejects(actuator.click({ x: 1, y: 1 }), /inactive/);
 });
 
-test('desktop observation does not wait for a second Allow after Grant desktop', () => {
+test('desktop observation and actuation do not wait for a second Allow after Grant desktop', () => {
   const id = 'cu-observe-permission';
   try {
-    custom.register(id, createComputerObserveTools({ computer: { status: () => ({ active: true }) }, observer: {} }));
+    custom.register(id, [
+      ...createComputerObserveTools({ computer: { status: () => ({ active: true }) }, observer: {} }),
+      ...createComputerActTools({ actuator: {} }),
+    ]);
     assert.equal(custom.needsPermission(id, 'cu_observe', 'ask'), false);
+    assert.equal(custom.needsPermission(id, 'cu_click', 'ask'), false);
+    assert.equal(custom.needsPermission(id, 'cu_type', 'ask'), false);
   } finally { custom.clear(id); }
 });
 
@@ -97,6 +103,29 @@ test('expired grants prevent desktop observation', async () => {
   await assert.rejects(observe.execute(), /grant/);
 });
 
+test('observe tools return compact tree nodes without accessibility state dumps', async () => {
+  const computer = { status: () => ({ active: true }) };
+  const observer = {
+    observe: async () => ({
+      focused: { name: 'Discord' },
+      windows: [{ id: 1 }],
+      tree: [{ ref: 'r1', role: 'frame', name: 'Discord', rect: [0, 0, 10, 10], state: ['1', '8', '24'] }],
+      frame_source: 'pipewire',
+      screenshot_path: '/tmp/x.webp',
+      unavailable: [],
+    }),
+    find: async () => [{ ref: 'r1', role: 'entry', name: 'Message', rect: [1, 2, 3, 4], score: 90, state: ['focused'] }],
+  };
+  const tools = Object.fromEntries(createComputerObserveTools({ computer, observer }).map((tool) => [tool.name, tool]));
+  const seen = await tools.cu_observe.execute({});
+  assert.equal(seen.tree[0].ref, 'r1');
+  assert.equal(seen.tree[0].state, undefined);
+  assert.match(seen.hint, /Do not paste/);
+  const hits = await tools.cu_find.execute({ query: 'message' });
+  assert.equal(hits[0].score, 90);
+  assert.equal(hits[0].state, undefined);
+});
+
 function helper() {
   const child = new EventEmitter();
   child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); child.stdin = new EventEmitter(); child.stdin.writable = true; child.stdin.write = () => {};

test/runtime-tools.test.js

@@ -227,7 +227,8 @@ test('voice prompt tells the model not to chain extra terminal commands', () =>
   assert.match(VOICE_SYSTEM_PROMPT, /Tool names, tool arguments/);
   assert.match(VOICE_SYSTEM_PROMPT, /File tools may read any path they accept/);
   assert.match(VOICE_SYSTEM_PROMPT, /Allow now/);
-  assert.match(VOICE_SYSTEM_PROMPT, /call cu_observe to read the live PipeWire frame buffer/);
+  assert.match(VOICE_SYSTEM_PROMPT, /call cu_observe, then cu_find or a tree ref, then cu_click and cu_type/);
+  assert.match(VOICE_SYSTEM_PROMPT, /Never paste tool JSON/);
   assert.doesNotMatch(VOICE_SYSTEM_PROMPT, /Never claim cloud access/);
 });
 

test/settings.test.js

@@ -122,6 +122,49 @@ test('Hold Talk only mode ignores wake and automatic listening', () => {
   loop.ptt = true; loop.pushAudio(Buffer.alloc(10)); assert.equal(vadFrames, 1);
 });
 
+test('mute stops capture and ignores wake, VAD, and push-to-talk', async () => {
+  const capture = new EventEmitter();
+  let started = 0;
+  let stopped = 0;
+  capture.start = () => { started += 1; };
+  capture.stop = () => { stopped += 1; };
+  let wakeFrames = 0;
+  let vadFrames = 0;
+  let heard = 0;
+  const wake = new EventEmitter();
+  wake.push = () => { wakeFrames += 1; };
+  wake.pause = () => {};
+  wake.resume = () => {};
+  wake.start = () => {};
+  wake.close = () => {};
+  const vad = new EventEmitter();
+  vad.push = () => { vadFrames += 1; };
+  vad.reset = () => {};
+  vad.end = () => {};
+  const daemon = new EventEmitter();
+  daemon.state = 'LISTENING';
+  daemon.arm = () => { heard += 1; };
+  const loop = new VoiceLoop({ daemon, capture, wake, vad, asr: { start: async () => {} } });
+  await loop.start();
+  assert.ok(started >= 1);
+  loop.setMuted(true);
+  assert.equal(loop.muted, true);
+  assert.equal(loop.status.muted, true);
+  assert.equal(loop.status.capture, false);
+  assert.equal(stopped, 1);
+  loop.pushAudio(Buffer.alloc(10));
+  assert.equal(wakeFrames, 0);
+  assert.equal(vadFrames, 0);
+  loop.wakeHeard('hey jarvis');
+  assert.equal(heard, 0);
+  await loop.setPushToTalk(true);
+  assert.equal(loop.ptt, false);
+  loop.interrupt();
+  loop.setMuted(false);
+  assert.equal(loop.status.capture, true);
+  await loop.stop();
+});
+
 test('VAD discards short noises and bounds the whole recording including pauses', () => {
   const vad = new VadSegmenter({ params: { minSpeechDurationMs: 300, minSilenceDurationMs: 500, maxSpeechDurationMs: 600 } });
   const speech = Buffer.alloc(3200); for (let i = 0; i < speech.length; i += 2) speech.writeInt16LE(20000, i);

test/tool-parse.test.js

@@ -89,3 +89,37 @@ Need a current answer.
   assert.equal(recovered.calls[0].name, 'web_search');
   assert.equal(recovered.calls[0].arguments.query, 'Ubuntu 26.04 release');
 });
+
+test('SDK thinking events use text and recover think tags from content', () => {
+  const events = require('../vendor/agent-harness/lib/events.js');
+  const sdkThink = events.normalizeCompletionEvent({ type: 'thinkingDelta', seq: 1, text: 'Considering the lookup.' });
+  assert.equal(sdkThink.type, 'thinkingDelta');
+  assert.equal(sdkThink.delta, 'Considering the lookup.');
+  const first = events.expandThinkEvents({ type: 'contentDelta', delta: 'Hello <think>reason' }, { inThink: false, carry: '' });
+  assert.deepEqual(first.events, [
+    { type: 'contentDelta', delta: 'Hello ' },
+    { type: 'thinkingDelta', delta: 'reason' },
+  ]);
+  assert.equal(first.state.inThink, true);
+  const next = events.expandThinkEvents({ type: 'contentDelta', delta: 'ing</think>Answer' }, first.state);
+  assert.deepEqual(next.events, [
+    { type: 'thinkingDelta', delta: 'ing' },
+    { type: 'contentDelta', delta: 'Answer' },
+  ]);
+  assert.equal(next.state.inThink, false);
+});
+
+test('click and type aliases recover as computer-use tools', () => {
+  const tools = [{ name: 'cu_click' }, { name: 'cu_type' }, { name: 'cu_observe' }];
+  const calls = toolParse.extractCalls(
+    '<tool_call><function=click><parameter=ref>\nr4\n</parameter></function></tool_call>',
+    tools,
+  );
+  assert.equal(calls[0].name, 'cu_click');
+  assert.equal(calls[0].arguments.ref, 'r4');
+  const typed = toolParse.extractCalls(
+    '<tool_call><function=type><parameter=text>\nhi\n</parameter></function></tool_call>',
+    tools,
+  );
+  assert.equal(typed[0].name, 'cu_type');
+});

vendor/agent-harness/agent-workspace/AGENTS.md

@@ -41,7 +41,7 @@ Workspace skills live in `skills/<name>/SKILL.md`. When a request matches a skil
 - `read_file` / `list_dir` / `grep` / `write_file` / `search_replace` — workspace files.
 - `run_terminal_cmd` — local shell. Public HTTP via curl or wget is blocked; use web tools.
 - `web_search` / `google_search` / `fetch_page` / `web_fetch` / `wiki_search` / `hn_search` / `code_search` — public reads, no extra keys.
-- Desktop and computer-use tools are registered by Jarvis. Observe and act only with an active grant.
+- Desktop and computer-use tools are registered by Jarvis. After Grant desktop, call `cu_observe`, then `cu_click` / `cu_type`. Do not paste tool JSON into chat.
 - `ask_user_question` — wait for a user choice.
 
 Keep going until the user’s request is fully complete. Never stop after announcing the next step. When the work is done, speak a short summary. A greeting does not need a long summary.

vendor/agent-harness/agent/loop.js

@@ -38,6 +38,7 @@ const pendingCustom = new Map();
 const pendingAsks = new Map();
 const pendingPlans = new Map();
 const MAX_TURNS = 24;
+const CU_FREE_ROUNDS = new Set(['todo_write', 'update_goal', 'cu_observe', 'cu_find', 'cu_tree', 'cu_status', 'cu_zoom']);
 const SUBAGENT_TURNS = 8;
 const CUSTOM_TOOL_TIMEOUT_MS = 60000;
 const ASK_TIMEOUT_MS = 10 * 60 * 1000;
@@ -877,7 +878,7 @@ async function runTurn(ctx) {
         }
         if (stopEarly) break;
       }
-      if (prepared.some((item) => item.name !== 'todo_write' && item.name !== 'update_goal')) {
+      if (prepared.some((item) => !CU_FREE_ROUNDS.has(item.name))) {
         toolBudget.markToolRound(budget);
       }
       if (stopEarly) {

vendor/agent-harness/lib/catalog.js

@@ -303,6 +303,7 @@ const COMPACT_TOOL_ALLOW = [
   'cu_status',
   'cu_observe',
   'cu_find',
+  'cu_tree',
   'cu_click',
   'cu_type',
   'cu_key',

vendor/agent-harness/lib/events.js

@@ -10,13 +10,59 @@ function clip(s) {
   return t.slice(0, MAX_DELTA);
 }
 
+function eventDelta(ev) {
+  if (ev == null) return '';
+  if (typeof ev === 'string' || typeof ev === 'number' || typeof ev === 'boolean') return clip(ev);
+  return clip(ev.text || ev.delta || ev.content || ev.chunk || '');
+}
+
+const THINK_OPEN = '<think>';
+const THINK_CLOSE = '</think>';
+
+function trailingPartial(buf, marker) {
+  const max = Math.min(buf.length, marker.length - 1);
+  for (let len = max; len > 0; len--) {
+    if (marker.startsWith(buf.slice(buf.length - len))) return len;
+  }
+  return 0;
+}
+
+function expandThinkEvents(n, state) {
+  const st = state && typeof state === 'object' ? state : { inThink: false, carry: '' };
+  if (!n) return { events: [], state: st };
+  if (n.type === 'thinkingDelta') {
+    const delta = n.delta || '';
+    return { events: delta ? [n] : [], state: st };
+  }
+  if (n.type !== 'contentDelta') return { events: [n], state: st };
+  let rest = String(st.carry || '') + String(n.delta || '');
+  const events = [];
+  let inThink = !!st.inThink;
+  while (rest) {
+    const marker = inThink ? THINK_CLOSE : THINK_OPEN;
+    const idx = rest.indexOf(marker);
+    if (idx < 0) {
+      const partial = trailingPartial(rest, marker);
+      const emit = partial ? rest.slice(0, rest.length - partial) : rest;
+      const carry = partial ? rest.slice(rest.length - partial) : '';
+      if (emit) events.push({ type: inThink ? 'thinkingDelta' : 'contentDelta', delta: emit });
+      return { events, state: { inThink, carry } };
+    }
+    const before = rest.slice(0, idx);
+    if (before) events.push({ type: inThink ? 'thinkingDelta' : 'contentDelta', delta: before });
+    rest = rest.slice(idx + marker.length);
+    inThink = !inThink;
+  }
+  return { events, state: { inThink, carry: '' } };
+}
+
 function normalizeCompletionEvent(ev) {
   if (!ev || !ev.type) return null;
   if (ev.type === 'contentDelta') {
-    return { type: 'contentDelta', delta: clip(ev.delta || ev.text || ev.content || '') };
+    return { type: 'contentDelta', delta: eventDelta(ev) };
   }
   if (ev.type === 'thinkingDelta') {
-    return { type: 'thinkingDelta', delta: clip(ev.delta || ev.text || ev.content || '') };
+    return { type: 'thinkingDelta', delta: eventDelta(ev) };
   }
   if (ev.type === 'toolCall') {
     const call = ev.call || ev.toolCall || ev;
@@ -35,4 +81,4 @@ function normalizeCompletionEvent(ev) {
   return { type: ev.type, delta: ev.delta != null ? clip(ev.delta) : undefined, call: ev.call };
 }
 
-module.exports = { normalizeCompletionEvent, MAX_DELTA };
+module.exports = { normalizeCompletionEvent, expandThinkEvents, eventDelta, MAX_DELTA };

vendor/agent-harness/lib/qvac.js

@@ -396,6 +396,7 @@ async function complete(opts, onEvent) {
 
   let text = '';
   let thinking = '';
+  let thinkState = { inThink: false, carry: '' };
   const toolCalls = [];
   const abortRun = () => {
     try {
@@ -420,41 +421,59 @@ async function complete(opts, onEvent) {
         const n = events.normalizeCompletionEvent(ev);
         if (!n) continue;
         watch.bump();
-        if (n.type === 'contentDelta') {
-          text += n.delta;
-          if (onEvent) onEvent(n);
+        const split = events.expandThinkEvents(n, thinkState);
+        thinkState = split.state;
+        for (const p of split.events) {
+          if (p.type === 'contentDelta') {
+            text += p.delta;
+            if (onEvent) onEvent(p);
             if (repeatLoop.isRepeating(text)) {
               abortRun();
               settleTimeout();
               return;
             }
-        } else if (n.type === 'thinkingDelta') {
-          thinking += n.delta;
-          if (onEvent) onEvent(n);
+          } else if (p.type === 'thinkingDelta') {
+            thinking += p.delta;
+            if (onEvent) onEvent(p);
             if (repeatLoop.isRepeating(thinking)) {
               abortRun();
               settleTimeout();
               return;
             }
-        } else if (n.type === 'toolCall') {
-          toolCalls.push(n.call);
-          if (onEvent) onEvent(n);
+          } else if (p.type === 'toolCall') {
+            toolCalls.push(p.call);
+            if (onEvent) onEvent(p);
           } else if (onEvent) {
-          onEvent(n);
+            onEvent(p);
+          }
         }
       }
     } else if (run.tokenStream) {
       for await (const token of run.tokenStream) {
         if (watch.timedOut()) return;
         watch.bump();
-        text += token;
-        if (onEvent) onEvent({ type: 'contentDelta', delta: token });
+        const split = events.expandThinkEvents({ type: 'contentDelta', delta: token }, thinkState);
+        thinkState = split.state;
+        for (const p of split.events) {
+          if (p.type === 'thinkingDelta') {
+            thinking += p.delta;
+            if (onEvent) onEvent(p);
+            if (repeatLoop.isRepeating(thinking)) {
+              abortRun();
+              settleTimeout();
+              return;
+            }
+          } else {
+            text += p.delta;
+            if (onEvent) onEvent({ type: 'contentDelta', delta: p.delta });
             if (repeatLoop.isRepeating(text)) {
               abortRun();
               settleTimeout();
               return;
             }
           }
+        }
+      }
       if (run.toolCallStream) {
         for await (const evt of run.toolCallStream) {
           if (watch.timedOut()) return;

vendor/agent-harness/lib/tool-parse.js

@@ -29,6 +29,17 @@ const ALIASES = {
   write: 'write_file',
   ls: 'list_dir',
   status: 'jarvis_status',
+  click: 'cu_click',
+  type: 'cu_type',
+  type_text: 'cu_type',
+  observe: 'cu_observe',
+  look: 'cu_observe',
+  screenshot: 'cu_observe',
+  find: 'cu_find',
+  hover: 'cu_hover',
+  scroll: 'cu_scroll',
+  key: 'cu_key',
+  press: 'cu_key',
 };
 
 function knownNames(tools) {