Fix libei

README.md

@@ -165,14 +165,10 @@ Computer use is a mode, not default chat behavior. Say “take the wheel” or
 select Computer in ARC to request a grant. Jarvis then prefers domain actions,
 app D-Bus/GIO, AT-SPI references, GNOME Shell helpers, and finally vision
 coordinates. RemoteDesktop plus EIS/libei is the primary Wayland path.
+Jarvis ships a built-in libei injector; `JARVIS_LIBEI_BRIDGE` is optional.
 
-Configure the local injector before live actuation:
-
-~~~bash
-export JARVIS_LIBEI_BRIDGE="<local-eis-injector-command>"
-~~~
-
-The helper will not claim input readiness without this bridge. Password fields,
+The helper claims input readiness after ConnectToEIS or a portal Notify
+fallback. Password fields,
 lock screens, dangerous keys, destructive targets, and actions outside the
 step budget are blocked or require confirmation. See the
 [computer-use acceptance procedure](docs/cu-acceptance.md).

computer-use/observer.js

@@ -8,20 +8,36 @@ import { ShellProvider } from './shell-provider.js';
 const normalize = (value) => String(value || '').toLowerCase().trim();
 const score = (query, node) => { const q = normalize(query); const text = `${normalize(node.name)} ${normalize(node.role)}`; if (!q || !text) return 0; if (text === q) return 100; if (text.includes(q)) return 75; const words = q.split(/\s+/).filter((w) => text.includes(w)); return words.length ? 40 + words.length * 10 : 0; };
 
+function timed(promise, ms, label) {
+  const timeoutMs = Number(ms) > 0 ? Number(ms) : 0;
+  if (!timeoutMs) return promise;
+  let timer;
+  return Promise.race([
+    promise,
+    new Promise((_, reject) => {
+      timer = setTimeout(() => reject(new Error(`${label} timed out`)), timeoutMs);
+    }),
+  ]).finally(() => { if (timer) clearTimeout(timer); });
+}
+
 export class DesktopObserver {
-  constructor({ screenshot = new PortalScreenshot(), normalizer = new FrameNormalizer(), atspi = new AtspiProvider(), shell = new ShellProvider(), ocr, vision, tmpDir = '/tmp/jarvis-cu' } = {}) { this.screenshot = screenshot; this.normalizer = normalizer; this.atspi = atspi; this.shell = shell; this.ocr = ocr; this.vision = vision; this.tmpDir = tmpDir; this.lastTree = []; }
-  async tree({ focusedOnly = true, maxNodes = 400 } = {}) { const nodes = await this.atspi.tree({ focusedOnly, maxNodes }); this.lastTree = nodes.map((node, index) => ({ ...node, ref: `r${index + 1}` })); return this.lastTree; }
-  async observe({ includeTree = true, includeOcr = true, includeVision = false } = {}) {
+  constructor({ screenshot = new PortalScreenshot(), normalizer = new FrameNormalizer(), atspi = new AtspiProvider(), shell = new ShellProvider(), ocr, vision, tmpDir = '/tmp/jarvis-cu', timeouts = {} } = {}) {
+    this.screenshot = screenshot; this.normalizer = normalizer; this.atspi = atspi; this.shell = shell; this.ocr = ocr; this.vision = vision; this.tmpDir = tmpDir;
+    this.lastTree = [];
+    this.timeouts = { screenshot: timeouts.screenshot ?? 8000, tree: timeouts.tree ?? 5000, ocr: timeouts.ocr ?? 8000, vision: timeouts.vision ?? 8000 };
+  }
+  async tree({ focusedOnly = true, maxNodes = 400 } = {}) { const nodes = await timed(this.atspi.tree({ focusedOnly, maxNodes }), this.timeouts.tree, 'AT-SPI'); this.lastTree = nodes.map((node, index) => ({ ...node, ref: `r${index + 1}` })); return this.lastTree; }
+  async observe({ includeTree = true, includeOcr = false, includeVision = false } = {}) {
     await mkdir(this.tmpDir, { recursive: true });
     const unavailable = [];
     let rawPath; let frame = { path: null };
-    try { rawPath = await this.screenshot.capture(path.join(this.tmpDir, `observe-${Date.now()}.png`)); frame = await this.normalizer.normalize(rawPath); } catch (error) { unavailable.push(`screenshot: ${error.message}`); }
+    try { rawPath = await timed(this.screenshot.capture(path.join(this.tmpDir, `observe-${Date.now()}.png`)), this.timeouts.screenshot, 'screenshot'); frame = await this.normalizer.normalize(rawPath); } catch (error) { unavailable.push(`screenshot: ${error.message}`); }
     await this.shell.connect?.().catch((error) => unavailable.push(`Shell helper: ${error.message}`));
     const [windows, focused, tree] = await Promise.all([this.shell.windows(), this.shell.focused(), includeTree ? this.tree().catch((error) => { unavailable.push(`AT-SPI: ${error.message}`); return []; }) : Promise.resolve([])]);
     const result = { monitor: null, focused, windows: windows.windows || [], tree, screenshot_path: frame.path, ocr_blocks: [], vision_hint: null, unavailable };
     if (!windows.available) result.unavailable.push(windows.reason);
-    if (frame.path && includeOcr && this.ocr) result.ocr_blocks = await this.ocr(frame.path).catch((error) => { result.unavailable.push(`OCR: ${error.message}`); return []; });
-    if (frame.path && includeVision && this.vision) result.vision_hint = await this.vision(frame.path).catch((error) => { result.unavailable.push(`vision: ${error.message}`); return null; });
+    if (frame.path && includeOcr && this.ocr) result.ocr_blocks = await timed(this.ocr(frame.path), this.timeouts.ocr, 'OCR').catch((error) => { result.unavailable.push(`OCR: ${error.message}`); return []; });
+    if (frame.path && includeVision && this.vision) result.vision_hint = await timed(this.vision(frame.path), this.timeouts.vision, 'vision').catch((error) => { result.unavailable.push(`vision: ${error.message}`); return null; });
     return result;
   }
   async zoom({ rect, ref } = {}) { await mkdir(this.tmpDir, { recursive: true }); const node = ref ? this.lastTree.find((item) => item.ref === ref) : null; const target = rect || node?.rect; if (!target) throw new Error('rect or current tree ref is required'); const raw = await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`)); const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), target); return { rect: target, screenshot_path: frame.path }; }

computer-use/portal-input.js

@@ -40,7 +40,7 @@ export class PortalInputBackend {
           if (event.type === 'error') { fail(new Error(event.reason || 'portal input unavailable')); return; }
           if (event.type === 'ready' && this.process === child && !settled) {
             clearTimeout(timer); settled = true; this.available = true;
-            resolve({ restore_token_present: Boolean(event.restore_token_present), monitors, backend: 'portal-ei' });
+            resolve({ restore_token_present: Boolean(event.restore_token_present), monitors, backend: event.backend || 'portal-ei' });
           }
         }
       });

computer-use/portal-screenshot.js

@@ -2,8 +2,17 @@ import { spawn } from 'node:child_process';
 import path from 'node:path';
 
 export class PortalScreenshot {
-  constructor({ helper = path.resolve(new URL('./py/portal_screenshot.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn, tmpDir = '/tmp/jarvis-cu' } = {}) { this.helper = helper; this.python = python; this.spawnImpl = spawnImpl; this.tmpDir = tmpDir; }
+  constructor({ helper = path.resolve(new URL('./py/portal_screenshot.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn, tmpDir = '/tmp/jarvis-cu', timeoutMs = 8000 } = {}) { this.helper = helper; this.python = python; this.spawnImpl = spawnImpl; this.tmpDir = tmpDir; this.timeoutMs = timeoutMs; }
   async capture(output = path.join(this.tmpDir, `portal-${Date.now()}.png`)) {
-    return new Promise((resolve, reject) => { const child = this.spawnImpl(this.python, [this.helper, output], { stdio: ['ignore', 'pipe', 'pipe'] }); let out = ''; let err = ''; child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err += d; }); child.on('error', reject); child.on('close', (code) => code === 0 ? resolve(out.trim() || output) : reject(new Error(err || `Screenshot portal exited ${code}`))); });
+    return new Promise((resolve, reject) => {
+      const child = this.spawnImpl(this.python, [this.helper, output], { stdio: ['ignore', 'pipe', 'pipe'] });
+      let out = ''; let err = ''; let settled = false;
+      const timer = setTimeout(() => { if (settled) return; settled = true; child.kill('SIGTERM'); reject(new Error('screenshot portal timed out')); }, this.timeoutMs);
+      const done = (fn) => (value) => { if (settled) return; settled = true; clearTimeout(timer); fn(value); };
+      child.stdout.on('data', (d) => { out += d; });
+      child.stderr.on('data', (d) => { err += d; });
+      child.on('error', done(reject));
+      child.on('close', (code) => done(code === 0 ? resolve : reject)(code === 0 ? (out.trim() || output) : new Error(err || `Screenshot portal exited ${code}`)));
+    });
   }
 }

computer-use/py/libei_sender.py

@@ -0,0 +1,315 @@
+"""libei sender for an EIS fd from xdg-desktop-portal ConnectToEIS."""
+import ctypes
+import select
+import time
+
+# libei.h: enum ei_event_type starts at 1, then increments.
+EI_EVENT_CONNECT = 1
+EI_EVENT_DISCONNECT = 2
+EI_EVENT_SEAT_ADDED = 3
+EI_EVENT_SEAT_REMOVED = 4
+EI_EVENT_DEVICE_ADDED = 5
+EI_EVENT_DEVICE_REMOVED = 6
+EI_EVENT_DEVICE_PAUSED = 7
+EI_EVENT_DEVICE_RESUMED = 8
+
+# libei.h: enum ei_device_capability values are bitmasks, not 1..n.
+CAP_POINTER = 1 << 0
+CAP_POINTER_ABSOLUTE = 1 << 1
+CAP_KEYBOARD = 1 << 2
+CAP_TOUCH = 1 << 3
+CAP_SCROLL = 1 << 4
+CAP_BUTTON = 1 << 5
+BIND_CAPS = (CAP_POINTER, CAP_POINTER_ABSOLUTE, CAP_KEYBOARD, CAP_BUTTON, CAP_SCROLL)
+
+BTN_LEFT = 0x110
+BTN_RIGHT = 0x111
+BTN_MIDDLE = 0x112
+
+KEY_ESC = 1
+KEY_ENTER = 28
+KEY_LEFTCTRL = 29
+KEY_LEFTSHIFT = 42
+KEY_LEFTALT = 56
+KEY_SPACE = 57
+KEY_F1 = 59
+KEY_BACKSPACE = 14
+KEY_TAB = 15
+KEY_LEFTMETA = 125
+F_KEYS = {
+    'f1': 59, 'f2': 60, 'f3': 61, 'f4': 62, 'f5': 63, 'f6': 64, 'f7': 65, 'f8': 66,
+    'f9': 67, 'f10': 68, 'f11': 87, 'f12': 88,
+}
+COMBO_KEYS = {
+    'enter': KEY_ENTER, 'return': KEY_ENTER, 'esc': KEY_ESC, 'escape': KEY_ESC,
+    'backspace': KEY_BACKSPACE, 'tab': KEY_TAB, 'space': KEY_SPACE,
+    'ctrl': KEY_LEFTCTRL, 'control': KEY_LEFTCTRL, 'alt': KEY_LEFTALT,
+    'shift': KEY_LEFTSHIFT, 'super': KEY_LEFTMETA, 'meta': KEY_LEFTMETA,
+    **F_KEYS,
+}
+LETTER_KEYS = {ch: code for ch, code in zip('qwertyuiopasdfghjklzxcvbnm', [
+    16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 30, 31, 32, 33, 34, 35, 36, 37, 38, 44, 45, 46, 47, 48, 49,
+])}
+DIGIT_KEYS = {str(d): code for d, code in enumerate([11, 2, 3, 4, 5, 6, 7, 8, 9, 10])}
+
+
+def _ptr(value):
+    return int(value) if value else 0
+
+
+def _lib():
+    lib = ctypes.CDLL('libei.so.1')
+    lib.ei_new_sender.restype = ctypes.c_void_p
+    lib.ei_new_sender.argtypes = [ctypes.c_void_p]
+    lib.ei_configure_name.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
+    lib.ei_setup_backend_fd.argtypes = [ctypes.c_void_p, ctypes.c_int]
+    lib.ei_setup_backend_fd.restype = ctypes.c_int
+    lib.ei_get_fd.argtypes = [ctypes.c_void_p]
+    lib.ei_get_fd.restype = ctypes.c_int
+    lib.ei_dispatch.argtypes = [ctypes.c_void_p]
+    lib.ei_get_event.argtypes = [ctypes.c_void_p]
+    lib.ei_get_event.restype = ctypes.c_void_p
+    lib.ei_event_get_type.argtypes = [ctypes.c_void_p]
+    lib.ei_event_get_type.restype = ctypes.c_int
+    lib.ei_event_get_seat.argtypes = [ctypes.c_void_p]
+    lib.ei_event_get_seat.restype = ctypes.c_void_p
+    lib.ei_event_get_device.argtypes = [ctypes.c_void_p]
+    lib.ei_event_get_device.restype = ctypes.c_void_p
+    lib.ei_event_unref.argtypes = [ctypes.c_void_p]
+    lib.ei_seat_bind_capabilities.restype = None
+    lib.ei_seat_has_capability.argtypes = [ctypes.c_void_p, ctypes.c_int]
+    lib.ei_seat_has_capability.restype = ctypes.c_int
+    lib.ei_device_has_capability.argtypes = [ctypes.c_void_p, ctypes.c_int]
+    lib.ei_device_has_capability.restype = ctypes.c_int
+    lib.ei_device_ref.argtypes = [ctypes.c_void_p]
+    lib.ei_device_ref.restype = ctypes.c_void_p
+    lib.ei_device_unref.argtypes = [ctypes.c_void_p]
+    lib.ei_device_start_emulating.argtypes = [ctypes.c_void_p, ctypes.c_uint]
+    lib.ei_device_stop_emulating.argtypes = [ctypes.c_void_p]
+    lib.ei_device_pointer_motion_absolute.argtypes = [ctypes.c_void_p, ctypes.c_double, ctypes.c_double]
+    lib.ei_device_pointer_motion.argtypes = [ctypes.c_void_p, ctypes.c_double, ctypes.c_double]
+    lib.ei_device_button_button.argtypes = [ctypes.c_void_p, ctypes.c_uint, ctypes.c_bool]
+    lib.ei_device_scroll_discrete.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int]
+    lib.ei_device_keyboard_key.argtypes = [ctypes.c_void_p, ctypes.c_uint, ctypes.c_bool]
+    lib.ei_device_frame.argtypes = [ctypes.c_void_p, ctypes.c_uint64]
+    lib.ei_now.argtypes = [ctypes.c_void_p]
+    lib.ei_now.restype = ctypes.c_uint64
+    lib.ei_unref.argtypes = [ctypes.c_void_p]
+    return lib
+
+
+class LibeiSender:
+    def __init__(self, fd, name='jarvis'):
+        self.lib = _lib()
+        self.ei = self.lib.ei_new_sender(None)
+        if not self.ei:
+            raise RuntimeError('ei_new_sender failed')
+        self.lib.ei_configure_name(self.ei, name.encode())
+        if self.lib.ei_setup_backend_fd(self.ei, int(fd)) != 0:
+            raise RuntimeError('ei_setup_backend_fd failed')
+        self.fd = self.lib.ei_get_fd(self.ei)
+        self.pointer_abs = None
+        self.pointer_rel = None
+        self.keyboard = None
+        self._emulating = set()
+        self._seq = 1
+
+    @property
+    def pointer(self):
+        return self.pointer_abs or self.pointer_rel
+
+    def _has(self, device, cap):
+        return bool(device) and bool(self.lib.ei_device_has_capability(device, cap))
+
+    def _adopt(self, current, device):
+        if _ptr(current) == _ptr(device):
+            return current
+        if current:
+            self.lib.ei_device_unref(current)
+        return self.lib.ei_device_ref(device)
+
+    def _drop(self, device):
+        key = _ptr(device)
+        self._emulating.discard(key)
+        dropped = []
+        for attr in ('pointer_abs', 'pointer_rel', 'keyboard'):
+            current = getattr(self, attr)
+            if _ptr(current) == key:
+                dropped.append(current)
+                setattr(self, attr, None)
+        for current in dropped:
+            self.lib.ei_device_unref(current)
+
+    def _remember(self, device):
+        if self._has(device, CAP_POINTER_ABSOLUTE):
+            self.pointer_abs = self._adopt(self.pointer_abs, device)
+        elif self._has(device, CAP_POINTER):
+            self.pointer_rel = self._adopt(self.pointer_rel, device)
+        if self._has(device, CAP_KEYBOARD):
+            self.keyboard = self._adopt(self.keyboard, device)
+
+    def wait_ready(self, timeout=8.0):
+        deadline = time.time() + timeout
+        collected = None
+        while time.time() < deadline:
+            remaining = max(0.05, deadline - time.time())
+            readable, _, _ = select.select([self.fd], [], [], remaining)
+            if readable:
+                self.dispatch()
+            if self._emulating and (self.pointer or self.keyboard):
+                if collected is None:
+                    collected = time.time()
+                elif time.time() - collected >= 0.35 or self.pointer_abs:
+                    return True
+        return bool(self._emulating and (self.pointer or self.keyboard))
+
+    def dispatch(self):
+        self.lib.ei_dispatch(self.ei)
+        while True:
+            event = self.lib.ei_get_event(self.ei)
+            if not event:
+                break
+            kind = self.lib.ei_event_get_type(event)
+            if kind == EI_EVENT_SEAT_ADDED:
+                seat = self.lib.ei_event_get_seat(event)
+                caps = [cap for cap in BIND_CAPS if self.lib.ei_seat_has_capability(seat, cap)] or list(BIND_CAPS)
+                self.lib.ei_seat_bind_capabilities(seat, *[ctypes.c_int(cap) for cap in caps], None)
+            elif kind == EI_EVENT_DEVICE_ADDED:
+                self._remember(self.lib.ei_event_get_device(event))
+            elif kind == EI_EVENT_DEVICE_RESUMED:
+                device = self.lib.ei_event_get_device(event)
+                self.lib.ei_device_start_emulating(device, self._seq)
+                self._seq += 1
+                self._emulating.add(_ptr(device))
+                self._remember(device)
+            elif kind == EI_EVENT_DEVICE_PAUSED:
+                self._emulating.discard(_ptr(self.lib.ei_event_get_device(event)))
+            elif kind in (EI_EVENT_DEVICE_REMOVED, EI_EVENT_DISCONNECT):
+                if kind == EI_EVENT_DEVICE_REMOVED:
+                    self._drop(self.lib.ei_event_get_device(event))
+                else:
+                    for current in (self.pointer_abs, self.pointer_rel, self.keyboard):
+                        if current:
+                            self._drop(current)
+            self.lib.ei_event_unref(event)
+
+    def _frame(self, device):
+        self.lib.ei_device_frame(device, self.lib.ei_now(self.ei))
+
+    def _button_code(self, name):
+        if name in ('right', 3, '3'):
+            return BTN_RIGHT
+        if name in ('middle', 2, '2'):
+            return BTN_MIDDLE
+        return BTN_LEFT
+
+    def move(self, x, y):
+        device = self.pointer
+        if not device:
+            raise RuntimeError('no EIS pointer device')
+        if self._has(device, CAP_POINTER_ABSOLUTE):
+            self.lib.ei_device_pointer_motion_absolute(device, float(x), float(y))
+        else:
+            self.lib.ei_device_pointer_motion(device, float(x), float(y))
+        self._frame(device)
+
+    def click(self, x, y, button='left', count=1):
+        self.move(x, y)
+        device = self.pointer
+        code = self._button_code(button)
+        for _ in range(max(1, int(count))):
+            self.lib.ei_device_button_button(device, code, True)
+            self._frame(device)
+            self.lib.ei_device_button_button(device, code, False)
+            self._frame(device)
+
+    def scroll(self, x, y, dx=0, dy=0):
+        self.move(x, y)
+        self.lib.ei_device_scroll_discrete(self.pointer, int(dx), int(dy))
+        self._frame(self.pointer)
+
+    def drag(self, start, end):
+        self.move(start[0], start[1])
+        self.lib.ei_device_button_button(self.pointer, BTN_LEFT, True)
+        self._frame(self.pointer)
+        self.move(end[0], end[1])
+        self.lib.ei_device_button_button(self.pointer, BTN_LEFT, False)
+        self._frame(self.pointer)
+
+    def _tap(self, code, mods=()):
+        if not self.keyboard:
+            raise RuntimeError('no EIS keyboard device')
+        for mod in mods:
+            self.lib.ei_device_keyboard_key(self.keyboard, mod, True)
+            self._frame(self.keyboard)
+        self.lib.ei_device_keyboard_key(self.keyboard, code, True)
+        self._frame(self.keyboard)
+        self.lib.ei_device_keyboard_key(self.keyboard, code, False)
+        self._frame(self.keyboard)
+        for mod in reversed(mods):
+            self.lib.ei_device_keyboard_key(self.keyboard, mod, False)
+            self._frame(self.keyboard)
+
+    def type_text(self, text, submit=False):
+        for ch in str(text):
+            if ch == '\n':
+                self._tap(KEY_ENTER)
+                continue
+            if ch == ' ':
+                self._tap(KEY_SPACE)
+                continue
+            if ch == '\t':
+                self._tap(KEY_TAB)
+                continue
+            lower = ch.lower()
+            code = LETTER_KEYS.get(lower) or DIGIT_KEYS.get(ch)
+            if code is None:
+                continue
+            self._tap(code, (KEY_LEFTSHIFT,) if ch.isupper() else ())
+        if submit:
+            self._tap(KEY_ENTER)
+
+    def key_combo(self, combo):
+        parts = [p.strip().lower() for p in str(combo).replace('-', '+').split('+') if p.strip()]
+        if not parts:
+            return
+        mods = [COMBO_KEYS[part] for part in parts[:-1] if part in COMBO_KEYS]
+        key = parts[-1]
+        code = COMBO_KEYS.get(key) or LETTER_KEYS.get(key) or DIGIT_KEYS.get(key)
+        if code is None:
+            return
+        self._tap(code, tuple(mods))
+
+    def handle(self, action):
+        self.dispatch()
+        kind = action.get('type')
+        name = action.get('action')
+        if kind == 'pointer' and name in ('click', 'double_click'):
+            self.click(action.get('x') or 0, action.get('y') or 0, action.get('button') or 'left', 2 if name == 'double_click' else 1)
+        elif kind == 'pointer' and name == 'move':
+            self.move(action.get('x') or 0, action.get('y') or 0)
+        elif kind == 'pointer' and name == 'scroll':
+            self.scroll(action.get('x') or 0, action.get('y') or 0, action.get('dx') or 0, action.get('dy') or 0)
+        elif kind == 'pointer' and name == 'drag':
+            self.drag(action.get('from') or [0, 0], action.get('to') or [0, 0])
+        elif kind == 'keyboard' and name == 'type':
+            self.type_text(action.get('text') or '', action.get('submit'))
+        elif kind == 'keyboard' and name == 'key':
+            self.key_combo(action.get('combo') or '')
+        else:
+            raise RuntimeError(f'unsupported EIS action {kind} {name}')
+
+    def close(self):
+        seen = []
+        for device in (self.pointer_abs, self.pointer_rel, self.keyboard):
+            if device and _ptr(device) not in seen:
+                seen.append(_ptr(device))
+                try:
+                    self.lib.ei_device_unref(device)
+                except Exception:
+                    pass
+        self.pointer_abs = self.pointer_rel = self.keyboard = None
+        self._emulating.clear()
+        if self.ei:
+            self.lib.ei_unref(self.ei)
+        self.ei = None

computer-use/py/portal_remote_desktop.py

@@ -1,9 +1,13 @@
 #!/usr/bin/env python3
-"""RemoteDesktop consent broker for the local EIS input worker."""
+"""RemoteDesktop consent broker with a built-in libei injector."""
 import json
-import sys
 import os
+import select
 import subprocess
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
 try:
     import gi
     gi.require_version('Gio', '2.0')
@@ -12,36 +16,186 @@ except Exception as exc:
     print(json.dumps({'type': 'error', 'reason': f'PyGObject unavailable: {exc}'}), flush=True)
     raise SystemExit(1)
 
+from libei_sender import LibeiSender
+
+BTN = {'left': 0x110, 'right': 0x111, 'middle': 0x112}
+
+
+def log_error(reason):
+    print(json.dumps({'type': 'error', 'reason': str(reason)}), flush=True)
+
+
 bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
-proxy = Gio.DBusProxy.new_sync(bus, Gio.DBusProxyFlags.NONE, None, 'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop', 'org.freedesktop.portal.RemoteDesktop', None)
-def request(method, signature, values):
-    request_path = proxy.call_sync(method, GLib.Variant(signature, values), Gio.DBusCallFlags.NONE, -1, None).unpack()[0]
-    loop = GLib.MainLoop(); result = {'code': 1, 'results': {}}
+proxy = Gio.DBusProxy.new_sync(
+    bus, Gio.DBusProxyFlags.NONE, None,
+    'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop',
+    'org.freedesktop.portal.RemoteDesktop', None,
+)
+
+
+def request(method, signature, values, timeout_ms=120000):
+    request_path = proxy.call_sync(method, GLib.Variant(signature, values), Gio.DBusCallFlags.NONE, timeout_ms, None).unpack()[0]
+    loop = GLib.MainLoop()
+    result = {'code': 1, 'results': {}}
+
     def response(_conn, _sender, _path, _interface, _member, params):
-        result['code'], result['results'] = params.unpack(); loop.quit()
+        result['code'], result['results'] = params.unpack()
+        loop.quit()
+
     sub = bus.signal_subscribe(None, 'org.freedesktop.portal.Request', 'Response', request_path, None, Gio.DBusSignalFlags.NONE, response)
-    loop.run(); bus.signal_unsubscribe(sub)
-    if result['code'] != 0: raise RuntimeError(f'{method} portal response {result["code"]}')
+    loop.run()
+    bus.signal_unsubscribe(sub)
+    if result['code'] != 0:
+        raise RuntimeError(f'{method} portal response {result["code"]}')
     return result['results']
+
+
+def connect_to_eis(session):
+    incoming = Gio.UnixFDList.new()
+    variant, outgoing = proxy.call_with_unix_fd_list_sync(
+        'ConnectToEIS',
+        GLib.Variant('(oa{sv})', (session, {})),
+        Gio.DBusCallFlags.NONE,
+        15000,
+        incoming,
+        None,
+    )
+    if outgoing is not None and outgoing.get_length() > 0:
+        unpacked = variant.unpack()
+        index = unpacked[0] if isinstance(unpacked, (tuple, list)) else unpacked
+        if isinstance(index, int) and 0 <= index < outgoing.get_length():
+            return outgoing.get(index)
+        return outgoing.get(0)
+    unpacked = variant.unpack()
+    fd = unpacked[0] if isinstance(unpacked, (tuple, list)) else unpacked
+    if isinstance(fd, int) and fd >= 0:
+        return fd
+    raise RuntimeError('ConnectToEIS returned no file descriptor')
+
+
+def notify(method, signature, values):
+    proxy.call_sync(method, GLib.Variant(signature, values), Gio.DBusCallFlags.NONE, 5000, None)
+
+
+class PortalNotify:
+    def __init__(self, session):
+        self.session = session
+
+    def handle(self, action):
+        kind = action.get('type')
+        name = action.get('action')
+        opts = {}
+        session = self.session
+        if kind == 'pointer' and name in ('click', 'double_click', 'move'):
+            x = float(action.get('x') or 0)
+            y = float(action.get('y') or 0)
+            notify('NotifyPointerMotionAbsolute', '(oa{sv}udd)', (session, opts, 0, x, y))
+            if name != 'move':
+                button = BTN.get(str(action.get('button') or 'left'), BTN['left'])
+                repeats = 2 if name == 'double_click' else 1
+                for _ in range(repeats):
+                    notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, button, 1))
+                    notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, button, 0))
+        elif kind == 'pointer' and name == 'scroll':
+            notify('NotifyPointerAxisDiscrete', '(oa{sv}ui)', (session, opts, 0, int(action.get('dy') or 0)))
+        elif kind == 'keyboard' and name == 'type':
+            for ch in str(action.get('text') or ''):
+                keysym = 0xff0d if ch == '\n' else (0x020 if ch == ' ' else ord(ch))
+                notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, keysym, 1))
+                notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, keysym, 0))
+            if action.get('submit'):
+                notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, 0xff0d, 1))
+                notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, 0xff0d, 0))
+        elif kind == 'keyboard' and name == 'key':
+            combo = str(action.get('combo') or '').lower()
+            keysym = 0xff1b if 'esc' in combo else 0xff0d
+            notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, keysym, 1))
+            notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, keysym, 0))
+        else:
+            raise RuntimeError(f'unsupported portal notify action {kind} {name}')
+
+
+def run_loop(handler, extra_fd=None):
+    stdin_fd = sys.stdin.fileno()
+    while True:
+        fds = [stdin_fd]
+        if extra_fd is not None:
+            fds.append(extra_fd)
+        readable, _, _ = select.select(fds, [], [], 0.25)
+        if extra_fd is not None and extra_fd in readable and hasattr(handler, 'dispatch'):
+            handler.dispatch()
+        if stdin_fd in readable:
+            line = sys.stdin.readline()
+            if not line:
+                break
+            try:
+                action = json.loads(line)
+            except Exception:
+                continue
+            if action.get('type') == 'close':
+                break
+            try:
+                handler.handle(action)
+            except Exception as exc:
+                log_error(exc)
+
+
+injector = None
+sender = None
+session = None
 try:
     token = f'jarvis{GLib.get_real_time()}'
     session = request('CreateSession', '(a{sv})', ({'session_handle_token': GLib.Variant('s', token)},))['session_handle']
-    request('SelectDevices', '(oa{sv})', (session, {'types': GLib.Variant('u', 3), 'persist_mode': GLib.Variant('u', 2)}))
+    request('SelectDevices', '(oa{sv})', (session, {'types': GLib.Variant('u', 7), 'persist_mode': GLib.Variant('u', 2)}))
     request('Start', '(osa{sv})', (session, '', {}))
-    if not os.environ.get('JARVIS_LIBEI_BRIDGE'):
-        raise RuntimeError('portal consent succeeded but no local libei injector is configured; refusing to claim input readiness')
-    injector = subprocess.Popen(os.environ['JARVIS_LIBEI_BRIDGE'], shell=True, stdin=subprocess.PIPE, text=True)
-    print(json.dumps({'type': 'ready', 'session': session, 'restore_token_present': True}), flush=True)
-except Exception as exc:
-    print(json.dumps({'type': 'error', 'reason': str(exc)}), flush=True)
-for line in sys.stdin:
+    backend = 'none'
+    extra_fd = None
+    handler = None
+    eis_error = None
     try:
-        action = json.loads(line)
-        if action.get('type') == 'close':
-            break
-        if 'injector' in globals() and injector.stdin:
-            injector.stdin.write(json.dumps(action) + '\n'); injector.stdin.flush()
+        eis_fd = connect_to_eis(session)
+        sender = LibeiSender(eis_fd)
+        if not sender.wait_ready():
+            raise RuntimeError('libei connected but no pointer or keyboard device appeared')
+        handler = sender
+        extra_fd = sender.fd
+        backend = 'portal-ei'
+    except Exception as exc:
+        eis_error = str(exc)
+        if sender:
+            try:
+                sender.close()
             except Exception:
-        continue
-if 'injector' in globals():
+                pass
+        sender = None
+    if handler is None and os.environ.get('JARVIS_LIBEI_BRIDGE'):
+        injector = subprocess.Popen(os.environ['JARVIS_LIBEI_BRIDGE'], shell=True, stdin=subprocess.PIPE, text=True)
+
+        class Bridge:
+            def handle(self, action):
+                injector.stdin.write(json.dumps(action) + '\n')
+                injector.stdin.flush()
+
+        handler = Bridge()
+        backend = 'portal-ei'
+    if handler is None:
+        handler = PortalNotify(session)
+        backend = 'portal-notify'
+    print(json.dumps({
+        'type': 'ready',
+        'session': session,
+        'restore_token_present': True,
+        'backend': backend,
+        'eis_error': eis_error,
+    }), flush=True)
+    run_loop(handler, extra_fd)
+except Exception as exc:
+    log_error(exc)
+finally:
+    if sender:
+        try:
+            sender.close()
+        except Exception:
+            pass
+    if injector:
         injector.terminate()

computer-use/py/portal_screenshot.py

@@ -1,27 +1,68 @@
 #!/usr/bin/env python3
-"""Capture one frame through the GNOME Screenshot portal and print its path."""
+"""Capture one frame through GNOME Shell or the Screenshot portal and print its path."""
 import sys
 from gi.repository import Gio, GLib
 
 target = sys.argv[1]
 bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
-proxy = Gio.DBusProxy.new_sync(bus, Gio.DBusProxyFlags.NONE, None, 'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop', 'org.freedesktop.portal.Screenshot', None)
-request = proxy.call_sync('Screenshot', GLib.Variant('(a{sv})', ({'interactive': GLib.Variant('b', False)},)), Gio.DBusCallFlags.NONE, -1, None).unpack()[0]
+
+
+def gnome_shell_screenshot():
+    proxy = Gio.DBusProxy.new_sync(
+        bus, Gio.DBusProxyFlags.NONE, None,
+        'org.gnome.Shell.Screenshot', '/org/gnome/Shell/Screenshot',
+        'org.gnome.Shell.Screenshot', None,
+    )
+    ok, path = proxy.call_sync(
+        'Screenshot',
+        GLib.Variant('(bbs)', (False, False, target)),
+        Gio.DBusCallFlags.NONE,
+        5000,
+        None,
+    ).unpack()
+    if not ok:
+        raise RuntimeError('GNOME Shell screenshot failed')
+    return path or target
+
+
+def portal_screenshot():
+    proxy = Gio.DBusProxy.new_sync(
+        bus, Gio.DBusProxyFlags.NONE, None,
+        'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop',
+        'org.freedesktop.portal.Screenshot', None,
+    )
+    options = {'interactive': GLib.Variant('b', False)}
+    try:
+        request = proxy.call_sync('Screenshot', GLib.Variant('(a{sv})', (options,)), Gio.DBusCallFlags.NONE, 8000, None).unpack()[0]
+    except Exception:
+        request = proxy.call_sync('Screenshot', GLib.Variant('(sa{sv})', ('', options)), Gio.DBusCallFlags.NONE, 8000, None).unpack()[0]
     loop = GLib.MainLoop()
-result = {'uri': None, 'code': 1}
+    result = {'uri': None}
+
     def response(_conn, _sender, _path, _interface, _member, params):
         code, values = params.unpack()
         if code == 0:
             result['uri'] = values.get('uri')
-        result['code'] = 0
         loop.quit()
-subscription = bus.signal_subscribe(None, 'org.freedesktop.portal.Request', 'Response', request, None, Gio.DBusSignalFlags.NONE, response)
+
+    sub = bus.signal_subscribe(None, 'org.freedesktop.portal.Request', 'Response', request, None, Gio.DBusSignalFlags.NONE, response)
+    GLib.timeout_add(7000, loop.quit)
     loop.run()
-bus.signal_unsubscribe(subscription)
+    bus.signal_unsubscribe(sub)
     if not result['uri']:
-    raise SystemExit('Screenshot portal returned no image URI')
+        raise RuntimeError('Screenshot portal returned no image URI')
     ok, contents, _etag = Gio.File.new_for_uri(result['uri']).load_contents(None)
     if not ok:
-    raise SystemExit('could not read screenshot portal URI')
+        raise RuntimeError('could not read screenshot portal URI')
     Gio.File.new_for_path(target).replace_contents(contents, None, False, Gio.FileCreateFlags.REPLACE_DESTINATION, None)
-print(target)
+    return target
+
+
+errors = []
+for capture in (gnome_shell_screenshot, portal_screenshot):
+    try:
+        print(capture())
+        raise SystemExit(0)
+    except Exception as exc:
+        errors.append(f'{capture.__name__}: {exc}')
+raise SystemExit('; '.join(errors) or 'screenshot unavailable')

computer-use/session.js

@@ -23,7 +23,7 @@ export class ComputerUseSession {
     return { session_id: this.sessionId, restore_token_present: Boolean(persist), monitors, backend: this.backend };
   }
 
-  setBackend(backend) { this.backend = ['portal-ei', 'ydotool', 'none'].includes(backend) ? backend : 'none'; return this.backend; }
+  setBackend(backend) { this.backend = ['portal-ei', 'portal-notify', 'ydotool', 'none'].includes(backend) ? backend : 'none'; return this.backend; }
 
   revoke() {
     this.active = false;

daemon/perception.js

@@ -1,8 +1,9 @@
-import { acquireQvac, releaseQvac, loadAuxiliaryModel, qvacSdk, withQvacMaster } from './qvac-master.js';
+import { acquireQvac, releaseQvac, loadAuxiliaryModel, qvacSdk, withQvacMaster, qvacBusy } from './qvac-master.js';
 
 export class QvacPerception {
   constructor({ model = process.env.JARVIS_OCR_MODEL || 'OCR_LATIN' } = {}) { this.model = model; this.modelId = null; }
   async ocr(image) {
+    if (qvacBusy()) throw new Error('OCR skipped while the language model is answering');
     await acquireQvac();
     try {
       this.modelId ||= await loadAuxiliaryModel(this.model, {});

daemon/qvac-master.js

@@ -127,6 +127,15 @@ export async function unloadAuxiliaryModel(modelId) {
 
 export function releaseQvac() { ownerCount = Math.max(0, ownerCount - 1); }
 
+export function qvacBusy() {
+  try {
+    const loaded = Agent.engine.getLoaded?.();
+    return Boolean(loaded && loaded.requestId);
+  } catch {
+    return false;
+  }
+}
+
 export async function closeQvac() {
   if (ownerCount > 0) return;
   for (const modelId of auxiliaryModels.values()) await unloadAuxiliaryModel(modelId).catch(() => {});

docs/computer-use.md

@@ -39,9 +39,11 @@ sequenceDiagram
 5. Vision coordinates for broken accessibility.
 6. Opt-in ydotool/X11 fallback.
 
-The primary Wayland input path is XDG RemoteDesktop plus EIS/libei. The
-portal helper refuses to report input readiness without a configured local EIS
-bridge (`JARVIS_LIBEI_BRIDGE`). Screenshot frames are downscaled for vision,
+The primary Wayland input path is XDG RemoteDesktop plus EIS/libei. After
+portal consent, Jarvis opens `ConnectToEIS` and injects pointer/keyboard
+events through a built-in libei sender. `JARVIS_LIBEI_BRIDGE` is an optional
+override. If EIS is unavailable, the helper falls back to portal Notify
+methods instead of refusing readiness. Screenshot frames are downscaled for vision,
 kept in temporary storage, and removed on revoke unless trace retention is
 enabled.
 

docs/cu-acceptance.md

@@ -3,12 +3,13 @@
 Run these on Ubuntu GNOME after `npm run cu-doctor` reports its required portal,
 PipeWire, AT-SPI, and libei checks as passed. The ydotool result is optional.
 The explicit computer-use grant is required, screenshots stay in
-`/tmp/jarvis-cu`, and actuation is enabled only after the local EIS injector
+`/tmp/jarvis-cu`, and actuation is enabled only after the portal helper
 reports ready.
 
-For live pointer and keyboard injection, configure `JARVIS_LIBEI_BRIDGE` with
-the local EIS injector command. The portal helper refuses readiness without it;
-this prevents a consent-only session from being mistaken for working input.
+For live pointer and keyboard injection, Jarvis uses xdg-desktop-portal
+`ConnectToEIS` plus a built-in libei sender. Set `JARVIS_LIBEI_BRIDGE` only to
+replace that sender. The helper no longer refuses readiness when the env var is
+unset.
 
 1. Observe Settings and toggle Night Light.
 2. Type a sentence into Text Editor and save it.

docs/troubleshooting.md

@@ -18,10 +18,11 @@ remains useful for typed or manually triggered testing.
 
 ## Portal consent succeeds but input is unavailable
 
-ScreenCast consent does not prove that RemoteDesktop/EIS injection is ready.
-Configure `JARVIS_LIBEI_BRIDGE`; the helper intentionally fails closed without
-it. Use observe-only mode until the injector reports ready. `ydotool` is an
-optional, explicitly enabled fallback.
+Grant desktop starts RemoteDesktop, then Jarvis connects to EIS with the
+bundled libei sender. An external `JARVIS_LIBEI_BRIDGE` is optional. If input
+still fails, run `npm run cu-doctor` and confirm `libei` is installed. Observation
+does not need the injector: after a grant, `cu_observe` should return a screenshot
+and accessibility tree even when pointer injection is down.
 
 ## The extension is missing
 

skills/computer-observe.js

@@ -1,10 +1,10 @@
-const PERMISSION = 'computer-use';
+const PERMISSION = 'read';
 const inactive = (computer) => !computer?.status?.().active ? { unavailable: 'computer-use grant required', action: 'cu.grant' } : null;
 
 export function createComputerObserveTools({ computer, observer } = {}) {
   const guard = () => { const result = inactive(computer); if (result) throw new Error(result.unavailable); };
   return [
-    { name: 'cu_observe', permission: PERMISSION, description: 'Observe the local desktop through the portal, AT-SPI, OCR, and optional local vision.', parameters: { type: 'object', properties: { include_tree: { type: 'boolean' }, include_ocr: { type: 'boolean' }, include_vision: { type: 'boolean' } } }, execute: async ({ include_tree = true, include_ocr = true, include_vision = false } = {}) => { guard(); return observer.observe({ includeTree: include_tree, includeOcr: include_ocr, includeVision: include_vision }); } },
+    { name: 'cu_observe', permission: PERMISSION, description: 'Observe the local desktop through the portal, AT-SPI, and the accessibility tree. OCR is off unless include_ocr is true.', parameters: { type: 'object', properties: { include_tree: { type: 'boolean' }, include_ocr: { type: 'boolean' }, include_vision: { type: 'boolean' } } }, execute: async ({ include_tree = true, include_ocr = false, include_vision = false } = {}) => { guard(); return observer.observe({ includeTree: include_tree, includeOcr: include_ocr, includeVision: include_vision }); } },
     { name: 'cu_zoom', permission: PERMISSION, description: 'Request a higher resolution local crop by rectangle or current AT-SPI ref.', parameters: { type: 'object', properties: { rect: { type: 'array', items: { type: 'number' } }, ref: { type: 'string' } } }, execute: async (args) => { guard(); return observer.zoom(args); } },
     { name: 'cu_tree', permission: PERMISSION, description: 'Return visible AT-SPI roles, names, states, bounds, and per-step refs.', parameters: { type: 'object', properties: { app: { type: 'string' }, focused_only: { type: 'boolean' }, max_nodes: { type: 'number' } } }, execute: async ({ focused_only = true, max_nodes = 400 } = {}) => { guard(); return observer.tree({ focusedOnly: focused_only, maxNodes: max_nodes }); } },
     { name: 'cu_find', permission: PERMISSION, description: 'Find a visible desktop element by accessible name/role.', parameters: { type: 'object', properties: { query: { type: 'string' }, role: { type: 'string' } }, required: ['query'] }, execute: async (args) => { guard(); return observer.find(args); } },

skills/voice-prompt.js

@@ -13,7 +13,7 @@ This computer can reach the internet. web_search and web_fetch are unrestricted
 
 File tools may read any path they accept. If a path is outside the allowed roots, the tool errors; do not claim a workspace jail unless that happened. Writes, including fs_write and overwrite, still need confirmation.
 
-Computer use requires an explicit user grant from Settings, Computer use, Allow now, or Grant desktop in the tray. Never click or type while it is inactive, locked, or revoked. Never ask for passwords or credentials.
+Computer use requires an explicit user grant from Settings, Computer use, Allow now, or Grant desktop in the tray. After a grant, call cu_observe to see the screen. Do not wait for a libei injector. Never click or type while it is inactive, locked, or revoked. Never ask for passwords or credentials.
 Destructive actions require confirmation in both the heads-up display and spoken conversation.
 Prefer structured tools and accessibility references over coordinates.
 

test/computer-use.test.js

@@ -1,11 +1,13 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
 import { ComputerUseSession } from '../computer-use/session.js';
 import { ComputerActuator } from '../computer-use/actuator.js';
 import { ComputerAudit } from '../computer-use/audit.js';
 import { mkdtemp, readFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import path from 'node:path';
+import { fileURLToPath } from 'node:url';
 
 test('computer use requires an explicit grant and enforces its step budget', () => {
   let now = 1000;
@@ -30,6 +32,13 @@ test('computer use expires its wall clock grant', () => {
   assert.equal(session.status().backend, 'none');
 });
 
+test('computer use records portal-notify as a live backend', () => {
+  const session = new ComputerUseSession();
+  session.grant();
+  assert.equal(session.setBackend('portal-notify'), 'portal-notify');
+  assert.equal(session.status().backend, 'portal-notify');
+});
+
 test('computer use semantic actuation previews, budgets, and audits actions', async () => {
   const dir = await mkdtemp(path.join(tmpdir(), 'jarvis-cu-audit-'));
   const audit = new ComputerAudit({ dir }); const session = new ComputerUseSession({ stepsMax: 1, audit }); const sent = [];
@@ -44,3 +53,11 @@ test('computer use refuses password targets and unconfirmed dangerous keys', asy
   await assert.rejects(() => actuator.type({ ref: 'password', text: 'secret' }), /password/);
   await assert.rejects(() => actuator.key({ combo: 'alt+f4' }), /confirmation/);
 });
+
+test('libei sender binds bitmask capabilities from libei.h', () => {
+  const pyDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../computer-use/py');
+  const compiled = spawnSync('python3', ['-m', 'py_compile', 'libei_sender.py', 'portal_remote_desktop.py', 'portal_screenshot.py'], { cwd: pyDir, encoding: 'utf8' });
+  assert.equal(compiled.status, 0, compiled.stderr);
+  const caps = spawnSync('python3', ['-c', 'from libei_sender import CAP_POINTER, CAP_KEYBOARD, CAP_BUTTON, F_KEYS; assert CAP_POINTER == 1 and CAP_KEYBOARD == 4 and CAP_BUTTON == 32 and F_KEYS["f11"] == 87'], { cwd: pyDir, encoding: 'utf8' });
+  assert.equal(caps.status, 0, caps.stderr);
+});

test/observer.test.js

@@ -15,6 +15,20 @@ test('Phase 6 observer returns stable refs and ranked semantic matches', async (
   const zoom = await observer.zoom({ ref: 'r1' }); assert.deepEqual(zoom.rect, [10, 20, 30, 30]);
 });
 
+test('observer returns after a hung screenshot instead of staying on THINKING', async () => {
+  const observer = new DesktopObserver({
+    screenshot: { capture: () => new Promise(() => {}) },
+    shell: { windows: async () => ({ available: true, windows: [] }), focused: async () => null },
+    atspi: { tree: async () => [] },
+    timeouts: { screenshot: 40 },
+  });
+  const started = Date.now();
+  const bundle = await observer.observe({ includeTree: false, includeOcr: false });
+  assert.ok(Date.now() - started < 1000);
+  assert.equal(bundle.screenshot_path, null);
+  assert.match(bundle.unavailable.join(' '), /screenshot/);
+});
+
 test('Phase 6 observer reports portal failure instead of inventing a frame', async () => {
   const observer = new DesktopObserver({ screenshot: { capture: async () => { throw new Error('no session bus'); } }, shell: { windows: async () => ({ available: false, windows: [] }), focused: async () => null }, atspi: { tree: async () => [] } });
   const bundle = await observer.observe({ includeTree: false, includeOcr: false });

test/review-regressions.test.js

@@ -83,6 +83,14 @@ test('stale semantic refs fail before input and revocation during preview preven
   await assert.rejects(actuator.click({ x: 1, y: 1 }), /inactive/);
 });
 
+test('desktop observation does not wait for a second Allow after Grant desktop', () => {
+  const id = 'cu-observe-permission';
+  try {
+    custom.register(id, createComputerObserveTools({ computer: { status: () => ({ active: true }) }, observer: {} }));
+    assert.equal(custom.needsPermission(id, 'cu_observe', 'ask'), false);
+  } finally { custom.clear(id); }
+});
+
 test('expired grants prevent desktop observation', async () => {
   let now = 0; const computer = new ComputerUseSession({ clock: () => now }); computer.grant(); now = 180001;
   const [observe] = createComputerObserveTools({ computer, observer: { observe: () => assert.fail('expired observation') } });
@@ -96,6 +104,13 @@ function helper() {
   return child;
 }
 
+test('portal helper reports a notify fallback as ready', async () => {
+  const child = helper(); const input = new PortalInputBackend({ spawnImpl: () => child });
+  const grant = input.grant();
+  child.stdout.emit('data', '{"type":"ready","restore_token_present":true,"backend":"portal-notify"}\n');
+  assert.equal((await grant).backend, 'portal-notify');
+});
+
 test('portal grants wait for a complete readiness message and clear on exit', async () => {
   const child = helper(); const input = new PortalInputBackend({ spawnImpl: () => child });
   const grant = input.grant();

test/runtime-tools.test.js

@@ -82,6 +82,7 @@ test('voice prompt tells the model not to chain extra terminal commands', () =>
   assert.match(VOICE_SYSTEM_PROMPT, /Tool names, tool arguments/);
   assert.match(VOICE_SYSTEM_PROMPT, /File tools may read any path they accept/);
   assert.match(VOICE_SYSTEM_PROMPT, /Allow now/);
+  assert.match(VOICE_SYSTEM_PROMPT, /call cu_observe to see the screen/);
   assert.doesNotMatch(VOICE_SYSTEM_PROMPT, /Never claim cloud access/);
 });