Computer Use Updates

computer-use/py/atspi_action.py

@@ -11,7 +11,7 @@ desktop = Atspi.get_desktop(0)
 found = None
 wanted_name = target.get('name') or ''
 wanted_role = target.get('role') or ''
-wanted_rect = target.get('rect') or None
+wanted_rect = target.get('raw_rect') or target.get('rect') or None
 
 def rect_close(node):
     if not wanted_rect or len(wanted_rect) < 4:
@@ -38,7 +38,21 @@ def walk(node, depth=0):
     except Exception:
         return
 
-walk(desktop)
+if target.get('pid') is not None and target.get('atspi_path') is not None:
+    for i in range(desktop.get_child_count()):
+        app = desktop.get_child_at_index(i)
+        if app.get_process_id() != target['pid']:
+            continue
+        candidate = app
+        for index in target['atspi_path']:
+            candidate = candidate.get_child_at_index(index)
+            if candidate is None:
+                break
+        if candidate and (candidate.get_name() or '') == wanted_name and (candidate.get_role_name() or '') == wanted_role and rect_close(candidate):
+            found = candidate
+            break
+else:
+    walk(desktop)
 if not found:
     raise SystemExit('AT-SPI target not found')
 actions = found.get_action()

computer-use/py/atspi_snapshot.py

@@ -9,7 +9,8 @@ mode, limit = sys.argv[1], int(sys.argv[2])
 Atspi.init()
 desktop = Atspi.get_desktop(0)
 nodes = []
-def walk(node, depth=0):
+def walk(node, depth=0, route=None, pid=None, window_rect=None):
+    route = route or []
     if len(nodes) >= limit or node is None or depth > 30:
         return
     try:
@@ -17,10 +18,13 @@ def walk(node, depth=0):
         name = node.get_name() or ''
         component = node.get_component()
         rect = component.get_extents(Atspi.CoordType.SCREEN) if component else None
+        bounds = [rect.x, rect.y, rect.width, rect.height] if rect else None
+        if window_rect is None and depth == 0:
+            window_rect = bounds
         if role and (name or rect):
-            nodes.append({'role': role, 'name': name, 'rect': [rect.x, rect.y, rect.width, rect.height] if rect else None, 'state': [str(s) for s in node.get_state_set().get_states()]})
+            nodes.append({'role': role, 'name': name, 'rect': [rect.x, rect.y, rect.width, rect.height] if rect else None, 'state': [str(s) for s in node.get_state_set().get_states()], 'pid': pid, 'atspi_path': route, 'window_rect': window_rect, 'raw_rect': bounds})
         for i in range(node.get_child_count()):
-            walk(node.get_child_at_index(i), depth + 1)
+            walk(node.get_child_at_index(i), depth + 1, route + [i], pid, window_rect)
     except Exception:
         return
 if mode == 'focused':
@@ -30,9 +34,12 @@ if mode == 'focused':
             for j in range(app.get_child_count()):
                 window = app.get_child_at_index(j)
                 if window.get_state_set().contains(Atspi.StateType.ACTIVE):
-                    walk(window)
+                    walk(window, route=[j], pid=app.get_process_id())
         except Exception:
             continue
 else:
-    walk(desktop)
+    for i in range(desktop.get_child_count()):
+        app = desktop.get_child_at_index(i)
+        for j in range(app.get_child_count()):
+            walk(app.get_child_at_index(j), route=[j], pid=app.get_process_id())
 print(json.dumps(nodes, ensure_ascii=False))

computer-use/py/libei_sender.py

@@ -45,6 +45,8 @@ COMBO_KEYS = {
     'backspace': KEY_BACKSPACE, 'tab': KEY_TAB, 'space': KEY_SPACE,
     'ctrl': KEY_LEFTCTRL, 'control': KEY_LEFTCTRL, 'alt': KEY_LEFTALT,
     'shift': KEY_LEFTSHIFT, 'super': KEY_LEFTMETA, 'meta': KEY_LEFTMETA,
+    'left': 105, 'right': 106, 'up': 103, 'down': 108,
+    'home': 102, 'end': 107, 'pageup': 104, 'pagedown': 109, 'delete': 111, 'insert': 110,
     **F_KEYS,
 }
 LETTER_KEYS = {ch: code for ch, code in zip('qwertyuiopasdfghjklzxcvbnm', [
@@ -52,6 +54,7 @@ LETTER_KEYS = {ch: code for ch, code in zip('qwertyuiopasdfghjklzxcvbnm', [
 ])}
 DIGIT_KEYS = {str(d): code for d, code in enumerate([11, 2, 3, 4, 5, 6, 7, 8, 9, 10])}
 PUNCT_KEYS = {
+    **{ch: (DIGIT_KEYS[d], True) for ch, d in zip('!@#$%^&*()', '1234567890')},
     '-': (12, False), '_': (12, True), '=': (13, False), '+': (13, True),
     '[': (26, False), '{': (26, True), ']': (27, False), '}': (27, True),
     ';': (39, False), ':': (39, True), "'": (40, False), '"': (40, True),
@@ -84,6 +87,7 @@ def _lib():
     lib.ei_event_get_device.argtypes = [ctypes.c_void_p]
     lib.ei_event_get_device.restype = ctypes.c_void_p
     lib.ei_event_unref.argtypes = [ctypes.c_void_p]
+    lib.ei_seat_bind_capabilities.argtypes = [ctypes.c_void_p]
     lib.ei_seat_bind_capabilities.restype = None
     lib.ei_seat_has_capability.argtypes = [ctypes.c_void_p, ctypes.c_int]
     lib.ei_seat_has_capability.restype = ctypes.c_int
@@ -164,12 +168,12 @@ class LibeiSender:
             readable, _, _ = select.select([self.fd], [], [], remaining)
             if readable:
                 self.dispatch()
-            if self._emulating and (self.pointer or self.keyboard):
+            if self.pointer_abs and self.keyboard and all(_ptr(d) in self._emulating for d in (self.pointer_abs, self.keyboard)):
                 if collected is None:
                     collected = time.time()
                 elif time.time() - collected >= 0.35 or self.pointer_abs:
                     return True
-        return bool(self._emulating and (self.pointer or self.keyboard))
+        return bool(self.pointer_abs and self.keyboard and all(_ptr(d) in self._emulating for d in (self.pointer_abs, self.keyboard)))
 
     def dispatch(self):
         self.lib.ei_dispatch(self.ei)
@@ -212,9 +216,9 @@ class LibeiSender:
         return BTN_LEFT
 
     def move(self, x, y):
-        device = self.pointer
-        if not device:
-            raise RuntimeError('no EIS pointer device')
+        device = self.pointer_abs
+        if not device or _ptr(device) not in self._emulating:
+            raise RuntimeError('no active absolute EIS pointer device')
         if self._has(device, CAP_POINTER_ABSOLUTE):
             self.lib.ei_device_pointer_motion_absolute(device, float(x), float(y))
         else:
@@ -233,19 +237,23 @@ class LibeiSender:
 
     def scroll(self, x, y, dx=0, dy=0):
         self.move(x, y)
-        self.lib.ei_device_scroll_discrete(self.pointer, int(dx), int(dy))
+        self.lib.ei_device_scroll_discrete(self.pointer, int(dx) * 120, int(dy) * 120)
         self._frame(self.pointer)
 
     def drag(self, start, end):
         self.move(start[0], start[1])
         self.lib.ei_device_button_button(self.pointer, BTN_LEFT, True)
         self._frame(self.pointer)
-        self.move(end[0], end[1])
-        self.lib.ei_device_button_button(self.pointer, BTN_LEFT, False)
-        self._frame(self.pointer)
+        try:
+            for step in range(1, 13):
+                self.move(start[0] + (end[0] - start[0]) * step / 12, start[1] + (end[1] - start[1]) * step / 12)
+                time.sleep(0.012)
+        finally:
+            self.lib.ei_device_button_button(self.pointer, BTN_LEFT, False)
+            self._frame(self.pointer)
 
     def _tap(self, code, mods=()):
-        if not self.keyboard:
+        if not self.keyboard or _ptr(self.keyboard) not in self._emulating:
             raise RuntimeError('no EIS keyboard device')
         for mod in mods:
             self.lib.ei_device_keyboard_key(self.keyboard, mod, True)
@@ -278,18 +286,30 @@ class LibeiSender:
             if punct:
                 code, shifted = punct
                 self._tap(code, (KEY_LEFTSHIFT,) if shifted else ())
+                continue
+            # GNOME/GTK Unicode input method; never silently drop characters.
+            self._tap(LETTER_KEYS['u'], (KEY_LEFTCTRL, KEY_LEFTSHIFT))
+            time.sleep(0.08)
+            for digit in format(ord(ch), 'x'):
+                self._tap(LETTER_KEYS.get(digit) or DIGIT_KEYS[digit])
+                time.sleep(0.01)
+            self._tap(KEY_ENTER)
+            time.sleep(0.1)
         if submit:
             self._tap(KEY_ENTER)
 
     def key_combo(self, combo):
         parts = [p.strip().lower() for p in str(combo).replace('-', '+').split('+') if p.strip()]
         if not parts:
-            return
-        mods = [COMBO_KEYS[part] for part in parts[:-1] if part in COMBO_KEYS]
+            raise ValueError('keyboard combination is required')
+        modifiers = {'ctrl', 'control', 'alt', 'shift', 'super', 'meta'}
+        if any(part not in modifiers for part in parts[:-1]):
+            raise ValueError('unsupported keyboard modifier')
+        mods = list(dict.fromkeys(COMBO_KEYS[part] for part in parts[:-1]))
         key = parts[-1]
         code = COMBO_KEYS.get(key) or LETTER_KEYS.get(key) or DIGIT_KEYS.get(key)
         if code is None:
-            return
+            raise ValueError(f'unsupported key: {key}')
         self._tap(code, tuple(mods))
 
     def handle(self, action):

computer-use/py/normalize_frame.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 import sys
+import json
 from PIL import Image
 
 source, target, cap, quality = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4])
 with Image.open(source) as image:
     image = image.convert('RGB')
+    source_width, source_height = image.size
     if len(sys.argv) == 9:
         x, y, width, height = [int(v) for v in sys.argv[5:9]]
         image = image.crop((x, y, x + width, y + height))
@@ -12,3 +14,5 @@ with Image.open(source) as image:
     if scale < 1.0:
         image = image.resize((round(image.width * scale), round(image.height * scale)), Image.Resampling.LANCZOS)
     image.save(target, 'WEBP', quality=quality, method=4)
+
+    print(json.dumps({'source_width': source_width, 'source_height': source_height, 'width': image.width, 'height': image.height, 'scale': scale}))

computer-use/py/portal_remote_desktop.py

@@ -3,8 +3,9 @@
 import json
 import os
 import select
-import subprocess
 import sys
+import time
+import signal
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
@@ -21,13 +22,16 @@ from pw_framebuffer import PipeWireFrameBuffer
 
 BTN = {'left': 0x110, 'right': 0x111, 'middle': 0x112}
 MODE = os.environ.get('JARVIS_CU_MODE', 'act')
+# persist_mode 2 made GNOME restore the previous share and skip the screen picker.
+PERSIST = os.environ.get('JARVIS_CU_PERSIST', '0') == '1'
+PERSIST_MODE = 2 if PERSIST else 0
 
 
 def log_error(reason):
     print(json.dumps({'type': 'error', 'reason': str(reason)}), flush=True)
 
 
-bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
+bus = None
 
 
 def portal_proxy(iface):
@@ -38,22 +42,35 @@ def portal_proxy(iface):
     )
 
 
-remote = portal_proxy('org.freedesktop.portal.RemoteDesktop')
-screencast = portal_proxy('org.freedesktop.portal.ScreenCast')
+remote = screencast = None
 
 
 def request(proxy, method, signature, values, timeout_ms=120000):
-    request_path = proxy.call_sync(method, GLib.Variant(signature, values), Gio.DBusCallFlags.NONE, timeout_ms, None).unpack()[0]
+    # Subscribe before sending the request: fast responses can precede the reply.
+    token = f'jarvis{GLib.get_real_time()}'
+    sender_name = bus.get_unique_name()[1:].replace('.', '_')
+    request_path = f'/org/freedesktop/portal/desktop/request/{sender_name}/{token}'
+    values = list(values)
+    values[-1] = dict(values[-1], handle_token=GLib.Variant('s', token))
     loop = GLib.MainLoop()
-    result = {'code': 1, 'results': {}}
-
+    result = {'code': None, 'results': {}}
     def response(_conn, _sender, _path, _interface, _member, params):
         result['code'], result['results'] = params.unpack()
         loop.quit()
-
+    def expired():
+        result['code'] = 'timeout'
+        loop.quit()
+        return False
     sub = bus.signal_subscribe(None, 'org.freedesktop.portal.Request', 'Response', request_path, None, Gio.DBusSignalFlags.NONE, response)
-    loop.run()
-    bus.signal_unsubscribe(sub)
+    timer = GLib.timeout_add(timeout_ms, expired)
+    try:
+        proxy.call_sync(method, GLib.Variant(signature, tuple(values)), Gio.DBusCallFlags.NONE, timeout_ms, None)
+        if result['code'] is None:
+            loop.run()
+    finally:
+        if result['code'] != 'timeout':
+            GLib.source_remove(timer)
+        bus.signal_unsubscribe(sub)
     if result['code'] != 0:
         raise RuntimeError(f'{method} portal response {result["code"]}')
     return result['results']
@@ -98,8 +115,10 @@ def stream_node_id(streams):
 
 
 class PortalNotify:
-    def __init__(self, session):
+    def __init__(self, session, node_id, origin=(0, 0)):
         self.session = session
+        self.node_id = node_id
+        self.origin = origin
 
     def handle(self, action):
         kind = action.get('type')
@@ -109,7 +128,7 @@ class PortalNotify:
         if kind == 'pointer' and name in ('click', 'double_click', 'move'):
             x = float(action.get('x') or 0)
             y = float(action.get('y') or 0)
-            notify('NotifyPointerMotionAbsolute', '(oa{sv}udd)', (session, opts, 0, x, y))
+            notify('NotifyPointerMotionAbsolute', '(oa{sv}udd)', (session, opts, self.node_id, x - self.origin[0], y - self.origin[1]))
             if name != 'move':
                 button = BTN.get(str(action.get('button') or 'left'), BTN['left'])
                 repeats = 2 if name == 'double_click' else 1
@@ -117,27 +136,56 @@ class PortalNotify:
                     notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, button, 1))
                     notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, button, 0))
         elif kind == 'pointer' and name == 'scroll':
-            notify('NotifyPointerAxisDiscrete', '(oa{sv}ui)', (session, opts, 0, int(action.get('dy') or 0)))
+            self.handle(dict(action, action='move'))
+            for axis, key in ((0, 'dy'), (1, 'dx')):
+                amount = int(action.get(key) or 0)
+                if amount:
+                    notify('NotifyPointerAxisDiscrete', '(oa{sv}ui)', (session, opts, axis, amount))
         elif kind == 'pointer' and name == 'drag':
             start = action.get('from') or [0, 0]
             end = action.get('to') or [0, 0]
-            notify('NotifyPointerMotionAbsolute', '(oa{sv}udd)', (session, opts, 0, float(start[0]), float(start[1])))
+            self.handle({'type': 'pointer', 'action': 'move', 'x': start[0], 'y': start[1]})
             notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, BTN['left'], 1))
-            notify('NotifyPointerMotionAbsolute', '(oa{sv}udd)', (session, opts, 0, float(end[0]), float(end[1])))
-            notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, BTN['left'], 0))
+            try:
+                for step in range(1, 13):
+                    self.handle({'type': 'pointer', 'action': 'move', 'x': start[0] + (end[0] - start[0]) * step / 12, 'y': start[1] + (end[1] - start[1]) * step / 12})
+                    time.sleep(0.012)
+            finally:
+                notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, BTN['left'], 0))
         elif kind == 'keyboard' and name == 'type':
             for ch in str(action.get('text') or ''):
-                keysym = 0xff0d if ch == '\n' else (0x020 if ch == ' ' else ord(ch))
+                if ord(ch) > 127:
+                    self.handle({'type': 'keyboard', 'action': 'key', 'combo': 'ctrl+shift+u'})
+                    time.sleep(0.08)
+                    for digit in format(ord(ch), 'x'):
+                        self.handle({'type': 'keyboard', 'action': 'type', 'text': digit})
+                        time.sleep(0.01)
+                    self.handle({'type': 'keyboard', 'action': 'key', 'combo': 'enter'})
+                    time.sleep(0.1)
+                    continue
+                keysym = {'\n': 0xff0d, '\t': 0xff09}.get(ch, ord(ch) if ord(ch) <= 255 else 0x01000000 | ord(ch))
                 notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, keysym, 1))
                 notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, keysym, 0))
             if action.get('submit'):
                 notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, 0xff0d, 1))
                 notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, 0xff0d, 0))
         elif kind == 'keyboard' and name == 'key':
-            combo = str(action.get('combo') or '').lower()
-            keysym = 0xff1b if 'esc' in combo else 0xff0d
-            notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, keysym, 1))
-            notify('NotifyKeyboardKeysym', '(oa{sv}iu)', (session, opts, keysym, 0))
+            from libei_sender import COMBO_KEYS, LETTER_KEYS, DIGIT_KEYS
+            parts = str(action.get('combo') or '').lower().replace('-', '+').split('+')
+            modifiers = {'ctrl', 'control', 'alt', 'shift', 'super', 'meta'}
+            if not parts or any(p not in modifiers for p in parts[:-1]):
+                raise ValueError('unsupported keyboard combination')
+            codes = [COMBO_KEYS.get(p) or LETTER_KEYS.get(p) or DIGIT_KEYS.get(p) for p in parts]
+            if any(code is None for code in codes):
+                raise ValueError('unsupported keyboard combination')
+            held = []
+            try:
+                for code in codes:
+                    notify('NotifyKeyboardKeycode', '(oa{sv}iu)', (session, opts, code, 1))
+                    held.append(code)
+            finally:
+                for code in reversed(held):
+                    notify('NotifyKeyboardKeycode', '(oa{sv}iu)', (session, opts, code, 0))
         else:
             raise RuntimeError(f'unsupported portal notify action {kind} {name}')
 
@@ -159,7 +207,7 @@ class SessionHandler:
             if not path:
                 raise RuntimeError('frame path is required')
             self.framebuffer.capture(path)
-            print(json.dumps({'type': 'frame', 'path': path, 'source': 'pipewire'}), flush=True)
+            print(json.dumps({'type': 'frame', 'path': path, 'source': 'pipewire', 'id': action.get('id')}), flush=True)
             return
         if self.input_handler is None:
             raise RuntimeError('desktop input is observe-only')
@@ -168,27 +216,35 @@ class SessionHandler:
 
 def run_loop(handler, extra_fd=None):
     stdin_fd = sys.stdin.fileno()
+    buffer = b''
     while True:
-        fds = [stdin_fd]
-        if extra_fd is not None:
-            fds.append(extra_fd)
-        readable, _, _ = select.select(fds, [], [], 0.25)
-        if extra_fd is not None and extra_fd in readable and hasattr(handler, 'dispatch'):
+        context = GLib.MainContext.default()
+        while context.pending():
+            context.iteration(False)
+        fds = [stdin_fd] + ([extra_fd] if extra_fd is not None else [])
+        readable, _, _ = select.select(fds, [], [], 0.1)
+        if extra_fd is not None and extra_fd in readable:
             handler.dispatch()
-        if stdin_fd in readable:
-            line = sys.stdin.readline()
-            if not line:
-                break
+        if stdin_fd not in readable:
+            continue
+        chunk = os.read(stdin_fd, 65536)
+        if not chunk:
+            break
+        buffer += chunk
+        if len(buffer) > 1024 * 1024:
+            raise RuntimeError('input message exceeds limit')
+        while b'\n' in buffer:
+            line, buffer = buffer.split(b'\n', 1)
+            action = {}
             try:
                 action = json.loads(line)
-            except Exception:
-                continue
-            if action.get('type') == 'close':
-                break
-            try:
+                if action.get('type') == 'close':
+                    return
                 handler.handle(action)
+                if action.get('type') != 'frame':
+                    print(json.dumps({'type': 'ack', 'id': action.get('id'), 'ok': True}), flush=True)
             except Exception as exc:
-                log_error(exc)
+                print(json.dumps({'type': 'error', 'id': action.get('id'), 'reason': str(exc)}), flush=True)
 
 
 def attach_screencast():
@@ -198,90 +254,74 @@ def attach_screencast():
         'types': GLib.Variant('u', 1),
         'multiple': GLib.Variant('b', False),
         'cursor_mode': GLib.Variant('u', 2),
-        'persist_mode': GLib.Variant('u', 2),
+        'persist_mode': GLib.Variant('u', PERSIST_MODE),
     }))
     return session
 
 
-def start_framebuffer(sc_session):
-    results = request(screencast, 'Start', '(osa{sv})', (sc_session, '', {}))
-    node_id = stream_node_id(results.get('streams') or [])
-    fd = unix_fd(screencast, 'OpenPipeWireRemote', sc_session)
-    return PipeWireFrameBuffer(fd, node_id)
-
-
-injector = None
-sender = None
-framebuffer = None
-session = None
-try:
-    sc_session = attach_screencast()
-    input_handler = None
-    extra_fd = None
-    backend = 'none'
-    eis_error = None
-    frame_error = None
-    if MODE != 'observe':
-        token = f'jarvisrd{GLib.get_real_time()}'
-        session = request(remote, 'CreateSession', '(a{sv})', ({'session_handle_token': GLib.Variant('s', token)},))['session_handle']
-        request(remote, 'SelectDevices', '(oa{sv})', (session, {'types': GLib.Variant('u', 7), 'persist_mode': GLib.Variant('u', 2)}))
-        request(remote, 'Start', '(osa{sv})', (session, '', {}))
-        try:
-            eis_fd = unix_fd(remote, 'ConnectToEIS', session)
-            sender = LibeiSender(eis_fd)
-            if not sender.wait_ready():
-                raise RuntimeError('libei connected but no pointer or keyboard device appeared')
-            input_handler = sender
-            extra_fd = sender.fd
-            backend = 'portal-ei'
-        except Exception as exc:
-            eis_error = str(exc)
-            if sender:
-                try:
-                    sender.close()
-                except Exception:
-                    pass
-            sender = None
-        if input_handler is None and os.environ.get('JARVIS_LIBEI_BRIDGE'):
-            injector = subprocess.Popen(os.environ['JARVIS_LIBEI_BRIDGE'], shell=True, stdin=subprocess.PIPE, text=True)
-
-            class Bridge:
-                def handle(self, action):
-                    injector.stdin.write(json.dumps(action) + '\n')
-                    injector.stdin.flush()
-
-            input_handler = Bridge()
-            backend = 'portal-ei'
-        if input_handler is None:
-            input_handler = PortalNotify(session)
-            backend = 'portal-notify'
+def main():
+    global bus, remote, screencast
+    bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
+    remote = portal_proxy('org.freedesktop.portal.RemoteDesktop')
+    screencast = portal_proxy('org.freedesktop.portal.ScreenCast')
+    sender = framebuffer = None
+    session = None
+    signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
     try:
-        framebuffer = start_framebuffer(sc_session)
+        if MODE == 'observe':
+            session = attach_screencast()
+            results = request(screencast, 'Start', '(osa{sv})', (session, '', {}))
+        else:
+            token = f'jarvisrd{GLib.get_real_time()}'
+            session = request(remote, 'CreateSession', '(a{sv})', ({'session_handle_token': GLib.Variant('s', token)},))['session_handle']
+            request(remote, 'SelectDevices', '(oa{sv})', (session, {'types': GLib.Variant('u', 3), 'persist_mode': GLib.Variant('u', PERSIST_MODE)}))
+            request(screencast, 'SelectSources', '(oa{sv})', (session, {
+                'types': GLib.Variant('u', 1), 'multiple': GLib.Variant('b', False), 'cursor_mode': GLib.Variant('u', 2),
+            }))
+            results = request(remote, 'Start', '(osa{sv})', (session, '', {}))
+        streams = results.get('streams') or []
+        node_id = stream_node_id(streams)
+        props = streams[0][1]
+        origin = props.get('position', (0, 0))
+        framebuffer = PipeWireFrameBuffer(unix_fd(screencast, 'OpenPipeWireRemote', session), node_id)
+        backend, extra_fd, input_handler, eis_error = 'none', None, None, None
+        if MODE != 'observe':
+            # Notify remains a tested fallback, selectable for diagnostics.
+            if os.environ.get('JARVIS_CU_INPUT_BACKEND') != 'notify':
+                try:
+                    sender = LibeiSender(unix_fd(remote, 'ConnectToEIS', session))
+                    if not sender.wait_ready():
+                        raise RuntimeError('EIS did not provide active absolute pointer and keyboard devices')
+                    input_handler, extra_fd, backend = sender, sender.fd, 'portal-ei'
+                except Exception as exc:
+                    eis_error = str(exc)
+                    if sender:
+                        sender.close()
+                        sender = None
+            if input_handler is None:
+                input_handler, backend = PortalNotify(session, node_id, origin), 'portal-notify'
+        def closed(*_):
+            raise SystemExit(0)
+        bus.signal_subscribe(None, 'org.freedesktop.portal.Session', 'Closed', session, None, Gio.DBusSignalFlags.NONE, closed)
+        print(json.dumps({
+            'type': 'ready', 'session': session, 'restore_token_present': bool(results.get('restore_token')),
+            'backend': backend, 'screen': True, 'eis_error': eis_error,
+            'streams': [{'node_id': int(n), **p} for n, p in streams],
+        }), flush=True)
+        run_loop(SessionHandler(input_handler, framebuffer), extra_fd)
     except Exception as exc:
-        frame_error = str(exc)
-        framebuffer = None
-    print(json.dumps({
-        'type': 'ready',
-        'session': session or sc_session,
-        'restore_token_present': True,
-        'backend': backend,
-        'screen': framebuffer is not None,
-        'eis_error': eis_error,
-        'frame_error': frame_error,
-    }), flush=True)
-    run_loop(SessionHandler(input_handler, framebuffer), extra_fd)
-except Exception as exc:
-    log_error(exc)
-finally:
-    if framebuffer:
-        try:
+        log_error(exc)
+    finally:
+        if framebuffer:
             framebuffer.close()
-        except Exception:
-            pass
-    if sender:
-        try:
+        if sender:
             sender.close()
-        except Exception:
-            pass
-    if injector:
-        injector.terminate()
+        if session:
+            try:
+                bus.call_sync('org.freedesktop.portal.Desktop', session, 'org.freedesktop.portal.Session', 'Close', None, None, Gio.DBusCallFlags.NONE, 2000, None)
+            except Exception:
+                pass
+
+
+if __name__ == '__main__':
+    main()