Updates

computer-use/py/atspi_action.py

@@ -1,31 +1,60 @@
 #!/usr/bin/env python3
 import json
 import sys
+import gi
+gi.require_version('Atspi', '2.0')
 from gi.repository import Atspi
 
 target, action = json.loads(sys.argv[1]), sys.argv[2]
 Atspi.init()
 desktop = Atspi.get_desktop(0)
 found = None
+wanted_name = target.get('name') or ''
+wanted_role = target.get('role') or ''
+wanted_rect = target.get('rect') or None
+
+def rect_close(node):
+    if not wanted_rect or len(wanted_rect) < 4:
+        return True
+    try:
+        component = node.get_component()
+        if not component:
+            return True
+        extents = component.get_extents(Atspi.CoordType.SCREEN)
+        return abs(extents.x - wanted_rect[0]) < 8 and abs(extents.y - wanted_rect[1]) < 8
+    except Exception:
+        return True
+
 def walk(node, depth=0):
     global found
     if found or node is None or depth > 30:
         return
     try:
-        if (node.get_name() or '') == target.get('name') and (node.get_role_name() or '') == target.get('role'):
+        if (node.get_name() or '') == wanted_name and (node.get_role_name() or '') == wanted_role and rect_close(node):
             found = node
             return
         for i in range(node.get_child_count()):
             walk(node.get_child_at_index(i), depth + 1)
     except Exception:
         return
+
 walk(desktop)
 if not found:
     raise SystemExit('AT-SPI target not found')
 actions = found.get_action()
-for i in range(actions.get_n_actions()):
-    if actions.get_action_name(i).lower() == action.lower():
-        if actions.do_action(i):
-            print(json.dumps({'ok': True, 'action': action}))
-            raise SystemExit(0)
+if actions is None:
+    raise SystemExit(f'AT-SPI action unavailable: {action}')
+aliases = [action.lower()]
+if action.lower() in ('click', 'press', 'activate'):
+    aliases.extend(['press', 'click', 'activate'])
+seen = set()
+for name in aliases:
+    if name in seen:
+        continue
+    seen.add(name)
+    for i in range(actions.get_n_actions()):
+        if actions.get_action_name(i).lower() == name:
+            if actions.do_action(i):
+                print(json.dumps({'ok': True, 'action': name}))
+                raise SystemExit(0)
 raise SystemExit(f'AT-SPI action unavailable: {action}')

computer-use/py/libei_sender.py

@@ -51,6 +51,14 @@ LETTER_KEYS = {ch: code for ch, code in zip('qwertyuiopasdfghjklzxcvbnm', [
     16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 30, 31, 32, 33, 34, 35, 36, 37, 38, 44, 45, 46, 47, 48, 49,
 ])}
 DIGIT_KEYS = {str(d): code for d, code in enumerate([11, 2, 3, 4, 5, 6, 7, 8, 9, 10])}
+PUNCT_KEYS = {
+    '-': (12, False), '_': (12, True), '=': (13, False), '+': (13, True),
+    '[': (26, False), '{': (26, True), ']': (27, False), '}': (27, True),
+    ';': (39, False), ':': (39, True), "'": (40, False), '"': (40, True),
+    '`': (41, False), '~': (41, True), '\\': (43, False), '|': (43, True),
+    ',': (51, False), '<': (51, True), '.': (52, False), '>': (52, True),
+    '/': (53, False), '?': (53, True),
+}
 
 
 def _ptr(value):
@@ -263,9 +271,13 @@ class LibeiSender:
                 continue
             lower = ch.lower()
             code = LETTER_KEYS.get(lower) or DIGIT_KEYS.get(ch)
-            if code is None:
+            if code is not None:
+                self._tap(code, (KEY_LEFTSHIFT,) if ch.isupper() else ())
                 continue
-            self._tap(code, (KEY_LEFTSHIFT,) if ch.isupper() else ())
+            punct = PUNCT_KEYS.get(ch)
+            if punct:
+                code, shifted = punct
+                self._tap(code, (KEY_LEFTSHIFT,) if shifted else ())
         if submit:
             self._tap(KEY_ENTER)
 

computer-use/py/portal_remote_desktop.py

@@ -118,6 +118,13 @@ class PortalNotify:
                     notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, button, 0))
         elif kind == 'pointer' and name == 'scroll':
             notify('NotifyPointerAxisDiscrete', '(oa{sv}ui)', (session, opts, 0, int(action.get('dy') or 0)))
+        elif kind == 'pointer' and name == 'drag':
+            start = action.get('from') or [0, 0]
+            end = action.get('to') or [0, 0]
+            notify('NotifyPointerMotionAbsolute', '(oa{sv}udd)', (session, opts, 0, float(start[0]), float(start[1])))
+            notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, BTN['left'], 1))
+            notify('NotifyPointerMotionAbsolute', '(oa{sv}udd)', (session, opts, 0, float(end[0]), float(end[1])))
+            notify('NotifyPointerButton', '(oa{sv}iu)', (session, opts, BTN['left'], 0))
         elif kind == 'keyboard' and name == 'type':
             for ch in str(action.get('text') or ''):
                 keysym = 0xff0d if ch == '\n' else (0x020 if ch == ' ' else ord(ch))