snxraven/gnome-jarvis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Better Search
Rolling release / release (push) Successful in 8m58s
Details

Browse Source
This commit is contained in:
snxraven
2026-09-12 12:33:52 -04:00
parent 0312fedaa7
commit 0e52942b8b
19 changed files with 1333 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
computer-use/py/portal_remote_desktop.py
+125 -46
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""RemoteDesktop consent broker with a built-in libei injector."""
"""RemoteDesktop + ScreenCast broker: EIS input and a live PipeWire frame buffer."""
import json
import os
import select
@@ -17,8 +17,10 @@ except Exception as exc:
raise SystemExit(1)
from libei_sender import LibeiSender
from pw_framebuffer import PipeWireFrameBuffer
BTN = {'left': 0x110, 'right': 0x111, 'middle': 0x112}
MODE = os.environ.get('JARVIS_CU_MODE', 'act')
def log_error(reason):
@@ -26,14 +28,21 @@ def log_error(reason):
bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
proxy = Gio.DBusProxy.new_sync(
bus, Gio.DBusProxyFlags.NONE, None,
'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop',
'org.freedesktop.portal.RemoteDesktop', None,
)
def request(method, signature, values, timeout_ms=120000):
def portal_proxy(iface):
return Gio.DBusProxy.new_sync(
bus, Gio.DBusProxyFlags.NONE, None,
'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop',
iface, None,
)
remote = portal_proxy('org.freedesktop.portal.RemoteDesktop')
screencast = portal_proxy('org.freedesktop.portal.ScreenCast')
def request(proxy, method, signature, values, timeout_ms=120000):
request_path = proxy.call_sync(method, GLib.Variant(signature, values), Gio.DBusCallFlags.NONE, timeout_ms, None).unpack()[0]
loop = GLib.MainLoop()
result = {'code': 1, 'results': {}}
@@ -50,10 +59,10 @@ def request(method, signature, values, timeout_ms=120000):
return result['results']
def connect_to_eis(session):
def unix_fd(proxy, method, session):
incoming = Gio.UnixFDList.new()
variant, outgoing = proxy.call_with_unix_fd_list_sync(
'ConnectToEIS',
method,
GLib.Variant('(oa{sv})', (session, {})),
Gio.DBusCallFlags.NONE,
15000,
@@ -70,11 +79,22 @@ def connect_to_eis(session):
fd = unpacked[0] if isinstance(unpacked, (tuple, list)) else unpacked
if isinstance(fd, int) and fd >= 0:
return fd
raise RuntimeError('ConnectToEIS returned no file descriptor')
raise RuntimeError(f'{method} returned no file descriptor')
def notify(method, signature, values):
proxy.call_sync(method, GLib.Variant(signature, values), Gio.DBusCallFlags.NONE, 5000, None)
remote.call_sync(method, GLib.Variant(signature, values), Gio.DBusCallFlags.NONE, 5000, None)
def stream_node_id(streams):
if not streams:
raise RuntimeError('ScreenCast Start returned no PipeWire streams')
first = streams[0]
if isinstance(first, (list, tuple)):
return int(first[0])
if isinstance(first, dict):
return int(first.get('node_id') or first.get('id'))
return int(first)
class PortalNotify:
@@ -115,6 +135,30 @@ class PortalNotify:
raise RuntimeError(f'unsupported portal notify action {kind} {name}')
class SessionHandler:
def __init__(self, input_handler, framebuffer):
self.input_handler = input_handler
self.framebuffer = framebuffer
def dispatch(self):
if hasattr(self.input_handler, 'dispatch'):
self.input_handler.dispatch()
def handle(self, action):
if action.get('type') == 'frame':
if not self.framebuffer:
raise RuntimeError('PipeWire ScreenCast is not attached')
path = action.get('path')
if not path:
raise RuntimeError('frame path is required')
self.framebuffer.capture(path)
print(json.dumps({'type': 'frame', 'path': path, 'source': 'pipewire'}), flush=True)
return
if self.input_handler is None:
raise RuntimeError('desktop input is observe-only')
self.input_handler.handle(action)
def run_loop(handler, extra_fd=None):
stdin_fd = sys.stdin.fileno()
while True:
@@ -140,58 +184,93 @@ def run_loop(handler, extra_fd=None):
log_error(exc)
def attach_screencast():
token = f'jarvissc{GLib.get_real_time()}'
session = request(screencast, 'CreateSession', '(a{sv})', ({'session_handle_token': GLib.Variant('s', token)},))['session_handle']
request(screencast, 'SelectSources', '(oa{sv})', (session, {
'types': GLib.Variant('u', 1),
'multiple': GLib.Variant('b', False),
'cursor_mode': GLib.Variant('u', 2),
'persist_mode': GLib.Variant('u', 2),
}))
return session
def start_framebuffer(sc_session):
results = request(screencast, 'Start', '(osa{sv})', (sc_session, '', {}))
node_id = stream_node_id(results.get('streams') or [])
fd = unix_fd(screencast, 'OpenPipeWireRemote', sc_session)
return PipeWireFrameBuffer(fd, node_id)
injector = None
sender = None
framebuffer = None
session = None
try:
token = f'jarvis{GLib.get_real_time()}'
session = request('CreateSession', '(a{sv})', ({'session_handle_token': GLib.Variant('s', token)},))['session_handle']
request('SelectDevices', '(oa{sv})', (session, {'types': GLib.Variant('u', 7), 'persist_mode': GLib.Variant('u', 2)}))
request('Start', '(osa{sv})', (session, '', {}))
backend = 'none'
sc_session = attach_screencast()
input_handler = None
extra_fd = None
handler = None
backend = 'none'
eis_error = None
frame_error = None
if MODE != 'observe':
token = f'jarvisrd{GLib.get_real_time()}'
session = request(remote, 'CreateSession', '(a{sv})', ({'session_handle_token': GLib.Variant('s', token)},))['session_handle']
request(remote, 'SelectDevices', '(oa{sv})', (session, {'types': GLib.Variant('u', 7), 'persist_mode': GLib.Variant('u', 2)}))
request(remote, 'Start', '(osa{sv})', (session, '', {}))
try:
eis_fd = unix_fd(remote, 'ConnectToEIS', session)
sender = LibeiSender(eis_fd)
if not sender.wait_ready():
raise RuntimeError('libei connected but no pointer or keyboard device appeared')
input_handler = sender
extra_fd = sender.fd
backend = 'portal-ei'
except Exception as exc:
eis_error = str(exc)
if sender:
try:
sender.close()
except Exception:
pass
sender = None
if input_handler is None and os.environ.get('JARVIS_LIBEI_BRIDGE'):
injector = subprocess.Popen(os.environ['JARVIS_LIBEI_BRIDGE'], shell=True, stdin=subprocess.PIPE, text=True)
class Bridge:
def handle(self, action):
injector.stdin.write(json.dumps(action) + '\n')
injector.stdin.flush()
input_handler = Bridge()
backend = 'portal-ei'
if input_handler is None:
input_handler = PortalNotify(session)
backend = 'portal-notify'
try:
eis_fd = connect_to_eis(session)
sender = LibeiSender(eis_fd)
if not sender.wait_ready():
raise RuntimeError('libei connected but no pointer or keyboard device appeared')
handler = sender
extra_fd = sender.fd
backend = 'portal-ei'
framebuffer = start_framebuffer(sc_session)
except Exception as exc:
eis_error = str(exc)
if sender:
try:
sender.close()
except Exception:
pass
sender = None
if handler is None and os.environ.get('JARVIS_LIBEI_BRIDGE'):
injector = subprocess.Popen(os.environ['JARVIS_LIBEI_BRIDGE'], shell=True, stdin=subprocess.PIPE, text=True)
class Bridge:
def handle(self, action):
injector.stdin.write(json.dumps(action) + '\n')
injector.stdin.flush()
handler = Bridge()
backend = 'portal-ei'
if handler is None:
handler = PortalNotify(session)
backend = 'portal-notify'
frame_error = str(exc)
framebuffer = None
print(json.dumps({
'type': 'ready',
'session': session,
'session': session or sc_session,
'restore_token_present': True,
'backend': backend,
'screen': framebuffer is not None,
'eis_error': eis_error,
'frame_error': frame_error,
}), flush=True)
run_loop(handler, extra_fd)
run_loop(SessionHandler(input_handler, framebuffer), extra_fd)
except Exception as exc:
log_error(exc)
finally:
if framebuffer:
try:
framebuffer.close()
except Exception:
pass
if sender:
try:
sender.close()
computer-use/py/pw_framebuffer.py
+57
View File
@@ -0,0 +1,57 @@
"""Pull one RGB frame from a portal PipeWire ScreenCast stream."""
from PIL import Image
class PipeWireFrameBuffer:
def __init__(self, fd, node_id):
import gi
gi.require_version('Gst', '1.0')
from gi.repository import Gst
Gst.init(None)
self.Gst = Gst
self.pipeline = Gst.parse_launch(
f'pipewiresrc fd={int(fd)} path={int(node_id)} always-copy=true do-timestamp=true '
'client-name=jarvis ! videoconvert ! video/x-raw,format=RGB ! '
'appsink name=sink max-buffers=1 drop=true sync=false enable-last-sample=true'
)
self.sink = self.pipeline.get_by_name('sink')
if not self.sink:
raise RuntimeError('GStreamer appsink missing')
self.pipeline.set_state(Gst.State.PLAYING)
change, state, _pending = self.pipeline.get_state(5 * Gst.SECOND)
if change == Gst.StateChangeReturn.FAILURE or state != Gst.State.PLAYING:
self.close()
raise RuntimeError('PipeWire ScreenCast pipeline failed to play')
def capture(self, path, timeout_ms=4000):
sample = self.sink.emit('try-pull-sample', timeout_ms * self.Gst.MSECOND)
if sample is None:
sample = self.sink.get_property('last-sample')
if sample is None:
raise RuntimeError('no PipeWire frame yet')
buf = sample.get_buffer()
caps = sample.get_caps().get_structure(0)
width = int(caps.get_value('width'))
height = int(caps.get_value('height'))
ok, mapped = buf.map(self.Gst.MapFlags.READ)
if not ok:
raise RuntimeError('could not map PipeWire frame')
try:
data = bytes(mapped.data)
finally:
buf.unmap(mapped)
stride = max(width * 3, len(data) // max(1, height))
if stride != width * 3:
rows = [data[i * stride:i * stride + width * 3] for i in range(height)]
data = b''.join(rows)
Image.frombytes('RGB', (width, height), data[:width * height * 3]).save(path)
return path
def close(self):
if getattr(self, 'pipeline', None):
try:
self.pipeline.set_state(self.Gst.State.NULL)
except Exception:
pass
self.pipeline = None
self.sink = None