Updates

computer-use/camera-session.js

@@ -0,0 +1,49 @@
+export class CameraSession {
+  constructor({ enabled = false, grantMinutes = 3, device = '', clock = () => Date.now(), audit } = {}) {
+    this.clock = clock;
+    this.enabled = Boolean(enabled);
+    this.grantMinutes = Math.min(15, Math.max(1, Number(grantMinutes) || 3));
+    this.device = String(device || '');
+    this.active = false;
+    this.expiresAt = null;
+    this.backend = 'none';
+    this.audit = audit;
+  }
+
+  grant() {
+    this.active = true;
+    this.expiresAt = this.clock() + this.grantMinutes * 60 * 1000;
+    return { active: true, grant_expires_at: this.expiresAt, device: this.device };
+  }
+
+  setBackend(backend) {
+    this.backend = ['portal', 'v4l2', 'grant', 'none'].includes(backend) ? backend : 'none';
+    return this.backend;
+  }
+
+  revoke() {
+    this.active = false;
+    this.expiresAt = null;
+    this.backend = 'none';
+    Promise.resolve(this.audit?.wipeTemp?.('/tmp/jarvis-webcam')).catch(() => {});
+  }
+
+  status() {
+    if (this.active && this.expiresAt != null && this.clock() >= this.expiresAt) this.revoke();
+    return {
+      enabled: this.enabled,
+      active: this.active,
+      device: this.device,
+      grant_expires_at: this.expiresAt,
+      backend: this.backend,
+    };
+  }
+
+  assertActive() {
+    if (!this.active) throw new Error('webcam grant is inactive');
+    if (this.expiresAt && this.clock() >= this.expiresAt) {
+      this.revoke();
+      throw new Error('webcam grant expired');
+    }
+  }
+}

computer-use/frame.js

@@ -26,6 +26,8 @@ export class FrameNormalizer {
       });
     });
     const info = await stat(output);
-    return { ...metadata, path: output, bytes: info.size, maxLongEdge: this.maxLongEdge, mime: 'image/webp' };
+    const ext = path.extname(output).toLowerCase();
+    const mime = metadata.mime || (ext === '.png' ? 'image/png' : ext === '.jpg' || ext === '.jpeg' ? 'image/jpeg' : 'image/webp');
+    return { ...metadata, path: output, bytes: info.size, maxLongEdge: this.maxLongEdge, mime };
   }
 }

computer-use/portal-camera.js

@@ -0,0 +1,61 @@
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+
+export class PortalCamera {
+  constructor({
+    helper = path.resolve(new URL('./py/portal_camera.py', import.meta.url).pathname),
+    python = 'python3',
+    spawnImpl = spawn,
+    tmpDir = '/tmp/jarvis-webcam',
+    timeoutMs = 8000,
+    accessTimeoutMs = 120_000,
+  } = {}) {
+    this.helper = helper;
+    this.python = python;
+    this.spawnImpl = spawnImpl;
+    this.tmpDir = tmpDir;
+    this.timeoutMs = timeoutMs;
+    this.accessTimeoutMs = accessTimeoutMs;
+  }
+
+  access({ device = '' } = {}) {
+    return this._run(['access'], this.accessTimeoutMs, device);
+  }
+
+  capture(output, { device = '' } = {}) {
+    return this._run(['capture', output], this.timeoutMs, device);
+  }
+
+  _run(args, timeoutMs, device) {
+    return new Promise((resolve, reject) => {
+      const child = this.spawnImpl(this.python, [this.helper, ...args], {
+        stdio: ['ignore', 'pipe', 'pipe'],
+        env: { ...process.env, JARVIS_WEBCAM_DEVICE: device || '' },
+      });
+      let out = '';
+      let err = '';
+      let settled = false;
+      const timer = setTimeout(() => {
+        if (settled) return;
+        settled = true;
+        child.kill('SIGTERM');
+        reject(new Error('webcam helper timed out'));
+      }, timeoutMs);
+      const done = (fn) => (value) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        fn(value);
+      };
+      child.stdout.on('data', (chunk) => { out += chunk; });
+      child.stderr.on('data', (chunk) => { err += chunk; });
+      child.on('error', done(reject));
+      child.on('close', (code) => {
+        let payload = {};
+        try { payload = JSON.parse(String(out).trim().split('\n').pop() || '{}'); } catch {}
+        if (code === 0 && payload.ok) done(resolve)(payload);
+        else done(reject)(new Error(payload.error || err.trim() || `webcam helper exited ${code}`));
+      });
+    });
+  }
+}

computer-use/py/normalize_frame.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
-import sys
 import json
+import os
+import sys
 from PIL import Image
 
 source, target, cap, quality = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4])
+ext = os.path.splitext(target)[1].lower()
 with Image.open(source) as image:
     image = image.convert('RGB')
     source_width, source_height = image.size
@@ -13,6 +15,21 @@ with Image.open(source) as image:
     scale = min(1.0, cap / max(image.width, image.height))
     if scale < 1.0:
         image = image.resize((round(image.width * scale), round(image.height * scale)), Image.Resampling.LANCZOS)
-    image.save(target, 'WEBP', quality=quality, method=4)
+    if ext in ('.jpg', '.jpeg'):
+        image.save(target, 'JPEG', quality=quality, optimize=True)
+        mime = 'image/jpeg'
+    elif ext == '.png':
+        image.save(target, 'PNG', optimize=True)
+        mime = 'image/png'
+    else:
+        image.save(target, 'WEBP', quality=quality, method=4)
+        mime = 'image/webp'
 
-    print(json.dumps({'source_width': source_width, 'source_height': source_height, 'width': image.width, 'height': image.height, 'scale': scale}))
+    print(json.dumps({
+        'source_width': source_width,
+        'source_height': source_height,
+        'width': image.width,
+        'height': image.height,
+        'scale': scale,
+        'mime': mime,
+    }))

computer-use/py/portal_camera.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+"""Capture one webcam frame through the Camera portal, then v4l2."""
+import json
+import os
+import sys
+
+try:
+    import gi
+    gi.require_version('Gio', '2.0')
+    from gi.repository import Gio, GLib
+except Exception as exc:
+    print(json.dumps({'ok': False, 'error': f'PyGObject unavailable: {exc}'}), flush=True)
+    raise SystemExit(1)
+
+ACTION = sys.argv[1] if len(sys.argv) > 1 else 'capture'
+TARGET = sys.argv[2] if len(sys.argv) > 2 else ''
+DEVICE = os.environ.get('JARVIS_WEBCAM_DEVICE', '').strip()
+bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
+
+
+def fail(reason):
+    print(json.dumps({'ok': False, 'error': str(reason)}), flush=True)
+    raise SystemExit(1)
+
+
+def portal_proxy():
+    return Gio.DBusProxy.new_sync(
+        bus, Gio.DBusProxyFlags.NONE, None,
+        'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop',
+        'org.freedesktop.portal.Camera', None,
+    )
+
+
+def request_access(proxy, timeout_ms=120000):
+    token = f'jarviscam{GLib.get_real_time()}'
+    sender_name = bus.get_unique_name()[1:].replace('.', '_')
+    request_path = f'/org/freedesktop/portal/desktop/request/{sender_name}/{token}'
+    options = {
+        'handle_token': GLib.Variant('s', token),
+        'parent_window': GLib.Variant('s', ''),
+    }
+    loop = GLib.MainLoop()
+    result = {'code': None}
+    def response(_conn, _sender, _path, _interface, _member, params):
+        result['code'] = params.unpack()[0]
+        loop.quit()
+    def expired():
+        result['code'] = 'timeout'
+        loop.quit()
+        return False
+    sub = bus.signal_subscribe(None, 'org.freedesktop.portal.Request', 'Response', request_path, None, Gio.DBusSignalFlags.NONE, response)
+    timer = GLib.timeout_add(timeout_ms, expired)
+    try:
+        proxy.call_sync('Access', GLib.Variant('(a{sv})', (options,)), Gio.DBusCallFlags.NONE, timeout_ms, None)
+        if result['code'] is None:
+            loop.run()
+    finally:
+        if result['code'] != 'timeout':
+            GLib.source_remove(timer)
+        bus.signal_unsubscribe(sub)
+    if result['code'] not in (0, None):
+        raise RuntimeError(f'Camera portal Access response {result["code"]}')
+
+
+def open_pipewire_fd(proxy):
+    incoming = Gio.UnixFDList.new()
+    variant, outgoing = proxy.call_with_unix_fd_list_sync(
+        'OpenPipeWireRemote',
+        GLib.Variant('(a{sv})', ({},)),
+        Gio.DBusCallFlags.NONE,
+        15000,
+        incoming,
+        None,
+    )
+    if outgoing is not None and outgoing.get_length() > 0:
+        unpacked = variant.unpack() if variant is not None else 0
+        index = unpacked[0] if isinstance(unpacked, (tuple, list)) else unpacked
+        if isinstance(index, int) and 0 <= index < outgoing.get_length():
+            return outgoing.get(index)
+        return outgoing.get(0)
+    unpacked = variant.unpack() if variant is not None else None
+    fd = unpacked[0] if isinstance(unpacked, (tuple, list)) else unpacked
+    if isinstance(fd, int) and fd >= 0:
+        return fd
+    raise RuntimeError('Camera portal returned no PipeWire file descriptor')
+
+
+def grab_rgb(pipeline_desc, path, timeout_ms=4000):
+    gi.require_version('Gst', '1.0')
+    from gi.repository import Gst
+    from PIL import Image
+    Gst.init(None)
+    pipeline = Gst.parse_launch(pipeline_desc)
+    sink = pipeline.get_by_name('sink')
+    if not sink:
+        raise RuntimeError('GStreamer appsink missing')
+    pipeline.set_state(Gst.State.PLAYING)
+    change, state, _pending = pipeline.get_state(5 * Gst.SECOND)
+    try:
+        if change == Gst.StateChangeReturn.FAILURE or state != Gst.State.PLAYING:
+            raise RuntimeError('webcam pipeline failed to play')
+        sample = sink.emit('try-pull-sample', timeout_ms * Gst.MSECOND)
+        if sample is None:
+            sample = sink.get_property('last-sample')
+        if sample is None:
+            raise RuntimeError('no webcam frame yet')
+        buf = sample.get_buffer()
+        caps = sample.get_caps().get_structure(0)
+        width = int(caps.get_value('width'))
+        height = int(caps.get_value('height'))
+        ok, mapped = buf.map(Gst.MapFlags.READ)
+        if not ok:
+            raise RuntimeError('could not map webcam frame')
+        try:
+            data = bytes(mapped.data)
+        finally:
+            buf.unmap(mapped)
+        stride = max(width * 3, len(data) // max(1, height))
+        if stride != width * 3:
+            rows = [data[i * stride:i * stride + width * 3] for i in range(height)]
+            data = b''.join(rows)
+        os.makedirs(os.path.dirname(path) or '.', exist_ok=True)
+        Image.frombytes('RGB', (width, height), data[:width * height * 3]).save(path)
+        return path
+    finally:
+        pipeline.set_state(Gst.State.NULL)
+
+
+def v4l2_device():
+    if DEVICE.startswith('/dev/video'):
+        return DEVICE
+    for index in range(0, 8):
+        candidate = f'/dev/video{index}'
+        if os.path.exists(candidate):
+            return candidate
+    return '/dev/video0'
+
+
+def pipewire_target():
+    if DEVICE.startswith('/dev/'):
+        return ''
+    safe = ''.join(ch for ch in DEVICE if ch.isalnum() or ch in '._:-')
+    return f' target-object={safe}' if safe else ''
+
+
+def capture_portal(path):
+    proxy = portal_proxy()
+    request_access(proxy)
+    fd = open_pipewire_fd(proxy)
+    desc = (
+        f'pipewiresrc fd={int(fd)}{pipewire_target()} always-copy=true do-timestamp=true client-name=jarvis-webcam ! '
+        'videoconvert ! video/x-raw,format=RGB ! appsink name=sink max-buffers=1 drop=true sync=false enable-last-sample=true'
+    )
+    grab_rgb(desc, path)
+    return 'portal'
+
+
+def capture_v4l2(path):
+    device = v4l2_device()
+    desc = (
+        f'v4l2src device={device} ! videoconvert ! video/x-raw,format=RGB ! '
+        'appsink name=sink max-buffers=1 drop=true sync=false enable-last-sample=true'
+    )
+    grab_rgb(desc, path)
+    return 'v4l2'
+
+
+if ACTION == 'access':
+    try:
+        request_access(portal_proxy())
+        print(json.dumps({'ok': True, 'via': 'portal'}), flush=True)
+        raise SystemExit(0)
+    except Exception as exc:
+        print(json.dumps({'ok': True, 'via': 'grant', 'warning': str(exc)}), flush=True)
+        raise SystemExit(0)
+
+if ACTION != 'capture' or not TARGET:
+    fail('usage: portal_camera.py access | capture <path>')
+
+errors = []
+for grab, via_name in ((capture_portal, 'portal'), (capture_v4l2, 'v4l2')):
+    try:
+        via = grab(TARGET)
+        print(json.dumps({'ok': True, 'path': TARGET, 'via': via or via_name}), flush=True)
+        raise SystemExit(0)
+    except Exception as exc:
+        errors.append(f'{via_name}: {exc}')
+fail('; '.join(errors) or 'webcam unavailable')