From 0e52942b8bde82536b93b28157c88e0d46209880 Mon Sep 17 00:00:00 2001 From: Raven Scott Date: Sat, 12 Sep 2026 12:33:52 -0400 Subject: [PATCH] Better Search --- computer-use/observer.js | 22 +- computer-use/portal-input.js | 43 +- computer-use/py/portal_remote_desktop.py | 171 +++-- computer-use/py/pw_framebuffer.py | 57 ++ daemon/harness-bridge.js | 2 +- daemon/index.js | 8 +- docs/computer-use.md | 8 +- skills/computer-observe.js | 2 +- skills/voice-prompt.js | 4 +- test/computer-use.test.js | 2 +- test/daemon.test.js | 5 + test/observer.test.js | 20 +- test/review-regressions.test.js | 25 + test/runtime-tools.test.js | 102 ++- vendor/agent-harness/agent/policy.js | 2 +- vendor/agent-harness/agent/tool-set.js | 7 +- vendor/agent-harness/agent/tools.js | 192 ++---- vendor/agent-harness/agent/web-search.js | 779 +++++++++++++++++++++++ vendor/agent-harness/test/test.js | 97 +++ 19 files changed, 1333 insertions(+), 215 deletions(-) create mode 100644 computer-use/py/pw_framebuffer.py create mode 100644 vendor/agent-harness/agent/web-search.js diff --git a/computer-use/observer.js b/computer-use/observer.js index dd8d196..92b2178 100644 --- a/computer-use/observer.js +++ b/computer-use/observer.js @@ -21,8 +21,8 @@ function timed(promise, ms, label) { } export class DesktopObserver { - constructor({ screenshot = new PortalScreenshot(), normalizer = new FrameNormalizer(), atspi = new AtspiProvider(), shell = new ShellProvider(), ocr, vision, tmpDir = '/tmp/jarvis-cu', timeouts = {} } = {}) { - this.screenshot = screenshot; this.normalizer = normalizer; this.atspi = atspi; this.shell = shell; this.ocr = ocr; this.vision = vision; this.tmpDir = tmpDir; + constructor({ screenshot = new PortalScreenshot(), normalizer = new FrameNormalizer(), atspi = new AtspiProvider(), shell = new ShellProvider(), framebuffer, ocr, vision, tmpDir = '/tmp/jarvis-cu', timeouts = {} } = {}) { + this.screenshot = screenshot; this.normalizer = normalizer; this.atspi = atspi; this.shell = shell; this.framebuffer = framebuffer; this.ocr = ocr; this.vision = vision; this.tmpDir = tmpDir; this.lastTree = []; this.timeouts = { screenshot: timeouts.screenshot ?? 8000, tree: timeouts.tree ?? 5000, shell: timeouts.shell ?? 2000, ocr: timeouts.ocr ?? 8000, vision: timeouts.vision ?? 8000 }; } @@ -34,8 +34,15 @@ export class DesktopObserver { } async _captureFrame() { - const rawPath = await this.screenshot.capture(path.join(this.tmpDir, `observe-${Date.now()}.png`)); - return this.normalizer.normalize(rawPath); + const dest = path.join(this.tmpDir, `observe-${Date.now()}.png`); + if (this.framebuffer?.capture) { + const rawPath = await this.framebuffer.capture(dest); + const frame = await this.normalizer.normalize(rawPath); + return { ...frame, source: 'pipewire' }; + } + const rawPath = await this.screenshot.capture(dest); + const frame = await this.normalizer.normalize(rawPath); + return { ...frame, source: 'screenshot' }; } async _shellSnapshot() { @@ -49,7 +56,7 @@ export class DesktopObserver { const unavailable = []; const note = (label) => (error) => { unavailable.push(`${label}: ${error.message}`); return null; }; const [frame, shell, tree] = await Promise.all([ - timed(this._captureFrame(), this.timeouts.screenshot, 'screenshot').catch(note('screenshot')), + timed(this._captureFrame(), this.timeouts.screenshot, 'frame').catch(note('frame')), timed(this._shellSnapshot(), this.timeouts.shell, 'Shell helper').catch(note('Shell helper')), includeTree ? this.tree().catch((error) => { unavailable.push(`AT-SPI: ${error.message}`); return []; }) : Promise.resolve([]), ]); @@ -60,6 +67,7 @@ export class DesktopObserver { windows: windows.windows || [], tree, screenshot_path: frame?.path || null, + frame_source: frame?.source || null, ocr_blocks: [], vision_hint: null, unavailable, @@ -75,7 +83,9 @@ export class DesktopObserver { const node = ref ? this.lastTree.find((item) => item.ref === ref) : null; const target = rect || node?.rect; if (!target) throw new Error('rect or current tree ref is required'); - const raw = await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`)); + const raw = this.framebuffer?.capture + ? await this.framebuffer.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`)) + : await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`)); const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), target); return { rect: target, screenshot_path: frame.path }; } diff --git a/computer-use/portal-input.js b/computer-use/portal-input.js index 879f9bc..6c2e4f0 100644 --- a/computer-use/portal-input.js +++ b/computer-use/portal-input.js @@ -1,22 +1,23 @@ import { spawn } from 'node:child_process'; import path from 'node:path'; -/** Wayland input boundary. The helper owns portal consent and the EIS fd; - * Node sends only bounded JSON actions and never uses hidden uinput. */ +/** Wayland input and ScreenCast boundary. The helper owns portal consent, the + * EIS fd, and the PipeWire remote; Node sends only bounded JSON actions. */ export class PortalInputBackend { constructor({ command = process.env.JARVIS_EI_HELPER || path.resolve(new URL('./py/portal_remote_desktop.py', import.meta.url).pathname), python = 'python3', spawnImpl = spawn, timeoutMs = 120_000 } = {}) { this.command = command; this.python = python; this.spawnImpl = spawnImpl; this.timeoutMs = timeoutMs; - this.process = null; this.available = false; this._grant = null; + this.process = null; this.available = false; this.screen = false; this._grant = null; this._frameWait = null; } - grant({ persist = false, monitors = 'focused' } = {}) { + grant({ persist = false, monitors = 'focused', mode = 'act' } = {}) { if (this._grant) return this._grant; - const child = this.process = this.spawnImpl(this.python, [this.command], { stdio: ['pipe', 'pipe', 'pipe'] }); + const child = this.process = this.spawnImpl(this.python, [this.command], { stdio: ['pipe', 'pipe', 'pipe'], env: { ...process.env, JARVIS_CU_MODE: mode } }); this._grant = new Promise((resolve, reject) => { let buffer = ''; let settled = false; const timer = setTimeout(() => fail(new Error('portal input consent timed out')), this.timeoutMs); const fail = (error) => { clearTimeout(timer); - if (this.process === child) { this.available = false; this.process = null; this._grant = null; } + if (this.process === child) { this.available = false; this.screen = false; this.process = null; this._grant = null; } + this._rejectFrame(error); if (!settled) { settled = true; reject(error); } child.kill('SIGTERM'); }; @@ -24,7 +25,8 @@ export class PortalInputBackend { child.on('error', fail); child.once('close', () => { clearTimeout(timer); - if (this.process === child) { this.available = false; this.process = null; this._grant = null; } + if (this.process === child) { this.available = false; this.screen = false; this.process = null; this._grant = null; } + this._rejectFrame(new Error('portal helper exited')); if (!settled) { settled = true; reject(new Error('portal input helper exited before readiness')); } }); child.stdin?.on('error', fail); @@ -37,10 +39,15 @@ export class PortalInputBackend { const line = buffer.slice(0, index); buffer = buffer.slice(index + 1); let event; try { event = JSON.parse(line); } catch { continue; } - if (event.type === 'error') { fail(new Error(event.reason || 'portal input unavailable')); return; } + if (event.type === 'error') { + if (!settled) { fail(new Error(event.reason || 'portal input unavailable')); return; } + this._rejectFrame(new Error(event.reason || 'portal helper error')); + continue; + } + if (event.type === 'frame') { this._resolveFrame(event.path || event); continue; } if (event.type === 'ready' && this.process === child && !settled) { - clearTimeout(timer); settled = true; this.available = true; - resolve({ restore_token_present: Boolean(event.restore_token_present), monitors, backend: event.backend || 'portal-ei' }); + clearTimeout(timer); settled = true; this.available = true; this.screen = Boolean(event.screen); + resolve({ restore_token_present: Boolean(event.restore_token_present), monitors, backend: event.backend || 'portal-ei', screen: this.screen }); } } }); @@ -48,5 +55,19 @@ export class PortalInputBackend { return this._grant; } send(action) { if (!this.available || !this.process?.stdin?.writable) throw new Error('portal EIS input backend is unavailable'); this.process.stdin.write(`${JSON.stringify(action)}\n`); } - revoke() { this._cancelGrant?.(); this._cancelGrant = null; this.available = false; this.process = null; this._grant = null; } + captureFrame(output) { + if (!this.available || !this.process?.stdin?.writable) throw new Error('PipeWire ScreenCast is unavailable until desktop access is granted'); + if (this._frameWait) throw new Error('a PipeWire frame grab is already in flight'); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => this._rejectFrame(new Error('PipeWire frame timed out')), 8000); + this._frameWait = { + resolve: (value) => { clearTimeout(timer); resolve(value); }, + reject: (error) => { clearTimeout(timer); reject(error); }, + }; + try { this.send({ type: 'frame', path: output }); } catch (error) { this._rejectFrame(error); } + }); + } + _resolveFrame(path) { const wait = this._frameWait; this._frameWait = null; wait?.resolve(path); } + _rejectFrame(error) { const wait = this._frameWait; this._frameWait = null; wait?.reject(error); } + revoke() { this._cancelGrant?.(); this._cancelGrant = null; this.available = false; this.screen = false; this.process = null; this._grant = null; } } diff --git a/computer-use/py/portal_remote_desktop.py b/computer-use/py/portal_remote_desktop.py index 0486ab8..6321189 100644 --- a/computer-use/py/portal_remote_desktop.py +++ b/computer-use/py/portal_remote_desktop.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""RemoteDesktop consent broker with a built-in libei injector.""" +"""RemoteDesktop + ScreenCast broker: EIS input and a live PipeWire frame buffer.""" import json import os import select @@ -17,8 +17,10 @@ except Exception as exc: raise SystemExit(1) from libei_sender import LibeiSender +from pw_framebuffer import PipeWireFrameBuffer BTN = {'left': 0x110, 'right': 0x111, 'middle': 0x112} +MODE = os.environ.get('JARVIS_CU_MODE', 'act') def log_error(reason): @@ -26,14 +28,21 @@ def log_error(reason): bus = Gio.bus_get_sync(Gio.BusType.SESSION, None) -proxy = Gio.DBusProxy.new_sync( - bus, Gio.DBusProxyFlags.NONE, None, - 'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop', - 'org.freedesktop.portal.RemoteDesktop', None, -) -def request(method, signature, values, timeout_ms=120000): +def portal_proxy(iface): + return Gio.DBusProxy.new_sync( + bus, Gio.DBusProxyFlags.NONE, None, + 'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop', + iface, None, + ) + + +remote = portal_proxy('org.freedesktop.portal.RemoteDesktop') +screencast = portal_proxy('org.freedesktop.portal.ScreenCast') + + +def request(proxy, method, signature, values, timeout_ms=120000): request_path = proxy.call_sync(method, GLib.Variant(signature, values), Gio.DBusCallFlags.NONE, timeout_ms, None).unpack()[0] loop = GLib.MainLoop() result = {'code': 1, 'results': {}} @@ -50,10 +59,10 @@ def request(method, signature, values, timeout_ms=120000): return result['results'] -def connect_to_eis(session): +def unix_fd(proxy, method, session): incoming = Gio.UnixFDList.new() variant, outgoing = proxy.call_with_unix_fd_list_sync( - 'ConnectToEIS', + method, GLib.Variant('(oa{sv})', (session, {})), Gio.DBusCallFlags.NONE, 15000, @@ -70,11 +79,22 @@ def connect_to_eis(session): fd = unpacked[0] if isinstance(unpacked, (tuple, list)) else unpacked if isinstance(fd, int) and fd >= 0: return fd - raise RuntimeError('ConnectToEIS returned no file descriptor') + raise RuntimeError(f'{method} returned no file descriptor') def notify(method, signature, values): - proxy.call_sync(method, GLib.Variant(signature, values), Gio.DBusCallFlags.NONE, 5000, None) + remote.call_sync(method, GLib.Variant(signature, values), Gio.DBusCallFlags.NONE, 5000, None) + + +def stream_node_id(streams): + if not streams: + raise RuntimeError('ScreenCast Start returned no PipeWire streams') + first = streams[0] + if isinstance(first, (list, tuple)): + return int(first[0]) + if isinstance(first, dict): + return int(first.get('node_id') or first.get('id')) + return int(first) class PortalNotify: @@ -115,6 +135,30 @@ class PortalNotify: raise RuntimeError(f'unsupported portal notify action {kind} {name}') +class SessionHandler: + def __init__(self, input_handler, framebuffer): + self.input_handler = input_handler + self.framebuffer = framebuffer + + def dispatch(self): + if hasattr(self.input_handler, 'dispatch'): + self.input_handler.dispatch() + + def handle(self, action): + if action.get('type') == 'frame': + if not self.framebuffer: + raise RuntimeError('PipeWire ScreenCast is not attached') + path = action.get('path') + if not path: + raise RuntimeError('frame path is required') + self.framebuffer.capture(path) + print(json.dumps({'type': 'frame', 'path': path, 'source': 'pipewire'}), flush=True) + return + if self.input_handler is None: + raise RuntimeError('desktop input is observe-only') + self.input_handler.handle(action) + + def run_loop(handler, extra_fd=None): stdin_fd = sys.stdin.fileno() while True: @@ -140,58 +184,93 @@ def run_loop(handler, extra_fd=None): log_error(exc) +def attach_screencast(): + token = f'jarvissc{GLib.get_real_time()}' + session = request(screencast, 'CreateSession', '(a{sv})', ({'session_handle_token': GLib.Variant('s', token)},))['session_handle'] + request(screencast, 'SelectSources', '(oa{sv})', (session, { + 'types': GLib.Variant('u', 1), + 'multiple': GLib.Variant('b', False), + 'cursor_mode': GLib.Variant('u', 2), + 'persist_mode': GLib.Variant('u', 2), + })) + return session + + +def start_framebuffer(sc_session): + results = request(screencast, 'Start', '(osa{sv})', (sc_session, '', {})) + node_id = stream_node_id(results.get('streams') or []) + fd = unix_fd(screencast, 'OpenPipeWireRemote', sc_session) + return PipeWireFrameBuffer(fd, node_id) + + injector = None sender = None +framebuffer = None session = None try: - token = f'jarvis{GLib.get_real_time()}' - session = request('CreateSession', '(a{sv})', ({'session_handle_token': GLib.Variant('s', token)},))['session_handle'] - request('SelectDevices', '(oa{sv})', (session, {'types': GLib.Variant('u', 7), 'persist_mode': GLib.Variant('u', 2)})) - request('Start', '(osa{sv})', (session, '', {})) - backend = 'none' + sc_session = attach_screencast() + input_handler = None extra_fd = None - handler = None + backend = 'none' eis_error = None + frame_error = None + if MODE != 'observe': + token = f'jarvisrd{GLib.get_real_time()}' + session = request(remote, 'CreateSession', '(a{sv})', ({'session_handle_token': GLib.Variant('s', token)},))['session_handle'] + request(remote, 'SelectDevices', '(oa{sv})', (session, {'types': GLib.Variant('u', 7), 'persist_mode': GLib.Variant('u', 2)})) + request(remote, 'Start', '(osa{sv})', (session, '', {})) + try: + eis_fd = unix_fd(remote, 'ConnectToEIS', session) + sender = LibeiSender(eis_fd) + if not sender.wait_ready(): + raise RuntimeError('libei connected but no pointer or keyboard device appeared') + input_handler = sender + extra_fd = sender.fd + backend = 'portal-ei' + except Exception as exc: + eis_error = str(exc) + if sender: + try: + sender.close() + except Exception: + pass + sender = None + if input_handler is None and os.environ.get('JARVIS_LIBEI_BRIDGE'): + injector = subprocess.Popen(os.environ['JARVIS_LIBEI_BRIDGE'], shell=True, stdin=subprocess.PIPE, text=True) + + class Bridge: + def handle(self, action): + injector.stdin.write(json.dumps(action) + '\n') + injector.stdin.flush() + + input_handler = Bridge() + backend = 'portal-ei' + if input_handler is None: + input_handler = PortalNotify(session) + backend = 'portal-notify' try: - eis_fd = connect_to_eis(session) - sender = LibeiSender(eis_fd) - if not sender.wait_ready(): - raise RuntimeError('libei connected but no pointer or keyboard device appeared') - handler = sender - extra_fd = sender.fd - backend = 'portal-ei' + framebuffer = start_framebuffer(sc_session) except Exception as exc: - eis_error = str(exc) - if sender: - try: - sender.close() - except Exception: - pass - sender = None - if handler is None and os.environ.get('JARVIS_LIBEI_BRIDGE'): - injector = subprocess.Popen(os.environ['JARVIS_LIBEI_BRIDGE'], shell=True, stdin=subprocess.PIPE, text=True) - - class Bridge: - def handle(self, action): - injector.stdin.write(json.dumps(action) + '\n') - injector.stdin.flush() - - handler = Bridge() - backend = 'portal-ei' - if handler is None: - handler = PortalNotify(session) - backend = 'portal-notify' + frame_error = str(exc) + framebuffer = None print(json.dumps({ 'type': 'ready', - 'session': session, + 'session': session or sc_session, 'restore_token_present': True, 'backend': backend, + 'screen': framebuffer is not None, 'eis_error': eis_error, + 'frame_error': frame_error, }), flush=True) - run_loop(handler, extra_fd) + run_loop(SessionHandler(input_handler, framebuffer), extra_fd) except Exception as exc: log_error(exc) finally: + if framebuffer: + try: + framebuffer.close() + except Exception: + pass if sender: try: sender.close() diff --git a/computer-use/py/pw_framebuffer.py b/computer-use/py/pw_framebuffer.py new file mode 100644 index 0000000..545681a --- /dev/null +++ b/computer-use/py/pw_framebuffer.py @@ -0,0 +1,57 @@ +"""Pull one RGB frame from a portal PipeWire ScreenCast stream.""" +from PIL import Image + + +class PipeWireFrameBuffer: + def __init__(self, fd, node_id): + import gi + gi.require_version('Gst', '1.0') + from gi.repository import Gst + Gst.init(None) + self.Gst = Gst + self.pipeline = Gst.parse_launch( + f'pipewiresrc fd={int(fd)} path={int(node_id)} always-copy=true do-timestamp=true ' + 'client-name=jarvis ! videoconvert ! video/x-raw,format=RGB ! ' + 'appsink name=sink max-buffers=1 drop=true sync=false enable-last-sample=true' + ) + self.sink = self.pipeline.get_by_name('sink') + if not self.sink: + raise RuntimeError('GStreamer appsink missing') + self.pipeline.set_state(Gst.State.PLAYING) + change, state, _pending = self.pipeline.get_state(5 * Gst.SECOND) + if change == Gst.StateChangeReturn.FAILURE or state != Gst.State.PLAYING: + self.close() + raise RuntimeError('PipeWire ScreenCast pipeline failed to play') + + def capture(self, path, timeout_ms=4000): + sample = self.sink.emit('try-pull-sample', timeout_ms * self.Gst.MSECOND) + if sample is None: + sample = self.sink.get_property('last-sample') + if sample is None: + raise RuntimeError('no PipeWire frame yet') + buf = sample.get_buffer() + caps = sample.get_caps().get_structure(0) + width = int(caps.get_value('width')) + height = int(caps.get_value('height')) + ok, mapped = buf.map(self.Gst.MapFlags.READ) + if not ok: + raise RuntimeError('could not map PipeWire frame') + try: + data = bytes(mapped.data) + finally: + buf.unmap(mapped) + stride = max(width * 3, len(data) // max(1, height)) + if stride != width * 3: + rows = [data[i * stride:i * stride + width * 3] for i in range(height)] + data = b''.join(rows) + Image.frombytes('RGB', (width, height), data[:width * height * 3]).save(path) + return path + + def close(self): + if getattr(self, 'pipeline', None): + try: + self.pipeline.set_state(self.Gst.State.NULL) + except Exception: + pass + self.pipeline = None + self.sink = None diff --git a/daemon/harness-bridge.js b/daemon/harness-bridge.js index 231aa4c..2e3cb3d 100644 --- a/daemon/harness-bridge.js +++ b/daemon/harness-bridge.js @@ -35,7 +35,7 @@ export class HarnessBridge extends EventEmitter { ...createPhase9GatewayTool(), ...tools, ], - builtinTools: ['read_file', 'list_dir', 'grep', 'run_terminal_cmd', 'web_fetch', 'web_search'], + builtinTools: ['read_file', 'list_dir', 'grep', 'run_terminal_cmd', 'web_fetch', 'fetch_page', 'google_search', 'web_search', 'wiki_search', 'hn_search', 'code_search'], webFetch: true, permissionMode, origin: 'jarvis-qvac', diff --git a/daemon/index.js b/daemon/index.js index 163e6b4..b7988ef 100644 --- a/daemon/index.js +++ b/daemon/index.js @@ -35,7 +35,11 @@ export class JarvisDaemon extends EventEmitter { this.computer = new ComputerUseSession({ audit: this.audit, stepsMax: this.settings.computerSteps, grantMinutes: this.settings.computerGrantMinutes, mode: this.settings.computerMode }); this.input = new PortalInputBackend(); this.perception = new QvacPerception(); - this.observer = new DesktopObserver({ normalizer: new FrameNormalizer({ maxLongEdge: this.settings.screenshotMaxEdge, quality: this.settings.screenshotQuality }), ocr: (image) => this.perception.ocr(image) }); + this.observer = new DesktopObserver({ + normalizer: new FrameNormalizer({ maxLongEdge: this.settings.screenshotMaxEdge, quality: this.settings.screenshotQuality }), + ocr: (image) => this.perception.ocr(image), + framebuffer: { capture: (output) => this.input.captureFrame(output) }, + }); this.actuator = new ComputerActuator({ session: this.computer, input: this.input, find: ({ ref }) => this.observer.lastTree.filter((node) => node.ref === ref), atspiAction: (target, action) => this.observer.atspi.action(target, action), highlight: async (target, action) => this.emit('ComputerHighlight', JSON.stringify({ rect: target?.rect || null, label: `${action} ${target?.name || ''}` })) , audit: this.audit }); this.harness = new HarnessBridge({ cwd: process.cwd(), computer: this.computer, observer: this.observer, actuator: this.actuator, fsAccess: this.settings.fsAccess }); this.log = new PrivacyLog(); @@ -185,7 +189,7 @@ export class JarvisDaemon extends EventEmitter { await this.voiceLoop.ensureAsr?.(); } cancel() { this._askGeneration += 1; this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computerRevoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); } - computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); if (this.settings.computerMode === 'observe') return result; this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; } + computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); this.input.grant({ persist, mode: this.settings.computerMode }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; } computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); } startVoice() { if (this._voiceStarting) return this._voiceStarting; diff --git a/docs/computer-use.md b/docs/computer-use.md index e4dfae7..815dfbc 100644 --- a/docs/computer-use.md +++ b/docs/computer-use.md @@ -41,9 +41,11 @@ sequenceDiagram The primary Wayland input path is XDG RemoteDesktop plus EIS/libei. After portal consent, Jarvis opens `ConnectToEIS` and injects pointer/keyboard -events through a built-in libei sender. `JARVIS_LIBEI_BRIDGE` is an optional -override. If EIS is unavailable, the helper falls back to portal Notify -methods instead of refusing readiness. Screenshot frames are downscaled for vision, +events through a built-in libei sender. Observation reads the live PipeWire +ScreenCast stream (the compositor frame buffer), not the Screenshot portal. +`JARVIS_LIBEI_BRIDGE` is an optional override. If EIS is unavailable, the helper +falls back to portal Notify methods instead of refusing readiness. Frames are +downscaled for vision, kept in temporary storage, and removed on revoke unless trace retention is enabled. diff --git a/skills/computer-observe.js b/skills/computer-observe.js index 398dd73..3413883 100644 --- a/skills/computer-observe.js +++ b/skills/computer-observe.js @@ -4,7 +4,7 @@ const inactive = (computer) => !computer?.status?.().active ? { unavailable: 'co export function createComputerObserveTools({ computer, observer } = {}) { const guard = () => { const result = inactive(computer); if (result) throw new Error(result.unavailable); }; return [ - { name: 'cu_observe', permission: PERMISSION, description: 'Observe the local desktop through the portal, AT-SPI, and the accessibility tree. OCR is off unless include_ocr is true.', parameters: { type: 'object', properties: { include_tree: { type: 'boolean' }, include_ocr: { type: 'boolean' }, include_vision: { type: 'boolean' } } }, execute: async ({ include_tree = true, include_ocr = false, include_vision = false } = {}) => { guard(); return observer.observe({ includeTree: include_tree, includeOcr: include_ocr, includeVision: include_vision }); } }, + { name: 'cu_observe', permission: PERMISSION, description: 'Read the live PipeWire ScreenCast frame buffer from the granted desktop session, plus AT-SPI. This is not the Screenshot portal. OCR is off unless include_ocr is true.', parameters: { type: 'object', properties: { include_tree: { type: 'boolean' }, include_ocr: { type: 'boolean' }, include_vision: { type: 'boolean' } } }, execute: async ({ include_tree = true, include_ocr = false, include_vision = false } = {}) => { guard(); return observer.observe({ includeTree: include_tree, includeOcr: include_ocr, includeVision: include_vision }); } }, { name: 'cu_zoom', permission: PERMISSION, description: 'Request a higher resolution local crop by rectangle or current AT-SPI ref.', parameters: { type: 'object', properties: { rect: { type: 'array', items: { type: 'number' } }, ref: { type: 'string' } } }, execute: async (args) => { guard(); return observer.zoom(args); } }, { name: 'cu_tree', permission: PERMISSION, description: 'Return visible AT-SPI roles, names, states, bounds, and per-step refs.', parameters: { type: 'object', properties: { app: { type: 'string' }, focused_only: { type: 'boolean' }, max_nodes: { type: 'number' } } }, execute: async ({ focused_only = true, max_nodes = 400 } = {}) => { guard(); return observer.tree({ focusedOnly: focused_only, maxNodes: max_nodes }); } }, { name: 'cu_find', permission: PERMISSION, description: 'Find a visible desktop element by accessible name/role.', parameters: { type: 'object', properties: { query: { type: 'string' }, role: { type: 'string' } }, required: ['query'] }, execute: async (args) => { guard(); return observer.find(args); } }, diff --git a/skills/voice-prompt.js b/skills/voice-prompt.js index 4ed4631..91c1f84 100644 --- a/skills/voice-prompt.js +++ b/skills/voice-prompt.js @@ -9,11 +9,11 @@ Tool names, tool arguments, paths, and U R L strings use ordinary spelling. Only Thinking is private. After thoughts, call a tool or speak the answer. Do not stop in thoughts or say you will search later. Ground desktop, file, memory, model, and network claims in a tool result. Do not invent limits the tools did not report. -This computer can reach the internet. web_search and web_fetch are unrestricted and do not wait for confirmation. Never say you will use a tool. Call the tool instead of announcing it. Use web_search to find pages. After web_search, call web_fetch on one real http or https page from the hits, then speak the answer. DuckDuckGo redirect links are not an answer. For this computer's public I P, call web_fetch on https://ifconfig.me/ip first. Use web_fetch for any website or I P lookup page. Do not use curl, wget, or run_terminal_cmd for websites. The shell blocks public H T T P; that is not a network outage. If a shell result says HTTP access is not allowed, call web_fetch or web_search next and answer from that result. Never say the network is unavailable unless web_fetch or web_search itself failed. +This computer can reach the internet. web_search, google_search, fetch_page, web_fetch, wiki_search, hn_search, and code_search are unrestricted and do not wait for confirmation. Never say you will use a tool. Call the tool instead of announcing it. Use web_search to find pages. Default engine auto tries several backends. If it fails, call web_search again with engine set to bing, jina, wikipedia, duckduckgo, or google. After web_search, call fetch_page on one real http or https page from the hits, then speak the answer. Use web_fetch for raw pages and this computer's public I P at https://ifconfig.me/ip. Use wiki_search, hn_search, or code_search when the question is about Wikipedia, Hacker News, GitHub, npm, or M D N. Redirect links are not an answer. Do not use curl, wget, or run_terminal_cmd for websites. The shell blocks public H T T P; that is not a network outage. If a shell result says HTTP access is not allowed, call web_fetch or web_search next and answer from that result. Never say the network is unavailable unless web_fetch or web_search itself failed. File tools may read any path they accept. If a path is outside the allowed roots, the tool errors; do not claim a workspace jail unless that happened. Writes, including fs_write and overwrite, still need confirmation. -Computer use requires an explicit user grant from Settings, Computer use, Allow now, or Grant desktop in the tray. After a grant, call cu_observe to see the screen. Do not wait for a libei injector. Never click or type while it is inactive, locked, or revoked. Never ask for passwords or credentials. +Computer use requires an explicit user grant from Settings, Computer use, Allow now, or Grant desktop in the tray. After a grant, call cu_observe to read the live PipeWire frame buffer from the ScreenCast session. Do not take a screenshot. Do not wait for a libei injector. Never click or type while it is inactive, locked, or revoked. Never ask for passwords or credentials. Destructive actions require confirmation in both the heads-up display and spoken conversation. Prefer structured tools and accessibility references over coordinates. diff --git a/test/computer-use.test.js b/test/computer-use.test.js index 0ac8fb7..048f064 100644 --- a/test/computer-use.test.js +++ b/test/computer-use.test.js @@ -56,7 +56,7 @@ test('computer use refuses password targets and unconfirmed dangerous keys', asy test('libei sender binds bitmask capabilities from libei.h', () => { const pyDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../computer-use/py'); - const compiled = spawnSync('python3', ['-m', 'py_compile', 'libei_sender.py', 'portal_remote_desktop.py', 'portal_screenshot.py'], { cwd: pyDir, encoding: 'utf8' }); + const compiled = spawnSync('python3', ['-m', 'py_compile', 'libei_sender.py', 'portal_remote_desktop.py', 'portal_screenshot.py', 'pw_framebuffer.py'], { cwd: pyDir, encoding: 'utf8' }); assert.equal(compiled.status, 0, compiled.stderr); const caps = spawnSync('python3', ['-c', 'from libei_sender import CAP_POINTER, CAP_KEYBOARD, CAP_BUTTON, F_KEYS; assert CAP_POINTER == 1 and CAP_KEYBOARD == 4 and CAP_BUTTON == 32 and F_KEYS["f11"] == 87'], { cwd: pyDir, encoding: 'utf8' }); assert.equal(caps.status, 0, caps.stderr); diff --git a/test/daemon.test.js b/test/daemon.test.js index 3ee9b4a..e1df688 100644 --- a/test/daemon.test.js +++ b/test/daemon.test.js @@ -80,7 +80,12 @@ test('harness bridge caps voice shell chaining', () => { 'grep', 'run_terminal_cmd', 'web_fetch', + 'fetch_page', + 'google_search', 'web_search', + 'wiki_search', + 'hn_search', + 'code_search', ]); assert.equal(bridge.options.webFetch, true); }); diff --git a/test/observer.test.js b/test/observer.test.js index a9429c2..81ef720 100644 --- a/test/observer.test.js +++ b/test/observer.test.js @@ -11,6 +11,7 @@ test('Phase 6 observer returns stable refs and ranked semantic matches', async ( }); const bundle = await observer.observe({ includeOcr: false }); assert.equal(bundle.screenshot_path, '/tmp/frame.webp'); assert.equal(bundle.tree[0].ref, 'r1'); + assert.equal(bundle.frame_source, 'screenshot'); assert.equal((await observer.find({ query: 'night light' }))[0].ref, 'r1'); const zoom = await observer.zoom({ ref: 'r1' }); assert.deepEqual(zoom.rect, [10, 20, 30, 30]); }); @@ -31,6 +32,21 @@ test('observer returns when the Shell helper never connects', async () => { assert.match(bundle.unavailable.join(' '), /Shell helper/); }); +test('observer reads the PipeWire frame buffer instead of the Screenshot portal', async () => { + let shots = 0; + const observer = new DesktopObserver({ + screenshot: { capture: async () => { shots += 1; return '/tmp/portal.png'; } }, + framebuffer: { capture: async () => '/tmp/pw.png' }, + normalizer: { normalize: async (input) => ({ path: `${input}.webp` }) }, + shell: { windows: async () => ({ available: true, windows: [] }), focused: async () => null }, + atspi: { tree: async () => [] }, + }); + const bundle = await observer.observe({ includeTree: false, includeOcr: false }); + assert.equal(shots, 0); + assert.equal(bundle.frame_source, 'pipewire'); + assert.equal(bundle.screenshot_path, '/tmp/pw.png.webp'); +}); + test('observer returns after a hung screenshot instead of staying on THINKING', async () => { const observer = new DesktopObserver({ screenshot: { capture: () => new Promise(() => {}) }, @@ -42,11 +58,11 @@ test('observer returns after a hung screenshot instead of staying on THINKING', const bundle = await observer.observe({ includeTree: false, includeOcr: false }); assert.ok(Date.now() - started < 1000); assert.equal(bundle.screenshot_path, null); - assert.match(bundle.unavailable.join(' '), /screenshot/); + assert.match(bundle.unavailable.join(' '), /frame/); }); test('Phase 6 observer reports portal failure instead of inventing a frame', async () => { const observer = new DesktopObserver({ screenshot: { capture: async () => { throw new Error('no session bus'); } }, shell: { windows: async () => ({ available: false, windows: [] }), focused: async () => null }, atspi: { tree: async () => [] } }); const bundle = await observer.observe({ includeTree: false, includeOcr: false }); - assert.equal(bundle.screenshot_path, null); assert.match(bundle.unavailable[0], /screenshot/); + assert.equal(bundle.screenshot_path, null); assert.match(bundle.unavailable[0], /frame/); }); diff --git a/test/review-regressions.test.js b/test/review-regressions.test.js index 91d6233..4173640 100644 --- a/test/review-regressions.test.js +++ b/test/review-regressions.test.js @@ -111,6 +111,31 @@ test('portal helper reports a notify fallback as ready', async () => { assert.equal((await grant).backend, 'portal-notify'); }); +test('portal helper returns a PipeWire frame without ending the grant', async () => { + const child = helper(); let written = ''; + child.stdin.write = (chunk) => { written += String(chunk); }; + const input = new PortalInputBackend({ spawnImpl: () => child }); + const grant = input.grant(); + child.stdout.emit('data', '{"type":"ready","backend":"portal-ei","screen":true}\n'); + assert.equal((await grant).screen, true); + const frame = input.captureFrame('/tmp/pw.png'); + assert.match(written, /"type":"frame"/); + child.stdout.emit('data', '{"type":"frame","path":"/tmp/pw.png","source":"pipewire"}\n'); + assert.equal(await frame, '/tmp/pw.png'); + assert.equal(input.available, true); +}); + +test('a PipeWire frame error does not revoke desktop access', async () => { + const child = helper(); const input = new PortalInputBackend({ spawnImpl: () => child }); + const grant = input.grant(); + child.stdout.emit('data', '{"type":"ready","backend":"portal-notify","screen":false}\n'); + await grant; + const frame = input.captureFrame('/tmp/pw.png'); + child.stdout.emit('data', '{"type":"error","reason":"no PipeWire frame yet"}\n'); + await assert.rejects(frame, /no PipeWire frame/); + assert.equal(input.available, true); +}); + test('portal grants wait for a complete readiness message and clear on exit', async () => { const child = helper(); const input = new PortalInputBackend({ spawnImpl: () => child }); const grant = input.grant(); diff --git a/test/runtime-tools.test.js b/test/runtime-tools.test.js index ddabd13..b7c6536 100644 --- a/test/runtime-tools.test.js +++ b/test/runtime-tools.test.js @@ -54,11 +54,108 @@ test('web_fetch times out instead of hanging the turn', async () => { } }); +test('google_search parses lite HTML and falls back to DuckDuckGo then Bing', async () => { + const require = createRequire(import.meta.url); + const tools = require('../vendor/agent-harness/agent/tools.js'); + const parsed = tools.parseGoogleHits( + '
Example Domain
' + ); + assert.equal(parsed[0].url, 'https://example.com/page'); + assert.equal(parsed[0].title, 'Example Domain'); + assert.equal(tools.parseGoogleHits('Google Search').length, 0); + + const bingHref = + 'https://www.bing.com/ck/a?!&&p=ae&u=a1aHR0cDovL3d3dy5leGFtcGxlLmNvbS8&ntb=1'; + const bingHits = tools.parseBingHits( + '
  • Example Domain

  • ' + ); + assert.equal(bingHits[0].url, 'http://www.example.com/'); + assert.equal(bingHits[0].title, 'Example Domain'); + + const orig = globalThis.fetch; + globalThis.fetch = async (url) => { + const href = String(url); + if (href.includes('google.com')) { + return { status: 200, url: href, text: async () => 'Google Search' }; + } + if (href.includes('duckduckgo.com')) { + return { + status: 202, + url: href, + text: async () => '
    Unfortunately, bots use DuckDuckGo too.
    ', + }; + } + return { + status: 200, + url: href, + text: async () => + '
  • Example Domain

  • ', + }; + }; + try { + const hits = await tools.googleSearchWithFallback('example', 200); + assert.equal(hits[0].source, 'bing'); + assert.equal(hits[0].url, 'http://www.example.com/'); + assert.equal(hits[0].title, 'Example Domain'); + } finally { + globalThis.fetch = orig; + } +}); + +test('web_search can pin an engine and fetch_page prefers Jina', async () => { + const require = createRequire(import.meta.url); + const tools = require('../vendor/agent-harness/agent/tools.js'); + const webSearch = require('../vendor/agent-harness/agent/web-search.js'); + const unknown = await tools.runWebSearch('example', { engine: 'nope' }); + assert.match(String(unknown.error), /unknown engine/); + assert.ok(Array.isArray(unknown.engines)); + + const rss = webSearch.parseRssItems( + 'Examplehttps://example.com/rssHello' + ); + assert.equal(rss[0].url, 'https://example.com/rss'); + assert.equal(rss[0].title, 'Example'); + + const orig = globalThis.fetch; + globalThis.fetch = async (url) => { + const href = String(url); + if (href.includes('wikipedia.org')) { + return { + status: 200, + url: href, + text: async () => JSON.stringify({ + query: { search: [{ title: 'Example.com', snippet: 'an example domain' }] }, + }), + }; + } + if (href.includes('r.jina.ai')) { + return { status: 200, url: href, text: async () => '# Example\nReadable article from Jina reader' }; + } + throw new Error('unexpected fetch ' + href); + }; + try { + const wiki = await tools.runWebSearch('example.com', { engine: 'wikipedia', timeoutMs: 200 }); + assert.equal(wiki[0].source, 'wikipedia'); + assert.equal(wiki[0].title, 'Example.com'); + assert.match(wiki[0].url, /wikipedia\.org\/wiki\/Example\.com/); + const page = await tools.fetchPage('https://example.com/article', 200); + assert.equal(page.via, 'jina'); + assert.match(page.text, /Readable article from Jina reader/); + } finally { + globalThis.fetch = orig; + } +}); + test('public web search and fetch do not require confirmation', () => { const require = createRequire(import.meta.url); const policy = require('../vendor/agent-harness/agent/policy.js'); assert.equal(policy.needsPermission('web_fetch', 'ask'), false); + assert.equal(policy.needsPermission('fetch_page', 'ask'), false); + assert.equal(policy.needsPermission('google_search', 'ask'), false); assert.equal(policy.needsPermission('web_search', 'ask'), false); + assert.equal(policy.needsPermission('wiki_search', 'ask'), false); + assert.equal(policy.needsPermission('hn_search', 'ask'), false); + assert.equal(policy.needsPermission('code_search', 'ask'), false); assert.equal(policy.needsPermission('run_terminal_cmd', 'ask'), true); assert.equal(policy.needsPermission('write_file', 'ask'), true); }); @@ -73,7 +170,8 @@ test('voice prompt tells the model not to chain extra terminal commands', () => assert.match(VOICE_SYSTEM_PROMPT, /Internet protocol addresses have no dots/); assert.match(VOICE_SYSTEM_PROMPT, /Spell them as separate letters/); assert.match(VOICE_SYSTEM_PROMPT, /web_fetch/); - assert.match(VOICE_SYSTEM_PROMPT, /web_search and web_fetch are unrestricted/); + assert.match(VOICE_SYSTEM_PROMPT, /web_search, google_search, fetch_page, web_fetch, wiki_search, hn_search, and code_search are unrestricted/); + assert.match(VOICE_SYSTEM_PROMPT, /Use web_search to find pages/); assert.match(VOICE_SYSTEM_PROMPT, /Never say you will use a tool/); assert.match(VOICE_SYSTEM_PROMPT, /Do not stop in thoughts/); assert.match(VOICE_SYSTEM_PROMPT, /ifconfig\.me\/ip/); @@ -82,7 +180,7 @@ test('voice prompt tells the model not to chain extra terminal commands', () => assert.match(VOICE_SYSTEM_PROMPT, /Tool names, tool arguments/); assert.match(VOICE_SYSTEM_PROMPT, /File tools may read any path they accept/); assert.match(VOICE_SYSTEM_PROMPT, /Allow now/); - assert.match(VOICE_SYSTEM_PROMPT, /call cu_observe to see the screen/); + assert.match(VOICE_SYSTEM_PROMPT, /call cu_observe to read the live PipeWire frame buffer/); assert.doesNotMatch(VOICE_SYSTEM_PROMPT, /Never claim cloud access/); }); diff --git a/vendor/agent-harness/agent/policy.js b/vendor/agent-harness/agent/policy.js index 42609a0..d2e8cb2 100644 --- a/vendor/agent-harness/agent/policy.js +++ b/vendor/agent-harness/agent/policy.js @@ -2,7 +2,7 @@ const WRITE_TOOLS = new Set(['search_replace', 'write_file', 'run_terminal_cmd', 'use_tool']); const ASK_TOOLS = new Set(['run_terminal_cmd', 'use_tool']); -// web_fetch and web_search are public reads; they do not prompt. +// web_fetch, fetch_page, google_search, web_search, wiki_search, hn_search, and code_search are public reads; they do not prompt. const SHELL_ALLOW = new Set([ 'git', 'rg', 'grep', 'ls', 'cat', 'head', 'tail', 'pwd', 'echo', 'node', 'npm', 'npx', 'python3', 'python', 'cargo', 'go', 'make', 'bare', 'wc', 'sort', 'uniq', 'find', 'sed', 'awk', diff --git a/vendor/agent-harness/agent/tool-set.js b/vendor/agent-harness/agent/tool-set.js index 54d0c15..d9be20e 100644 --- a/vendor/agent-harness/agent/tool-set.js +++ b/vendor/agent-harness/agent/tool-set.js @@ -21,8 +21,13 @@ const ALWAYS_RESERVED = ['image_gen', 'image_edit', 'image_to_video', 'deploy_ap const ALWAYS_BUILTIN_RESERVED = [ 'todo_write', + 'google_search', 'web_search', 'web_fetch', + 'fetch_page', + 'wiki_search', + 'hn_search', + 'code_search', 'enter_plan_mode', 'exit_plan_mode', 'ask_user_question', @@ -61,7 +66,7 @@ function filterBuiltinSchemas(schemas, opts) { list = list.filter((t) => t.name !== 'run_terminal_cmd'); } if (opts.webFetch !== true) { - list = list.filter((t) => t.name !== 'web_fetch'); + list = list.filter((t) => t.name !== 'web_fetch' && t.name !== 'fetch_page'); } return list; } diff --git a/vendor/agent-harness/agent/tools.js b/vendor/agent-harness/agent/tools.js index 46042d5..d4bc949 100644 --- a/vendor/agent-harness/agent/tools.js +++ b/vendor/agent-harness/agent/tools.js @@ -13,7 +13,7 @@ const todos = require('./todos.js'); const goalMod = require('./goal.js'); const sr = require('./search-replace.js'); const grepUtil = require('./grep-util.js'); -const truncate = require('./truncate.js'); +const web = require('./web-search.js'); const MAX_READ = 400 * 1024; const MAX_GREP_HITS = 50; @@ -217,8 +217,13 @@ const SCHEMAS = [ { type: 'function', name: 'list_dir', description: 'List a directory.', parameters: { type: 'object', properties: { path: { type: 'string' }, recursive: { type: 'boolean' } } } }, { type: 'function', name: 'run_terminal_cmd', description: 'Run a shell command in the workspace cwd.', parameters: { type: 'object', properties: { command: { type: 'string' }, timeout_ms: { type: 'number' } }, required: ['command'] } }, { type: 'function', name: 'todo_write', description: 'Merge or replace session todos. Status: pending | in_progress | completed | cancelled.', parameters: { type: 'object', properties: { todos: { type: 'array', items: { type: 'object', properties: { id: { type: 'string' }, content: { type: 'string' }, status: { type: 'string', enum: ['pending', 'in_progress', 'completed', 'cancelled'] } } } }, merge: { type: 'boolean', description: 'If true (default), merge by id. If false, replace the list.' } }, required: ['todos'] } }, - { type: 'function', name: 'web_search', description: 'Search the public web (DuckDuckGo HTML).', parameters: { type: 'object', properties: { query: { type: 'string' } }, required: ['query'] } }, - { type: 'function', name: 'web_fetch', description: 'Fetch any http or https URL as text, including public internet hosts.', parameters: { type: 'object', properties: { url: { type: 'string' } }, required: ['url'] } }, + { type: 'function', name: 'web_search', description: 'Search the public web with no API key. Default engine auto walks DuckDuckGo, Jina, Bing, Google, then Wikipedia. Pin engine to retry one backend: auto, duckduckgo, ddg_lite, ddg_instant, google, bing, bing_rss, jina, wikipedia, hn, github, npm, mdn, stackoverflow, arxiv.', parameters: { type: 'object', properties: { query: { type: 'string' }, engine: { type: 'string' }, limit: { type: 'number' } }, required: ['query'] } }, + { type: 'function', name: 'google_search', description: 'Same as web_search but tries Google HTML first, then the auto fallback chain.', parameters: { type: 'object', properties: { query: { type: 'string' }, limit: { type: 'number' } }, required: ['query'] } }, + { type: 'function', name: 'fetch_page', description: 'Fetch a URL as readable text. Tries Jina Reader, then a direct HTML strip. Use for articles. Use web_fetch for raw pages and I P lookups.', parameters: { type: 'object', properties: { url: { type: 'string' } }, required: ['url'] } }, + { type: 'function', name: 'web_fetch', description: 'Fetch any http or https URL as stripped text, including public internet hosts. Use this for I P lookup pages such as ifconfig.me.', parameters: { type: 'object', properties: { url: { type: 'string' } }, required: ['url'] } }, + { type: 'function', name: 'wiki_search', description: 'Search Wikipedia (official MediaWiki JSON, no key).', parameters: { type: 'object', properties: { query: { type: 'string' }, limit: { type: 'number' } }, required: ['query'] } }, + { type: 'function', name: 'hn_search', description: 'Search Hacker News discussions (Algolia, no key).', parameters: { type: 'object', properties: { query: { type: 'string' }, limit: { type: 'number' } }, required: ['query'] } }, + { type: 'function', name: 'code_search', description: 'Search GitHub repositories, npm packages, and MDN docs in parallel (no key).', parameters: { type: 'object', properties: { query: { type: 'string' }, limit: { type: 'number' } }, required: ['query'] } }, { type: 'function', name: 'memory_search', description: 'Search local agent memory notes.', parameters: { type: 'object', properties: { query: { type: 'string' } } } }, { type: 'function', name: 'memory_get', description: 'Read a memory note by name.', parameters: { type: 'object', properties: { name: { type: 'string' } }, required: ['name'] } }, { type: 'function', name: 'memory_write', description: 'Write a local agent memory note.', parameters: { type: 'object', properties: { name: { type: 'string' }, text: { type: 'string' } }, required: ['name', 'text'] } }, @@ -239,138 +244,22 @@ function defs(opts) { return toolSet.filterBuiltinSchemas(SCHEMAS, opts); } -const WEB_TIMEOUT_MS = 12000; -const BROWSER_UA = - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; +const WEB_TIMEOUT_MS = web.WEB_TIMEOUT_MS; +const BROWSER_UA = web.BROWSER_UA; +const GOOGLE_UA = web.GOOGLE_UA; +const fetchWithTimeout = web.fetchWithTimeout; +const htmlToText = web.htmlToText; +const decodeSearchUrl = web.decodeSearchUrl; +const parseGoogleHits = web.parseGoogleHits; +const parseBingHits = web.parseBingHits; +const duckDuckGoSearch = web.duckDuckGoSearch; +const bingSearch = web.bingSearch; +const googleSearch = web.googleSearch; +const googleSearchWithFallback = web.googleSearchWithFallback; +const webSearch = web.webSearch; +const webFetch = web.webFetch; +const fetchPage = web.fetchPage; -function abortError(timeoutMs) { - const err = new Error('timed out after ' + timeoutMs + 'ms'); - err.name = 'AbortError'; - return err; -} - -function fetchWithTimeout(url, opts, timeoutMs) { - const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS; - const headers = Object.assign({ 'user-agent': BROWSER_UA }, (opts && opts.headers) || {}); - const controller = typeof AbortController === 'function' ? new AbortController() : null; - let timer; - const init = Object.assign({}, opts || {}, { headers }); - if (controller) init.signal = controller.signal; - const timeout = new Promise((_, reject) => { - timer = setTimeout(() => { - try { - if (controller) controller.abort(); - } catch (_) {} - reject(abortError(ms)); - }, ms); - }); - const pending = fetch(url, init); - pending.catch(() => {}); - return Promise.race([pending, timeout]).finally(() => { - if (timer) clearTimeout(timer); - }); -} - -function readBodyWithTimeout(res, timeoutMs) { - const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS; - if (!res || typeof res.text !== 'function') return Promise.resolve(''); - let timer; - const timeout = new Promise((_, reject) => { - timer = setTimeout(() => reject(abortError(ms)), ms); - }); - const pending = res.text(); - pending.catch(() => {}); - return Promise.race([pending, timeout]).finally(() => { - if (timer) clearTimeout(timer); - }); -} - -function decodeSearchUrl(href) { - let raw = String(href || '').replace(/&/g, '&').trim(); - if (!raw) return raw; - if (raw.startsWith('//')) raw = 'https:' + raw; - try { - const u = new URL(raw); - const host = u.hostname.replace(/^www\./, ''); - if (host === 'duckduckgo.com') { - const uddg = u.searchParams.get('uddg'); - if (uddg) { - let dest = String(uddg); - try { - dest = decodeURIComponent(dest); - } catch (_) {} - dest = dest.replace(/&/g, '&'); - if (dest.startsWith('//')) dest = 'https:' + dest; - return dest; - } - } - return u.toString(); - } catch (_) { - return raw; - } -} - -async function webSearch(query, timeoutMs) { - const net = require('../lib/net.js'); - const url = 'https://html.duckduckgo.com/html/?q=' + encodeURIComponent(query); - try { - net.assertPublicHttpUrl(url); - } catch (err) { - return { error: String(err && err.message || err), url }; - } - try { - const res = await fetchWithTimeout(url, {}, timeoutMs); - if (res.status >= 400) { - return { error: 'HTTP ' + res.status, url: String(res.url || url), status: res.status }; - } - const text = await readBodyWithTimeout(res, timeoutMs); - const hits = []; - const re = /]*class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi; - let m; - while ((m = re.exec(text)) && hits.length < 8) { - hits.push({ url: decodeSearchUrl(m[1]), title: m[2].replace(/<[^>]+>/g, '').trim() }); - } - return hits; - } catch (err) { - return { error: String(err && err.message || err), url }; - } -} - -function htmlToText(html) { - const raw = String(html || ''); - if (!/<(?:html|body|div|p|script|head)\b/i.test(raw) && !//gi, ' ') - .replace(//gi, ' ') - .replace(/<[^>]+>/g, ' ') - .replace(/ /gi, ' ') - .replace(/&/gi, '&') - .replace(/</gi, '<') - .replace(/>/gi, '>') - .replace(/\s+/g, ' ') - .trim(); -} - -async function webFetch(url, timeoutMs) { - const net = require('../lib/net.js'); - try { - net.assertHttpUrl(url); - } catch (err) { - return { error: String(err && err.message || err), url: String(url || '') }; - } - try { - const res = await fetchWithTimeout(url, {}, timeoutMs); - let text = htmlToText(await readBodyWithTimeout(res, timeoutMs)); - text = truncate.truncateWithMarker(text, 12000); - const href = String(res.url || url); - if (res.status >= 400) { - return { error: 'HTTP ' + res.status, url: href, status: res.status, text }; - } - return { status: res.status, url: href, text }; - } catch (err) { - return { error: String(err && err.message || err), url: String(url) }; - } -} async function execute(ctx, name, args) { const origin = ctx.origin; @@ -460,9 +349,27 @@ async function execute(ctx, name, args) { return { ok: true, todos: ctx.session.plan }; } case 'web_search': - return webSearch(args.query); + return web.runWebSearch(args.query, { + engine: args.engine, + limit: args.limit, + timeoutMs: args.timeout_ms || args.timeoutMs, + }); + case 'google_search': + return web.runWebSearch(args.query, { + prefer: ['google'], + limit: args.limit, + timeoutMs: args.timeout_ms || args.timeoutMs, + }); + case 'fetch_page': + return web.fetchPage(args.url, args.timeout_ms || args.timeoutMs); case 'web_fetch': - return webFetch(args.url); + return web.webFetch(args.url, args.timeout_ms || args.timeoutMs); + case 'wiki_search': + return web.runWebSearch(args.query, { engine: 'wikipedia', limit: args.limit, timeoutMs: args.timeout_ms || args.timeoutMs }); + case 'hn_search': + return web.runWebSearch(args.query, { engine: 'hn', limit: args.limit, timeoutMs: args.timeout_ms || args.timeoutMs }); + case 'code_search': + return web.codeSearch(args.query, args.timeout_ms || args.timeoutMs, args.limit); case 'memory_search': return memory.search(origin, args.query); case 'memory_get': @@ -542,10 +449,23 @@ module.exports = { runShell, formatShellResult, webFetch, + fetchPage, webSearch, + googleSearch, + duckDuckGoSearch, + bingSearch, + googleSearchWithFallback, + parseGoogleHits, + parseBingHits, decodeSearchUrl, htmlToText, fetchWithTimeout, WEB_TIMEOUT_MS, BROWSER_UA, + GOOGLE_UA, + runWebSearch: web.runWebSearch, + wikiSearch: web.wikiSearch, + hnSearch: web.hnSearch, + codeSearch: web.codeSearch, + ENGINE_NAMES: web.ENGINE_NAMES, }; diff --git a/vendor/agent-harness/agent/web-search.js b/vendor/agent-harness/agent/web-search.js new file mode 100644 index 0000000..361da22 --- /dev/null +++ b/vendor/agent-harness/agent/web-search.js @@ -0,0 +1,779 @@ +/** + * Zero-key public search / page fetch for Bare (no cheerio, jsdom, Playwright). + * Official JSON APIs plus HTML/RSS scrapes. Scrapers break; the auto chain + * walks several backends and the agent can pin `engine` to retry one. + */ + +const net = require('../lib/net.js'); +const truncate = require('./truncate.js'); + +const WEB_TIMEOUT_MS = 12000; +const PAGE_TIMEOUT_MS = 20000; +const BROWSER_UA = + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'; +const GOOGLE_UA = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0'; +const AGENT_UA = 'Jarvis-QVAC/1.0 (local GNOME voice assistant)'; + +const ENGINE_NAMES = [ + 'auto', + 'duckduckgo', + 'ddg_lite', + 'ddg_instant', + 'google', + 'bing', + 'bing_rss', + 'jina', + 'wikipedia', + 'hn', + 'github', + 'npm', + 'mdn', + 'stackoverflow', + 'arxiv', +]; + +const AUTO_ENGINES = [ + 'duckduckgo', + 'ddg_lite', + 'jina', + 'bing', + 'bing_rss', + 'google', + 'wikipedia', + 'ddg_instant', +]; + +const ENGINE_ALIASES = { + ddg: 'duckduckgo', + ddg_html: 'duckduckgo', + wiki: 'wikipedia', + wikipedia: 'wikipedia', + hackernews: 'hn', + 'hacker-news': 'hn', + so: 'stackoverflow', + stackoverflow: 'stackoverflow', +}; + +function abortError(timeoutMs) { + const err = new Error('timed out after ' + timeoutMs + 'ms'); + err.name = 'AbortError'; + return err; +} + +function fetchWithTimeout(url, opts, timeoutMs) { + const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS; + const headers = Object.assign({ 'user-agent': BROWSER_UA }, (opts && opts.headers) || {}); + const controller = typeof AbortController === 'function' ? new AbortController() : null; + let timer; + const init = Object.assign({}, opts || {}, { headers }); + if (controller) init.signal = controller.signal; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => { + try { + if (controller) controller.abort(); + } catch (_) {} + reject(abortError(ms)); + }, ms); + }); + const pending = fetch(url, init); + pending.catch(() => {}); + return Promise.race([pending, timeout]).finally(() => { + if (timer) clearTimeout(timer); + }); +} + +function readBodyWithTimeout(res, timeoutMs) { + const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS; + if (!res || typeof res.text !== 'function') return Promise.resolve(''); + let timer; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(abortError(ms)), ms); + }); + const pending = res.text(); + pending.catch(() => {}); + return Promise.race([pending, timeout]).finally(() => { + if (timer) clearTimeout(timer); + }); +} + +function stripSearchHtml(s) { + return String(s || '') + .replace(//gi, '$1') + .replace(/<[^>]+>/g, ' ') + .replace(/ /gi, ' ') + .replace(/&/gi, '&') + .replace(/"/gi, '"') + .replace(/'/g, "'") + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/\s+/g, ' ') + .trim(); +} + +function htmlToText(html) { + const raw = String(html || ''); + if (!/<(?:html|body|div|p|script|head)\b/i.test(raw) && !//gi, ' ') + .replace(//gi, ' ') + .replace(/<[^>]+>/g, ' ') + .replace(/ /gi, ' ') + .replace(/&/gi, '&') + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/\s+/g, ' ') + .trim(); +} + +function decodeSearchUrl(href) { + let raw = String(href || '').replace(/&/g, '&').trim(); + if (!raw) return raw; + if (raw.startsWith('/url?') || raw.startsWith('/search?')) raw = 'https://www.google.com' + raw; + if (raw.startsWith('//')) raw = 'https:' + raw; + try { + const u = new URL(raw); + const host = u.hostname.replace(/^www\./, ''); + if (host === 'duckduckgo.com') { + const uddg = u.searchParams.get('uddg'); + if (uddg) { + let dest = String(uddg); + try { + dest = decodeURIComponent(dest); + } catch (_) {} + dest = dest.replace(/&/g, '&'); + if (dest.startsWith('//')) dest = 'https:' + dest; + return dest; + } + } + if (host === 'google.com' || host.endsWith('.google.com')) { + const dest = u.searchParams.get('q') || u.searchParams.get('url'); + if (dest) { + let out = String(dest); + try { + out = decodeURIComponent(out); + } catch (_) {} + out = out.replace(/&/g, '&'); + if (out.startsWith('//')) out = 'https:' + out; + if (/^https?:\/\//i.test(out)) return out; + } + } + return u.toString(); + } catch (_) { + return raw; + } +} + +function isOrganicResultUrl(href) { + let u; + try { + u = new URL(href); + } catch (_) { + return false; + } + if (u.protocol !== 'http:' && u.protocol !== 'https:') return false; + const h = u.hostname.replace(/^www\./, '').toLowerCase(); + if (h === 'google.com') return false; + if (h === 'googleusercontent.com' || h.endsWith('.googleusercontent.com')) return false; + if (h === 'gstatic.com' || h.endsWith('.gstatic.com')) return false; + if (h === 'bing.com' || h.endsWith('.bing.com')) return false; + if (h === 'duckduckgo.com' && u.pathname.indexOf('/y.js') === 0) return false; + if (h === 'youtube.com' && u.pathname.indexOf('/redirect') === 0) return false; + return true; +} + +function decodeBase64Utf8(raw) { + const s = String(raw || ''); + try { + if (typeof Buffer !== 'undefined') return Buffer.from(s, 'base64').toString('utf8'); + } catch (_) {} + try { + if (typeof atob === 'function') return atob(s); + } catch (_) {} + return ''; +} + +function decodeBingClickUrl(href) { + const raw = String(href || '').replace(/&/g, '&').trim(); + try { + const u = new URL(raw, 'https://www.bing.com'); + const host = u.hostname.replace(/^www\./, ''); + if (host === 'bing.com' || host.endsWith('.bing.com')) { + const dest = u.searchParams.get('u'); + if (dest) { + let payload = dest; + if (/^a1/i.test(payload)) payload = payload.slice(2); + const decoded = decodeBase64Utf8(payload); + if (/^https?:\/\//i.test(decoded)) return decoded; + } + } + } catch (_) {} + return decodeSearchUrl(href); +} + +function searchHasHits(result) { + return Array.isArray(result) && result.length > 0; +} + +function clampLimit(limit) { + const n = Number(limit); + if (!n || n < 1) return 8; + return n > 15 ? 15 : Math.floor(n); +} + +function tagSearchHits(hits, source) { + return hits.map((hit) => Object.assign({ source: hit.source || source }, hit)); +} + +function pushHit(hits, seen, href, title, snippet, limit) { + const url = decodeSearchUrl(href); + if (!isOrganicResultUrl(url)) return; + const key = url.split('#')[0]; + if (seen.has(key)) return; + seen.add(key); + const item = { url, title: stripSearchHtml(title) || url }; + const snip = stripSearchHtml(snippet); + if (snip) item.snippet = snip; + hits.push(item); +} + +async function fetchText(url, timeoutMs, opts) { + try { + net.assertPublicHttpUrl(url); + } catch (err) { + return { error: String(err && err.message || err), url }; + } + try { + const res = await fetchWithTimeout(url, opts || {}, timeoutMs); + const text = await readBodyWithTimeout(res, timeoutMs); + if (res.status >= 400) { + return { error: 'HTTP ' + res.status, url: String(res.url || url), status: res.status, text }; + } + return { url: String(res.url || url), text, status: res.status }; + } catch (err) { + return { error: String(err && err.message || err), url }; + } +} + +async function fetchJson(url, timeoutMs, opts) { + const page = await fetchText(url, timeoutMs, opts); + if (page.error) return page; + try { + return { url: page.url, json: JSON.parse(page.text), status: page.status }; + } catch (_) { + return { error: 'invalid json', url: page.url, text: page.text }; + } +} + +function parseGoogleHits(html, limit) { + const text = String(html || ''); + const max = clampLimit(limit); + const hits = []; + const seen = new Set(); + const cardRe = /]+href="([^"]+)"[^>]*>[\s\S]*?
    ]*>([\s\S]*?)<\/div>/gi; + let m; + while ((m = cardRe.exec(text)) && hits.length < max) pushHit(hits, seen, m[1], m[2], '', max); + const deskRe = /]*class="[^"]*yuRUbf[^"]*"[^>]*>[\s\S]*?]+href="([^"]+)"[^>]*>[\s\S]*?]*>([\s\S]*?)<\/h3>/gi; + while ((m = deskRe.exec(text)) && hits.length < max) pushHit(hits, seen, m[1], m[2], '', max); + const urlqRe = /\/url\?q=(https?:\/\/[^&"'<>]+)/gi; + while ((m = urlqRe.exec(text)) && hits.length < max) { + let dest = m[1]; + try { + dest = decodeURIComponent(dest); + } catch (_) {} + pushHit(hits, seen, dest, dest, '', max); + } + return hits.slice(0, max); +} + +function parseBingHits(html, limit) { + const text = String(html || ''); + const max = clampLimit(limit); + const hits = []; + const seen = new Set(); + const re = /
  • ]*>[\s\S]*?]*>\s*]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi; + let m; + while ((m = re.exec(text)) && hits.length < max) { + const url = decodeBingClickUrl(m[1]); + if (!isOrganicResultUrl(url)) continue; + const key = url.split('#')[0]; + if (seen.has(key)) continue; + seen.add(key); + hits.push({ url, title: stripSearchHtml(m[2]) || url }); + } + return hits; +} + +function parseDdgHtmlHits(html, limit) { + const text = String(html || ''); + const max = clampLimit(limit); + const hits = []; + const seen = new Set(); + const re = /]*class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi; + let m; + while ((m = re.exec(text)) && hits.length < max) { + const after = text.slice(m.index, m.index + 800); + const snip = (after.match(/class="result__snippet[^"]*"[^>]*>([\s\S]*?)<\/(?:td|div|a|span)>/i) || [])[1] || ''; + pushHit(hits, seen, m[1], m[2], snip, max); + } + return hits; +} + +function parseDdgLiteHits(html, limit) { + const text = String(html || ''); + const max = clampLimit(limit); + const hits = []; + const seen = new Set(); + const re = /]+(?:class="[^"]*result-link[^"]*"|rel="nofollow")[^>]*href="(https?:[^"]+)"[^>]*>([\s\S]*?)<\/a>/gi; + let m; + while ((m = re.exec(text)) && hits.length < max) pushHit(hits, seen, m[1], m[2], '', max); + return hits; +} + +function parseRssItems(xml, limit) { + const text = String(xml || ''); + const max = clampLimit(limit); + const hits = []; + const seen = new Set(); + const re = /([\s\S]*?)<\/item>/gi; + let m; + while ((m = re.exec(text)) && hits.length < max) { + const block = m[1]; + const title = stripSearchHtml((block.match(/([\s\S]*?)<\/title>/i) || [])[1] || ''); + let link = stripSearchHtml((block.match(/<link>([\s\S]*?)<\/link>/i) || [])[1] || ''); + const desc = stripSearchHtml((block.match(/<description>([\s\S]*?)<\/description>/i) || [])[1] || ''); + if (!link) continue; + link = decodeBingClickUrl(link); + if (!isOrganicResultUrl(link)) continue; + const key = link.split('#')[0]; + if (seen.has(key)) continue; + seen.add(key); + const item = { url: link, title: title || link }; + if (desc) item.snippet = desc.slice(0, 280); + hits.push(item); + } + return hits; +} + +function parseAtomEntries(xml, limit) { + const text = String(xml || ''); + const max = clampLimit(limit); + const hits = []; + const re = /<entry>([\s\S]*?)<\/entry>/gi; + let m; + while ((m = re.exec(text)) && hits.length < max) { + const block = m[1]; + const title = stripSearchHtml((block.match(/<title[^>]*>([\s\S]*?)<\/title>/i) || [])[1] || ''); + const linkM = block.match(/<link[^>]+href="([^"]+)"/i) || block.match(/<id>([\s\S]*?)<\/id>/i); + const url = linkM ? String(linkM[1]).trim() : ''; + const summary = stripSearchHtml((block.match(/<(?:summary|content)[^>]*>([\s\S]*?)<\/(?:summary|content)>/i) || [])[1] || ''); + if (!url || !/^https?:\/\//i.test(url)) continue; + const item = { url, title: title || url }; + if (summary) item.snippet = summary.slice(0, 280); + hits.push(item); + } + return hits; +} + +function isDdgChallenge(html) { + const text = String(html || ''); + return /anomaly-modal|Unfortunately, bots use DuckDuckGo/i.test(text) && !/result__a/i.test(text); +} + +async function duckDuckGoSearch(query, timeoutMs, limit) { + const url = 'https://html.duckduckgo.com/html/'; + const body = 'q=' + encodeURIComponent(query) + '&b=&kl=us-en'; + let page = await fetchText(url, timeoutMs, { + method: 'POST', + headers: { + 'content-type': 'application/x-www-form-urlencoded', + 'user-agent': BROWSER_UA, + accept: 'text/html', + }, + body, + }); + if (!page.error) { + if (isDdgChallenge(page.text)) return { error: 'duckduckgo bot challenge', url: page.url, status: page.status }; + const posted = parseDdgHtmlHits(page.text, limit); + if (searchHasHits(posted)) return posted; + } + page = await fetchText(url + '?q=' + encodeURIComponent(query), timeoutMs); + if (page.error) return page; + if (isDdgChallenge(page.text)) return { error: 'duckduckgo bot challenge', url: page.url, status: page.status }; + return parseDdgHtmlHits(page.text, limit); +} + +async function ddgLiteSearch(query, timeoutMs, limit) { + const url = 'https://lite.duckduckgo.com/lite/?q=' + encodeURIComponent(query); + const page = await fetchText(url, timeoutMs); + if (page.error) return page; + if (isDdgChallenge(page.text)) return { error: 'duckduckgo bot challenge', url: page.url, status: page.status }; + const hits = parseDdgLiteHits(page.text, limit); + if (searchHasHits(hits)) return hits; + return parseDdgHtmlHits(page.text, limit); +} + +async function ddgInstantSearch(query, timeoutMs, limit) { + const url = + 'https://api.duckduckgo.com/?q=' + + encodeURIComponent(query) + + '&format=json&no_html=1&skip_disambig=1'; + const page = await fetchJson(url, timeoutMs, { headers: { accept: 'application/json', 'user-agent': AGENT_UA } }); + if (page.error) return page; + const j = page.json || {}; + const hits = []; + if (j.AbstractURL && (j.AbstractText || j.Heading)) { + hits.push({ + url: j.AbstractURL, + title: j.Heading || j.AbstractURL, + snippet: j.AbstractText || '', + }); + } + const related = j.RelatedTopics || []; + for (let i = 0; i < related.length && hits.length < clampLimit(limit); i++) { + const row = related[i]; + const topics = row.Topics || [row]; + for (let t = 0; t < topics.length && hits.length < clampLimit(limit); t++) { + const item = topics[t]; + if (!item || !item.FirstURL) continue; + hits.push({ url: item.FirstURL, title: stripSearchHtml(item.Text || item.FirstURL), snippet: item.Text || '' }); + } + } + return hits; +} + +async function googleSearch(query, timeoutMs, limit) { + const url = + 'https://www.google.com/search?q=' + + encodeURIComponent(query) + + '&num=' + + clampLimit(limit) + + '&hl=en&pws=0&gbv=1'; + const page = await fetchText(url, timeoutMs, { headers: { 'user-agent': GOOGLE_UA } }); + if (page.error) return page; + const hits = parseGoogleHits(page.text, limit); + if (searchHasHits(hits)) return hits; + if (/enablejs|Please click/i.test(page.text || '')) return { error: 'google javascript challenge', url: page.url }; + return hits; +} + +async function bingSearch(query, timeoutMs, limit) { + const url = 'https://www.bing.com/search?q=' + encodeURIComponent(query); + const page = await fetchText(url, timeoutMs); + if (page.error) return page; + return parseBingHits(page.text, limit); +} + +async function bingRssSearch(query, timeoutMs, limit) { + const url = 'https://www.bing.com/search?q=' + encodeURIComponent(query) + '&format=rss'; + const page = await fetchText(url, timeoutMs, { headers: { accept: 'application/rss+xml, application/xml, text/xml, */*' } }); + if (page.error) return page; + return parseRssItems(page.text, limit); +} + +async function jinaSearch(query, timeoutMs, limit) { + const url = 'https://s.jina.ai/' + encodeURIComponent(query); + const page = await fetchJson(url, timeoutMs, { + headers: { accept: 'application/json', 'user-agent': AGENT_UA }, + }); + if (page.error) return page; + const j = page.json || {}; + let rows = j.data || j.results || []; + if (rows && !Array.isArray(rows) && Array.isArray(rows.results)) rows = rows.results; + if (!Array.isArray(rows)) rows = []; + return rows.slice(0, clampLimit(limit)).map((row) => ({ + url: row.url || row.link, + title: row.title || row.url, + snippet: stripSearchHtml(row.description || row.content || '').slice(0, 280), + })).filter((h) => h.url); +} + +async function wikiSearch(query, timeoutMs, limit) { + const url = + 'https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch=' + + encodeURIComponent(query) + + '&srlimit=' + + clampLimit(limit) + + '&format=json&utf8=1'; + const page = await fetchJson(url, timeoutMs, { + headers: { accept: 'application/json', 'user-agent': AGENT_UA }, + }); + if (page.error) return page; + const rows = (page.json && page.json.query && page.json.query.search) || []; + return rows.map((row) => ({ + url: 'https://en.wikipedia.org/wiki/' + encodeURIComponent(String(row.title || '').replace(/ /g, '_')), + title: row.title, + snippet: stripSearchHtml(row.snippet || ''), + })); +} + +async function hnSearch(query, timeoutMs, limit) { + const url = + 'https://hn.algolia.com/api/v1/search?query=' + + encodeURIComponent(query) + + '&tags=story&hitsPerPage=' + + clampLimit(limit); + const page = await fetchJson(url, timeoutMs, { headers: { accept: 'application/json', 'user-agent': AGENT_UA } }); + if (page.error) return page; + const rows = (page.json && page.json.hits) || []; + return rows.map((row) => ({ + url: row.url || 'https://news.ycombinator.com/item?id=' + row.objectID, + title: row.title || row.story_title || String(row.objectID), + snippet: (row.author ? 'by ' + row.author + '. ' : '') + (row.points != null ? row.points + ' points' : ''), + points: row.points, + comments: row.num_comments, + })); +} + +async function githubSearch(query, timeoutMs, limit) { + const url = + 'https://api.github.com/search/repositories?q=' + + encodeURIComponent(query) + + '&per_page=' + + clampLimit(limit); + const page = await fetchJson(url, timeoutMs, { + headers: { + 'user-agent': AGENT_UA, + accept: 'application/vnd.github+json', + }, + }); + if (page.error) return page; + const rows = (page.json && page.json.items) || []; + return rows.map((row) => ({ + url: row.html_url, + title: row.full_name || row.name, + snippet: row.description || '', + })); +} + +async function npmSearch(query, timeoutMs, limit) { + const url = + 'https://registry.npmjs.org/-/v1/search?text=' + + encodeURIComponent(query) + + '&size=' + + clampLimit(limit); + const page = await fetchJson(url, timeoutMs, { headers: { accept: 'application/json', 'user-agent': AGENT_UA } }); + if (page.error) return page; + const rows = (page.json && page.json.objects) || []; + return rows.map((row) => { + const pkg = row.package || {}; + return { + url: 'https://www.npmjs.com/package/' + pkg.name, + title: pkg.name, + snippet: pkg.description || '', + version: pkg.version, + }; + }); +} + +async function mdnSearch(query, timeoutMs, limit) { + const url = 'https://developer.mozilla.org/api/v1/search?q=' + encodeURIComponent(query); + const page = await fetchJson(url, timeoutMs, { headers: { accept: 'application/json', 'user-agent': AGENT_UA } }); + if (page.error) return page; + const rows = (page.json && page.json.documents) || []; + return rows.slice(0, clampLimit(limit)).map((row) => ({ + url: row.mdn_url ? 'https://developer.mozilla.org' + row.mdn_url : row.url, + title: row.title, + snippet: stripSearchHtml(row.summary || ''), + })).filter((h) => h.url); +} + +async function stackOverflowSearch(query, timeoutMs, limit) { + const url = + 'https://api.stackexchange.com/2.3/search/advanced?order=desc&sort=relevance&site=stackoverflow&q=' + + encodeURIComponent(query) + + '&pagesize=' + + clampLimit(limit); + const page = await fetchJson(url, timeoutMs, { headers: { accept: 'application/json', 'user-agent': AGENT_UA } }); + if (page.error) return page; + const rows = (page.json && page.json.items) || []; + return rows.map((row) => ({ + url: row.link, + title: stripSearchHtml(row.title || ''), + snippet: row.score != null ? String(row.score) + ' score' : '', + })).filter((h) => h.url); +} + +async function arxivSearch(query, timeoutMs, limit) { + const url = + 'https://export.arxiv.org/api/query?search_query=all:' + + encodeURIComponent(query) + + '&start=0&max_results=' + + clampLimit(limit); + const page = await fetchText(url, timeoutMs, { headers: { accept: 'application/atom+xml, application/xml, text/xml', 'user-agent': AGENT_UA } }); + if (page.error) return page; + return parseAtomEntries(page.text, limit); +} + +const SEARCH_ENGINES = { + duckduckgo: duckDuckGoSearch, + ddg_lite: ddgLiteSearch, + ddg_instant: ddgInstantSearch, + google: googleSearch, + bing: bingSearch, + bing_rss: bingRssSearch, + jina: jinaSearch, + wikipedia: wikiSearch, + hn: hnSearch, + github: githubSearch, + npm: npmSearch, + mdn: mdnSearch, + stackoverflow: stackOverflowSearch, + arxiv: arxivSearch, +}; + +function resolveEngine(name) { + const raw = String(name || 'auto').trim().toLowerCase(); + if (!raw || raw === 'auto') return 'auto'; + return ENGINE_ALIASES[raw] || raw; +} + +async function runWebSearch(query, opts) { + opts = opts || {}; + const q = String(query || '').trim(); + if (!q) return { error: 'query required' }; + const limit = clampLimit(opts.limit); + const timeoutMs = opts.timeoutMs; + const engine = resolveEngine(opts.engine); + if (engine !== 'auto') { + const fn = SEARCH_ENGINES[engine]; + if (!fn) return { error: 'unknown engine', engine: opts.engine, engines: ENGINE_NAMES }; + const result = await fn(q, timeoutMs, limit); + if (searchHasHits(result)) return tagSearchHits(result, engine).slice(0, limit); + return { + error: (result && result.error) || 'no search results', + url: result && result.url, + tried: [engine], + engines: ENGINE_NAMES, + }; + } + const prefer = Array.isArray(opts.prefer) ? opts.prefer.map(resolveEngine).filter((n) => SEARCH_ENGINES[n]) : []; + const chain = prefer.concat(AUTO_ENGINES.filter((name) => prefer.indexOf(name) < 0)); + const tried = []; + const errors = {}; + for (let i = 0; i < chain.length; i++) { + const name = chain[i]; + tried.push(name); + const result = await SEARCH_ENGINES[name](q, timeoutMs, limit); + if (searchHasHits(result)) return tagSearchHits(result, name).slice(0, limit); + errors[name] = result && result.error ? result.error : 'no results'; + } + return { error: 'no search results', tried, errors, engines: ENGINE_NAMES }; +} + +async function googleSearchWithFallback(query, timeoutMs) { + return runWebSearch(query, { timeoutMs, prefer: ['google'] }); +} + +async function webSearch(query, timeoutMs) { + return runWebSearch(query, { timeoutMs }); +} + +async function codeSearch(query, timeoutMs, limit) { + const q = String(query || '').trim(); + if (!q) return { error: 'query required' }; + const [github, npm, mdn] = await Promise.all([ + githubSearch(q, timeoutMs, limit), + npmSearch(q, timeoutMs, limit), + mdnSearch(q, timeoutMs, limit), + ]); + const out = { github: [], npm: [], mdn: [] }; + if (searchHasHits(github)) out.github = tagSearchHits(github, 'github'); + else if (github && github.error) out.github_error = github.error; + if (searchHasHits(npm)) out.npm = tagSearchHits(npm, 'npm'); + else if (npm && npm.error) out.npm_error = npm.error; + if (searchHasHits(mdn)) out.mdn = tagSearchHits(mdn, 'mdn'); + else if (mdn && mdn.error) out.mdn_error = mdn.error; + if (!out.github.length && !out.npm.length && !out.mdn.length) { + return { error: 'no code search results', github_error: out.github_error, npm_error: out.npm_error, mdn_error: out.mdn_error }; + } + return out; +} + +async function webFetch(url, timeoutMs) { + try { + net.assertHttpUrl(url); + } catch (err) { + return { error: String(err && err.message || err), url: String(url || '') }; + } + try { + const res = await fetchWithTimeout(url, {}, timeoutMs); + let text = htmlToText(await readBodyWithTimeout(res, timeoutMs)); + text = truncate.truncateWithMarker(text, 12000); + const href = String(res.url || url); + if (res.status >= 400) { + return { error: 'HTTP ' + res.status, url: href, status: res.status, text, via: 'raw' }; + } + return { status: res.status, url: href, text, via: 'raw' }; + } catch (err) { + return { error: String(err && err.message || err), url: String(url), via: 'raw' }; + } +} + +async function fetchPage(url, timeoutMs) { + try { + net.assertHttpUrl(url); + } catch (err) { + return { error: String(err && err.message || err), url: String(url || '') }; + } + const target = String(url); + const jinaUrl = 'https://r.jina.ai/' + target; + const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : PAGE_TIMEOUT_MS; + try { + net.assertPublicHttpUrl(jinaUrl); + const res = await fetchWithTimeout(jinaUrl, { headers: { accept: 'text/plain', 'user-agent': AGENT_UA } }, ms); + if (res.status < 400) { + let text = await readBodyWithTimeout(res, ms); + text = truncate.truncateWithMarker(text, 12000); + if (text && text.length > 24 && !/verifying you are (a )?human/i.test(text)) { + return { status: res.status, url: target, text, via: 'jina' }; + } + } + } catch (_) {} + const raw = await webFetch(target, timeoutMs); + if (raw && !raw.via) raw.via = 'raw'; + return raw; +} + +module.exports = { + WEB_TIMEOUT_MS, + PAGE_TIMEOUT_MS, + BROWSER_UA, + GOOGLE_UA, + AGENT_UA, + ENGINE_NAMES, + AUTO_ENGINES, + SEARCH_ENGINES, + fetchWithTimeout, + readBodyWithTimeout, + stripSearchHtml, + htmlToText, + decodeSearchUrl, + decodeBingClickUrl, + parseGoogleHits, + parseBingHits, + parseDdgHtmlHits, + parseDdgLiteHits, + parseRssItems, + parseAtomEntries, + duckDuckGoSearch, + ddgLiteSearch, + ddgInstantSearch, + googleSearch, + bingSearch, + bingRssSearch, + jinaSearch, + wikiSearch, + hnSearch, + githubSearch, + npmSearch, + mdnSearch, + stackOverflowSearch, + arxivSearch, + runWebSearch, + googleSearchWithFallback, + webSearch, + codeSearch, + webFetch, + fetchPage, +}; diff --git a/vendor/agent-harness/test/test.js b/vendor/agent-harness/test/test.js index 622b662..ea5211c 100644 --- a/vendor/agent-harness/test/test.js +++ b/vendor/agent-harness/test/test.js @@ -262,7 +262,9 @@ testTruncateAndPerm(); testPaths(); testQvacWorkerDeps(); testDevicePrefersGpu(); +testGoogleSearchParseAndFallback(); testWebFetchTimeout() + .then(() => testGoogleSearchFallsBackToDuckDuckGo()) .then(() => { console.log('ok'); }) @@ -310,3 +312,98 @@ async function testWebFetchTimeout() { globalThis.fetch = orig; } } + +function testGoogleSearchParseAndFallback() { + const parsed = tools.parseGoogleHits( + '<a href="/url?q=https://example.com/page&sa=U"><div class="BNeawe vvjwJb AP7Wnd">Example Domain</div></a>' + ); + assert.strictEqual(parsed.length, 1); + assert.strictEqual(parsed[0].url, 'https://example.com/page'); + assert.strictEqual(parsed[0].title, 'Example Domain'); + assert.strictEqual(tools.parseGoogleHits('<title>Google Search').length, 0); + assert.ok(tools.SCHEMAS.find((t) => t.name === 'google_search')); + assert.ok(tools.SCHEMAS.find((t) => t.name === 'fetch_page')); + assert.ok(tools.SCHEMAS.find((t) => t.name === 'wiki_search')); + const rss = require('../agent/web-search.js').parseRssItems( + 'Examplehttps://example.com/rss' + ); + assert.strictEqual(rss[0].url, 'https://example.com/rss'); +} + +async function testGoogleSearchFallsBackToDuckDuckGo() { + const orig = globalThis.fetch; + globalThis.fetch = async (url) => { + const href = String(url); + if (href.indexOf('google.com') >= 0) { + return { + status: 200, + url: href, + text: async () => 'Google Search', + }; + } + return { + status: 200, + url: href, + text: async () => 'DDG Example', + }; + }; + try { + const hits = await tools.webSearch('example domain', 200); + assert.ok(Array.isArray(hits)); + assert.strictEqual(hits.length, 1); + assert.strictEqual(hits[0].source, 'duckduckgo'); + assert.strictEqual(hits[0].url, 'https://example.com/ddg'); + assert.strictEqual(hits[0].title, 'DDG Example'); + } finally { + globalThis.fetch = orig; + } + + globalThis.fetch = async (url) => { + const href = String(url); + if (href.indexOf('google.com') >= 0) { + return { + status: 200, + url: href, + text: async () => + '
    From Google
    ', + }; + } + throw new Error('duckduckgo should not run when google hits'); + }; + try { + const hits = await tools.googleSearchWithFallback('example domain', 200); + assert.strictEqual(hits[0].source, 'google'); + assert.strictEqual(hits[0].url, 'https://example.com/google'); + assert.strictEqual(hits[0].title, 'From Google'); + } finally { + globalThis.fetch = orig; + } + + const bingHref = + 'https://www.bing.com/ck/a?!&&p=ae&u=a1aHR0cDovL3d3dy5leGFtcGxlLmNvbS8&ntb=1'; + globalThis.fetch = async (url) => { + const href = String(url); + if (href.indexOf('google.com') >= 0) { + return { status: 200, url: href, text: async () => 'Google Search' }; + } + if (href.indexOf('duckduckgo.com') >= 0) { + return { + status: 202, + url: href, + text: async () => '
    Unfortunately, bots use DuckDuckGo too.
    ', + }; + } + return { + status: 200, + url: href, + text: async () => '
  • Example Domain

  • ', + }; + }; + try { + const hits = await tools.googleSearchWithFallback('example domain', 200); + assert.strictEqual(hits[0].source, 'bing'); + assert.strictEqual(hits[0].url, 'http://www.example.com/'); + } finally { + globalThis.fetch = orig; + } +}