Updates

browser-use/helper.js

@@ -0,0 +1,337 @@
+#!/usr/bin/env node
+/** Node Playwright sidecar. The Bare daemon never loads Playwright itself. */
+import { createRequire } from 'node:module';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import readline from 'node:readline';
+
+const require = createRequire(import.meta.url);
+
+const SEARCH_BUDGET_MS = 25_000;
+const FETCH_BUDGET_MS = 30_000;
+const CHALLENGE_MS = 20_000;
+const CHALLENGE_RE = /just a moment|attention required|verify (?:that )?you are human|checking your browser|enable javascript|security check|captcha|cloudflare|unusual traffic|please wait/i;
+
+const SEARCH_URLS = {
+  duckduckgo: (q) => `https://duckduckgo.com/?q=${encodeURIComponent(q)}`,
+  ddg_lite: (q) => `https://duckduckgo.com/?q=${encodeURIComponent(q)}`,
+  google: (q) => `https://www.google.com/search?q=${encodeURIComponent(q)}`,
+  bing: (q) => `https://www.bing.com/search?q=${encodeURIComponent(q)}`,
+  bing_rss: (q) => `https://www.bing.com/search?q=${encodeURIComponent(q)}`,
+  wikipedia: (q) => `https://en.wikipedia.org/w/index.php?search=${encodeURIComponent(q)}`,
+  hn: (q) => `https://hn.algolia.com/?q=${encodeURIComponent(q)}`,
+  github: (q) => `https://github.com/search?q=${encodeURIComponent(q)}&type=repositories`,
+  npm: (q) => `https://www.npmjs.com/search?q=${encodeURIComponent(q)}`,
+  mdn: (q) => `https://developer.mozilla.org/en-US/search?q=${encodeURIComponent(q)}`,
+  stackoverflow: (q) => `https://stackoverflow.com/search?q=${encodeURIComponent(q)}`,
+  arxiv: (q) => `https://arxiv.org/search/?query=${encodeURIComponent(q)}&searchtype=all`,
+};
+
+function stateRoot() {
+  return path.join(process.env.XDG_STATE_HOME || path.join(os.homedir(), '.local/state'), 'jarvis', 'browser');
+}
+
+function isBlockedHostname(hostname) {
+  if (!hostname) return true;
+  const h = String(hostname).toLowerCase().replace(/^\[|\]$/g, '');
+  if (h === 'localhost' || h.endsWith('.localhost') || h === '0.0.0.0' || h === '::' || h === '::1' || h === 'metadata.google.internal' || h.endsWith('.internal')) return true;
+  const v4 = h.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
+  if (v4) {
+    const a = Number(v4[1]);
+    const b = Number(v4[2]);
+    if (a === 0 || a === 10 || a === 127) return true;
+    if (a === 169 && b === 254) return true;
+    if (a === 172 && b >= 16 && b <= 31) return true;
+    if (a === 192 && b === 168) return true;
+    if (a === 100 && b >= 64 && b <= 127) return true;
+  }
+  if (h.includes(':') && (h === '::1' || h.startsWith('fe80:') || h.startsWith('fc') || h.startsWith('fd'))) return true;
+  return false;
+}
+
+function assertPublicHttpUrl(raw) {
+  let url;
+  try { url = new URL(String(raw)); } catch { throw new Error('invalid URL'); }
+  if (url.protocol !== 'https:' && url.protocol !== 'http:') throw new Error('only http(s) URLs are allowed');
+  if (isBlockedHostname(url.hostname)) throw new Error('private, loopback, and metadata hosts are blocked');
+  return url.href;
+}
+
+function reply(id, payload) {
+  process.stdout.write(`${JSON.stringify({ id, ...payload })}\n`);
+}
+
+function fail(id, error) {
+  reply(id, { ok: false, error: String(error && error.message || error) });
+}
+
+async function loadPlaywright() {
+  try {
+    return require('playwright');
+  } catch (error) {
+    throw new Error(`Playwright is not installed (${error.message}). From the Jarvis tree run: npm install && npm run browser:install`);
+  }
+}
+
+function challengeText(title, text) {
+  return CHALLENGE_RE.test(String(title || '')) || CHALLENGE_RE.test(String(text || '').slice(0, 1200));
+}
+
+async function waitOutChallenge(page, timeoutMs) {
+  const deadline = Date.now() + Math.max(0, Number(timeoutMs) || 0);
+  while (Date.now() < deadline) {
+    const title = await page.title().catch(() => '');
+    const text = await page.locator('body').innerText({ timeout: 1000 }).catch(() => '');
+    if (!challengeText(title, text)) return false;
+    await page.waitForTimeout(400);
+  }
+  const title = await page.title().catch(() => '');
+  const text = await page.locator('body').innerText({ timeout: 1000 }).catch(() => '');
+  return challengeText(title, text);
+}
+
+async function extractHits(page, limit) {
+  const max = Math.min(15, Math.max(1, Number(limit) || 8));
+  return page.evaluate((cap) => {
+    const out = [];
+    const seen = new Set();
+    const unwrap = (href) => {
+      try {
+        const u = new URL(href);
+        const host = u.hostname.replace(/^www\./, '');
+        const dest = u.searchParams.get('uddg') || u.searchParams.get('url');
+        if (dest && /^https?:/i.test(dest)) return dest;
+        if ((host === 'google.com' || host.endsWith('.google.com')) && u.pathname === '/url') {
+          const q = u.searchParams.get('q');
+          if (q && /^https?:/i.test(q)) return q;
+        }
+        if (['google.com', 'bing.com', 'googleusercontent.com'].includes(host)) return '';
+        return u.href;
+      } catch {
+        return '';
+      }
+    };
+    const push = (href, title, snippet) => {
+      const url = unwrap(href);
+      if (!url || !/^https?:/i.test(url) || seen.has(url)) return;
+      seen.add(url);
+      out.push({
+        url,
+        title: String(title || url).replace(/\s+/g, ' ').trim().slice(0, 200) || url,
+        snippet: String(snippet || '').replace(/\s+/g, ' ').trim().slice(0, 280),
+      });
+    };
+    const blocks = document.querySelectorAll('article[data-testid="result"], [data-testid="result"], li.b_algo, div.g, a.result__a');
+    for (const node of blocks) {
+      const a = node.tagName === 'A' ? node : node.querySelector('a[href]');
+      if (!a) continue;
+      const heading = node.querySelector('h3, h2') || a;
+      const snip = node.querySelector('[data-result="snippet"], .b_caption p, .result__snippet, .VwiC3b');
+      push(a.href, heading.innerText, snip ? snip.innerText : '');
+      if (out.length >= cap) return out;
+    }
+    if (!out.length) {
+      for (const a of document.querySelectorAll('a[href^="http"]')) {
+        const label = (a.innerText || '').replace(/\s+/g, ' ').trim();
+        if (label.length < 8) continue;
+        push(a.href, label, '');
+        if (out.length >= cap) break;
+      }
+    }
+    return out.slice(0, cap);
+  }, max);
+}
+
+async function snapshot(page) {
+  let aria = '';
+  try {
+    aria = await page.locator('body').ariaSnapshot({ timeout: 2000 });
+  } catch {
+    aria = '';
+  }
+  const refs = await page.evaluate(() => {
+    const items = [];
+    const nodes = document.querySelectorAll('a, button, input, textarea, select, [role="button"], [role="link"], [contenteditable="true"]');
+    let i = 1;
+    for (const el of nodes) {
+      if (items.length >= 80) break;
+      const name = (el.innerText || el.value || el.getAttribute('aria-label') || el.getAttribute('placeholder') || el.getAttribute('name') || '').replace(/\s+/g, ' ').trim().slice(0, 120);
+      if (!name && el.tagName !== 'INPUT' && el.tagName !== 'TEXTAREA' && el.tagName !== 'SELECT') continue;
+      el.setAttribute('data-jarvis-ref', String(i));
+      items.push({ ref: String(i), role: (el.getAttribute('role') || el.tagName.toLowerCase()), name, href: el.href || undefined });
+      i += 1;
+    }
+    return items;
+  });
+  return { url: page.url(), title: await page.title(), refs, aria: String(aria || '').slice(0, 8000) };
+}
+
+async function locate(page, input) {
+  const ref = input.ref != null ? String(input.ref) : '';
+  if (ref) {
+    const handle = page.locator(`[data-jarvis-ref="${ref}"]`).first();
+    if (await handle.count()) return handle;
+  }
+  const selector = input.selector != null ? String(input.selector) : '';
+  if (selector) return page.locator(selector).first();
+  const text = input.text != null && !input.submit ? String(input.text) : '';
+  if (text && input.action === 'click') return page.getByText(text, { exact: false }).first();
+  return null;
+}
+
+let context;
+let page;
+
+async function currentPage() {
+  if (page && !page.isClosed()) return page;
+  page = context.pages().find((item) => !item.isClosed()) || await context.newPage();
+  return page;
+}
+
+async function gotoUrl(target, timeoutMs) {
+  const url = assertPublicHttpUrl(target);
+  const tab = await currentPage();
+  const response = await tab.goto(url, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
+  await tab.waitForLoadState('networkidle', { timeout: Math.min(8000, timeoutMs) }).catch(() => {});
+  const challenged = await waitOutChallenge(tab, Math.min(CHALLENGE_MS, timeoutMs));
+  return { tab, url: tab.url(), status: response ? response.status() : 0, challenged };
+}
+
+async function fetchPage(input, timeoutMs) {
+  const opened = await gotoUrl(input.url, timeoutMs);
+  const tab = opened.tab;
+  const url = opened.url;
+  const status = opened.status;
+  const challenged = opened.challenged;
+  const title = await tab.title().catch(() => '');
+  const text = await tab.locator('body').innerText().catch(() => '');
+  let html = '';
+  try { html = await tab.content(); } catch { html = ''; }
+  if (html.length > 2 * 1024 * 1024) html = html.slice(0, 2 * 1024 * 1024);
+  const result = {
+    url,
+    title,
+    status,
+    html,
+    text: String(text || '').slice(0, 30_000),
+    via: 'browser',
+  };
+  if (challenged) {
+    result.challenge = true;
+    result.next_action = 'Complete the prompt in the Jarvis browser window, then call the tool again.';
+  }
+  if (status >= 400 && !challenged) result.error = `HTTP ${status}`;
+  return result;
+}
+
+async function handle(message) {
+  const action = String(message.action || '').trim();
+  const timeoutMs = Number(message.timeoutMs) > 0 ? Number(message.timeoutMs) : (action === 'search' ? SEARCH_BUDGET_MS : FETCH_BUDGET_MS);
+  const tab = await currentPage();
+  if (action === 'search') {
+    const engine = String(message.engine || 'duckduckgo').trim() || 'duckduckgo';
+    const build = SEARCH_URLS[engine];
+    if (!build) throw new Error(`unknown engine ${engine}`);
+    const query = String(message.query || '').trim();
+    if (!query) throw new Error('query required');
+    const { tab: searchTab, url, challenged } = await gotoUrl(build(query), timeoutMs);
+    if (challenged) {
+      return {
+        error: 'Provider requires a browser security challenge',
+        code: 'bot_challenge',
+        challenge: true,
+        url,
+        next_action: 'Complete the prompt in the Jarvis browser window, then call the tool again.',
+      };
+    }
+    const hits = await extractHits(searchTab, message.limit);
+    return hits.map((hit) => ({ ...hit, source: engine }));
+  }
+  if (action === 'fetch' || action === 'navigate') {
+    const result = await fetchPage(message, timeoutMs);
+    if (action === 'navigate') {
+      const shot = await snapshot(await currentPage());
+      return { ...shot, status: result.status, challenge: result.challenge, next_action: result.next_action, error: result.error };
+    }
+    return result;
+  }
+  if (action === 'snapshot') return snapshot(tab);
+  if (action === 'click') {
+    await snapshot(tab);
+    const target = await locate(tab, message);
+    if (!target) throw new Error('click target not found');
+    await target.click({ timeout: Math.min(8000, timeoutMs) });
+    await tab.waitForLoadState('domcontentloaded', { timeout: 8000 }).catch(() => {});
+    return snapshot(tab);
+  }
+  if (action === 'type') {
+    const value = String(message.text || '');
+    await snapshot(tab);
+    const target = await locate(tab, { ...message, action: 'type' });
+    if (target) {
+      await target.click({ timeout: 4000 }).catch(() => {});
+      await target.fill(value, { timeout: Math.min(8000, timeoutMs) }).catch(async () => {
+        await tab.keyboard.type(value, { delay: 20 });
+      });
+    } else {
+      await tab.keyboard.type(value, { delay: 20 });
+    }
+    if (message.submit) await tab.keyboard.press('Enter');
+    return snapshot(tab);
+  }
+  if (action === 'press') {
+    const key = String(message.key || message.combo || '').trim();
+    if (!key) throw new Error('key required');
+    await tab.keyboard.press(key);
+    return snapshot(tab);
+  }
+  if (action === 'scroll') {
+    await tab.mouse.wheel(Number(message.dx) || 0, Number(message.dy) || 600);
+    return snapshot(tab);
+  }
+  if (action === 'wait') {
+    const ms = Math.min(20_000, Math.max(0, Number(message.ms) || 1000));
+    await tab.waitForTimeout(ms);
+    if (message.text) await tab.getByText(String(message.text), { exact: false }).first().waitFor({ timeout: timeoutMs }).catch(() => {});
+    return snapshot(tab);
+  }
+  throw new Error(`unsupported browser action ${action}`);
+}
+
+async function main() {
+  const root = stateRoot();
+  const profile = process.env.JARVIS_BROWSER_PROFILE || path.join(root, 'profile');
+  const browsers = process.env.PLAYWRIGHT_BROWSERS_PATH || path.join(root, 'ms-playwright');
+  process.env.PLAYWRIGHT_BROWSERS_PATH = browsers;
+  fs.mkdirSync(profile, { recursive: true });
+  fs.mkdirSync(browsers, { recursive: true });
+  const { chromium } = await loadPlaywright();
+  const canShow = Boolean(process.env.DISPLAY || process.env.WAYLAND_DISPLAY);
+  const headless = process.env.JARVIS_BROWSER_HEADLESS === '1' || !canShow;
+  context = await chromium.launchPersistentContext(profile, {
+    headless,
+    viewport: { width: 1280, height: 800 },
+    args: ['--disable-blink-features=AutomationControlled'],
+  });
+  page = context.pages()[0] || await context.newPage();
+  process.stdout.write(`${JSON.stringify({ type: 'ready', headless })}\n`);
+  const rl = readline.createInterface({ input: process.stdin });
+  for await (const line of rl) {
+    if (!line.trim()) continue;
+    let message;
+    try { message = JSON.parse(line); } catch { continue; }
+    const id = message.id;
+    try {
+      const result = await handle(message);
+      reply(id, { ok: true, result });
+    } catch (error) {
+      fail(id, error);
+    }
+  }
+}
+
+main().catch((error) => {
+  process.stdout.write(`${JSON.stringify({ type: 'error', reason: String(error && error.message || error) })}\n`);
+  process.exit(1);
+});