Updates

vendor/agent-harness/agent/tools.js

@@ -239,28 +239,137 @@ function defs(opts) {
   return toolSet.filterBuiltinSchemas(SCHEMAS, opts);
 }
 
-async function webSearch(query) {
-  const net = require('../lib/net.js');
-  const url = 'https://html.duckduckgo.com/html/?q=' + encodeURIComponent(query);
-  net.assertPublicHttpUrl(url);
-  const res = await fetch(url, { headers: { 'user-agent': 'agent-harness/0.1' } });
-  const text = await res.text();
-  const hits = [];
-  const re = /<a[^>]*class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
-  let m;
-  while ((m = re.exec(text)) && hits.length < 8) {
-    hits.push({ url: m[1], title: m[2].replace(/<[^>]+>/g, '').trim() });
-  }
-  return hits;
+const WEB_TIMEOUT_MS = 12000;
+const BROWSER_UA =
+  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
+
+function abortError(timeoutMs) {
+  const err = new Error('timed out after ' + timeoutMs + 'ms');
+  err.name = 'AbortError';
+  return err;
 }
 
-async function webFetch(url) {
+function fetchWithTimeout(url, opts, timeoutMs) {
+  const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS;
+  const headers = Object.assign({ 'user-agent': BROWSER_UA }, (opts && opts.headers) || {});
+  const controller = typeof AbortController === 'function' ? new AbortController() : null;
+  let timer;
+  const init = Object.assign({}, opts || {}, { headers });
+  if (controller) init.signal = controller.signal;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => {
+      try {
+        if (controller) controller.abort();
+      } catch (_) {}
+      reject(abortError(ms));
+    }, ms);
+  });
+  const pending = fetch(url, init);
+  pending.catch(() => {});
+  return Promise.race([pending, timeout]).finally(() => {
+    if (timer) clearTimeout(timer);
+  });
+}
+
+function readBodyWithTimeout(res, timeoutMs) {
+  const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS;
+  if (!res || typeof res.text !== 'function') return Promise.resolve('');
+  let timer;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(abortError(ms)), ms);
+  });
+  const pending = res.text();
+  pending.catch(() => {});
+  return Promise.race([pending, timeout]).finally(() => {
+    if (timer) clearTimeout(timer);
+  });
+}
+
+function decodeSearchUrl(href) {
+  let raw = String(href || '').replace(/&amp;/g, '&').trim();
+  if (!raw) return raw;
+  if (raw.startsWith('//')) raw = 'https:' + raw;
+  try {
+    const u = new URL(raw);
+    const host = u.hostname.replace(/^www\./, '');
+    if (host === 'duckduckgo.com') {
+      const uddg = u.searchParams.get('uddg');
+      if (uddg) {
+        let dest = String(uddg);
+        try {
+          dest = decodeURIComponent(dest);
+        } catch (_) {}
+        dest = dest.replace(/&amp;/g, '&');
+        if (dest.startsWith('//')) dest = 'https:' + dest;
+        return dest;
+      }
+    }
+    return u.toString();
+  } catch (_) {
+    return raw;
+  }
+}
+
+async function webSearch(query, timeoutMs) {
   const net = require('../lib/net.js');
-  net.assertHttpUrl(url);
-  const res = await fetch(url, { headers: { 'user-agent': 'agent-harness/0.1' } });
-  let text = await res.text();
-  text = truncate.truncateWithMarker(text, 80000);
-  return { status: res.status, url: String(res.url || url), text };
+  const url = 'https://html.duckduckgo.com/html/?q=' + encodeURIComponent(query);
+  try {
+    net.assertPublicHttpUrl(url);
+  } catch (err) {
+    return { error: String(err && err.message || err), url };
+  }
+  try {
+    const res = await fetchWithTimeout(url, {}, timeoutMs);
+    if (res.status >= 400) {
+      return { error: 'HTTP ' + res.status, url: String(res.url || url), status: res.status };
+    }
+    const text = await readBodyWithTimeout(res, timeoutMs);
+    const hits = [];
+    const re = /<a[^>]*class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
+    let m;
+    while ((m = re.exec(text)) && hits.length < 8) {
+      hits.push({ url: decodeSearchUrl(m[1]), title: m[2].replace(/<[^>]+>/g, '').trim() });
+    }
+    return hits;
+  } catch (err) {
+    return { error: String(err && err.message || err), url };
+  }
+}
+
+function htmlToText(html) {
+  const raw = String(html || '');
+  if (!/<(?:html|body|div|p|script|head)\b/i.test(raw) && !/<!DOCTYPE/i.test(raw)) return raw;
+  return raw
+    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/&nbsp;/gi, ' ')
+    .replace(/&amp;/gi, '&')
+    .replace(/&lt;/gi, '<')
+    .replace(/&gt;/gi, '>')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+async function webFetch(url, timeoutMs) {
+  const net = require('../lib/net.js');
+  try {
+    net.assertHttpUrl(url);
+  } catch (err) {
+    return { error: String(err && err.message || err), url: String(url || '') };
+  }
+  try {
+    const res = await fetchWithTimeout(url, {}, timeoutMs);
+    let text = htmlToText(await readBodyWithTimeout(res, timeoutMs));
+    text = truncate.truncateWithMarker(text, 12000);
+    const href = String(res.url || url);
+    if (res.status >= 400) {
+      return { error: 'HTTP ' + res.status, url: href, status: res.status, text };
+    }
+    return { status: res.status, url: href, text };
+  } catch (err) {
+    return { error: String(err && err.message || err), url: String(url) };
+  }
 }
 
 async function execute(ctx, name, args) {
@@ -425,4 +534,18 @@ function ensureParent(abs) {
   } catch (_) {}
 }
 
-module.exports = { defs, execute, SCHEMAS, HOST_WORKSPACE_TOOLS: toolSet.HOST_WORKSPACE_TOOLS, runShell, formatShellResult };
+module.exports = {
+  defs,
+  execute,
+  SCHEMAS,
+  HOST_WORKSPACE_TOOLS: toolSet.HOST_WORKSPACE_TOOLS,
+  runShell,
+  formatShellResult,
+  webFetch,
+  webSearch,
+  decodeSearchUrl,
+  htmlToText,
+  fetchWithTimeout,
+  WEB_TIMEOUT_MS,
+  BROWSER_UA,
+};