Better Search

2026-09-12 12:33:52 -04:00
parent 0312fedaa7
commit 0e52942b8b
19 changed files with 1333 additions and 215 deletions
@@ -13,7 +13,7 @@ const todos = require('./todos.js');
 const goalMod = require('./goal.js');
 const sr = require('./search-replace.js');
 const grepUtil = require('./grep-util.js');
-const truncate = require('./truncate.js');
+const web = require('./web-search.js');

 const MAX_READ = 400 * 1024;
 const MAX_GREP_HITS = 50;
@@ -217,8 +217,13 @@ const SCHEMAS = [
  { type: 'function', name: 'list_dir', description: 'List a directory.', parameters: { type: 'object', properties: { path: { type: 'string' }, recursive: { type: 'boolean' } } } },
  { type: 'function', name: 'run_terminal_cmd', description: 'Run a shell command in the workspace cwd.', parameters: { type: 'object', properties: { command: { type: 'string' }, timeout_ms: { type: 'number' } }, required: ['command'] } },
  { type: 'function', name: 'todo_write', description: 'Merge or replace session todos. Status: pending | in_progress | completed | cancelled.', parameters: { type: 'object', properties: { todos: { type: 'array', items: { type: 'object', properties: { id: { type: 'string' }, content: { type: 'string' }, status: { type: 'string', enum: ['pending', 'in_progress', 'completed', 'cancelled'] } } } }, merge: { type: 'boolean', description: 'If true (default), merge by id. If false, replace the list.' } }, required: ['todos'] } },
-  { type: 'function', name: 'web_search', description: 'Search the public web (DuckDuckGo HTML).', parameters: { type: 'object', properties: { query: { type: 'string' } }, required: ['query'] } },
-  { type: 'function', name: 'web_fetch', description: 'Fetch any http or https URL as text, including public internet hosts.', parameters: { type: 'object', properties: { url: { type: 'string' } }, required: ['url'] } },
+  { type: 'function', name: 'web_search', description: 'Search the public web with no API key. Default engine auto walks DuckDuckGo, Jina, Bing, Google, then Wikipedia. Pin engine to retry one backend: auto, duckduckgo, ddg_lite, ddg_instant, google, bing, bing_rss, jina, wikipedia, hn, github, npm, mdn, stackoverflow, arxiv.', parameters: { type: 'object', properties: { query: { type: 'string' }, engine: { type: 'string' }, limit: { type: 'number' } }, required: ['query'] } },
+  { type: 'function', name: 'google_search', description: 'Same as web_search but tries Google HTML first, then the auto fallback chain.', parameters: { type: 'object', properties: { query: { type: 'string' }, limit: { type: 'number' } }, required: ['query'] } },
+  { type: 'function', name: 'fetch_page', description: 'Fetch a URL as readable text. Tries Jina Reader, then a direct HTML strip. Use for articles. Use web_fetch for raw pages and I P lookups.', parameters: { type: 'object', properties: { url: { type: 'string' } }, required: ['url'] } },
+  { type: 'function', name: 'web_fetch', description: 'Fetch any http or https URL as stripped text, including public internet hosts. Use this for I P lookup pages such as ifconfig.me.', parameters: { type: 'object', properties: { url: { type: 'string' } }, required: ['url'] } },
+  { type: 'function', name: 'wiki_search', description: 'Search Wikipedia (official MediaWiki JSON, no key).', parameters: { type: 'object', properties: { query: { type: 'string' }, limit: { type: 'number' } }, required: ['query'] } },
+  { type: 'function', name: 'hn_search', description: 'Search Hacker News discussions (Algolia, no key).', parameters: { type: 'object', properties: { query: { type: 'string' }, limit: { type: 'number' } }, required: ['query'] } },
+  { type: 'function', name: 'code_search', description: 'Search GitHub repositories, npm packages, and MDN docs in parallel (no key).', parameters: { type: 'object', properties: { query: { type: 'string' }, limit: { type: 'number' } }, required: ['query'] } },
  { type: 'function', name: 'memory_search', description: 'Search local agent memory notes.', parameters: { type: 'object', properties: { query: { type: 'string' } } } },
  { type: 'function', name: 'memory_get', description: 'Read a memory note by name.', parameters: { type: 'object', properties: { name: { type: 'string' } }, required: ['name'] } },
  { type: 'function', name: 'memory_write', description: 'Write a local agent memory note.', parameters: { type: 'object', properties: { name: { type: 'string' }, text: { type: 'string' } }, required: ['name', 'text'] } },
@@ -239,138 +244,22 @@ function defs(opts) {
  return toolSet.filterBuiltinSchemas(SCHEMAS, opts);
 }

-const WEB_TIMEOUT_MS = 12000;
-const BROWSER_UA =
-  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
+const WEB_TIMEOUT_MS = web.WEB_TIMEOUT_MS;
+const BROWSER_UA = web.BROWSER_UA;
+const GOOGLE_UA = web.GOOGLE_UA;
+const fetchWithTimeout = web.fetchWithTimeout;
+const htmlToText = web.htmlToText;
+const decodeSearchUrl = web.decodeSearchUrl;
+const parseGoogleHits = web.parseGoogleHits;
+const parseBingHits = web.parseBingHits;
+const duckDuckGoSearch = web.duckDuckGoSearch;
+const bingSearch = web.bingSearch;
+const googleSearch = web.googleSearch;
+const googleSearchWithFallback = web.googleSearchWithFallback;
+const webSearch = web.webSearch;
+const webFetch = web.webFetch;
+const fetchPage = web.fetchPage;

-function abortError(timeoutMs) {
-  const err = new Error('timed out after ' + timeoutMs + 'ms');
-  err.name = 'AbortError';
-  return err;
-}
-
-function fetchWithTimeout(url, opts, timeoutMs) {
-  const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS;
-  const headers = Object.assign({ 'user-agent': BROWSER_UA }, (opts && opts.headers) || {});
-  const controller = typeof AbortController === 'function' ? new AbortController() : null;
-  let timer;
-  const init = Object.assign({}, opts || {}, { headers });
-  if (controller) init.signal = controller.signal;
-  const timeout = new Promise((_, reject) => {
-    timer = setTimeout(() => {
-      try {
-        if (controller) controller.abort();
-      } catch (_) {}
-      reject(abortError(ms));
-    }, ms);
-  });
-  const pending = fetch(url, init);
-  pending.catch(() => {});
-  return Promise.race([pending, timeout]).finally(() => {
-    if (timer) clearTimeout(timer);
-  });
-}
-
-function readBodyWithTimeout(res, timeoutMs) {
-  const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS;
-  if (!res || typeof res.text !== 'function') return Promise.resolve('');
-  let timer;
-  const timeout = new Promise((_, reject) => {
-    timer = setTimeout(() => reject(abortError(ms)), ms);
-  });
-  const pending = res.text();
-  pending.catch(() => {});
-  return Promise.race([pending, timeout]).finally(() => {
-    if (timer) clearTimeout(timer);
-  });
-}
-
-function decodeSearchUrl(href) {
-  let raw = String(href || '').replace(/&amp;/g, '&').trim();
-  if (!raw) return raw;
-  if (raw.startsWith('//')) raw = 'https:' + raw;
-  try {
-    const u = new URL(raw);
-    const host = u.hostname.replace(/^www\./, '');
-    if (host === 'duckduckgo.com') {
-      const uddg = u.searchParams.get('uddg');
-      if (uddg) {
-        let dest = String(uddg);
-        try {
-          dest = decodeURIComponent(dest);
-        } catch (_) {}
-        dest = dest.replace(/&amp;/g, '&');
-        if (dest.startsWith('//')) dest = 'https:' + dest;
-        return dest;
-      }
-    }
-    return u.toString();
-  } catch (_) {
-    return raw;
-  }
-}
-
-async function webSearch(query, timeoutMs) {
-  const net = require('../lib/net.js');
-  const url = 'https://html.duckduckgo.com/html/?q=' + encodeURIComponent(query);
-  try {
-    net.assertPublicHttpUrl(url);
-  } catch (err) {
-    return { error: String(err && err.message || err), url };
-  }
-  try {
-    const res = await fetchWithTimeout(url, {}, timeoutMs);
-    if (res.status >= 400) {
-      return { error: 'HTTP ' + res.status, url: String(res.url || url), status: res.status };
-    }
-    const text = await readBodyWithTimeout(res, timeoutMs);
-    const hits = [];
-    const re = /<a[^>]*class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
-    let m;
-    while ((m = re.exec(text)) && hits.length < 8) {
-      hits.push({ url: decodeSearchUrl(m[1]), title: m[2].replace(/<[^>]+>/g, '').trim() });
-    }
-    return hits;
-  } catch (err) {
-    return { error: String(err && err.message || err), url };
-  }
-}
-
-function htmlToText(html) {
-  const raw = String(html || '');
-  if (!/<(?:html|body|div|p|script|head)\b/i.test(raw) && !/<!DOCTYPE/i.test(raw)) return raw;
-  return raw
-    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
-    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
-    .replace(/<[^>]+>/g, ' ')
-    .replace(/&nbsp;/gi, ' ')
-    .replace(/&amp;/gi, '&')
-    .replace(/&lt;/gi, '<')
-    .replace(/&gt;/gi, '>')
-    .replace(/\s+/g, ' ')
-    .trim();
-}
-
-async function webFetch(url, timeoutMs) {
-  const net = require('../lib/net.js');
-  try {
-    net.assertHttpUrl(url);
-  } catch (err) {
-    return { error: String(err && err.message || err), url: String(url || '') };
-  }
-  try {
-    const res = await fetchWithTimeout(url, {}, timeoutMs);
-    let text = htmlToText(await readBodyWithTimeout(res, timeoutMs));
-    text = truncate.truncateWithMarker(text, 12000);
-    const href = String(res.url || url);
-    if (res.status >= 400) {
-      return { error: 'HTTP ' + res.status, url: href, status: res.status, text };
-    }
-    return { status: res.status, url: href, text };
-  } catch (err) {
-    return { error: String(err && err.message || err), url: String(url) };
-  }
-}

 async function execute(ctx, name, args) {
  const origin = ctx.origin;
@@ -460,9 +349,27 @@ async function execute(ctx, name, args) {
      return { ok: true, todos: ctx.session.plan };
    }
    case 'web_search':
-      return webSearch(args.query);
+      return web.runWebSearch(args.query, {
+        engine: args.engine,
+        limit: args.limit,
+        timeoutMs: args.timeout_ms || args.timeoutMs,
+      });
+    case 'google_search':
+      return web.runWebSearch(args.query, {
+        prefer: ['google'],
+        limit: args.limit,
+        timeoutMs: args.timeout_ms || args.timeoutMs,
+      });
+    case 'fetch_page':
+      return web.fetchPage(args.url, args.timeout_ms || args.timeoutMs);
    case 'web_fetch':
-      return webFetch(args.url);
+      return web.webFetch(args.url, args.timeout_ms || args.timeoutMs);
+    case 'wiki_search':
+      return web.runWebSearch(args.query, { engine: 'wikipedia', limit: args.limit, timeoutMs: args.timeout_ms || args.timeoutMs });
+    case 'hn_search':
+      return web.runWebSearch(args.query, { engine: 'hn', limit: args.limit, timeoutMs: args.timeout_ms || args.timeoutMs });
+    case 'code_search':
+      return web.codeSearch(args.query, args.timeout_ms || args.timeoutMs, args.limit);
    case 'memory_search':
      return memory.search(origin, args.query);
    case 'memory_get':
@@ -542,10 +449,23 @@ module.exports = {
  runShell,
  formatShellResult,
  webFetch,
+  fetchPage,
  webSearch,
+  googleSearch,
+  duckDuckGoSearch,
+  bingSearch,
+  googleSearchWithFallback,
+  parseGoogleHits,
+  parseBingHits,
  decodeSearchUrl,
  htmlToText,
  fetchWithTimeout,
  WEB_TIMEOUT_MS,
  BROWSER_UA,
+  GOOGLE_UA,
+  runWebSearch: web.runWebSearch,
+  wikiSearch: web.wikiSearch,
+  hnSearch: web.hnSearch,
+  codeSearch: web.codeSearch,
+  ENGINE_NAMES: web.ENGINE_NAMES,
 };