gnome-jarvis/vendor/agent-harness/agent/web-search.js

/**
 * Zero-key public search / page fetch for Bare (no cheerio, jsdom, Playwright).
 * Official JSON APIs plus HTML/RSS scrapes. Scrapers break; the auto chain
 * walks several backends and the agent can pin `engine` to retry one.
 */

const net = require('../lib/net.js');
const truncate = require('./truncate.js');

const WEB_TIMEOUT_MS = 3500;
const PAGE_TIMEOUT_MS = 8000;
const SEARCH_BUDGET_MS = 8000;
const ENGINE_TIMEOUT_MS = 3000;
const BROWSER_UA =
  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
const GOOGLE_UA =
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0';
const AGENT_UA = 'Jarvis-QVAC/1.0 (local GNOME voice assistant)';

const ENGINE_NAMES = [
  'auto',
  'duckduckgo',
  'ddg_lite',
  'ddg_instant',
  'google',
  'bing',
  'bing_rss',
  'jina',
  'wikipedia',
  'hn',
  'github',
  'npm',
  'mdn',
  'stackoverflow',
  'arxiv',
];

const AUTO_ENGINES = [
  'jina',
  'wikipedia',
  'ddg_instant',
  'ddg_lite',
  'bing_rss',
  'duckduckgo',
  'bing',
  'google',
];

const ENGINE_ALIASES = {
  ddg: 'duckduckgo',
  ddg_html: 'duckduckgo',
  wiki: 'wikipedia',
  wikipedia: 'wikipedia',
  hackernews: 'hn',
  'hacker-news': 'hn',
  so: 'stackoverflow',
  stackoverflow: 'stackoverflow',
};

function abortError(timeoutMs) {
  const err = new Error('timed out after ' + timeoutMs + 'ms');
  err.name = 'AbortError';
  return err;
}

function remainingMs(deadline) {
  return Math.max(0, Number(deadline) - Date.now());
}

function timeoutErrorResult(ms, extra) {
  return Object.assign({ error: 'timed out after ' + ms + 'ms' }, extra || {});
}

function budgetMs(timeoutMs, fallback, max) {
  const fallbackMs = Number(fallback) > 0 ? Number(fallback) : SEARCH_BUDGET_MS;
  const cap = Number(max) > 0 ? Number(max) : fallbackMs;
  const n = Number(timeoutMs);
  if (!(n > 0)) return fallbackMs;
  return n > cap ? cap : n;
}

function withDeadline(work, deadline, fallback) {
  const left = remainingMs(deadline);
  if (left <= 0) return Promise.resolve(typeof fallback === 'function' ? fallback() : fallback);
  let timer;
  const timeout = new Promise((resolve) => {
    timer = setTimeout(() => resolve(typeof fallback === 'function' ? fallback() : fallback), left);
  });
  return Promise.race([Promise.resolve().then(work), timeout]).finally(() => {
    if (timer) clearTimeout(timer);
  });
}

function linkAbort(parent, child) {
  if (!parent || !child) return;
  if (parent.aborted) {
    try {
      child.abort();
    } catch (_) {}
    return;
  }
  parent.addEventListener(
    'abort',
    () => {
      try {
        child.abort();
      } catch (_) {}
    },
    { once: true },
  );
}

function fetchWithTimeout(url, opts, timeoutMs) {
  const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS;
  if (!(ms > 0)) return Promise.reject(abortError(0));
  const headers = Object.assign({ 'user-agent': BROWSER_UA }, (opts && opts.headers) || {});
  const controller = typeof AbortController === 'function' ? new AbortController() : null;
  let timer;
  const init = Object.assign({}, opts || {}, { headers });
  if (controller) {
    init.signal = controller.signal;
    if (opts && opts.signal) linkAbort(opts.signal, controller);
  }
  const timeout = new Promise((_, reject) => {
    timer = setTimeout(() => {
      try {
        if (controller) controller.abort();
      } catch (_) {}
      reject(abortError(ms));
    }, ms);
  });
  const pending = fetch(url, init);
  pending.catch(() => {});
  return Promise.race([pending, timeout]).finally(() => {
    if (timer) clearTimeout(timer);
  });
}

function readBodyWithTimeout(res, timeoutMs) {
  const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : 0;
  if (!(ms > 0)) return Promise.reject(abortError(0));
  if (!res || typeof res.text !== 'function') return Promise.resolve('');
  let timer;
  const timeout = new Promise((_, reject) => {
    timer = setTimeout(() => reject(abortError(ms)), ms);
  });
  const pending = res.text();
  pending.catch(() => {});
  return Promise.race([pending, timeout]).finally(() => {
    if (timer) clearTimeout(timer);
  });
}

function stripSearchHtml(s) {
  return String(s || '')
    .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/gi, '$1')
    .replace(/<[^>]+>/g, ' ')
    .replace(/&nbsp;/gi, ' ')
    .replace(/&amp;/gi, '&')
    .replace(/&quot;/gi, '"')
    .replace(/&#39;/g, "'")
    .replace(/&lt;/gi, '<')
    .replace(/&gt;/gi, '>')
    .replace(/\s+/g, ' ')
    .trim();
}

function htmlToText(html) {
  const raw = String(html || '');
  if (!/<(?:html|body|div|p|script|head)\b/i.test(raw) && !/<!DOCTYPE/i.test(raw)) return raw;
  return raw
    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
    .replace(/<[^>]+>/g, ' ')
    .replace(/&nbsp;/gi, ' ')
    .replace(/&amp;/gi, '&')
    .replace(/&lt;/gi, '<')
    .replace(/&gt;/gi, '>')
    .replace(/\s+/g, ' ')
    .trim();
}

function decodeSearchUrl(href) {
  let raw = String(href || '').replace(/&amp;/g, '&').trim();
  if (!raw) return raw;
  if (raw.startsWith('/url?') || raw.startsWith('/search?')) raw = 'https://www.google.com' + raw;
  if (raw.startsWith('//')) raw = 'https:' + raw;
  try {
    const u = new URL(raw);
    const host = u.hostname.replace(/^www\./, '');
    if (host === 'duckduckgo.com') {
      const uddg = u.searchParams.get('uddg');
      if (uddg) {
        let dest = String(uddg);
        try {
          dest = decodeURIComponent(dest);
        } catch (_) {}
        dest = dest.replace(/&amp;/g, '&');
        if (dest.startsWith('//')) dest = 'https:' + dest;
        return dest;
      }
    }
    if (host === 'google.com' || host.endsWith('.google.com')) {
      const dest = u.searchParams.get('q') || u.searchParams.get('url');
      if (dest) {
        let out = String(dest);
        try {
          out = decodeURIComponent(out);
        } catch (_) {}
        out = out.replace(/&amp;/g, '&');
        if (out.startsWith('//')) out = 'https:' + out;
        if (/^https?:\/\//i.test(out)) return out;
      }
    }
    return u.toString();
  } catch (_) {
    return raw;
  }
}

function isOrganicResultUrl(href) {
  let u;
  try {
    u = new URL(href);
  } catch (_) {
    return false;
  }
  if (u.protocol !== 'http:' && u.protocol !== 'https:') return false;
  const h = u.hostname.replace(/^www\./, '').toLowerCase();
  if (h === 'google.com') return false;
  if (h === 'googleusercontent.com' || h.endsWith('.googleusercontent.com')) return false;
  if (h === 'gstatic.com' || h.endsWith('.gstatic.com')) return false;
  if (h === 'bing.com' || h.endsWith('.bing.com')) return false;
  if (h === 'duckduckgo.com' && u.pathname.indexOf('/y.js') === 0) return false;
  if (h === 'youtube.com' && u.pathname.indexOf('/redirect') === 0) return false;
  return true;
}

function decodeBase64Utf8(raw) {
  const s = String(raw || '');
  try {
    if (typeof Buffer !== 'undefined') return Buffer.from(s, 'base64').toString('utf8');
  } catch (_) {}
  try {
    if (typeof atob === 'function') return atob(s);
  } catch (_) {}
  return '';
}

function decodeBingClickUrl(href) {
  const raw = String(href || '').replace(/&amp;/g, '&').trim();
  try {
    const u = new URL(raw, 'https://www.bing.com');
    const host = u.hostname.replace(/^www\./, '');
    if (host === 'bing.com' || host.endsWith('.bing.com')) {
      const dest = u.searchParams.get('u');
      if (dest) {
        let payload = dest;
        if (/^a1/i.test(payload)) payload = payload.slice(2);
        const decoded = decodeBase64Utf8(payload);
        if (/^https?:\/\//i.test(decoded)) return decoded;
      }
    }
  } catch (_) {}
  return decodeSearchUrl(href);
}

function searchHasHits(result) {
  return Array.isArray(result) && result.length > 0;
}

function clampLimit(limit) {
  const n = Number(limit);
  if (!n || n < 1) return 8;
  return n > 15 ? 15 : Math.floor(n);
}

function tagSearchHits(hits, source) {
  return hits.map((hit) => Object.assign({ source: hit.source || source }, hit));
}

function pushHit(hits, seen, href, title, snippet, limit) {
  const url = decodeSearchUrl(href);
  if (!isOrganicResultUrl(url)) return;
  const key = url.split('#')[0];
  if (seen.has(key)) return;
  seen.add(key);
  const item = { url, title: stripSearchHtml(title) || url };
  const snip = stripSearchHtml(snippet);
  if (snip) item.snippet = snip;
  hits.push(item);
}

async function fetchText(url, timeoutMs, opts) {
  const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS;
  const deadline = Date.now() + ms;
  try {
    net.assertPublicHttpUrl(url);
  } catch (err) {
    return { error: String(err && err.message || err), url };
  }
  try {
    const res = await fetchWithTimeout(url, opts || {}, remainingMs(deadline));
    const text = await readBodyWithTimeout(res, remainingMs(deadline));
    if (res.status >= 400) {
      return { error: 'HTTP ' + res.status, url: String(res.url || url), status: res.status, text };
    }
    return { url: String(res.url || url), text, status: res.status };
  } catch (err) {
    return { error: String(err && err.message || err), url };
  }
}

async function fetchJson(url, timeoutMs, opts) {
  const page = await fetchText(url, timeoutMs, opts);
  if (page.error) return page;
  try {
    return { url: page.url, json: JSON.parse(page.text), status: page.status };
  } catch (_) {
    return { error: 'invalid json', url: page.url, text: page.text };
  }
}

function parseGoogleHits(html, limit) {
  const text = String(html || '');
  const max = clampLimit(limit);
  const hits = [];
  const seen = new Set();
  const cardRe = /<a[^>]+href="([^"]+)"[^>]*>[\s\S]*?<div class="BNeawe vvjwJb AP7Wnd"[^>]*>([\s\S]*?)<\/div>/gi;
  let m;
  while ((m = cardRe.exec(text)) && hits.length < max) pushHit(hits, seen, m[1], m[2], '', max);
  const deskRe = /<div[^>]*class="[^"]*yuRUbf[^"]*"[^>]*>[\s\S]*?<a[^>]+href="([^"]+)"[^>]*>[\s\S]*?<h3[^>]*>([\s\S]*?)<\/h3>/gi;
  while ((m = deskRe.exec(text)) && hits.length < max) pushHit(hits, seen, m[1], m[2], '', max);
  const urlqRe = /\/url\?q=(https?:\/\/[^&"'<>]+)/gi;
  while ((m = urlqRe.exec(text)) && hits.length < max) {
    let dest = m[1];
    try {
      dest = decodeURIComponent(dest);
    } catch (_) {}
    pushHit(hits, seen, dest, dest, '', max);
  }
  return hits.slice(0, max);
}

function parseBingHits(html, limit) {
  const text = String(html || '');
  const max = clampLimit(limit);
  const hits = [];
  const seen = new Set();
  const re = /<li class="b_algo"[^>]*>[\s\S]*?<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
  let m;
  while ((m = re.exec(text)) && hits.length < max) {
    const url = decodeBingClickUrl(m[1]);
    if (!isOrganicResultUrl(url)) continue;
    const key = url.split('#')[0];
    if (seen.has(key)) continue;
    seen.add(key);
    hits.push({ url, title: stripSearchHtml(m[2]) || url });
  }
  return hits;
}

function parseDdgHtmlHits(html, limit) {
  const text = String(html || '');
  const max = clampLimit(limit);
  const hits = [];
  const seen = new Set();
  const re = /<a[^>]*class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
  let m;
  while ((m = re.exec(text)) && hits.length < max) {
    const after = text.slice(m.index, m.index + 800);
    const snip = (after.match(/class="result__snippet[^"]*"[^>]*>([\s\S]*?)<\/(?:td|div|a|span)>/i) || [])[1] || '';
    pushHit(hits, seen, m[1], m[2], snip, max);
  }
  return hits;
}

function parseDdgLiteHits(html, limit) {
  const text = String(html || '');
  const max = clampLimit(limit);
  const hits = [];
  const seen = new Set();
  const re = /<a[^>]+(?:class="[^"]*result-link[^"]*"|rel="nofollow")[^>]*href="(https?:[^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
  let m;
  while ((m = re.exec(text)) && hits.length < max) pushHit(hits, seen, m[1], m[2], '', max);
  return hits;
}

function parseRssItems(xml, limit) {
  const text = String(xml || '');
  const max = clampLimit(limit);
  const hits = [];
  const seen = new Set();
  const re = /<item>([\s\S]*?)<\/item>/gi;
  let m;
  while ((m = re.exec(text)) && hits.length < max) {
    const block = m[1];
    const title = stripSearchHtml((block.match(/<title>([\s\S]*?)<\/title>/i) || [])[1] || '');
    let link = stripSearchHtml((block.match(/<link>([\s\S]*?)<\/link>/i) || [])[1] || '');
    const desc = stripSearchHtml((block.match(/<description>([\s\S]*?)<\/description>/i) || [])[1] || '');
    if (!link) continue;
    link = decodeBingClickUrl(link);
    if (!isOrganicResultUrl(link)) continue;
    const key = link.split('#')[0];
    if (seen.has(key)) continue;
    seen.add(key);
    const item = { url: link, title: title || link };
    if (desc) item.snippet = desc.slice(0, 280);
    hits.push(item);
  }
  return hits;
}

function parseAtomEntries(xml, limit) {
  const text = String(xml || '');
  const max = clampLimit(limit);
  const hits = [];
  const re = /<entry>([\s\S]*?)<\/entry>/gi;
  let m;
  while ((m = re.exec(text)) && hits.length < max) {
    const block = m[1];
    const title = stripSearchHtml((block.match(/<title[^>]*>([\s\S]*?)<\/title>/i) || [])[1] || '');
    const linkM = block.match(/<link[^>]+href="([^"]+)"/i) || block.match(/<id>([\s\S]*?)<\/id>/i);
    const url = linkM ? String(linkM[1]).trim() : '';
    const summary = stripSearchHtml((block.match(/<(?:summary|content)[^>]*>([\s\S]*?)<\/(?:summary|content)>/i) || [])[1] || '');
    if (!url || !/^https?:\/\//i.test(url)) continue;
    const item = { url, title: title || url };
    if (summary) item.snippet = summary.slice(0, 280);
    hits.push(item);
  }
  return hits;
}

function isDdgChallenge(html) {
  const text = String(html || '');
  return /anomaly-modal|Unfortunately, bots use DuckDuckGo/i.test(text) && !/result__a/i.test(text);
}

async function duckDuckGoSearch(query, timeoutMs, limit) {
  const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS;
  const deadline = Date.now() + ms;
  const url = 'https://html.duckduckgo.com/html/';
  const body = 'q=' + encodeURIComponent(query) + '&b=&kl=us-en';
  let page = await fetchText(url, remainingMs(deadline), {
    method: 'POST',
    headers: {
      'content-type': 'application/x-www-form-urlencoded',
      'user-agent': BROWSER_UA,
      accept: 'text/html',
    },
    body,
  });
  if (!page.error) {
    if (isDdgChallenge(page.text)) return { error: 'duckduckgo bot challenge', url: page.url, status: page.status };
    const posted = parseDdgHtmlHits(page.text, limit);
    if (searchHasHits(posted)) return posted;
  }
  if (remainingMs(deadline) <= 0) return { error: 'timed out after ' + ms + 'ms', url };
  page = await fetchText(url + '?q=' + encodeURIComponent(query), remainingMs(deadline));
  if (page.error) return page;
  if (isDdgChallenge(page.text)) return { error: 'duckduckgo bot challenge', url: page.url, status: page.status };
  return parseDdgHtmlHits(page.text, limit);
}

async function ddgLiteSearch(query, timeoutMs, limit) {
  const url = 'https://lite.duckduckgo.com/lite/?q=' + encodeURIComponent(query);
  const page = await fetchText(url, timeoutMs);
  if (page.error) return page;
  if (isDdgChallenge(page.text)) return { error: 'duckduckgo bot challenge', url: page.url, status: page.status };
  const hits = parseDdgLiteHits(page.text, limit);
  if (searchHasHits(hits)) return hits;
  return parseDdgHtmlHits(page.text, limit);
}

async function ddgInstantSearch(query, timeoutMs, limit) {
  const url =
    'https://api.duckduckgo.com/?q=' +
    encodeURIComponent(query) +
    '&format=json&no_html=1&skip_disambig=1';
  const page = await fetchJson(url, timeoutMs, { headers: { accept: 'application/json', 'user-agent': AGENT_UA } });
  if (page.error) return page;
  const j = page.json || {};
  const hits = [];
  if (j.AbstractURL && (j.AbstractText || j.Heading)) {
    hits.push({
      url: j.AbstractURL,
      title: j.Heading || j.AbstractURL,
      snippet: j.AbstractText || '',
    });
  }
  const related = j.RelatedTopics || [];
  for (let i = 0; i < related.length && hits.length < clampLimit(limit); i++) {
    const row = related[i];
    const topics = row.Topics || [row];
    for (let t = 0; t < topics.length && hits.length < clampLimit(limit); t++) {
      const item = topics[t];
      if (!item || !item.FirstURL) continue;
      hits.push({ url: item.FirstURL, title: stripSearchHtml(item.Text || item.FirstURL), snippet: item.Text || '' });
    }
  }
  return hits;
}

async function googleSearch(query, timeoutMs, limit) {
  const url =
    'https://www.google.com/search?q=' +
    encodeURIComponent(query) +
    '&num=' +
    clampLimit(limit) +
    '&hl=en&pws=0&gbv=1';
  const page = await fetchText(url, timeoutMs, { headers: { 'user-agent': GOOGLE_UA } });
  if (page.error) return page;
  const hits = parseGoogleHits(page.text, limit);
  if (searchHasHits(hits)) return hits;
  if (/enablejs|Please click/i.test(page.text || '')) return { error: 'google javascript challenge', url: page.url };
  return hits;
}

async function bingSearch(query, timeoutMs, limit) {
  const url = 'https://www.bing.com/search?q=' + encodeURIComponent(query);
  const page = await fetchText(url, timeoutMs);
  if (page.error) return page;
  return parseBingHits(page.text, limit);
}

async function bingRssSearch(query, timeoutMs, limit) {
  const url = 'https://www.bing.com/search?q=' + encodeURIComponent(query) + '&format=rss';
  const page = await fetchText(url, timeoutMs, { headers: { accept: 'application/rss+xml, application/xml, text/xml, */*' } });
  if (page.error) return page;
  return parseRssItems(page.text, limit);
}

async function jinaSearch(query, timeoutMs, limit) {
  const url = 'https://s.jina.ai/' + encodeURIComponent(query);
  const page = await fetchJson(url, timeoutMs, {
    headers: { accept: 'application/json', 'user-agent': AGENT_UA },
  });
  if (page.error) return page;
  const j = page.json || {};
  let rows = j.data || j.results || [];
  if (rows && !Array.isArray(rows) && Array.isArray(rows.results)) rows = rows.results;
  if (!Array.isArray(rows)) rows = [];
  return rows.slice(0, clampLimit(limit)).map((row) => ({
    url: row.url || row.link,
    title: row.title || row.url,
    snippet: stripSearchHtml(row.description || row.content || '').slice(0, 280),
  })).filter((h) => h.url);
}

async function wikiSearch(query, timeoutMs, limit) {
  const url =
    'https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch=' +
    encodeURIComponent(query) +
    '&srlimit=' +
    clampLimit(limit) +
    '&format=json&utf8=1';
  const page = await fetchJson(url, timeoutMs, {
    headers: { accept: 'application/json', 'user-agent': AGENT_UA },
  });
  if (page.error) return page;
  const rows = (page.json && page.json.query && page.json.query.search) || [];
  return rows.map((row) => ({
    url: 'https://en.wikipedia.org/wiki/' + encodeURIComponent(String(row.title || '').replace(/ /g, '_')),
    title: row.title,
    snippet: stripSearchHtml(row.snippet || ''),
  }));
}

async function hnSearch(query, timeoutMs, limit) {
  const url =
    'https://hn.algolia.com/api/v1/search?query=' +
    encodeURIComponent(query) +
    '&tags=story&hitsPerPage=' +
    clampLimit(limit);
  const page = await fetchJson(url, timeoutMs, { headers: { accept: 'application/json', 'user-agent': AGENT_UA } });
  if (page.error) return page;
  const rows = (page.json && page.json.hits) || [];
  return rows.map((row) => ({
    url: row.url || 'https://news.ycombinator.com/item?id=' + row.objectID,
    title: row.title || row.story_title || String(row.objectID),
    snippet: (row.author ? 'by ' + row.author + '. ' : '') + (row.points != null ? row.points + ' points' : ''),
    points: row.points,
    comments: row.num_comments,
  }));
}

async function githubSearch(query, timeoutMs, limit) {
  const url =
    'https://api.github.com/search/repositories?q=' +
    encodeURIComponent(query) +
    '&per_page=' +
    clampLimit(limit);
  const page = await fetchJson(url, timeoutMs, {
    headers: {
      'user-agent': AGENT_UA,
      accept: 'application/vnd.github+json',
    },
  });
  if (page.error) return page;
  const rows = (page.json && page.json.items) || [];
  return rows.map((row) => ({
    url: row.html_url,
    title: row.full_name || row.name,
    snippet: row.description || '',
  }));
}

async function npmSearch(query, timeoutMs, limit) {
  const url =
    'https://registry.npmjs.org/-/v1/search?text=' +
    encodeURIComponent(query) +
    '&size=' +
    clampLimit(limit);
  const page = await fetchJson(url, timeoutMs, { headers: { accept: 'application/json', 'user-agent': AGENT_UA } });
  if (page.error) return page;
  const rows = (page.json && page.json.objects) || [];
  return rows.map((row) => {
    const pkg = row.package || {};
    return {
      url: 'https://www.npmjs.com/package/' + pkg.name,
      title: pkg.name,
      snippet: pkg.description || '',
      version: pkg.version,
    };
  });
}

async function mdnSearch(query, timeoutMs, limit) {
  const url = 'https://developer.mozilla.org/api/v1/search?q=' + encodeURIComponent(query);
  const page = await fetchJson(url, timeoutMs, { headers: { accept: 'application/json', 'user-agent': AGENT_UA } });
  if (page.error) return page;
  const rows = (page.json && page.json.documents) || [];
  return rows.slice(0, clampLimit(limit)).map((row) => ({
    url: row.mdn_url ? 'https://developer.mozilla.org' + row.mdn_url : row.url,
    title: row.title,
    snippet: stripSearchHtml(row.summary || ''),
  })).filter((h) => h.url);
}

async function stackOverflowSearch(query, timeoutMs, limit) {
  const url =
    'https://api.stackexchange.com/2.3/search/advanced?order=desc&sort=relevance&site=stackoverflow&q=' +
    encodeURIComponent(query) +
    '&pagesize=' +
    clampLimit(limit);
  const page = await fetchJson(url, timeoutMs, { headers: { accept: 'application/json', 'user-agent': AGENT_UA } });
  if (page.error) return page;
  const rows = (page.json && page.json.items) || [];
  return rows.map((row) => ({
    url: row.link,
    title: stripSearchHtml(row.title || ''),
    snippet: row.score != null ? String(row.score) + ' score' : '',
  })).filter((h) => h.url);
}

async function arxivSearch(query, timeoutMs, limit) {
  const url =
    'https://export.arxiv.org/api/query?search_query=all:' +
    encodeURIComponent(query) +
    '&start=0&max_results=' +
    clampLimit(limit);
  const page = await fetchText(url, timeoutMs, { headers: { accept: 'application/atom+xml, application/xml, text/xml', 'user-agent': AGENT_UA } });
  if (page.error) return page;
  return parseAtomEntries(page.text, limit);
}

const SEARCH_ENGINES = {
  duckduckgo: duckDuckGoSearch,
  ddg_lite: ddgLiteSearch,
  ddg_instant: ddgInstantSearch,
  google: googleSearch,
  bing: bingSearch,
  bing_rss: bingRssSearch,
  jina: jinaSearch,
  wikipedia: wikiSearch,
  hn: hnSearch,
  github: githubSearch,
  npm: npmSearch,
  mdn: mdnSearch,
  stackoverflow: stackOverflowSearch,
  arxiv: arxivSearch,
};

function resolveEngine(name) {
  const raw = String(name || 'auto').trim().toLowerCase();
  if (!raw || raw === 'auto') return 'auto';
  return ENGINE_ALIASES[raw] || raw;
}

async function runWebSearch(query, opts) {
  opts = opts || {};
  const q = String(query || '').trim();
  if (!q) return { error: 'query required' };
  const limit = clampLimit(opts.limit);
  const budget = budgetMs(opts.timeoutMs, SEARCH_BUDGET_MS, SEARCH_BUDGET_MS);
  const deadline = Date.now() + budget;
  const engine = resolveEngine(opts.engine);
  const tried = [];
  const errors = {};
  const timedOut = () => timeoutErrorResult(budget, {
    tried: tried.slice(),
    errors: Object.assign({}, errors),
    engines: ENGINE_NAMES,
  });
  return withDeadline(async () => {
    try {
    if (engine !== 'auto') {
      const fn = SEARCH_ENGINES[engine];
      if (!fn) return { error: 'unknown engine', engine: opts.engine, engines: ENGINE_NAMES };
      tried.push(engine);
      const result = await fn(q, remainingMs(deadline), limit);
      if (searchHasHits(result)) return tagSearchHits(result, engine).slice(0, limit);
      return {
        error: (result && result.error) || 'no search results',
        url: result && result.url,
        tried: [engine],
        engines: ENGINE_NAMES,
      };
    }
    const prefer = Array.isArray(opts.prefer) ? opts.prefer.map(resolveEngine).filter((n) => SEARCH_ENGINES[n]) : [];
    const chain = prefer.concat(AUTO_ENGINES.filter((name) => prefer.indexOf(name) < 0));
    for (let i = 0; i < chain.length; i++) {
      const left = remainingMs(deadline);
      if (left <= 0) return timedOut();
      if (tried.length && left < 50) break;
      const name = chain[i];
      tried.push(name);
      const result = await SEARCH_ENGINES[name](q, Math.min(ENGINE_TIMEOUT_MS, left), limit);
      if (searchHasHits(result)) return tagSearchHits(result, name).slice(0, limit);
      errors[name] = result && result.error ? result.error : 'no results';
    }
    return { error: 'no search results', tried, errors, engines: ENGINE_NAMES };
    } catch (err) {
      return { error: String(err && err.message || err), tried, errors, engines: ENGINE_NAMES };
    }
  }, deadline, timedOut);
}

async function googleSearchWithFallback(query, timeoutMs) {
  return runWebSearch(query, { timeoutMs, prefer: ['google'] });
}

async function webSearch(query, timeoutMs) {
  return runWebSearch(query, { timeoutMs });
}

async function codeSearch(query, timeoutMs, limit) {
  const q = String(query || '').trim();
  if (!q) return { error: 'query required' };
  const budget = budgetMs(timeoutMs, SEARCH_BUDGET_MS, SEARCH_BUDGET_MS);
  const deadline = Date.now() + budget;
  return withDeadline(async () => {
    const slice = remainingMs(deadline);
    const [github, npm, mdn] = await Promise.all([
      githubSearch(q, slice, limit),
      npmSearch(q, slice, limit),
      mdnSearch(q, slice, limit),
    ]);
    const out = { github: [], npm: [], mdn: [] };
    if (searchHasHits(github)) out.github = tagSearchHits(github, 'github');
    else if (github && github.error) out.github_error = github.error;
    if (searchHasHits(npm)) out.npm = tagSearchHits(npm, 'npm');
    else if (npm && npm.error) out.npm_error = npm.error;
    if (searchHasHits(mdn)) out.mdn = tagSearchHits(mdn, 'mdn');
    else if (mdn && mdn.error) out.mdn_error = mdn.error;
    if (!out.github.length && !out.npm.length && !out.mdn.length) {
      return { error: 'no code search results', github_error: out.github_error, npm_error: out.npm_error, mdn_error: out.mdn_error };
    }
    return out;
  }, deadline, () => timeoutErrorResult(budget));
}

async function webFetch(url, timeoutMs) {
  try {
    net.assertHttpUrl(url);
  } catch (err) {
    return { error: String(err && err.message || err), url: String(url || '') };
  }
  const ms = budgetMs(timeoutMs, WEB_TIMEOUT_MS, SEARCH_BUDGET_MS);
  const deadline = Date.now() + ms;
  try {
    const res = await fetchWithTimeout(url, {}, remainingMs(deadline));
    let text = htmlToText(await readBodyWithTimeout(res, remainingMs(deadline)));
    text = truncate.truncateWithMarker(text, 12000);
    const href = String(res.url || url);
    if (res.status >= 400) {
      return { error: 'HTTP ' + res.status, url: href, status: res.status, text, via: 'raw' };
    }
    return { status: res.status, url: href, text, via: 'raw' };
  } catch (err) {
    return { error: String(err && err.message || err), url: String(url), via: 'raw' };
  }
}

async function fetchPage(url, timeoutMs) {
  try {
    net.assertHttpUrl(url);
  } catch (err) {
    return { error: String(err && err.message || err), url: String(url || '') };
  }
  const target = String(url);
  const jinaUrl = 'https://r.jina.ai/' + target;
  const ms = budgetMs(timeoutMs, PAGE_TIMEOUT_MS, SEARCH_BUDGET_MS);
  const deadline = Date.now() + ms;
  const timedOut = () => timeoutErrorResult(ms, { url: target });
  return withDeadline(async () => {
    try {
      net.assertPublicHttpUrl(jinaUrl);
      const left = remainingMs(deadline);
      if (left > 0) {
        const res = await fetchWithTimeout(jinaUrl, { headers: { accept: 'text/plain', 'user-agent': AGENT_UA } }, left);
        if (res.status < 400) {
          let text = await readBodyWithTimeout(res, remainingMs(deadline));
          text = truncate.truncateWithMarker(text, 12000);
          if (text && text.length > 24 && !/verifying you are (a )?human/i.test(text)) {
            return { status: res.status, url: target, text, via: 'jina' };
          }
        }
      }
    } catch (_) {}
    const left = remainingMs(deadline);
    if (left <= 0) return timedOut();
    const raw = await webFetch(target, left);
    if (raw && !raw.via) raw.via = 'raw';
    return raw;
  }, deadline, timedOut);
}

module.exports = {
  WEB_TIMEOUT_MS,
  PAGE_TIMEOUT_MS,
  SEARCH_BUDGET_MS,
  ENGINE_TIMEOUT_MS,
  BROWSER_UA,
  GOOGLE_UA,
  AGENT_UA,
  ENGINE_NAMES,
  AUTO_ENGINES,
  SEARCH_ENGINES,
  fetchWithTimeout,
  readBodyWithTimeout,
  stripSearchHtml,
  htmlToText,
  decodeSearchUrl,
  decodeBingClickUrl,
  parseGoogleHits,
  parseBingHits,
  parseDdgHtmlHits,
  parseDdgLiteHits,
  parseRssItems,
  parseAtomEntries,
  duckDuckGoSearch,
  ddgLiteSearch,
  ddgInstantSearch,
  googleSearch,
  bingSearch,
  bingRssSearch,
  jinaSearch,
  wikiSearch,
  hnSearch,
  githubSearch,
  npmSearch,
  mdnSearch,
  stackOverflowSearch,
  arxivSearch,
  runWebSearch,
  googleSearchWithFallback,
  webSearch,
  codeSearch,
  webFetch,
  fetchPage,
};