Fix CI

vendor/agent-harness/agent/web-search.js

@@ -46,6 +46,24 @@ const ENGINE_ALIASES = {
   stackoverflow: 'stackoverflow',
 };
 
+// Share cooldowns across searches, including DuckDuckGo's HTML/lite hosts.
+const blockedProviders = new Map();
+function providerKey(url) {
+  const host = new URL(url).hostname;
+  return /(^|\.)duckduckgo\.com$/.test(host) ? 'duckduckgo.com' : host;
+}
+function challengePage(text) {
+  const html = String(text || '');
+  return /anomaly-modal|Unfortunately, bots use DuckDuckGo|id=["']challenge-form|\/cdn-cgi\/challenge-platform\/|<title>\s*(?:Just a moment|Attention Required)/i.test(html) ||
+    (reader.readableText(html).length < 2000 && /verify (?:that )?you are human|unusual traffic from your computer network|checking your browser|complete the security check/i.test(reader.readableText(html)));
+}
+function blockedResult(url, status, code, retryAfterMs) {
+  return { error: code === 'rate_limited' ? 'Provider rate limited requests' : 'Provider requires a browser security challenge',
+    code, url, status, retry_after_ms: retryAfterMs,
+    warning: 'Page is blocked by a rate limit or bot challenge; content is not verified.',
+    next_action: 'Try another search engine, or open this URL in your browser and complete any required verification.' };
+}
+
 function abortError(timeoutMs) {
   const err = new Error('timed out after ' + timeoutMs + 'ms');
   err.name = 'AbortError';
@@ -102,7 +120,7 @@ function linkAbort(parent, child) {
 function fetchWithTimeout(url, opts, timeoutMs) {
   const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS;
   if (!(ms > 0)) return Promise.reject(abortError(0));
-  const headers = Object.assign({ 'user-agent': BROWSER_UA }, (opts && opts.headers) || {});
+  const headers = Object.assign({ 'user-agent': BROWSER_UA, accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'accept-language': 'en-US,en;q=0.9' }, (opts && opts.headers) || {});
   const controller = typeof AbortController === 'function' ? new AbortController() : null;
   let timer;
   const init = Object.assign({}, opts || {}, { headers });
@@ -284,6 +302,10 @@ async function fetchText(url, timeoutMs, opts) {
     let target = String(url), res;
     for (let hop = 0; hop <= 5; hop++) {
       net.assertPublicHttpUrl(target);
+      if (remainingMs(deadline) <= 0) throw abortError(ms);
+      const cooldown = blockedProviders.get(providerKey(target));
+      if (cooldown && cooldown.until > Date.now()) return blockedResult(target, cooldown.status, cooldown.code, cooldown.until - Date.now());
+      blockedProviders.delete(providerKey(target));
       res = await fetchWithTimeout(target, Object.assign({}, opts, { redirect: 'manual' }), remainingMs(deadline));
       if (![301, 302, 303, 307, 308].includes(res.status)) break;
       const location = res.headers && res.headers.get('location');
@@ -296,6 +318,15 @@ async function fetchText(url, timeoutMs, opts) {
     const type = res.headers && res.headers.get('content-type') || '';
     if (type && !/text\/|json|xml|javascript/i.test(type)) throw new Error('unsupported content type: ' + type);
     const text = await readBodyWithTimeout(res, remainingMs(deadline));
+    if (res.status === 429 || challengePage(text)) {
+      const code = res.status === 429 ? 'rate_limited' : 'bot_challenge';
+      const retry = res.headers && res.headers.get('retry-after');
+      const delay = retry ? (/^\d+$/.test(retry) ? Number(retry) * 1000 : Date.parse(retry) - Date.now()) : 60000;
+      const retryAfterMs = Math.min(3600000, Math.max(1000, Number.isFinite(delay) ? delay : 60000));
+      if (blockedProviders.size >= 128) blockedProviders.delete(blockedProviders.keys().next().value);
+      blockedProviders.set(providerKey(target), { until: Date.now() + retryAfterMs, status: res.status, code });
+      return blockedResult(target, res.status, code, retryAfterMs);
+    }
     if (res.status >= 400) {
       return { error: 'HTTP ' + res.status, url: String(res.url || target), status: res.status, text };
     }
@@ -440,6 +471,7 @@ async function duckDuckGoSearch(query, timeoutMs, limit) {
     },
     body,
   });
+  if (page.code) return page;
   if (!page.error) {
     if (isDdgChallenge(page.text)) return { error: 'duckduckgo bot challenge', url: page.url, status: page.status };
     const posted = parseDdgHtmlHits(page.text, limit);
@@ -570,6 +602,7 @@ async function runWebSearch(query, opts) {
       const result = await fn(q, remainingMs(deadline), limit);
       if (searchHasHits(result)) return tagSearchHits(result, engine).slice(0, limit);
       return {
+        ...(result && !Array.isArray(result) ? result : {}),
         error: (result && result.error) || 'no search results',
         url: result && result.url,
         tried: [engine],
@@ -646,7 +679,7 @@ async function codeSearch(query, timeoutMs, limit) {
 
 async function webFetch(url, timeoutMs, opts) {
   const page = await fetchText(url, budgetMs(timeoutMs, PAGE_TIMEOUT_MS, SEARCH_BUDGET_MS));
-  if (page.error) return { error: page.error, url: page.url, status: page.status, via: 'raw' };
+  if (page.error) { const { text, ...failure } = page; return { ...failure, via: 'raw' }; }
   return Object.assign({ status: page.status, url: page.url, via: 'raw' }, reader.extractPage(page.text, page.url, opts));
 }