699 lines
25 KiB
JavaScript
699 lines
25 KiB
JavaScript
/**
|
|
* Zero-key public search / page fetch for Bare (no cheerio, jsdom, Playwright).
|
|
* Direct HTML/RSS scraping only. Scrapers break; the auto chain
|
|
* walks several backends and the agent can pin `engine` to retry one.
|
|
*/
|
|
|
|
const net = require('../lib/net.js');
|
|
const reader = require('./web-reader.js');
|
|
|
|
const WEB_TIMEOUT_MS = 3500;
|
|
const PAGE_TIMEOUT_MS = 8000;
|
|
const SEARCH_BUDGET_MS = 8000;
|
|
const ENGINE_TIMEOUT_MS = 3000;
|
|
const BROWSER_UA =
|
|
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
|
|
const GOOGLE_UA =
|
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0';
|
|
const AGENT_UA = 'Jarvis-QVAC/1.0 (local GNOME voice assistant)';
|
|
|
|
const ENGINE_NAMES = [
|
|
'auto',
|
|
'duckduckgo',
|
|
'ddg_lite',
|
|
'google',
|
|
'bing',
|
|
'bing_rss',
|
|
'wikipedia',
|
|
'hn',
|
|
'github',
|
|
'npm',
|
|
'mdn',
|
|
'stackoverflow',
|
|
'arxiv',
|
|
];
|
|
|
|
const AUTO_ENGINES = ['duckduckgo', 'bing_rss', 'google', 'ddg_lite', 'bing'];
|
|
|
|
const ENGINE_ALIASES = {
|
|
ddg: 'duckduckgo',
|
|
ddg_html: 'duckduckgo',
|
|
wiki: 'wikipedia',
|
|
wikipedia: 'wikipedia',
|
|
hackernews: 'hn',
|
|
'hacker-news': 'hn',
|
|
so: 'stackoverflow',
|
|
stackoverflow: 'stackoverflow',
|
|
};
|
|
|
|
function abortError(timeoutMs) {
|
|
const err = new Error('timed out after ' + timeoutMs + 'ms');
|
|
err.name = 'AbortError';
|
|
return err;
|
|
}
|
|
|
|
function remainingMs(deadline) {
|
|
return Math.max(0, Number(deadline) - Date.now());
|
|
}
|
|
|
|
function timeoutErrorResult(ms, extra) {
|
|
return Object.assign({ error: 'timed out after ' + ms + 'ms' }, extra || {});
|
|
}
|
|
|
|
function budgetMs(timeoutMs, fallback, max) {
|
|
const fallbackMs = Number(fallback) > 0 ? Number(fallback) : SEARCH_BUDGET_MS;
|
|
const cap = Number(max) > 0 ? Number(max) : fallbackMs;
|
|
const n = Number(timeoutMs);
|
|
if (!(n > 0)) return fallbackMs;
|
|
return n > cap ? cap : n;
|
|
}
|
|
|
|
function withDeadline(work, deadline, fallback) {
|
|
const left = remainingMs(deadline);
|
|
if (left <= 0) return Promise.resolve(typeof fallback === 'function' ? fallback() : fallback);
|
|
let timer;
|
|
const timeout = new Promise((resolve) => {
|
|
timer = setTimeout(() => resolve(typeof fallback === 'function' ? fallback() : fallback), left);
|
|
});
|
|
return Promise.race([Promise.resolve().then(work), timeout]).finally(() => {
|
|
if (timer) clearTimeout(timer);
|
|
});
|
|
}
|
|
|
|
function linkAbort(parent, child) {
|
|
if (!parent || !child) return;
|
|
if (parent.aborted) {
|
|
try {
|
|
child.abort();
|
|
} catch (_) {}
|
|
return;
|
|
}
|
|
parent.addEventListener(
|
|
'abort',
|
|
() => {
|
|
try {
|
|
child.abort();
|
|
} catch (_) {}
|
|
},
|
|
{ once: true },
|
|
);
|
|
}
|
|
|
|
function fetchWithTimeout(url, opts, timeoutMs) {
|
|
const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS;
|
|
if (!(ms > 0)) return Promise.reject(abortError(0));
|
|
const headers = Object.assign({ 'user-agent': BROWSER_UA }, (opts && opts.headers) || {});
|
|
const controller = typeof AbortController === 'function' ? new AbortController() : null;
|
|
let timer;
|
|
const init = Object.assign({}, opts || {}, { headers });
|
|
if (controller) {
|
|
init.signal = controller.signal;
|
|
if (opts && opts.signal) linkAbort(opts.signal, controller);
|
|
}
|
|
const timeout = new Promise((_, reject) => {
|
|
timer = setTimeout(() => {
|
|
try {
|
|
if (controller) controller.abort();
|
|
} catch (_) {}
|
|
reject(abortError(ms));
|
|
}, ms);
|
|
});
|
|
const pending = fetch(url, init);
|
|
pending.catch(() => {});
|
|
return Promise.race([pending, timeout]).finally(() => {
|
|
if (timer) clearTimeout(timer);
|
|
});
|
|
}
|
|
|
|
function readBodyWithTimeout(res, timeoutMs) {
|
|
const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : 0;
|
|
if (!(ms > 0)) return Promise.reject(abortError(0));
|
|
if (!res || typeof res.text !== 'function') return Promise.resolve('');
|
|
let timer, activeReader;
|
|
const timeout = new Promise((_, reject) => {
|
|
timer = setTimeout(() => { if (activeReader) activeReader.cancel().catch(() => {}); reject(abortError(ms)); }, ms);
|
|
});
|
|
const pending = (async () => {
|
|
const max = 2 * 1024 * 1024;
|
|
if (res.body && typeof res.body.getReader === 'function') {
|
|
const stream = res.body.getReader();
|
|
activeReader = stream;
|
|
const decoder = new TextDecoder();
|
|
let text = '', bytes = 0;
|
|
try {
|
|
while (true) {
|
|
const chunk = await stream.read();
|
|
if (chunk.done) break;
|
|
bytes += chunk.value.byteLength;
|
|
if (bytes > max) throw new Error('response exceeds 2 MiB limit');
|
|
text += decoder.decode(chunk.value, { stream: true });
|
|
}
|
|
return text + decoder.decode();
|
|
} finally { await stream.cancel().catch(() => {}); }
|
|
}
|
|
const text = await res.text();
|
|
if (text.length > max) throw new Error('response exceeds 2 MiB limit');
|
|
return text;
|
|
})();
|
|
pending.catch(() => {});
|
|
return Promise.race([pending, timeout]).finally(() => {
|
|
if (timer) clearTimeout(timer);
|
|
});
|
|
}
|
|
|
|
function stripSearchHtml(s) { return reader.decodeEntities(String(s || '').replace(/<!\[CDATA\[([\s\S]*?)\]\]>/gi, '$1').replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim(); }
|
|
|
|
function htmlToText(html) { return reader.readableText(html); }
|
|
|
|
function decodeSearchUrl(href) {
|
|
let raw = String(href || '').replace(/&/g, '&').trim();
|
|
if (!raw) return raw;
|
|
if (raw.startsWith('/url?') || raw.startsWith('/search?')) raw = 'https://www.google.com' + raw;
|
|
if (raw.startsWith('//')) raw = 'https:' + raw;
|
|
try {
|
|
const u = new URL(raw);
|
|
const host = u.hostname.replace(/^www\./, '');
|
|
if (host === 'duckduckgo.com') {
|
|
const uddg = u.searchParams.get('uddg');
|
|
if (uddg) {
|
|
let dest = String(uddg);
|
|
|
|
dest = dest.replace(/&/g, '&');
|
|
if (dest.startsWith('//')) dest = 'https:' + dest;
|
|
return dest;
|
|
}
|
|
}
|
|
if (host === 'google.com' || host.endsWith('.google.com')) {
|
|
const dest = u.searchParams.get('q') || u.searchParams.get('url');
|
|
if (dest) {
|
|
let out = String(dest);
|
|
|
|
out = out.replace(/&/g, '&');
|
|
if (out.startsWith('//')) out = 'https:' + out;
|
|
if (/^https?:\/\//i.test(out)) return out;
|
|
}
|
|
}
|
|
return u.toString();
|
|
} catch (_) {
|
|
return raw;
|
|
}
|
|
}
|
|
|
|
function isOrganicResultUrl(href) {
|
|
let u;
|
|
try {
|
|
u = new URL(href);
|
|
} catch (_) {
|
|
return false;
|
|
}
|
|
if (u.protocol !== 'http:' && u.protocol !== 'https:') return false;
|
|
const h = u.hostname.replace(/^www\./, '').toLowerCase();
|
|
if (h === 'google.com') return false;
|
|
if (h === 'googleusercontent.com' || h.endsWith('.googleusercontent.com')) return false;
|
|
if (h === 'gstatic.com' || h.endsWith('.gstatic.com')) return false;
|
|
if (h === 'bing.com' || h.endsWith('.bing.com')) return false;
|
|
if (h === 'duckduckgo.com' && u.pathname.indexOf('/y.js') === 0) return false;
|
|
if (h === 'youtube.com' && u.pathname.indexOf('/redirect') === 0) return false;
|
|
return true;
|
|
}
|
|
|
|
function decodeBase64Utf8(raw) {
|
|
const s = String(raw || '');
|
|
try {
|
|
if (typeof Buffer !== 'undefined') return Buffer.from(s, 'base64').toString('utf8');
|
|
} catch (_) {}
|
|
try {
|
|
if (typeof atob === 'function') return atob(s);
|
|
} catch (_) {}
|
|
return '';
|
|
}
|
|
|
|
function decodeBingClickUrl(href) {
|
|
const raw = String(href || '').replace(/&/g, '&').trim();
|
|
try {
|
|
const u = new URL(raw, 'https://www.bing.com');
|
|
const host = u.hostname.replace(/^www\./, '');
|
|
if (host === 'bing.com' || host.endsWith('.bing.com')) {
|
|
const dest = u.searchParams.get('u');
|
|
if (dest) {
|
|
let payload = dest;
|
|
if (/^a1/i.test(payload)) payload = payload.slice(2);
|
|
const decoded = decodeBase64Utf8(payload);
|
|
if (/^https?:\/\//i.test(decoded)) return decoded;
|
|
}
|
|
}
|
|
} catch (_) {}
|
|
return decodeSearchUrl(href);
|
|
}
|
|
|
|
function searchHasHits(result) {
|
|
return Array.isArray(result) && result.length > 0;
|
|
}
|
|
|
|
function clampLimit(limit) {
|
|
const n = Number(limit);
|
|
if (!n || n < 1) return 8;
|
|
return n > 15 ? 15 : Math.floor(n);
|
|
}
|
|
|
|
function tagSearchHits(hits, source) {
|
|
return hits.map((hit) => Object.assign({}, hit, { source }));
|
|
}
|
|
|
|
function pushHit(hits, seen, href, title, snippet, limit) {
|
|
const url = decodeSearchUrl(href);
|
|
if (!isOrganicResultUrl(url)) return;
|
|
const key = reader.canonicalUrl(url);
|
|
if (seen.has(key)) return;
|
|
seen.add(key);
|
|
const item = { url, title: stripSearchHtml(title) || url };
|
|
const snip = stripSearchHtml(snippet);
|
|
if (snip) item.snippet = snip;
|
|
hits.push(item);
|
|
}
|
|
|
|
async function fetchText(url, timeoutMs, opts) {
|
|
const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS;
|
|
const deadline = Date.now() + ms;
|
|
try {
|
|
net.assertPublicHttpUrl(url);
|
|
} catch (err) {
|
|
return { error: String(err && err.message || err), url };
|
|
}
|
|
try {
|
|
let target = String(url), res;
|
|
for (let hop = 0; hop <= 5; hop++) {
|
|
net.assertPublicHttpUrl(target);
|
|
res = await fetchWithTimeout(target, Object.assign({}, opts, { redirect: 'manual' }), remainingMs(deadline));
|
|
if (![301, 302, 303, 307, 308].includes(res.status)) break;
|
|
const location = res.headers && res.headers.get('location');
|
|
if (res.body && res.body.cancel) await res.body.cancel();
|
|
if (!location) throw new Error('redirect missing location');
|
|
if (hop === 5) throw new Error('too many redirects');
|
|
target = new URL(location, target).href;
|
|
if (res.status === 303 || ((res.status === 301 || res.status === 302) && opts && opts.method === 'POST')) opts = { method: 'GET' };
|
|
}
|
|
const type = res.headers && res.headers.get('content-type') || '';
|
|
if (type && !/text\/|json|xml|javascript/i.test(type)) throw new Error('unsupported content type: ' + type);
|
|
const text = await readBodyWithTimeout(res, remainingMs(deadline));
|
|
if (res.status >= 400) {
|
|
return { error: 'HTTP ' + res.status, url: String(res.url || target), status: res.status, text };
|
|
}
|
|
return { url: String(res.url || target), text, status: res.status };
|
|
} catch (err) {
|
|
return { error: String(err && err.message || err), url };
|
|
}
|
|
}
|
|
|
|
function parseGoogleHits(html, limit) {
|
|
const text = String(html || '');
|
|
const max = clampLimit(limit);
|
|
const hits = [];
|
|
const seen = new Set();
|
|
const cardRe = /<a[^>]+href="([^"]+)"[^>]*>[\s\S]*?<div class="BNeawe vvjwJb AP7Wnd"[^>]*>([\s\S]*?)<\/div>/gi;
|
|
let m;
|
|
while ((m = cardRe.exec(text)) && hits.length < max) pushHit(hits, seen, m[1], m[2], '', max);
|
|
const deskRe = /<div[^>]*class="[^"]*yuRUbf[^"]*"[^>]*>[\s\S]*?<a[^>]+href="([^"]+)"[^>]*>[\s\S]*?<h3[^>]*>([\s\S]*?)<\/h3>/gi;
|
|
while ((m = deskRe.exec(text)) && hits.length < max) pushHit(hits, seen, m[1], m[2], '', max);
|
|
const anchors = /<a\b([^>]*)>([\s\S]*?)<\/a>/gi;
|
|
while ((m = anchors.exec(text)) && hits.length < max) {
|
|
const heading = (m[2].match(/<h3\b[^>]*>([\s\S]*?)<\/h3>/i) || [])[1];
|
|
if (heading) pushHit(hits, seen, reader.attributes(m[1]).href, heading, '', max);
|
|
}
|
|
const urlqRe = /\/url\?q=(https?:\/\/[^&"'<>]+)/gi;
|
|
while ((m = urlqRe.exec(text)) && hits.length < max) {
|
|
let dest = m[1];
|
|
try {
|
|
dest = decodeURIComponent(dest);
|
|
} catch (_) {}
|
|
pushHit(hits, seen, dest, dest, '', max);
|
|
}
|
|
return hits.slice(0, max);
|
|
}
|
|
|
|
function parseBingHits(html, limit) {
|
|
const hits = [], seen = new Set();
|
|
const cards = String(html || '').match(/<li\b[^>]*class=["'][^"']*\bb_algo\b[^"']*["'][^>]*>[\s\S]*?<\/li>/gi) || [];
|
|
for (const card of cards) {
|
|
const heading = (card.match(/<h2\b[^>]*>([\s\S]*?)<\/h2>/i) || [])[1] || '';
|
|
const link = heading.match(/<a\b([^>]*)>([\s\S]*?)<\/a>/i);
|
|
if (!link) continue;
|
|
const attrs = reader.attributes(link[1]);
|
|
const snippet = (card.match(/<p\b[^>]*>([\s\S]*?)<\/p>/i) || [])[1] || '';
|
|
pushHit(hits, seen, decodeBingClickUrl(attrs.href), link[2], snippet, limit);
|
|
if (hits.length >= clampLimit(limit)) break;
|
|
}
|
|
return hits;
|
|
}
|
|
|
|
function parseDdgHtmlHits(html, limit) {
|
|
const text = String(html || ''), hits = [], seen = new Set();
|
|
const re = /<a\b([^>]*)>([\s\S]*?)<\/a>/gi;
|
|
let m;
|
|
while ((m = re.exec(text)) && hits.length < clampLimit(limit)) {
|
|
const a = reader.attributes(m[1]);
|
|
if (!/(?:^|\s)result__a(?:\s|$)/.test(a.class || '')) continue;
|
|
const after = text.slice(re.lastIndex, re.lastIndex + 1800);
|
|
const snippet = (after.match(/<(?:a|td|div|span)\b[^>]*class=["'][^"']*(?:result__snippet|result-snippet)[^"']*["'][^>]*>([\s\S]*?)<\/(?:a|td|div|span)>/i) || [])[1] || '';
|
|
const href = a.href && reader.canonicalUrl(a.href, 'https://duckduckgo.com');
|
|
pushHit(hits, seen, href, m[2], snippet, limit);
|
|
}
|
|
return hits;
|
|
}
|
|
|
|
function parseDdgLiteHits(html, limit) {
|
|
const text = String(html || ''), hits = [], seen = new Set();
|
|
const re = /<a\b([^>]*)>([\s\S]*?)<\/a>/gi;
|
|
let m;
|
|
while ((m = re.exec(text)) && hits.length < clampLimit(limit)) {
|
|
const a = reader.attributes(m[1]);
|
|
if (!/(?:^|\s)result-link(?:\s|$)/.test(a.class || '')) continue;
|
|
const after = text.slice(re.lastIndex, re.lastIndex + 1800);
|
|
const snippet = (after.match(/<(?:a|td|div|span)\b[^>]*class=["'][^"']*(?:result__snippet|result-snippet)[^"']*["'][^>]*>([\s\S]*?)<\/(?:a|td|div|span)>/i) || [])[1] || '';
|
|
const href = a.href && new URL(a.href, 'https://duckduckgo.com').href;
|
|
pushHit(hits, seen, href, m[2], snippet, limit);
|
|
}
|
|
return hits;
|
|
}
|
|
|
|
function parseRssItems(xml, limit) {
|
|
const text = String(xml || '');
|
|
const max = clampLimit(limit);
|
|
const hits = [];
|
|
const seen = new Set();
|
|
const re = /<item>([\s\S]*?)<\/item>/gi;
|
|
let m;
|
|
while ((m = re.exec(text)) && hits.length < max) {
|
|
const block = m[1];
|
|
const title = stripSearchHtml((block.match(/<title>([\s\S]*?)<\/title>/i) || [])[1] || '');
|
|
let link = stripSearchHtml((block.match(/<link>([\s\S]*?)<\/link>/i) || [])[1] || '');
|
|
const desc = stripSearchHtml((block.match(/<description>([\s\S]*?)<\/description>/i) || [])[1] || '');
|
|
if (!link) continue;
|
|
link = decodeBingClickUrl(link);
|
|
if (!isOrganicResultUrl(link)) continue;
|
|
const key = link.split('#')[0];
|
|
if (seen.has(key)) continue;
|
|
seen.add(key);
|
|
const item = { url: link, title: title || link };
|
|
if (desc) item.snippet = desc.slice(0, 280);
|
|
hits.push(item);
|
|
}
|
|
return hits;
|
|
}
|
|
|
|
function parseAtomEntries(xml, limit) {
|
|
const text = String(xml || '');
|
|
const max = clampLimit(limit);
|
|
const hits = [];
|
|
const re = /<entry>([\s\S]*?)<\/entry>/gi;
|
|
let m;
|
|
while ((m = re.exec(text)) && hits.length < max) {
|
|
const block = m[1];
|
|
const title = stripSearchHtml((block.match(/<title[^>]*>([\s\S]*?)<\/title>/i) || [])[1] || '');
|
|
const linkM = block.match(/<link[^>]+href="([^"]+)"/i) || block.match(/<id>([\s\S]*?)<\/id>/i);
|
|
const url = linkM ? String(linkM[1]).trim() : '';
|
|
const summary = stripSearchHtml((block.match(/<(?:summary|content)[^>]*>([\s\S]*?)<\/(?:summary|content)>/i) || [])[1] || '');
|
|
if (!url || !/^https?:\/\//i.test(url)) continue;
|
|
const item = { url, title: title || url };
|
|
if (summary) item.snippet = summary.slice(0, 280);
|
|
hits.push(item);
|
|
}
|
|
return hits;
|
|
}
|
|
|
|
function isDdgChallenge(html) {
|
|
const text = String(html || '');
|
|
return /anomaly-modal|Unfortunately, bots use DuckDuckGo/i.test(text) && !/result__a/i.test(text);
|
|
}
|
|
|
|
async function duckDuckGoSearch(query, timeoutMs, limit) {
|
|
const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : WEB_TIMEOUT_MS;
|
|
const deadline = Date.now() + ms;
|
|
const url = 'https://html.duckduckgo.com/html/';
|
|
const body = 'q=' + encodeURIComponent(query) + '&b=&kl=us-en';
|
|
let page = await fetchText(url, remainingMs(deadline), {
|
|
method: 'POST',
|
|
headers: {
|
|
'content-type': 'application/x-www-form-urlencoded',
|
|
'user-agent': BROWSER_UA,
|
|
accept: 'text/html',
|
|
},
|
|
body,
|
|
});
|
|
if (!page.error) {
|
|
if (isDdgChallenge(page.text)) return { error: 'duckduckgo bot challenge', url: page.url, status: page.status };
|
|
const posted = parseDdgHtmlHits(page.text, limit);
|
|
if (searchHasHits(posted)) return posted;
|
|
}
|
|
if (remainingMs(deadline) <= 0) return { error: 'timed out after ' + ms + 'ms', url };
|
|
page = await fetchText(url + '?q=' + encodeURIComponent(query), remainingMs(deadline));
|
|
if (page.error) return page;
|
|
if (isDdgChallenge(page.text)) return { error: 'duckduckgo bot challenge', url: page.url, status: page.status };
|
|
return parseDdgHtmlHits(page.text, limit);
|
|
}
|
|
|
|
async function ddgLiteSearch(query, timeoutMs, limit) {
|
|
const url = 'https://lite.duckduckgo.com/lite/?q=' + encodeURIComponent(query);
|
|
const page = await fetchText(url, timeoutMs);
|
|
if (page.error) return page;
|
|
if (isDdgChallenge(page.text)) return { error: 'duckduckgo bot challenge', url: page.url, status: page.status };
|
|
const hits = parseDdgLiteHits(page.text, limit);
|
|
if (searchHasHits(hits)) return hits;
|
|
return parseDdgHtmlHits(page.text, limit);
|
|
}
|
|
|
|
async function googleSearch(query, timeoutMs, limit) {
|
|
const url =
|
|
'https://www.google.com/search?q=' +
|
|
encodeURIComponent(query) +
|
|
'&num=' +
|
|
clampLimit(limit) +
|
|
'&hl=en&pws=0&gbv=1';
|
|
const page = await fetchText(url, timeoutMs, { headers: { 'user-agent': GOOGLE_UA } });
|
|
if (page.error) return page;
|
|
const hits = parseGoogleHits(page.text, limit);
|
|
if (searchHasHits(hits)) return hits;
|
|
if (/enablejs|Please click/i.test(page.text || '')) return { error: 'google javascript challenge', url: page.url };
|
|
return hits;
|
|
}
|
|
|
|
async function bingSearch(query, timeoutMs, limit) {
|
|
const url = 'https://www.bing.com/search?q=' + encodeURIComponent(query);
|
|
const page = await fetchText(url, timeoutMs);
|
|
if (page.error) return page;
|
|
return parseBingHits(page.text, limit);
|
|
}
|
|
|
|
async function bingRssSearch(query, timeoutMs, limit) {
|
|
const url = 'https://www.bing.com/search?q=' + encodeURIComponent(query) + '&format=rss';
|
|
const page = await fetchText(url, timeoutMs, { headers: { accept: 'application/rss+xml, application/xml, text/xml, */*' } });
|
|
if (page.error) return page;
|
|
return parseRssItems(page.text, limit);
|
|
}
|
|
|
|
async function wikiSearch(query, timeoutMs, limit) {
|
|
return siteSearch('en.wikipedia.org', query, timeoutMs, limit);
|
|
}
|
|
|
|
async function hnSearch(query, timeoutMs, limit) {
|
|
return siteSearch('news.ycombinator.com', query, timeoutMs, limit);
|
|
}
|
|
|
|
async function githubSearch(query, timeoutMs, limit) {
|
|
return siteSearch('github.com', query, timeoutMs, limit);
|
|
}
|
|
|
|
async function npmSearch(query, timeoutMs, limit) {
|
|
return siteSearch('npmjs.com', query, timeoutMs, limit);
|
|
}
|
|
|
|
async function mdnSearch(query, timeoutMs, limit) {
|
|
return siteSearch('developer.mozilla.org', query, timeoutMs, limit);
|
|
}
|
|
|
|
async function stackOverflowSearch(query, timeoutMs, limit) {
|
|
return siteSearch('stackoverflow.com', query, timeoutMs, limit);
|
|
}
|
|
|
|
async function arxivSearch(query, timeoutMs, limit) {
|
|
return siteSearch('arxiv.org', query, timeoutMs, limit);
|
|
}
|
|
|
|
async function siteSearch(site, query, timeoutMs, limit) {
|
|
const hits = await runWebSearch('site:' + site + ' ' + query, { timeoutMs, limit });
|
|
if (!Array.isArray(hits)) return hits;
|
|
return hits.filter(hit => { try { const h = new URL(hit.url).hostname; return h === site || h.endsWith('.' + site); } catch (_) { return false; } });
|
|
}
|
|
|
|
const SEARCH_ENGINES = {
|
|
duckduckgo: duckDuckGoSearch,
|
|
ddg_lite: ddgLiteSearch,
|
|
google: googleSearch,
|
|
bing: bingSearch,
|
|
bing_rss: bingRssSearch,
|
|
wikipedia: wikiSearch,
|
|
hn: hnSearch,
|
|
github: githubSearch,
|
|
npm: npmSearch,
|
|
mdn: mdnSearch,
|
|
stackoverflow: stackOverflowSearch,
|
|
arxiv: arxivSearch,
|
|
};
|
|
|
|
function resolveEngine(name) {
|
|
const raw = String(name || 'auto').trim().toLowerCase();
|
|
if (!raw || raw === 'auto') return 'auto';
|
|
return ENGINE_ALIASES[raw] || raw;
|
|
}
|
|
|
|
async function runWebSearch(query, opts) {
|
|
opts = opts || {};
|
|
const q = String(query || '').trim();
|
|
if (!q) return { error: 'query required' };
|
|
const limit = clampLimit(opts.limit);
|
|
const budget = budgetMs(opts.timeoutMs, SEARCH_BUDGET_MS, SEARCH_BUDGET_MS);
|
|
const deadline = Date.now() + budget;
|
|
const engine = resolveEngine(opts.engine);
|
|
const tried = [];
|
|
const errors = {};
|
|
const timedOut = () => timeoutErrorResult(budget, {
|
|
tried: tried.slice(),
|
|
errors: Object.assign({}, errors),
|
|
engines: ENGINE_NAMES,
|
|
});
|
|
return withDeadline(async () => {
|
|
try {
|
|
if (engine !== 'auto') {
|
|
const fn = SEARCH_ENGINES[engine];
|
|
if (!fn) return { error: 'unknown engine', engine: opts.engine, engines: ENGINE_NAMES };
|
|
tried.push(engine);
|
|
const result = await fn(q, remainingMs(deadline), limit);
|
|
if (searchHasHits(result)) return tagSearchHits(result, engine).slice(0, limit);
|
|
return {
|
|
error: (result && result.error) || 'no search results',
|
|
url: result && result.url,
|
|
tried: [engine],
|
|
engines: ENGINE_NAMES,
|
|
};
|
|
}
|
|
const prefer = Array.isArray(opts.prefer) ? opts.prefer.map(resolveEngine).filter((n) => SEARCH_ENGINES[n]) : [];
|
|
const chain = prefer.concat(AUTO_ENGINES.filter((name) => prefer.indexOf(name) < 0));
|
|
const merged = new Map();
|
|
for (let i = 0; i < chain.length; i += 3) {
|
|
const left = remainingMs(deadline);
|
|
if (left <= 10) break;
|
|
const batch = chain.slice(i, i + 3);
|
|
const results = await Promise.all(batch.map(async name => {
|
|
tried.push(name);
|
|
try { return await withDeadline(() => SEARCH_ENGINES[name](q, Math.min(ENGINE_TIMEOUT_MS, left), limit), Math.min(deadline - 5, Date.now() + ENGINE_TIMEOUT_MS), { error: 'engine timed out' }); }
|
|
catch (err) { return { error: String(err.message || err) }; }
|
|
}));
|
|
results.forEach((result, index) => {
|
|
const name = batch[index];
|
|
if (!searchHasHits(result)) { errors[name] = result && result.error || 'no results'; return; }
|
|
result.forEach((hit, rank) => {
|
|
const key = reader.canonicalUrl(hit.url);
|
|
if (!key) return;
|
|
const old = merged.get(key);
|
|
if (old) { old.score += 1 / (60 + rank); if (!old.sources.includes(name)) old.sources.push(name); if ((hit.snippet || '').length > (old.snippet || '').length) old.snippet = hit.snippet; }
|
|
else merged.set(key, Object.assign({}, hit, { url: key, source: name, sources: [name], score: 1 / (60 + rank) }));
|
|
});
|
|
});
|
|
if (merged.size >= limit) break;
|
|
}
|
|
if (merged.size) return [...merged.values()].sort((a, b) => b.score - a.score).slice(0, limit).map(({ score, ...hit }) => hit);
|
|
if (remainingMs(deadline) <= 10) return timedOut();
|
|
return { error: 'no search results', tried, errors, engines: ENGINE_NAMES };
|
|
} catch (err) {
|
|
return { error: String(err && err.message || err), tried, errors, engines: ENGINE_NAMES };
|
|
}
|
|
}, deadline, timedOut);
|
|
}
|
|
|
|
async function googleSearchWithFallback(query, timeoutMs) {
|
|
return runWebSearch(query, { timeoutMs, prefer: ['google'] });
|
|
}
|
|
|
|
async function webSearch(query, timeoutMs) {
|
|
return runWebSearch(query, { timeoutMs });
|
|
}
|
|
|
|
async function codeSearch(query, timeoutMs, limit) {
|
|
const q = String(query || '').trim();
|
|
if (!q) return { error: 'query required' };
|
|
const budget = budgetMs(timeoutMs, SEARCH_BUDGET_MS, SEARCH_BUDGET_MS);
|
|
const deadline = Date.now() + budget;
|
|
return withDeadline(async () => {
|
|
const slice = remainingMs(deadline);
|
|
const [github, npm, mdn] = await Promise.all([
|
|
githubSearch(q, slice, limit),
|
|
npmSearch(q, slice, limit),
|
|
mdnSearch(q, slice, limit),
|
|
]);
|
|
const out = { github: [], npm: [], mdn: [] };
|
|
if (searchHasHits(github)) out.github = tagSearchHits(github, 'github');
|
|
else if (github && github.error) out.github_error = github.error;
|
|
if (searchHasHits(npm)) out.npm = tagSearchHits(npm, 'npm');
|
|
else if (npm && npm.error) out.npm_error = npm.error;
|
|
if (searchHasHits(mdn)) out.mdn = tagSearchHits(mdn, 'mdn');
|
|
else if (mdn && mdn.error) out.mdn_error = mdn.error;
|
|
if (!out.github.length && !out.npm.length && !out.mdn.length) {
|
|
return { error: 'no code search results', github_error: out.github_error, npm_error: out.npm_error, mdn_error: out.mdn_error };
|
|
}
|
|
return out;
|
|
}, deadline, () => timeoutErrorResult(budget));
|
|
}
|
|
|
|
async function webFetch(url, timeoutMs, opts) {
|
|
const page = await fetchText(url, budgetMs(timeoutMs, PAGE_TIMEOUT_MS, SEARCH_BUDGET_MS));
|
|
if (page.error) return { error: page.error, url: page.url, status: page.status, via: 'raw' };
|
|
return Object.assign({ status: page.status, url: page.url, via: 'raw' }, reader.extractPage(page.text, page.url, opts));
|
|
}
|
|
|
|
async function fetchPage(url, timeoutMs, opts) {
|
|
return webFetch(url, timeoutMs, opts);
|
|
}
|
|
|
|
module.exports = {
|
|
WEB_TIMEOUT_MS,
|
|
PAGE_TIMEOUT_MS,
|
|
SEARCH_BUDGET_MS,
|
|
ENGINE_TIMEOUT_MS,
|
|
BROWSER_UA,
|
|
GOOGLE_UA,
|
|
AGENT_UA,
|
|
ENGINE_NAMES,
|
|
AUTO_ENGINES,
|
|
SEARCH_ENGINES,
|
|
fetchWithTimeout,
|
|
readBodyWithTimeout,
|
|
stripSearchHtml,
|
|
htmlToText,
|
|
decodeSearchUrl,
|
|
decodeBingClickUrl,
|
|
parseGoogleHits,
|
|
parseBingHits,
|
|
parseDdgHtmlHits,
|
|
parseDdgLiteHits,
|
|
parseRssItems,
|
|
parseAtomEntries,
|
|
duckDuckGoSearch,
|
|
ddgLiteSearch,
|
|
googleSearch,
|
|
bingSearch,
|
|
bingRssSearch,
|
|
wikiSearch,
|
|
hnSearch,
|
|
githubSearch,
|
|
npmSearch,
|
|
mdnSearch,
|
|
stackOverflowSearch,
|
|
arxivSearch,
|
|
runWebSearch,
|
|
googleSearchWithFallback,
|
|
webSearch,
|
|
codeSearch,
|
|
webFetch,
|
|
fetchPage,
|
|
};
|