237 lines
7.9 KiB
JavaScript
237 lines
7.9 KiB
JavaScript
/**
|
|
* Public search / page fetch through the Jarvis Playwright helper.
|
|
* The Bare daemon never loads Playwright; Node Chromium runs in browser-use/helper.js.
|
|
*/
|
|
|
|
const net = require('../lib/net.js');
|
|
const reader = require('./web-reader.js');
|
|
|
|
const WEB_TIMEOUT_MS = 25_000;
|
|
const PAGE_TIMEOUT_MS = 30_000;
|
|
const SEARCH_BUDGET_MS = 25_000;
|
|
const ENGINE_NAMES = [
|
|
'auto',
|
|
'duckduckgo',
|
|
'ddg_lite',
|
|
'google',
|
|
'bing',
|
|
'bing_rss',
|
|
'wikipedia',
|
|
'hn',
|
|
'github',
|
|
'npm',
|
|
'mdn',
|
|
'stackoverflow',
|
|
'arxiv',
|
|
];
|
|
const ENGINE_ALIASES = {
|
|
ddg: 'duckduckgo',
|
|
ddg_html: 'duckduckgo',
|
|
wiki: 'wikipedia',
|
|
wikipedia: 'wikipedia',
|
|
hackernews: 'hn',
|
|
'hacker-news': 'hn',
|
|
so: 'stackoverflow',
|
|
stackoverflow: 'stackoverflow',
|
|
};
|
|
|
|
let backend = null;
|
|
function setBrowserBackend(next) {
|
|
backend = next || null;
|
|
}
|
|
|
|
function htmlToText(html) {
|
|
return reader.readableText(html);
|
|
}
|
|
|
|
function decodeSearchUrl(href) {
|
|
let raw = String(href || '').replace(/&/g, '&').trim();
|
|
if (!raw) return raw;
|
|
if (raw.startsWith('/url?') || raw.startsWith('/search?')) raw = 'https://www.google.com' + raw;
|
|
if (raw.startsWith('//')) raw = 'https:' + raw;
|
|
try {
|
|
const u = new URL(raw);
|
|
const host = u.hostname.replace(/^www\./, '');
|
|
if (host === 'duckduckgo.com') {
|
|
const uddg = u.searchParams.get('uddg');
|
|
if (uddg) {
|
|
let dest = String(uddg).replace(/&/g, '&');
|
|
if (dest.startsWith('//')) dest = 'https:' + dest;
|
|
return dest;
|
|
}
|
|
}
|
|
if (host === 'google.com' || host.endsWith('.google.com')) {
|
|
const dest = u.searchParams.get('q') || u.searchParams.get('url');
|
|
if (dest) {
|
|
let out = String(dest).replace(/&/g, '&');
|
|
if (out.startsWith('//')) out = 'https:' + out;
|
|
if (/^https?:\/\//i.test(out)) return out;
|
|
}
|
|
}
|
|
return u.toString();
|
|
} catch (_) {
|
|
return raw;
|
|
}
|
|
}
|
|
|
|
function clampLimit(limit) {
|
|
const n = Number(limit);
|
|
if (!n || n < 1) return 8;
|
|
return n > 15 ? 15 : Math.floor(n);
|
|
}
|
|
|
|
function budgetMs(timeoutMs, fallback, max) {
|
|
const fallbackMs = Number(fallback) > 0 ? Number(fallback) : SEARCH_BUDGET_MS;
|
|
const cap = Number(max) > 0 ? Number(max) : fallbackMs;
|
|
const n = Number(timeoutMs);
|
|
if (!(n > 0)) return fallbackMs;
|
|
return n > cap ? cap : n;
|
|
}
|
|
|
|
function resolveEngine(name) {
|
|
const raw = String(name || 'auto').trim().toLowerCase();
|
|
if (!raw || raw === 'auto') return 'auto';
|
|
return ENGINE_ALIASES[raw] || raw;
|
|
}
|
|
|
|
function unavailable(extra) {
|
|
return Object.assign({ error: 'Jarvis browser helper unavailable' }, extra || {});
|
|
}
|
|
|
|
async function callBrowser(action, payload, timeoutMs, local) {
|
|
const impl = local || backend;
|
|
if (!impl || typeof impl.call !== 'function') return unavailable();
|
|
const ms = Number(timeoutMs) > 0 ? Number(timeoutMs) : SEARCH_BUDGET_MS;
|
|
return new Promise((resolve) => {
|
|
const timer = setTimeout(() => resolve({ error: 'timed out after ' + ms + 'ms' }), ms);
|
|
Promise.resolve()
|
|
.then(() => impl.call(action, payload, ms))
|
|
.then((value) => { clearTimeout(timer); resolve(value); }, (error) => {
|
|
clearTimeout(timer);
|
|
resolve({ error: String(error && error.message || error) });
|
|
});
|
|
});
|
|
}
|
|
|
|
function searchHasHits(result) {
|
|
return Array.isArray(result) && result.length > 0;
|
|
}
|
|
|
|
async function runWebSearch(query, opts) {
|
|
opts = opts || {};
|
|
const q = String(query || '').trim();
|
|
if (!q) return { error: 'query required' };
|
|
const limit = clampLimit(opts.limit);
|
|
const budget = budgetMs(opts.timeoutMs, SEARCH_BUDGET_MS, SEARCH_BUDGET_MS);
|
|
const engine = resolveEngine(opts.engine);
|
|
if (engine !== 'auto' && ENGINE_NAMES.indexOf(engine) < 0) {
|
|
return { error: 'unknown engine', engine: opts.engine, engines: ENGINE_NAMES };
|
|
}
|
|
const prefer = Array.isArray(opts.prefer) ? resolveEngine(opts.prefer[0]) : '';
|
|
const first = engine === 'auto' ? (prefer && prefer !== 'auto' ? prefer : 'duckduckgo') : engine;
|
|
const started = Date.now();
|
|
const result = await callBrowser('search', { query: q, engine: first, limit }, budget, opts.backend);
|
|
if (searchHasHits(result)) return result.slice(0, limit);
|
|
const remaining = budget - (Date.now() - started);
|
|
if (engine === 'auto' && remaining > 0 && (first === 'duckduckgo' || first === 'google')) {
|
|
const fallbackEngine = first === 'google' ? 'duckduckgo' : 'google';
|
|
const fallback = await callBrowser('search', { query: q, engine: fallbackEngine, limit }, remaining, opts.backend);
|
|
if (searchHasHits(fallback)) return fallback.slice(0, limit);
|
|
if (fallback && fallback.error) return fallback;
|
|
}
|
|
if (result && result.error) return result;
|
|
return { error: 'no search results', tried: [first], engines: ENGINE_NAMES };
|
|
}
|
|
|
|
async function googleSearchWithFallback(query, timeoutMs) {
|
|
return runWebSearch(query, { timeoutMs, prefer: ['google'] });
|
|
}
|
|
|
|
async function webSearch(query, timeoutMs) {
|
|
return runWebSearch(query, { timeoutMs });
|
|
}
|
|
|
|
async function wikiSearch(query, timeoutMs, limit, local) {
|
|
return runWebSearch(query, { engine: 'wikipedia', timeoutMs, limit, backend: local });
|
|
}
|
|
|
|
async function hnSearch(query, timeoutMs, limit, local) {
|
|
return runWebSearch(query, { engine: 'hn', timeoutMs, limit, backend: local });
|
|
}
|
|
|
|
async function codeSearch(query, timeoutMs, limit, local) {
|
|
const q = String(query || '').trim();
|
|
if (!q) return { error: 'query required' };
|
|
const budget = budgetMs(timeoutMs, SEARCH_BUDGET_MS, SEARCH_BUDGET_MS);
|
|
const started = Date.now();
|
|
const slice = () => Math.max(500, budget - (Date.now() - started));
|
|
const github = await runWebSearch(q, { engine: 'github', timeoutMs: slice(), limit, backend: local });
|
|
const npm = await runWebSearch(q, { engine: 'npm', timeoutMs: slice(), limit, backend: local });
|
|
const mdn = await runWebSearch(q, { engine: 'mdn', timeoutMs: slice(), limit, backend: local });
|
|
const out = { github: [], npm: [], mdn: [] };
|
|
if (searchHasHits(github)) out.github = github;
|
|
else if (github && github.error) out.github_error = github.error;
|
|
if (searchHasHits(npm)) out.npm = npm;
|
|
else if (npm && npm.error) out.npm_error = npm.error;
|
|
if (searchHasHits(mdn)) out.mdn = mdn;
|
|
else if (mdn && mdn.error) out.mdn_error = mdn.error;
|
|
if (!out.github.length && !out.npm.length && !out.mdn.length) {
|
|
return { error: 'no code search results', github_error: out.github_error, npm_error: out.npm_error, mdn_error: out.mdn_error };
|
|
}
|
|
return out;
|
|
}
|
|
|
|
async function webFetch(url, timeoutMs, opts) {
|
|
opts = opts || {};
|
|
try {
|
|
net.assertPublicHttpUrl(url);
|
|
} catch (error) {
|
|
return { error: String(error && error.message || error), url };
|
|
}
|
|
const budget = budgetMs(timeoutMs, PAGE_TIMEOUT_MS, PAGE_TIMEOUT_MS);
|
|
const page = await callBrowser('fetch', {
|
|
url,
|
|
offset: opts.offset,
|
|
max_chars: opts.max_chars,
|
|
find: opts.find,
|
|
}, budget, opts.backend);
|
|
if (!page || page.error && !page.html && !page.text) {
|
|
return Object.assign({ url, via: 'browser' }, page && page.error ? page : unavailable({ url }));
|
|
}
|
|
const extracted = reader.extractPage(page.html || `<title>${page.title || ''}</title><body>${page.text || ''}</body>`, page.url || url, opts);
|
|
const out = Object.assign({
|
|
status: page.status,
|
|
url: page.url || url,
|
|
via: 'browser',
|
|
}, extracted);
|
|
if (page.challenge) {
|
|
out.challenge = true;
|
|
out.next_action = page.next_action || 'Complete the prompt in the Jarvis browser window, then call the tool again.';
|
|
out.warning = out.warning || 'Page may still be a bot challenge; complete it in the Jarvis browser.';
|
|
}
|
|
if (page.error && !out.challenge) out.error = page.error;
|
|
return out;
|
|
}
|
|
|
|
async function fetchPage(url, timeoutMs, opts) {
|
|
return webFetch(url, timeoutMs, opts);
|
|
}
|
|
|
|
module.exports = {
|
|
WEB_TIMEOUT_MS,
|
|
PAGE_TIMEOUT_MS,
|
|
SEARCH_BUDGET_MS,
|
|
ENGINE_NAMES,
|
|
htmlToText,
|
|
decodeSearchUrl,
|
|
setBrowserBackend,
|
|
runWebSearch,
|
|
googleSearchWithFallback,
|
|
webSearch,
|
|
wikiSearch,
|
|
hnSearch,
|
|
codeSearch,
|
|
webFetch,
|
|
fetchPage,
|
|
};
|