import test from 'node:test';
import assert from 'node:assert/strict';
import { createRequire } from 'node:module';
const require = createRequire(import.meta.url);
const web = require('../vendor/agent-harness/agent/web-search.js');
const reader = require('../vendor/agent-harness/agent/web-reader.js');
const response = (url, text, status = 200, headers = {}) => ({ url: String(url), status, text: async () => text, headers: { get: key => headers[key] || null } });
test('reader extracts structured content, relative links, entities, pagination and find', () => {
const html = `A & BGuide 🚀
${'Useful text. '.repeat(40)}
Next`;
const page = reader.extractPage(html, 'https://example.com/docs/start', { max_chars: 200, find: 'Useful' });
assert.equal(page.title, 'A & B');
assert.equal(page.headings[0].text, 'Guide 🚀');
assert.equal(page.links[0].url, 'https://example.com/next?q=one');
assert.equal(page.metadata.description, 'A guide');
assert.equal(page.next_offset, 200);
assert.equal(page.matches.length, 20);
assert.doesNotMatch(page.text, /navigation|malicious/);
assert.equal(reader.extractPage(html, 'https://example.com', { offset: 200 }).text, reader.extractPage(html, 'https://example.com').text.slice(200));
});
test('DDG parses reordered, single-quoted attributes and snippets', () => {
const hits = web.parseDdgHtmlHits(`The snippet
`);
assert.equal(hits[0].url, 'https://example.com/a%3Fb');
assert.equal(hits[0].title, 'Title — test');
assert.equal(hits[0].snippet, 'The snippet');
assert.equal(web.parseDdgLiteHits(`Lite`)[0].title, 'Lite');
});
test('auto merges scraped engines and removes tracking duplicates without calling APIs', async () => {
const orig = globalThis.fetch; const calls = [];
globalThis.fetch = async url => {
calls.push(String(url));
if (String(url).includes('duckduckgo')) return response(url, 'Example');
if (String(url).includes('bing')) return response(url, '- Examplehttps://example.com/a
- Otherhttps://other.example/b
');
return response(url, '');
};
try {
const hits = await web.runWebSearch('example', { limit: 2, timeoutMs: 500 });
assert.equal(hits.length, 2);
assert.deepEqual(hits[0].sources, ['duckduckgo', 'bing_rss']);
assert.equal(hits[0].url, 'https://example.com/a');
assert.ok(calls.every(url => !/jina|api\.|\/api\//.test(url)));
assert.match((await web.runWebSearch('test', { engine: 'jina' })).error, /unknown engine/);
} finally { globalThis.fetch = orig; }
});
test('direct fetch checks redirects, rejects binary pages, and reports challenges', async () => {
const orig = globalThis.fetch; let calls = 0;
try {
globalThis.fetch = async url => { calls++; return response(url, '', 302, { location: 'http://127.0.0.1/secret' }); };
assert.match((await web.fetchPage('https://example.com')).error, /blocked/);
assert.equal(calls, 1);
globalThis.fetch = async url => response(url, 'binary', 200, { 'content-type': 'application/pdf' });
assert.match((await web.fetchPage('https://example.com')).error, /unsupported content type/);
globalThis.fetch = async url => response(url, 'Verify you are human
');
assert.match((await web.fetchPage('https://challenge.example.com')).warning, /challenge/);
globalThis.fetch = async url => response(url, 'x'.repeat(2 * 1024 * 1024 + 1));
assert.match((await web.fetchPage('https://example.com')).error, /limit/);
} finally { globalThis.fetch = orig; }
});
test('stalled streaming bodies are cancelled at the deadline', async () => {
let cancelled = false;
const res = new Response(new ReadableStream({ cancel() { cancelled = true; } }));
await assert.rejects(web.readBodyWithTimeout(res, 30), /timed out/);
assert.equal(cancelled, true);
});
test('challenge responses fall back and cool down related search hosts', async () => {
const orig = globalThis.fetch; const calls = [];
globalThis.fetch = async url => {
calls.push(String(url));
if (String(url).includes('duckduckgo')) return response(url, '');
if (String(url).includes('bing')) return response(url, '- Availablehttps://available.example/page
');
return response(url, '');
};
try {
const hits = await web.runWebSearch('fallback', { limit: 1 });
assert.equal(hits[0].source, 'bing_rss');
const blocked = await web.runWebSearch('fallback', { engine: 'ddg_lite' });
assert.equal(blocked.code, 'bot_challenge');
assert.ok(blocked.retry_after_ms > 0);
assert.equal(calls.filter(url => url.includes('duckduckgo')).length, 1);
} finally { globalThis.fetch = orig; }
});
test('rate limits preserve retry guidance and never expose challenge content', async () => {
const orig = globalThis.fetch; let calls = 0;
globalThis.fetch = async url => { calls++; return response(url, 'blocked content', 429, { 'retry-after': '120' }); };
try {
const result = await web.fetchPage('https://limited.example/page');
assert.equal(result.code, 'rate_limited');
assert.equal(result.retry_after_ms, 120000);
assert.equal(result.text, undefined);
assert.match(result.next_action, /browser/);
await web.fetchPage('https://limited.example/other');
assert.equal(calls, 1);
} finally { globalThis.fetch = orig; }
});