102 lines
5.8 KiB
JavaScript
102 lines
5.8 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { createRequire } from 'node:module';
|
|
const require = createRequire(import.meta.url);
|
|
const web = require('../vendor/agent-harness/agent/web-search.js');
|
|
const reader = require('../vendor/agent-harness/agent/web-reader.js');
|
|
const response = (url, text, status = 200, headers = {}) => ({ url: String(url), status, text: async () => text, headers: { get: key => headers[key] || null } });
|
|
|
|
test('reader extracts structured content, relative links, entities, pagination and find', () => {
|
|
const html = `<html><head><title>A & B</title><meta name='description' content='A guide'></head><body><nav>Ignore navigation</nav><main><h1>Guide 🚀</h1><p>${'Useful text. '.repeat(40)}</p><a href='../next?utm_source=x&q=one'>Next</a><script>malicious()</script></main></body></html>`;
|
|
const page = reader.extractPage(html, 'https://example.com/docs/start', { max_chars: 200, find: 'Useful' });
|
|
assert.equal(page.title, 'A & B');
|
|
assert.equal(page.headings[0].text, 'Guide 🚀');
|
|
assert.equal(page.links[0].url, 'https://example.com/next?q=one');
|
|
assert.equal(page.metadata.description, 'A guide');
|
|
assert.equal(page.next_offset, 200);
|
|
assert.equal(page.matches.length, 20);
|
|
assert.doesNotMatch(page.text, /navigation|malicious/);
|
|
assert.equal(reader.extractPage(html, 'https://example.com', { offset: 200 }).text, reader.extractPage(html, 'https://example.com').text.slice(200));
|
|
});
|
|
|
|
test('DDG parses reordered, single-quoted attributes and snippets', () => {
|
|
const hits = web.parseDdgHtmlHits(`<a href='/l/?uddg=https%3A%2F%2Fexample.com%2Fa%253Fb' class='extra result__a'>Title — test</a><div class='result__snippet'>The snippet</div>`);
|
|
assert.equal(hits[0].url, 'https://example.com/a%3Fb');
|
|
assert.equal(hits[0].title, 'Title — test');
|
|
assert.equal(hits[0].snippet, 'The snippet');
|
|
assert.equal(web.parseDdgLiteHits(`<a href='https://example.com' class='result-link'>Lite</a>`)[0].title, 'Lite');
|
|
});
|
|
|
|
test('auto merges scraped engines and removes tracking duplicates without calling APIs', async () => {
|
|
const orig = globalThis.fetch; const calls = [];
|
|
globalThis.fetch = async url => {
|
|
calls.push(String(url));
|
|
if (String(url).includes('duckduckgo')) return response(url, '<a class="result__a" href="https://example.com/a?utm_source=ddg">Example</a>');
|
|
if (String(url).includes('bing')) return response(url, '<rss><item><title>Example</title><link>https://example.com/a</link></item><item><title>Other</title><link>https://other.example/b</link></item></rss>');
|
|
return response(url, '');
|
|
};
|
|
try {
|
|
const hits = await web.runWebSearch('example', { limit: 2, timeoutMs: 500 });
|
|
assert.equal(hits.length, 2);
|
|
assert.deepEqual(hits[0].sources, ['duckduckgo', 'bing_rss']);
|
|
assert.equal(hits[0].url, 'https://example.com/a');
|
|
assert.ok(calls.every(url => !/jina|api\.|\/api\//.test(url)));
|
|
assert.match((await web.runWebSearch('test', { engine: 'jina' })).error, /unknown engine/);
|
|
} finally { globalThis.fetch = orig; }
|
|
});
|
|
|
|
test('direct fetch checks redirects, rejects binary pages, and reports challenges', async () => {
|
|
const orig = globalThis.fetch; let calls = 0;
|
|
try {
|
|
globalThis.fetch = async url => { calls++; return response(url, '', 302, { location: 'http://127.0.0.1/secret' }); };
|
|
assert.match((await web.fetchPage('https://example.com')).error, /blocked/);
|
|
assert.equal(calls, 1);
|
|
globalThis.fetch = async url => response(url, 'binary', 200, { 'content-type': 'application/pdf' });
|
|
assert.match((await web.fetchPage('https://example.com')).error, /unsupported content type/);
|
|
globalThis.fetch = async url => response(url, '<p>Verify you are human</p>');
|
|
assert.match((await web.fetchPage('https://challenge.example.com')).warning, /challenge/);
|
|
globalThis.fetch = async url => response(url, 'x'.repeat(2 * 1024 * 1024 + 1));
|
|
assert.match((await web.fetchPage('https://example.com')).error, /limit/);
|
|
} finally { globalThis.fetch = orig; }
|
|
});
|
|
|
|
test('stalled streaming bodies are cancelled at the deadline', async () => {
|
|
let cancelled = false;
|
|
const res = new Response(new ReadableStream({ cancel() { cancelled = true; } }));
|
|
await assert.rejects(web.readBodyWithTimeout(res, 30), /timed out/);
|
|
assert.equal(cancelled, true);
|
|
});
|
|
|
|
|
|
test('challenge responses fall back and cool down related search hosts', async () => {
|
|
const orig = globalThis.fetch; const calls = [];
|
|
globalThis.fetch = async url => {
|
|
calls.push(String(url));
|
|
if (String(url).includes('duckduckgo')) return response(url, '<form id="challenge-form">Verify you are human</form>');
|
|
if (String(url).includes('bing')) return response(url, '<rss><item><title>Available</title><link>https://available.example/page</link></item></rss>');
|
|
return response(url, '');
|
|
};
|
|
try {
|
|
const hits = await web.runWebSearch('fallback', { limit: 1 });
|
|
assert.equal(hits[0].source, 'bing_rss');
|
|
const blocked = await web.runWebSearch('fallback', { engine: 'ddg_lite' });
|
|
assert.equal(blocked.code, 'bot_challenge');
|
|
assert.ok(blocked.retry_after_ms > 0);
|
|
assert.equal(calls.filter(url => url.includes('duckduckgo')).length, 1);
|
|
} finally { globalThis.fetch = orig; }
|
|
});
|
|
|
|
test('rate limits preserve retry guidance and never expose challenge content', async () => {
|
|
const orig = globalThis.fetch; let calls = 0;
|
|
globalThis.fetch = async url => { calls++; return response(url, 'blocked content', 429, { 'retry-after': '120' }); };
|
|
try {
|
|
const result = await web.fetchPage('https://limited.example/page');
|
|
assert.equal(result.code, 'rate_limited');
|
|
assert.equal(result.retry_after_ms, 120000);
|
|
assert.equal(result.text, undefined);
|
|
assert.match(result.next_action, /browser/);
|
|
await web.fetchPage('https://limited.example/other');
|
|
assert.equal(calls, 1);
|
|
} finally { globalThis.fetch = orig; }
|
|
});
|