242 lines
7.4 KiB
JavaScript
242 lines
7.4 KiB
JavaScript
/**
|
|
* Loads lib/agent/agent-web-fetch.js in a VM (same bundle shape as /bin/agent preamble).
|
|
*/
|
|
import test from 'brittle'
|
|
import { readFileSync } from 'node:fs'
|
|
import vm from 'node:vm'
|
|
|
|
const CODE = readFileSync(
|
|
new URL('../lib/agent/agent-web-fetch.js', import.meta.url),
|
|
'utf8'
|
|
)
|
|
|
|
/** @returns {Record<string, unknown>} */
|
|
function loadSandbox(extra = {}) {
|
|
const sandbox = {
|
|
URL,
|
|
TextDecoder,
|
|
TextEncoder,
|
|
Uint8Array,
|
|
AbortController,
|
|
AbortSignal,
|
|
ReadableStream,
|
|
Response,
|
|
Request,
|
|
setTimeout: globalThis.setTimeout.bind(globalThis),
|
|
clearTimeout: globalThis.clearTimeout.bind(globalThis),
|
|
console,
|
|
...extra
|
|
}
|
|
vm.createContext(sandbox)
|
|
vm.runInContext(CODE, sandbox, { filename: 'agent-web-fetch.js' })
|
|
return sandbox
|
|
}
|
|
|
|
test('bareWebExtractHtmlText strips scripts and tags', async (t) => {
|
|
const s = loadSandbox()
|
|
const fn = /** @type {(h: string) => string} */ (s.bareWebExtractHtmlText)
|
|
const html =
|
|
'<html><head><title>x</title></head><body><script>evil()</script><p>Hello & <!--c--><b>world</b></p></body></html>'
|
|
const out = fn(html)
|
|
t.ok(out.includes('Hello'))
|
|
t.ok(out.includes('world'))
|
|
t.absent(out.includes('script'))
|
|
t.ok(out.includes('&') || out.includes('world'))
|
|
})
|
|
|
|
test('bareWebExtractLinks resolves relative hrefs', async (t) => {
|
|
const s = loadSandbox()
|
|
const fn = /** @type {(h: string, b: string) => { links: string[] }} */ (
|
|
s.bareWebExtractLinks
|
|
)
|
|
const r = fn(
|
|
'<a href="/doc">x</a><a href="https://ex.org/a">y</a>',
|
|
'https://example.com/pages/'
|
|
)
|
|
t.ok(r.links.includes('https://example.com/doc'))
|
|
t.ok(r.links.includes('https://ex.org/a'))
|
|
})
|
|
|
|
test('bareWebExtractMeta title and og', async (t) => {
|
|
const s = loadSandbox()
|
|
const fn = /** @type {(h: string) => Record<string, string>} */ (
|
|
s.bareWebExtractMeta
|
|
)
|
|
const html = `<!doctype html><title>T < Test</title>
|
|
<meta name="description" content="Desc here">
|
|
<meta property="og:title" content="OG Title">
|
|
<meta property='og:description' content='OG Desc'>`
|
|
const m = fn(html)
|
|
t.is(m.title, 'T < Test')
|
|
t.ok(String(m.description).includes('Desc'))
|
|
t.is(m.og_title, 'OG Title')
|
|
t.is(m.og_description, 'OG Desc')
|
|
})
|
|
|
|
test('bareWebMaybeParseJson', async (t) => {
|
|
const s = loadSandbox()
|
|
const fn = /** @type {(x: string) => { ok: boolean, value?: unknown }} */ (
|
|
s.bareWebMaybeParseJson
|
|
)
|
|
t.ok(fn('{"a":1}').ok)
|
|
t.is(/** @type {{a:number}} */ (fn('{"a":1}').value).a, 1)
|
|
t.absent(fn('{').ok)
|
|
})
|
|
|
|
test('bareWebRunTool follows redirect then reads body', async (t) => {
|
|
let calls = 0
|
|
const fetchFn = async (url, init) => {
|
|
calls++
|
|
if (calls === 1) {
|
|
t.is(String(url), 'https://a.test/start')
|
|
return new Response('', {
|
|
status: 302,
|
|
headers: { Location: '/final' }
|
|
})
|
|
}
|
|
t.is(String(url), 'https://a.test/final')
|
|
t.is(init.method, 'GET')
|
|
return new Response('<html><body><p>OK</p></body></html>', {
|
|
status: 200,
|
|
headers: { 'Content-Type': 'text/html; charset=utf-8' }
|
|
})
|
|
}
|
|
const s = loadSandbox({ fetch: fetchFn })
|
|
const run = /** @type {typeof bareWebRunTool} */ (s.bareWebRunTool)
|
|
const ctx = { httpFetch: fetchFn }
|
|
const out = await run({
|
|
ctx,
|
|
url: 'https://a.test/start',
|
|
format: 'markdownish'
|
|
})
|
|
t.ok(out.ok)
|
|
t.is(out.status, 200)
|
|
t.ok(String(out.url_final).includes('final'))
|
|
t.ok(out.extract && typeof out.extract === 'object')
|
|
const ex = /** @type {{ text?: string }} */ (out.extract)
|
|
t.ok(ex.text && ex.text.includes('OK'))
|
|
})
|
|
|
|
test('bareWebRunTool truncates large body stream', async (t) => {
|
|
const chunk = new Uint8Array(60000).fill(65)
|
|
const fetchFn = async () =>
|
|
new Response(
|
|
new ReadableStream({
|
|
start(controller) {
|
|
controller.enqueue(chunk)
|
|
controller.enqueue(chunk)
|
|
controller.close()
|
|
}
|
|
}),
|
|
{
|
|
status: 200,
|
|
headers: { 'Content-Type': 'text/plain' }
|
|
}
|
|
)
|
|
const s = loadSandbox({ fetch: fetchFn })
|
|
const run = /** @type {typeof bareWebRunTool} */ (s.bareWebRunTool)
|
|
const out = await run({
|
|
ctx: { httpFetch: fetchFn },
|
|
url: 'https://big.test/x',
|
|
format: 'raw',
|
|
max_response_bytes: 50000
|
|
})
|
|
t.ok(out.ok)
|
|
t.ok(out.truncated)
|
|
})
|
|
|
|
test('bareWebFmtErr null/undefined and node-like errors', async (t) => {
|
|
const s = loadSandbox()
|
|
const fmt = /** @type {(e: unknown) => string} */ (s.bareWebFmtErr)
|
|
t.ok(fmt(null).includes('rejected_with_null'))
|
|
t.ok(fmt(undefined).includes('rejected_with_undefined'))
|
|
t.absent(fmt({ message: undefined }) === 'undefined')
|
|
t.ok(fmt({ name: 'TypeError', code: 'ETIMEDOUT' }).includes('ETIMEDOUT'))
|
|
const withCause = new Error('outer')
|
|
withCause.cause = new Error('inner')
|
|
t.ok(fmt(withCause).includes('inner'))
|
|
})
|
|
|
|
test('bareWebRunTool surfaces response.type and Headers.getSetCookie', async (t) => {
|
|
const fetchFn = async () => {
|
|
const res = new Response('ok', {
|
|
status: 200,
|
|
headers: {
|
|
'Content-Type': 'text/plain',
|
|
'Set-Cookie': 'a=1; Path=/'
|
|
}
|
|
})
|
|
if (res.type == null) Object.defineProperty(res, 'type', { value: 'basic' })
|
|
if (typeof res.headers.getSetCookie !== 'function') {
|
|
res.headers.getSetCookie = () => ['a=1; Path=/']
|
|
}
|
|
return res
|
|
}
|
|
const s = loadSandbox({ fetch: fetchFn })
|
|
const run = /** @type {typeof bareWebRunTool} */ (s.bareWebRunTool)
|
|
const out = await run({
|
|
ctx: { httpFetch: fetchFn },
|
|
url: 'https://x.test/',
|
|
format: 'raw'
|
|
})
|
|
t.ok(out.ok)
|
|
t.ok(out.response_type === 'basic' || typeof out.response_type === 'string')
|
|
t.ok(Array.isArray(out.set_cookie) || out.set_cookie === undefined)
|
|
})
|
|
|
|
test('bareWebRunTool without timeout_ms uses finite default (not NaN)', async (t) => {
|
|
const fetchFn = async () =>
|
|
new Response('<html><head><title>ok</title></head></html>', {
|
|
status: 200,
|
|
headers: { 'Content-Type': 'text/html; charset=utf-8' }
|
|
})
|
|
const s = loadSandbox({ fetch: fetchFn })
|
|
const run = /** @type {typeof bareWebRunTool} */ (s.bareWebRunTool)
|
|
const out = await run({
|
|
ctx: { httpFetch: fetchFn },
|
|
url: 'https://x.test/',
|
|
format: 'meta'
|
|
})
|
|
t.ok(out.ok)
|
|
t.absent(JSON.stringify(out).includes('NaN'))
|
|
})
|
|
|
|
test('bareWebRunTool timeout passes abort reason (matches bare-fetch contract)', async (t) => {
|
|
/** Simulates bare-fetch: rejects with `signal.reason` when aborted. */
|
|
const fetchFn = async (_url, init) => {
|
|
const sig = init && /** @type {{ signal?: AbortSignal }} */ (init).signal
|
|
if (!sig)
|
|
return new Promise(() => {
|
|
/* hang */
|
|
})
|
|
return new Promise((_res, rej) => {
|
|
if (sig.aborted) {
|
|
rej(/** @type {AbortSignal} */ (sig).reason)
|
|
return
|
|
}
|
|
sig.addEventListener(
|
|
'abort',
|
|
() => rej(/** @type {AbortSignal} */ (sig).reason),
|
|
{ once: true }
|
|
)
|
|
})
|
|
}
|
|
const s = loadSandbox({ fetch: fetchFn })
|
|
const run = /** @type {typeof bareWebRunTool} */ (s.bareWebRunTool)
|
|
const out = await run({
|
|
ctx: { httpFetch: fetchFn },
|
|
url: 'https://slow.example/hang',
|
|
format: 'meta',
|
|
timeout_ms: 25
|
|
})
|
|
t.absent(out.ok)
|
|
const err = String(/** @type {{ error?: string }} */ (out).error || '')
|
|
t.ok(err.includes('web_fetch: request failed'))
|
|
t.ok(
|
|
err.includes('timeout') ||
|
|
err.includes('TimeoutError') ||
|
|
err.includes('exceeded')
|
|
)
|
|
t.absent(err.includes('promise_rejected_with_undefined'))
|
|
})
|