Raven Scott
9bdecc4170
Release rolling / release (push) Successful in 11m16s
Bump published pins (compact-encoding 3, bare-fetch/tls/https/ws 3, bare-subprocess 6, bare-signals 5, corestore 7.12, protomux 3.11, hypercore-crypto 3.7, bare-runtime 1.31) and regenerate catalogs, manifests, and kernel/seeder bundles. Adapt call sites to the new APIs: - Corestore: explicit session flush before suspend(); treeCache ctor opts - bare-crypto: KeyObject.export() instead of removed ._key - Protomux 3.11: wait for fullyOpened()/fullyClosed() on chat channels - bare-fetch: surface response.type and Headers.getSetCookie - host snapshots: bare-os 3.9 / bare-posix / bare-fs.statfs frsize - bare-subprocess 6: optional IPC channel + json serialization Keep catalog sync from wiping curated pearEntries. Teach the Node test shim to stub bare-thread/bare-worker (ESM absolute paths) and chain Bare.on so bare-timers can load. Booter 479, protocol 34, seeder 14.
714 lines
19 KiB
JavaScript
714 lines
19 KiB
JavaScript
/** HTTP fetch + HTML extract helpers for agent `web_fetch` tool (preamble for /bin/agent). */
|
|
|
|
/**
|
|
* @param {unknown} e
|
|
* @returns {string}
|
|
*/
|
|
function bareWebFmtErr(e) {
|
|
if (e === undefined)
|
|
return 'promise_rejected_with_undefined (no rejection reason)'
|
|
if (e === null) return 'promise_rejected_with_null'
|
|
if (typeof e === 'string') return e
|
|
if (typeof e !== 'object') return String(e)
|
|
const o = /** @type {Record<string, unknown>} */ (e)
|
|
const msg = o.message
|
|
if (typeof msg === 'string' && msg.trim())
|
|
return bareWebFmtErrAugment(o, msg.trim())
|
|
if (typeof msg === 'number' || typeof msg === 'boolean')
|
|
return bareWebFmtErrAugment(o, String(msg))
|
|
const nm = o.name
|
|
const code = o.code
|
|
const errno = o.errno
|
|
/** @type {string[]} */
|
|
const bits = []
|
|
if (typeof nm === 'string' && nm.trim()) bits.push(nm)
|
|
if (code !== undefined && code !== null && String(code) !== '')
|
|
bits.push('code=' + String(code))
|
|
if (errno !== undefined && errno !== null && String(errno) !== '')
|
|
bits.push('errno=' + String(errno))
|
|
const cause = o.cause
|
|
if (cause !== undefined && cause !== null && cause !== e) {
|
|
const cs = bareWebFmtErr(cause)
|
|
if (cs && cs !== 'unknown_error')
|
|
bits.push('cause=(' + cs.slice(0, 280) + ')')
|
|
}
|
|
const errs = o.errors
|
|
if (Array.isArray(errs) && errs.length) {
|
|
errs.slice(0, 5).forEach((sub, i) => {
|
|
bits.push('agg' + i + '=' + bareWebFmtErr(sub).slice(0, 120))
|
|
})
|
|
}
|
|
if (bits.length) return bits.join(' ')
|
|
try {
|
|
const j = JSON.stringify(o)
|
|
if (j && j !== '{}' && j !== '[]') return j.slice(0, 400)
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
try {
|
|
if (
|
|
typeof /** @type {{ toString?: () => string }} */ (o).toString ===
|
|
'function'
|
|
) {
|
|
const t = /** @type {{ toString: () => string }} */ (o).toString()
|
|
if (t && t !== '[object Object]') return t.slice(0, 400)
|
|
}
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
return 'unknown_error'
|
|
}
|
|
|
|
/**
|
|
* Append errno/syscall from Node-ish errors when message alone is vague.
|
|
* @param {Record<string, unknown>} o
|
|
* @param {string} base
|
|
*/
|
|
function bareWebFmtErrAugment(o, base) {
|
|
const syscall = o.syscall
|
|
const code = o.code
|
|
const errno = o.errno
|
|
/** @type {string[]} */
|
|
const tail = []
|
|
if (typeof syscall === 'string' && syscall.trim())
|
|
tail.push('syscall=' + syscall)
|
|
if (code !== undefined && code !== null && String(code) !== '')
|
|
tail.push(String(code))
|
|
if (errno !== undefined && errno !== null && String(errno) !== '')
|
|
tail.push('errno=' + String(errno))
|
|
const cause = o.cause
|
|
if (cause !== undefined && cause !== null) {
|
|
const cs = bareWebFmtErr(cause)
|
|
if (cs && cs !== 'unknown_error')
|
|
tail.push('cause=(' + cs.slice(0, 240) + ')')
|
|
}
|
|
return tail.length ? base + ' [' + tail.join(', ') + ']' : base
|
|
}
|
|
|
|
/**
|
|
* Tool args often omit numeric fields; `Number(undefined)` is NaN and `NaN ?? d` is still NaN.
|
|
* @param {unknown} n
|
|
* @param {number} def
|
|
*/
|
|
function bareWebFiniteOr(n, def) {
|
|
const x = Number(n)
|
|
return Number.isFinite(x) ? x : def
|
|
}
|
|
|
|
/**
|
|
* @param {Record<string, unknown>} ctx
|
|
* @returns {typeof fetch | null}
|
|
*/
|
|
function bareWebResolveFetch(ctx) {
|
|
if (typeof ctx.httpFetch === 'function')
|
|
return /** @type {typeof fetch} */ (ctx.httpFetch.bind(ctx))
|
|
const bare = ctx.bare && typeof ctx.bare === 'object' ? ctx.bare : null
|
|
let f =
|
|
bare && typeof bare.fetch === 'function'
|
|
? bare.fetch
|
|
: bare &&
|
|
bare.default &&
|
|
typeof bare.default === 'object' &&
|
|
typeof bare.default.fetch === 'function'
|
|
? bare.default.fetch
|
|
: null
|
|
if (typeof f === 'function') return /** @type {typeof fetch} */ (f.bind(bare))
|
|
if (typeof globalThis.fetch === 'function')
|
|
return globalThis.fetch.bind(globalThis)
|
|
return null
|
|
}
|
|
|
|
/**
|
|
* bundled bare-fetch rejects the fetch promise with `signal.reason` on abort.
|
|
* `controller.abort()` with no argument sets `reason === undefined`, so callers
|
|
* see `promise_rejected_with_undefined`. Always pass an explicit reason.
|
|
* @param {number} timeoutMs
|
|
*/
|
|
function bareWebTimeoutAbortReason(timeoutMs) {
|
|
const msg = 'web_fetch: exceeded ' + timeoutMs + 'ms (timeout)'
|
|
try {
|
|
if (typeof DOMException === 'function')
|
|
return new DOMException(msg, 'TimeoutError')
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
const e = new Error(msg)
|
|
e.name = 'TimeoutError'
|
|
return e
|
|
}
|
|
|
|
/**
|
|
* @param {AbortSignal} sig
|
|
*/
|
|
function bareWebSignalAbortReason(sig) {
|
|
try {
|
|
const r = /** @type {{ reason?: unknown }} */ (sig).reason
|
|
if (r !== undefined && r !== null) return r
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
const e = new Error('web_fetch aborted (signal)')
|
|
e.name = 'AbortError'
|
|
return e
|
|
}
|
|
|
|
/**
|
|
* @param {AbortSignal | null | undefined} a
|
|
* @param {AbortSignal | null | undefined} b
|
|
*/
|
|
function bareWebUnionAbort(a, b) {
|
|
if (!a) return b || undefined
|
|
if (!b) return a
|
|
if (typeof AbortSignal.any === 'function') return AbortSignal.any([a, b])
|
|
const c = new AbortController()
|
|
/**
|
|
* @param {AbortSignal} sig
|
|
*/
|
|
const forward = (sig) => {
|
|
try {
|
|
c.abort(bareWebSignalAbortReason(sig))
|
|
} catch {
|
|
/* ignore — second source may fire after controller already aborted */
|
|
}
|
|
}
|
|
try {
|
|
const as = /** @type {AbortSignal} */ (a)
|
|
const bs = /** @type {AbortSignal} */ (b)
|
|
if (as.aborted) forward(as)
|
|
else as.addEventListener('abort', () => forward(as), { once: true })
|
|
if (bs.aborted) forward(bs)
|
|
else bs.addEventListener('abort', () => forward(bs), { once: true })
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
return c.signal
|
|
}
|
|
|
|
/**
|
|
* @param {Uint8Array[]} parts
|
|
*/
|
|
function bareWebConcatUint8(parts) {
|
|
let n = 0
|
|
for (const p of parts) n += p.length
|
|
const out = new Uint8Array(n)
|
|
let o = 0
|
|
for (const p of parts) {
|
|
out.set(p, o)
|
|
o += p.length
|
|
}
|
|
return out
|
|
}
|
|
|
|
/**
|
|
* @param {Response} res
|
|
* @param {number} maxBytes
|
|
* @param {AbortSignal | undefined} signal
|
|
*/
|
|
async function bareWebReadBodyLimited(res, maxBytes, signal) {
|
|
if (!res.body || typeof res.body.getReader !== 'function') {
|
|
try {
|
|
const ab = await res.arrayBuffer()
|
|
const u8 = new Uint8Array(ab)
|
|
return {
|
|
bytes: u8.byteLength > maxBytes ? u8.slice(0, maxBytes) : u8,
|
|
truncated: u8.byteLength > maxBytes
|
|
}
|
|
} catch {
|
|
return { bytes: new Uint8Array(0), truncated: false }
|
|
}
|
|
}
|
|
const reader = res.body.getReader()
|
|
/** @type {Uint8Array[]} */
|
|
const chunks = []
|
|
let total = 0
|
|
try {
|
|
for (;;) {
|
|
if (signal && signal.aborted) {
|
|
try {
|
|
await reader.cancel()
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
break
|
|
}
|
|
const { done, value } = await reader.read()
|
|
if (done) break
|
|
if (!value || !value.length) continue
|
|
total += value.length
|
|
if (total > maxBytes) {
|
|
const prev = total - value.length
|
|
const take = Math.max(0, maxBytes - prev)
|
|
if (take > 0) chunks.push(value.subarray(0, take))
|
|
try {
|
|
await reader.cancel()
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
return { bytes: bareWebConcatUint8(chunks), truncated: true }
|
|
}
|
|
chunks.push(value)
|
|
}
|
|
} finally {
|
|
try {
|
|
reader.releaseLock()
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
return { bytes: bareWebConcatUint8(chunks), truncated: false }
|
|
}
|
|
|
|
/**
|
|
* @param {string | null | undefined} ct
|
|
*/
|
|
function bareWebCharsetFromContentType(ct) {
|
|
const m = /charset\s*=\s*["']?([^"';\s]+)/i.exec(String(ct || ''))
|
|
return (m ? m[1] : 'utf-8').trim().toLowerCase()
|
|
}
|
|
|
|
/**
|
|
* @param {Uint8Array} bytes
|
|
* @param {string} label
|
|
*/
|
|
function bareWebDecodeBytes(bytes, label) {
|
|
try {
|
|
const dec = new TextDecoder(label || 'utf-8', {
|
|
fatal: false,
|
|
ignoreBOM: true
|
|
})
|
|
return dec.decode(bytes)
|
|
} catch {
|
|
return new TextDecoder('utf-8', { fatal: false }).decode(bytes)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param {string} s
|
|
*/
|
|
function bareWebDecodeHtmlEntities(s) {
|
|
let t = String(s || '')
|
|
t = t.replace(/ /gi, ' ')
|
|
t = t.replace(/"/gi, '"')
|
|
t = t.replace(/'/g, "'")
|
|
t = t.replace(/'/gi, "'")
|
|
t = t.replace(/&/gi, '&')
|
|
t = t.replace(/</gi, '<')
|
|
t = t.replace(/>/gi, '>')
|
|
t = t.replace(/&#x([0-9a-f]+);/gi, (_, h) => {
|
|
const c = parseInt(h, 16)
|
|
return Number.isFinite(c) ? String.fromCodePoint(c) : _
|
|
})
|
|
t = t.replace(/&#(\d+);/g, (_, d) => {
|
|
const c = parseInt(d, 10)
|
|
return Number.isFinite(c) ? String.fromCodePoint(c) : _
|
|
})
|
|
return t
|
|
}
|
|
|
|
/**
|
|
* @param {string} html
|
|
*/
|
|
function bareWebExtractHtmlText(html) {
|
|
let s = String(html || '')
|
|
s = s.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '')
|
|
s = s.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '')
|
|
s = s.replace(/<noscript\b[^<]*(?:(?!<\/noscript>)<[^<]*)*<\/noscript>/gi, '')
|
|
s = s.replace(/<!--[\s\S]*?-->/g, '')
|
|
s = s.replace(/<[^>]+>/g, ' ')
|
|
s = bareWebDecodeHtmlEntities(s)
|
|
s = s.replace(/\s+/g, ' ').trim()
|
|
return s
|
|
}
|
|
|
|
/**
|
|
* @param {string} html
|
|
* @param {string} baseUrl
|
|
* @param {number} maxLinks
|
|
*/
|
|
function bareWebExtractLinks(html, baseUrl, maxLinks) {
|
|
const cap = Math.min(Math.max(Number(maxLinks) || 200, 1), 500)
|
|
let uBase = null
|
|
try {
|
|
uBase = baseUrl ? new URL(String(baseUrl)) : null
|
|
} catch {
|
|
uBase = null
|
|
}
|
|
const seen = new Set()
|
|
/** @type {string[]} */
|
|
const out = []
|
|
const re = /<a\b[^>]*\bhref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi
|
|
let m
|
|
const h = String(html || '')
|
|
while ((m = re.exec(h)) !== null) {
|
|
const href = (m[1] || m[2] || m[3] || '').trim()
|
|
if (!href || href.startsWith('javascript:') || href.startsWith('#'))
|
|
continue
|
|
try {
|
|
const abs = uBase ? new URL(href, uBase).href : new URL(href).href
|
|
const proto = new URL(abs).protocol
|
|
if (proto !== 'http:' && proto !== 'https:') continue
|
|
if (!seen.has(abs)) {
|
|
seen.add(abs)
|
|
out.push(abs)
|
|
}
|
|
} catch {
|
|
/* skip */
|
|
}
|
|
if (out.length >= cap) break
|
|
}
|
|
return { links: out, links_truncated: out.length >= cap }
|
|
}
|
|
|
|
/**
|
|
* @param {string} tag
|
|
*/
|
|
function bareWebMetaContent(tag) {
|
|
const q =
|
|
/content\s*=\s*"([^"]*)"/i.exec(tag) ||
|
|
/content\s*=\s*'([^']*)'/i.exec(tag) ||
|
|
/content\s*=\s*([^\s>]+)/i.exec(tag)
|
|
return q ? bareWebDecodeHtmlEntities(q[1]).trim() : ''
|
|
}
|
|
|
|
/**
|
|
* @param {string} html
|
|
*/
|
|
function bareWebExtractMeta(html) {
|
|
const h = String(html || '')
|
|
const titleM = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(h)
|
|
const title = titleM
|
|
? bareWebDecodeHtmlEntities(titleM[1].replace(/<[^>]+>/g, ' ')).trim()
|
|
: ''
|
|
let description = ''
|
|
const metaDescRe = /<meta[^>]*\bname\s*=\s*["']description["'][^>]*>/i.exec(h)
|
|
if (metaDescRe) description = bareWebMetaContent(metaDescRe[0])
|
|
|
|
let og_title = ''
|
|
const ogT = /<meta[^>]*\bproperty\s*=\s*["']og:title["'][^>]*>/i.exec(h)
|
|
if (ogT) og_title = bareWebMetaContent(ogT[0])
|
|
|
|
let og_description = ''
|
|
const ogD = /<meta[^>]*\bproperty\s*=\s*["']og:description["'][^>]*>/i.exec(h)
|
|
if (ogD) og_description = bareWebMetaContent(ogD[0])
|
|
|
|
return {
|
|
title,
|
|
description,
|
|
og_title,
|
|
og_description
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param {string} text
|
|
*/
|
|
function bareWebMaybeParseJson(text) {
|
|
try {
|
|
return { ok: true, value: JSON.parse(String(text)) }
|
|
} catch {
|
|
return { ok: false }
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param {string} ct
|
|
*/
|
|
function bareWebLooksLikeHtml(ct) {
|
|
return /\btext\/html\b/i.test(String(ct || ''))
|
|
}
|
|
|
|
/**
|
|
* @param {string} ct
|
|
*/
|
|
function bareWebLooksLikeJson(ct) {
|
|
const s = String(ct || '').toLowerCase()
|
|
return (
|
|
/\bapplication\/json\b/.test(s) ||
|
|
/\bapplication\/.*\+json\b/.test(s) ||
|
|
/\btext\/json\b/.test(s)
|
|
)
|
|
}
|
|
|
|
/**
|
|
* @param {{
|
|
* ctx: Record<string, unknown>,
|
|
* url: string,
|
|
* method?: string,
|
|
* headers?: Record<string, unknown>,
|
|
* body?: string,
|
|
* content_type?: string,
|
|
* max_response_bytes?: number,
|
|
* max_redirects?: number,
|
|
* timeout_ms?: number,
|
|
* format?: string,
|
|
* max_links?: number,
|
|
* signal?: AbortSignal | null
|
|
* }} o
|
|
*/
|
|
async function bareWebRunTool(o) {
|
|
const ctx = o.ctx
|
|
const fetchFn = bareWebResolveFetch(ctx)
|
|
if (!fetchFn) {
|
|
return {
|
|
ok: false,
|
|
error:
|
|
'web_fetch: no HTTP client (set ctx.httpFetch, bare.fetch, or global fetch)'
|
|
}
|
|
}
|
|
|
|
let startUrl = String(o.url || '').trim()
|
|
let method = String(o.method || 'GET').toUpperCase()
|
|
if (
|
|
!['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'].includes(
|
|
method
|
|
)
|
|
) {
|
|
return { ok: false, error: 'web_fetch: unsupported method' }
|
|
}
|
|
|
|
let body =
|
|
o.body != null && method !== 'GET' && method !== 'HEAD'
|
|
? String(o.body)
|
|
: undefined
|
|
|
|
let u0
|
|
try {
|
|
u0 = new URL(startUrl)
|
|
} catch {
|
|
return { ok: false, error: 'web_fetch: invalid URL' }
|
|
}
|
|
if (u0.protocol !== 'http:' && u0.protocol !== 'https:') {
|
|
return { ok: false, error: 'web_fetch: only http(s) URLs are allowed' }
|
|
}
|
|
|
|
const maxRedirects = Math.min(
|
|
Math.max(bareWebFiniteOr(o.max_redirects, 5), 0),
|
|
20
|
|
)
|
|
const maxBytes = Math.min(
|
|
Math.max(bareWebFiniteOr(o.max_response_bytes, 524288), 1024),
|
|
2 * 1024 * 1024
|
|
)
|
|
const timeoutMs = Math.min(
|
|
Math.max(bareWebFiniteOr(o.timeout_ms, 30000), 500),
|
|
120000
|
|
)
|
|
const fmtRaw = String(o.format || 'auto').toLowerCase()
|
|
const maxLinks = Number(o.max_links) || 200
|
|
|
|
/** @type {string[]} */
|
|
const redirectChain = [startUrl]
|
|
let currentUrl = startUrl
|
|
let redirectsUsed = 0
|
|
|
|
for (;;) {
|
|
const controller = new AbortController()
|
|
const timer = setTimeout(() => {
|
|
try {
|
|
controller.abort(bareWebTimeoutAbortReason(timeoutMs))
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}, timeoutMs)
|
|
const signal = bareWebUnionAbort(o.signal || undefined, controller.signal)
|
|
|
|
/** @type {Record<string, string>} */
|
|
const hdrObj = {}
|
|
const hin = o.headers
|
|
if (hin && typeof hin === 'object' && !Array.isArray(hin)) {
|
|
for (const [k, v] of Object.entries(hin)) {
|
|
if (typeof v === 'string' && k) hdrObj[k] = v
|
|
}
|
|
}
|
|
if (
|
|
body != null &&
|
|
method !== 'GET' &&
|
|
method !== 'HEAD' &&
|
|
!Object.keys(hdrObj).some((k) => k.toLowerCase() === 'content-type')
|
|
) {
|
|
hdrObj['Content-Type'] =
|
|
typeof o.content_type === 'string' && o.content_type.trim()
|
|
? o.content_type.trim()
|
|
: 'application/octet-stream'
|
|
}
|
|
|
|
/** @type {RequestInit} */
|
|
const init = {
|
|
method,
|
|
headers: hdrObj,
|
|
signal: signal || undefined,
|
|
redirect: 'manual'
|
|
}
|
|
if (body != null && method !== 'GET' && method !== 'HEAD') {
|
|
init.body = body
|
|
}
|
|
|
|
let res
|
|
try {
|
|
res = await fetchFn(currentUrl, init)
|
|
} catch (e) {
|
|
clearTimeout(timer)
|
|
const msg = bareWebFmtErr(
|
|
e === undefined
|
|
? new Error(
|
|
'web_fetch: fetch rejected with undefined (bare-fetch uses signal.reason; upstream abort() had no reason)'
|
|
)
|
|
: e
|
|
)
|
|
return {
|
|
ok: false,
|
|
error: 'web_fetch: request failed: ' + msg.slice(0, 400),
|
|
url_final: currentUrl,
|
|
redirect_chain: redirectChain
|
|
}
|
|
}
|
|
clearTimeout(timer)
|
|
|
|
const st = res.status
|
|
if (st >= 300 && st < 400) {
|
|
if (redirectsUsed >= maxRedirects) {
|
|
return {
|
|
ok: false,
|
|
error: 'web_fetch: too many redirects',
|
|
status: st,
|
|
url_final: currentUrl,
|
|
redirect_chain: redirectChain
|
|
}
|
|
}
|
|
const loc = res.headers.get('Location')
|
|
if (!loc) {
|
|
return {
|
|
ok: false,
|
|
error: 'web_fetch: redirect without Location',
|
|
status: st,
|
|
url_final: currentUrl,
|
|
redirect_chain: redirectChain
|
|
}
|
|
}
|
|
let nextUrl
|
|
try {
|
|
nextUrl = new URL(loc, currentUrl).href
|
|
} catch {
|
|
return {
|
|
ok: false,
|
|
error: 'web_fetch: bad redirect URL',
|
|
status: st,
|
|
url_final: currentUrl,
|
|
redirect_chain: redirectChain
|
|
}
|
|
}
|
|
redirectChain.push(nextUrl)
|
|
currentUrl = nextUrl
|
|
redirectsUsed++
|
|
if (st === 301 || st === 302 || st === 303) {
|
|
method = 'GET'
|
|
body = undefined
|
|
}
|
|
continue
|
|
}
|
|
|
|
const ct = res.headers.get('content-type') || ''
|
|
const url_final = currentUrl
|
|
const responseType = typeof res.type === 'string' ? res.type : undefined
|
|
let setCookie
|
|
try {
|
|
const hdrs = res.headers
|
|
if (hdrs && typeof hdrs.getSetCookie === 'function') {
|
|
const sc = hdrs.getSetCookie()
|
|
if (Array.isArray(sc) && sc.length) setCookie = sc.slice(0, 32)
|
|
}
|
|
} catch {
|
|
setCookie = undefined
|
|
}
|
|
|
|
if (method === 'HEAD') {
|
|
return {
|
|
ok: true,
|
|
url_final,
|
|
status: st,
|
|
content_type: ct,
|
|
response_type: responseType,
|
|
set_cookie: setCookie,
|
|
redirect_chain: redirectChain.length > 1 ? redirectChain : undefined,
|
|
truncated: false,
|
|
extract: { note: 'HEAD — body omitted' }
|
|
}
|
|
}
|
|
|
|
let bodyRead
|
|
try {
|
|
bodyRead = await bareWebReadBodyLimited(
|
|
res,
|
|
maxBytes,
|
|
signal || undefined
|
|
)
|
|
} catch (e) {
|
|
const msg = bareWebFmtErr(e)
|
|
return {
|
|
ok: false,
|
|
error: 'web_fetch: read body failed: ' + msg.slice(0, 400),
|
|
url_final,
|
|
status: st,
|
|
redirect_chain: redirectChain.length > 1 ? redirectChain : undefined
|
|
}
|
|
}
|
|
|
|
const charset = bareWebCharsetFromContentType(ct)
|
|
const text = bareWebDecodeBytes(bodyRead.bytes, charset)
|
|
|
|
const fmt =
|
|
fmtRaw === 'auto'
|
|
? bareWebLooksLikeJson(ct)
|
|
? 'json'
|
|
: bareWebLooksLikeHtml(ct)
|
|
? 'markdownish'
|
|
: 'raw'
|
|
: fmtRaw
|
|
|
|
/** @type {unknown} */
|
|
let extract
|
|
|
|
if (fmt === 'json') {
|
|
const p = bareWebMaybeParseJson(text)
|
|
extract = p.ok
|
|
? { json: p.value }
|
|
: { parse_error: true, text_slice: text.slice(0, 8000) }
|
|
} else if (fmt === 'links') {
|
|
extract = bareWebExtractLinks(text, url_final, maxLinks)
|
|
} else if (fmt === 'meta') {
|
|
extract = bareWebExtractMeta(text)
|
|
} else if (fmt === 'markdownish' || fmt === 'text') {
|
|
const plain = bareWebExtractHtmlText(text)
|
|
extract = {
|
|
text: plain,
|
|
approx_chars: plain.length
|
|
}
|
|
} else if (fmt === 'raw') {
|
|
extract = {
|
|
raw_text: text.length > 12000 ? text.slice(0, 12000) + '\n…' : text,
|
|
char_count: text.length
|
|
}
|
|
} else {
|
|
extract = {
|
|
text: text.length > 12000 ? text.slice(0, 12000) + '\n…' : text,
|
|
char_count: text.length
|
|
}
|
|
}
|
|
|
|
const raw_preview = text.slice(0, 2000)
|
|
|
|
return {
|
|
ok: true,
|
|
url_final,
|
|
status: st,
|
|
content_type: ct,
|
|
response_type: responseType,
|
|
set_cookie: setCookie,
|
|
redirect_chain: redirectChain.length > 1 ? redirectChain : undefined,
|
|
truncated: bodyRead.truncated,
|
|
extract,
|
|
raw_preview: fmt === 'raw' || fmt === 'json' ? undefined : raw_preview
|
|
}
|
|
}
|
|
}
|