Orginize

packages/bare-os-coreutils/lib/agent/agent-web-fetch.js

@@ -0,0 +1,713 @@
+/** HTTP fetch + HTML extract helpers for agent `web_fetch` tool (preamble for /bin/agent). */
+
+/**
+ * @param {unknown} e
+ * @returns {string}
+ */
+function bareWebFmtErr(e) {
+  if (e === undefined)
+    return 'promise_rejected_with_undefined (no rejection reason)'
+  if (e === null) return 'promise_rejected_with_null'
+  if (typeof e === 'string') return e
+  if (typeof e !== 'object') return String(e)
+  const o = /** @type {Record<string, unknown>} */ (e)
+  const msg = o.message
+  if (typeof msg === 'string' && msg.trim())
+    return bareWebFmtErrAugment(o, msg.trim())
+  if (typeof msg === 'number' || typeof msg === 'boolean')
+    return bareWebFmtErrAugment(o, String(msg))
+  const nm = o.name
+  const code = o.code
+  const errno = o.errno
+  /** @type {string[]} */
+  const bits = []
+  if (typeof nm === 'string' && nm.trim()) bits.push(nm)
+  if (code !== undefined && code !== null && String(code) !== '')
+    bits.push('code=' + String(code))
+  if (errno !== undefined && errno !== null && String(errno) !== '')
+    bits.push('errno=' + String(errno))
+  const cause = o.cause
+  if (cause !== undefined && cause !== null && cause !== e) {
+    const cs = bareWebFmtErr(cause)
+    if (cs && cs !== 'unknown_error')
+      bits.push('cause=(' + cs.slice(0, 280) + ')')
+  }
+  const errs = o.errors
+  if (Array.isArray(errs) && errs.length) {
+    errs.slice(0, 5).forEach((sub, i) => {
+      bits.push('agg' + i + '=' + bareWebFmtErr(sub).slice(0, 120))
+    })
+  }
+  if (bits.length) return bits.join(' ')
+  try {
+    const j = JSON.stringify(o)
+    if (j && j !== '{}' && j !== '[]') return j.slice(0, 400)
+  } catch {
+    /* ignore */
+  }
+  try {
+    if (
+      typeof /** @type {{ toString?: () => string }} */ (o).toString ===
+      'function'
+    ) {
+      const t = /** @type {{ toString: () => string }} */ (o).toString()
+      if (t && t !== '[object Object]') return t.slice(0, 400)
+    }
+  } catch {
+    /* ignore */
+  }
+  return 'unknown_error'
+}
+
+/**
+ * Append errno/syscall from Node-ish errors when message alone is vague.
+ * @param {Record<string, unknown>} o
+ * @param {string} base
+ */
+function bareWebFmtErrAugment(o, base) {
+  const syscall = o.syscall
+  const code = o.code
+  const errno = o.errno
+  /** @type {string[]} */
+  const tail = []
+  if (typeof syscall === 'string' && syscall.trim())
+    tail.push('syscall=' + syscall)
+  if (code !== undefined && code !== null && String(code) !== '')
+    tail.push(String(code))
+  if (errno !== undefined && errno !== null && String(errno) !== '')
+    tail.push('errno=' + String(errno))
+  const cause = o.cause
+  if (cause !== undefined && cause !== null) {
+    const cs = bareWebFmtErr(cause)
+    if (cs && cs !== 'unknown_error')
+      tail.push('cause=(' + cs.slice(0, 240) + ')')
+  }
+  return tail.length ? base + ' [' + tail.join(', ') + ']' : base
+}
+
+/**
+ * Tool args often omit numeric fields; `Number(undefined)` is NaN and `NaN ?? d` is still NaN.
+ * @param {unknown} n
+ * @param {number} def
+ */
+function bareWebFiniteOr(n, def) {
+  const x = Number(n)
+  return Number.isFinite(x) ? x : def
+}
+
+/**
+ * @param {Record<string, unknown>} ctx
+ * @returns {typeof fetch | null}
+ */
+function bareWebResolveFetch(ctx) {
+  if (typeof ctx.httpFetch === 'function')
+    return /** @type {typeof fetch} */ (ctx.httpFetch.bind(ctx))
+  const bare = ctx.bare && typeof ctx.bare === 'object' ? ctx.bare : null
+  let f =
+    bare && typeof bare.fetch === 'function'
+      ? bare.fetch
+      : bare &&
+          bare.default &&
+          typeof bare.default === 'object' &&
+          typeof bare.default.fetch === 'function'
+        ? bare.default.fetch
+        : null
+  if (typeof f === 'function') return /** @type {typeof fetch} */ (f.bind(bare))
+  if (typeof globalThis.fetch === 'function')
+    return globalThis.fetch.bind(globalThis)
+  return null
+}
+
+/**
+ * bundled bare-fetch rejects the fetch promise with `signal.reason` on abort.
+ * `controller.abort()` with no argument sets `reason === undefined`, so callers
+ * see `promise_rejected_with_undefined`. Always pass an explicit reason.
+ * @param {number} timeoutMs
+ */
+function bareWebTimeoutAbortReason(timeoutMs) {
+  const msg = 'web_fetch: exceeded ' + timeoutMs + 'ms (timeout)'
+  try {
+    if (typeof DOMException === 'function')
+      return new DOMException(msg, 'TimeoutError')
+  } catch {
+    /* ignore */
+  }
+  const e = new Error(msg)
+  e.name = 'TimeoutError'
+  return e
+}
+
+/**
+ * @param {AbortSignal} sig
+ */
+function bareWebSignalAbortReason(sig) {
+  try {
+    const r = /** @type {{ reason?: unknown }} */ (sig).reason
+    if (r !== undefined && r !== null) return r
+  } catch {
+    /* ignore */
+  }
+  const e = new Error('web_fetch aborted (signal)')
+  e.name = 'AbortError'
+  return e
+}
+
+/**
+ * @param {AbortSignal | null | undefined} a
+ * @param {AbortSignal | null | undefined} b
+ */
+function bareWebUnionAbort(a, b) {
+  if (!a) return b || undefined
+  if (!b) return a
+  if (typeof AbortSignal.any === 'function') return AbortSignal.any([a, b])
+  const c = new AbortController()
+  /**
+   * @param {AbortSignal} sig
+   */
+  const forward = (sig) => {
+    try {
+      c.abort(bareWebSignalAbortReason(sig))
+    } catch {
+      /* ignore — second source may fire after controller already aborted */
+    }
+  }
+  try {
+    const as = /** @type {AbortSignal} */ (a)
+    const bs = /** @type {AbortSignal} */ (b)
+    if (as.aborted) forward(as)
+    else as.addEventListener('abort', () => forward(as), { once: true })
+    if (bs.aborted) forward(bs)
+    else bs.addEventListener('abort', () => forward(bs), { once: true })
+  } catch {
+    /* ignore */
+  }
+  return c.signal
+}
+
+/**
+ * @param {Uint8Array[]} parts
+ */
+function bareWebConcatUint8(parts) {
+  let n = 0
+  for (const p of parts) n += p.length
+  const out = new Uint8Array(n)
+  let o = 0
+  for (const p of parts) {
+    out.set(p, o)
+    o += p.length
+  }
+  return out
+}
+
+/**
+ * @param {Response} res
+ * @param {number} maxBytes
+ * @param {AbortSignal | undefined} signal
+ */
+async function bareWebReadBodyLimited(res, maxBytes, signal) {
+  if (!res.body || typeof res.body.getReader !== 'function') {
+    try {
+      const ab = await res.arrayBuffer()
+      const u8 = new Uint8Array(ab)
+      return {
+        bytes: u8.byteLength > maxBytes ? u8.slice(0, maxBytes) : u8,
+        truncated: u8.byteLength > maxBytes
+      }
+    } catch {
+      return { bytes: new Uint8Array(0), truncated: false }
+    }
+  }
+  const reader = res.body.getReader()
+  /** @type {Uint8Array[]} */
+  const chunks = []
+  let total = 0
+  try {
+    for (;;) {
+      if (signal && signal.aborted) {
+        try {
+          await reader.cancel()
+        } catch {
+          /* ignore */
+        }
+        break
+      }
+      const { done, value } = await reader.read()
+      if (done) break
+      if (!value || !value.length) continue
+      total += value.length
+      if (total > maxBytes) {
+        const prev = total - value.length
+        const take = Math.max(0, maxBytes - prev)
+        if (take > 0) chunks.push(value.subarray(0, take))
+        try {
+          await reader.cancel()
+        } catch {
+          /* ignore */
+        }
+        return { bytes: bareWebConcatUint8(chunks), truncated: true }
+      }
+      chunks.push(value)
+    }
+  } finally {
+    try {
+      reader.releaseLock()
+    } catch {
+      /* ignore */
+    }
+  }
+  return { bytes: bareWebConcatUint8(chunks), truncated: false }
+}
+
+/**
+ * @param {string | null | undefined} ct
+ */
+function bareWebCharsetFromContentType(ct) {
+  const m = /charset\s*=\s*["']?([^"';\s]+)/i.exec(String(ct || ''))
+  return (m ? m[1] : 'utf-8').trim().toLowerCase()
+}
+
+/**
+ * @param {Uint8Array} bytes
+ * @param {string} label
+ */
+function bareWebDecodeBytes(bytes, label) {
+  try {
+    const dec = new TextDecoder(label || 'utf-8', {
+      fatal: false,
+      ignoreBOM: true
+    })
+    return dec.decode(bytes)
+  } catch {
+    return new TextDecoder('utf-8', { fatal: false }).decode(bytes)
+  }
+}
+
+/**
+ * @param {string} s
+ */
+function bareWebDecodeHtmlEntities(s) {
+  let t = String(s || '')
+  t = t.replace(/&nbsp;/gi, ' ')
+  t = t.replace(/&quot;/gi, '"')
+  t = t.replace(/&#39;/g, "'")
+  t = t.replace(/&apos;/gi, "'")
+  t = t.replace(/&amp;/gi, '&')
+  t = t.replace(/&lt;/gi, '<')
+  t = t.replace(/&gt;/gi, '>')
+  t = t.replace(/&#x([0-9a-f]+);/gi, (_, h) => {
+    const c = parseInt(h, 16)
+    return Number.isFinite(c) ? String.fromCodePoint(c) : _
+  })
+  t = t.replace(/&#(\d+);/g, (_, d) => {
+    const c = parseInt(d, 10)
+    return Number.isFinite(c) ? String.fromCodePoint(c) : _
+  })
+  return t
+}
+
+/**
+ * @param {string} html
+ */
+function bareWebExtractHtmlText(html) {
+  let s = String(html || '')
+  s = s.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '')
+  s = s.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '')
+  s = s.replace(/<noscript\b[^<]*(?:(?!<\/noscript>)<[^<]*)*<\/noscript>/gi, '')
+  s = s.replace(/<!--[\s\S]*?-->/g, '')
+  s = s.replace(/<[^>]+>/g, ' ')
+  s = bareWebDecodeHtmlEntities(s)
+  s = s.replace(/\s+/g, ' ').trim()
+  return s
+}
+
+/**
+ * @param {string} html
+ * @param {string} baseUrl
+ * @param {number} maxLinks
+ */
+function bareWebExtractLinks(html, baseUrl, maxLinks) {
+  const cap = Math.min(Math.max(Number(maxLinks) || 200, 1), 500)
+  let uBase = null
+  try {
+    uBase = baseUrl ? new URL(String(baseUrl)) : null
+  } catch {
+    uBase = null
+  }
+  const seen = new Set()
+  /** @type {string[]} */
+  const out = []
+  const re = /<a\b[^>]*\bhref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi
+  let m
+  const h = String(html || '')
+  while ((m = re.exec(h)) !== null) {
+    const href = (m[1] || m[2] || m[3] || '').trim()
+    if (!href || href.startsWith('javascript:') || href.startsWith('#'))
+      continue
+    try {
+      const abs = uBase ? new URL(href, uBase).href : new URL(href).href
+      const proto = new URL(abs).protocol
+      if (proto !== 'http:' && proto !== 'https:') continue
+      if (!seen.has(abs)) {
+        seen.add(abs)
+        out.push(abs)
+      }
+    } catch {
+      /* skip */
+    }
+    if (out.length >= cap) break
+  }
+  return { links: out, links_truncated: out.length >= cap }
+}
+
+/**
+ * @param {string} tag
+ */
+function bareWebMetaContent(tag) {
+  const q =
+    /content\s*=\s*"([^"]*)"/i.exec(tag) ||
+    /content\s*=\s*'([^']*)'/i.exec(tag) ||
+    /content\s*=\s*([^\s>]+)/i.exec(tag)
+  return q ? bareWebDecodeHtmlEntities(q[1]).trim() : ''
+}
+
+/**
+ * @param {string} html
+ */
+function bareWebExtractMeta(html) {
+  const h = String(html || '')
+  const titleM = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(h)
+  const title = titleM
+    ? bareWebDecodeHtmlEntities(titleM[1].replace(/<[^>]+>/g, ' ')).trim()
+    : ''
+  let description = ''
+  const metaDescRe = /<meta[^>]*\bname\s*=\s*["']description["'][^>]*>/i.exec(h)
+  if (metaDescRe) description = bareWebMetaContent(metaDescRe[0])
+
+  let og_title = ''
+  const ogT = /<meta[^>]*\bproperty\s*=\s*["']og:title["'][^>]*>/i.exec(h)
+  if (ogT) og_title = bareWebMetaContent(ogT[0])
+
+  let og_description = ''
+  const ogD = /<meta[^>]*\bproperty\s*=\s*["']og:description["'][^>]*>/i.exec(h)
+  if (ogD) og_description = bareWebMetaContent(ogD[0])
+
+  return {
+    title,
+    description,
+    og_title,
+    og_description
+  }
+}
+
+/**
+ * @param {string} text
+ */
+function bareWebMaybeParseJson(text) {
+  try {
+    return { ok: true, value: JSON.parse(String(text)) }
+  } catch {
+    return { ok: false }
+  }
+}
+
+/**
+ * @param {string} ct
+ */
+function bareWebLooksLikeHtml(ct) {
+  return /\btext\/html\b/i.test(String(ct || ''))
+}
+
+/**
+ * @param {string} ct
+ */
+function bareWebLooksLikeJson(ct) {
+  const s = String(ct || '').toLowerCase()
+  return (
+    /\bapplication\/json\b/.test(s) ||
+    /\bapplication\/.*\+json\b/.test(s) ||
+    /\btext\/json\b/.test(s)
+  )
+}
+
+/**
+ * @param {{
+ *   ctx: Record<string, unknown>,
+ *   url: string,
+ *   method?: string,
+ *   headers?: Record<string, unknown>,
+ *   body?: string,
+ *   content_type?: string,
+ *   max_response_bytes?: number,
+ *   max_redirects?: number,
+ *   timeout_ms?: number,
+ *   format?: string,
+ *   max_links?: number,
+ *   signal?: AbortSignal | null
+ * }} o
+ */
+async function bareWebRunTool(o) {
+  const ctx = o.ctx
+  const fetchFn = bareWebResolveFetch(ctx)
+  if (!fetchFn) {
+    return {
+      ok: false,
+      error:
+        'web_fetch: no HTTP client (set ctx.httpFetch, bare.fetch, or global fetch)'
+    }
+  }
+
+  let startUrl = String(o.url || '').trim()
+  let method = String(o.method || 'GET').toUpperCase()
+  if (
+    !['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'].includes(
+      method
+    )
+  ) {
+    return { ok: false, error: 'web_fetch: unsupported method' }
+  }
+
+  let body =
+    o.body != null && method !== 'GET' && method !== 'HEAD'
+      ? String(o.body)
+      : undefined
+
+  let u0
+  try {
+    u0 = new URL(startUrl)
+  } catch {
+    return { ok: false, error: 'web_fetch: invalid URL' }
+  }
+  if (u0.protocol !== 'http:' && u0.protocol !== 'https:') {
+    return { ok: false, error: 'web_fetch: only http(s) URLs are allowed' }
+  }
+
+  const maxRedirects = Math.min(
+    Math.max(bareWebFiniteOr(o.max_redirects, 5), 0),
+    20
+  )
+  const maxBytes = Math.min(
+    Math.max(bareWebFiniteOr(o.max_response_bytes, 524288), 1024),
+    2 * 1024 * 1024
+  )
+  const timeoutMs = Math.min(
+    Math.max(bareWebFiniteOr(o.timeout_ms, 30000), 500),
+    120000
+  )
+  const fmtRaw = String(o.format || 'auto').toLowerCase()
+  const maxLinks = Number(o.max_links) || 200
+
+  /** @type {string[]} */
+  const redirectChain = [startUrl]
+  let currentUrl = startUrl
+  let redirectsUsed = 0
+
+  for (;;) {
+    const controller = new AbortController()
+    const timer = setTimeout(() => {
+      try {
+        controller.abort(bareWebTimeoutAbortReason(timeoutMs))
+      } catch {
+        /* ignore */
+      }
+    }, timeoutMs)
+    const signal = bareWebUnionAbort(o.signal || undefined, controller.signal)
+
+    /** @type {Record<string, string>} */
+    const hdrObj = {}
+    const hin = o.headers
+    if (hin && typeof hin === 'object' && !Array.isArray(hin)) {
+      for (const [k, v] of Object.entries(hin)) {
+        if (typeof v === 'string' && k) hdrObj[k] = v
+      }
+    }
+    if (
+      body != null &&
+      method !== 'GET' &&
+      method !== 'HEAD' &&
+      !Object.keys(hdrObj).some((k) => k.toLowerCase() === 'content-type')
+    ) {
+      hdrObj['Content-Type'] =
+        typeof o.content_type === 'string' && o.content_type.trim()
+          ? o.content_type.trim()
+          : 'application/octet-stream'
+    }
+
+    /** @type {RequestInit} */
+    const init = {
+      method,
+      headers: hdrObj,
+      signal: signal || undefined,
+      redirect: 'manual'
+    }
+    if (body != null && method !== 'GET' && method !== 'HEAD') {
+      init.body = body
+    }
+
+    let res
+    try {
+      res = await fetchFn(currentUrl, init)
+    } catch (e) {
+      clearTimeout(timer)
+      const msg = bareWebFmtErr(
+        e === undefined
+          ? new Error(
+              'web_fetch: fetch rejected with undefined (bare-fetch uses signal.reason; upstream abort() had no reason)'
+            )
+          : e
+      )
+      return {
+        ok: false,
+        error: 'web_fetch: request failed: ' + msg.slice(0, 400),
+        url_final: currentUrl,
+        redirect_chain: redirectChain
+      }
+    }
+    clearTimeout(timer)
+
+    const st = res.status
+    if (st >= 300 && st < 400) {
+      if (redirectsUsed >= maxRedirects) {
+        return {
+          ok: false,
+          error: 'web_fetch: too many redirects',
+          status: st,
+          url_final: currentUrl,
+          redirect_chain: redirectChain
+        }
+      }
+      const loc = res.headers.get('Location')
+      if (!loc) {
+        return {
+          ok: false,
+          error: 'web_fetch: redirect without Location',
+          status: st,
+          url_final: currentUrl,
+          redirect_chain: redirectChain
+        }
+      }
+      let nextUrl
+      try {
+        nextUrl = new URL(loc, currentUrl).href
+      } catch {
+        return {
+          ok: false,
+          error: 'web_fetch: bad redirect URL',
+          status: st,
+          url_final: currentUrl,
+          redirect_chain: redirectChain
+        }
+      }
+      redirectChain.push(nextUrl)
+      currentUrl = nextUrl
+      redirectsUsed++
+      if (st === 301 || st === 302 || st === 303) {
+        method = 'GET'
+        body = undefined
+      }
+      continue
+    }
+
+    const ct = res.headers.get('content-type') || ''
+    const url_final = currentUrl
+    const responseType = typeof res.type === 'string' ? res.type : undefined
+    let setCookie
+    try {
+      const hdrs = res.headers
+      if (hdrs && typeof hdrs.getSetCookie === 'function') {
+        const sc = hdrs.getSetCookie()
+        if (Array.isArray(sc) && sc.length) setCookie = sc.slice(0, 32)
+      }
+    } catch {
+      setCookie = undefined
+    }
+
+    if (method === 'HEAD') {
+      return {
+        ok: true,
+        url_final,
+        status: st,
+        content_type: ct,
+        response_type: responseType,
+        set_cookie: setCookie,
+        redirect_chain: redirectChain.length > 1 ? redirectChain : undefined,
+        truncated: false,
+        extract: { note: 'HEAD — body omitted' }
+      }
+    }
+
+    let bodyRead
+    try {
+      bodyRead = await bareWebReadBodyLimited(
+        res,
+        maxBytes,
+        signal || undefined
+      )
+    } catch (e) {
+      const msg = bareWebFmtErr(e)
+      return {
+        ok: false,
+        error: 'web_fetch: read body failed: ' + msg.slice(0, 400),
+        url_final,
+        status: st,
+        redirect_chain: redirectChain.length > 1 ? redirectChain : undefined
+      }
+    }
+
+    const charset = bareWebCharsetFromContentType(ct)
+    const text = bareWebDecodeBytes(bodyRead.bytes, charset)
+
+    const fmt =
+      fmtRaw === 'auto'
+        ? bareWebLooksLikeJson(ct)
+          ? 'json'
+          : bareWebLooksLikeHtml(ct)
+            ? 'markdownish'
+            : 'raw'
+        : fmtRaw
+
+    /** @type {unknown} */
+    let extract
+
+    if (fmt === 'json') {
+      const p = bareWebMaybeParseJson(text)
+      extract = p.ok
+        ? { json: p.value }
+        : { parse_error: true, text_slice: text.slice(0, 8000) }
+    } else if (fmt === 'links') {
+      extract = bareWebExtractLinks(text, url_final, maxLinks)
+    } else if (fmt === 'meta') {
+      extract = bareWebExtractMeta(text)
+    } else if (fmt === 'markdownish' || fmt === 'text') {
+      const plain = bareWebExtractHtmlText(text)
+      extract = {
+        text: plain,
+        approx_chars: plain.length
+      }
+    } else if (fmt === 'raw') {
+      extract = {
+        raw_text: text.length > 12000 ? text.slice(0, 12000) + '\n…' : text,
+        char_count: text.length
+      }
+    } else {
+      extract = {
+        text: text.length > 12000 ? text.slice(0, 12000) + '\n…' : text,
+        char_count: text.length
+      }
+    }
+
+    const raw_preview = text.slice(0, 2000)
+
+    return {
+      ok: true,
+      url_final,
+      status: st,
+      content_type: ct,
+      response_type: responseType,
+      set_cookie: setCookie,
+      redirect_chain: redirectChain.length > 1 ? redirectChain : undefined,
+      truncated: bodyRead.truncated,
+      extract,
+      raw_preview: fmt === 'raw' || fmt === 'json' ? undefined : raw_preview
+    }
+  }
+}