bare-operating-system/packages/bare-os-booter/lib/tools/warc-cli.js

/**
 * Minimal WARC 1.0 tooling for Bare OS (no external WARC libraries).
 * Supports listing record summaries and appending a single response record from a VFS file.
 */

import b4a from 'b4a'

const CRLF = '\r\n'

/**
 * @param {Record<string, string>} fields
 */
function warcHeaderBlock(fields) {
  const lines = ['WARC/1.0']
  for (const [k, v] of Object.entries(fields)) {
    const val = String(v).replace(/\r?\n/g, ' ')
    lines.push(`${k}: ${val}`)
  }
  lines.push('')
  lines.push('')
  return b4a.from(lines.join(CRLF), 'utf8')
}

/**
 * Parse WARC file buffer: yield { type, targetUri, contentLength } per record (best-effort).
 * @param {Uint8Array} buf
 */
function* iterateWarcRecords(buf) {
  let off = 0
  const u8 = buf instanceof Uint8Array ? buf : new Uint8Array(buf)
  while (off < u8.length) {
    const start = off
    const idx = indexOfHeaderEnd(u8, off)
    if (idx < 0) break
    const headerBytes = u8.subarray(off, idx)
    off = idx + 4
    const headerText = b4a.toString(headerBytes, 'utf8')
    const cl = parseWarcHeaderNumber(headerText, 'Content-Length')
    const warcType = parseWarcHeaderString(headerText, 'WARC-Type')
    const targetUri = parseWarcHeaderString(headerText, 'WARC-Target-URI')
    if (cl > 0 && off + cl <= u8.length) {
      yield {
        type: warcType || 'unknown',
        targetUri: targetUri || '',
        contentLength: cl,
        headerOffset: start
      }
      off += cl
      if (off < u8.length) {
        const pad = (4 - (cl % 4 || 4)) % 4
        off += pad
      }
    } else {
      break
    }
  }
}

/**
 * @param {Uint8Array} u8
 * @param {number} from
 */
function indexOfHeaderEnd(u8, from) {
  for (let i = from; i + 3 < u8.length; i++) {
    if (
      u8[i] === 0xd &&
      u8[i + 1] === 0xa &&
      u8[i + 2] === 0xd &&
      u8[i + 3] === 0xa
    ) {
      return i
    }
  }
  return -1
}

/**
 * @param {string} headerText
 * @param {string} name
 */
function parseWarcHeaderNumber(headerText, name) {
  const re = new RegExp('^' + name + ':\\s*(\\d+)\\s*$', 'mi')
  const m = headerText.match(re)
  return m ? Number.parseInt(m[1], 10) : 0
}

/**
 * @param {string} headerText
 * @param {string} name
 */
function parseWarcHeaderString(headerText, name) {
  const re = new RegExp('^' + name + ':\\s*(.+)\\s*$', 'mi')
  const m = headerText.match(re)
  return m ? m[1].trim() : ''
}

/**
 * @param {Record<string, unknown>} ctx
 * @param {string[]} argv
 */
export async function runWarcCli(ctx, argv) {
  const args = argv.slice(1)
  if (
    !args.length ||
    args[0] === '-h' ||
    args[0] === '--help' ||
    args[0] === 'help'
  ) {
    ctx.console.log(
      'Usage:\n' +
        '  warc list FILE.warc           — summarize WARC records\n' +
        '  warc write-response -o OUT -u URI -i INPUT  — append one response record (VFS paths)\n'
    )
    return
  }

  const vfs = ctx.vfs
  if (!vfs || typeof vfs.readFile !== 'function' || typeof vfs.writeFile !== 'function') {
    ctx.console.error('warc: vfs unavailable')
    ctx.exitCode = 1
    return
  }

  if (args[0] === 'list' && args[1]) {
    const p = String(args[1])
    const abs = vfs.resolveLogical(p)
    const buf = await vfs.readFile(abs)
    if (!buf) {
      ctx.console.error('warc: cannot read ' + p)
      ctx.exitCode = 1
      return
    }
    const u8 = buf instanceof Uint8Array ? buf : new Uint8Array(buf)
    let n = 0
    for (const rec of iterateWarcRecords(u8)) {
      n++
      ctx.console.log(
        `${n}\t${rec.type}\t${rec.targetUri || '-'}\t${rec.contentLength}b`
      )
    }
    if (n === 0) ctx.console.log('(no records parsed)')
    return
  }

  if (args[0] === 'write-response') {
    let outPath = ''
    let uri = ''
    let inPath = ''
    for (let i = 1; i < args.length; i++) {
      const a = args[i]
      if (a === '-o') outPath = String(args[++i] || '')
      else if (a === '-u') uri = String(args[++i] || '')
      else if (a === '-i') inPath = String(args[++i] || '')
      else {
        ctx.console.error('warc: unknown argument ' + a)
        ctx.exitCode = 1
        return
      }
    }
    if (!outPath || !uri || !inPath) {
      ctx.console.error('warc: write-response requires -o OUT -u URI -i INPUT')
      ctx.exitCode = 1
      return
    }
    const absIn = vfs.resolveLogical(inPath)
    const body = await vfs.readFile(absIn)
    if (!body) {
      ctx.console.error('warc: cannot read input ' + inPath)
      ctx.exitCode = 1
      return
    }
    const payload = body instanceof Uint8Array ? body : new Uint8Array(body)
    const date = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z')
    const recordId = `<urn:uuid:${hexRandom(32)}>`
    const fields = {
      'WARC-Type': 'response',
      'WARC-Target-URI': uri,
      'WARC-Date': date,
      'WARC-Record-ID': recordId,
      'Content-Type': 'application/octet-stream',
      'Content-Length': String(payload.length)
    }
    const head = warcHeaderBlock(fields)
    const absOut = vfs.resolveLogical(outPath)
    let existing = new Uint8Array(0)
    try {
      const prev = await vfs.readFile(absOut)
      if (prev) existing = prev instanceof Uint8Array ? prev : new Uint8Array(prev)
    } catch {
      /* new file */
    }
    const pad = (4 - (payload.length % 4 || 4)) % 4
    const tail = new Uint8Array(pad)
    const merged = new Uint8Array(
      existing.length + head.length + payload.length + tail.length
    )
    merged.set(existing, 0)
    merged.set(head, existing.length)
    merged.set(payload, existing.length + head.length)
    merged.set(tail, existing.length + head.length + payload.length)
    await vfs.writeFile(absOut, merged)
    return
  }

  ctx.console.error('warc: unknown subcommand (try warc --help)')
  ctx.exitCode = 1
}

function hexRandom(nibbles) {
  const bytes = Math.ceil(nibbles / 2)
  const u = new Uint8Array(bytes)
  if (typeof globalThis.crypto?.getRandomValues === 'function') {
    globalThis.crypto.getRandomValues(u)
  } else {
    for (let i = 0; i < bytes; i++) u[i] = (Math.random() * 256) | 0
  }
  let s = ''
  for (let i = 0; i < bytes; i++) s += u[i].toString(16).padStart(2, '0')
  return s.slice(0, nibbles)
}