/** * Minimal WARC 1.0 tooling for Bare OS (no external WARC libraries). * Supports listing record summaries and appending a single response record from a VFS file. */ import b4a from 'b4a' const CRLF = '\r\n' /** * @param {Record} fields */ function warcHeaderBlock(fields) { const lines = ['WARC/1.0'] for (const [k, v] of Object.entries(fields)) { const val = String(v).replace(/\r?\n/g, ' ') lines.push(`${k}: ${val}`) } lines.push('') lines.push('') return b4a.from(lines.join(CRLF), 'utf8') } /** * Parse WARC file buffer: yield { type, targetUri, contentLength } per record (best-effort). * @param {Uint8Array} buf */ function* iterateWarcRecords(buf) { let off = 0 const u8 = buf instanceof Uint8Array ? buf : new Uint8Array(buf) while (off < u8.length) { const start = off const idx = indexOfHeaderEnd(u8, off) if (idx < 0) break const headerBytes = u8.subarray(off, idx) off = idx + 4 const headerText = b4a.toString(headerBytes, 'utf8') const cl = parseWarcHeaderNumber(headerText, 'Content-Length') const warcType = parseWarcHeaderString(headerText, 'WARC-Type') const targetUri = parseWarcHeaderString(headerText, 'WARC-Target-URI') if (cl > 0 && off + cl <= u8.length) { yield { type: warcType || 'unknown', targetUri: targetUri || '', contentLength: cl, headerOffset: start } off += cl if (off < u8.length) { const pad = (4 - (cl % 4 || 4)) % 4 off += pad } } else { break } } } /** * @param {Uint8Array} u8 * @param {number} from */ function indexOfHeaderEnd(u8, from) { for (let i = from; i + 3 < u8.length; i++) { if ( u8[i] === 0xd && u8[i + 1] === 0xa && u8[i + 2] === 0xd && u8[i + 3] === 0xa ) { return i } } return -1 } /** * @param {string} headerText * @param {string} name */ function parseWarcHeaderNumber(headerText, name) { const re = new RegExp('^' + name + ':\\s*(\\d+)\\s*$', 'mi') const m = headerText.match(re) return m ? Number.parseInt(m[1], 10) : 0 } /** * @param {string} headerText * @param {string} name */ function parseWarcHeaderString(headerText, name) { const re = new RegExp('^' + name + ':\\s*(.+)\\s*$', 'mi') const m = headerText.match(re) return m ? m[1].trim() : '' } /** * @param {Record} ctx * @param {string[]} argv */ export async function runWarcCli(ctx, argv) { const args = argv.slice(1) if ( !args.length || args[0] === '-h' || args[0] === '--help' || args[0] === 'help' ) { ctx.console.log( 'Usage:\n' + ' warc list FILE.warc — summarize WARC records\n' + ' warc write-response -o OUT -u URI -i INPUT — append one response record (VFS paths)\n' ) return } const vfs = ctx.vfs if (!vfs || typeof vfs.readFile !== 'function' || typeof vfs.writeFile !== 'function') { ctx.console.error('warc: vfs unavailable') ctx.exitCode = 1 return } if (args[0] === 'list' && args[1]) { const p = String(args[1]) const abs = vfs.resolveLogical(p) const buf = await vfs.readFile(abs) if (!buf) { ctx.console.error('warc: cannot read ' + p) ctx.exitCode = 1 return } const u8 = buf instanceof Uint8Array ? buf : new Uint8Array(buf) let n = 0 for (const rec of iterateWarcRecords(u8)) { n++ ctx.console.log( `${n}\t${rec.type}\t${rec.targetUri || '-'}\t${rec.contentLength}b` ) } if (n === 0) ctx.console.log('(no records parsed)') return } if (args[0] === 'write-response') { let outPath = '' let uri = '' let inPath = '' for (let i = 1; i < args.length; i++) { const a = args[i] if (a === '-o') outPath = String(args[++i] || '') else if (a === '-u') uri = String(args[++i] || '') else if (a === '-i') inPath = String(args[++i] || '') else { ctx.console.error('warc: unknown argument ' + a) ctx.exitCode = 1 return } } if (!outPath || !uri || !inPath) { ctx.console.error('warc: write-response requires -o OUT -u URI -i INPUT') ctx.exitCode = 1 return } const absIn = vfs.resolveLogical(inPath) const body = await vfs.readFile(absIn) if (!body) { ctx.console.error('warc: cannot read input ' + inPath) ctx.exitCode = 1 return } const payload = body instanceof Uint8Array ? body : new Uint8Array(body) const date = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z') const recordId = `` const fields = { 'WARC-Type': 'response', 'WARC-Target-URI': uri, 'WARC-Date': date, 'WARC-Record-ID': recordId, 'Content-Type': 'application/octet-stream', 'Content-Length': String(payload.length) } const head = warcHeaderBlock(fields) const absOut = vfs.resolveLogical(outPath) let existing = new Uint8Array(0) try { const prev = await vfs.readFile(absOut) if (prev) existing = prev instanceof Uint8Array ? prev : new Uint8Array(prev) } catch { /* new file */ } const pad = (4 - (payload.length % 4 || 4)) % 4 const tail = new Uint8Array(pad) const merged = new Uint8Array( existing.length + head.length + payload.length + tail.length ) merged.set(existing, 0) merged.set(head, existing.length) merged.set(payload, existing.length + head.length) merged.set(tail, existing.length + head.length + payload.length) await vfs.writeFile(absOut, merged) return } ctx.console.error('warc: unknown subcommand (try warc --help)') ctx.exitCode = 1 } function hexRandom(nibbles) { const bytes = Math.ceil(nibbles / 2) const u = new Uint8Array(bytes) if (typeof globalThis.crypto?.getRandomValues === 'function') { globalThis.crypto.getRandomValues(u) } else { for (let i = 0; i < bytes; i++) u[i] = (Math.random() * 256) | 0 } let s = '' for (let i = 0; i < bytes; i++) s += u[i].toString(16).padStart(2, '0') return s.slice(0, nibbles) }