/** * Bare OS wget — HTTP(S) download via Fetch (not GNU wget2 / libwget). * @see https://www.gnu.org/software/wget/manual/ for UX inspiration; behavior is a small subset. */ import path from '#host-path' import { DEFAULT_WGET_USER_AGENT, defaultFetchSaveName, isSupportedFetchUrl, normalizeFetchUrl, applyBareOsCurlResolveMap } from './http-fetch-url.js' import { assertFetchUrlHostAllowedByDnsPolicy } from './bare-os-dns-policy.js' import { ensureBareFetchGlobals, resolveBareOsFetchFn } from './bare-os-ensure-bare-fetch.js' function usage() { return ( 'usage: wget [options] URL...\n' + 'Bare OS wget is fetch-based, not GNU wget2. See man wget.' ) } /** * One argv token of short options: -qO-, -T30, -O file (via -O then next arg). * @returns {number} next index into args, or -1 on error, -2 on help/version done */ function wgetConsumeShortCluster(a, args, i, st, ctx) { let j = 1 while (j < a.length) { const c = a[j] if (c === 'q') { st.quiet = true j++ continue } if (c === 'c') { st.continuePartial = true j++ continue } if (c === 'N') { st.timestamping = true j++ continue } if (c === 'h') { ctx.console.log(usage()) ctx.exitCode = 0 return -2 } if (c === 'V') { ctx.console.log('wget (Bare OS fetch subset) 0.1 — not GNU wget2') ctx.exitCode = 0 return -2 } if (c === 'O') { const tail = a.slice(j + 1) if (tail !== '') { st.outputDocument = tail return i + 1 } if (i + 1 >= args.length) { ctx.console.error('wget: option requires an argument: -O') ctx.exitCode = 2 return -1 } st.outputDocument = String(args[i + 1]) return i + 2 } if (c === 'P') { const tail = a.slice(j + 1) if (tail !== '') { st.directoryPrefix = tail return i + 1 } if (i + 1 >= args.length) { ctx.console.error('wget: option requires an argument: -P') ctx.exitCode = 2 return -1 } st.directoryPrefix = String(args[i + 1]) return i + 2 } if (c === 'U') { const tail = a.slice(j + 1) if (tail !== '') { st.userAgent = tail return i + 1 } if (i + 1 >= args.length) { ctx.console.error('wget: option requires an argument: -U') ctx.exitCode = 2 return -1 } st.userAgent = String(args[i + 1]) return i + 2 } if (c === 'T') { const tail = a.slice(j + 1) if (tail !== '') { const sec = Number(tail) if (!Number.isFinite(sec) || sec < 0) { ctx.console.error('wget: invalid timeout') ctx.exitCode = 2 return -1 } st.timeoutMs = Math.round(sec * 1000) return i + 1 } if (i + 1 >= args.length) { ctx.console.error('wget: option requires an argument: -T') ctx.exitCode = 2 return -1 } const sec = Number(args[i + 1]) if (!Number.isFinite(sec) || sec < 0) { ctx.console.error('wget: invalid timeout') ctx.exitCode = 2 return -1 } st.timeoutMs = Math.round(sec * 1000) return i + 2 } ctx.console.error('wget: invalid option: -' + c) ctx.exitCode = 2 return -1 } return i + 1 } /** * @param {Record} ctx * @param {string[]} argv */ export async function runWgetCli(ctx, argv) { const vfs = ctx.vfs const args = argv.slice(1) const st = { quiet: false, outputDocument: /** @type {string | null} */ (null), directoryPrefix: /** @type {string | null} */ (null), userAgent: /** @type {string | null} */ (null), timeoutMs: 0, continuePartial: false, timestamping: false, noClobber: false, extraHeaders: /** @type {string[]} */ ([]), postData: /** @type {string | null} */ (null), postFile: /** @type {string | null} */ (null) } /** @type {string[]} */ const urls = [] let i = 0 while (i < args.length) { const a = args[i] if (a === '--') { i++ while (i < args.length) { urls.push(normalizeFetchUrl(String(args[i++]))) } break } if (!a.startsWith('-') || a === '-') { urls.push(normalizeFetchUrl(String(a))) i++ continue } if (a === '-h' || a === '--help') { ctx.console.log(usage()) return } if (a === '-V' || a === '--version') { ctx.console.log('wget (Bare OS fetch subset) 0.1 — not GNU wget2') return } if (a === '-q' || a === '--quiet') { st.quiet = true i++ continue } if (a === '-c' || a === '--continue') { st.continuePartial = true i++ continue } if (a === '-N' || a === '--timestamping') { st.timestamping = true i++ continue } if (a === '-nc' || a === '--no-clobber') { st.noClobber = true i++ continue } if (a === '-O' || a === '--output-document') { if (i + 1 >= args.length) { ctx.console.error('wget: option requires an argument: ' + a) ctx.exitCode = 2 return } st.outputDocument = String(args[++i]) i++ continue } if (a.startsWith('--output-document=')) { st.outputDocument = a.slice('--output-document='.length) i++ continue } if (a === '-P' || a === '--directory-prefix') { if (i + 1 >= args.length) { ctx.console.error('wget: option requires an argument: ' + a) ctx.exitCode = 2 return } st.directoryPrefix = String(args[++i]) i++ continue } if (a.startsWith('--directory-prefix=')) { st.directoryPrefix = a.slice('--directory-prefix='.length) i++ continue } if (a === '-U' || a === '--user-agent') { if (i + 1 >= args.length) { ctx.console.error('wget: option requires an argument: ' + a) ctx.exitCode = 2 return } st.userAgent = String(args[++i]) i++ continue } if (a.startsWith('--user-agent=')) { st.userAgent = a.slice('--user-agent='.length) i++ continue } if (a === '-T' || a === '--timeout') { if (i + 1 >= args.length) { ctx.console.error('wget: option requires an argument: ' + a) ctx.exitCode = 2 return } const sec = Number(args[++i]) if (!Number.isFinite(sec) || sec < 0) { ctx.console.error('wget: invalid timeout') ctx.exitCode = 2 return } st.timeoutMs = Math.round(sec * 1000) i++ continue } if (a.startsWith('--timeout=')) { const sec = Number(a.slice('--timeout='.length)) if (!Number.isFinite(sec) || sec < 0) { ctx.console.error('wget: invalid timeout') ctx.exitCode = 2 return } st.timeoutMs = Math.round(sec * 1000) i++ continue } if (a === '--header') { if (i + 1 >= args.length) { ctx.console.error('wget: option requires an argument: --header') ctx.exitCode = 2 return } st.extraHeaders.push(String(args[++i])) i++ continue } if (a.startsWith('--header=')) { st.extraHeaders.push(a.slice('--header='.length)) i++ continue } if (a === '--post-data') { if (i + 1 >= args.length) { ctx.console.error('wget: option requires an argument: --post-data') ctx.exitCode = 2 return } st.postData = String(args[++i]) i++ continue } if (a.startsWith('--post-data=')) { st.postData = a.slice('--post-data='.length) i++ continue } if (a === '--post-file') { if (i + 1 >= args.length) { ctx.console.error('wget: option requires an argument: --post-file') ctx.exitCode = 2 return } st.postFile = String(args[++i]) i++ continue } if (a.startsWith('--')) { ctx.console.error('wget: unknown option: ' + a) ctx.exitCode = 2 return } const ni = wgetConsumeShortCluster(a, args, i, st, ctx) if (ni === -1) return if (ni === -2) return i = ni } const { quiet, outputDocument, directoryPrefix, userAgent, timeoutMs, continuePartial, timestamping, noClobber, extraHeaders, postData, postFile } = st if (urls.length === 0) { ctx.console.error(usage()) ctx.exitCode = 2 return } for (const u of urls) { if (!isSupportedFetchUrl(u)) { ctx.console.error( 'wget: URL rejected (need http(s)://, data:, or file://): ' + u ) ctx.exitCode = 2 return } try { assertFetchUrlHostAllowedByDnsPolicy(u, ctx.vfs?.env || {}) } catch (e) { ctx.console.error((e && e.message) || String(e)) ctx.exitCode = 2 return } } if (urls.length > 1 && outputDocument != null) { ctx.console.error( 'wget: cannot use -O/--output-document with multiple URLs' ) ctx.exitCode = 2 return } if (continuePartial && outputDocument === '-') { ctx.console.error( 'wget: --continue cannot be used with output to stdout (-O -)' ) ctx.exitCode = 2 return } if (noClobber && continuePartial) { ctx.console.error('wget: --no-clobber cannot be used with --continue') ctx.exitCode = 2 return } if (timestamping) { ctx.console.error( 'wget: --timestamping (-N) is not supported in Bare OS wget yet' ) ctx.exitCode = 2 return } if (outputDocument && directoryPrefix) { ctx.console.error('wget: cannot combine -O and -P') ctx.exitCode = 2 return } await ensureBareFetchGlobals(ctx) const fetchFn = resolveBareOsFetchFn(ctx) if (!fetchFn) { ctx.console.error( 'wget: no HTTP client (need global fetch, ctx.bare.fetch from /lib/bare/bundles, or bare-fetch on Pear/Bare)' ) ctx.exitCode = 4 return } let body = undefined if (postFile) { const buf = await vfs.readFile(postFile) if (!buf) { if (!quiet) ctx.console.error('wget: cannot read --post-file: ' + postFile) ctx.exitCode = 3 return } body = new Uint8Array(buf) } else if (postData != null) { body = postData } const method = body !== undefined ? 'POST' : 'GET' const hdr = new Headers() for (const line of extraHeaders) { const colon = line.indexOf(':') if (colon < 1) { ctx.console.error('wget: malformed header: ' + line) ctx.exitCode = 2 return } hdr.set(line.slice(0, colon).trim(), line.slice(colon + 1).trim()) } if (userAgent) hdr.set('User-Agent', userAgent) else if (!hdr.has('User-Agent')) hdr.set('User-Agent', DEFAULT_WGET_USER_AGENT) if (body !== undefined && !hdr.has('Content-Type')) hdr.set('Content-Type', 'application/x-www-form-urlencoded') for (let ui = 0; ui < urls.length; ui++) { const url = applyBareOsCurlResolveMap(urls[ui], vfs?.env) /** @type {string | null} */ let targetPath = null if (outputDocument === '-') { targetPath = null } else if (outputDocument != null) { targetPath = outputDocument } else { const name = defaultFetchSaveName(url) const dir = directoryPrefix || '.' targetPath = path.posix.join(dir.replace(/\/$/, '') || '.', name) } const reqHdr = new Headers(hdr) let rangeStart = 0 if (continuePartial && targetPath != null && !reqHdr.has('Range')) { const fst = await vfs.stat(targetPath) if ( fst && fst.type === 'file' && typeof fst.size === 'number' && fst.size > 0 ) { rangeStart = fst.size reqHdr.set('Range', `bytes=${rangeStart}-`) } } const ac = timeoutMs > 0 ? new AbortController() : null const t = timeoutMs > 0 ? setTimeout(() => { try { ac.abort() } catch { /* ignore */ } }, timeoutMs) : null let res try { res = await fetchFn(url, { method, headers: reqHdr, body: method === 'GET' ? undefined : body, signal: ac ? ac.signal : undefined }) } catch (e) { if (t) clearTimeout(t) const msg = e && e.message ? e.message : String(e) if (!quiet) ctx.console.error('wget: ' + msg) ctx.exitCode = 4 return } if (t) clearTimeout(t) if (!res.ok) { if (continuePartial && rangeStart > 0 && res.status === 416) { continue } if (!quiet) ctx.console.error('wget: HTTP error ' + res.status + ' for ' + url) ctx.exitCode = 8 return } let buf = new Uint8Array(await res.arrayBuffer()) if (continuePartial && rangeStart > 0 && res.status === 206) { const prev = await vfs.readFile(targetPath) const prevU8 = prev ? new Uint8Array(prev) : new Uint8Array(0) const combined = new Uint8Array(prevU8.length + buf.length) combined.set(prevU8, 0) combined.set(buf, prevU8.length) buf = combined } if ( noClobber && targetPath != null && outputDocument !== '-' && !continuePartial ) { try { const ex = await vfs.stat(targetPath) if (ex && ex.type === 'file' && (ex.size ?? 0) > 0) { if (!quiet) ctx.console.error( 'wget: ' + targetPath + ': File exists (--no-clobber)' ) continue } } catch { /* treat as absent */ } } if (outputDocument === '-') { ctx.console.log(ctx.b4a.toString(buf)) } else if (outputDocument != null) { await vfs.writeFile(outputDocument, buf) if (!quiet) ctx.console.error(outputDocument) } else { await vfs.writeFile(targetPath, buf) if (!quiet) ctx.console.error(targetPath) } } ctx.exitCode = 0 }