568 lines
14 KiB
JavaScript
568 lines
14 KiB
JavaScript
/**
|
|
* Bare OS wget — HTTP(S) download via Fetch (not GNU wget2 / libwget).
|
|
* @see https://www.gnu.org/software/wget/manual/ for UX inspiration; behavior is a small subset.
|
|
*/
|
|
|
|
import path from '#host-path'
|
|
import {
|
|
DEFAULT_WGET_USER_AGENT,
|
|
defaultFetchSaveName,
|
|
isSupportedFetchUrl,
|
|
normalizeFetchUrl,
|
|
applyBareOsCurlResolveMap
|
|
} from './http-fetch-url.js'
|
|
import { assertFetchUrlHostAllowedByDnsPolicy } from '../security/bare-os-dns-policy.js'
|
|
import {
|
|
ensureBareFetchGlobals,
|
|
resolveBareOsFetchFn
|
|
} from '../ctx/bare-os-ensure-bare-fetch.js'
|
|
|
|
function usage() {
|
|
return (
|
|
'usage: wget [options] URL...\n' +
|
|
'Bare OS wget is fetch-based, not GNU wget2. See man wget.'
|
|
)
|
|
}
|
|
|
|
/**
|
|
* One argv token of short options: -qO-, -T30, -O file (via -O then next arg).
|
|
* @returns {number} next index into args, or -1 on error, -2 on help/version done
|
|
*/
|
|
function wgetConsumeShortCluster(a, args, i, st, ctx) {
|
|
let j = 1
|
|
while (j < a.length) {
|
|
const c = a[j]
|
|
if (c === 'q') {
|
|
st.quiet = true
|
|
j++
|
|
continue
|
|
}
|
|
if (c === 'c') {
|
|
st.continuePartial = true
|
|
j++
|
|
continue
|
|
}
|
|
if (c === 'N') {
|
|
st.timestamping = true
|
|
j++
|
|
continue
|
|
}
|
|
if (c === 'h') {
|
|
ctx.console.log(usage())
|
|
ctx.exitCode = 0
|
|
return -2
|
|
}
|
|
if (c === 'V') {
|
|
ctx.console.log('wget (Bare OS fetch subset) 0.1 — not GNU wget2')
|
|
ctx.exitCode = 0
|
|
return -2
|
|
}
|
|
if (c === 'O') {
|
|
const tail = a.slice(j + 1)
|
|
if (tail !== '') {
|
|
st.outputDocument = tail
|
|
return i + 1
|
|
}
|
|
if (i + 1 >= args.length) {
|
|
ctx.console.error('wget: option requires an argument: -O')
|
|
ctx.exitCode = 2
|
|
return -1
|
|
}
|
|
st.outputDocument = String(args[i + 1])
|
|
return i + 2
|
|
}
|
|
if (c === 'P') {
|
|
const tail = a.slice(j + 1)
|
|
if (tail !== '') {
|
|
st.directoryPrefix = tail
|
|
return i + 1
|
|
}
|
|
if (i + 1 >= args.length) {
|
|
ctx.console.error('wget: option requires an argument: -P')
|
|
ctx.exitCode = 2
|
|
return -1
|
|
}
|
|
st.directoryPrefix = String(args[i + 1])
|
|
return i + 2
|
|
}
|
|
if (c === 'U') {
|
|
const tail = a.slice(j + 1)
|
|
if (tail !== '') {
|
|
st.userAgent = tail
|
|
return i + 1
|
|
}
|
|
if (i + 1 >= args.length) {
|
|
ctx.console.error('wget: option requires an argument: -U')
|
|
ctx.exitCode = 2
|
|
return -1
|
|
}
|
|
st.userAgent = String(args[i + 1])
|
|
return i + 2
|
|
}
|
|
if (c === 'T') {
|
|
const tail = a.slice(j + 1)
|
|
if (tail !== '') {
|
|
const sec = Number(tail)
|
|
if (!Number.isFinite(sec) || sec < 0) {
|
|
ctx.console.error('wget: invalid timeout')
|
|
ctx.exitCode = 2
|
|
return -1
|
|
}
|
|
st.timeoutMs = Math.round(sec * 1000)
|
|
return i + 1
|
|
}
|
|
if (i + 1 >= args.length) {
|
|
ctx.console.error('wget: option requires an argument: -T')
|
|
ctx.exitCode = 2
|
|
return -1
|
|
}
|
|
const sec = Number(args[i + 1])
|
|
if (!Number.isFinite(sec) || sec < 0) {
|
|
ctx.console.error('wget: invalid timeout')
|
|
ctx.exitCode = 2
|
|
return -1
|
|
}
|
|
st.timeoutMs = Math.round(sec * 1000)
|
|
return i + 2
|
|
}
|
|
ctx.console.error('wget: invalid option: -' + c)
|
|
ctx.exitCode = 2
|
|
return -1
|
|
}
|
|
return i + 1
|
|
}
|
|
|
|
/**
|
|
* @param {Record<string, unknown>} ctx
|
|
* @param {string[]} argv
|
|
*/
|
|
export async function runWgetCli(ctx, argv) {
|
|
const vfs = ctx.vfs
|
|
const args = argv.slice(1)
|
|
|
|
const st = {
|
|
quiet: false,
|
|
outputDocument: /** @type {string | null} */ (null),
|
|
directoryPrefix: /** @type {string | null} */ (null),
|
|
userAgent: /** @type {string | null} */ (null),
|
|
timeoutMs: 0,
|
|
continuePartial: false,
|
|
timestamping: false,
|
|
noClobber: false,
|
|
extraHeaders: /** @type {string[]} */ ([]),
|
|
postData: /** @type {string | null} */ (null),
|
|
postFile: /** @type {string | null} */ (null)
|
|
}
|
|
|
|
/** @type {string[]} */
|
|
const urls = []
|
|
|
|
let i = 0
|
|
while (i < args.length) {
|
|
const a = args[i]
|
|
if (a === '--') {
|
|
i++
|
|
while (i < args.length) {
|
|
urls.push(normalizeFetchUrl(String(args[i++])))
|
|
}
|
|
break
|
|
}
|
|
if (!a.startsWith('-') || a === '-') {
|
|
urls.push(normalizeFetchUrl(String(a)))
|
|
i++
|
|
continue
|
|
}
|
|
|
|
if (a === '-h' || a === '--help') {
|
|
ctx.console.log(usage())
|
|
return
|
|
}
|
|
if (a === '-V' || a === '--version') {
|
|
ctx.console.log('wget (Bare OS fetch subset) 0.1 — not GNU wget2')
|
|
return
|
|
}
|
|
|
|
if (a === '-q' || a === '--quiet') {
|
|
st.quiet = true
|
|
i++
|
|
continue
|
|
}
|
|
|
|
if (a === '-c' || a === '--continue') {
|
|
st.continuePartial = true
|
|
i++
|
|
continue
|
|
}
|
|
if (a === '-N' || a === '--timestamping') {
|
|
st.timestamping = true
|
|
i++
|
|
continue
|
|
}
|
|
|
|
if (a === '-nc' || a === '--no-clobber') {
|
|
st.noClobber = true
|
|
i++
|
|
continue
|
|
}
|
|
|
|
if (a === '-O' || a === '--output-document') {
|
|
if (i + 1 >= args.length) {
|
|
ctx.console.error('wget: option requires an argument: ' + a)
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
st.outputDocument = String(args[++i])
|
|
i++
|
|
continue
|
|
}
|
|
if (a.startsWith('--output-document=')) {
|
|
st.outputDocument = a.slice('--output-document='.length)
|
|
i++
|
|
continue
|
|
}
|
|
|
|
if (a === '-P' || a === '--directory-prefix') {
|
|
if (i + 1 >= args.length) {
|
|
ctx.console.error('wget: option requires an argument: ' + a)
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
st.directoryPrefix = String(args[++i])
|
|
i++
|
|
continue
|
|
}
|
|
if (a.startsWith('--directory-prefix=')) {
|
|
st.directoryPrefix = a.slice('--directory-prefix='.length)
|
|
i++
|
|
continue
|
|
}
|
|
|
|
if (a === '-U' || a === '--user-agent') {
|
|
if (i + 1 >= args.length) {
|
|
ctx.console.error('wget: option requires an argument: ' + a)
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
st.userAgent = String(args[++i])
|
|
i++
|
|
continue
|
|
}
|
|
if (a.startsWith('--user-agent=')) {
|
|
st.userAgent = a.slice('--user-agent='.length)
|
|
i++
|
|
continue
|
|
}
|
|
|
|
if (a === '-T' || a === '--timeout') {
|
|
if (i + 1 >= args.length) {
|
|
ctx.console.error('wget: option requires an argument: ' + a)
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
const sec = Number(args[++i])
|
|
if (!Number.isFinite(sec) || sec < 0) {
|
|
ctx.console.error('wget: invalid timeout')
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
st.timeoutMs = Math.round(sec * 1000)
|
|
i++
|
|
continue
|
|
}
|
|
if (a.startsWith('--timeout=')) {
|
|
const sec = Number(a.slice('--timeout='.length))
|
|
if (!Number.isFinite(sec) || sec < 0) {
|
|
ctx.console.error('wget: invalid timeout')
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
st.timeoutMs = Math.round(sec * 1000)
|
|
i++
|
|
continue
|
|
}
|
|
|
|
if (a === '--header') {
|
|
if (i + 1 >= args.length) {
|
|
ctx.console.error('wget: option requires an argument: --header')
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
st.extraHeaders.push(String(args[++i]))
|
|
i++
|
|
continue
|
|
}
|
|
if (a.startsWith('--header=')) {
|
|
st.extraHeaders.push(a.slice('--header='.length))
|
|
i++
|
|
continue
|
|
}
|
|
|
|
if (a === '--post-data') {
|
|
if (i + 1 >= args.length) {
|
|
ctx.console.error('wget: option requires an argument: --post-data')
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
st.postData = String(args[++i])
|
|
i++
|
|
continue
|
|
}
|
|
if (a.startsWith('--post-data=')) {
|
|
st.postData = a.slice('--post-data='.length)
|
|
i++
|
|
continue
|
|
}
|
|
|
|
if (a === '--post-file') {
|
|
if (i + 1 >= args.length) {
|
|
ctx.console.error('wget: option requires an argument: --post-file')
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
st.postFile = String(args[++i])
|
|
i++
|
|
continue
|
|
}
|
|
|
|
if (a.startsWith('--')) {
|
|
ctx.console.error('wget: unknown option: ' + a)
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
|
|
const ni = wgetConsumeShortCluster(a, args, i, st, ctx)
|
|
if (ni === -1) return
|
|
if (ni === -2) return
|
|
i = ni
|
|
}
|
|
|
|
const {
|
|
quiet,
|
|
outputDocument,
|
|
directoryPrefix,
|
|
userAgent,
|
|
timeoutMs,
|
|
continuePartial,
|
|
timestamping,
|
|
noClobber,
|
|
extraHeaders,
|
|
postData,
|
|
postFile
|
|
} = st
|
|
|
|
if (urls.length === 0) {
|
|
ctx.console.error(usage())
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
|
|
for (const u of urls) {
|
|
if (!isSupportedFetchUrl(u)) {
|
|
ctx.console.error(
|
|
'wget: URL rejected (need http(s)://, data:, or file://): ' + u
|
|
)
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
try {
|
|
assertFetchUrlHostAllowedByDnsPolicy(u, ctx.vfs?.env || {})
|
|
} catch (e) {
|
|
ctx.console.error((e && e.message) || String(e))
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
}
|
|
|
|
if (urls.length > 1 && outputDocument != null) {
|
|
ctx.console.error(
|
|
'wget: cannot use -O/--output-document with multiple URLs'
|
|
)
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
|
|
if (continuePartial && outputDocument === '-') {
|
|
ctx.console.error(
|
|
'wget: --continue cannot be used with output to stdout (-O -)'
|
|
)
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
|
|
if (noClobber && continuePartial) {
|
|
ctx.console.error('wget: --no-clobber cannot be used with --continue')
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
|
|
if (timestamping) {
|
|
ctx.console.error(
|
|
'wget: --timestamping (-N) is not supported in Bare OS wget yet'
|
|
)
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
|
|
if (outputDocument && directoryPrefix) {
|
|
ctx.console.error('wget: cannot combine -O and -P')
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
|
|
await ensureBareFetchGlobals(ctx)
|
|
const fetchFn = resolveBareOsFetchFn(ctx)
|
|
|
|
if (!fetchFn) {
|
|
ctx.console.error(
|
|
'wget: no HTTP client (need global fetch, ctx.bare.fetch from /lib/bare/bundles, or bare-fetch on Pear/Bare)'
|
|
)
|
|
ctx.exitCode = 4
|
|
return
|
|
}
|
|
|
|
let body = undefined
|
|
if (postFile) {
|
|
const buf = await vfs.readFile(postFile)
|
|
if (!buf) {
|
|
if (!quiet)
|
|
ctx.console.error('wget: cannot read --post-file: ' + postFile)
|
|
ctx.exitCode = 3
|
|
return
|
|
}
|
|
body = new Uint8Array(buf)
|
|
} else if (postData != null) {
|
|
body = postData
|
|
}
|
|
|
|
const method = body !== undefined ? 'POST' : 'GET'
|
|
|
|
const hdr = new Headers()
|
|
for (const line of extraHeaders) {
|
|
const colon = line.indexOf(':')
|
|
if (colon < 1) {
|
|
ctx.console.error('wget: malformed header: ' + line)
|
|
ctx.exitCode = 2
|
|
return
|
|
}
|
|
hdr.set(line.slice(0, colon).trim(), line.slice(colon + 1).trim())
|
|
}
|
|
if (userAgent) hdr.set('User-Agent', userAgent)
|
|
else if (!hdr.has('User-Agent'))
|
|
hdr.set('User-Agent', DEFAULT_WGET_USER_AGENT)
|
|
if (body !== undefined && !hdr.has('Content-Type'))
|
|
hdr.set('Content-Type', 'application/x-www-form-urlencoded')
|
|
|
|
for (let ui = 0; ui < urls.length; ui++) {
|
|
const url = applyBareOsCurlResolveMap(urls[ui], vfs?.env)
|
|
|
|
/** @type {string | null} */
|
|
let targetPath = null
|
|
if (outputDocument === '-') {
|
|
targetPath = null
|
|
} else if (outputDocument != null) {
|
|
targetPath = outputDocument
|
|
} else {
|
|
const name = defaultFetchSaveName(url)
|
|
const dir = directoryPrefix || '.'
|
|
targetPath = path.posix.join(dir.replace(/\/$/, '') || '.', name)
|
|
}
|
|
|
|
const reqHdr = new Headers(hdr)
|
|
let rangeStart = 0
|
|
if (continuePartial && targetPath != null && !reqHdr.has('Range')) {
|
|
const fst = await vfs.stat(targetPath)
|
|
if (
|
|
fst &&
|
|
fst.type === 'file' &&
|
|
typeof fst.size === 'number' &&
|
|
fst.size > 0
|
|
) {
|
|
rangeStart = fst.size
|
|
reqHdr.set('Range', `bytes=${rangeStart}-`)
|
|
}
|
|
}
|
|
|
|
const ac = timeoutMs > 0 ? new AbortController() : null
|
|
const t =
|
|
timeoutMs > 0
|
|
? setTimeout(() => {
|
|
try {
|
|
ac.abort()
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}, timeoutMs)
|
|
: null
|
|
|
|
let res
|
|
try {
|
|
res = await fetchFn(url, {
|
|
method,
|
|
headers: reqHdr,
|
|
body: method === 'GET' ? undefined : body,
|
|
signal: ac ? ac.signal : undefined
|
|
})
|
|
} catch (e) {
|
|
if (t) clearTimeout(t)
|
|
const msg = e && e.message ? e.message : String(e)
|
|
if (!quiet) ctx.console.error('wget: ' + msg)
|
|
ctx.exitCode = 4
|
|
return
|
|
}
|
|
if (t) clearTimeout(t)
|
|
|
|
if (!res.ok) {
|
|
if (continuePartial && rangeStart > 0 && res.status === 416) {
|
|
continue
|
|
}
|
|
if (!quiet)
|
|
ctx.console.error('wget: HTTP error ' + res.status + ' for ' + url)
|
|
ctx.exitCode = 8
|
|
return
|
|
}
|
|
|
|
let buf = new Uint8Array(await res.arrayBuffer())
|
|
|
|
if (continuePartial && rangeStart > 0 && res.status === 206) {
|
|
const prev = await vfs.readFile(targetPath)
|
|
const prevU8 = prev ? new Uint8Array(prev) : new Uint8Array(0)
|
|
const combined = new Uint8Array(prevU8.length + buf.length)
|
|
combined.set(prevU8, 0)
|
|
combined.set(buf, prevU8.length)
|
|
buf = combined
|
|
}
|
|
|
|
if (
|
|
noClobber &&
|
|
targetPath != null &&
|
|
outputDocument !== '-' &&
|
|
!continuePartial
|
|
) {
|
|
try {
|
|
const ex = await vfs.stat(targetPath)
|
|
if (ex && ex.type === 'file' && (ex.size ?? 0) > 0) {
|
|
if (!quiet)
|
|
ctx.console.error(
|
|
'wget: ' + targetPath + ': File exists (--no-clobber)'
|
|
)
|
|
continue
|
|
}
|
|
} catch {
|
|
/* treat as absent */
|
|
}
|
|
}
|
|
|
|
if (outputDocument === '-') {
|
|
ctx.console.log(ctx.b4a.toString(buf))
|
|
} else if (outputDocument != null) {
|
|
await vfs.writeFile(outputDocument, buf)
|
|
if (!quiet) ctx.console.error(outputDocument)
|
|
} else {
|
|
await vfs.writeFile(targetPath, buf)
|
|
if (!quiet) ctx.console.error(targetPath)
|
|
}
|
|
}
|
|
|
|
ctx.exitCode = 0
|
|
}
|