/* BARE_OS_BIN_API 1.0.0 — bump when staged /bin script semantics change (see developer guide). */ /** Shared helpers for drive-resident /bin scripts (prepended before each command). */ function bareStdin(ctx) { return typeof ctx.shellStdin === 'string' ? ctx.shellStdin : '' } /** @param {number} mode @param {'file' | 'directory' | 'symlink'} type */ function bareFormatModeString(mode, type) { const typeChar = type === 'directory' ? 'd' : type === 'symlink' ? 'l' : '-' const perm = mode & 0o777 const r = (bit) => (perm & bit ? 'r' : '-') const w = (bit) => (perm & bit ? 'w' : '-') const x = (bit) => (perm & bit ? 'x' : '-') return ( typeChar + r(0o400) + w(0o200) + x(0o100) + r(0o040) + w(0o020) + x(0o010) + r(0o004) + w(0o002) + x(0o001) ) } /** @param {number} mtimeMs @param {number} [nowMs] */ function bareFormatLsMtime(mtimeMs, nowMs) { const now = nowMs != null ? nowMs : Date.now() const d = new Date(mtimeMs) const months = [ 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec' ] const mon = months[d.getMonth()] const day = String(d.getDate()).padStart(2, ' ') const sixMo = 180 * 24 * 3600 * 1000 if (Math.abs(now - mtimeMs) > sixMo) { const yr = String(d.getFullYear()).padStart(4, ' ') return mon + ' ' + day + ' ' + yr } const hh = String(d.getHours()).padStart(2, '0') const mm = String(d.getMinutes()).padStart(2, '0') return mon + ' ' + day + ' ' + hh + ':' + mm } /** @param {number} size */ function barePosixBlocks(size) { return Math.ceil(Number(size) / 512) || 0 } /** * Raw stdout for NUL/binary when **`process.stdout.write`** is missing. * If **`ctx.bareOsBinWrite(Uint8Array|string)`** is set (tests / host), use it. * @param {Record} ctx * @param {string | Uint8Array} chunk * @returns {boolean} */ function bareOsEmitRaw(ctx, chunk) { if (typeof ctx.bareOsBinWrite === 'function') { const b4 = ctx.b4a const u8 = typeof chunk === 'string' ? b4 && typeof b4.from === 'function' ? b4.from(chunk) : new TextEncoder().encode(chunk) : chunk ctx.bareOsBinWrite(u8 instanceof Uint8Array ? u8 : new Uint8Array(u8)) return true } const w = globalThis.process?.stdout?.write if (typeof w === 'function') { w.call(globalThis.process.stdout, chunk) return true } return false } /** Session env map (`vfs.env`, then `ctx.env`). Never throws. */ function bareOsEnv(ctx) { const v = ctx && ctx.vfs && ctx.vfs.env if (v && typeof v === 'object') return v const e = ctx && ctx.env if (e && typeof e === 'object') return e return {} } /** * Strict POSIX-ish decimal integer (no octal, no exponent, no empty). * @param {unknown} s * @returns {number} */ function bareOsParseDecInt(s) { const t = String(s == null ? '' : s).trim() if (!/^[+-]?(?:0|[1-9][0-9]*)$/.test(t)) return NaN const n = Number.parseInt(t, 10) return Number.isSafeInteger(n) ? n : NaN } /** @param {unknown} s */ function bareOsParseNonNegInt(s) { const n = bareOsParseDecInt(s) return n >= 0 ? n : NaN } /** * @param {Record} ctx * @param {string} name * @param {number} fallback * @param {number} [min] * @param {number} [max] */ function bareOsEnvInt(ctx, name, fallback, min, max) { const raw = bareOsEnv(ctx)[name] if (raw == null || raw === '') return fallback const n = Number.parseInt(String(raw), 10) if (!Number.isFinite(n)) return fallback let v = n if (min != null && v < min) v = min if (max != null && v > max) v = max return v } /** * @param {Record} ctx * @param {string} msg * @param {number} [code] */ function bareOsFail(ctx, msg, code) { if (msg) ctx.console.error(msg) ctx.exitCode = code == null ? 1 : code } /** @param {unknown} e */ function bareOsIsNotFoundErr(e) { const code = e && typeof e === 'object' ? e.code : '' if (code === 'ENOENT') return true const msg = String((e && e.message) || e || '') return /ENOENT|No such file|not found/i.test(msg) } /** * @param {Record} ctx * @param {unknown} buf * @returns {Uint8Array} */ function bareOsToU8(ctx, buf) { if (!buf) return new Uint8Array(0) if (buf instanceof Uint8Array) return buf if (ctx && ctx.b4a && typeof ctx.b4a.from === 'function') return ctx.b4a.from(buf) return new Uint8Array(buf) } /** @param {string} dir @param {string} name */ function bareOsJoinPath(dir, name) { const d = String(dir || '').replace(/\/+$/, '') const n = String(name || '').replace(/^\/+/, '') if (!d || d === '/') return '/' + n return d + '/' + n } /** @param {string} p */ function bareOsBaseName(p) { const t = String(p || '').replace(/\/+$/, '') if (!t || t === '/') return t === '/' ? '/' : '' const i = t.lastIndexOf('/') return i < 0 ? t : t.slice(i + 1) || t } /** @param {string} p */ function bareOsParentDir(p) { const t = String(p || '').replace(/\/+$/, '') || '/' if (t === '/') return '/' const i = t.lastIndexOf('/') return i <= 0 ? '/' : t.slice(0, i) || '/' } /** @param {string} p */ function bareOsNormPath(p) { return String(p || '').replace(/\/+$/, '') || '/' } /** * @param {Record} ctx * @param {string} p */ function bareOsResolvePath(ctx, p) { if (ctx && ctx.vfs && typeof ctx.vfs.resolveLogical === 'function') { try { return String(ctx.vfs.resolveLogical(p) || p) } catch { /* fall through */ } } return String(p || '') } /** * True when dest is src or lives under src (self-copy / self-move). * @param {Record} ctx * @param {string} src * @param {string} dest */ function bareOsDestInsideSrc(ctx, src, dest) { const s = bareOsNormPath(bareOsResolvePath(ctx, src)) const d = bareOsNormPath(bareOsResolvePath(ctx, dest)) if (s === d) return true if (s === '/') return d !== '/' return d === s || d.startsWith(s + '/') } const BARE_OS_B64_ALPH = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' /** @param {Uint8Array} u8 */ function bareOsB64Encode(u8) { let out = '' let i = 0 for (; i + 2 < u8.length; i += 3) { const n = (u8[i] << 16) | (u8[i + 1] << 8) | u8[i + 2] out += BARE_OS_B64_ALPH[(n >> 18) & 63] + BARE_OS_B64_ALPH[(n >> 12) & 63] + BARE_OS_B64_ALPH[(n >> 6) & 63] + BARE_OS_B64_ALPH[n & 63] } const rest = u8.length - i if (rest === 1) { const n = u8[i] << 16 out += BARE_OS_B64_ALPH[(n >> 18) & 63] + BARE_OS_B64_ALPH[(n >> 12) & 63] + '==' } else if (rest === 2) { const n = (u8[i] << 16) | (u8[i + 1] << 8) out += BARE_OS_B64_ALPH[(n >> 18) & 63] + BARE_OS_B64_ALPH[(n >> 12) & 63] + BARE_OS_B64_ALPH[(n >> 6) & 63] + '=' } return out } /** * RFC 4648 Base64 decode (also accepts URL-safe alphabet). Rejects junk. * @param {string} s * @returns {Uint8Array} */ function bareOsB64Decode(s) { const t = String(s).replace(/\s+/g, '') if (!t) return new Uint8Array(0) if (t.length % 4 === 1) throw new Error('invalid base64 length') let pad = 0 if (t.endsWith('==')) pad = 2 else if (t.endsWith('=')) pad = 1 const body = pad ? t.slice(0, t.length - pad) : t const bytes = [] let buf = 0 let bits = 0 for (let i = 0; i < body.length; i++) { const c = body[i] let v = BARE_OS_B64_ALPH.indexOf(c) if (v < 0) { if (c === '-') v = 62 else if (c === '_') v = 63 else throw new Error('invalid base64 character') } buf = (buf << 6) | v bits += 6 if (bits >= 8) { bits -= 8 bytes.push((buf >> bits) & 255) } } if (pad) { const want = Math.floor((body.length * 6) / 8) if (bytes.length > want) bytes.length = want } return new Uint8Array(bytes) } /** * @param {string} s * @returns {Uint8Array} */ function bareOsHexDecode(s) { const t = String(s).replace(/\s+/g, '') if (t.length % 2 !== 0) throw new Error('odd hex length') const out = new Uint8Array(t.length / 2) for (let i = 0; i < out.length; i++) { const pair = t.slice(i * 2, i * 2 + 2) if (!/^[0-9a-fA-F]{2}$/.test(pair)) throw new Error('invalid hex') out[i] = Number.parseInt(pair, 16) } return out } /** @param {Uint8Array} u8 */ function bareOsHexEncode(u8) { let s = '' for (let i = 0; i < u8.length; i++) s += u8[i].toString(16).padStart(2, '0') return s } // Bare OS: vendored from @sscots/jqjs@1.3.2 (https://github.com/sscots/jqjs). ISC License. Top-level compile() and prettyPrint(); not the C jq binary (https://github.com/jqlang/jq). // Triage: treat bug reports as jqjs semantic gaps; prefer golden tests before expanding builtins. Large JSON is bounded by host memory and stdin string caps (see handbook §3). // jqjs - jq JSON query language in JavaScript // Copyright (C) 2018-2023 Michael Homer /* * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ function compile(prog) { let filter = parse(tokenise(prog).tokens) return (input) => filter.node.apply(input, { userFuncs: {}, userFuncArgs: {}, variables: {} }) } function compileNode(prog) { return parse(tokenise(prog).tokens).node } function isAlpha(c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c === '_' } function isDigit(c) { return c >= '0' && c <= '9' } function isQuote(c) { return c === '"' || c === "'" } function prettyPrint(val, indent = '', step = ' ', LF = '\n') { let SP = step ? ' ' : '' if (typeof val == 'undefined') return val if (val === null) { return 'null' } else if (val instanceof Array) { let ret = '[' let first = true for (let v of Object.values(val)) { ret += (first ? '' : ',') + LF + indent + step + prettyPrint(v, indent + step, step, LF) first = false } ret += LF + indent + ']' return ret } else if (typeof val == 'object') { let ret = '{' let first = true for (let k of Object.keys(val)) { ret += (first ? '' : ',') + LF + indent + step + '"' + k + '":' + SP + prettyPrint(val[k], indent + step, step, LF) first = false } ret += LF + indent + '}' return ret } else if (typeof val == 'string') { return '"' + escapeString(val) + '"' } else if (typeof val == 'number') { return '' + val } else if (typeof val == 'boolean') { return val ? 'true' : 'false' } } function escapeString(s) { s = s.replace(/\\/g, '\\\\') s = s.replace(/"/g, '\\"') s = s.replace(/\n/g, '\\n') s = s.replace( /[\x00-\x1f]/g, (x) => '\\u00' + x.charCodeAt(0).toString(16).padStart(2, '0') ) return s } function* zip(a, b) { let aa = a[Symbol.iterator]() let bb = b[Symbol.iterator]() let v1 = aa.next() let v2 = bb.next() while (!v1.done && !v2.done) { yield [v1.value, v2.value] v1 = aa.next() v2 = bb.next() } } // Implements the jq ordering algorithm, which is terrible. function compareValues(a, b) { let at = nameType(a) let bt = nameType(b) if (at != bt) { return ( compareValues.typeOrder.indexOf(at) - compareValues.typeOrder.indexOf(bt) ) } if (at == 'boolean') { if (a && !b) return 1 if (!a && b) return -1 } else if (at == 'number') { return a - b } else if (at == 'string') { if (a < b) return -1 if (b < a) return 1 } else if (at == 'array') { for (let i = 0; i < a.length; i++) { let v1 = a[i] let v2 = b[i] if (typeof v1 == 'undefined' && typeof v2 != 'undefined') return -1 else if (typeof v1 != 'undefined' && typeof v2 == 'undefined') return 1 else if (typeof v1 == 'undefined') return 0 let c = compareValues(v1, v2) if (c != 0) return c } return 0 } else if (at == 'object') { let ka = Object.keys(a).sort() let kb = Object.keys(b).sort() let c = compareValues(ka, kb) if (c) return c for (let k of ka) { c = compareValues(a[k], b[k]) if (c) return c } } return 0 } compareValues.typeOrder = [ 'null', 'boolean', 'number', 'string', 'array', 'object' ] // Create a function from a program string. // // params is an array of parameter names // body is a jq program as a string, which may use the parameters // // For example: // makeFunc(['f'], '[.[] | f]') // defines the map function. function makeFunc(params, body, pathFunc = false) { let c = compileNode(body) let f = (x, conf) => c.apply(x, conf) if (pathFunc) f = (x, conf) => c.paths(x, conf) return function* (input, conf, args) { let origArgs = conf.userFuncArgs conf.userFuncArgs = Object.create(origArgs) for (let i = 0; i < params.length; i++) { let pn = params[i] let pv = args[i] conf.userFuncArgs[pn + '/0'] = pv } yield* f(input, conf) conf.userFuncArgs = origArgs } } // Define and save a function that is shorthand for a longer expression // // name is the name of the function // params is an array of parameters, or a string of one-character names // body is a jq program as a string function defineShorthandFunction(name, params, body) { let fname = name + '/' + params.length functions[fname] = makeFunc(params, body) functions[fname + '-paths'] = makeFunc(params, body, true) } // Recursive-descent parser for JQ query language // Split input program into tokens. Tokens are: // quote, number, identifier-index, dot-square, dot, left-square, // right-square, left-paren, right-paren, pipe, comma, // identifier, colon, left-brace, right-brace, semicolon, at, // variable, as, reduce, foreach, def, import, include, question // if, then, else, end, elif function tokenise(str, startAt = 0, parenDepth) { let ret = [] function error(msg) { throw msg } let i toplevel: for (i = startAt; i < str.length; i++) { let location = i let c = str[i] if (c == ' ') continue if (c == '"' || c == "'") { let st = c let tok = '' let escaped = false let uniEsc let cu = 0 for (i++; i < str.length; i++) { if (uniEsc) { uniEsc-- cu *= 16 cu += Number.parseInt(str[i], 16) if (uniEsc == 0) { tok += String.fromCharCode(cu) cu = 0 } } else if (escaped) { let q = str[i] if (q == '"' || q == "'") tok += q else if (q == 'n') tok += '\n' else if (q == 't') tok += '\t' else if (q == 'r') tok += '\r' else if (q == 'b') tok += '\b' else if (q == 'f') tok += '\f' else if (q == '/') tok += '/' else if (q == '\\') tok += '\\' else if (q == 'u') uniEsc = 4 else if (q == '(') { // Interpolation let r = tokenise(str, i + 1, 0) ret.push({ type: 'quote-interp', value: tok, location }) tok = '' ret = ret.concat(r.tokens) i = r.i } else throw 'invalid escape ' + q escaped = false } else if (str[i] == '\\') { escaped = true } else if (str[i] == st) { ret.push({ type: 'quote', value: tok, location }) continue toplevel } else { escaped = false tok += str[i] } } error('unterminated string literal') } else if (isDigit(c)) { let tok = '' while (isDigit(str[i]) || str[i] == '.') tok += str[i++] ret.push({ type: 'number', value: Number.parseFloat(tok), location }) i-- } else if (c == '.') { let d = str[i + 1] if (isAlpha(d) || isQuote(d)) { i++ let tok = '' while (isAlpha(str[i]) || isDigit(str[i]) || isQuote(str[i])) tok += str[i++] tok = tok.split('"').join('').split("'").join('') // Remove quotes ret.push({ type: 'identifier-index', value: tok, location }) i-- } else if (d == '[') { i++ ret.push({ type: 'dot-square', location }) } else if (d == '.') { i++ ret.push({ type: 'dot-dot', location }) } else { ret.push({ type: 'dot', location }) } } else if (c == '$') { let d = str[i + 1] i++ let tok = '' while (isAlpha(str[i]) || isDigit(str[i])) { tok += str[i] i++ } ret.push({ type: 'variable', name: tok, location }) i-- } else if (c == '[') { ret.push({ type: 'left-square', location }) } else if (c == ']') { ret.push({ type: 'right-square', location }) } else if (c == '(') { ret.push({ type: 'left-paren', location }) parenDepth++ } else if (c == ')') { ret.push({ type: 'right-paren', location }) parenDepth-- if (parenDepth < 0) return { tokens: ret, i } } else if (c == '{') { ret.push({ type: 'left-brace', location }) } else if (c == '}') { ret.push({ type: 'right-brace', location }) } else if (c == ',') { ret.push({ type: 'comma', location }) } else if (c == ';') { ret.push({ type: 'semicolon', location }) } else if (c == '@') { ret.push({ type: 'at', location }) } else if (c == '?') { ret.push({ type: 'question', location }) } else if (c == '|') { let d = str[i + 1] if (d == '=') { ret.push({ type: 'pipe-equals', location }) i++ } else ret.push({ type: 'pipe', location }) // Infix operators } else if ( c == '+' || c == '*' || c == '-' || c == '/' || c == '%' || c == '<' || c == '>' ) { if (c == '/' && str[i + 1] == '/') { c = '//' i++ } if (str[i + 1] == '=') { if (c == '<' || c == '>') ret.push({ type: 'op', op: c + '=', location }) else ret.push({ type: 'op-equals', op: c, location }) i++ } else ret.push({ type: 'op', op: c }) } else if (c == '=') { if (str[i + 1] != '=') throw 'plain assignment = is not supported' i++ ret.push({ type: 'op', op: '==', location }) } else if (c == '!') { if (str[i + 1] != '=') throw 'unexpected ! at ' + location i++ ret.push({ type: 'op', op: '!=', location }) } else if (isAlpha(c)) { let tok = '' while (isAlpha(str[i]) || isDigit(str[i]) || str[i] == '_') tok += str[i++] if ( tok == 'as' || tok == 'reduce' || tok == 'foreach' || tok == 'import' || tok == 'include' || tok == 'def' || tok == 'if' || tok == 'then' || tok == 'else' || tok == 'end' || tok == 'elif' ) { ret.push({ type: tok, location }) } else { ret.push({ type: 'identifier', value: tok, location }) } i-- } else if (c == ':') { ret.push({ type: 'colon', location }) } } ret.push({ type: '', location: i }) return { tokens: ret, i } } function describeLocation(token) { if (token) { return token.location } return '' } // Parse a token stream by recursive descent. // // Returns {node: {*apply(input, conf)}, i}, where i is the position in the // token stream and node is one of the filtering nodes defined below. // Returns at end of stream or when a token of type until is found. function parse(tokens, startAt = 0, until = 'none') { let i = startAt let t = tokens[i] let ret = [] let commaAccum = [] while (t && until.indexOf(t.type) == -1) { // Simple cases if (t.type == 'identifier-index') { ret.push(new IdentifierIndex(t.value)) } else if (t.type == 'number') { ret.push(new NumberNode(t.value)) } else if (t.type == 'quote') { ret.push(new StringNode(t.value)) } else if (t.type == 'dot') { ret.push(new IdentityNode()) } else if (t.type == 'dot-dot') { ret.push(new RecursiveDescent()) } else if (t.type == 'identifier') { if (t.value == 'true' || t.value == 'false') ret.push(new BooleanNode(t.value == 'true')) else if (t.value == 'null') ret.push(new BooleanNode(null)) else { // Named function let fname = t.value let args = [] if (tokens[i + 1] && tokens[i + 1].type == 'left-paren') { i++ while (tokens[i].type != 'right-paren') { let arg = parse(tokens, i + 1, ['semicolon', 'right-paren']) args.push(arg.node) i = arg.i } } ret.push(new FunctionCall(fname + '/' + args.length, args)) } // Recursive square bracket cases } else if (t.type == 'dot-square') { let r = parseDotSquare(tokens, i) ret.push(r.node) i = r.i } else if (t.type == 'left-square') { // Find the body of the brackets first let r = parse(tokens, i + 1, ['right-square', 'colon']) if (ret.length) { let lhs = makeFilterNode(ret) ret = [] if (tokens[r.i].type == 'colon') { // Slice if (r.node.length === 0) r.node = new NumberNode(0) let e = parse(tokens, r.i + 1, ['right-square']) if (e.node.length === 0) e.node = new NumberNode(-1) ret.push(new SliceNode(lhs, r.node, e.node)) r = e } else if (r.node.length === 0) ret.push(new SpecificValueIterator(lhs)) else ret.push(new IndexNode(lhs, r.node)) } else { ret.push(new ArrayNode(r.node)) } i = r.i // Recursive parenthesis case } else if (t.type == 'left-paren') { // Find the body of the brackets first let r = parse(tokens, i + 1, 'right-paren') ret.push(r.node) i = r.i // Object literal } else if (t.type == 'left-brace') { let r = parseObject(tokens, i + 1) ret.push(r.node) i = r.i // Format @x } else if (t.type == 'at') { let n = tokens[++i] if (!n || n.type != 'identifier') throw 'expected identifier after @ at ' + describeLocation(n) let fmt = n.value if (!formats[fmt]) throw 'not a valid format: ' + fmt let q if (tokens[i + 1] && tokens[i + 1].type == 'quote-interp') { ;({ q, i } = parseStringInterpolation(tokens, i + 1)) i = i } ret.push(new FormatNode(fmt, q)) // Comma consumes everything previous and splits in-place // (parsing carries on in this method) } else if (t.type == 'comma') { commaAccum.push(makeFilterNode(ret)) ret = [] // Pipe consumes everything previous *including* commas // and splits by recursing for the right-hand side. // "as" indicates a variable binding. } else if (t.type == 'pipe' || t.type == 'as') { if (commaAccum.length) { // .x,.y | .[1] is the same as (.x,.y) | .[1] commaAccum.push(makeFilterNode(ret)) ret = [new CommaNode(commaAccum)] commaAccum = [] } let lhs = makeFilterNode(ret) if (t.type == 'as') { let nameTok = tokens[i + 1] lhs = new VariableBinding(lhs, nameTok.name) i += 2 } let r = parse(tokens, i + 1, until) let rhs = r.node i = r.i if (tokens[i] && until.indexOf(tokens[i].type) != -1) i-- ret = [new PipeNode(lhs, rhs)] // Question mark suppresses errors on the preceding filter } else if (t.type == 'question') { let p = ret.pop() if (!p) throw 'unexpected ? without preceding filter at ' + describeLocation(t) ret.push(new ErrorSuppression(p)) // Infix operators } else if (t.type == 'op') { if (ret.length == 0 && t.op == '-' && tokens[i + 1].type == 'number') { tokens[i + 1].value = -tokens[i + 1].value t = tokens[++i] continue } let lhs = makeFilterNode(ret) let op = t.op let stream = [lhs, t] let r = parse( tokens, i + 1, [ 'op', 'comma', 'pipe', 'right-paren', 'right-brace', 'right-square', '' ].concat(until) ) i = r.i stream.push(r.node) while (i < tokens.length && tokens[i].type == 'op') { stream.push(tokens[i]) let r = parse( tokens, i + 1, [ 'op', 'comma', 'pipe', 'right-paren', 'right-brace', 'right-square', '' ].concat(until) ) i = r.i stream.push(r.node) } ret = [shuntingYard(stream)] if (tokens[i]) i-- // Update-assignment } else if (t.type == 'pipe-equals') { let lhs = makeFilterNode(ret) let r = parse( tokens, i + 1, [ 'comma', 'pipe', 'right-paren', 'right-brace', 'right-square', '' ].concat(until) ) i = r.i let rhs = r.node ret = [new UpdateAssignment(lhs, rhs)] // Arithmetic update-assignment } else if (t.type == 'op-equals') { let lhs = makeFilterNode(ret) let r = parse( tokens, i + 1, [ 'comma', 'pipe', 'right-paren', 'right-brace', 'right-square', '' ].concat(until) ) i = r.i let rhs = r.node rhs = shuntingYard([new IdentityNode(), { type: 'op', op: t.op }, rhs]) ret = [new UpdateAssignment(lhs, rhs)] // reduce .[] as $item (0, . + $item) } else if (t.type == 'reduce') { let r = parse(tokens, i + 1, ['as']) i = r.i let generator = r.node i++ // 'as' let name = tokens[i].name i++ if (tokens[i].type != 'left-paren') throw 'expected left-paren in reduce at ' + describeLocation(tokens[i]) r = parse(tokens, i + 1, ['semicolon']) i = r.i let init = r.node r = parse(tokens, i + 1, ['right-paren']) i = r.i let expr = r.node ret.push(new ReduceNode(generator, name, init, expr)) // Interpolated string literal } else if (t.type == 'quote-interp') { let q ;({ q, i } = parseStringInterpolation(tokens, i)) ret.push(q) // Variable reference } else if (t.type == 'variable') { ret.push(new VariableReference(t.name)) // Conditional if-then-(elif-then)*-else?-end } else if (t.type == 'if') { let conds = [] let trueExprs = [] let falseExpr = null while ( tokens[i] && (tokens[i].type == 'if' || tokens[i].type == 'elif') ) { let cond = parse(tokens, i + 1, ['then']) if (!tokens[cond.i] || tokens[cond.i].type != 'then') throw ( 'expected then at ' + describeLocation(tokens[cond.i]) + ', from ' + tokens[i].type + ' at ' + tokens[i].location ) let trueExpr = parse(tokens, cond.i + 1, ['else', 'elif', 'end']) if (trueExpr.i == cond.i + 1) throw ( 'expected expression after then at ' + describeLocation(tokens[cond.i + 1]) + ', not ' + tokens[cond.i + 1].type ) i = trueExpr.i conds.push(cond.node) trueExprs.push(trueExpr.node) } if (tokens[i] && tokens[i].type == 'else') { let elseCase = parse(tokens, i + 1, ['end']) i = elseCase.i falseExpr = elseCase.node } if (!tokens[i] || tokens[i].type != 'end') throw ( 'expected end at ' + describeLocation(tokens[i]) + ' from if at ' + t.location ) ret.push(new IfNode(conds, trueExprs, falseExpr)) } else if (t.type == '' && until == 'none') { break } else { throw ( 'could not handle token ' + t.type + ' at ' + describeLocation(t) + (until != 'none' ? ', expected ' + until.join('/') : '') ) } t = tokens[++i] } // If a comma appeared this array is non-empty and contains all // previous branches. if (commaAccum.length) { commaAccum.push(makeFilterNode(ret)) return { node: new CommaNode(commaAccum), i } } return { node: makeFilterNode(ret), i } } function makeFilterNode(ret) { if (ret.length == 1) return ret[0] return new FilterNode(ret) } // Consumes pairs (quote-interp, expression up to rparen)* followed by // a bare string and returns a StringLiteral node with the interleaving // lists. function parseStringInterpolation(tokens, i) { let t = tokens[i] let strings = [] let interps = [] strings.push(t.value) // Always followed by a paren expression afterwards let inner = parse(tokens, i + 1, ['right-paren']) i = inner.i + 1 interps.push(inner.node) while (tokens[i].type == 'quote-interp') { strings.push(tokens[i].value) inner = parse(tokens, i + 1, ['right-paren']) i = inner.i + 1 interps.push(inner.node) } // Must be the ending quote now strings.push(tokens[i].value) return { q: new StringLiteral(strings, interps), i } } function parseDotSquare(tokens, startAt = 0) { let i = startAt let ds = tokens[i] i++ if (tokens[i].type == 'right-square') return { node: new GenericValueIterator(), i } let r = parse(tokens, i, ['right-square', 'colon']) if (tokens[r.i].type == 'colon') { // Slice let fr = r if (fr.length === 0) fr.node = new NumberNode(0) r = parse(tokens, r.i + 1, ['right-square']) if (r.length === 0) r.node = new NumberNode(-1) return { node: new GenericSlice(fr.node, r.node), i: r.i } } return { node: new GenericIndex(r.node), i: r.i } } // Parse an object literal, expecting to start immediately inside the // left brace and to consume up to and including the right brace. function parseObject(tokens, startAt = 0) { let i = startAt let fields = [] while (tokens[i].type != 'right-brace') { if (tokens[i].type == 'identifier') { // bare name x let ident = tokens[i++] if (tokens[i].type == 'colon') { // with value x: val let r = parse(tokens, i + 1, ['comma', 'right-brace']) i = r.i fields.push({ key: new StringNode(ident.value), value: r.node }) i-- } else if (tokens[i].type == 'comma') { // no value: equivalent to x : .x fields.push({ key: new StringNode(ident.value), value: new IdentifierIndex(ident.value) }) } else if (tokens[i].type == 'right-brace') { // ditto, last field: equivalent to x : .x fields.push({ key: new StringNode(ident.value), value: new IdentifierIndex(ident.value) }) i-- } } else if (tokens[i].type == 'quote') { // quoted-string key: "x" : val let ident = tokens[i++] if (tokens[i].type == 'colon') { let r = parse(tokens, i + 1, ['comma', 'right-brace']) i = r.i fields.push({ key: new StringNode(ident.value), value: r.node }) i-- } else { throw ( 'unexpected ' + tokens[i].type + ', expected colon at ' + describeLocation(tokens[i]) ) } } else if (tokens[i].type == 'left-paren') { // computed key: (.x | .y) : val let kr = parse(tokens, i + 1, 'right-paren') i = kr.i + 1 if (tokens[i].type == 'colon') { let r = parse(tokens, i + 1, ['comma', 'right-brace']) i = r.i fields.push({ key: kr.node, value: r.node }) i-- } else { throw ( 'unexpected ' + tokens[i].type + ', expected colon at ' + describeLocation(tokens[i]) ) } } else { throw ( 'unexpected ' + tokens[i].type + ' at ' + describeLocation(tokens[i]) + ' in object at ' + describeLocation(tokens[startAt - 1]) ) } i++ // Consume a comma after a field if (tokens[i].type == 'comma') i++ } return { node: new ObjectNode(fields), i } } function shuntingYard(stream) { const prec = { '+': 5, '-': 5, '*': 10, '/': 10, '%': 10, '//': 2, '==': 3, '!=': 3, '>': 3, '<': 3, '>=': 3, '<=': 3 } let output = [] let operators = [] for (let x of stream) { if (x.type == 'op') { while (operators.length && prec[operators[0].op] >= prec[x.op]) output.push(operators.shift()) operators.unshift(x) } else { output.push(x) } } for (let o of operators) output.push(o) let constructors = { '+': AdditionOperator, '*': MultiplicationOperator, '-': SubtractionOperator, '/': DivisionOperator, '%': ModuloOperator, '//': AlternativeOperator, '==': EqualsOperator, '!=': NotEqualsOperator, '<': LessThanOperator, '>': GreaterThanOperator, '<=': LessEqualsOperator, '>=': GreaterEqualsOperator } let stack = [] for (let o of output) { if (o.type == 'op') { let r = stack.pop() let l = stack.pop() stack.push(new constructors[o.op](l, r)) } else { stack.push(o) } } return stack[0] } // Convert a value to a consistent type name, addressing the issue // that arrays are objects. function nameType(o) { if (o === null) return 'null' if (typeof o == 'number') return 'number' if (typeof o == 'string') return 'string' if (typeof o == 'boolean') return 'boolean' if (o instanceof Array) return 'array' if (typeof o == 'object') return 'object' } // Parse node classes follow. Parse nodes are: // FilterNode, generic juxtaposition combination // IndexNode, lhs[rhs] // SliceNode, lhs[from:to] // GenericIndex, .[index] // IdentifierIndex .index (delegates to GenericIndex("index")) // GenericSlice, .[from:to] // IdentityNode, . // ValueNode, parent of string/number/boolean // StringNode, "abc" // NumberNode, 123.45 // BooleanNode, true/false // SpecificValueIterator, lhs[] (yields values from lhs) // GenericValueIterator, .[] (yields values from input) // CommaNode, .x, .y, .z // ArrayNode, [...] // PipeNode, a | b | c // ObjectNode { x : y, z, "a b" : 12, (.x.y) : .z } // RecursiveDescent, .. // OperatorNode, a binary infix operator // AdditionOperator, a + b // MultiplicationOperator, a * b // SubtractionOperator, a - b // DivisionOperator, a / b // ModuloOperator, a % b // EqualsOperator, a == b // NotEqualsOperator, a != b // AlternativeOperator, a // b // UpdateAssignment, .x.y |= .z // FunctionCall, fname(arg1, arg2) // FormatNode, @format, @format "a\(...)" // ErrorSuppression, foo? // VariableBinding, ... as $x (not the pipe) // VariableReference, $x // ReduceNode, reduce .[] as $x (0; . + $x) // IfNode, if a then b elif c then d else e end class ParseNode {} class FilterNode extends ParseNode { constructor(nodes) { super() this.length = nodes.length let p = nodes.pop() if (p) { this.filter = p this.source = nodes.length == 1 ? nodes[0] : new FilterNode(nodes) } } *apply(input, conf) { if (!this.filter) return for (let v of this.source.apply(input, conf)) { yield* this.filter.apply(v, conf) } } *paths(input, conf) { if (!this.filter) { return [] } for (let v of this.source.paths(input, conf)) { for (let w of this.filter.paths(input, conf)) { yield v.concat(w) } } } } class IndexNode extends ParseNode { constructor(lhs, index) { super() this.lhs = lhs this.index = index } *apply(input, conf) { for (let l of this.lhs.apply(input, conf)) { let t = nameType(l) for (let i of this.index.apply(input, conf)) { if (t == 'array' && nameType(i) != 'number') throw ( 'Cannot index array with ' + nameType(i) + ' ' + JSON.stringify(i) ) else if (t == 'object' && nameType(i) != 'string') throw ( 'Cannot index object with ' + nameType(i) + ' ' + JSON.stringify(i) ) if (typeof i == 'number' && i < 0 && nameType(l) == 'array') yield l[l.length + i] else yield l[i] yield typeof l[i] == 'undefined' ? null : l[i] } } } *paths(input, conf) { for (let l of this.lhs.paths(input, conf)) for (let a of this.index.apply(input, conf)) yield l.concat([a]) } } class SliceNode extends ParseNode { constructor(lhs, from, to) { super() this.lhs = lhs this.from = from this.to = to } *apply(input, conf) { for (let l of this.lhs.apply(input, conf)) for (let s of this.from.apply(input, conf)) { if (s < 0) s += l.length for (let e of this.to.apply(input, conf)) { if (e < 0) e += l.length yield l.slice(s, e) } } } *paths(input, conf) { for (let l of this.lhs.paths(input, conf)) for (let a of this.from.apply(input, conf)) for (let b of this.to.apply(input, conf)) yield l.concat([{ start: a, end: b }]) } } class GenericIndex extends ParseNode { constructor(innerNode) { super() this.index = innerNode } *apply(input, conf) { let t = nameType(input) for (let i of this.index.apply(input, conf)) { if (t == 'array' && nameType(i) != 'number') throw 'Cannot index array with ' + nameType(i) + ' ' + JSON.stringify(i) else if (t == 'object' && nameType(i) != 'string') throw ( 'Cannot index object with ' + nameType(i) + ' ' + JSON.stringify(i) ) if (typeof i == 'number' && i < 0 && nameType(input) == 'array') yield input[input.length + i] else yield typeof input[i] == 'undefined' ? null : input[i] } } *paths(input, conf) { for (let a of this.index.apply(input, conf)) yield [a] } } class IdentifierIndex extends GenericIndex { constructor(v) { super(new StringNode(v)) } } class GenericSlice extends ParseNode { constructor(fr, to) { super() this.from = fr this.to = to } *apply(input, conf) { for (let l of this.from.apply(input, conf)) { if (l < 0) l += input.length for (let r of this.to.apply(input, conf)) { if (r < 0) r += input.length yield input.slice(l, r) } } } *paths(input, conf) { for (let l of this.from.apply(input, conf)) for (let r of this.to.apply(input, conf)) yield [{ start: l, end: r }] } } class IdentityNode extends ParseNode { constructor() { super() } *apply(input, conf) { yield input } *paths(input, conf) { yield [] } } class ValueNode extends ParseNode { constructor(v) { super() this.value = v } *apply() { yield this.value } *paths(input, conf) { yield this.value } } class StringNode extends ValueNode { constructor(v) { super(v) } } class StringLiteral extends ParseNode { constructor(strings, interpolations) { super() this.strings = strings this.interpolations = interpolations } *apply(input, conf) { yield* this.applyEscape(input, formats.text, conf) } *applyEscape(input, esc, conf, startAt = 0) { let s = this.strings[startAt] let i = this.interpolations[startAt] if (!i) return yield s for (let v of this.interpolations[startAt].apply(input, conf)) { for (let r of this.applyEscape(input, esc, conf, startAt + 1)) { yield s + esc(v) + r } } } } class NumberNode extends ValueNode { constructor(v) { super(v) } } class BooleanNode extends ValueNode { constructor(v) { super(v) } } class SpecificValueIterator extends ParseNode { constructor(source) { super() this.source = source } *apply(input, conf) { for (let o of this.source.apply(input, conf)) yield* Object.values(o) } *paths(input, conf) { for (let [p, v] of this.zip( this.source.paths(input, conf), this.source.apply(input, conf) )) { if (nameType(v) == 'array') for (let i = 0; i < v.length; i++) yield p.concat([i]) else for (let i of Object.keys(v)) { yield p.concat([i]) } } } *zip(a, b) { let aa = a[Symbol.iterator]() let bb = b[Symbol.iterator]() let v1 = aa.next() let v2 = bb.next() while (!v1.done && !v2.done) { yield [v1.value, v2.value] v1 = aa.next() v2 = bb.next() } } } class GenericValueIterator extends ParseNode { constructor() { super() } *apply(input, conf) { if (nameType(input) == 'array') yield* input else yield* Object.values(input) } *paths(input, conf) { if (nameType(input) == 'array') for (let i = 0; i < input.length; i++) yield [i] else for (let o of Object.keys(input)) yield [o] } } class CommaNode extends ParseNode { constructor(branches) { super() this.branches = branches } *apply(input, conf) { for (let b of this.branches) yield* b.apply(input, conf) } *paths(input, conf) { for (let b of this.branches) yield* b.paths(input, conf) } } class ArrayNode extends ParseNode { constructor(body) { super() this.body = body } *apply(input, conf) { yield Array.from(this.body.apply(input, conf)) } } class PipeNode extends ParseNode { constructor(lhs, rhs) { super() this.lhs = lhs this.rhs = rhs } *apply(input, conf) { for (let v of this.lhs.apply(input, conf)) for (let q of this.rhs.apply(v, conf)) yield q } *paths(input, conf) { for (let [p, v] of this.zip( this.lhs.paths(input, conf), this.lhs.apply(input, conf) )) { for (let p2 of this.rhs.paths(v, conf)) { yield p.concat(p2) } } } *zip(a, b) { let aa = a[Symbol.iterator]() let bb = b[Symbol.iterator]() let v1 = aa.next() let v2 = bb.next() while (!v1.done && !v2.done) { yield [v1.value, v2.value] v1 = aa.next() v2 = bb.next() } } } class ObjectNode extends ParseNode { constructor(fields) { super() this.fields = fields } *apply(input, conf) { let obj = {} let values = {} let keys = [] for (let { key, value } of this.fields) { for (let k of key.apply(input, conf)) { keys.push(k) values[k] = [] for (let v of value.apply(input, conf)) values[k].push(v) } } yield* this.helper(keys, values, 0, {}) } *helper(keys, values, startAt, obj) { if (startAt >= keys.length) { yield Object.assign({}, obj) return } let k = keys[startAt] for (let v of values[k]) { obj[k] = v yield* this.helper(keys, values, startAt + 1, obj) } } } class RecursiveDescent extends ParseNode { constructor() { super() } *apply(input, conf) { yield* this.recurse(input) } *recurse(s) { yield s let t = nameType(s) if (t == 'array' || t == 'object') for (let v of Object.values(s)) yield* this.recurse(v) } *paths(input, conf) { yield* this.recursePaths(input, []) } *recursePaths(s, prefix) { yield prefix let t = nameType(s) if (t == 'array') for (let i = 0; i < s.length; i++) yield* this.recursePaths(s[i], prefix.concat([i])) else if (t == 'object') for (let [k, v] of Object.entries(s)) yield* this.recursePaths(v, prefix.concat([k])) } } class OperatorNode extends ParseNode { constructor(l, r) { super() this.l = l this.r = r } *apply(input, conf) { for (let rr of this.r.apply(input, conf)) for (let ll of this.l.apply(input, conf)) yield this.combine(ll, rr, nameType(ll), nameType(rr)) } } class AdditionOperator extends OperatorNode { constructor(l, r) { super(l, r) } combine(l, r, lt, rt) { if (lt == 'number' && rt == 'number') return l + r if (l === null) return r if (r === null) return l if (lt == 'string' && rt == 'string') return l + r if (lt == 'array' && rt == 'array') return l.concat(r) if (lt == 'object' && rt == 'object') return Object.assign(Object.assign({}, l), r) throw 'type mismatch in +:' + lt + ' and ' + rt + ' cannot be added' } } class MultiplicationOperator extends OperatorNode { constructor(l, r) { super(l, r) } combine(l, r, lt, rt) { if (lt == 'number' && rt == 'number') return l * r if (lt == 'number' && rt == 'string') return this.repeat(r, l) if (lt == 'string' && rt == 'number') return this.repeat(l, r) if (lt == 'object' && rt == 'object') return this.merge(Object.assign({}, l), r) throw 'type mismatch in *:' + lt + ' and ' + rt + ' cannot be multiplied' } repeat(s, n) { if (n == 0) return null let r = [] for (let i = 0; i < n; i++) r.push(s) return r.join('') } merge(l, r) { for (let k of Object.keys(r)) { if (!l.hasOwnProperty(k)) l[k] = r[k] else if (nameType(l[k]) != 'object' || nameType(r[k]) != 'object') l[k] = r[k] else this.merge(l[k], r[k]) } return l } } class SubtractionOperator extends OperatorNode { constructor(l, r) { super(l, r) } combine(l, r, lt, rt) { if (lt == 'number' && rt == 'number') return l - r if (l == null || r == null) throw 'type mismatch in -' if (lt == 'array' && rt == 'array') return l.filter((x) => r.indexOf(x) == -1) throw 'type mismatch in -:' + lt + ' and ' + rt + ' cannot be subtracted' } } class DivisionOperator extends OperatorNode { constructor(l, r) { super(l, r) } combine(l, r, lt, rt) { if (lt == 'number' && rt == 'number') return l / r if (lt == 'string' && rt == 'string') return l.split(r) throw 'type mismatch in -:' + lt + ' and ' + rt + ' cannot be divided' } } class ModuloOperator extends OperatorNode { constructor(l, r) { super(l, r) } combine(l, r, lt, rt) { if (lt == 'number' && rt == 'number') return l % r throw ( 'type mismatch in -:' + lt + ' and ' + rt + ' cannot be divided (remainder)' ) } } class LessThanOperator extends OperatorNode { constructor(l, r) { super(l, r) } combine(l, r, lt, rt) { return compareValues(l, r) < 0 } } class GreaterThanOperator extends OperatorNode { constructor(l, r) { super(l, r) } combine(l, r, lt, rt) { return compareValues(l, r) > 0 } } class LessEqualsOperator extends OperatorNode { constructor(l, r) { super(l, r) } combine(l, r, lt, rt) { return compareValues(l, r) <= 0 } } class GreaterEqualsOperator extends OperatorNode { constructor(l, r) { super(l, r) } combine(l, r, lt, rt) { return compareValues(l, r) >= 0 } } class EqualsOperator extends OperatorNode { constructor(l, r) { super(l, r) } combine(l, r, lt, rt) { if (lt != rt) return false if (lt == 'number' || lt == 'string' || lt == 'boolean' || lt == 'null') return l == r if (lt == 'array') { if (l.length != r.length) return false for (let i = 0; i < l.length; i++) if (!this.combine(l[i], r[i], nameType(l[i]), nameType(r[i]))) return false return true } let lk = Object.keys(l) let rk = Object.keys(r) if (lk.length != rk.length) return false for (let k of lk) { if (!r.hasOwnProperty(k)) return false if (!this.combine(l[k], r[k], nameType(l[k]), nameType(r[k]))) return false } return true } } class NotEqualsOperator extends EqualsOperator { constructor(l, r) { super(l, r) } combine(l, r, lt, rt) { return !super.combine(l, r, lt, rt) } } class AlternativeOperator extends ParseNode { constructor(l, r) { super() this.lhs = l this.rhs = r } *apply(input, conf) { let found = false for (let v of this.lhs.apply(input, conf)) { if (v !== null) found = true yield v } if (!found) yield* this.rhs.apply(input, conf) } } class UpdateAssignment extends ParseNode { constructor(l, r) { super() this.l = l this.r = r } *apply(input, conf) { input = JSON.parse(JSON.stringify(input)) for (let p of this.l.paths(input, conf)) { let it = this.r.apply(this.get(input, p), conf).next() if (it.done) input = this.update(input, p, null, true) else input = this.update(input, p, it.value) } yield input } // Pluck the value at a path out of an object get(obj, p) { let o = obj for (let i of p) o = o[i] return o } // Set the value at path p to v in obj, // or delete the key if del is true. update(obj, p, v, del = false) { let o = obj let last = p.pop() for (let i of p) o = o[i] if (typeof last == 'undefined') return v o[last] = v if (del) delete o[last] return obj } } class FunctionCall extends ParseNode { constructor(fname, args) { super() this.name = fname this.args = args } apply(input, conf) { let func let ufa = conf.userFuncArgs[this.name] if (ufa) func = function (input, conf, args) { return ufa.apply(input, conf) } else if (!func) func = functions[this.name] if (!func) throw 'no such function ' + this.name let argStack = [] return func(input, conf, this.args) } paths(input, conf) { let ufa = conf.userFuncArgs[this.name] if (ufa) return ufa.paths(input, conf) let func = functions[this.name + '-paths'] if (!func) throw 'no paths for ' + this.name return func(input, conf, this.args) } } class FormatNode extends ParseNode { constructor(fname, quote) { super() this.name = fname this.string = quote } *apply(input, conf) { if (!this.string) return yield formats[this.name](input) yield* this.string.applyEscape(input, formats[this.name], conf) } } class ErrorSuppression extends ParseNode { constructor(inner) { super() this.inner = inner } *apply(input, conf) { try { for (let o of this.inner.apply(input, conf)) if (o !== null) yield o } catch {} } *paths(input, conf) { try { for (let [o, p] of zip( this.inner.apply(input, conf), this.inner.paths(input, conf) )) if (o !== null) yield p } catch {} } } class VariableBinding extends ParseNode { constructor(lhs, name) { super() this.value = lhs this.name = name } *apply(input, conf) { for (let v of this.value.apply(input, conf)) { conf.variables[this.name] = v yield input } delete conf.variables[this.name] } } class VariableReference extends ParseNode { constructor(name) { super() this.name = name } *apply(input, conf) { yield conf.variables[this.name] } } class ReduceNode extends ParseNode { constructor(generator, name, init, expr) { super() this.generator = generator this.name = name this.init = init this.expr = expr } *apply(input, conf) { // This uses all values of the initialiser, but only the // last value of the reduction expression is retained. This // seems to match jq proper's behaviour, but jq has odd // errors in mixed cases that seem unnecessary. for (let accum of this.init.apply(input, conf)) { for (let v of this.generator.apply(input, conf)) { conf.variables[this.name] = v for (let a of this.expr.apply(accum, conf)) accum = a } delete conf.variables[this.name] yield accum } } } class IfNode extends ParseNode { constructor(conditions, thens, elseBranch) { super() this.conditions = conditions this.thens = thens this.elseBranch = elseBranch } *apply(input, conf) { for (let [c, t] of zip(this.conditions, this.thens)) { for (let cond of c.apply(input, conf)) { if (cond) { for (let o of t.apply(input, conf)) yield o return } } } if (this.elseBranch) { yield* this.elseBranch.apply(input, conf) return } yield input } } const formats = { text(v) { if (typeof v == 'string') return v return prettyPrint(v, '', '', '') }, json(v) { return prettyPrint(v, '', '', '') }, html(v) { if (typeof v != 'string') v = prettyPrint(v, '', '', '') return v .replace(//g, '>') .replace(/&/g, '&') .replace(/'/g, ''') .replace(/"/g, '"') }, uri(v) { if (typeof v != 'string') v = prettyPrint(v, '', '', '') return escape(v) }, csv(v) { if (nameType(v) != 'array') throw 'cannot csv-format ' + nameType(v) + ', only array' return v .map((x) => { if (typeof x == 'string') return '"' + x.replace(/"/g, '""') + '"' else if (typeof x == 'number') return '' + x else if (x === null) return '' else throw 'type ' + nameType(x) + ' not valid in a csv row' }) .join(',') }, tsv(v) { if (nameType(v) != 'array') throw 'cannot tsv-format ' + nameType(v) + ', only array' return v .map((x) => { if (typeof x == 'string') return escapeString(x) else if (typeof x == 'number') return '' + x else if (x === null) return '' else throw 'type ' + nameType(x) + ' not valid in a tsv row' }) .join('\t') }, base64(v) { if (typeof v != 'string') v = prettyPrint(v, '', '', '') return btoa(v) }, base64d(v) { if (typeof v != 'string') throw 'can only base64-decode strings' return atob(v) }, sh(v) { let t = nameType(v) if (t == 'string') return "'" + t.replace(/'/g, "'\\''") + "'" else if (t == 'number') return '' + v else if (t == 'boolean') return '' + v else if (v === null) return 'null' else if (t === 'array') { return v .map((v) => { let t = nameType(v) if (t == 'string') return "'" + v.replace(/'/g, "'\\''") + "'" else if (t == 'number') return '' + v else if (t == 'boolean') return '' + v else if (v === null) return 'null' else throw t + ' cannot be escaped for shell' }) .join(' ') } else throw t + ' cannot be escaped for shell' } } const functions = { 'tostring/0': function* (input) { yield formats.text(input) }, 'empty/0': function* (input) {}, 'path/1': function* (input, conf, args) { let f = args[0] yield* f.paths(input, conf) }, 'select/1': function* (input, conf, args) { let selector = args[0] for (let b of selector.apply(input, conf)) if (b) yield input }, 'select/1-paths': function* (input, conf, args) { let selector = args[0] for (let b of selector.apply(input, conf)) if (b) yield [] }, 'length/0': function* (input) { let t = nameType(input) if (t == 'string' || t == 'array') return yield input.length if (t == 'null') return yield 0 if (t == 'object') return yield Object.keys(input).length throw 'cannot compute length of ' + t }, 'keys/0': function* (input) { yield* Object.keys(input).sort() }, 'has/1': function* (input, conf, args) { let f = args[0] for (let k of f.apply(input, conf)) yield input.hasOwnProperty(k) }, 'has/1-paths': function* (input, conf, args) { let f = args[0] for (let k of f.apply(input, conf)) if (input.hasOwnProperty(k)) yield [] }, 'in/1': function* (input, conf, args) { let f = args[0] for (let o of f.apply(input, conf)) yield o.hasOwnProperty(input) }, 'in/1-paths': function* (input, conf, args) { let f = args[0] for (let o of f.apply(input, conf)) if (o.hasOwnProperty(input)) yield [] }, 'contains/1': function* (input, conf, args) { let f = args[0] let t = nameType(input) for (let o of f.apply(input, conf)) { let ot = nameType(o) if (t != ot) { throw t + ' and ' + ot + ' cannot have their containment checked' } else yield containsHelper(input, o) } }, 'inside/1': function* (input, conf, args) { let f = args[0] let t = nameType(input) for (let o of f.apply(input, conf)) { let ot = nameType(o) if (t != ot) { throw t + ' and ' + ot + ' cannot have their containment checked' } else yield containsHelper(o, input) } }, 'to_entries/0': function* (input, conf) { let t = nameType(input) if (t == 'array') { let ret = [] for (let i = 0; i < input.length; i++) ret.push({ key: i, value: input[i] }) yield ret } else if (t == 'object') yield Object.entries(input).map((a) => ({ key: a[0], value: a[1] })) else throw 'cannot make entries from ' + t }, 'from_entries/0': function* (input, conf) { let t = nameType(input) if (t == 'array') { let obj = {} for (let { key, value } of input) obj[key] = value yield obj } else throw 'cannot use entries from ' + t }, 'type/0': function* (input) { yield nameType(input) }, 'range/1': function* (input, conf, args) { for (let m of args[0].apply(input, conf)) for (let i = 0; i < m; i++) yield i }, 'range/2': function* (input, conf, args) { for (let min of args[0].apply(input, conf)) for (let max of args[1].apply(input, conf)) for (let i = min; i < max; i++) yield i }, 'range/3': function* (input, conf, args) { for (let min of args[0].apply(input, conf)) for (let max of args[1].apply(input, conf)) for (let step of args[2].apply(input, conf)) for (let i = min; i < max; i += step) yield i }, 'any/0': function* (input, conf) { if (nameType(input) != 'array') throw 'any/0 requires array as input, not ' + nameType(input) for (let b of input) if (b) return yield true yield false }, 'any/1': function* (input, conf, args) { if (nameType(input) != 'array') throw 'any/1 requires array as input, not ' + nameType(input) for (let v of input) for (let b of args[0].apply(v, conf)) if (b) return yield true yield false }, 'any/2': function* (input, conf, args) { let gen = args[0] let cond = args[1] for (let v of gen.apply(input, conf)) for (let b of cond.apply(v, conf)) if (b) return yield true yield false }, 'all/0': function* (input, conf) { if (nameType(input) != 'array') throw 'all/0 requires array as input, not ' + nameType(input) for (let b of input) if (!b) return yield false yield true }, 'all/1': function* (input, conf, args) { if (nameType(input) != 'array') throw 'all/1 requires array as input, not ' + nameType(input) for (let v of input) for (let b of args[0].apply(v, conf)) if (!b) return yield false yield true }, 'all/2': function* (input, conf, args) { let gen = args[0] let cond = args[1] for (let v of gen.apply(input, conf)) for (let b of cond.apply(v, conf)) if (!b) return yield false yield true }, 'add/0': function* (input, conf) { if (nameType(input) != 'array') throw 'can only add up arrays' if (input.length == 0) return yield null if (input.length == 1) return yield input[0] let ret = AdditionOperator.prototype.combine( input[0], input[1], nameType(input[0]), nameType(input[1]) ) for (let i = 2; i < input.length; i++) ret = AdditionOperator.prototype.combine( ret, input[i], nameType(ret), nameType(input[i]) ) yield ret }, 'round/0': function* (input) { yield Math.round(input) }, 'round/1': function* (input, conf, args) { yield input.toFixed(args[0].value) }, 'tonumber/0': function* (input) { yield Number.parseFloat(input) }, 'sort/0': function* (input, conf) { if (nameType(input) != 'array') throw 'can only sort arrays, not ' + nameType(input) let r = Array.from(input) yield r.sort(compareValues) }, 'sort_by/1': function* (input, conf, args) { if (nameType(input) != 'array') throw 'can only sort arrays, not ' + nameType(input) let key = args[0] let r = input.map((v) => ({ key: key.apply(v, conf).next().value, value: v })) r.sort((a, b) => compareValues(a.key, b.key)) yield r.map((a) => a.value) }, 'explode/0': function* (input, conf) { if (nameType(input) != 'string') throw 'can only explode string, not ' + nameType(input) let ret = [] for (let i = 0; i < input.length; i++) { let c = input.charCodeAt(i) ret.push(c) if (c > 0xffff) i++ } yield ret }, 'implode/0': function* (input, conf) { if (nameType(input) != 'array') throw 'can only implode array, not ' + nameType(input) yield input.map((x) => String.fromCodePoint(x)).join('') }, 'split/1': function* (input, conf, args) { if (nameType(input) != 'string') throw 'can only split string, not ' + nameType(input) for (let s of args[0].apply(input, conf)) { yield input.split(s) } }, 'join/1': function* (input, conf, args) { if (nameType(input) != 'array') throw 'can only join array, not ' + nameType(input) let a = input.map((x) => { if (typeof x == 'number') return '' + x if (typeof x == 'string') return x if (typeof x == 'boolean') return '' + x if (x === null) return '' throw 'cannot join ' + nameType(x) }) for (let s of args[0].apply(input, conf)) yield a.join(s) }, 'test/1': function* (input, conf, args) { const regex = new RegExp(args[0].value) const isValid = regex.test(input) if (isValid) yield [input] } } // Implements the containment algorithm, returning whether haystack // contains needle: // * Strings are contained if they are substrings // * Arrays if each element is contained in some element of other // * Object if values contained by values in matching key // * All others, if they are equal. // This helper function is necessary because the recursive case // has different error behaviour to the user-exposed function. function containsHelper(haystack, needle) { let haystackType = nameType(haystack) let needleType = nameType(needle) if (haystackType != needleType) { return false } else if (haystackType == 'string') { return haystack.indexOf(needle) != -1 } else if (haystackType == 'array') { for (let b of needle) { let found = false for (let a of haystack) { if (containsHelper(a, b)) { found = true break } } if (!found) return false } return true } else if (haystackType == 'object') { for (let k of Object.keys(needle)) { if (!haystack.hasOwnProperty(k)) return false if (!containsHelper(haystack[k], needle[k])) return false } return true } else { return haystack === needle } } defineShorthandFunction('map', 'f', '[.[] | f]') defineShorthandFunction('map_values', 'f', '.[] |= f') defineShorthandFunction('del', 'p', 'p |= empty') defineShorthandFunction( 'with_entries', 'w', 'to_entries | map(w) | from_entries' ) defineShorthandFunction('arrays', '', 'select(type == "array")') defineShorthandFunction('objects', '', 'select(type == "object")') defineShorthandFunction('booleans', '', 'select(type == "boolean")') defineShorthandFunction('strings', '', 'select(type == "string")') defineShorthandFunction('numbers', '', 'select(type == "number")') defineShorthandFunction('nulls', '', 'select(type == "null")') /** * JSON filter command using the vendored jqjs engine (jq language subset). * Not bit-identical to https://github.com/jqlang/jq — see man jq bareOsNotes. */ async function run(ctx, argv) { const vfs = ctx.vfs let nullInput = false let slurp = false let rawInput = false let compact = false let rawOut = false let exitStatus = false let programPath = null const args = argv.slice(1) let i = 0 function usage() { ctx.console.error( 'usage: jq [-n] [-R] [-s] [-c] [-r] [-e] [-f file] filter [file...]' ) ctx.exitCode = 2 } while (i < args.length) { const a = args[i] if (a === '--') { i++ break } if (a === '-' || !a.startsWith('-')) break if (a === '-n' || a === '--null-input') { nullInput = true i++ continue } if (a === '-R' || a === '--raw-input') { rawInput = true i++ continue } if (a === '-s' || a === '--slurp') { slurp = true i++ continue } if (a === '-c' || a === '--compact-output') { compact = true i++ continue } if (a === '-r' || a === '--raw-output') { rawOut = true i++ continue } if (a === '-e' || a === '--exit-status') { exitStatus = true i++ continue } if (a === '-f' || a === '--from-file') { if (i + 1 >= args.length) { usage() return } programPath = args[i + 1] i += 2 continue } if (a.startsWith('--')) { ctx.console.error('jq: unknown option ' + a) ctx.exitCode = 2 return } const rest = a.slice(1) let consumeNextArg = 0 for (let j = 0; j < rest.length; j++) { const c = rest[j] switch (c) { case 'n': nullInput = true break case 'R': rawInput = true break case 's': slurp = true break case 'c': compact = true break case 'r': rawOut = true break case 'e': exitStatus = true break case 'f': if (j + 1 < rest.length) { programPath = rest.slice(j + 1) j = rest.length } else { if (i + 1 >= args.length) { usage() return } programPath = args[i + 1] consumeNextArg = 1 j = rest.length } break default: ctx.console.error('jq: invalid option -- ' + c) ctx.exitCode = 2 return } } i += 1 + consumeNextArg } let program if (programPath) { const buf = await vfs.readFile(programPath) if (!buf) { ctx.console.error('jq: Could not open ' + programPath) ctx.exitCode = 1 return } program = ctx.b4a.toString(buf).replace(/\r\n/g, '\n') } else { if (i >= args.length) { usage() return } program = args[i++] } const fileArgs = args.slice(i) if (nullInput && rawInput) { ctx.console.error('jq: -n and -R are mutually exclusive') ctx.exitCode = 2 return } let runFn try { runFn = compile(program) } catch (e) { ctx.console.error('jq: compile error: ' + (e && e.message ? e.message : e)) ctx.exitCode = 3 return } /** @type {unknown[]} */ let inputs try { inputs = await bareJqBuildInputs(ctx, vfs, fileArgs, { nullInput, slurp, rawInput }) } catch (e) { ctx.console.error('jq: ' + (e && e.message ? e.message : e)) ctx.exitCode = 1 return } let lastOut let anyOut = false function formatOne(val) { if (typeof val === 'undefined') return null if (rawOut && typeof val === 'string') return val if (compact) return JSON.stringify(val) return prettyPrint(val) } for (const input of inputs) { let iter try { iter = runFn(input) } catch (e) { ctx.console.error('jq: error: ' + (e && e.message ? e.message : e)) ctx.exitCode = 1 return } try { for (const val of iter) { const line = formatOne(val) if (line !== null) { ctx.console.log(line) anyOut = true lastOut = val } } } catch (e) { ctx.console.error('jq: error: ' + (e && e.message ? e.message : e)) ctx.exitCode = 1 return } } if (exitStatus) { if (!anyOut) ctx.exitCode = 4 else if (lastOut === false || lastOut === null) ctx.exitCode = 1 } } function bareJqSkipWs(str, i) { while (i < str.length && /\s/.test(str[i])) i++ return i } function bareJqEndOfJsonValue(str, start) { let i = bareJqSkipWs(str, start) if (i >= str.length) throw new SyntaxError('Unexpected end of JSON input') const c = str[i] if (c === '"') { let j = i + 1 while (j < str.length) { if (str[j] === '\\') { j += 2 continue } if (str[j] === '"') return j + 1 j++ } throw new SyntaxError('Unclosed string in JSON') } if (c === '{' || c === '[') { const stack = [c === '{' ? '}' : ']'] let j = i + 1 let inStr = false let esc = false while (j < str.length) { const ch = str[j] if (inStr) { if (esc) esc = false else if (ch === '\\') esc = true else if (ch === '"') inStr = false } else { if (ch === '"') inStr = true else if (ch === '{') stack.push('}') else if (ch === '[') stack.push(']') else if (ch === '}' || ch === ']') { const want = stack.pop() if (ch !== want) throw new SyntaxError('Mismatched bracket in JSON') if (stack.length === 0) return j + 1 } } j++ } throw new SyntaxError('Unclosed array or object in JSON') } if (c === '-' || (c >= '0' && c <= '9')) { let j = i if (str[j] === '-') { j++ if (j >= str.length) throw new SyntaxError('Invalid number') } if (str[j] === '0') { j++ if (j < str.length && str[j] >= '1' && str[j] <= '9') throw new SyntaxError('Invalid number') } else if (str[j] >= '1' && str[j] <= '9') { while (j < str.length && str[j] >= '0' && str[j] <= '9') j++ } else if (str[j] === '0') { j++ } else { throw new SyntaxError('Invalid number') } if (j < str.length && str[j] === '.') { j++ if (j >= str.length || str[j] < '0' || str[j] > '9') throw new SyntaxError('Invalid number') while (j < str.length && str[j] >= '0' && str[j] <= '9') j++ } if (j < str.length && (str[j] === 'e' || str[j] === 'E')) { j++ if (j < str.length && (str[j] === '+' || str[j] === '-')) j++ if (j >= str.length || str[j] < '0' || str[j] > '9') throw new SyntaxError('Invalid number') while (j < str.length && str[j] >= '0' && str[j] <= '9') j++ } return j } if (str.startsWith('null', i)) return i + 4 if (str.startsWith('true', i)) return i + 4 if (str.startsWith('false', i)) return i + 5 throw new SyntaxError('Unexpected token in JSON at position ' + i) } function bareJqParseJsonValues(text) { const values = [] let i = 0 while (true) { i = bareJqSkipWs(text, i) if (i >= text.length) break const end = bareJqEndOfJsonValue(text, i) values.push(JSON.parse(text.slice(i, end))) i = end } return values } function bareJqRawLines(text) { const lines = text.replace(/\r\n/g, '\n').split('\n') if (lines.length && lines[lines.length - 1] === '') lines.pop() return lines } /** * @param {unknown} ctx * @param {unknown} vfs * @param {string[]} fileArgs * @param {{ nullInput: boolean, slurp: boolean, rawInput: boolean }} opts */ async function bareJqBuildInputs(ctx, vfs, fileArgs, opts) { if (opts.nullInput) return [null] const parts = [] if (fileArgs.length === 0) { const stdin = bareStdin(ctx) parts.push(stdin == null ? '' : String(stdin)) } else { for (const name of fileArgs) { if (name === '-') { parts.push(String(bareStdin(ctx) ?? '')) } else { const buf = await vfs.readFile(name) if (!buf) throw new Error(name + ': No such file or directory') parts.push(ctx.b4a.toString(buf)) } } } if (opts.rawInput) { const lines = [] for (const p of parts) { for (const line of bareJqRawLines(p)) lines.push(line) } if (opts.slurp) return [lines] return lines } const merged = parts.join('') const values = bareJqParseJsonValues(merged) if (opts.slurp) return [values] return values }