Add jq

2026-04-03 05:55:25 -04:00
parent 6259c25d4c
commit 0dd01be4ab
15 changed files with 7165 additions and 7 deletions
@@ -62,7 +62,7 @@ function barePosixBlocks(size) {

 async function run(ctx, argv) {
  ctx.console.log(
-    'Bare OS — default user: guest | shell builtins: alias, cd, command, export, exit, login, logout, readonly, type, umask, unalias, unset, : | /bin: awk basename cat chgrp chmod chown cksum clear cp crontab cut date dirname du echo env exit false find getconf grep head hdms help hostname id ln login logout logname ls man mkdir mkfifo mv nl od pathchk printenv printf pwd readlink rm rmdir savevault sed seq sleep sort stat tail tee test time touch tr true tty uname wc which whoami xargs'
+    'Bare OS — default user: guest | shell builtins: alias, cd, command, export, exit, login, logout, readonly, type, umask, unalias, unset, : | /bin: awk basename cat chgrp chmod chown cksum clear cp crontab cut date dirname du echo env exit false find getconf grep head hdms help hostname id jq ln login logout logname ls man mkdir mkfifo mv nl od pathchk printenv printf pwd readlink rm rmdir savevault sed seq sleep sort stat tail tee test time touch tr true tty uname wc which whoami xargs'
  )
  ctx.console.log(
    'Docs: man <command> | man handbook (full handbook, section 7) | man bare-os-shell | man -k <word> | man -l'
@@ -685,6 +685,45 @@ test('tier-1 grep from system drive', async (t) => {
  rmSync(dir, { recursive: true, force: true })
 })

+test('tier-1 jq from system drive', async (t) => {
+  const dir = testCorestoreDir('jq')
+  const store = new Corestore(dir)
+  const drive = new Hyperdrive(store)
+  const personal = new Hyperdrive(store.namespace('pjq'))
+  await drive.ready()
+  await personal.ready()
+  await drive.put('/bin/jq', b4a.from(await readBuiltBin('jq')))
+  const lines = []
+  const ctx = testCtx(drive, personal)
+  ctx.exitCode = 0
+  ctx.console = {
+    log(s) {
+      lines.push(String(s))
+    },
+    error() {}
+  }
+  await ctx.vfs.writeFile('data.json', b4a.from('{"x":42,"name":"hi"}'))
+  await runBinCommand(ctx, ['jq', '.x', 'data.json'])
+  t.is(ctx.exitCode, 0)
+  t.is(lines[0], '42')
+
+  lines.length = 0
+  ctx.exitCode = 0
+  await runBinCommand(ctx, ['jq', '-c', '.', 'data.json'])
+  t.is(ctx.exitCode, 0)
+  t.is(lines[0], '{"x":42,"name":"hi"}')
+
+  lines.length = 0
+  ctx.exitCode = 0
+  ctx.shellStdin = '{"a":1}'
+  await runBinCommand(ctx, ['jq', '.a'])
+  t.is(ctx.exitCode, 0)
+  t.is(lines[0], '1')
+
+  await store.close()
+  rmSync(dir, { recursive: true, force: true })
+})
+
 test('tier-1 rm -rf removes directory tree on personal drive', async (t) => {
  const dir = testCorestoreDir('rmrf')
  const store = new Corestore(dir)
@@ -14,6 +14,7 @@ const seederKernelBin = join(repoRoot, 'packages/bare-os-seeder/kernel/bin')
 const preamble = {
  sed: ['sed-engine.js'],
  awk: ['awk-engine.js'],
+  jq: ['jq-engine.js'],
  man: ['man-render.js']
 }

@@ -29,6 +29,7 @@ export const COREUTILS_COMMANDS = [
  'help',
  'hostname',
  'id',
+  'jq',
  'ln',
  'login',
  'logout',
@@ -0,0 +1,76 @@
+{
+  "name": "jq",
+  "section": 1,
+  "title": "command-line JSON processor (jq language subset)",
+  "synopsis": [
+    "jq [-n] [-R] [-s] [-c] [-r] [-e] [-f file] filter [file...]",
+    "jq reads JSON (concatenated values or NDJSON-style streams) from files or stdin."
+  ],
+  "description": "Runs a jq filter program against JSON values. The engine is vendored jqjs (pure JavaScript), not the C implementation at https://github.com/jqlang/jq — language coverage and edge cases differ.",
+  "options": [
+    {
+      "flag": "-n, --null-input",
+      "meaning": "Use null as the sole input (ignore file/stdin for input)"
+    },
+    {
+      "flag": "-R, --raw-input",
+      "meaning": "Treat each line as a string instead of JSON"
+    },
+    {
+      "flag": "-s, --slurp",
+      "meaning": "Read all inputs into one array; run the filter once"
+    },
+    {
+      "flag": "-c, --compact-output",
+      "meaning": "Compact JSON on output"
+    },
+    {
+      "flag": "-r, --raw-output",
+      "meaning": "Print strings without JSON quotes"
+    },
+    {
+      "flag": "-e, --exit-status",
+      "meaning": "Set exit status from outputs (no output → 4; last false/null → 1)"
+    },
+    {
+      "flag": "-f, --from-file",
+      "meaning": "Read filter program from file"
+    }
+  ],
+  "keywords": [
+    "jq",
+    "json",
+    "query",
+    "filter",
+    "jqjs"
+  ],
+  "seeAlso": [
+    {
+      "name": "grep",
+      "section": 1
+    },
+    {
+      "name": "awk",
+      "section": 1
+    }
+  ],
+  "bareOsNotes": "Engine: lib/jq-engine.js from @sscots/jqjs (mwh/jqjs). Missing vs C jq: try/catch, user-defined functions, recurse, many builtins, modules, full Unicode. See upstream jqjs README for the feature matrix.",
+  "examples": [
+    {
+      "caption": "pretty-print",
+      "code": "jq . data.json"
+    },
+    {
+      "caption": "field",
+      "code": "jq .version package.json"
+    },
+    {
+      "caption": "slurp array",
+      "code": "jq -s 'map(.x) | add' parts.jsonl"
+    },
+    {
+      "caption": "compact",
+      "code": "jq -c '.[] | select(.ok)' items.json"
+    }
+  ]
+}
@@ -6,7 +6,7 @@
    "man [-k keyword] [-f name] [-l] [[section] name]",
    "man reads /share/man/man.json on the system drive."
  ],
-  "description": "Displays manual pages from the merged JSON database. Section 1 only in this release.",
+  "description": "Displays manual pages from the merged JSON database. Section 1: /bin and git/shell pages. Section 7: handbook (man handbook) and developer guide (man devguide), merged at build from handbook/*.md and developer-guide/*.md.",
  "options": [
    {
      "flag": "-k, --apropos",
@@ -18,7 +18,7 @@
    },
    {
      "flag": "-l, --list",
-      "meaning": "List all manual page names"
+      "meaning": "List pages grouped by category (/bin, git/shell, handbook, developer guide), then alphabetically"
    }
  ],
  "keywords": [
@@ -39,6 +39,14 @@
    {
      "name": "help",
      "section": 1
+    },
+    {
+      "name": "bare-os-handbook",
+      "section": 7
+    },
+    {
+      "name": "bare-os-developer-guide",
+      "section": 7
    }
  ],
  "bareOsNotes": "No troff; no embedded DB fallback in v1.",
@@ -43,6 +43,7 @@ const POSIX_TITLE = {
  help: 'Bare OS help summary',
  hostname: 'set or print hostname',
  id: 'return user identity',
+  jq: 'command-line JSON processor (jq language subset)',
  ln: 'link files',
  login: 'begin a session on the system',
  logout: 'end session (save vault)',
@@ -496,6 +497,36 @@ EXTRA.find = {
  keywords: ['find', 'directory', 'walk', 'search'],
  bareOsNotes: 'Expression syntax is a simplified subset.'
 }
+EXTRA.jq = {
+  synopsis: [
+    'jq [-n] [-R] [-s] [-c] [-r] [-e] [-f file] filter [file...]',
+    'jq reads JSON (concatenated values or NDJSON-style streams) from files or stdin.'
+  ],
+  description:
+    'Runs a jq filter program against JSON values. The engine is vendored jqjs (pure JavaScript), not the C implementation at https://github.com/jqlang/jq — language coverage and edge cases differ.',
+  options: [
+    { flag: '-n, --null-input', meaning: 'Use null as the sole input (ignore file/stdin for input)' },
+    { flag: '-R, --raw-input', meaning: 'Treat each line as a string instead of JSON' },
+    { flag: '-s, --slurp', meaning: 'Read all inputs into one array; run the filter once' },
+    { flag: '-c, --compact-output', meaning: 'Compact JSON on output' },
+    { flag: '-r, --raw-output', meaning: 'Print strings without JSON quotes' },
+    { flag: '-e, --exit-status', meaning: 'Set exit status from outputs (no output → 4; last false/null → 1)' },
+    { flag: '-f, --from-file', meaning: 'Read filter program from file' }
+  ],
+  keywords: ['jq', 'json', 'query', 'filter', 'jqjs'],
+  seeAlso: [
+    { name: 'grep', section: 1 },
+    { name: 'awk', section: 1 }
+  ],
+  bareOsNotes:
+    'Engine: lib/jq-engine.js from @sscots/jqjs (mwh/jqjs). Missing vs C jq: try/catch, user-defined functions, recurse, many builtins, modules, full Unicode. See upstream jqjs README for the feature matrix.',
+  examples: [
+    { caption: 'pretty-print', code: 'jq . data.json' },
+    { caption: 'field', code: 'jq .version package.json' },
+    { caption: 'slurp array', code: 'jq -s \'map(.x) | add\' parts.jsonl' },
+    { caption: 'compact', code: 'jq -c \'.[] | select(.ok)\' items.json' }
+  ]
+}
 EXTRA.login = {
  description:
    'When invoked from /bin, behavior aligns with session identity hooks (see booter). Prefer the shell builtin for passphrase entry.',
@@ -1,6 +1,6 @@
 async function run(ctx, argv) {
  ctx.console.log(
-    'Bare OS — default user: guest | shell builtins: alias, cd, command, export, exit, login, logout, readonly, type, umask, unalias, unset, : | /bin: awk basename cat chgrp chmod chown cksum clear cp crontab cut date dirname du echo env exit false find getconf grep head hdms help hostname id ln login logout logname ls man mkdir mkfifo mv nl od pathchk printenv printf pwd readlink rm rmdir savevault sed seq sleep sort stat tail tee test time touch tr true tty uname wc which whoami xargs'
+    'Bare OS — default user: guest | shell builtins: alias, cd, command, export, exit, login, logout, readonly, type, umask, unalias, unset, : | /bin: awk basename cat chgrp chmod chown cksum clear cp crontab cut date dirname du echo env exit false find getconf grep head hdms help hostname id jq ln login logout logname ls man mkdir mkfifo mv nl od pathchk printenv printf pwd readlink rm rmdir savevault sed seq sleep sort stat tail tee test time touch tr true tty uname wc which whoami xargs'
  )
  ctx.console.log(
    'Docs: man <command> | man handbook (full handbook, section 7) | man bare-os-shell | man -k <word> | man -l'
@@ -0,0 +1,355 @@
+/**
+ * JSON filter command using the vendored jqjs engine (jq language subset).
+ * Not bit-identical to https://github.com/jqlang/jq — see man jq bareOsNotes.
+ */
+async function run(ctx, argv) {
+  const vfs = ctx.vfs
+
+  let nullInput = false
+  let slurp = false
+  let rawInput = false
+  let compact = false
+  let rawOut = false
+  let exitStatus = false
+  let programPath = null
+
+  const args = argv.slice(1)
+  let i = 0
+
+  function usage() {
+    ctx.console.error(
+      'usage: jq [-n] [-R] [-s] [-c] [-r] [-e] [-f file] filter [file...]'
+    )
+    ctx.exitCode = 2
+  }
+
+  while (i < args.length) {
+    const a = args[i]
+    if (a === '--') {
+      i++
+      break
+    }
+    if (a === '-' || !a.startsWith('-')) break
+
+    if (a === '-n' || a === '--null-input') {
+      nullInput = true
+      i++
+      continue
+    }
+    if (a === '-R' || a === '--raw-input') {
+      rawInput = true
+      i++
+      continue
+    }
+    if (a === '-s' || a === '--slurp') {
+      slurp = true
+      i++
+      continue
+    }
+    if (a === '-c' || a === '--compact-output') {
+      compact = true
+      i++
+      continue
+    }
+    if (a === '-r' || a === '--raw-output') {
+      rawOut = true
+      i++
+      continue
+    }
+    if (a === '-e' || a === '--exit-status') {
+      exitStatus = true
+      i++
+      continue
+    }
+    if (a === '-f' || a === '--from-file') {
+      if (i + 1 >= args.length) {
+        usage()
+        return
+      }
+      programPath = args[i + 1]
+      i += 2
+      continue
+    }
+    if (a.startsWith('--')) {
+      ctx.console.error('jq: unknown option ' + a)
+      ctx.exitCode = 2
+      return
+    }
+
+    const rest = a.slice(1)
+    let consumeNextArg = 0
+    for (let j = 0; j < rest.length; j++) {
+      const c = rest[j]
+      switch (c) {
+        case 'n':
+          nullInput = true
+          break
+        case 'R':
+          rawInput = true
+          break
+        case 's':
+          slurp = true
+          break
+        case 'c':
+          compact = true
+          break
+        case 'r':
+          rawOut = true
+          break
+        case 'e':
+          exitStatus = true
+          break
+        case 'f':
+          if (j + 1 < rest.length) {
+            programPath = rest.slice(j + 1)
+            j = rest.length
+          } else {
+            if (i + 1 >= args.length) {
+              usage()
+              return
+            }
+            programPath = args[i + 1]
+            consumeNextArg = 1
+            j = rest.length
+          }
+          break
+        default:
+          ctx.console.error('jq: invalid option -- ' + c)
+          ctx.exitCode = 2
+          return
+      }
+    }
+    i += 1 + consumeNextArg
+  }
+
+  let program
+  if (programPath) {
+    const buf = await vfs.readFile(programPath)
+    if (!buf) {
+      ctx.console.error('jq: Could not open ' + programPath)
+      ctx.exitCode = 1
+      return
+    }
+    program = ctx.b4a.toString(buf).replace(/\r\n/g, '\n')
+  } else {
+    if (i >= args.length) {
+      usage()
+      return
+    }
+    program = args[i++]
+  }
+
+  const fileArgs = args.slice(i)
+
+  if (nullInput && rawInput) {
+    ctx.console.error('jq: -n and -R are mutually exclusive')
+    ctx.exitCode = 2
+    return
+  }
+
+  let runFn
+  try {
+    runFn = compile(program)
+  } catch (e) {
+    ctx.console.error('jq: compile error: ' + (e && e.message ? e.message : e))
+    ctx.exitCode = 3
+    return
+  }
+
+  /** @type {unknown[]} */
+  let inputs
+  try {
+    inputs = await bareJqBuildInputs(ctx, vfs, fileArgs, {
+      nullInput,
+      slurp,
+      rawInput
+    })
+  } catch (e) {
+    ctx.console.error('jq: ' + (e && e.message ? e.message : e))
+    ctx.exitCode = 1
+    return
+  }
+
+  let lastOut
+  let anyOut = false
+
+  function formatOne(val) {
+    if (typeof val === 'undefined') return null
+    if (rawOut && typeof val === 'string') return val
+    if (compact) return JSON.stringify(val)
+    return prettyPrint(val)
+  }
+
+  for (const input of inputs) {
+    let iter
+    try {
+      iter = runFn(input)
+    } catch (e) {
+      ctx.console.error('jq: error: ' + (e && e.message ? e.message : e))
+      ctx.exitCode = 1
+      return
+    }
+    try {
+      for (const val of iter) {
+        const line = formatOne(val)
+        if (line !== null) {
+          ctx.console.log(line)
+          anyOut = true
+          lastOut = val
+        }
+      }
+    } catch (e) {
+      ctx.console.error('jq: error: ' + (e && e.message ? e.message : e))
+      ctx.exitCode = 1
+      return
+    }
+  }
+
+  if (exitStatus) {
+    if (!anyOut) ctx.exitCode = 4
+    else if (lastOut === false || lastOut === null) ctx.exitCode = 1
+  }
+}
+
+function bareJqSkipWs(str, i) {
+  while (i < str.length && /\s/.test(str[i])) i++
+  return i
+}
+
+function bareJqEndOfJsonValue(str, start) {
+  let i = bareJqSkipWs(str, start)
+  if (i >= str.length) throw new SyntaxError('Unexpected end of JSON input')
+  const c = str[i]
+  if (c === '"') {
+    let j = i + 1
+    while (j < str.length) {
+      if (str[j] === '\\') {
+        j += 2
+        continue
+      }
+      if (str[j] === '"') return j + 1
+      j++
+    }
+    throw new SyntaxError('Unclosed string in JSON')
+  }
+  if (c === '{' || c === '[') {
+    const stack = [c === '{' ? '}' : ']']
+    let j = i + 1
+    let inStr = false
+    let esc = false
+    while (j < str.length) {
+      const ch = str[j]
+      if (inStr) {
+        if (esc) esc = false
+        else if (ch === '\\') esc = true
+        else if (ch === '"') inStr = false
+      } else {
+        if (ch === '"') inStr = true
+        else if (ch === '{') stack.push('}')
+        else if (ch === '[') stack.push(']')
+        else if (ch === '}' || ch === ']') {
+          const want = stack.pop()
+          if (ch !== want) throw new SyntaxError('Mismatched bracket in JSON')
+          if (stack.length === 0) return j + 1
+        }
+      }
+      j++
+    }
+    throw new SyntaxError('Unclosed array or object in JSON')
+  }
+  if (c === '-' || (c >= '0' && c <= '9')) {
+    let j = i
+    if (str[j] === '-') {
+      j++
+      if (j >= str.length) throw new SyntaxError('Invalid number')
+    }
+    if (str[j] === '0') {
+      j++
+      if (j < str.length && str[j] >= '1' && str[j] <= '9')
+        throw new SyntaxError('Invalid number')
+    } else if (str[j] >= '1' && str[j] <= '9') {
+      while (j < str.length && str[j] >= '0' && str[j] <= '9') j++
+    } else if (str[j] === '0') {
+      j++
+    } else {
+      throw new SyntaxError('Invalid number')
+    }
+    if (j < str.length && str[j] === '.') {
+      j++
+      if (j >= str.length || str[j] < '0' || str[j] > '9')
+        throw new SyntaxError('Invalid number')
+      while (j < str.length && str[j] >= '0' && str[j] <= '9') j++
+    }
+    if (j < str.length && (str[j] === 'e' || str[j] === 'E')) {
+      j++
+      if (j < str.length && (str[j] === '+' || str[j] === '-')) j++
+      if (j >= str.length || str[j] < '0' || str[j] > '9')
+        throw new SyntaxError('Invalid number')
+      while (j < str.length && str[j] >= '0' && str[j] <= '9') j++
+    }
+    return j
+  }
+  if (str.startsWith('null', i)) return i + 4
+  if (str.startsWith('true', i)) return i + 4
+  if (str.startsWith('false', i)) return i + 5
+  throw new SyntaxError('Unexpected token in JSON at position ' + i)
+}
+
+function bareJqParseJsonValues(text) {
+  const values = []
+  let i = 0
+  while (true) {
+    i = bareJqSkipWs(text, i)
+    if (i >= text.length) break
+    const end = bareJqEndOfJsonValue(text, i)
+    values.push(JSON.parse(text.slice(i, end)))
+    i = end
+  }
+  return values
+}
+
+function bareJqRawLines(text) {
+  const lines = text.replace(/\r\n/g, '\n').split('\n')
+  if (lines.length && lines[lines.length - 1] === '') lines.pop()
+  return lines
+}
+
+/**
+ * @param {unknown} ctx
+ * @param {unknown} vfs
+ * @param {string[]} fileArgs
+ * @param {{ nullInput: boolean, slurp: boolean, rawInput: boolean }} opts
+ */
+async function bareJqBuildInputs(ctx, vfs, fileArgs, opts) {
+  if (opts.nullInput) return [null]
+
+  const parts = []
+  if (fileArgs.length === 0) {
+    const stdin = bareStdin(ctx)
+    parts.push(stdin == null ? '' : String(stdin))
+  } else {
+    for (const name of fileArgs) {
+      if (name === '-') {
+        parts.push(String(bareStdin(ctx) ?? ''))
+      } else {
+        const buf = await vfs.readFile(name)
+        if (!buf) throw new Error(name + ': No such file or directory')
+        parts.push(ctx.b4a.toString(buf))
+      }
+    }
+  }
+
+  if (opts.rawInput) {
+    const lines = []
+    for (const p of parts) {
+      for (const line of bareJqRawLines(p)) lines.push(line)
+    }
+    if (opts.slurp) return [lines]
+    return lines
+  }
+
+  const merged = parts.join('')
+  const values = bareJqParseJsonValues(merged)
+  if (opts.slurp) return [values]
+  return values
+}
@@ -62,7 +62,7 @@ function barePosixBlocks(size) {

 async function run(ctx, argv) {
  ctx.console.log(
-    'Bare OS — default user: guest | shell builtins: alias, cd, command, export, exit, login, logout, readonly, type, umask, unalias, unset, : | /bin: awk basename cat chgrp chmod chown cksum clear cp crontab cut date dirname du echo env exit false find getconf grep head hdms help hostname id ln login logout logname ls man mkdir mkfifo mv nl od pathchk printenv printf pwd readlink rm rmdir savevault sed seq sleep sort stat tail tee test time touch tr true tty uname wc which whoami xargs'
+    'Bare OS — default user: guest | shell builtins: alias, cd, command, export, exit, login, logout, readonly, type, umask, unalias, unset, : | /bin: awk basename cat chgrp chmod chown cksum clear cp crontab cut date dirname du echo env exit false find getconf grep head hdms help hostname id jq ln login logout logname ls man mkdir mkfifo mv nl od pathchk printenv printf pwd readlink rm rmdir savevault sed seq sleep sort stat tail tee test time touch tr true tty uname wc which whoami xargs'
  )
  ctx.console.log(
    'Docs: man <command> | man handbook (full handbook, section 7) | man bare-os-shell | man -k <word> | man -l'