Agent Harness Updates
Release rolling / release (push) Successful in 10m1s

This commit is contained in:
2026-08-19 14:04:03 -04:00
parent 05852180d1
commit d047d26f0a
10 changed files with 606 additions and 42 deletions
@@ -222,6 +222,8 @@ test('agent-tui embeds operating contract appendix', async (t) => {
t.ok(TUI.includes('YOU RUN THE TOOLS'))
t.ok(TUI.includes('YOU CAN EXECUTE'))
t.ok(TUI.includes('There is no bash, shell, cli, or terminal tool'))
t.ok(TUI.includes('never wrap them in run_js_script'))
t.ok(TUI.includes('Do not call task_complete after a failed tool'))
t.ok(TUI.includes('The user never runs your tools'))
t.ok(TUI.includes('discord_send_message'))
t.ok(TUI.includes('DISCORD CHANNEL'))
@@ -29,6 +29,40 @@ test('mutate paths use a denylist (base system read-only, rest writable)', async
t.absent(s.bareAgentPathAllowedMutate('/'))
})
test('js script wrapping curl/fetch diverts to a shell command', async (t) => {
const s = load()
const fromJs = s.bareAgentShellCommandFromJsScript
t.is(
fromJs("await ctx.execLine('curl http://api.ipify.org')"),
'curl http://api.ipify.org'
)
t.is(
fromJs("const r = await fetch('https://api.ipify.org')"),
'curl -s https://api.ipify.org'
)
t.is(fromJs('curl http://api.ipify.org'), 'curl http://api.ipify.org')
t.is(fromJs('const vfs = ctx.vfs\nawait vfs.readFile("/tmp/x")'), '')
})
test('failed task_complete detector matches screenshot surrender', async (t) => {
const s = load()
const fn = s.bareAgentLooksLikeFailedTaskComplete
t.ok(
fn(
'Done: Attempted to retrieve IP address using JavaScript. Error occurred due to async/await in top-level scope. Alternative method required.'
)
)
t.absent(fn('Fetched public IP via curl; stdout is 1.2.3.4'))
})
test('guest CLI request detector matches run curl for IP', async (t) => {
const s = load()
const fn = s.bareAgentLooksLikeGuestCliRequest
t.ok(fn('run curl to find our IP Address'))
t.ok(fn('get our public ip'))
t.absent(fn('refactor the agent prompt'))
})
test('normalize guest script strips ctx redeclare and export', async (t) => {
const s = load()
const n = s.bareAgentNormalizeGuestScript
@@ -177,3 +177,23 @@ test('extract ignores javascript fences that are not shell', () => {
const out = bareAgentExtractToolCallsFromText(text)
assert.equal(out.length, 0)
})
test('extract recovers curl glued to following prose', () => {
const { bareAgentExtractToolCallsFromText } = loadToolCallFns()
const text =
'For example:\ncurl http://api.ipify.orgBut the assistant needs to make sure that the script is correct.'
const out = bareAgentExtractToolCallsFromText(text)
assert.ok(out.length >= 1)
assert.equal(out[0].function.name, 'run_command')
assert.equal(JSON.parse(out[0].function.arguments).command, 'curl http://api.ipify.org')
})
test('third-person planning dump is treated as instruction dump', () => {
const { bareAgentLooksLikeInstructionDump } = loadToolCallFns()
assert.equal(
bareAgentLooksLikeInstructionDump(
'But the assistant is supposed to use the run_js_script tool. So the assistant would generate a script that uses curl.'
),
true
)
})
@@ -3,6 +3,8 @@ import { readFileSync } from 'node:fs'
import vm from 'node:vm'
const CODE =
readFileSync(new URL('../lib/agent/agent-helpers.js', import.meta.url), 'utf8') +
'\n' +
readFileSync(new URL('../lib/agent/agent-tools.js', import.meta.url), 'utf8') +
'\n' +
readFileSync(
@@ -168,3 +170,41 @@ test('run_command accepts cmd alias field', async (t) => {
t.ok(seen.length >= 1)
t.ok(seen[0].startsWith('echo hi > '))
})
test('run_js_script wrapping curl diverts to run_command', async (t) => {
const { ctx, seen } = makeCtx()
const out = await callNamed(ctx, 'run_js_script', {
code: "await ctx.execLine('curl http://api.ipify.org')"
})
t.ok(out.ok)
t.ok(seen.length >= 1)
t.ok(String(seen[0]).includes('curl http://api.ipify.org'))
})
test('task_complete after a failed attempt is rejected', async (t) => {
const { ctx } = makeCtx()
let completed = false
const s = load()
const dispatch = /** @type {(o: object) => Promise<string>} */ (s.bareAgentDispatchTool)
const raw = await dispatch({
ctx,
paths: { cmdOut: '/tmp/agent.out', dir: '/tmp' },
toolName: 'task_complete',
argsJson: JSON.stringify({
summary:
'Done: Attempted to retrieve IP address using JavaScript. Error occurred due to async/await in top-level scope. Alternative method required.'
}),
configRef: { current: {} },
appendProgress: () => {},
signal: null,
onTaskComplete: () => {
completed = true
},
bareWebRunTool: async () => ({ ok: false }),
bareWebFmtErr: (e) => String(e)
})
const out = JSON.parse(raw)
t.absent(out.ok)
t.is(out.error, 'task_not_complete')
t.absent(completed)
})