325 lines
12 KiB
JavaScript
325 lines
12 KiB
JavaScript
import test from 'brittle'
|
|
import { readFileSync } from 'node:fs'
|
|
import vm from 'node:vm'
|
|
|
|
const QVAC_SRC = readFileSync(new URL('../lib/agent/agent-qvac.js', import.meta.url), 'utf8')
|
|
|
|
/** Eval concat-style preamble helpers into a sandbox (same shape as /bin/agent). */
|
|
function loadAgentQvacHelpers() {
|
|
const sandbox = { console }
|
|
vm.createContext(sandbox)
|
|
vm.runInContext(
|
|
QVAC_SRC +
|
|
'\n;this.__exports = { bareAgentFlattenToolsForQvac, bareAgentSanitizeHistoryForQvac, bareAgentQvacDetectToolDialect, bareAgentQvacGetProfile, bareAgentQvacProfileList, bareAgentResolveBackend, bareAgentQvacBridgeAvailable, bareAgentQvacResolveCtxSize, bareAgentQvacResolveDeviceOpts, bareAgentQvacModelCardCtxSize, bareAgentSanitizeConfigForBackend, bareAgentRestGetProvider, bareAgentRestProviderList, bareAgentIsQvacModelId, bareAgentMaskSecretPreview, BARE_AGENT_QVAC_CTX_QWEN3, BARE_AGENT_QVAC_CTX_LLAMA32_1B }',
|
|
sandbox
|
|
)
|
|
return sandbox.__exports
|
|
}
|
|
|
|
test('full agent tool list flattens to QVAC-safe property schemas', async (t) => {
|
|
const defs = readFileSync(
|
|
new URL('../lib/agent/agent-tool-definitions.js', import.meta.url),
|
|
'utf8'
|
|
)
|
|
const sandbox = { console }
|
|
vm.createContext(sandbox)
|
|
vm.runInContext(
|
|
QVAC_SRC +
|
|
'\n' +
|
|
defs +
|
|
'\n;this.__exports = { bareAgentFlattenToolsForQvac, bareAgentToolDefinitions }',
|
|
sandbox
|
|
)
|
|
const { bareAgentFlattenToolsForQvac, bareAgentToolDefinitions } = sandbox.__exports
|
|
const flat = bareAgentFlattenToolsForQvac(bareAgentToolDefinitions())
|
|
t.ok(flat.length > 20)
|
|
t.ok(flat.some((x) => x.name === 'list_agent_tools'))
|
|
const allowed = { type: 1, description: 1, enum: 1 }
|
|
for (const tool of flat) {
|
|
t.ok(tool.name, 'tool missing name')
|
|
t.is(tool.type, 'function')
|
|
t.ok(tool.parameters && tool.parameters.type === 'object')
|
|
const props = tool.parameters.properties || {}
|
|
for (const key of Object.keys(props)) {
|
|
const prop = props[key]
|
|
for (const pk of Object.keys(prop)) {
|
|
t.ok(allowed[pk], tool.name + '.' + key + ' has forbidden key ' + pk)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
test('live tool catalog lists every defined tool and not skills', async (t) => {
|
|
const defs = readFileSync(
|
|
new URL('../lib/agent/agent-tool-definitions.js', import.meta.url),
|
|
'utf8'
|
|
)
|
|
const skills = readFileSync(
|
|
new URL('../lib/agent/agent-skills.js', import.meta.url),
|
|
'utf8'
|
|
)
|
|
const tui = readFileSync(
|
|
new URL('../lib/agent/agent-tui.js', import.meta.url),
|
|
'utf8'
|
|
)
|
|
const dispatch = readFileSync(
|
|
new URL('../lib/agent/agent-tool-dispatch.js', import.meta.url),
|
|
'utf8'
|
|
)
|
|
const sandbox = { console }
|
|
vm.createContext(sandbox)
|
|
vm.runInContext(
|
|
defs +
|
|
'\n;this.__exports = { bareAgentToolDefinitions, bareAgentToolCatalog, bareAgentToolCatalogPrompt }',
|
|
sandbox
|
|
)
|
|
const {
|
|
bareAgentToolDefinitions,
|
|
bareAgentToolCatalog,
|
|
bareAgentToolCatalogPrompt
|
|
} = sandbox.__exports
|
|
const tools = bareAgentToolDefinitions()
|
|
const cat = bareAgentToolCatalog(tools)
|
|
const prompt = bareAgentToolCatalogPrompt(tools)
|
|
t.ok(cat.length >= 80)
|
|
t.is(cat.length, tools.length)
|
|
const names = cat.map((x) => x.name)
|
|
t.ok(names.includes('read_file'))
|
|
t.ok(names.includes('run_command'))
|
|
t.ok(names.includes('write_file'))
|
|
t.ok(names.includes('list_agent_tools'))
|
|
t.ok(names.includes('discord_send_message'))
|
|
t.ok(names.includes('task_complete'))
|
|
t.absent(names.includes('ctx-api-change'))
|
|
t.absent(names.includes('docs-contract-update'))
|
|
t.absent(names.includes('ctx-baredoctor'))
|
|
for (const n of names) {
|
|
t.ok(prompt.includes(n), 'catalog prompt missing ' + n)
|
|
t.ok(
|
|
dispatch.includes("toolName === '" + n + "'") || n === 'remember',
|
|
'dispatch missing ' + n
|
|
)
|
|
}
|
|
t.ok(skills.includes('NOT callable tools'))
|
|
t.ok(tui.includes('bareAgentToolCatalogPrompt()'))
|
|
t.ok(tui.includes('bareAgentLooksLikeToolInventoryQuestion'))
|
|
})
|
|
|
|
test('flatten OpenAI nested tools for QVAC', async (t) => {
|
|
const { bareAgentFlattenToolsForQvac } = loadAgentQvacHelpers()
|
|
const flat = bareAgentFlattenToolsForQvac([
|
|
{
|
|
type: 'function',
|
|
function: {
|
|
name: 'read_file',
|
|
description: 'Read a file',
|
|
parameters: { type: 'object', properties: { path: { type: 'string' } }, required: ['path'] }
|
|
}
|
|
}
|
|
])
|
|
t.is(flat.length, 1)
|
|
t.is(flat[0].name, 'read_file')
|
|
t.is(flat[0].type, 'function')
|
|
t.ok(flat[0].parameters && flat[0].parameters.properties)
|
|
t.is(flat[0].function, undefined)
|
|
})
|
|
|
|
test('flatten strips nested JSON-schema fields QVAC rejects', async (t) => {
|
|
const { bareAgentFlattenToolsForQvac } = loadAgentQvacHelpers()
|
|
const flat = bareAgentFlattenToolsForQvac([
|
|
{
|
|
type: 'function',
|
|
function: {
|
|
name: 'todo_write',
|
|
description: 'todos',
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
todos: {
|
|
type: 'array',
|
|
description: 'items',
|
|
items: { type: 'object', properties: { id: { type: 'string' } } }
|
|
},
|
|
merge: { type: 'boolean', default: true }
|
|
},
|
|
required: ['todos', 'missing']
|
|
}
|
|
}
|
|
}
|
|
])
|
|
t.is(flat.length, 1)
|
|
const todos = flat[0].parameters.properties.todos
|
|
t.is(todos.type, 'array')
|
|
t.is(todos.description, 'items')
|
|
t.absent(todos.items)
|
|
t.absent(flat[0].parameters.properties.merge.default)
|
|
t.alike(flat[0].parameters.required, ['todos'])
|
|
})
|
|
|
|
test('sanitize QVAC history stringifies tool_calls and never sends null content', async (t) => {
|
|
const { bareAgentSanitizeHistoryForQvac } = loadAgentQvacHelpers()
|
|
const out = bareAgentSanitizeHistoryForQvac([
|
|
{ role: 'system', content: 'sys' },
|
|
{
|
|
role: 'assistant',
|
|
content: null,
|
|
tool_calls: [
|
|
{
|
|
id: 'c1',
|
|
function: { name: 'read_file', arguments: '{"path":"/x"}' }
|
|
}
|
|
]
|
|
},
|
|
{ role: 'tool', tool_call_id: 'c1', content: 'ok' }
|
|
])
|
|
t.is(out.length, 3)
|
|
t.is(out[1].role, 'assistant')
|
|
t.ok(out[1].content.includes('<tool_call>'))
|
|
t.ok(out[1].content.includes('read_file'))
|
|
t.absent('tool_calls' in out[1])
|
|
t.is(out[2].content, 'ok')
|
|
})
|
|
|
|
test('sanitize QVAC history keeps prose and serializes tool_calls', async (t) => {
|
|
const { bareAgentSanitizeHistoryForQvac } = loadAgentQvacHelpers()
|
|
const out = bareAgentSanitizeHistoryForQvac([
|
|
{
|
|
role: 'assistant',
|
|
content: 'Working.',
|
|
tool_calls: [
|
|
{ function: { name: 'read_file', arguments: '{"path":"/x"}' } }
|
|
]
|
|
}
|
|
])
|
|
t.ok(out[0].content.includes('Working.'))
|
|
t.ok(out[0].content.includes('<tool_call>'))
|
|
t.ok(out[0].content.includes('read_file'))
|
|
})
|
|
|
|
test('detect QVAC tool dialect from model id', async (t) => {
|
|
const { bareAgentQvacDetectToolDialect } = loadAgentQvacHelpers()
|
|
t.is(bareAgentQvacDetectToolDialect('QWEN3_1_7B_INST_Q4'), 'hermes')
|
|
t.is(bareAgentQvacDetectToolDialect('QWEN3_5_4B_MULTIMODAL_Q4_K_M'), 'qwen35')
|
|
t.is(bareAgentQvacDetectToolDialect('GEMMA4_2B_MULTIMODAL_Q4_K_M'), 'gemma4')
|
|
t.is(bareAgentQvacDetectToolDialect('GPT_OSS_20B_INST_Q4_K_M'), 'harmony')
|
|
t.is(bareAgentQvacDetectToolDialect('LFM2_1_2B_Q4_K_M'), 'pythonic')
|
|
t.is(bareAgentQvacDetectToolDialect('DEEPSEEK_V3_2_CHAT'), 'dsml')
|
|
})
|
|
|
|
test('qvac profiles include recommended + lite', async (t) => {
|
|
const { bareAgentQvacProfileList, bareAgentQvacGetProfile } = loadAgentQvacHelpers()
|
|
const list = bareAgentQvacProfileList()
|
|
t.ok(list.length >= 4)
|
|
t.is(bareAgentQvacGetProfile('recommended').chatModel, 'QWEN3_1_7B_INST_Q4')
|
|
t.is(bareAgentQvacGetProfile('recommended').ctxSize, 32768)
|
|
t.is(bareAgentQvacGetProfile('lite').id, 'lite')
|
|
t.is(bareAgentQvacGetProfile('lite').ctxSize, 32768)
|
|
t.is(bareAgentQvacGetProfile('tool-tiny').ctxSize, 131072)
|
|
t.is(bareAgentQvacGetProfile('strong').ctxSize, 32768)
|
|
t.is(bareAgentQvacGetProfile('nope').id, 'recommended')
|
|
})
|
|
|
|
test('model-card ctx sizes match upstream cards', async (t) => {
|
|
const { bareAgentQvacModelCardCtxSize } = loadAgentQvacHelpers()
|
|
t.is(bareAgentQvacModelCardCtxSize('QWEN3_600M_INST_Q4'), 32768)
|
|
t.is(bareAgentQvacModelCardCtxSize('QWEN3_1_7B_INST_Q4'), 32768)
|
|
t.is(bareAgentQvacModelCardCtxSize('QWEN3_4B_INST_Q4_K_M'), 32768)
|
|
t.is(bareAgentQvacModelCardCtxSize('LLAMA_TOOL_CALLING_1B_INST_Q4_K'), 131072)
|
|
})
|
|
|
|
test('resolve qvac ctx size from profile or config override', async (t) => {
|
|
const { bareAgentQvacGetProfile, bareAgentQvacResolveCtxSize } = loadAgentQvacHelpers()
|
|
const rec = bareAgentQvacGetProfile('recommended')
|
|
const lite = bareAgentQvacGetProfile('lite')
|
|
const tiny = bareAgentQvacGetProfile('tool-tiny')
|
|
t.is(bareAgentQvacResolveCtxSize({}, rec), 32768)
|
|
// Legacy undersized overrides are ignored (auto model-card).
|
|
t.is(bareAgentQvacResolveCtxSize({ qvac_ctx_size: 4096 }, rec), 32768)
|
|
t.is(bareAgentQvacResolveCtxSize({ qvac_ctx_size: 8192 }, lite), 32768)
|
|
t.is(bareAgentQvacResolveCtxSize({ qvac_ctx_size: 65536 }, rec), 65536)
|
|
t.is(bareAgentQvacResolveCtxSize({ qvac_ctx_size: 999999 }, rec), 131072)
|
|
t.is(bareAgentQvacResolveCtxSize({}, lite), 32768)
|
|
t.is(bareAgentQvacResolveCtxSize({}, tiny), 131072)
|
|
})
|
|
|
|
test('resolve qvac device opts', async (t) => {
|
|
const { bareAgentQvacResolveDeviceOpts } = loadAgentQvacHelpers()
|
|
const d = bareAgentQvacResolveDeviceOpts({})
|
|
t.is(d.mainGpu, 'auto')
|
|
t.absent(d.device)
|
|
t.is(bareAgentQvacResolveDeviceOpts({ qvac_device: 'cpu' }).device, 'cpu')
|
|
t.is(bareAgentQvacResolveDeviceOpts({ qvac_main_gpu: '1' }).mainGpu, 1)
|
|
t.is(bareAgentQvacResolveDeviceOpts({ qvac_gpu_layers: 0 }).gpuLayers, 0)
|
|
})
|
|
|
|
test('resolve backend defaults to qvac', async (t) => {
|
|
const { bareAgentResolveBackend } = loadAgentQvacHelpers()
|
|
t.is(bareAgentResolveBackend({}), 'qvac')
|
|
t.is(bareAgentResolveBackend({ backend: 'qvac' }), 'qvac')
|
|
t.is(bareAgentResolveBackend({ backend: 'rest' }), 'rest')
|
|
t.is(bareAgentResolveBackend({ provider: 'groq', rest_api_key: 'x' }), 'rest')
|
|
t.is(bareAgentResolveBackend({ provider: 'openai' }), 'rest')
|
|
t.is(bareAgentResolveBackend({ provider: 'custom' }), 'rest')
|
|
t.is(bareAgentResolveBackend({ provider: 'qvac' }), 'qvac')
|
|
})
|
|
|
|
test('sanitize drops the other backend keys', async (t) => {
|
|
const { bareAgentSanitizeConfigForBackend } = loadAgentQvacHelpers()
|
|
const rest = bareAgentSanitizeConfigForBackend({
|
|
backend: 'rest',
|
|
provider: 'qvac',
|
|
model: 'QWEN3_1_7B_INST_Q4',
|
|
qvac_model: 'QWEN3_1_7B_INST_Q4',
|
|
qvac_profile: 'recommended',
|
|
qvac_ctx_size: 0,
|
|
rest_api_key: 'gsk_test',
|
|
rest_base_url: 'https://api.groq.com/openai/v1'
|
|
})
|
|
t.is(rest.backend, 'rest')
|
|
t.is(rest.provider, 'groq')
|
|
t.is(rest.model, 'llama-3.3-70b-versatile')
|
|
t.absent('qvac_model' in rest)
|
|
t.absent('qvac_profile' in rest)
|
|
t.ok(rest.rest_api_key)
|
|
|
|
const qvac = bareAgentSanitizeConfigForBackend({
|
|
backend: 'qvac',
|
|
provider: 'groq',
|
|
rest_api_key: 'secret',
|
|
rest_base_url: 'https://api.groq.com/openai/v1',
|
|
qvac_profile: 'strong',
|
|
qvac_model: 'QWEN3_4B_INST_Q4_K_M'
|
|
})
|
|
t.is(qvac.backend, 'qvac')
|
|
t.is(qvac.provider, 'qvac')
|
|
t.absent('rest_api_key' in qvac)
|
|
t.absent('rest_base_url' in qvac)
|
|
t.is(qvac.qvac_profile, 'strong')
|
|
})
|
|
|
|
test('REST provider catalog has groq xai openai custom', async (t) => {
|
|
const { bareAgentRestProviderList, bareAgentRestGetProvider, bareAgentMaskSecretPreview } =
|
|
loadAgentQvacHelpers()
|
|
const list = bareAgentRestProviderList()
|
|
t.ok(list.some((p) => p.id === 'groq'))
|
|
t.ok(list.some((p) => p.id === 'xai'))
|
|
t.ok(list.some((p) => p.id === 'openai'))
|
|
t.ok(list.some((p) => p.id === 'custom'))
|
|
t.is(bareAgentRestGetProvider('xai').rest_base_url, 'https://api.x.ai/v1')
|
|
t.is(bareAgentMaskSecretPreview('gsk_abcdefghijk'), 'gsk…hijk')
|
|
t.is(bareAgentMaskSecretPreview(''), '(not set)')
|
|
})
|
|
|
|
test('qvac bridge available checks ctx hooks', async (t) => {
|
|
const { bareAgentQvacBridgeAvailable } = loadAgentQvacHelpers()
|
|
t.is(bareAgentQvacBridgeAvailable({}), false)
|
|
t.is(
|
|
bareAgentQvacBridgeAvailable({
|
|
bareOsQvacAvailable: () => true,
|
|
bareOsQvacComplete: async () => {}
|
|
}),
|
|
true
|
|
)
|
|
t.is(bareAgentQvacBridgeAvailable({ bareOsQvacAvailable: () => false }), false)
|
|
})
|