snxraven/gnome-jarvis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity
Files
a097adf4ebd08439318d9226cf670ee61670c6dc
gnome-jarvis/vendor/agent-harness/test/test.js
T
snxraven a097adf4eb
Rolling release / release (push) Successful in 8m31s
Details
Updates
2026-09-13 22:24:14 -04:00

481 lines
19 KiB
JavaScript
Raw Blame History

'use strict';
const assert = require('assert');
const path = require('path');
const os = require('os');
const fs = require('fs');
const catalog = require('../lib/catalog.js');
const compaction = require('../agent/compaction.js');
const sr = require('../agent/search-replace.js');
const toolSet = require('../agent/tool-set.js');
const prompts = require('../agent/prompts.js');
const net = require('../lib/net.js');
const customTools = require('../agent/custom-tools.js');
const planMode = require('../agent/plan-mode.js');
const todos = require('../agent/todos.js');
const stationarity = require('../agent/stationarity.js');
const truncate = require('../agent/truncate.js');
const permRules = require('../agent/perm-rules.js');
const policy = require('../agent/policy.js');
const toolBudget = require('../agent/tool-budget.js');
const paths = require('../lib/paths.js');
const device = require('../lib/device.js');
const tools = require('../agent/tools.js');
function testCatalog() {
assert.strictEqual(catalog.resolveModelConstant('qwen3.5-4b'), 'QWEN3_5_4B_MULTIMODAL_Q4_K_M');
assert.strictEqual(catalog.resolveModelConstant('gemma4-4b'), 'GEMMA4_4B_MULTIMODAL_Q4_K_M');
assert.strictEqual(catalog.resolveModelConstant('qwen3-8b'), 'QWEN3_8B_INST_Q4_K_M');
assert.strictEqual(catalog.resolveModelConstant('qwen3vl-2b'), 'QWEN3_VL_2B_INSTRUCT_Q4_K_M');
assert.strictEqual(catalog.findCatalogEntry('gemma4-4b').vision, true);
assert.strictEqual(catalog.findCatalogEntry('qwen3-8b').vision, false);
assert.strictEqual(catalog.toolDialectFor('gemma4-4b'), 'gemma4');
assert.strictEqual(catalog.toolDialectFor('qwen3-8b'), 'hermes');
assert.strictEqual(catalog.toolDialectFor('qwen3.5-2b'), 'qwen35');
assert.strictEqual(catalog.toolDialectFor('qwen3.5-0.8b'), 'qwen35');
assert.strictEqual(catalog.toolDialectFor('QWEN3_5_2B_MULTIMODAL_Q4_K_M'), 'qwen35');
assert.strictEqual(catalog.findCatalogEntry('qwen3.5-2b').tools, true);
assert.strictEqual(catalog.findCatalogEntry('qwen3.5-2b').compactTools, true);
assert.strictEqual(catalog.findCatalogEntry('qwen3.5-0.8b').tools, true);
assert.strictEqual(catalog.findCatalogEntry('qwen3.5-0.8b').compactTools, true);
assert.strictEqual(catalog.isCompactToolModel('qwen3.5-2b'), true);
assert.strictEqual(catalog.isCompactToolModel('qwen3.5-0.8b'), true);
assert.strictEqual(catalog.isCompactToolModel('qwen3.5-4b'), false);
assert.strictEqual(catalog.findCatalogEntry('qwen3.5-0.8b').ctxSize, 16384);
assert.strictEqual(catalog.findCatalogEntry('qwen3.5-2b').ctxSize, 16384);
assert.strictEqual(catalog.findCatalogEntry('qwen3-1.7b').ctxSize, 16384);
assert.ok(catalog.findCatalogEntry('qwen3.5-4b').ctxSize >= 8192);
const tiny = catalog.filterToolsForModel(
[{ name: 'web_search' }, { name: 'browser' }, { name: 'webcam' }, { name: 'qvac_capability' }, { name: 'cu_drag' }],
'qwen3.5-0.8b',
);
assert.deepStrictEqual(tiny.map((t) => t.name), ['web_search', 'browser', 'webcam']);
assert.strictEqual(
catalog.filterToolsForModel([{ name: 'todo_write' }, { name: 'cu_drag' }], 'qwen3.5-0.8b')[0].name,
'todo_write',
);
assert.strictEqual(catalog.filterToolsForModel([{ name: 'cu_drag' }], 'qwen3.5-4b')[0].name, 'cu_drag');
assert.deepStrictEqual(catalog.compactGenerationParams('qwen3.5-0.8b'), { temp: 0.55 });
assert.strictEqual(catalog.reasoningBudgetForModel('qwen3.5-0.8b'), 128);
assert.strictEqual(catalog.reasoningBudgetForModel('qwen3.5-4b'), 320);
const largeGen = catalog.generationParamsForModel('qwen3.5-4b', { predict: 64 });
assert.strictEqual(largeGen.reasoning_budget, 320);
assert.strictEqual(largeGen.predict, 64);
assert.strictEqual(largeGen.repeat_penalty, 1.15);
assert.ok(/~5 GB/.test(catalog.catalogLabel(catalog.findCatalogEntry('qwen3-8b'))));
assert.ok(catalog.FALLBACK_LLM_IDS.indexOf('gemma4-4b') >= 0);
const listed = catalog.listCatalog().find((m) => m.id === 'gemma4-2b');
assert.ok(listed && listed.label.indexOf('~3.5 GB') >= 0);
}
function hugeToolDefs() {
return Array.from({ length: 24 }, (_, i) => ({
name: 'tool_' + i,
description: 'd'.repeat(400),
parameters: { type: 'object', properties: { q: { type: 'string' } } },
}));
}
function testCompaction() {
const sys = { role: 'system', content: 'sys' };
const user = { role: 'user', content: 'hello' };
const hist = [sys, user];
for (let i = 0; i < 40; i++) {
hist.push({ role: 'assistant', content: 'x'.repeat(200) });
hist.push({ role: 'tool', name: 'read_file', content: 'y'.repeat(200) });
}
assert.ok(compaction.shouldCompact(hist, [], 1024));
const out = compaction.compact(hist, { budgetTokens: 400, tools: [] });
assert.ok(out.length < hist.length);
assert.strictEqual(out[0].role, 'system');
assert.ok(compaction.isOverflowError(new Error('prompt too long for context window')));
const tools = hugeToolDefs();
const short = [
{ role: 'system', content: 'You are Jarvis' },
{ role: 'assistant', content: 'Hello! How can I help you today?' },
{ role: 'user', content: 'Hi, please tell me about my computer.' },
];
assert.ok(compaction.toolTokens(tools) > 2000);
assert.equal(compaction.shouldCompact(short, tools, 8192), false);
const kept = compaction.compact(short, {
budgetTokens: compaction.historyBudget(8192, tools, 0),
tools,
});
assert.strictEqual(kept.length, short.length);
assert.strictEqual(kept[2].content, short[2].content);
assert.ok(JSON.stringify(kept).indexOf('Earlier turns were compacted') < 0);
const four = short.concat([
{ role: 'assistant', content: 'Let me look.' },
{ role: 'user', content: 'Go ahead.' },
]);
assert.ok(compaction.nonSystemCount(four) >= compaction.MIN_COMPACT_MESSAGES);
const stillKept = compaction.heuristicCompact(four, {
budgetTokens: compaction.historyBudget(8192, tools, 0),
tools,
});
assert.ok(stillKept.some((m) => String(m.content).indexOf('tell me about my computer') >= 0));
assert.ok(JSON.stringify(stillKept).indexOf('Earlier turns were compacted') < 0);
assert.strictEqual(compaction.autoContinue([{ role: 'assistant', content: 'hi' }], { voice: true }), null);
assert.ok(compaction.autoContinue([{ role: 'assistant', content: 'hi' }]));
assert.ok(compaction.compactReminder({ voice: true }).indexOf('Do not greet') >= 0);
assert.ok(compaction.VOICE_COMPACT_PROMPT.indexOf('Do not greet') >= 0);
}
function testSearchReplace() {
const applied = sr.applySearchReplace('aaa bbb aaa', 'bbb', 'ccc', false);
assert.strictEqual(applied.text, 'aaa ccc aaa');
assert.strictEqual(applied.replacements, 1);
}
function testToolSet() {
assert.ok(toolSet.isHostWorkspaceTool('read_file'));
assert.ok(!toolSet.parseHostWorkspace({ hostWorkspace: false }));
const all = [
{ name: 'read_file' },
{ name: 'todo_write' },
{ name: 'web_fetch' },
];
const page = toolSet.filterBuiltinSchemas(all, { hostWorkspace: false, builtinTools: ['todo_write'] });
assert.ok(!page.find((t) => t.name === 'read_file'));
const withFetch = toolSet.filterBuiltinSchemas(all, {
hostWorkspace: true,
builtinTools: ['web_fetch'],
webFetch: true,
});
assert.ok(withFetch.find((t) => t.name === 'web_fetch'));
const noFetch = toolSet.filterBuiltinSchemas(all, {
hostWorkspace: true,
builtinTools: ['web_fetch'],
});
assert.ok(!noFetch.find((t) => t.name === 'web_fetch'));
}
function testPrompts() {
assert.ok(prompts.DEFAULT_SYSTEM.indexOf('QVAC') >= 0);
assert.ok(prompts.DEFAULT_SYSTEM.indexOf('BridgeSwarm') < 0);
const page = prompts.assemble({ cwd: 'container', hostWorkspace: false });
assert.ok(page.indexOf('host filesystem') >= 0);
const voice = prompts.assemble({
personality: 'voice',
extra: 'You are Jarvis, a local Ubuntu GNOME voice assistant.',
cwd: '/tmp/jarvis',
hostWorkspace: true,
fsRead: (_c, n) => (n === 'AGENTS.md' ? '# AGENTS.md\nFollow Jarvis workspace rules.' : ''),
});
assert.ok(voice.indexOf('You are Jarvis') >= 0);
assert.ok(voice.indexOf(prompts.DEFAULT_SYSTEM) < 0);
assert.ok(voice.indexOf('coding agent') < 0);
assert.ok(voice.indexOf('AGENTS.md') >= 0);
assert.ok(voice.indexOf('Current workspace:') >= 0);
}
function testNet() {
assert.throws(() => net.assertPublicHttpUrl('http://127.0.0.1/x'));
assert.throws(() => net.assertPublicHttpUrl('http://192.168.1.1/x'));
const u = net.assertPublicHttpUrl('https://example.com/a');
assert.strictEqual(u.hostname, 'example.com');
assert.strictEqual(net.assertHttpUrl('https://ifconfig.me/ip').hostname, 'ifconfig.me');
assert.strictEqual(net.assertHttpUrl('http://192.168.1.1/status').hostname, '192.168.1.1');
assert.throws(() => net.assertHttpUrl('file:///etc/passwd'));
}
function testCustomTools() {
customTools.clear('s1');
customTools.setSession('s1', { hostWorkspace: false });
customTools.register('s1', {
name: 'ping',
description: 'ping',
parameters: { type: 'object', properties: {} },
execute: () => ({ ok: true }),
});
assert.ok(customTools.has('s1', 'ping'));
assert.strictEqual(typeof customTools.getHandler('s1', 'ping'), 'function');
customTools.clear('s1');
}
function testPlanTodosStationarity() {
const pm = planMode.create();
planMode.activate(pm);
assert.ok(planMode.isActive(pm));
const blocked = planMode.gateWrite(pm, 'write_file', { path: 'foo.txt' });
assert.ok(blocked);
const ok = planMode.gateWrite(pm, 'write_file', { path: 'plan.md' });
assert.ok(!ok);
const list = todos.merge([], [{ id: '1', content: 'a', status: 'pending' }], 'replace');
assert.ok(todos.hasOpen(list));
const st = stationarity.create();
stationarity.observe(st, 'read_file', { path: 'a' });
stationarity.observe(st, 'read_file', { path: 'a' });
stationarity.observe(st, 'read_file', { path: 'a' });
assert.ok(stationarity.shouldNudge(st) || !stationarity.shouldStop(st));
}
function testTruncateAndPerm() {
const t = truncate.truncateWithMarker('x'.repeat(5000), 400);
assert.ok(t.length < 5000);
assert.ok(t.indexOf('truncated') >= 0);
const rendered = truncate.renderToolResult({ ok: true, note: 'attached', images: [{ path: '/tmp/secret.png' }] });
assert.ok(rendered.indexOf('attached') >= 0);
assert.ok(rendered.indexOf('/tmp/secret.png') < 0);
const pat = permRules.patternFromArgs('run_terminal_cmd', { command: 'git status -sb' });
assert.strictEqual(pat, 'git status');
assert.ok(policy.shellSafe('git status'));
assert.ok(!policy.shellSafe('rm -rf /'));
assert.ok(permRules.globish('uname -a', '*'));
assert.strictEqual(permRules.patternFromArgs('run_terminal_cmd', { command: '*' }), '*');
const voice = toolBudget.fromPayload({}, 'jarvis-qvac');
toolBudget.markShell(voice);
assert.strictEqual(voice.answerOnly, true);
assert.ok(toolBudget.shouldSkipShell(voice));
}
function testVisionFollowUp() {
const qvac = require('../lib/qvac.js');
const frame = path.join(os.tmpdir(), 'agent-harness-webcam-test.webp');
fs.writeFileSync(frame, Buffer.from('RIFF'));
const hoisted = qvac.prepareVisionHistory([
{ role: 'user', content: 'Do you see anything?' },
{ role: 'assistant', content: '', tool_calls: [{ name: 'webcam' }] },
{ role: 'tool', name: 'webcam', content: '{"ok":true,"note":"attached"}', images: [{ path: frame }] },
]);
const last = hoisted[hoisted.length - 1];
const tool = hoisted[hoisted.length - 2];
assert.strictEqual(tool.role, 'tool');
assert.ok(!tool.images);
assert.ok(!tool.attachments);
assert.strictEqual(last.role, 'user');
assert.ok(String(last.content).trim().length > 0);
assert.strictEqual(last.content, qvac.VISION_FOLLOWUP_QUESTION);
assert.strictEqual(last.attachments.length, 1);
assert.strictEqual(last.attachments[0].path, frame);
const blank = qvac.prepareVisionHistory([{ role: 'user', content: '', images: [{ path: frame }] }])[0];
assert.strictEqual(blank.role, 'user');
assert.strictEqual(blank.content, qvac.VISION_FOLLOWUP_QUESTION);
assert.strictEqual(blank.attachments[0].path, frame);
}
function testPaths() {
const dir = paths.ensureDir(path.join(os.tmpdir(), 'agent-harness-test'));
assert.ok(fs.existsSync(dir));
assert.ok(paths.isPathInside(dir, path.join(dir, 'a.txt')));
assert.ok(!paths.isPathInside(dir, path.join(dir, '..', 'escape')));
}
function testQvacWorkerDeps() {
assert.doesNotThrow(() => require('hyperdispatch/runtime'));
assert.doesNotThrow(() => require('@qvac/registry-schema'));
}
function testDevicePrefersGpu() {
const empty = { gpus: [], drivers: {}, vramBytes: 0 };
assert.deepStrictEqual(device.pickDevice('auto', empty), { device: 'gpu', gpu_layers: 99 });
assert.deepStrictEqual(device.pickDevice(undefined, empty), { device: 'gpu', gpu_layers: 99 });
assert.deepStrictEqual(device.pickDevice('gpu', empty), { device: 'gpu', gpu_layers: 99 });
assert.deepStrictEqual(device.pickDevice('cpu', { gpus: [{ name: 'NVIDIA' }], drivers: { vulkan: true } }), {
device: 'cpu',
gpu_layers: 0,
});
assert.strictEqual(device.mmprojOnGpu({}, empty, true), true);
assert.strictEqual(device.mmprojOnGpu({ mmprojUseGpu: false }, empty, true), false);
assert.strictEqual(device.mmprojOnGpu({}, empty, false), false);
assert.strictEqual(device.gpuLayers({}, { device: 'gpu', gpu_layers: 99 }), 99);
const sdkShape = {
capabilities: {
memory: { totalBytes: { status: 'supported', value: 32e9, provenance: { source: 'test' } } },
gpus: {
status: 'supported',
value: [
{
id: '0',
name: { status: 'supported', value: 'GeForce', provenance: { source: 'test' } },
type: { status: 'supported', value: 2, provenance: { source: 'test' } },
memoryTotalBytes: { status: 'supported', value: 24e9, provenance: { source: 'test' } },
drivers: { vulkan: { status: 'supported', value: true, provenance: { source: 'test' } } },
},
],
provenance: { source: 'test' },
},
},
};
const norm = device.normalizeResources(sdkShape);
assert.ok(device.hasGpu(norm));
assert.ok(norm.vramBytes > 1e9);
assert.strictEqual(device.backendLabel(norm).backend, 'vulkan');
}
function testToolParse() {
const parse = require('../lib/tool-parse.js');
const tools = [
{ name: 'web_search', parameters: { type: 'object', properties: { query: { type: 'string' } } } },
{ name: 'run_terminal_cmd', parameters: { type: 'object', properties: { command: { type: 'string' } } } },
];
const xml = [
'<think>',
'I should look that up.',
'<tool_call>',
'<function=web_search>',
'<parameter=query>weather in Paris</parameter>',
'</function>',
'</tool_call>',
'</think>',
].join('\n');
const nested = parse.extractCalls(xml, tools);
assert.strictEqual(nested.length, 1);
assert.strictEqual(nested[0].name, 'web_search');
assert.strictEqual(nested[0].arguments.query, 'weather in Paris');
const hermes = parse.extractCalls(
'<think><tool_call>{"name":"web_search","arguments":{"query":"news"}}</tool_call></think>',
tools
);
assert.strictEqual(hermes[0].name, 'web_search');
assert.strictEqual(hermes[0].arguments.query, 'news');
const aliased = parse.extractCalls(
'<tool_call><function=search><parameter=query>headlines</parameter></function></tool_call>',
tools
);
assert.strictEqual(aliased[0].name, 'web_search');
const recovered = parse.recover({
thinking: xml,
text: 'I will search now.\n<tool_call><function=web_search><parameter=query>x</parameter></function></tool_call>',
tools,
});
assert.strictEqual(recovered.calls[0].name, 'web_search');
assert.strictEqual(recovered.text, 'I will search now.');
assert.ok(parse.FORMAT_REMINDER.indexOf('<function=TOOL_NAME>') >= 0);
}
testCatalog();
testToolParse();
testCompaction();
testSearchReplace();
testToolSet();
testPrompts();
testNet();
testCustomTools();
testPlanTodosStationarity();
testTruncateAndPerm();
testVisionFollowUp();
testPaths();
testQvacWorkerDeps();
testDevicePrefersGpu();
testGoogleSearchParseAndFallback();
testWebFetchTimeout()
.then(() => testGoogleSearchFallsBackToDuckDuckGo())
.then(() => {
console.log('ok');
})
.catch((err) => {
console.error(err);
process.exitCode = 1;
});
async function testWebFetchTimeout() {
tools.setBrowserBackend({ call: () => new Promise(() => {}) });
const started = Date.now();
try {
const hung = await tools.webFetch('https://example.com/ip', 40);
assert.ok(hung.error);
assert.ok(/timed out/i.test(hung.error));
assert.strictEqual(hung.url, 'https://example.com/ip');
assert.ok(Date.now() - started < 2000);
} finally {
tools.setBrowserBackend(null);
}
tools.setBrowserBackend({
call: async (_action, payload) => ({
url: payload.url,
status: 200,
html: '<html><body>203.0.113.8</body></html>',
text: '203.0.113.8',
}),
});
try {
const ok = await tools.webFetch('https://ifconfig.me/ip', 200);
assert.strictEqual(ok.status, 200);
assert.strictEqual(ok.url, 'https://ifconfig.me/ip');
assert.ok(/203\.0\.113\.8/.test(ok.text));
assert.ok(!ok.error);
} finally {
tools.setBrowserBackend(null);
}
tools.setBrowserBackend({
call: async () => ({ status: 503, url: 'https://example.com', html: '<p>down</p>', text: 'down', error: 'HTTP 503' }),
});
try {
const failed = await tools.webFetch('https://example.com/status', 200);
assert.ok(failed.error);
assert.strictEqual(failed.status, 503);
assert.strictEqual(failed.url, 'https://example.com');
} finally {
tools.setBrowserBackend(null);
}
}
function testGoogleSearchParseAndFallback() {
assert.ok(tools.SCHEMAS.find((t) => t.name === 'google_search'));
assert.ok(tools.SCHEMAS.find((t) => t.name === 'fetch_page'));
assert.ok(tools.SCHEMAS.find((t) => t.name === 'wiki_search'));
assert.ok(tools.SCHEMAS.find((t) => t.name === 'web_search'));
assert.ok(tools.SCHEMAS.find((t) => t.name === 'web_fetch'));
}
async function testGoogleSearchFallsBackToDuckDuckGo() {
tools.setBrowserBackend({
call: async (_action, payload) => {
if (payload.engine === 'google') return [];
return [{ url: 'https://example.com/ddg', title: 'DDG Example', source: payload.engine }];
},
});
try {
const hits = await tools.webSearch('example domain', 200);
assert.ok(Array.isArray(hits));
assert.strictEqual(hits.length, 1);
assert.strictEqual(hits[0].source, 'duckduckgo');
assert.strictEqual(hits[0].url, 'https://example.com/ddg');
assert.strictEqual(hits[0].title, 'DDG Example');
} finally {
tools.setBrowserBackend(null);
}
tools.setBrowserBackend({
call: async (_action, payload) => {
if (payload.engine === 'duckduckgo') throw new Error('duckduckgo should not run when google hits');
return [{ url: 'https://example.com/google', title: 'From Google', source: payload.engine }];
},
});
try {
const hits = await tools.googleSearchWithFallback('example domain', 200);
assert.strictEqual(hits[0].source, 'google');
assert.strictEqual(hits[0].url, 'https://example.com/google');
assert.strictEqual(hits[0].title, 'From Google');
} finally {
tools.setBrowserBackend(null);
}
tools.setBrowserBackend({
call: async (_action, payload) => {
if (payload.engine === 'google') return [];
return [{ url: 'http://www.example.com/', title: 'Example Domain', source: payload.engine }];
},
});
try {
const hits = await tools.googleSearchWithFallback('example domain', 200);
assert.strictEqual(hits[0].source, 'duckduckgo');
assert.strictEqual(hits[0].url, 'http://www.example.com/');
} finally {
tools.setBrowserBackend(null);
}
}
Reference in New Issue View Git Blame Copy Permalink