import test from 'brittle' import { Methods, MethodRoles, Roles } from '../shared/protocol.js' import { validateMethodArgs } from '../shared/schema.js' import { suggestProfile, getProfile, QVAC_PROFILES } from '../ui/qvac/profiles.js' import { buildSystemPrompt } from '../ui/qvac/prompts.js' import { buildRagContext, retrieve, catalogDocs, GUIDE_DOCS } from '../ui/qvac/rag.js' import { getStore } from '../server/services/store.js' import { searchCharts, summarizeChart, getHostSnapshot, investigateHost, hotMetrics, relatedCharts, compareChartWindows, summarizeCharts, } from '../server/services/ai-tools.js' import { initAuthKeys } from '../server/core/auth-keys.js' import crypto from 'hypercore-crypto' import b4a from 'b4a' const seed = crypto.randomBytes(32) initAuthKeys({ seedHex: b4a.toString(seed, 'hex'), publicKeyHex: b4a.toString(crypto.keyPair(seed).publicKey, 'hex'), }) const AI_METHODS = [ 'getHostSnapshot', 'searchCharts', 'summarizeChart', 'investigateHost', 'hotMetrics', 'relatedCharts', 'compareChartWindows', 'summarizeCharts', ] test('AI composite methods are viewer role', (t) => { for (const m of AI_METHODS) { t.is(MethodRoles[m], Roles.viewer, m) t.is(Methods[m], m) } }) test('schema validates searchCharts + summarizeChart + deep AI', (t) => { t.ok(validateMethodArgs('getHostSnapshot', {}).ok) t.ok(validateMethodArgs('investigateHost', {}).ok) const s = validateMethodArgs('searchCharts', { q: 'cpu', limit: 10 }) t.ok(s.ok) t.is(s.args.q, 'cpu') const bad = validateMethodArgs('searchCharts', { limit: 999 }) t.absent(bad.ok) const sum = validateMethodArgs('summarizeChart', { chart: 'system.cpu', points: 60 }) t.ok(sum.ok) t.is(sum.args.chart, 'system.cpu') const miss = validateMethodArgs('summarizeChart', {}) t.absent(miss.ok) const hot = validateMethodArgs('hotMetrics', { limit: 10, window: 60 }) t.ok(hot.ok) const hotBad = validateMethodArgs('hotMetrics', { window: 5 }) t.absent(hotBad.ok) const rel = validateMethodArgs('relatedCharts', { chart: 'system.cpu' }) t.ok(rel.ok) const relMiss = validateMethodArgs('relatedCharts', {}) t.absent(relMiss.ok) const cmp = validateMethodArgs('compareChartWindows', { chart: 'system.cpu', points: 40 }) t.ok(cmp.ok) const batch = validateMethodArgs('summarizeCharts', { charts: ['system.cpu', 'system.ram'], points: 30, }) t.ok(batch.ok) t.is(batch.args.charts.length, 2) const batchStr = validateMethodArgs('summarizeCharts', { charts: 'system.cpu,system.ram' }) t.ok(batchStr.ok) t.is(batchStr.args.charts.length, 2) }) test('QVAC profiles suggest by RAM', (t) => { t.is(suggestProfile({ totalRamBytes: 4e9 }), 'lite') t.is(suggestProfile({ totalRamBytes: 10e9 }), 'recommended') t.is(suggestProfile({ totalRamBytes: 32e9 }), 'strong') t.ok(getProfile('recommended').tools) t.ok(QVAC_PROFILES.lite.chatModel.includes('QWEN')) }) test('system prompt forbids inventing metrics and mentions investigate', (t) => { const p = buildSystemPrompt({ peerAlias: 'lab', role: 'viewer', connected: true }) t.ok(p.includes('Never invent')) t.ok(p.includes('lab')) t.ok(p.includes('viewer')) t.ok(p.includes('investigate_host')) }) test('RAG retrieves guide + catalog', (t) => { t.ok(GUIDE_DOCS.length >= 5) const hits = retrieve('charts time presets pin', GUIDE_DOCS, 3) t.ok(hits.length >= 1) t.ok(hits[0].score > 0) const cat = catalogDocs({ 'system.cpu': { title: 'CPU', context: 'system.cpu', family: 'cpu', units: '%' }, 'disk_io.sda': { title: 'Disk sda', context: 'disk', family: 'io' }, }) t.is(cat.length, 2) const ctx = buildRagContext({ query: 'disk io', catalog: { 'disk_io.sda': { title: 'Disk' } } }) t.ok(ctx.includes('disk') || ctx.includes('Disk') || ctx.includes('local knowledge')) }) test('searchCharts and summarizeChart against store', async (t) => { const now = Date.now() for (let i = 0; i < 8; i++) { getStore().ingest([ { chart: 'system.cpu', context: 'system.cpu', ts: now - (8 - i) * 1000, values: { user: 10 + i, system: 2, nice: 0, iowait: 0, irq: 0, softirq: 0, idle: 80 - i, }, }, ]) } const found = searchCharts({ q: 'cpu', limit: 20 }) t.ok(found.results.some((r) => r.id === 'system.cpu')) const sum = await summarizeChart({ chart: 'system.cpu', after: -30, points: 20 }) t.is(sum.chart, 'system.cpu') t.ok(sum.dims) const snap = await getHostSnapshot() t.ok(snap.hostname) t.ok(snap.kpis) const inv = await investigateHost({ processLimit: 5, hotLimit: 8 }) t.ok(inv.summary) t.ok(Array.isArray(inv.findings)) t.ok(inv.kpis) const hot = await hotMetrics({ limit: 10, window: 60 }) t.ok(Array.isArray(hot.results)) const batch = await summarizeCharts({ charts: ['system.cpu'], after: -30, points: 20 }) t.ok(batch.results?.some((r) => r.chart === 'system.cpu' && r.dims)) const cmp = await compareChartWindows({ chart: 'system.cpu', points: 20 }) t.ok(cmp.delta || cmp.error == null) t.ok(cmp.highlight || cmp.error) // related may be empty without full catalog entry — should not throw const rel = await relatedCharts({ chart: 'system.cpu', limit: 5 }) t.ok(rel.chart === 'system.cpu' || rel.error) }) test('tool runner gates offline + silence role + tiers', async (t) => { const { createToolRunner, fallbackComplete, TOOL_DEFS, toolDepthForProfile, } = await import('../ui/qvac/tools.js') // Flat @qvac/sdk shape (not nested function.name) t.ok(TOOL_DEFS.some((d) => d.name === 'host_snapshot')) t.ok(TOOL_DEFS.some((d) => d.name === 'investigate_host')) t.ok(TOOL_DEFS.some((d) => d.name === 'hot_metrics')) t.ok(TOOL_DEFS.some((d) => d.name === 'fleet_health')) t.ok(TOOL_DEFS.every((d) => d.type === 'function' && d.name && d.parameters)) t.absent(TOOL_DEFS.some((d) => d.function)) t.is(toolDepthForProfile('tool-tiny'), 'core') t.is(toolDepthForProfile('recommended'), 'deep') t.is(toolDepthForProfile('strong'), 'deep') let connected = false /** @type {string} */ let role = 'viewer' const calls = [] const tools = createToolRunner({ manager: { request: async (m, a) => { calls.push({ m, a }) if (m === 'getHostSnapshot') { return { hostname: 'lab', health: { status: 'ok' }, kpis: {} } } if (m === 'investigateHost') { return { hostname: 'lab', summary: { findingCount: 0, topSeverity: 'ok', health: 'ok' }, findings: [], kpis: {}, } } if (m === 'listAnomalies') return { anomalies: [] } if (m === 'hotMetrics') return { window: 120, count: 0, results: [] } if (m === 'listAlerts') return { alerts: [] } if (m === 'getHealth') return { status: 'ok' } if (m === 'listJobs') return { jobs: [] } return { ok: true } }, active: null, }, getRole: () => role, isConnected: () => connected, getCatalog: () => ({ 'system.cpu': { title: 'CPU' } }), }) const offline = await tools.run('host_snapshot', {}) t.ok(offline.error) t.ok(String(offline.error).toLowerCase().includes('connect')) connected = true const snap = await tools.run('host_snapshot', {}) t.is(snap.hostname, 'lab') const inv = await tools.run('investigate_host', {}) t.is(inv.hostname, 'lab') t.ok(inv.summary) const silence = await tools.run('silence_alert', { id: 'a1', confirmed: true }) t.ok(String(silence.error || '').includes('Operator') || silence.error) role = 'operator' const silence2 = await tools.run('silence_alert', { id: 'a1', confirmed: false }) t.is(silence2.error, 'confirmation_required') role = 'viewer' const defsViewer = tools.defsForRole({ profileId: 'recommended' }).map((d) => d.name) t.absent(defsViewer.includes('silence_alert')) t.ok(defsViewer.includes('investigate_host')) t.ok(defsViewer.includes('hot_metrics')) t.ok(defsViewer.includes('related_charts')) const defsTiny = tools.defsForRole({ profileId: 'tool-tiny' }).map((d) => d.name) t.ok(defsTiny.includes('investigate_host')) t.absent(defsTiny.includes('hot_metrics')) t.absent(defsTiny.includes('query_metric')) role = 'operator' const defsOp = tools.defsForRole({ profileId: 'strong' }).map((d) => d.name) t.ok(defsOp.includes('silence_alert')) t.ok(defsOp.includes('ack_alert')) t.ok(defsOp.includes('run_job')) // Wire shape has no internal tier field t.ok(tools.defsForRole().every((d) => d.tier == null)) const kn = await tools.run('local_knowledge', { q: 'charts' }) t.ok(kn.context) const fb = await fallbackComplete('any open alerts?', tools) t.is(fb.mode, 'fallback') t.ok(fb.contentText) t.ok((fb.toolCalls || []).length >= 1) const fb2 = await fallbackComplete("what's wrong with this host", tools) t.ok((fb2.toolCalls || []).some((c) => c.name === 'investigate_host')) })