peardata/test/ai-tools.test.js

import test from 'brittle'
import { Methods, MethodRoles, Roles } from '../shared/protocol.js'
import { validateMethodArgs } from '../shared/schema.js'
import { suggestProfile, getProfile, QVAC_PROFILES } from '../ui/qvac/profiles.js'
import { buildSystemPrompt } from '../ui/qvac/prompts.js'
import { buildRagContext, retrieve, catalogDocs, GUIDE_DOCS } from '../ui/qvac/rag.js'
import { getStore } from '../server/services/store.js'
import {
  searchCharts,
  summarizeChart,
  getHostSnapshot,
  investigateHost,
  hotMetrics,
  relatedCharts,
  compareChartWindows,
  summarizeCharts,
} from '../server/services/ai-tools.js'
import { initAuthKeys } from '../server/core/auth-keys.js'
import crypto from 'hypercore-crypto'
import b4a from 'b4a'

const seed = crypto.randomBytes(32)
initAuthKeys({
  seedHex: b4a.toString(seed, 'hex'),
  publicKeyHex: b4a.toString(crypto.keyPair(seed).publicKey, 'hex'),
})

const AI_METHODS = [
  'getHostSnapshot',
  'searchCharts',
  'summarizeChart',
  'investigateHost',
  'hotMetrics',
  'relatedCharts',
  'compareChartWindows',
  'summarizeCharts',
]

test('AI composite methods are viewer role', (t) => {
  for (const m of AI_METHODS) {
    t.is(MethodRoles[m], Roles.viewer, m)
    t.is(Methods[m], m)
  }
})

test('schema validates searchCharts + summarizeChart + deep AI', (t) => {
  t.ok(validateMethodArgs('getHostSnapshot', {}).ok)
  t.ok(validateMethodArgs('investigateHost', {}).ok)

  const s = validateMethodArgs('searchCharts', { q: 'cpu', limit: 10 })
  t.ok(s.ok)
  t.is(s.args.q, 'cpu')
  const bad = validateMethodArgs('searchCharts', { limit: 999 })
  t.absent(bad.ok)

  const sum = validateMethodArgs('summarizeChart', { chart: 'system.cpu', points: 60 })
  t.ok(sum.ok)
  t.is(sum.args.chart, 'system.cpu')
  const miss = validateMethodArgs('summarizeChart', {})
  t.absent(miss.ok)

  const hot = validateMethodArgs('hotMetrics', { limit: 10, window: 60 })
  t.ok(hot.ok)
  const hotBad = validateMethodArgs('hotMetrics', { window: 5 })
  t.absent(hotBad.ok)

  const rel = validateMethodArgs('relatedCharts', { chart: 'system.cpu' })
  t.ok(rel.ok)
  const relMiss = validateMethodArgs('relatedCharts', {})
  t.absent(relMiss.ok)

  const cmp = validateMethodArgs('compareChartWindows', { chart: 'system.cpu', points: 40 })
  t.ok(cmp.ok)

  const batch = validateMethodArgs('summarizeCharts', {
    charts: ['system.cpu', 'system.ram'],
    points: 30,
  })
  t.ok(batch.ok)
  t.is(batch.args.charts.length, 2)

  const batchStr = validateMethodArgs('summarizeCharts', { charts: 'system.cpu,system.ram' })
  t.ok(batchStr.ok)
  t.is(batchStr.args.charts.length, 2)
})

test('QVAC profiles suggest by RAM', (t) => {
  t.is(suggestProfile({ totalRamBytes: 4e9 }), 'lite')
  t.is(suggestProfile({ totalRamBytes: 10e9 }), 'recommended')
  t.is(suggestProfile({ totalRamBytes: 32e9 }), 'strong')
  t.ok(getProfile('recommended').tools)
  t.ok(QVAC_PROFILES.lite.chatModel.includes('QWEN'))
})

test('system prompt forbids inventing metrics and mentions investigate', (t) => {
  const p = buildSystemPrompt({ peerAlias: 'lab', role: 'viewer', connected: true })
  t.ok(p.includes('Never invent'))
  t.ok(p.includes('lab'))
  t.ok(p.includes('viewer'))
  t.ok(p.includes('investigate_host'))
})

test('RAG retrieves guide + catalog', (t) => {
  t.ok(GUIDE_DOCS.length >= 5)
  const hits = retrieve('charts time presets pin', GUIDE_DOCS, 3)
  t.ok(hits.length >= 1)
  t.ok(hits[0].score > 0)
  const cat = catalogDocs({
    'system.cpu': { title: 'CPU', context: 'system.cpu', family: 'cpu', units: '%' },
    'disk_io.sda': { title: 'Disk sda', context: 'disk', family: 'io' },
  })
  t.is(cat.length, 2)
  const ctx = buildRagContext({ query: 'disk io', catalog: { 'disk_io.sda': { title: 'Disk' } } })
  t.ok(ctx.includes('disk') || ctx.includes('Disk') || ctx.includes('local knowledge'))
})

test('searchCharts and summarizeChart against store', async (t) => {
  const now = Date.now()
  for (let i = 0; i < 8; i++) {
    getStore().ingest([
      {
        chart: 'system.cpu',
        context: 'system.cpu',
        ts: now - (8 - i) * 1000,
        values: {
          user: 10 + i,
          system: 2,
          nice: 0,
          iowait: 0,
          irq: 0,
          softirq: 0,
          idle: 80 - i,
        },
      },
    ])
  }
  const found = searchCharts({ q: 'cpu', limit: 20 })
  t.ok(found.results.some((r) => r.id === 'system.cpu'))
  const sum = await summarizeChart({ chart: 'system.cpu', after: -30, points: 20 })
  t.is(sum.chart, 'system.cpu')
  t.ok(sum.dims)
  const snap = await getHostSnapshot()
  t.ok(snap.hostname)
  t.ok(snap.kpis)

  const inv = await investigateHost({ processLimit: 5, hotLimit: 8 })
  t.ok(inv.summary)
  t.ok(Array.isArray(inv.findings))
  t.ok(inv.kpis)

  const hot = await hotMetrics({ limit: 10, window: 60 })
  t.ok(Array.isArray(hot.results))

  const batch = await summarizeCharts({ charts: ['system.cpu'], after: -30, points: 20 })
  t.ok(batch.results?.some((r) => r.chart === 'system.cpu' && r.dims))

  const cmp = await compareChartWindows({ chart: 'system.cpu', points: 20 })
  t.ok(cmp.delta || cmp.error == null)
  t.ok(cmp.highlight || cmp.error)

  // related may be empty without full catalog entry — should not throw
  const rel = await relatedCharts({ chart: 'system.cpu', limit: 5 })
  t.ok(rel.chart === 'system.cpu' || rel.error)
})

test('tool runner gates offline + silence role + tiers', async (t) => {
  const {
    createToolRunner,
    fallbackComplete,
    TOOL_DEFS,
    toolDepthForProfile,
  } = await import('../ui/qvac/tools.js')

  // Flat @qvac/sdk shape (not nested function.name)
  t.ok(TOOL_DEFS.some((d) => d.name === 'host_snapshot'))
  t.ok(TOOL_DEFS.some((d) => d.name === 'investigate_host'))
  t.ok(TOOL_DEFS.some((d) => d.name === 'hot_metrics'))
  t.ok(TOOL_DEFS.some((d) => d.name === 'fleet_health'))
  t.ok(TOOL_DEFS.every((d) => d.type === 'function' && d.name && d.parameters))
  t.absent(TOOL_DEFS.some((d) => d.function))

  t.is(toolDepthForProfile('tool-tiny'), 'core')
  t.is(toolDepthForProfile('recommended'), 'deep')
  t.is(toolDepthForProfile('strong'), 'deep')

  let connected = false
  /** @type {string} */
  let role = 'viewer'
  const calls = []
  const tools = createToolRunner({
    manager: {
      request: async (m, a) => {
        calls.push({ m, a })
        if (m === 'getHostSnapshot') {
          return { hostname: 'lab', health: { status: 'ok' }, kpis: {} }
        }
        if (m === 'investigateHost') {
          return {
            hostname: 'lab',
            summary: { findingCount: 0, topSeverity: 'ok', health: 'ok' },
            findings: [],
            kpis: {},
          }
        }
        if (m === 'listAnomalies') return { anomalies: [] }
        if (m === 'hotMetrics') return { window: 120, count: 0, results: [] }
        if (m === 'listAlerts') return { alerts: [] }
        if (m === 'getHealth') return { status: 'ok' }
        if (m === 'listJobs') return { jobs: [] }
        return { ok: true }
      },
      active: null,
    },
    getRole: () => role,
    isConnected: () => connected,
    getCatalog: () => ({ 'system.cpu': { title: 'CPU' } }),
  })

  const offline = await tools.run('host_snapshot', {})
  t.ok(offline.error)
  t.ok(String(offline.error).toLowerCase().includes('connect'))

  connected = true
  const snap = await tools.run('host_snapshot', {})
  t.is(snap.hostname, 'lab')

  const inv = await tools.run('investigate_host', {})
  t.is(inv.hostname, 'lab')
  t.ok(inv.summary)

  const silence = await tools.run('silence_alert', { id: 'a1', confirmed: true })
  t.ok(String(silence.error || '').includes('Operator') || silence.error)

  role = 'operator'
  const silence2 = await tools.run('silence_alert', { id: 'a1', confirmed: false })
  t.is(silence2.error, 'confirmation_required')

  role = 'viewer'
  const defsViewer = tools.defsForRole({ profileId: 'recommended' }).map((d) => d.name)
  t.absent(defsViewer.includes('silence_alert'))
  t.ok(defsViewer.includes('investigate_host'))
  t.ok(defsViewer.includes('hot_metrics'))
  t.ok(defsViewer.includes('related_charts'))

  const defsTiny = tools.defsForRole({ profileId: 'tool-tiny' }).map((d) => d.name)
  t.ok(defsTiny.includes('investigate_host'))
  t.absent(defsTiny.includes('hot_metrics'))
  t.absent(defsTiny.includes('query_metric'))

  role = 'operator'
  const defsOp = tools.defsForRole({ profileId: 'strong' }).map((d) => d.name)
  t.ok(defsOp.includes('silence_alert'))
  t.ok(defsOp.includes('ack_alert'))
  t.ok(defsOp.includes('run_job'))

  // Wire shape has no internal tier field
  t.ok(tools.defsForRole().every((d) => d.tier == null))

  const kn = await tools.run('local_knowledge', { q: 'charts' })
  t.ok(kn.context)

  const fb = await fallbackComplete('any open alerts?', tools)
  t.is(fb.mode, 'fallback')
  t.ok(fb.contentText)
  t.ok((fb.toolCalls || []).length >= 1)

  const fb2 = await fallbackComplete("what's wrong with this host", tools)
  t.ok((fb2.toolCalls || []).some((c) => c.name === 'investigate_host'))
})