peardata/test/retention.test.js

import test from 'brittle'
import fs from 'fs'
import path from 'path'
import { fileURLToPath } from 'url'
import {
  defaultRetentionConfig,
  normalizeRetentionConfig,
  retentionCutoffMs,
  RETENTION_PRESETS,
  DISK_BUDGET_PRESETS,
  matchPointsPreset,
  HOT_PRESETS,
} from '../shared/retention.js'
import { formatBytes } from '../shared/format.js'
import { MetricStore } from '../server/services/store.js'
import { PearDataModel } from '../server/db/model.js'
import { isWarmPersistableChart, registerChart } from '../shared/metrics.js'
import { enqueueWarmPoint, warmFlushStats } from '../server/services/warm-flush.js'
import Corestore from 'corestore'
import { validateMethodArgs } from '../shared/schema.js'
import { MethodRoles, Methods, Roles } from '../shared/protocol.js'

const __dirname = path.dirname(fileURLToPath(import.meta.url))
const TMP = path.join(__dirname, '..', 'tmp-retention-test')

function dirSize(dir) {
  let total = 0
  const stack = [dir]
  while (stack.length) {
    const cur = stack.pop()
    for (const ent of fs.readdirSync(cur, { withFileTypes: true })) {
      const full = path.join(cur, ent.name)
      if (ent.isDirectory()) stack.push(full)
      else total += fs.statSync(full).size || 0
    }
  }
  return total
}

test('normalizeRetentionConfig applies presets', (t) => {
  const cfg = normalizeRetentionConfig({
    warmRetentionPreset: '1m',
    tier0Points: 900,
  })
  t.is(cfg.warmRetentionMs, RETENTION_PRESETS['1m'].ms)
  t.is(cfg.tier0Points, 900)
  t.is(matchPointsPreset(900, HOT_PRESETS), '15m')
})

test('retentionCutoffMs respects forever / disabled', (t) => {
  const now = 1_700_000_000_000
  t.is(retentionCutoffMs({ warmRetentionEnabled: false, warmRetentionMs: 1000 }, now), null)
  t.is(retentionCutoffMs({ warmRetentionEnabled: true, warmRetentionMs: 0 }, now), null)
  t.is(
    retentionCutoffMs({ warmRetentionEnabled: true, warmRetentionMs: 1000 }, now),
    now - 1000
  )
})

test('formatBytes scales', (t) => {
  t.is(formatBytes(0), '0 B')
  t.is(formatBytes(1024), '1 KiB')
  t.is(formatBytes(1536 * 1024 * 1024), '1.5 GiB')
})

test('MetricStore setRetention trims rings', (t) => {
  const store = new MetricStore()
  const ts = Date.now()
  for (let i = 0; i < 100; i++) {
    store.ingest([
      {
        chart: 'system.cpu',
        context: 'system.cpu',
        ts: ts + i * 1000,
        values: { user: 1, idle: 99 },
      },
    ])
  }
  t.ok(store.series.get('system.cpu').points.length >= 100)
  store.setRetention({ tier0Max: 10, tier1Max: 5, tier1Every: 60 })
  t.is(store.tier0Max, 10)
  t.is(store.series.get('system.cpu').points.length, 10)
  const mem = store.memoryStats()
  t.is(mem.tier0Max, 10)
  t.ok(mem.tier0Points <= 10)
})

test('protocol exposes data manager methods', (t) => {
  for (const m of ['getStorageInfo', 'getRetentionConfig', 'setRetentionConfig', 'pruneHistory']) {
    t.ok(MethodRoles[m], m)
    t.is(Methods[m], m)
  }
  t.is(MethodRoles.setRetentionConfig, Roles.admin)
  t.is(MethodRoles.pruneHistory, Roles.admin)
  t.ok(validateMethodArgs('getStorageInfo', {}).ok)
  t.ok(validateMethodArgs('setRetentionConfig', { tier0Points: 900 }).ok)
  t.ok(validateMethodArgs('pruneHistory', { dryRun: true }).ok)
})

test('hyperdb deleteMetricPointsBefore removes old rows', async (t) => {
  fs.rmSync(TMP, { recursive: true, force: true })
  const store = new Corestore(path.join(TMP, 'corestore'))
  await store.ready()
  const core = store.get({ name: 'peardata-meta' })
  const model = new PearDataModel(core, { autoUpdate: true })
  await model.ready()
  try {
    const now = Date.now()
    await model.putMetricPoints([
      {
        chart: 'system.cpu',
        context: 'system.cpu',
        ts: now - 10_000,
        values: { user: 1 },
        tier: 1,
      },
      {
        chart: 'system.cpu',
        context: 'system.cpu',
        ts: now - 1000,
        values: { user: 2 },
        tier: 1,
      },
      {
        chart: 'system.cpu',
        context: 'system.cpu',
        ts: now,
        values: { user: 3 },
        tier: 1,
      },
    ])
    const dry = await model.deleteMetricPointsBefore(now - 2000, {
      charts: ['system.cpu'],
      dryRun: true,
    })
    t.is(dry.deleted, 1)
    const del = await model.deleteMetricPointsBefore(now - 2000, {
      charts: ['system.cpu'],
      dryRun: false,
    })
    t.is(del.deleted, 1)
    const rows = await model.queryMetricPoints({
      chart: 'system.cpu',
      afterMs: now - 20_000,
      beforeMs: now + 1000,
    })
    t.is(rows.length, 2)
    t.is(rows[0].values.user, 2)
  } finally {
    await model.close().catch(() => {})
    await store.close().catch(() => {})
    fs.rmSync(TMP, { recursive: true, force: true })
  }
})

test('defaultRetentionConfig has sane defaults', (t) => {
  const d = defaultRetentionConfig()
  t.is(d.warmRetentionPreset, '3m')
  t.is(d.warmRetentionMs, RETENTION_PRESETS['3m'].ms)
  t.is(d.warmMaxBytes, DISK_BUDGET_PRESETS['1gb'].bytes)
  t.ok(d.autoPruneEnabled)
  t.ok(d.tier0Points > 0)
})

test('isWarmPersistableChart gates high-cardinality instance charts', (t) => {
  t.ok(isWarmPersistableChart('system.cpu'))
  t.ok(isWarmPersistableChart('system.ram'))
  registerChart({
    id: 'disk_io.sda',
    name: 'disk_io.sda',
    context: 'disk.io',
    title: 'sda',
    units: 'KiB/s',
    family: 'disk',
    chartType: 'line',
    priority: 1,
    dimensions: [{ id: 'read', name: 'read', algorithm: 'incremental' }],
  })
  t.is(isWarmPersistableChart('disk_io.sda'), false)
  t.is(isWarmPersistableChart('docker.cpu.abc'), false)
})

test('enqueueWarmPoint skips non-persistable charts', (t) => {
  const before = warmFlushStats().skippedHighCardinality
  enqueueWarmPoint({
    chart: 'cgroup.cpu.foo',
    context: 'cgroup.cpu',
    ts: Date.now(),
    values: { usage: 1 },
  })
  t.ok(warmFlushStats().skippedHighCardinality > before)
  const pendingBefore = warmFlushStats().pending
  enqueueWarmPoint({
    chart: 'system.cpu',
    context: 'system.cpu',
    ts: Date.now(),
    values: { user: 1 },
  })
  t.is(warmFlushStats().pending, pendingBefore + 1)
})

test('reclaimStorage runs after delete (clearUnlinked + compact)', async (t) => {
  fs.rmSync(TMP, { recursive: true, force: true })
  const storePath = path.join(TMP, 'corestore')
  const store = new Corestore(storePath)
  await store.ready()
  const core = store.get({ name: 'peardata-meta' })
  const model = new PearDataModel(core, { autoUpdate: true })
  await model.ready()
  try {
    const now = Date.now()
    const points = []
    for (let i = 0; i < 400; i++) {
      points.push({
        chart: 'system.cpu',
        context: 'system.cpu',
        ts: now - (400 - i) * 60_000,
        values: { user: i % 50, idle: 50 },
        tier: 1,
      })
    }
    await model.putMetricPoints(points)
    const sizeAfterInsert = dirSize(storePath)

    const del = await model.deleteMetricPointsBefore(now - 200 * 60_000, {
      charts: ['system.cpu'],
      dryRun: false,
      maxPasses: 20,
    })
    t.ok(del.deleted >= 200, `deleted ${del.deleted}`)
    const sizeAfterDelete = dirSize(storePath)
    t.ok(sizeAfterDelete >= sizeAfterInsert, 'logical delete should not shrink (often grows)')

    const reclaim = await model.reclaimStorage({ batchSize: 512 })
    t.ok(reclaim.cleared)
    t.ok(reclaim.compacted)
    t.ok(reclaim.clearMs >= 0)
    t.ok(reclaim.compactMs >= 0)

    const remaining = await model.queryMetricPoints({
      chart: 'system.cpu',
      afterMs: now - 500 * 60_000,
      beforeMs: now + 1000,
      limit: 10_000,
    })
    t.ok(remaining.length > 0 && remaining.length < 400)
  } finally {
    await model.close().catch(() => {})
    await store.close().catch(() => {})
    fs.rmSync(TMP, { recursive: true, force: true })
  }
})

test('deleteMetricPointsBefore multi-pass drains large backlog', async (t) => {
  fs.rmSync(TMP, { recursive: true, force: true })
  const store = new Corestore(path.join(TMP, 'corestore'))
  await store.ready()
  const core = store.get({ name: 'peardata-meta' })
  const model = new PearDataModel(core, { autoUpdate: true })
  await model.ready()
  try {
    const now = Date.now()
    const batch = []
    for (let i = 0; i < 50; i++) {
      batch.push({
        chart: 'system.ram',
        context: 'system.ram',
        ts: now - (50 - i) * 1000,
        values: { free: i },
        tier: 1,
      })
    }
    await model.putMetricPoints(batch)
    const del = await model.deleteMetricPointsBefore(now + 1, {
      charts: ['system.ram'],
      dryRun: false,
      limitPerChart: 10,
      maxPasses: 20,
    })
    t.is(del.deleted, 50)
    t.ok(del.passes >= 5)
    const left = await model.queryMetricPoints({
      chart: 'system.ram',
      afterMs: 0,
      beforeMs: now + 10_000,
    })
    t.is(left.length, 0)
  } finally {
    await model.close().catch(() => {})
    await store.close().catch(() => {})
    fs.rmSync(TMP, { recursive: true, force: true })
  }
})

test('metric-point secondary indexes are not written (deprecated)', async (t) => {
  fs.rmSync(TMP, { recursive: true, force: true })
  const store = new Corestore(path.join(TMP, 'corestore'))
  await store.ready()
  const core = store.get({ name: 'peardata-meta' })
  const model = new PearDataModel(core, { autoUpdate: true })
  await model.ready()
  try {
    const lengthBefore = core.length
    await model.putMetricPoints([
      {
        chart: 'system.cpu',
        context: 'system.cpu',
        ts: Date.now(),
        values: { user: 1 },
        tier: 1,
      },
    ])
    // Without secondary indexes: ~1 bee put for the collection row (+ tree nodes), not 3× index fan-out
    const delta = core.length - lengthBefore
    t.ok(delta > 0 && delta < 10, `core growth per point should be small, got ${delta}`)
  } finally {
    await model.close().catch(() => {})
    await store.close().catch(() => {})
    fs.rmSync(TMP, { recursive: true, force: true })
  }
})