import test from 'brittle' import fs from 'fs' import path from 'path' import { fileURLToPath } from 'url' import { defaultRetentionConfig, normalizeRetentionConfig, retentionCutoffMs, RETENTION_PRESETS, DISK_BUDGET_PRESETS, matchPointsPreset, HOT_PRESETS, } from '../shared/retention.js' import { formatBytes } from '../shared/format.js' import { MetricStore } from '../server/services/store.js' import { PearDataModel } from '../server/db/model.js' import { isWarmPersistableChart, registerChart } from '../shared/metrics.js' import { enqueueWarmPoint, warmFlushStats } from '../server/services/warm-flush.js' import Corestore from 'corestore' import { validateMethodArgs } from '../shared/schema.js' import { MethodRoles, Methods, Roles } from '../shared/protocol.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const TMP = path.join(__dirname, '..', 'tmp-retention-test') function dirSize(dir) { let total = 0 const stack = [dir] while (stack.length) { const cur = stack.pop() for (const ent of fs.readdirSync(cur, { withFileTypes: true })) { const full = path.join(cur, ent.name) if (ent.isDirectory()) stack.push(full) else total += fs.statSync(full).size || 0 } } return total } test('normalizeRetentionConfig applies presets', (t) => { const cfg = normalizeRetentionConfig({ warmRetentionPreset: '1m', tier0Points: 900, }) t.is(cfg.warmRetentionMs, RETENTION_PRESETS['1m'].ms) t.is(cfg.tier0Points, 900) t.is(matchPointsPreset(900, HOT_PRESETS), '15m') }) test('retentionCutoffMs respects forever / disabled', (t) => { const now = 1_700_000_000_000 t.is(retentionCutoffMs({ warmRetentionEnabled: false, warmRetentionMs: 1000 }, now), null) t.is(retentionCutoffMs({ warmRetentionEnabled: true, warmRetentionMs: 0 }, now), null) t.is( retentionCutoffMs({ warmRetentionEnabled: true, warmRetentionMs: 1000 }, now), now - 1000 ) }) test('formatBytes scales', (t) => { t.is(formatBytes(0), '0 B') t.is(formatBytes(1024), '1 KiB') t.is(formatBytes(1536 * 1024 * 1024), '1.5 GiB') }) test('MetricStore setRetention trims rings', (t) => { const store = new MetricStore() const ts = Date.now() for (let i = 0; i < 100; i++) { store.ingest([ { chart: 'system.cpu', context: 'system.cpu', ts: ts + i * 1000, values: { user: 1, idle: 99 }, }, ]) } t.ok(store.series.get('system.cpu').points.length >= 100) store.setRetention({ tier0Max: 10, tier1Max: 5, tier1Every: 60 }) t.is(store.tier0Max, 10) t.is(store.series.get('system.cpu').points.length, 10) const mem = store.memoryStats() t.is(mem.tier0Max, 10) t.ok(mem.tier0Points <= 10) }) test('protocol exposes data manager methods', (t) => { for (const m of ['getStorageInfo', 'getRetentionConfig', 'setRetentionConfig', 'pruneHistory']) { t.ok(MethodRoles[m], m) t.is(Methods[m], m) } t.is(MethodRoles.setRetentionConfig, Roles.admin) t.is(MethodRoles.pruneHistory, Roles.admin) t.ok(validateMethodArgs('getStorageInfo', {}).ok) t.ok(validateMethodArgs('setRetentionConfig', { tier0Points: 900 }).ok) t.ok(validateMethodArgs('pruneHistory', { dryRun: true }).ok) }) test('hyperdb deleteMetricPointsBefore removes old rows', async (t) => { fs.rmSync(TMP, { recursive: true, force: true }) const store = new Corestore(path.join(TMP, 'corestore')) await store.ready() const core = store.get({ name: 'peardata-meta' }) const model = new PearDataModel(core, { autoUpdate: true }) await model.ready() try { const now = Date.now() await model.putMetricPoints([ { chart: 'system.cpu', context: 'system.cpu', ts: now - 10_000, values: { user: 1 }, tier: 1, }, { chart: 'system.cpu', context: 'system.cpu', ts: now - 1000, values: { user: 2 }, tier: 1, }, { chart: 'system.cpu', context: 'system.cpu', ts: now, values: { user: 3 }, tier: 1, }, ]) const dry = await model.deleteMetricPointsBefore(now - 2000, { charts: ['system.cpu'], dryRun: true, }) t.is(dry.deleted, 1) const del = await model.deleteMetricPointsBefore(now - 2000, { charts: ['system.cpu'], dryRun: false, }) t.is(del.deleted, 1) const rows = await model.queryMetricPoints({ chart: 'system.cpu', afterMs: now - 20_000, beforeMs: now + 1000, }) t.is(rows.length, 2) t.is(rows[0].values.user, 2) } finally { await model.close().catch(() => {}) await store.close().catch(() => {}) fs.rmSync(TMP, { recursive: true, force: true }) } }) test('defaultRetentionConfig has sane defaults', (t) => { const d = defaultRetentionConfig() t.is(d.warmRetentionPreset, '3m') t.is(d.warmRetentionMs, RETENTION_PRESETS['3m'].ms) t.is(d.warmMaxBytes, DISK_BUDGET_PRESETS['1gb'].bytes) t.ok(d.autoPruneEnabled) t.ok(d.tier0Points > 0) }) test('isWarmPersistableChart gates high-cardinality instance charts', (t) => { t.ok(isWarmPersistableChart('system.cpu')) t.ok(isWarmPersistableChart('system.ram')) registerChart({ id: 'disk_io.sda', name: 'disk_io.sda', context: 'disk.io', title: 'sda', units: 'KiB/s', family: 'disk', chartType: 'line', priority: 1, dimensions: [{ id: 'read', name: 'read', algorithm: 'incremental' }], }) t.is(isWarmPersistableChart('disk_io.sda'), false) t.is(isWarmPersistableChart('docker.cpu.abc'), false) }) test('enqueueWarmPoint skips non-persistable charts', (t) => { const before = warmFlushStats().skippedHighCardinality enqueueWarmPoint({ chart: 'cgroup.cpu.foo', context: 'cgroup.cpu', ts: Date.now(), values: { usage: 1 }, }) t.ok(warmFlushStats().skippedHighCardinality > before) const pendingBefore = warmFlushStats().pending enqueueWarmPoint({ chart: 'system.cpu', context: 'system.cpu', ts: Date.now(), values: { user: 1 }, }) t.is(warmFlushStats().pending, pendingBefore + 1) }) test('reclaimStorage runs after delete (clearUnlinked + compact)', async (t) => { fs.rmSync(TMP, { recursive: true, force: true }) const storePath = path.join(TMP, 'corestore') const store = new Corestore(storePath) await store.ready() const core = store.get({ name: 'peardata-meta' }) const model = new PearDataModel(core, { autoUpdate: true }) await model.ready() try { const now = Date.now() const points = [] for (let i = 0; i < 400; i++) { points.push({ chart: 'system.cpu', context: 'system.cpu', ts: now - (400 - i) * 60_000, values: { user: i % 50, idle: 50 }, tier: 1, }) } await model.putMetricPoints(points) const sizeAfterInsert = dirSize(storePath) const del = await model.deleteMetricPointsBefore(now - 200 * 60_000, { charts: ['system.cpu'], dryRun: false, maxPasses: 20, }) t.ok(del.deleted >= 200, `deleted ${del.deleted}`) const sizeAfterDelete = dirSize(storePath) t.ok(sizeAfterDelete >= sizeAfterInsert, 'logical delete should not shrink (often grows)') const reclaim = await model.reclaimStorage({ batchSize: 512 }) t.ok(reclaim.cleared) t.ok(reclaim.compacted) t.ok(reclaim.clearMs >= 0) t.ok(reclaim.compactMs >= 0) const remaining = await model.queryMetricPoints({ chart: 'system.cpu', afterMs: now - 500 * 60_000, beforeMs: now + 1000, limit: 10_000, }) t.ok(remaining.length > 0 && remaining.length < 400) } finally { await model.close().catch(() => {}) await store.close().catch(() => {}) fs.rmSync(TMP, { recursive: true, force: true }) } }) test('deleteMetricPointsBefore multi-pass drains large backlog', async (t) => { fs.rmSync(TMP, { recursive: true, force: true }) const store = new Corestore(path.join(TMP, 'corestore')) await store.ready() const core = store.get({ name: 'peardata-meta' }) const model = new PearDataModel(core, { autoUpdate: true }) await model.ready() try { const now = Date.now() const batch = [] for (let i = 0; i < 50; i++) { batch.push({ chart: 'system.ram', context: 'system.ram', ts: now - (50 - i) * 1000, values: { free: i }, tier: 1, }) } await model.putMetricPoints(batch) const del = await model.deleteMetricPointsBefore(now + 1, { charts: ['system.ram'], dryRun: false, limitPerChart: 10, maxPasses: 20, }) t.is(del.deleted, 50) t.ok(del.passes >= 5) const left = await model.queryMetricPoints({ chart: 'system.ram', afterMs: 0, beforeMs: now + 10_000, }) t.is(left.length, 0) } finally { await model.close().catch(() => {}) await store.close().catch(() => {}) fs.rmSync(TMP, { recursive: true, force: true }) } }) test('metric-point secondary indexes are not written (deprecated)', async (t) => { fs.rmSync(TMP, { recursive: true, force: true }) const store = new Corestore(path.join(TMP, 'corestore')) await store.ready() const core = store.get({ name: 'peardata-meta' }) const model = new PearDataModel(core, { autoUpdate: true }) await model.ready() try { const lengthBefore = core.length await model.putMetricPoints([ { chart: 'system.cpu', context: 'system.cpu', ts: Date.now(), values: { user: 1 }, tier: 1, }, ]) // Without secondary indexes: ~1 bee put for the collection row (+ tree nodes), not 3× index fan-out const delta = core.length - lengthBefore t.ok(delta > 0 && delta < 10, `core growth per point should be small, got ${delta}`) } finally { await model.close().catch(() => {}) await store.close().catch(() => {}) fs.rmSync(TMP, { recursive: true, force: true }) } })