497 lines
14 KiB
JavaScript
497 lines
14 KiB
JavaScript
/**
|
|
* Agent data manager — retention policy, auto-prune, storage usage.
|
|
*
|
|
* Persists under $PEARDATA_DATA_DIR/retention.json (survives restart).
|
|
* Env PEARDATA_TIER* seeds defaults on first boot when no file exists.
|
|
*
|
|
* Prune path:
|
|
* 1. Logical HyperDB deletes (age / point cap / disk budget multi-pass)
|
|
* 2. Physical reclaim: hyperbee.clearUnlinked + hypercore.compact
|
|
* 3. Record real corestore bytes before/after
|
|
*/
|
|
import fs from 'fs'
|
|
import path from 'path'
|
|
import {
|
|
defaultRetentionConfig,
|
|
normalizeRetentionConfig,
|
|
retentionCutoffMs,
|
|
} from '../../shared/retention.js'
|
|
import { getDataDir, getDb, isHyperDbEnabled } from '../db/index.js'
|
|
import { getStore } from './store.js'
|
|
import { CHART_BY_ID } from '../../shared/metrics.js'
|
|
import logger from '../utils/logger.js'
|
|
|
|
const log = logger.child('retention')
|
|
|
|
/** Max outer loops when corestore is still over warmMaxBytes after a delete pass. */
|
|
const DISK_BUDGET_MAX_PASSES = 8
|
|
/** Max age-delete passes inside deleteMetricPointsBefore per cutoff. */
|
|
const AGE_DELETE_MAX_PASSES = 50
|
|
|
|
/** @type {import('../../shared/retention.js').RetentionConfig} */
|
|
let config = defaultRetentionConfig()
|
|
/** @type {ReturnType<typeof setInterval>|null} */
|
|
let pruneTimer = null
|
|
let pruneRunning = false
|
|
|
|
function envInt(name, fallback) {
|
|
const n = Number(process.env[name])
|
|
return Number.isFinite(n) && n > 0 ? n : fallback
|
|
}
|
|
|
|
function configPath() {
|
|
return path.join(getDataDir(), 'retention.json')
|
|
}
|
|
|
|
function corestorePath() {
|
|
return path.join(getDataDir(), 'corestore')
|
|
}
|
|
|
|
/**
|
|
* Parse optional env override. `0` means unlimited; unset uses default.
|
|
* @param {string} name
|
|
* @param {number} fallback
|
|
*/
|
|
function envBytesOrDefault(name, fallback) {
|
|
const raw = process.env[name]
|
|
if (raw == null || raw === '') return fallback
|
|
const n = Number(raw)
|
|
if (!Number.isFinite(n) || n < 0) return fallback
|
|
return Math.floor(n)
|
|
}
|
|
|
|
function seedFromEnv() {
|
|
const d = defaultRetentionConfig()
|
|
const warmMsEnv = process.env.PEARDATA_WARM_RETENTION_MS
|
|
const warmRetentionMs =
|
|
warmMsEnv != null ? envInt('PEARDATA_WARM_RETENTION_MS', d.warmRetentionMs) : d.warmRetentionMs
|
|
return normalizeRetentionConfig({
|
|
...d,
|
|
tier0Points: envInt('PEARDATA_TIER0_POINTS', d.tier0Points),
|
|
tier1Points: envInt('PEARDATA_TIER1_POINTS', d.tier1Points),
|
|
tier1Every: envInt('PEARDATA_TIER1_EVERY', d.tier1Every),
|
|
warmRetentionMs,
|
|
warmRetentionPreset: warmMsEnv != null ? 'custom' : d.warmRetentionPreset,
|
|
warmMaxBytes: envBytesOrDefault('PEARDATA_WARM_MAX_BYTES', d.warmMaxBytes),
|
|
warmMaxPoints: envBytesOrDefault('PEARDATA_WARM_MAX_POINTS', d.warmMaxPoints),
|
|
autoPruneEnabled: process.env.PEARDATA_AUTO_PRUNE === '0' ? false : d.autoPruneEnabled,
|
|
})
|
|
}
|
|
|
|
function save() {
|
|
try {
|
|
const dir = getDataDir()
|
|
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true, mode: 0o700 })
|
|
fs.writeFileSync(configPath(), JSON.stringify(config, null, 2), { mode: 0o600 })
|
|
} catch (err) {
|
|
log.warn('Failed to save retention config', { error: err.message })
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Load persisted config (or env seeds) and apply to MetricStore.
|
|
*/
|
|
export function loadRetentionConfig() {
|
|
try {
|
|
const p = configPath()
|
|
if (fs.existsSync(p)) {
|
|
const raw = JSON.parse(fs.readFileSync(p, 'utf8'))
|
|
config = normalizeRetentionConfig(raw)
|
|
log.info('Loaded retention config', {
|
|
tier0: config.tier0Points,
|
|
tier1: config.tier1Points,
|
|
warmPreset: config.warmRetentionPreset,
|
|
warmMaxBytes: config.warmMaxBytes,
|
|
autoPrune: config.autoPruneEnabled,
|
|
})
|
|
} else {
|
|
config = seedFromEnv()
|
|
save()
|
|
log.info('Seeded retention config from defaults/env', {
|
|
warmPreset: config.warmRetentionPreset,
|
|
warmMaxBytes: config.warmMaxBytes,
|
|
})
|
|
}
|
|
} catch (err) {
|
|
log.warn('Failed to load retention config; using defaults', { error: err.message })
|
|
config = seedFromEnv()
|
|
}
|
|
applyRetentionToStore()
|
|
return getRetentionConfig()
|
|
}
|
|
|
|
export function getRetentionConfig() {
|
|
return { ...config }
|
|
}
|
|
|
|
/**
|
|
* @param {Partial<import('../../shared/retention.js').RetentionConfig>} patch
|
|
*/
|
|
export function setRetentionConfig(patch = {}) {
|
|
const next = normalizeRetentionConfig({
|
|
...config,
|
|
...patch,
|
|
// preserve prune stats unless explicitly cleared
|
|
lastPruneAt: patch.lastPruneAt !== undefined ? patch.lastPruneAt : config.lastPruneAt,
|
|
lastPruneDeleted:
|
|
patch.lastPruneDeleted !== undefined ? patch.lastPruneDeleted : config.lastPruneDeleted,
|
|
lastPruneBytesFreed:
|
|
patch.lastPruneBytesFreed !== undefined
|
|
? patch.lastPruneBytesFreed
|
|
: config.lastPruneBytesFreed,
|
|
})
|
|
config = next
|
|
save()
|
|
applyRetentionToStore()
|
|
restartAutoPrune()
|
|
return getRetentionConfig()
|
|
}
|
|
|
|
export function applyRetentionToStore() {
|
|
try {
|
|
getStore().setRetention({
|
|
tier0Max: config.tier0Points,
|
|
tier1Max: config.tier1Points,
|
|
tier1Every: config.tier1Every,
|
|
})
|
|
} catch (err) {
|
|
log.warn('Failed to apply retention to store', { error: err.message })
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Recursive directory size (bytes).
|
|
* @param {string} dir
|
|
* @param {number} [budget] max entries walked
|
|
*/
|
|
export function directorySizeBytes(dir, budget = 250_000) {
|
|
let total = 0
|
|
let walked = 0
|
|
/** @type {string[]} */
|
|
const stack = [dir]
|
|
while (stack.length && walked < budget) {
|
|
const cur = stack.pop()
|
|
let entries
|
|
try {
|
|
entries = fs.readdirSync(cur, { withFileTypes: true })
|
|
} catch {
|
|
continue
|
|
}
|
|
for (const ent of entries) {
|
|
walked++
|
|
if (walked >= budget) break
|
|
const full = path.join(cur, ent.name)
|
|
try {
|
|
if (ent.isDirectory()) stack.push(full)
|
|
else if (ent.isFile() || ent.isSymbolicLink()) {
|
|
const st = fs.statSync(full)
|
|
total += st.size || 0
|
|
}
|
|
} catch {
|
|
// ignore races
|
|
}
|
|
}
|
|
}
|
|
return { bytes: total, truncated: walked >= budget, walked }
|
|
}
|
|
|
|
function measureCorestoreBytes() {
|
|
const p = corestorePath()
|
|
try {
|
|
if (fs.existsSync(p)) return directorySizeBytes(p).bytes
|
|
} catch {
|
|
// ignore
|
|
}
|
|
return 0
|
|
}
|
|
|
|
/**
|
|
* Snapshot of agent storage usage + effective retention.
|
|
*/
|
|
export async function getStorageInfo() {
|
|
const dataDir = getDataDir()
|
|
const csPath = corestorePath()
|
|
const mem = getStore().memoryStats()
|
|
|
|
let dataDirSize = { bytes: 0, truncated: false, walked: 0 }
|
|
let corestoreSize = { bytes: 0, truncated: false, walked: 0 }
|
|
try {
|
|
if (fs.existsSync(dataDir)) dataDirSize = directorySizeBytes(dataDir)
|
|
} catch {
|
|
// ignore
|
|
}
|
|
try {
|
|
if (fs.existsSync(csPath)) corestoreSize = directorySizeBytes(csPath)
|
|
} catch {
|
|
// ignore
|
|
}
|
|
|
|
const db = getDb()
|
|
let warm = {
|
|
enabled: isHyperDbEnabled(),
|
|
open: Boolean(db),
|
|
pointsSampled: 0,
|
|
chartsSampled: 0,
|
|
oldestTs: null,
|
|
newestTs: null,
|
|
}
|
|
if (db) {
|
|
try {
|
|
const sample = await db.sampleMetricPointsStats({ limit: 50_000 })
|
|
warm = { ...warm, ...sample }
|
|
} catch (err) {
|
|
warm.error = err.message
|
|
}
|
|
}
|
|
|
|
const otherFiles = []
|
|
for (const name of ['retention.json', 'peer-policy.json', 'audit.log']) {
|
|
const p = path.join(dataDir, name)
|
|
try {
|
|
if (fs.existsSync(p)) {
|
|
const st = fs.statSync(p)
|
|
otherFiles.push({ name, bytes: st.size })
|
|
}
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
|
|
return {
|
|
dataDir,
|
|
hyperDbEnabled: isHyperDbEnabled(),
|
|
usage: {
|
|
dataDirBytes: dataDirSize.bytes,
|
|
corestoreBytes: corestoreSize.bytes,
|
|
dataDirTruncated: dataDirSize.truncated,
|
|
corestoreTruncated: corestoreSize.truncated,
|
|
otherFiles,
|
|
},
|
|
memory: mem,
|
|
warm,
|
|
retention: getRetentionConfig(),
|
|
measuredAt: Date.now(),
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Resolve chart ids for age prune (catalog + live series + HyperDB sample).
|
|
* @param {import('../db/model.js').PearDataModel} db
|
|
*/
|
|
async function resolveChartIds(db) {
|
|
const chartIds = new Set([...CHART_BY_ID.keys(), ...getStore().series.keys()])
|
|
try {
|
|
const fromDb = await db.listMetricPointCharts({ limit: 100_000 })
|
|
for (const c of fromDb) chartIds.add(c)
|
|
} catch {
|
|
// ignore — fall back to catalog + memory
|
|
}
|
|
return [...chartIds]
|
|
}
|
|
|
|
/**
|
|
* Prune warm HyperDB history by age / max points / soft disk budget,
|
|
* then reclaim physical corestore space (clearUnlinked + compact).
|
|
*
|
|
* @param {{ dryRun?: boolean, beforeMs?: number|null, force?: boolean, gc?: boolean }} [opts]
|
|
*/
|
|
export async function pruneHistory(opts = {}) {
|
|
if (pruneRunning) return { success: false, error: 'prune already running' }
|
|
pruneRunning = true
|
|
const dryRun = Boolean(opts.dryRun)
|
|
/** Physical GC unless explicitly disabled (dry-run never GC's). */
|
|
const doGc = !dryRun && opts.gc !== false
|
|
const started = Date.now()
|
|
const bytesBefore = measureCorestoreBytes()
|
|
|
|
try {
|
|
const db = getDb()
|
|
if (!db) {
|
|
return {
|
|
success: true,
|
|
dryRun,
|
|
deleted: 0,
|
|
skipped: true,
|
|
reason: 'hyperdb disabled or not open',
|
|
bytesBefore,
|
|
bytesAfter: bytesBefore,
|
|
bytesFreed: 0,
|
|
elapsedMs: Date.now() - started,
|
|
}
|
|
}
|
|
|
|
let beforeMs =
|
|
opts.beforeMs != null ? Number(opts.beforeMs) : retentionCutoffMs(config, started)
|
|
|
|
const chartIds = await resolveChartIds(db)
|
|
let deleted = 0
|
|
let scanned = 0
|
|
let agePasses = 0
|
|
let capPasses = 0
|
|
let budgetPasses = 0
|
|
|
|
// ── 1. Age prune (multi-pass inside model) ─────────────────
|
|
if (beforeMs != null && Number.isFinite(beforeMs) && beforeMs > 0) {
|
|
const age = await db.deleteMetricPointsBefore(beforeMs, {
|
|
charts: chartIds,
|
|
dryRun,
|
|
limitPerChart: 20_000,
|
|
maxPasses: AGE_DELETE_MAX_PASSES,
|
|
})
|
|
deleted += age.deleted
|
|
scanned += age.scanned
|
|
agePasses += age.passes || 0
|
|
}
|
|
|
|
// ── 2. Cap total warm points ───────────────────────────────
|
|
if (config.warmMaxPoints > 0) {
|
|
const cap = await db.enforceMetricPointCap(config.warmMaxPoints, {
|
|
dryRun,
|
|
charts: chartIds,
|
|
maxPasses: 20,
|
|
})
|
|
deleted += cap.deleted
|
|
scanned += cap.scanned
|
|
capPasses += cap.passes || 0
|
|
}
|
|
|
|
// ── 3. Soft disk budget — multi-pass tighter age windows ───
|
|
if (config.warmMaxBytes > 0 && !dryRun) {
|
|
const retentionMs = config.warmRetentionMs || 90 * 24 * 60 * 60 * 1000
|
|
let cutoff =
|
|
beforeMs != null && Number.isFinite(beforeMs) && beforeMs > 0
|
|
? beforeMs
|
|
: started - Math.floor(retentionMs * 0.5)
|
|
|
|
for (let i = 0; i < DISK_BUDGET_MAX_PASSES; i++) {
|
|
const bytes = measureCorestoreBytes()
|
|
if (bytes <= config.warmMaxBytes) break
|
|
|
|
// Drop another 25% of retention window (or 1 day floor) each pass
|
|
const extraMs = Math.max(24 * 60 * 60 * 1000, Math.floor(retentionMs * 0.25))
|
|
cutoff = cutoff - extraMs
|
|
if (cutoff <= 0) break
|
|
|
|
const extra = await db.deleteMetricPointsBefore(cutoff, {
|
|
charts: chartIds,
|
|
dryRun: false,
|
|
limitPerChart: 20_000,
|
|
maxPasses: AGE_DELETE_MAX_PASSES,
|
|
})
|
|
deleted += extra.deleted
|
|
scanned += extra.scanned
|
|
budgetPasses++
|
|
agePasses += extra.passes || 0
|
|
|
|
if (extra.deleted === 0) {
|
|
// No more logical rows under this cutoff — still over budget?
|
|
// Physical GC may help; break logical loop.
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
// Memory rings already self-trim via setRetention; force trim
|
|
getStore().trimToRetention()
|
|
|
|
// ── 4. Physical reclaim (clearUnlinked + compact) ──────────
|
|
/** @type {null | Awaited<ReturnType<import('../db/model.js').PearDataModel['reclaimStorage']>>} */
|
|
let reclaim = null
|
|
if (doGc && (deleted > 0 || opts.force || config.warmMaxBytes > 0)) {
|
|
try {
|
|
reclaim = await db.reclaimStorage({ batchSize: 4096 })
|
|
log.info('Corestore reclaim finished', reclaim)
|
|
} catch (err) {
|
|
log.warn('Corestore reclaim failed', { error: err.message })
|
|
reclaim = { error: err.message, cleared: false, compacted: false }
|
|
}
|
|
}
|
|
|
|
const bytesAfter = measureCorestoreBytes()
|
|
// clearUnlinked can temporarily expand; report signed delta + non-negative "freed"
|
|
const bytesDelta = bytesAfter - bytesBefore
|
|
const bytesFreed = Math.max(0, -bytesDelta)
|
|
|
|
if (!dryRun && (deleted > 0 || reclaim?.cleared || reclaim?.compacted)) {
|
|
config = normalizeRetentionConfig({
|
|
...config,
|
|
lastPruneAt: started,
|
|
lastPruneDeleted: deleted,
|
|
lastPruneBytesFreed: bytesFreed,
|
|
})
|
|
save()
|
|
}
|
|
|
|
log.info(dryRun ? 'Dry-run prune' : 'Pruned warm history', {
|
|
deleted,
|
|
scanned,
|
|
beforeMs,
|
|
bytesBefore,
|
|
bytesAfter,
|
|
bytesFreed,
|
|
bytesDelta,
|
|
agePasses,
|
|
budgetPasses,
|
|
gc: Boolean(reclaim && !reclaim.error),
|
|
})
|
|
|
|
return {
|
|
success: true,
|
|
dryRun,
|
|
deleted,
|
|
scanned,
|
|
beforeMs: beforeMs ?? null,
|
|
agePasses,
|
|
capPasses,
|
|
budgetPasses,
|
|
/** @deprecated use bytesFreed — kept for older UI */
|
|
approxBytesFreed: bytesFreed,
|
|
bytesBefore,
|
|
bytesAfter,
|
|
bytesFreed,
|
|
bytesDelta,
|
|
reclaim,
|
|
elapsedMs: Date.now() - started,
|
|
retention: getRetentionConfig(),
|
|
}
|
|
} catch (err) {
|
|
log.warn('Prune failed', { error: err.message })
|
|
return {
|
|
success: false,
|
|
error: err.message,
|
|
dryRun,
|
|
bytesBefore,
|
|
bytesAfter: measureCorestoreBytes(),
|
|
elapsedMs: Date.now() - started,
|
|
}
|
|
} finally {
|
|
pruneRunning = false
|
|
}
|
|
}
|
|
|
|
export function startAutoPrune() {
|
|
stopAutoPrune()
|
|
if (!config.autoPruneEnabled) return
|
|
const interval = config.autoPruneIntervalMs || 60 * 60 * 1000
|
|
pruneTimer = setInterval(() => {
|
|
pruneHistory({ force: true }).catch(() => {})
|
|
}, interval)
|
|
if (typeof pruneTimer.unref === 'function') pruneTimer.unref()
|
|
// Kick once shortly after boot (let HyperDB settle)
|
|
setTimeout(() => {
|
|
pruneHistory({ force: true }).catch(() => {})
|
|
}, 30_000).unref?.()
|
|
log.info('Auto-prune scheduled', { intervalMs: interval })
|
|
}
|
|
|
|
export function stopAutoPrune() {
|
|
if (pruneTimer) {
|
|
clearInterval(pruneTimer)
|
|
pruneTimer = null
|
|
}
|
|
}
|
|
|
|
function restartAutoPrune() {
|
|
startAutoPrune()
|
|
}
|