437 lines
13 KiB
JavaScript
437 lines
13 KiB
JavaScript
/**
|
|
* Threshold-based anomaly detection (+ optional z-score hybrid).
|
|
*
|
|
* PEARDATA_ANOMALY_MODE:
|
|
* threshold (default) — fixed warn/crit
|
|
* zscore — fire when |z| exceeds warnZ/critZ
|
|
* hybrid — threshold OR z-score
|
|
*/
|
|
import { EventEmitter } from 'events'
|
|
import { normalizeAnomaly } from '../../shared/data-model.js'
|
|
import { stats, zScore, thresholdsFromStats, seriesFromStore } from './zscore.js'
|
|
|
|
/** @typedef {import('../../shared/data-model.js').AlertConfig} AlertConfig */
|
|
/** @typedef {import('../../shared/data-model.js').AnomalyEvent} AnomalyEvent */
|
|
|
|
/** Default thresholds — tunable via setAlertConfig / REST. */
|
|
export const DEFAULT_THRESHOLDS = [
|
|
{
|
|
id: 'cpu_user_high',
|
|
chart: 'system.cpu',
|
|
dimension: 'user',
|
|
warn: 80,
|
|
crit: 95,
|
|
comparator: '>',
|
|
enabled: true,
|
|
info: 'CPU user time high',
|
|
},
|
|
{
|
|
id: 'cpu_iowait_high',
|
|
chart: 'system.cpu',
|
|
dimension: 'iowait',
|
|
warn: 40,
|
|
crit: 70,
|
|
comparator: '>',
|
|
enabled: true,
|
|
info: 'CPU iowait high',
|
|
},
|
|
{
|
|
id: 'cpu_steal_high',
|
|
chart: 'system.cpu',
|
|
dimension: 'steal',
|
|
warn: 10,
|
|
crit: 25,
|
|
comparator: '>',
|
|
enabled: true,
|
|
info: 'CPU steal time high (hypervisor contention)',
|
|
},
|
|
{
|
|
id: 'load1_high',
|
|
chart: 'system.load',
|
|
dimension: 'load1',
|
|
warn: null, // set dynamically from cpu count in evaluate
|
|
crit: null,
|
|
comparator: '>',
|
|
enabled: true,
|
|
info: 'Load average high vs CPU count',
|
|
_dynamicLoad: true,
|
|
},
|
|
{
|
|
id: 'mem_avail_low',
|
|
chart: 'mem.available',
|
|
dimension: 'avail',
|
|
warn: 512,
|
|
crit: 256,
|
|
comparator: '<',
|
|
enabled: true,
|
|
info: 'Available memory low (MiB)',
|
|
},
|
|
]
|
|
|
|
function compare(op, value, threshold) {
|
|
if (threshold == null || value == null || Number.isNaN(value)) return false
|
|
switch (op) {
|
|
case '<':
|
|
return value < threshold
|
|
case '<=':
|
|
return value <= threshold
|
|
case '>=':
|
|
return value >= threshold
|
|
case '>':
|
|
default:
|
|
return value > threshold
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Continuous score in [0, 1] from how far value sits past warn→crit.
|
|
* @param {number} value
|
|
* @param {number|null} warn
|
|
* @param {number|null} crit
|
|
* @param {string} op
|
|
* @param {'warning'|'critical'} severity
|
|
*/
|
|
export function scoreSeverity(value, warn, crit, op, severity) {
|
|
if (severity === 'critical') {
|
|
if (warn == null || crit == null || warn === crit) return 1
|
|
const span = Math.abs(crit - warn) || 1
|
|
if (op === '<' || op === '<=') {
|
|
// lower is worse: crit < warn
|
|
const over = Math.max(0, warn - value)
|
|
return Math.min(1, 0.6 + 0.4 * Math.min(1, over / span))
|
|
}
|
|
const over = Math.max(0, value - warn)
|
|
return Math.min(1, 0.6 + 0.4 * Math.min(1, over / span))
|
|
}
|
|
// warning band
|
|
if (warn == null) return 0.6
|
|
if (crit == null) return 0.6
|
|
const span = Math.abs(crit - warn) || 1
|
|
if (op === '<' || op === '<=') {
|
|
const into = Math.max(0, warn - value)
|
|
return Math.min(0.99, 0.35 + 0.25 * Math.min(1, into / span))
|
|
}
|
|
const into = Math.max(0, value - warn)
|
|
return Math.min(0.99, 0.35 + 0.25 * Math.min(1, into / span))
|
|
}
|
|
|
|
export class AnomalyEngine extends EventEmitter {
|
|
/**
|
|
* @param {{ cpuCount?: number }} [opts]
|
|
*/
|
|
constructor(opts = {}) {
|
|
super()
|
|
this.cpuCount = opts.cpuCount || 1
|
|
/** @type {Map<string, AlertConfig & { _dynamicLoad?: boolean, _mean?: number, _stdev?: number }>} */
|
|
this.configs = new Map(DEFAULT_THRESHOLDS.map((c) => [c.id, { ...c }]))
|
|
/** @type {Map<string, string>} status CLEAR|WARNING|CRITICAL */
|
|
this.status = new Map()
|
|
/** @type {AnomalyEvent[]} */
|
|
this.recent = []
|
|
this.recentMax = 500
|
|
/** @type {Map<string, number[]>} rolling values for z-score */
|
|
this._windows = new Map()
|
|
this.windowMax = Number(process.env.PEARDATA_ANOMALY_WINDOW) || 120
|
|
this.warnZ = Number(process.env.PEARDATA_ANOMALY_WARN_Z) || 2
|
|
this.critZ = Number(process.env.PEARDATA_ANOMALY_CRIT_Z) || 3
|
|
}
|
|
|
|
anomalyMode() {
|
|
const m = String(process.env.PEARDATA_ANOMALY_MODE || 'threshold').toLowerCase()
|
|
if (m === 'zscore' || m === 'hybrid') return m
|
|
return 'threshold'
|
|
}
|
|
|
|
/**
|
|
* Push a sample into the rolling window for a config.
|
|
* @param {string} cfgId
|
|
* @param {number} value
|
|
*/
|
|
_pushWindow(cfgId, value) {
|
|
let arr = this._windows.get(cfgId)
|
|
if (!arr) {
|
|
arr = []
|
|
this._windows.set(cfgId, arr)
|
|
}
|
|
arr.push(value)
|
|
while (arr.length > this.windowMax) arr.shift()
|
|
}
|
|
|
|
/**
|
|
* Retrain warn/crit from MetricStore history (or rolling windows).
|
|
* @param {import('./store.js').MetricStore} [store]
|
|
* @param {{ warnZ?: number, critZ?: number, minPoints?: number }} [opts]
|
|
*/
|
|
retrain(store = null, opts = {}) {
|
|
const warnZ = opts.warnZ ?? this.warnZ
|
|
const critZ = opts.critZ ?? this.critZ
|
|
const minPoints = opts.minPoints ?? 30
|
|
/** @type {Array<{ id: string, n: number, mean: number, stdev: number, warn: number, crit: number }>} */
|
|
const updated = []
|
|
|
|
for (const cfg of this.configs.values()) {
|
|
if (cfg._dynamicLoad) continue
|
|
let values = []
|
|
if (store) {
|
|
values = seriesFromStore(store, cfg.chart, cfg.dimension, this.windowMax)
|
|
}
|
|
if (values.length < minPoints) {
|
|
values = this._windows.get(cfg.id) || values
|
|
}
|
|
if (values.length < minPoints) continue
|
|
|
|
const s = stats(values)
|
|
const thr = thresholdsFromStats(s, cfg.comparator || '>', { warnZ, critZ })
|
|
this.setConfig({
|
|
...cfg,
|
|
warn: thr.warn,
|
|
crit: thr.crit,
|
|
_mean: thr.mean,
|
|
_stdev: thr.stdev,
|
|
info: cfg.info,
|
|
})
|
|
updated.push({
|
|
id: cfg.id,
|
|
n: s.n,
|
|
mean: thr.mean,
|
|
stdev: thr.stdev,
|
|
warn: thr.warn,
|
|
crit: thr.crit,
|
|
})
|
|
}
|
|
return { ok: true, mode: this.anomalyMode(), updated, warnZ, critZ }
|
|
}
|
|
|
|
/**
|
|
* @param {AlertConfig} cfg
|
|
*/
|
|
setConfig(cfg) {
|
|
const prev = this.configs.get(cfg.id) || {}
|
|
this.configs.set(cfg.id, { ...prev, ...cfg, id: cfg.id })
|
|
return this.configs.get(cfg.id)
|
|
}
|
|
|
|
listConfigs() {
|
|
return [...this.configs.values()]
|
|
}
|
|
|
|
listRecent(limit = 50) {
|
|
return this.recent.slice(-limit).reverse()
|
|
}
|
|
|
|
/**
|
|
* @param {Array<{ chart: string, context: string, ts: number, values: Record<string, number|null> }>} batch
|
|
*/
|
|
evaluate(batch) {
|
|
/** @type {AnomalyEvent[]} */
|
|
const fired = []
|
|
if (!this.configs.size) return fired
|
|
const byChart = new Map(batch.map((s) => [s.chart, s]))
|
|
|
|
for (const id of [...this.configs.keys()]) {
|
|
let cfg = this.configs.get(id)
|
|
// Auto-reenable after silence TTL
|
|
if (cfg.enabled === false && cfg._silencedUntil) {
|
|
if (Date.now() >= Number(cfg._silencedUntil)) {
|
|
this.setConfig({ ...cfg, enabled: true, _silencedUntil: null })
|
|
cfg = this.configs.get(id)
|
|
} else {
|
|
continue
|
|
}
|
|
}
|
|
if (cfg.enabled === false) continue
|
|
const sample = byChart.get(cfg.chart)
|
|
if (!sample) continue
|
|
const value = sample.values[cfg.dimension]
|
|
if (value == null) continue
|
|
|
|
this._pushWindow(cfg.id, value)
|
|
|
|
let warn = cfg.warn
|
|
let crit = cfg.crit
|
|
if (cfg._dynamicLoad) {
|
|
warn = this.cpuCount * 1.5
|
|
crit = this.cpuCount * 3
|
|
}
|
|
|
|
const op = cfg.comparator || '>'
|
|
const mode = this.anomalyMode()
|
|
let severity = null
|
|
let threshold = null
|
|
let z = null
|
|
|
|
if (mode === 'threshold' || mode === 'hybrid') {
|
|
if (compare(op, value, crit)) {
|
|
severity = 'critical'
|
|
threshold = crit
|
|
} else if (compare(op, value, warn)) {
|
|
severity = 'warning'
|
|
threshold = warn
|
|
}
|
|
}
|
|
|
|
if (mode === 'zscore' || mode === 'hybrid') {
|
|
const win = this._windows.get(cfg.id) || []
|
|
if (win.length >= 15) {
|
|
const s = stats(win.slice(0, -1)) // exclude current for baseline
|
|
z = zScore(value, s.mean, s.stdev)
|
|
const absZ = Math.abs(z)
|
|
let zSev = null
|
|
if (absZ >= this.critZ) zSev = 'critical'
|
|
else if (absZ >= this.warnZ) zSev = 'warning'
|
|
if (zSev) {
|
|
const rank = { warning: 1, critical: 2 }
|
|
if (!severity || rank[zSev] > rank[severity]) {
|
|
severity = zSev
|
|
threshold = zSev === 'critical' ? this.critZ : this.warnZ
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
const prev = this.status.get(cfg.id) || 'CLEAR'
|
|
const next = severity === 'critical' ? 'CRITICAL' : severity === 'warning' ? 'WARNING' : 'CLEAR'
|
|
|
|
if (next !== prev) {
|
|
this.status.set(cfg.id, next)
|
|
if (next === 'CLEAR') {
|
|
const cleared = normalizeAnomaly({
|
|
id: `${cfg.id}:${sample.ts}`,
|
|
chart: cfg.chart,
|
|
context: sample.context,
|
|
dimension: cfg.dimension,
|
|
severity: prev === 'CRITICAL' ? 'critical' : 'warning',
|
|
score: 0,
|
|
value,
|
|
threshold: threshold ?? warn ?? crit ?? 0,
|
|
comparator: op,
|
|
message: `${cfg.info || cfg.id} cleared`,
|
|
ts: sample.ts,
|
|
cleared: true,
|
|
})
|
|
this._push(cleared)
|
|
fired.push(cleared)
|
|
} else if (severity) {
|
|
const score =
|
|
z != null
|
|
? Math.min(1, Math.abs(z) / Math.max(this.critZ, 1))
|
|
: scoreSeverity(value, warn, crit, op, severity)
|
|
const zPart = z != null ? ` z=${round2(z)}` : ''
|
|
const ev = normalizeAnomaly({
|
|
id: `${cfg.id}:${sample.ts}`,
|
|
chart: cfg.chart,
|
|
context: sample.context,
|
|
dimension: cfg.dimension,
|
|
severity,
|
|
score,
|
|
value,
|
|
threshold,
|
|
comparator: op,
|
|
message: `${cfg.info || cfg.id}: ${cfg.dimension}=${round2(value)} ${op} ${threshold}${zPart} (score ${score.toFixed(2)})`,
|
|
ts: sample.ts,
|
|
})
|
|
this._push(ev)
|
|
fired.push(ev)
|
|
this.emit('anomaly', ev)
|
|
}
|
|
}
|
|
}
|
|
return fired
|
|
}
|
|
|
|
/** @param {AnomalyEvent} ev */
|
|
_push(ev) {
|
|
this.recent.push(ev)
|
|
if (this.recent.length > this.recentMax) {
|
|
this.recent.splice(0, this.recent.length - this.recentMax)
|
|
}
|
|
}
|
|
|
|
getHealth() {
|
|
let critical = 0
|
|
let warning = 0
|
|
const checks = []
|
|
for (const [id, st] of this.status) {
|
|
const cfg = this.configs.get(id)
|
|
checks.push({
|
|
id,
|
|
ok: st === 'CLEAR',
|
|
detail: `${cfg?.info || id}: ${st}`,
|
|
})
|
|
if (st === 'CRITICAL') critical++
|
|
else if (st === 'WARNING') warning++
|
|
}
|
|
const status = critical ? 'critical' : warning ? 'degraded' : 'ok'
|
|
const score = critical ? 0.2 : warning ? 0.7 : 1
|
|
return { status, score, checks, ts: Date.now() }
|
|
}
|
|
|
|
/**
|
|
* Per-chart investigation weights for related-metrics / UI ranking.
|
|
* Higher weight = more anomalous / interesting right now.
|
|
* @param {{ chart?: string, limit?: number }} [opts]
|
|
*/
|
|
getWeights(opts = {}) {
|
|
/** @type {Map<string, { id: string, weight: number, severity: string|null, score: number, info: string }>} */
|
|
const byChart = new Map()
|
|
|
|
const bump = (chart, weight, severity, score, info) => {
|
|
if (!chart) return
|
|
const prev = byChart.get(chart)
|
|
if (!prev || weight > prev.weight) {
|
|
byChart.set(chart, {
|
|
id: chart,
|
|
weight,
|
|
severity: severity || null,
|
|
score: score ?? weight,
|
|
info: info || chart,
|
|
})
|
|
}
|
|
}
|
|
|
|
for (const [cfgId, st] of this.status) {
|
|
const cfg = this.configs.get(cfgId)
|
|
if (!cfg?.chart) continue
|
|
if (st === 'CRITICAL') bump(cfg.chart, 1, 'critical', 1, cfg.info || cfgId)
|
|
else if (st === 'WARNING') bump(cfg.chart, 0.65, 'warning', 0.65, cfg.info || cfgId)
|
|
else bump(cfg.chart, 0, null, 0, cfg.info || cfgId)
|
|
}
|
|
|
|
for (const ev of this.recent.slice(-80)) {
|
|
if (ev.cleared || !ev.chart) continue
|
|
const w =
|
|
ev.severity === 'critical'
|
|
? Math.max(0.85, Number(ev.score) || 0.85)
|
|
: Math.max(0.45, Number(ev.score) || 0.45)
|
|
bump(ev.chart, w, ev.severity || null, Number(ev.score) || w, ev.message || ev.chart)
|
|
}
|
|
|
|
let results = [...byChart.values()].sort((a, b) => b.weight - a.weight || a.id.localeCompare(b.id))
|
|
if (opts.chart) {
|
|
const seed = String(opts.chart)
|
|
results = results.filter((r) => r.id === seed || r.weight > 0)
|
|
}
|
|
const limit = Math.max(1, Math.min(500, Number(opts.limit) || 100))
|
|
const health = this.getHealth()
|
|
return {
|
|
status: health.status,
|
|
score: health.score,
|
|
results: results.slice(0, limit),
|
|
ts: Date.now(),
|
|
}
|
|
}
|
|
}
|
|
|
|
function round2(n) {
|
|
return Math.round(Number(n) * 100) / 100
|
|
}
|
|
|
|
/** @type {AnomalyEngine|null} */
|
|
let singleton = null
|
|
|
|
export function getAnomalyEngine(cpuCount) {
|
|
if (!singleton) singleton = new AnomalyEngine({ cpuCount })
|
|
else if (cpuCount) singleton.cpuCount = cpuCount
|
|
return singleton
|
|
}
|