Raven Scott
c5015a66e8
Release rolling / release (push) Successful in 8m4s
1. Removed docker.df() entirely 2. Poll only what rules need • docker_daemon → cheap ping • container_health / stack_health → listContainers • resource → host statfs only (no Docker) • Event-only rules → no poll timer 3. Poll floor 60s, default 120s (was 15s min / 60s default) 4. No overlapping polls (pollInFlight) 5. Event path: drop noisy actions (exec_*, attach, …), match only relevant rules, serialize work (bounded queue) 6. UI status shows whether poll is active
1602 lines
47 KiB
JavaScript
1602 lines
47 KiB
JavaScript
/**
|
||
* PearDock server alerting engine (experimental).
|
||
*
|
||
* Fully customizable rules + webhook channels (Discord, Slack, Teams,
|
||
* generic HTTP, ntfy, Gotify, Telegram). Evaluates Docker events and
|
||
* light periodic health polls even when no client is connected.
|
||
*
|
||
* Host-load rules (keep this cheap on dockerd):
|
||
* - Never call docker.df() (expensive, unused).
|
||
* - Poll only kinds that have enabled rules (skip listContainers when unused).
|
||
* - Min poll interval 60s (default 120s); never overlap polls.
|
||
* - Docker event path filters noise (exec_*, attach, …) and serializes work.
|
||
* - Event-only rules need no Docker polling.
|
||
*
|
||
* Config (mode 0600), first match wins:
|
||
* PEARDOCK_ALERTS_PATH
|
||
* $PEARDOCK_HOME/.config/peardock/cache/alerts.json
|
||
* ~/.config/peardock/cache/alerts.json
|
||
* ./peardock-alerts.json (legacy cwd; migrated once into cache path)
|
||
*/
|
||
|
||
import fs from 'fs'
|
||
import path from 'path'
|
||
import os from 'os'
|
||
import { randomBytes } from 'crypto'
|
||
import { docker } from './docker.js'
|
||
import { peers } from '../core/peer-registry.js'
|
||
import { Pushes } from '../../shared/protocol.js'
|
||
import logger from '../utils/logger.js'
|
||
|
||
/**
|
||
* Resolve durable alerts config path (cache-aligned with client peers/settings).
|
||
* @returns {string}
|
||
*/
|
||
function resolveAlertsFilePath() {
|
||
if (process.env.PEARDOCK_ALERTS_PATH) {
|
||
return process.env.PEARDOCK_ALERTS_PATH
|
||
}
|
||
const home =
|
||
process.env.PEARDOCK_HOME ||
|
||
process.env.HOME ||
|
||
process.env.USERPROFILE ||
|
||
(typeof os.homedir === 'function' ? os.homedir() : '') ||
|
||
''
|
||
if (home) {
|
||
return path.join(home, '.config', 'peardock', 'cache', 'alerts.json')
|
||
}
|
||
return path.join(process.cwd(), 'peardock-alerts.json')
|
||
}
|
||
|
||
const FILE = resolveAlertsFilePath()
|
||
const LEGACY_CWD_FILE = path.join(process.cwd(), 'peardock-alerts.json')
|
||
|
||
const HISTORY_MAX = 200
|
||
/** Default health poll — 2 minutes (was 60s). */
|
||
const DEFAULT_POLL_MS = 120_000
|
||
/** Hard floor — sub-minute listContainers/ping loops thrash dockerd. */
|
||
const MIN_POLL_MS = 60_000
|
||
const MAX_POLL_MS = 3_600_000
|
||
/** Drop oldest under event flood rather than unbounded memory. */
|
||
const EVENT_QUEUE_MAX = 200
|
||
const SEVERITIES = ['info', 'warning', 'critical']
|
||
const SEV_RANK = { info: 1, warning: 2, critical: 3 }
|
||
|
||
/**
|
||
* Docker event actions that never match default peardock alert rules and are
|
||
* extremely chatty (terminals, attach, top). Skip before evaluateRules.
|
||
*/
|
||
const NOISY_DOCKER_ACTIONS = new Set([
|
||
'exec_create',
|
||
'exec_start',
|
||
'exec_die',
|
||
'exec_detach',
|
||
'attach',
|
||
'detach',
|
||
'resize',
|
||
'top',
|
||
'export',
|
||
'commit',
|
||
'copy',
|
||
'archive-path',
|
||
'extract-to-dir',
|
||
])
|
||
|
||
const CHANNEL_TYPES = new Set([
|
||
'discord',
|
||
'slack',
|
||
'teams',
|
||
'generic',
|
||
'ntfy',
|
||
'gotify',
|
||
'telegram',
|
||
])
|
||
|
||
const RULE_KINDS = new Set([
|
||
'docker_event',
|
||
'container_health',
|
||
'container_state',
|
||
'docker_daemon',
|
||
'stack_health',
|
||
'resource',
|
||
'peardock',
|
||
])
|
||
|
||
/** @type {ReturnType<typeof defaultConfig>} */
|
||
let config = defaultConfig()
|
||
/** @type {Array<object>} */
|
||
const history = []
|
||
/** @type {Map<string, number>} ruleKey → last fire ms */
|
||
const cooldowns = new Map()
|
||
/** @type {Map<string, string>} entity → last known state (health/daemon/stack) */
|
||
const stateCache = new Map()
|
||
/** @type {NodeJS.Timeout|null} */
|
||
let pollTimer = null
|
||
/** @type {NodeJS.Timeout|null} */
|
||
let saveTimer = null
|
||
let deliveriesThisMinute = 0
|
||
let deliveriesMinuteStart = Date.now()
|
||
let lastDaemonOk = true
|
||
let started = false
|
||
/** Prevent overlapping pollHealth (setInterval + slow listContainers). */
|
||
let pollInFlight = false
|
||
/** Serialize docker-event alert evaluation under noisy fleets. */
|
||
let eventQueue = []
|
||
let eventDraining = false
|
||
|
||
function newId(prefix) {
|
||
return `${prefix}_${Date.now().toString(36)}_${randomBytes(3).toString('hex')}`
|
||
}
|
||
|
||
function defaultChannels() {
|
||
return []
|
||
}
|
||
|
||
function defaultRules() {
|
||
return [
|
||
{
|
||
id: 'rule_container_die',
|
||
name: 'Container died / OOM / killed',
|
||
enabled: true,
|
||
severity: 'critical',
|
||
kind: 'docker_event',
|
||
match: {
|
||
types: ['container'],
|
||
actions: ['die', 'oom', 'kill'],
|
||
},
|
||
channelIds: null,
|
||
cooldownSeconds: 120,
|
||
notifyOnRecover: false,
|
||
},
|
||
{
|
||
id: 'rule_container_unhealthy',
|
||
name: 'Container healthcheck unhealthy',
|
||
enabled: true,
|
||
severity: 'warning',
|
||
kind: 'container_health',
|
||
match: { healthStatus: ['unhealthy'] },
|
||
channelIds: null,
|
||
cooldownSeconds: 300,
|
||
notifyOnRecover: true,
|
||
},
|
||
{
|
||
id: 'rule_container_stop',
|
||
name: 'Unexpected container stop',
|
||
enabled: false,
|
||
severity: 'warning',
|
||
kind: 'docker_event',
|
||
match: {
|
||
types: ['container'],
|
||
actions: ['stop', 'die'],
|
||
},
|
||
channelIds: null,
|
||
cooldownSeconds: 180,
|
||
notifyOnRecover: false,
|
||
},
|
||
{
|
||
id: 'rule_docker_daemon',
|
||
name: 'Docker daemon unreachable',
|
||
enabled: true,
|
||
severity: 'critical',
|
||
kind: 'docker_daemon',
|
||
match: {},
|
||
channelIds: null,
|
||
cooldownSeconds: 120,
|
||
notifyOnRecover: true,
|
||
},
|
||
{
|
||
id: 'rule_stack_degraded',
|
||
name: 'Compose stack degraded',
|
||
enabled: true,
|
||
severity: 'warning',
|
||
kind: 'stack_health',
|
||
match: {},
|
||
channelIds: null,
|
||
cooldownSeconds: 300,
|
||
notifyOnRecover: true,
|
||
},
|
||
{
|
||
id: 'rule_disk_high',
|
||
name: 'Host disk usage high',
|
||
enabled: false,
|
||
severity: 'warning',
|
||
kind: 'resource',
|
||
match: { diskPercent: 90 },
|
||
channelIds: null,
|
||
cooldownSeconds: 1800,
|
||
notifyOnRecover: true,
|
||
},
|
||
{
|
||
id: 'rule_restart_loop',
|
||
name: 'Container restart',
|
||
enabled: true,
|
||
severity: 'warning',
|
||
kind: 'docker_event',
|
||
match: {
|
||
types: ['container'],
|
||
actions: ['restart'],
|
||
},
|
||
channelIds: null,
|
||
cooldownSeconds: 300,
|
||
notifyOnRecover: false,
|
||
},
|
||
]
|
||
}
|
||
|
||
function defaultConfig() {
|
||
return {
|
||
version: 1,
|
||
enabled: true,
|
||
pollIntervalMs: DEFAULT_POLL_MS,
|
||
minSeverity: 'info',
|
||
rateLimitPerMinute: 40,
|
||
quietHours: {
|
||
enabled: false,
|
||
start: '22:00',
|
||
end: '07:00',
|
||
timezone: 'UTC',
|
||
},
|
||
includeHostname: true,
|
||
channels: defaultChannels(),
|
||
rules: defaultRules(),
|
||
updatedAt: null,
|
||
}
|
||
}
|
||
|
||
function sanitizeString(v, max = 500) {
|
||
return String(v ?? '')
|
||
.trim()
|
||
.slice(0, max)
|
||
}
|
||
|
||
function normalizeSeverity(s, fallback = 'info') {
|
||
const v = String(s || '').toLowerCase()
|
||
return SEVERITIES.includes(v) ? v : fallback
|
||
}
|
||
|
||
function severityAtLeast(sev, min) {
|
||
return (SEV_RANK[sev] || 0) >= (SEV_RANK[min] || 0)
|
||
}
|
||
|
||
/**
|
||
* @param {unknown} raw
|
||
* @returns {object}
|
||
*/
|
||
function normalizeChannel(raw) {
|
||
const c = raw && typeof raw === 'object' ? raw : {}
|
||
const type = CHANNEL_TYPES.has(c.type) ? c.type : 'generic'
|
||
return {
|
||
id:
|
||
c.id && String(c.id).startsWith('ch_')
|
||
? String(c.id)
|
||
: newId('ch'),
|
||
name: sanitizeString(c.name || type, 80) || type,
|
||
type,
|
||
enabled: c.enabled !== false,
|
||
url: sanitizeString(c.url || '', 2000),
|
||
token: sanitizeString(c.token || '', 500),
|
||
topic: sanitizeString(c.topic || '', 200),
|
||
botToken: sanitizeString(c.botToken || '', 200),
|
||
chatId: sanitizeString(c.chatId || '', 100),
|
||
username: sanitizeString(c.username || 'PearDock', 80),
|
||
avatarUrl: sanitizeString(c.avatarUrl || '', 500),
|
||
minSeverity: normalizeSeverity(c.minSeverity, 'info'),
|
||
headers:
|
||
c.headers && typeof c.headers === 'object' && !Array.isArray(c.headers)
|
||
? Object.fromEntries(
|
||
Object.entries(c.headers)
|
||
.slice(0, 20)
|
||
.map(([k, v]) => [String(k).slice(0, 64), String(v).slice(0, 500)])
|
||
)
|
||
: {},
|
||
createdAt: c.createdAt || new Date().toISOString(),
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @param {unknown} raw
|
||
* @returns {object}
|
||
*/
|
||
function normalizeRule(raw) {
|
||
const r = raw && typeof raw === 'object' ? raw : {}
|
||
const kind = RULE_KINDS.has(r.kind) ? r.kind : 'docker_event'
|
||
const match =
|
||
r.match && typeof r.match === 'object' && !Array.isArray(r.match) ? { ...r.match } : {}
|
||
// Normalize match arrays
|
||
for (const key of ['types', 'actions', 'healthStatus', 'states', 'nameInclude', 'nameExclude']) {
|
||
if (match[key] != null && !Array.isArray(match[key])) {
|
||
match[key] = [match[key]].filter(Boolean).map(String)
|
||
} else if (Array.isArray(match[key])) {
|
||
match[key] = match[key].map(String).slice(0, 50)
|
||
}
|
||
}
|
||
if (match.nameRegex != null) match.nameRegex = sanitizeString(match.nameRegex, 200)
|
||
if (match.diskPercent != null) {
|
||
const n = Number(match.diskPercent)
|
||
match.diskPercent = Number.isFinite(n) ? Math.min(100, Math.max(1, n)) : 90
|
||
}
|
||
let channelIds = null
|
||
if (Array.isArray(r.channelIds)) {
|
||
channelIds = r.channelIds.map(String).filter(Boolean).slice(0, 50)
|
||
}
|
||
return {
|
||
id:
|
||
r.id && String(r.id).startsWith('rule_')
|
||
? String(r.id)
|
||
: newId('rule'),
|
||
name: sanitizeString(r.name || kind, 120) || kind,
|
||
enabled: r.enabled !== false,
|
||
severity: normalizeSeverity(r.severity, 'warning'),
|
||
kind,
|
||
match,
|
||
channelIds,
|
||
cooldownSeconds: Math.max(0, Math.min(86400, Number(r.cooldownSeconds) || 300)),
|
||
notifyOnRecover: r.notifyOnRecover !== false,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @param {unknown} raw
|
||
*/
|
||
function normalizeConfig(raw) {
|
||
const base = defaultConfig()
|
||
const r = raw && typeof raw === 'object' ? raw : {}
|
||
const qh = r.quietHours && typeof r.quietHours === 'object' ? r.quietHours : {}
|
||
const channels = Array.isArray(r.channels)
|
||
? r.channels.map(normalizeChannel).slice(0, 50)
|
||
: base.channels
|
||
// Merge default rules by id if missing on first load of empty rules
|
||
let rules = Array.isArray(r.rules) ? r.rules.map(normalizeRule).slice(0, 100) : null
|
||
if (!rules || rules.length === 0) {
|
||
rules = base.rules
|
||
}
|
||
return {
|
||
version: 1,
|
||
enabled: r.enabled !== false,
|
||
pollIntervalMs: Math.max(
|
||
MIN_POLL_MS,
|
||
Math.min(MAX_POLL_MS, Number(r.pollIntervalMs) || DEFAULT_POLL_MS)
|
||
),
|
||
minSeverity: normalizeSeverity(r.minSeverity, 'info'),
|
||
rateLimitPerMinute: Math.max(1, Math.min(300, Number(r.rateLimitPerMinute) || 40)),
|
||
quietHours: {
|
||
enabled: Boolean(qh.enabled),
|
||
start: sanitizeString(qh.start || '22:00', 8) || '22:00',
|
||
end: sanitizeString(qh.end || '07:00', 8) || '07:00',
|
||
timezone: sanitizeString(qh.timezone || 'UTC', 64) || 'UTC',
|
||
},
|
||
includeHostname: r.includeHostname !== false,
|
||
channels,
|
||
rules,
|
||
updatedAt: r.updatedAt || null,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Persist config promptly. Coalesce bursts so rapid UI edits don't thrash disk,
|
||
* but never wait more than ~120ms so process death can't strand live config.
|
||
*/
|
||
function scheduleSave() {
|
||
if (saveTimer) return
|
||
saveTimer = setTimeout(() => {
|
||
saveTimer = null
|
||
saveDisk()
|
||
}, 120)
|
||
if (typeof saveTimer.unref === 'function') saveTimer.unref()
|
||
}
|
||
|
||
/** Flush any pending debounced save immediately (e.g. before critical ops). */
|
||
function flushSave() {
|
||
if (saveTimer) {
|
||
clearTimeout(saveTimer)
|
||
saveTimer = null
|
||
}
|
||
saveDisk()
|
||
}
|
||
|
||
function saveDisk() {
|
||
try {
|
||
const payload = {
|
||
...config,
|
||
updatedAt: new Date().toISOString(),
|
||
}
|
||
const tmp = `${FILE}.${process.pid}.tmp`
|
||
fs.writeFileSync(tmp, JSON.stringify(payload, null, 2), { mode: 0o600 })
|
||
fs.renameSync(tmp, FILE)
|
||
try {
|
||
fs.chmodSync(FILE, 0o600)
|
||
} catch {
|
||
// ignore
|
||
}
|
||
} catch (err) {
|
||
logger.warn('alerts: failed to save config', { error: err.message })
|
||
}
|
||
}
|
||
|
||
function loadDisk() {
|
||
try {
|
||
// One-time migrate legacy cwd peardock-alerts.json → cache path
|
||
if (
|
||
!fs.existsSync(FILE) &&
|
||
LEGACY_CWD_FILE !== FILE &&
|
||
fs.existsSync(LEGACY_CWD_FILE)
|
||
) {
|
||
try {
|
||
const dir = path.dirname(FILE)
|
||
fs.mkdirSync(dir, { recursive: true })
|
||
fs.copyFileSync(LEGACY_CWD_FILE, FILE)
|
||
try {
|
||
fs.chmodSync(FILE, 0o600)
|
||
} catch {
|
||
// ignore
|
||
}
|
||
logger.info('alerts: migrated config', {
|
||
from: LEGACY_CWD_FILE,
|
||
to: FILE,
|
||
})
|
||
} catch (migErr) {
|
||
logger.warn('alerts: legacy migrate failed', { error: migErr.message })
|
||
}
|
||
}
|
||
|
||
if (!fs.existsSync(FILE)) {
|
||
config = defaultConfig()
|
||
saveDisk()
|
||
return
|
||
}
|
||
const raw = JSON.parse(fs.readFileSync(FILE, 'utf8'))
|
||
// Support versioned envelope { version, updatedAt, data|settings|config }
|
||
const body =
|
||
raw &&
|
||
typeof raw === 'object' &&
|
||
(raw.config || raw.data || raw.settings) &&
|
||
(raw.version != null || raw.updatedAt != null)
|
||
? raw.config || raw.data || raw.settings
|
||
: raw
|
||
config = normalizeConfig(body)
|
||
} catch (err) {
|
||
logger.warn('alerts: failed to load config, using defaults', { error: err.message })
|
||
config = defaultConfig()
|
||
}
|
||
}
|
||
|
||
/** Absolute path to the on-disk alerts config (for ops / tests). */
|
||
export function getAlertsFilePath() {
|
||
return FILE
|
||
}
|
||
|
||
/**
|
||
* Redact secrets for API responses.
|
||
* @param {object} ch
|
||
*/
|
||
function redactChannel(ch) {
|
||
const mask = (s) => {
|
||
const v = String(s || '')
|
||
if (v.length <= 8) return v ? '••••' : ''
|
||
return `${v.slice(0, 4)}…${v.slice(-4)}`
|
||
}
|
||
return {
|
||
...ch,
|
||
url: ch.url ? mask(ch.url) : '',
|
||
token: ch.token ? '••••••••' : '',
|
||
botToken: ch.botToken ? '••••••••' : '',
|
||
headers: Object.fromEntries(
|
||
Object.keys(ch.headers || {}).map((k) => [k, '••••'])
|
||
),
|
||
hasUrl: Boolean(ch.url),
|
||
hasToken: Boolean(ch.token || ch.botToken),
|
||
}
|
||
}
|
||
|
||
function hostLabel() {
|
||
if (!config.includeHostname) return 'PearDock'
|
||
try {
|
||
return os.hostname() || 'PearDock'
|
||
} catch {
|
||
return 'PearDock'
|
||
}
|
||
}
|
||
|
||
function parseHm(s) {
|
||
const m = String(s || '').match(/^(\d{1,2}):(\d{2})$/)
|
||
if (!m) return null
|
||
const h = Number(m[1])
|
||
const min = Number(m[2])
|
||
if (h > 23 || min > 59) return null
|
||
return h * 60 + min
|
||
}
|
||
|
||
function inQuietHours() {
|
||
const q = config.quietHours
|
||
if (!q?.enabled) return false
|
||
const start = parseHm(q.start)
|
||
const end = parseHm(q.end)
|
||
if (start == null || end == null) return false
|
||
// Approximate with local server time (timezone field is documentary for now)
|
||
const now = new Date()
|
||
const cur = now.getHours() * 60 + now.getMinutes()
|
||
if (start === end) return true
|
||
if (start < end) return cur >= start && cur < end
|
||
// Overnight window e.g. 22:00–07:00
|
||
return cur >= start || cur < end
|
||
}
|
||
|
||
function rateLimitOk() {
|
||
const now = Date.now()
|
||
if (now - deliveriesMinuteStart > 60_000) {
|
||
deliveriesMinuteStart = now
|
||
deliveriesThisMinute = 0
|
||
}
|
||
if (deliveriesThisMinute >= (config.rateLimitPerMinute || 40)) return false
|
||
deliveriesThisMinute += 1
|
||
return true
|
||
}
|
||
|
||
function matchNameFilters(name, match) {
|
||
const n = String(name || '')
|
||
if (match.nameRegex) {
|
||
try {
|
||
if (!new RegExp(match.nameRegex, 'i').test(n)) return false
|
||
} catch {
|
||
// invalid regex → ignore
|
||
}
|
||
}
|
||
if (Array.isArray(match.nameInclude) && match.nameInclude.length) {
|
||
if (!match.nameInclude.some((s) => n.toLowerCase().includes(String(s).toLowerCase()))) {
|
||
return false
|
||
}
|
||
}
|
||
if (Array.isArray(match.nameExclude) && match.nameExclude.length) {
|
||
if (match.nameExclude.some((s) => n.toLowerCase().includes(String(s).toLowerCase()))) {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
function matchLabels(labels, match) {
|
||
if (!match.labelMatch || typeof match.labelMatch !== 'object') return true
|
||
const entries = Object.entries(match.labelMatch)
|
||
if (!entries.length) return true
|
||
const lab = labels && typeof labels === 'object' ? labels : {}
|
||
for (const [k, v] of entries) {
|
||
if (String(lab[k] ?? '') !== String(v)) return false
|
||
}
|
||
return true
|
||
}
|
||
|
||
function cooldownAllows(key, seconds) {
|
||
if (!seconds || seconds <= 0) return true
|
||
const last = cooldowns.get(key) || 0
|
||
if (Date.now() - last < seconds * 1000) return false
|
||
cooldowns.set(key, Date.now())
|
||
return true
|
||
}
|
||
|
||
function sevColor(severity) {
|
||
if (severity === 'critical') return 0xe74c3c
|
||
if (severity === 'warning') return 0xf39c12
|
||
return 0x3498db
|
||
}
|
||
|
||
function buildGenericPayload(alert) {
|
||
return {
|
||
source: 'peardock',
|
||
host: hostLabel(),
|
||
title: alert.title,
|
||
message: alert.message,
|
||
severity: alert.severity,
|
||
kind: alert.kind,
|
||
ruleId: alert.ruleId,
|
||
ruleName: alert.ruleName,
|
||
timestamp: alert.timestamp,
|
||
details: alert.details || {},
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @param {object} channel
|
||
* @param {object} alert
|
||
*/
|
||
async function deliverToChannel(channel, alert) {
|
||
const payload = buildGenericPayload(alert)
|
||
const title = `[${alert.severity.toUpperCase()}] ${alert.title}`
|
||
const body = `${alert.message}\nHost: ${hostLabel()}`
|
||
|
||
if (channel.type === 'discord') {
|
||
if (!channel.url) throw new Error('Discord webhook URL required')
|
||
const res = await fetch(channel.url, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json', ...channel.headers },
|
||
body: JSON.stringify({
|
||
username: channel.username || 'PearDock',
|
||
avatar_url: channel.avatarUrl || undefined,
|
||
embeds: [
|
||
{
|
||
title,
|
||
description: alert.message,
|
||
color: sevColor(alert.severity),
|
||
fields: [
|
||
{ name: 'Host', value: hostLabel(), inline: true },
|
||
{ name: 'Kind', value: alert.kind || '—', inline: true },
|
||
{ name: 'Rule', value: alert.ruleName || '—', inline: true },
|
||
],
|
||
timestamp: alert.timestamp,
|
||
footer: { text: 'PearDock Alerts' },
|
||
},
|
||
],
|
||
}),
|
||
})
|
||
if (!res.ok) throw new Error(`Discord HTTP ${res.status}`)
|
||
return
|
||
}
|
||
|
||
if (channel.type === 'slack') {
|
||
if (!channel.url) throw new Error('Slack webhook URL required')
|
||
const res = await fetch(channel.url, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json', ...channel.headers },
|
||
body: JSON.stringify({
|
||
text: title,
|
||
blocks: [
|
||
{
|
||
type: 'header',
|
||
text: { type: 'plain_text', text: title.slice(0, 150) },
|
||
},
|
||
{
|
||
type: 'section',
|
||
text: { type: 'mrkdwn', text: `${alert.message}\n*Host:* ${hostLabel()} · *Kind:* ${alert.kind}` },
|
||
},
|
||
],
|
||
attachments: [
|
||
{
|
||
color:
|
||
alert.severity === 'critical'
|
||
? '#e74c3c'
|
||
: alert.severity === 'warning'
|
||
? '#f39c12'
|
||
: '#3498db',
|
||
text: body,
|
||
},
|
||
],
|
||
}),
|
||
})
|
||
if (!res.ok) throw new Error(`Slack HTTP ${res.status}`)
|
||
return
|
||
}
|
||
|
||
if (channel.type === 'teams') {
|
||
if (!channel.url) throw new Error('Teams webhook URL required')
|
||
const res = await fetch(channel.url, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json', ...channel.headers },
|
||
body: JSON.stringify({
|
||
'@type': 'MessageCard',
|
||
'@context': 'http://schema.org/extensions',
|
||
themeColor:
|
||
alert.severity === 'critical'
|
||
? 'E74C3C'
|
||
: alert.severity === 'warning'
|
||
? 'F39C12'
|
||
: '3498DB',
|
||
summary: title,
|
||
title,
|
||
text: `${alert.message}<br/>Host: ${hostLabel()} · Kind: ${alert.kind}`,
|
||
}),
|
||
})
|
||
if (!res.ok) throw new Error(`Teams HTTP ${res.status}`)
|
||
return
|
||
}
|
||
|
||
if (channel.type === 'ntfy') {
|
||
const base = (channel.url || 'https://ntfy.sh').replace(/\/$/, '')
|
||
const topic = channel.topic || 'peardock'
|
||
const headers = {
|
||
Title: title.slice(0, 250),
|
||
Priority:
|
||
alert.severity === 'critical' ? '5' : alert.severity === 'warning' ? '4' : '3',
|
||
Tags: 'docker,peardock',
|
||
...channel.headers,
|
||
}
|
||
if (channel.token) headers.Authorization = `Bearer ${channel.token}`
|
||
const res = await fetch(`${base}/${encodeURIComponent(topic)}`, {
|
||
method: 'POST',
|
||
headers,
|
||
body: body,
|
||
})
|
||
if (!res.ok) throw new Error(`ntfy HTTP ${res.status}`)
|
||
return
|
||
}
|
||
|
||
if (channel.type === 'gotify') {
|
||
if (!channel.url || !channel.token) throw new Error('Gotify url + token required')
|
||
const base = channel.url.replace(/\/$/, '')
|
||
const res = await fetch(`${base}/message?token=${encodeURIComponent(channel.token)}`, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json', ...channel.headers },
|
||
body: JSON.stringify({
|
||
title: title.slice(0, 200),
|
||
message: body,
|
||
priority: alert.severity === 'critical' ? 8 : alert.severity === 'warning' ? 5 : 2,
|
||
}),
|
||
})
|
||
if (!res.ok) throw new Error(`Gotify HTTP ${res.status}`)
|
||
return
|
||
}
|
||
|
||
if (channel.type === 'telegram') {
|
||
if (!channel.botToken || !channel.chatId) {
|
||
throw new Error('Telegram botToken + chatId required')
|
||
}
|
||
const res = await fetch(
|
||
`https://api.telegram.org/bot${channel.botToken}/sendMessage`,
|
||
{
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json' },
|
||
body: JSON.stringify({
|
||
chat_id: channel.chatId,
|
||
text: `${title}\n\n${body}`,
|
||
disable_web_page_preview: true,
|
||
}),
|
||
}
|
||
)
|
||
if (!res.ok) throw new Error(`Telegram HTTP ${res.status}`)
|
||
return
|
||
}
|
||
|
||
// generic webhook
|
||
if (!channel.url) throw new Error('Webhook URL required')
|
||
const res = await fetch(channel.url, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json', ...channel.headers },
|
||
body: JSON.stringify(payload),
|
||
})
|
||
if (!res.ok) throw new Error(`Webhook HTTP ${res.status}`)
|
||
}
|
||
|
||
/**
|
||
* Fire an alert through matching rules/channels.
|
||
* @param {object} alert
|
||
*/
|
||
async function fireAlert(alert) {
|
||
if (!config.enabled) return
|
||
if (!severityAtLeast(alert.severity, config.minSeverity)) return
|
||
if (inQuietHours() && alert.severity !== 'critical') {
|
||
// Critical always breaks quiet hours
|
||
return
|
||
}
|
||
if (!rateLimitOk()) {
|
||
logger.warn('alerts: rate limit hit, dropping', { title: alert.title })
|
||
return
|
||
}
|
||
|
||
const channels = config.channels.filter((c) => {
|
||
if (!c.enabled) return false
|
||
if (!severityAtLeast(alert.severity, c.minSeverity)) return false
|
||
if (Array.isArray(alert.channelIds) && alert.channelIds.length) {
|
||
return alert.channelIds.includes(c.id)
|
||
}
|
||
return true
|
||
})
|
||
|
||
const results = []
|
||
for (const ch of channels) {
|
||
try {
|
||
await deliverToChannel(ch, alert)
|
||
results.push({ channelId: ch.id, name: ch.name, ok: true })
|
||
} catch (err) {
|
||
results.push({
|
||
channelId: ch.id,
|
||
name: ch.name,
|
||
ok: false,
|
||
error: err.message || String(err),
|
||
})
|
||
logger.warn('alerts: delivery failed', {
|
||
channel: ch.name,
|
||
type: ch.type,
|
||
error: err.message,
|
||
})
|
||
}
|
||
}
|
||
|
||
const record = {
|
||
id: newId('evt'),
|
||
timestamp: alert.timestamp || new Date().toISOString(),
|
||
title: alert.title,
|
||
message: alert.message,
|
||
severity: alert.severity,
|
||
kind: alert.kind,
|
||
ruleId: alert.ruleId || null,
|
||
ruleName: alert.ruleName || null,
|
||
details: alert.details || {},
|
||
deliveries: results,
|
||
}
|
||
history.unshift(record)
|
||
if (history.length > HISTORY_MAX) history.length = HISTORY_MAX
|
||
|
||
// Push to connected clients for in-app bell
|
||
try {
|
||
peers.broadcast(Pushes.alert || 'push:alert', {
|
||
type: 'alert',
|
||
data: record,
|
||
})
|
||
} catch {
|
||
// ignore
|
||
}
|
||
|
||
logger.info('alerts: fired', {
|
||
title: alert.title,
|
||
severity: alert.severity,
|
||
channels: results.length,
|
||
ok: results.filter((r) => r.ok).length,
|
||
})
|
||
|
||
return record
|
||
}
|
||
|
||
/**
|
||
* Evaluate rules of a given kind against a context object.
|
||
* @param {string} kind
|
||
* @param {object} ctx
|
||
*/
|
||
async function evaluateRules(kind, ctx) {
|
||
if (!config.enabled) return
|
||
for (const rule of config.rules) {
|
||
if (!rule.enabled || rule.kind !== kind) continue
|
||
if (!severityAtLeast(rule.severity, config.minSeverity)) continue
|
||
|
||
const match = rule.match || {}
|
||
const name = ctx.name || ctx.Actor?.Attributes?.name || ctx.id || ''
|
||
|
||
if (!matchNameFilters(name, match)) continue
|
||
if (!matchLabels(ctx.labels || ctx.Actor?.Attributes || {}, match)) continue
|
||
|
||
if (kind === 'docker_event') {
|
||
const type = String(ctx.Type || ctx.type || '')
|
||
const action = String(ctx.Action || ctx.status || '').split(':')[0]
|
||
if (Array.isArray(match.types) && match.types.length && !match.types.includes(type)) {
|
||
continue
|
||
}
|
||
if (
|
||
Array.isArray(match.actions) &&
|
||
match.actions.length &&
|
||
!match.actions.includes(action)
|
||
) {
|
||
continue
|
||
}
|
||
const cdKey = `${rule.id}:${type}:${action}:${ctx.id || name}`
|
||
if (!cooldownAllows(cdKey, rule.cooldownSeconds)) continue
|
||
await fireAlert({
|
||
title: `${rule.name}: ${name || ctx.id || 'resource'}`,
|
||
message: `Docker ${type}/${action}${name ? ` on ${name}` : ''}`,
|
||
severity: rule.severity,
|
||
kind: 'docker_event',
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
channelIds: rule.channelIds,
|
||
timestamp: new Date().toISOString(),
|
||
details: {
|
||
type,
|
||
action,
|
||
id: ctx.id,
|
||
name,
|
||
from: ctx.from || ctx.Actor?.Attributes?.image,
|
||
},
|
||
})
|
||
}
|
||
|
||
if (kind === 'container_health') {
|
||
const health = String(ctx.health || '')
|
||
if (
|
||
Array.isArray(match.healthStatus) &&
|
||
match.healthStatus.length &&
|
||
!match.healthStatus.includes(health)
|
||
) {
|
||
continue
|
||
}
|
||
const entityKey = `health:${ctx.id}`
|
||
const prev = stateCache.get(entityKey)
|
||
if (prev === health && health !== 'unhealthy') continue
|
||
// Recover notification
|
||
if (prev === 'unhealthy' && health === 'healthy' && rule.notifyOnRecover) {
|
||
const cdKey = `${rule.id}:recover:${ctx.id}`
|
||
if (!cooldownAllows(cdKey, rule.cooldownSeconds)) continue
|
||
stateCache.set(entityKey, health)
|
||
await fireAlert({
|
||
title: `Recovered: ${name || ctx.id}`,
|
||
message: `Container health is healthy again`,
|
||
severity: 'info',
|
||
kind: 'container_health',
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
channelIds: rule.channelIds,
|
||
timestamp: new Date().toISOString(),
|
||
details: { id: ctx.id, name, health, recovered: true },
|
||
})
|
||
continue
|
||
}
|
||
if (health !== 'unhealthy' && health !== 'starting') {
|
||
stateCache.set(entityKey, health)
|
||
continue
|
||
}
|
||
if (health === 'unhealthy') {
|
||
const cdKey = `${rule.id}:unhealthy:${ctx.id}`
|
||
if (!cooldownAllows(cdKey, rule.cooldownSeconds)) continue
|
||
stateCache.set(entityKey, health)
|
||
await fireAlert({
|
||
title: `${rule.name}: ${name || ctx.id}`,
|
||
message: `Container healthcheck is ${health}`,
|
||
severity: rule.severity,
|
||
kind: 'container_health',
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
channelIds: rule.channelIds,
|
||
timestamp: new Date().toISOString(),
|
||
details: { id: ctx.id, name, health },
|
||
})
|
||
} else {
|
||
stateCache.set(entityKey, health)
|
||
}
|
||
}
|
||
|
||
if (kind === 'docker_daemon') {
|
||
const ok = Boolean(ctx.ok)
|
||
const prev = stateCache.get('docker_daemon')
|
||
const prevOk = prev !== 'down'
|
||
if (ok && prevOk && prev != null) continue
|
||
if (!ok && !prevOk && prev != null) {
|
||
// still down — cooldown re-alert
|
||
const cdKey = `${rule.id}:still-down`
|
||
if (!cooldownAllows(cdKey, rule.cooldownSeconds)) continue
|
||
} else {
|
||
const cdKey = `${rule.id}:${ok ? 'up' : 'down'}`
|
||
if (!cooldownAllows(cdKey, Math.min(60, rule.cooldownSeconds))) continue
|
||
}
|
||
stateCache.set('docker_daemon', ok ? 'up' : 'down')
|
||
if (ok && rule.notifyOnRecover && prev === 'down') {
|
||
await fireAlert({
|
||
title: 'Docker daemon recovered',
|
||
message: 'Docker Engine is reachable again',
|
||
severity: 'info',
|
||
kind: 'docker_daemon',
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
channelIds: rule.channelIds,
|
||
timestamp: new Date().toISOString(),
|
||
details: { recovered: true },
|
||
})
|
||
} else if (!ok) {
|
||
await fireAlert({
|
||
title: rule.name,
|
||
message: ctx.error
|
||
? `Docker Engine unreachable: ${ctx.error}`
|
||
: 'Docker Engine is unreachable',
|
||
severity: rule.severity,
|
||
kind: 'docker_daemon',
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
channelIds: rule.channelIds,
|
||
timestamp: new Date().toISOString(),
|
||
details: { error: ctx.error || null },
|
||
})
|
||
}
|
||
}
|
||
|
||
if (kind === 'stack_health') {
|
||
const status = String(ctx.status || '')
|
||
if (status === 'running' || status === 'ok') {
|
||
const prev = stateCache.get(`stack:${ctx.name}`)
|
||
if (prev && prev !== 'ok' && rule.notifyOnRecover) {
|
||
const cdKey = `${rule.id}:recover:${ctx.name}`
|
||
if (cooldownAllows(cdKey, rule.cooldownSeconds)) {
|
||
await fireAlert({
|
||
title: `Stack recovered: ${ctx.name}`,
|
||
message: `Stack is healthy again`,
|
||
severity: 'info',
|
||
kind: 'stack_health',
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
channelIds: rule.channelIds,
|
||
timestamp: new Date().toISOString(),
|
||
details: { name: ctx.name, status, recovered: true },
|
||
})
|
||
}
|
||
}
|
||
stateCache.set(`stack:${ctx.name}`, 'ok')
|
||
continue
|
||
}
|
||
const cdKey = `${rule.id}:${ctx.name}:${status}`
|
||
if (!cooldownAllows(cdKey, rule.cooldownSeconds)) continue
|
||
stateCache.set(`stack:${ctx.name}`, status || 'degraded')
|
||
await fireAlert({
|
||
title: `${rule.name}: ${ctx.name}`,
|
||
message: ctx.message || `Stack status: ${status || 'degraded'}`,
|
||
severity: rule.severity,
|
||
kind: 'stack_health',
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
channelIds: rule.channelIds,
|
||
timestamp: new Date().toISOString(),
|
||
details: {
|
||
name: ctx.name,
|
||
status,
|
||
running: ctx.running,
|
||
desired: ctx.desired,
|
||
},
|
||
})
|
||
}
|
||
|
||
if (kind === 'resource') {
|
||
const threshold = Number(match.diskPercent) || 90
|
||
const used = Number(ctx.diskPercent)
|
||
if (!Number.isFinite(used)) continue
|
||
const entityKey = `disk:${ctx.mount || 'root'}`
|
||
if (used < threshold) {
|
||
const prev = stateCache.get(entityKey)
|
||
if (prev === 'high' && rule.notifyOnRecover) {
|
||
if (cooldownAllows(`${rule.id}:recover:${entityKey}`, rule.cooldownSeconds)) {
|
||
await fireAlert({
|
||
title: 'Disk usage recovered',
|
||
message: `Disk usage now ${used.toFixed(1)}% (threshold ${threshold}%)`,
|
||
severity: 'info',
|
||
kind: 'resource',
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
channelIds: rule.channelIds,
|
||
timestamp: new Date().toISOString(),
|
||
details: { diskPercent: used, threshold, recovered: true },
|
||
})
|
||
}
|
||
}
|
||
stateCache.set(entityKey, 'ok')
|
||
continue
|
||
}
|
||
if (!cooldownAllows(`${rule.id}:${entityKey}`, rule.cooldownSeconds)) continue
|
||
stateCache.set(entityKey, 'high')
|
||
await fireAlert({
|
||
title: rule.name,
|
||
message: `Disk usage ${used.toFixed(1)}% exceeds ${threshold}%${ctx.mount ? ` on ${ctx.mount}` : ''}`,
|
||
severity: rule.severity,
|
||
kind: 'resource',
|
||
ruleId: rule.id,
|
||
ruleName: rule.name,
|
||
channelIds: rule.channelIds,
|
||
timestamp: new Date().toISOString(),
|
||
details: { diskPercent: used, threshold, mount: ctx.mount },
|
||
})
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @param {string} kind
|
||
* @returns {boolean}
|
||
*/
|
||
function hasEnabledRuleKind(kind) {
|
||
return config.rules.some(
|
||
(r) =>
|
||
r.enabled &&
|
||
r.kind === kind &&
|
||
severityAtLeast(r.severity, config.minSeverity)
|
||
)
|
||
}
|
||
|
||
/**
|
||
* Kinds that require a periodic Docker / host poll (not pure events).
|
||
*/
|
||
function pollNeeds() {
|
||
return {
|
||
daemon: hasEnabledRuleKind('docker_daemon'),
|
||
list: hasEnabledRuleKind('container_health') || hasEnabledRuleKind('stack_health'),
|
||
resource: hasEnabledRuleKind('resource'),
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Whether this Docker event could fire any enabled alert rule.
|
||
* Drops terminal/exec noise that was previously evaluated on every event.
|
||
* @param {object} event
|
||
*/
|
||
function isAlertRelevantEvent(event) {
|
||
if (!event) return false
|
||
const type = String(event.Type || event.type || '')
|
||
const actionRaw = String(event.Action || event.status || '')
|
||
const action = actionRaw.split(':')[0]
|
||
|
||
if (NOISY_DOCKER_ACTIONS.has(action)) return false
|
||
|
||
// Healthcheck transitions → container_health rules
|
||
if (
|
||
type === 'container' &&
|
||
actionRaw.startsWith('health_status') &&
|
||
hasEnabledRuleKind('container_health')
|
||
) {
|
||
return true
|
||
}
|
||
|
||
if (!hasEnabledRuleKind('docker_event')) return false
|
||
|
||
for (const rule of config.rules) {
|
||
if (!rule.enabled || rule.kind !== 'docker_event') continue
|
||
if (!severityAtLeast(rule.severity, config.minSeverity)) continue
|
||
const match = rule.match || {}
|
||
if (Array.isArray(match.types) && match.types.length && !match.types.includes(type)) {
|
||
continue
|
||
}
|
||
if (
|
||
Array.isArray(match.actions) &&
|
||
match.actions.length &&
|
||
!match.actions.includes(action)
|
||
) {
|
||
continue
|
||
}
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
/**
|
||
* Process one Docker event for alert rules (awaited).
|
||
* @param {object} event
|
||
*/
|
||
async function processDockerEvent(event) {
|
||
await evaluateRules('docker_event', event)
|
||
const action = String(event.Action || event.status || '')
|
||
if (event.Type === 'container' && action.startsWith('health_status')) {
|
||
const health = action.includes(':')
|
||
? action.split(':').slice(1).join(':').trim()
|
||
: event.status
|
||
await evaluateRules('container_health', {
|
||
id: event.id || event.Actor?.ID,
|
||
name: event.Actor?.Attributes?.name,
|
||
labels: event.Actor?.Attributes,
|
||
health: health || event.Actor?.Attributes?.health_status,
|
||
})
|
||
}
|
||
}
|
||
|
||
async function drainEventQueue() {
|
||
if (eventDraining) return
|
||
eventDraining = true
|
||
try {
|
||
while (eventQueue.length) {
|
||
const ev = eventQueue.shift()
|
||
if (!ev || !config.enabled) continue
|
||
try {
|
||
await processDockerEvent(ev)
|
||
} catch (err) {
|
||
logger.warn('alerts: onDockerEvent failed', { error: err.message })
|
||
}
|
||
}
|
||
} finally {
|
||
eventDraining = false
|
||
// Work may have been enqueued while draining
|
||
if (eventQueue.length) {
|
||
drainEventQueue().catch(() => {})
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Called from Docker event stream (fire-and-forget from events.js).
|
||
* Cheap sync filter + bounded queue; never piles concurrent evaluate/fire work.
|
||
* @param {object} event
|
||
*/
|
||
export function onDockerEvent(event) {
|
||
if (!event || !config.enabled) return
|
||
if (!isAlertRelevantEvent(event)) return
|
||
if (eventQueue.length >= EVENT_QUEUE_MAX) {
|
||
eventQueue.shift()
|
||
}
|
||
eventQueue.push(event)
|
||
drainEventQueue().catch((err) => {
|
||
logger.debug('alerts: event drain error', { error: err?.message })
|
||
})
|
||
}
|
||
|
||
/**
|
||
* Periodic health poll. Skips Docker work for disabled rule kinds.
|
||
* Never calls docker.df (expensive and unused).
|
||
*/
|
||
async function pollHealth() {
|
||
if (!config.enabled) return
|
||
if (pollInFlight) {
|
||
logger.debug('alerts: poll skipped (previous still running)')
|
||
return
|
||
}
|
||
const needs = pollNeeds()
|
||
if (!needs.daemon && !needs.list && !needs.resource) {
|
||
// Only docker_event / peardock rules — pure event path, no polling work
|
||
return
|
||
}
|
||
|
||
pollInFlight = true
|
||
try {
|
||
if (needs.daemon) {
|
||
try {
|
||
await docker.ping()
|
||
lastDaemonOk = true
|
||
await evaluateRules('docker_daemon', { ok: true })
|
||
} catch (err) {
|
||
lastDaemonOk = false
|
||
await evaluateRules('docker_daemon', {
|
||
ok: false,
|
||
error: err.message || String(err),
|
||
})
|
||
return // skip list/resource if docker is down
|
||
}
|
||
}
|
||
|
||
/** @type {object[]} */
|
||
let containers = []
|
||
if (needs.list) {
|
||
try {
|
||
containers = (await docker.listContainers({ all: true })) || []
|
||
} catch (err) {
|
||
logger.debug('alerts: container list poll failed', { error: err.message })
|
||
}
|
||
}
|
||
|
||
if (needs.list && hasEnabledRuleKind('container_health')) {
|
||
try {
|
||
for (const c of containers) {
|
||
const health =
|
||
c.Status?.match(/\((healthy|unhealthy|health: starting)\)/i)?.[1] || c.State
|
||
const h =
|
||
c.Health?.Status ||
|
||
(typeof health === 'string'
|
||
? health.toLowerCase().replace('health: ', '')
|
||
: null)
|
||
if (!h || h === 'none') continue
|
||
// Skip pure state strings that aren't healthcheck results
|
||
if (h === 'running' || h === 'exited' || h === 'created' || h === 'paused') {
|
||
continue
|
||
}
|
||
const name = (c.Names?.[0] || '').replace(/^\//, '') || c.Id?.slice(0, 12)
|
||
await evaluateRules('container_health', {
|
||
id: c.Id,
|
||
name,
|
||
labels: c.Labels,
|
||
health: h === 'starting' ? 'starting' : h,
|
||
})
|
||
}
|
||
} catch (err) {
|
||
logger.debug('alerts: container health poll failed', { error: err.message })
|
||
}
|
||
}
|
||
|
||
if (needs.list && hasEnabledRuleKind('stack_health')) {
|
||
try {
|
||
/** @type {Map<string, { name: string, running: number, total: number, unhealthy: number }>} */
|
||
const stacks = new Map()
|
||
for (const c of containers) {
|
||
const project =
|
||
c.Labels?.['com.docker.compose.project'] ||
|
||
c.Labels?.['com.docker.stack.namespace']
|
||
if (!project) continue
|
||
let s = stacks.get(project)
|
||
if (!s) {
|
||
s = { name: project, running: 0, total: 0, unhealthy: 0 }
|
||
stacks.set(project, s)
|
||
}
|
||
s.total += 1
|
||
if (c.State === 'running') s.running += 1
|
||
const st = String(c.Status || '')
|
||
if (/\(unhealthy\)/i.test(st)) s.unhealthy += 1
|
||
}
|
||
for (const s of stacks.values()) {
|
||
const degraded = s.running < s.total || s.unhealthy > 0
|
||
await evaluateRules('stack_health', {
|
||
name: s.name,
|
||
status: degraded ? 'degraded' : 'ok',
|
||
running: s.running,
|
||
desired: s.total,
|
||
message: degraded
|
||
? `${s.running}/${s.total} running${s.unhealthy ? `, ${s.unhealthy} unhealthy` : ''}`
|
||
: 'all services running',
|
||
})
|
||
}
|
||
} catch (err) {
|
||
logger.debug('alerts: stack poll failed', { error: err.message })
|
||
}
|
||
}
|
||
|
||
// Host disk only — never docker.df() (expensive Engine inventory scan).
|
||
if (needs.resource) {
|
||
try {
|
||
const { statfsSync } = await import('fs')
|
||
if (typeof statfsSync === 'function') {
|
||
const st = statfsSync('/')
|
||
const total = Number(st.blocks) * Number(st.bsize)
|
||
const free = Number(st.bfree) * Number(st.bsize)
|
||
if (total > 0) {
|
||
const usedPct = ((total - free) / total) * 100
|
||
await evaluateRules('resource', {
|
||
diskPercent: usedPct,
|
||
mount: '/',
|
||
})
|
||
}
|
||
}
|
||
} catch (err) {
|
||
logger.debug('alerts: resource poll failed', { error: err?.message })
|
||
}
|
||
}
|
||
} finally {
|
||
pollInFlight = false
|
||
}
|
||
}
|
||
|
||
function armPoll() {
|
||
if (pollTimer) {
|
||
clearInterval(pollTimer)
|
||
pollTimer = null
|
||
}
|
||
if (!config.enabled) return
|
||
const needs = pollNeeds()
|
||
if (!needs.daemon && !needs.list && !needs.resource) {
|
||
logger.debug('alerts: poll timer off (no poll-backed rules enabled)')
|
||
return
|
||
}
|
||
const ms = Math.max(MIN_POLL_MS, config.pollIntervalMs || DEFAULT_POLL_MS)
|
||
pollTimer = setInterval(() => {
|
||
pollHealth().catch((err) => {
|
||
logger.debug('alerts: poll error', { error: err.message })
|
||
})
|
||
}, ms)
|
||
if (typeof pollTimer.unref === 'function') pollTimer.unref()
|
||
}
|
||
|
||
/**
|
||
* Apply in-memory config to runtime (poll timer, etc.) without restart.
|
||
* Called after every config mutation so enable/disable and intervals are live.
|
||
*/
|
||
function applyRuntimeConfig({ kickPoll = false } = {}) {
|
||
armPoll()
|
||
if (kickPoll && config.enabled) {
|
||
pollHealth().catch((err) => {
|
||
logger.debug('alerts: kick poll failed', { error: err.message })
|
||
})
|
||
}
|
||
logger.info('alerts: runtime config applied', {
|
||
enabled: config.enabled,
|
||
pollIntervalMs: config.pollIntervalMs,
|
||
minSeverity: config.minSeverity,
|
||
rateLimitPerMinute: config.rateLimitPerMinute,
|
||
quietHours: Boolean(config.quietHours?.enabled),
|
||
channels: config.channels.filter((c) => c.enabled).length,
|
||
rules: config.rules.filter((r) => r.enabled).length,
|
||
})
|
||
}
|
||
|
||
export function startAlerts() {
|
||
if (started) return
|
||
started = true
|
||
loadDisk()
|
||
applyRuntimeConfig({ kickPoll: false })
|
||
// Initial poll shortly after boot (only if enabled)
|
||
setTimeout(() => {
|
||
if (config.enabled) pollHealth().catch(() => {})
|
||
}, 5_000)
|
||
logger.info('Alerts engine started', {
|
||
channels: config.channels.length,
|
||
rules: config.rules.filter((r) => r.enabled).length,
|
||
enabled: config.enabled,
|
||
})
|
||
}
|
||
|
||
export function stopAlerts() {
|
||
started = false
|
||
if (pollTimer) {
|
||
clearInterval(pollTimer)
|
||
pollTimer = null
|
||
}
|
||
eventQueue = []
|
||
eventDraining = false
|
||
pollInFlight = false
|
||
if (saveTimer) {
|
||
clearTimeout(saveTimer)
|
||
saveTimer = null
|
||
saveDisk()
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Public config (secrets redacted).
|
||
*/
|
||
export function getAlertsConfig() {
|
||
return {
|
||
...config,
|
||
channels: config.channels.map(redactChannel),
|
||
rules: config.rules,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Full config for internal use (includes secrets). Not for clients.
|
||
*/
|
||
export function getAlertsConfigRaw() {
|
||
return { ...config, channels: [...config.channels], rules: [...config.rules] }
|
||
}
|
||
|
||
/**
|
||
* Replace or merge full config.
|
||
* @param {object} partial
|
||
* @param {{ replace?: boolean }} [opts]
|
||
*/
|
||
export function updateAlertsConfig(partial, opts = {}) {
|
||
const wasEnabled = config.enabled
|
||
if (opts.replace) {
|
||
config = normalizeConfig(partial)
|
||
} else {
|
||
const merged = {
|
||
...config,
|
||
...partial,
|
||
quietHours: {
|
||
...config.quietHours,
|
||
...(partial.quietHours || {}),
|
||
},
|
||
channels: Array.isArray(partial.channels)
|
||
? partial.channels.map(normalizeChannel)
|
||
: config.channels,
|
||
rules: Array.isArray(partial.rules)
|
||
? partial.rules.map(normalizeRule)
|
||
: config.rules,
|
||
}
|
||
config = normalizeConfig(merged)
|
||
}
|
||
config.updatedAt = new Date().toISOString()
|
||
flushSave()
|
||
// Live apply: poll interval, enable/disable — no process restart
|
||
applyRuntimeConfig({
|
||
kickPoll: config.enabled && (!wasEnabled || partial.pollIntervalMs != null),
|
||
})
|
||
return getAlertsConfig()
|
||
}
|
||
|
||
export function upsertAlertChannel(input) {
|
||
const ch = normalizeChannel(input)
|
||
const idx = config.channels.findIndex((c) => c.id === ch.id)
|
||
// Preserve secrets if client sent masked/empty on update
|
||
if (idx >= 0) {
|
||
const prev = config.channels[idx]
|
||
if (!ch.url || ch.url.includes('…') || ch.url.includes('••••')) ch.url = prev.url
|
||
if (!ch.token || ch.token.includes('•')) ch.token = prev.token
|
||
if (!ch.botToken || ch.botToken.includes('•')) ch.botToken = prev.botToken
|
||
if (ch.headers && Object.values(ch.headers).every((v) => String(v).includes('•'))) {
|
||
ch.headers = prev.headers
|
||
}
|
||
config.channels[idx] = ch
|
||
} else {
|
||
config.channels.push(ch)
|
||
}
|
||
// In-memory config is used on next fire; flush disk promptly (no restart)
|
||
flushSave()
|
||
logger.debug('alerts: channel upserted live', {
|
||
id: ch.id,
|
||
enabled: ch.enabled,
|
||
type: ch.type,
|
||
})
|
||
return redactChannel(ch)
|
||
}
|
||
|
||
export function deleteAlertChannel(id) {
|
||
const before = config.channels.length
|
||
config.channels = config.channels.filter((c) => c.id !== id)
|
||
// Clear channel refs from rules
|
||
for (const rule of config.rules) {
|
||
if (Array.isArray(rule.channelIds)) {
|
||
rule.channelIds = rule.channelIds.filter((x) => x !== id)
|
||
}
|
||
}
|
||
flushSave()
|
||
return before !== config.channels.length
|
||
}
|
||
|
||
export function upsertAlertRule(input) {
|
||
const rule = normalizeRule(input)
|
||
const idx = config.rules.findIndex((r) => r.id === rule.id)
|
||
if (idx >= 0) config.rules[idx] = rule
|
||
else config.rules.push(rule)
|
||
flushSave()
|
||
// Rule kind/enable changes which Docker polls we need
|
||
armPoll()
|
||
logger.debug('alerts: rule upserted live', {
|
||
id: rule.id,
|
||
enabled: rule.enabled,
|
||
kind: rule.kind,
|
||
})
|
||
return rule
|
||
}
|
||
|
||
export function deleteAlertRule(id) {
|
||
const before = config.rules.length
|
||
config.rules = config.rules.filter((r) => r.id !== id)
|
||
flushSave()
|
||
armPoll()
|
||
return before !== config.rules.length
|
||
}
|
||
|
||
export function listAlertHistory(limit = 50) {
|
||
const n = Math.max(1, Math.min(HISTORY_MAX, Number(limit) || 50))
|
||
return history.slice(0, n)
|
||
}
|
||
|
||
export function getAlertsStatus() {
|
||
const needs = pollNeeds()
|
||
return {
|
||
enabled: config.enabled,
|
||
channelCount: config.channels.filter((c) => c.enabled).length,
|
||
ruleCount: config.rules.filter((r) => r.enabled).length,
|
||
lastDaemonOk,
|
||
historyCount: history.length,
|
||
pollIntervalMs: config.pollIntervalMs,
|
||
pollActive: Boolean(pollTimer),
|
||
pollInFlight,
|
||
pollNeeds: needs,
|
||
eventQueueDepth: eventQueue.length,
|
||
quietHoursActive: inQuietHours(),
|
||
rateLimitPerMinute: config.rateLimitPerMinute,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Send a test message to one channel (uses raw secrets).
|
||
* @param {string} channelId
|
||
*/
|
||
export async function testAlertChannel(channelId) {
|
||
const ch = config.channels.find((c) => c.id === channelId)
|
||
if (!ch) throw Object.assign(new Error('Channel not found'), { code: 'NOT_FOUND' })
|
||
const alert = {
|
||
title: 'PearDock test alert',
|
||
message: `Test notification from PearDock at ${new Date().toISOString()}`,
|
||
severity: 'info',
|
||
kind: 'peardock',
|
||
ruleId: null,
|
||
ruleName: 'Test',
|
||
timestamp: new Date().toISOString(),
|
||
details: { test: true },
|
||
}
|
||
await deliverToChannel(ch, alert)
|
||
return { success: true, channelId: ch.id, name: ch.name }
|
||
}
|
||
|
||
/**
|
||
* Manually fire a peardock-kind rule (e.g. from client).
|
||
* @param {{ title: string, message?: string, severity?: string }} input
|
||
*/
|
||
export async function fireManualAlert(input) {
|
||
return fireAlert({
|
||
title: sanitizeString(input.title || 'Alert', 200),
|
||
message: sanitizeString(input.message || '', 2000),
|
||
severity: normalizeSeverity(input.severity, 'info'),
|
||
kind: 'peardock',
|
||
ruleId: null,
|
||
ruleName: 'Manual',
|
||
channelIds: null,
|
||
timestamp: new Date().toISOString(),
|
||
details: input.details || {},
|
||
})
|
||
}
|
||
|
||
export const ALERT_CHANNEL_TYPES = [...CHANNEL_TYPES]
|
||
export const ALERT_RULE_KINDS = [...RULE_KINDS]
|
||
export const ALERT_SEVERITIES = [...SEVERITIES]
|
||
|
||
export default {
|
||
startAlerts,
|
||
stopAlerts,
|
||
onDockerEvent,
|
||
getAlertsConfig,
|
||
getAlertsConfigRaw,
|
||
getAlertsFilePath,
|
||
updateAlertsConfig,
|
||
upsertAlertChannel,
|
||
deleteAlertChannel,
|
||
upsertAlertRule,
|
||
deleteAlertRule,
|
||
listAlertHistory,
|
||
getAlertsStatus,
|
||
testAlertChannel,
|
||
fireManualAlert,
|
||
}
|