+
@@ -3098,7 +3108,8 @@ services:
diff --git a/server/services/alerts.js b/server/services/alerts.js
index 507ea05..6de7ba2 100644
--- a/server/services/alerts.js
+++ b/server/services/alerts.js
@@ -3,7 +3,14 @@
*
* Fully customizable rules + webhook channels (Discord, Slack, Teams,
* generic HTTP, ntfy, Gotify, Telegram). Evaluates Docker events and
- * periodic health polls even when no client is connected.
+ * light periodic health polls even when no client is connected.
+ *
+ * Host-load rules (keep this cheap on dockerd):
+ * - Never call docker.df() (expensive, unused).
+ * - Poll only kinds that have enabled rules (skip listContainers when unused).
+ * - Min poll interval 60s (default 120s); never overlap polls.
+ * - Docker event path filters noise (exec_*, attach, …) and serializes work.
+ * - Event-only rules need no Docker polling.
*
* Config (mode 0600), first match wins:
* PEARDOCK_ALERTS_PATH
@@ -45,10 +52,36 @@ const FILE = resolveAlertsFilePath()
const LEGACY_CWD_FILE = path.join(process.cwd(), 'peardock-alerts.json')
const HISTORY_MAX = 200
-const DEFAULT_POLL_MS = 60_000
+/** Default health poll — 2 minutes (was 60s). */
+const DEFAULT_POLL_MS = 120_000
+/** Hard floor — sub-minute listContainers/ping loops thrash dockerd. */
+const MIN_POLL_MS = 60_000
+const MAX_POLL_MS = 3_600_000
+/** Drop oldest under event flood rather than unbounded memory. */
+const EVENT_QUEUE_MAX = 200
const SEVERITIES = ['info', 'warning', 'critical']
const SEV_RANK = { info: 1, warning: 2, critical: 3 }
+/**
+ * Docker event actions that never match default peardock alert rules and are
+ * extremely chatty (terminals, attach, top). Skip before evaluateRules.
+ */
+const NOISY_DOCKER_ACTIONS = new Set([
+ 'exec_create',
+ 'exec_start',
+ 'exec_die',
+ 'exec_detach',
+ 'attach',
+ 'detach',
+ 'resize',
+ 'top',
+ 'export',
+ 'commit',
+ 'copy',
+ 'archive-path',
+ 'extract-to-dir',
+])
+
const CHANNEL_TYPES = new Set([
'discord',
'slack',
@@ -85,6 +118,11 @@ let deliveriesThisMinute = 0
let deliveriesMinuteStart = Date.now()
let lastDaemonOk = true
let started = false
+/** Prevent overlapping pollHealth (setInterval + slow listContainers). */
+let pollInFlight = false
+/** Serialize docker-event alert evaluation under noisy fleets. */
+let eventQueue = []
+let eventDraining = false
function newId(prefix) {
return `${prefix}_${Date.now().toString(36)}_${randomBytes(3).toString('hex')}`
@@ -316,8 +354,8 @@ function normalizeConfig(raw) {
version: 1,
enabled: r.enabled !== false,
pollIntervalMs: Math.max(
- 15_000,
- Math.min(3_600_000, Number(r.pollIntervalMs) || DEFAULT_POLL_MS)
+ MIN_POLL_MS,
+ Math.min(MAX_POLL_MS, Number(r.pollIntervalMs) || DEFAULT_POLL_MS)
),
minSeverity: normalizeSeverity(r.minSeverity, 'info'),
rateLimitPerMinute: Math.max(1, Math.min(300, Number(r.rateLimitPerMinute) || 40)),
@@ -1031,141 +1069,260 @@ async function evaluateRules(kind, ctx) {
}
/**
- * Called from Docker event stream.
- * @param {object} event
+ * @param {string} kind
+ * @returns {boolean}
*/
-export async function onDockerEvent(event) {
- if (!event || !config.enabled) return
- try {
- await evaluateRules('docker_event', event)
- // Health status often arrives as container health_status event
- const action = String(event.Action || event.status || '')
- if (event.Type === 'container' && action.startsWith('health_status')) {
- const health = action.includes(':')
- ? action.split(':').slice(1).join(':').trim()
- : event.status
- await evaluateRules('container_health', {
- id: event.id || event.Actor?.ID,
- name: event.Actor?.Attributes?.name,
- labels: event.Actor?.Attributes,
- health: health || event.Actor?.Attributes?.health_status,
- })
- }
- } catch (err) {
- logger.warn('alerts: onDockerEvent failed', { error: err.message })
+function hasEnabledRuleKind(kind) {
+ return config.rules.some(
+ (r) =>
+ r.enabled &&
+ r.kind === kind &&
+ severityAtLeast(r.severity, config.minSeverity)
+ )
+}
+
+/**
+ * Kinds that require a periodic Docker / host poll (not pure events).
+ */
+function pollNeeds() {
+ return {
+ daemon: hasEnabledRuleKind('docker_daemon'),
+ list: hasEnabledRuleKind('container_health') || hasEnabledRuleKind('stack_health'),
+ resource: hasEnabledRuleKind('resource'),
}
}
+/**
+ * Whether this Docker event could fire any enabled alert rule.
+ * Drops terminal/exec noise that was previously evaluated on every event.
+ * @param {object} event
+ */
+function isAlertRelevantEvent(event) {
+ if (!event) return false
+ const type = String(event.Type || event.type || '')
+ const actionRaw = String(event.Action || event.status || '')
+ const action = actionRaw.split(':')[0]
+
+ if (NOISY_DOCKER_ACTIONS.has(action)) return false
+
+ // Healthcheck transitions → container_health rules
+ if (
+ type === 'container' &&
+ actionRaw.startsWith('health_status') &&
+ hasEnabledRuleKind('container_health')
+ ) {
+ return true
+ }
+
+ if (!hasEnabledRuleKind('docker_event')) return false
+
+ for (const rule of config.rules) {
+ if (!rule.enabled || rule.kind !== 'docker_event') continue
+ if (!severityAtLeast(rule.severity, config.minSeverity)) continue
+ const match = rule.match || {}
+ if (Array.isArray(match.types) && match.types.length && !match.types.includes(type)) {
+ continue
+ }
+ if (
+ Array.isArray(match.actions) &&
+ match.actions.length &&
+ !match.actions.includes(action)
+ ) {
+ continue
+ }
+ return true
+ }
+ return false
+}
+
+/**
+ * Process one Docker event for alert rules (awaited).
+ * @param {object} event
+ */
+async function processDockerEvent(event) {
+ await evaluateRules('docker_event', event)
+ const action = String(event.Action || event.status || '')
+ if (event.Type === 'container' && action.startsWith('health_status')) {
+ const health = action.includes(':')
+ ? action.split(':').slice(1).join(':').trim()
+ : event.status
+ await evaluateRules('container_health', {
+ id: event.id || event.Actor?.ID,
+ name: event.Actor?.Attributes?.name,
+ labels: event.Actor?.Attributes,
+ health: health || event.Actor?.Attributes?.health_status,
+ })
+ }
+}
+
+async function drainEventQueue() {
+ if (eventDraining) return
+ eventDraining = true
+ try {
+ while (eventQueue.length) {
+ const ev = eventQueue.shift()
+ if (!ev || !config.enabled) continue
+ try {
+ await processDockerEvent(ev)
+ } catch (err) {
+ logger.warn('alerts: onDockerEvent failed', { error: err.message })
+ }
+ }
+ } finally {
+ eventDraining = false
+ // Work may have been enqueued while draining
+ if (eventQueue.length) {
+ drainEventQueue().catch(() => {})
+ }
+ }
+}
+
+/**
+ * Called from Docker event stream (fire-and-forget from events.js).
+ * Cheap sync filter + bounded queue; never piles concurrent evaluate/fire work.
+ * @param {object} event
+ */
+export function onDockerEvent(event) {
+ if (!event || !config.enabled) return
+ if (!isAlertRelevantEvent(event)) return
+ if (eventQueue.length >= EVENT_QUEUE_MAX) {
+ eventQueue.shift()
+ }
+ eventQueue.push(event)
+ drainEventQueue().catch((err) => {
+ logger.debug('alerts: event drain error', { error: err?.message })
+ })
+}
+
+/**
+ * Periodic health poll. Skips Docker work for disabled rule kinds.
+ * Never calls docker.df (expensive and unused).
+ */
async function pollHealth() {
if (!config.enabled) return
-
- // Docker daemon
- try {
- await docker.ping()
- lastDaemonOk = true
- await evaluateRules('docker_daemon', { ok: true })
- } catch (err) {
- lastDaemonOk = false
- await evaluateRules('docker_daemon', {
- ok: false,
- error: err.message || String(err),
- })
- return // skip further polls if docker is down
+ if (pollInFlight) {
+ logger.debug('alerts: poll skipped (previous still running)')
+ return
+ }
+ const needs = pollNeeds()
+ if (!needs.daemon && !needs.list && !needs.resource) {
+ // Only docker_event / peardock rules — pure event path, no polling work
+ return
}
- // One listContainers for both container_health and stack_health (half the socket work)
- /** @type {object[]} */
- let containers = []
+ pollInFlight = true
try {
- containers = (await docker.listContainers({ all: true })) || []
- } catch (err) {
- logger.debug('alerts: container list poll failed', { error: err.message })
- }
-
- // Container health
- try {
- for (const c of containers) {
- const health = c.Status?.match(/\((healthy|unhealthy|health: starting)\)/i)?.[1]
- || c.State // running/exited — Health may be in inspect only
- // Prefer Health from inspect-lite if present
- const h =
- c.Health?.Status ||
- (typeof health === 'string' ? health.toLowerCase().replace('health: ', '') : null)
- if (!h || h === 'none') continue
- const name = (c.Names?.[0] || '').replace(/^\//, '') || c.Id?.slice(0, 12)
- await evaluateRules('container_health', {
- id: c.Id,
- name,
- labels: c.Labels,
- health: h === 'starting' ? 'starting' : h,
- })
- }
- } catch (err) {
- logger.debug('alerts: container health poll failed', { error: err.message })
- }
-
- // Stack / compose project health (group by com.docker.compose.project)
- try {
- /** @type {Map} */
- const stacks = new Map()
- for (const c of containers) {
- const project =
- c.Labels?.['com.docker.compose.project'] ||
- c.Labels?.['com.docker.stack.namespace']
- if (!project) continue
- let s = stacks.get(project)
- if (!s) {
- s = { name: project, running: 0, total: 0, unhealthy: 0 }
- stacks.set(project, s)
+ if (needs.daemon) {
+ try {
+ await docker.ping()
+ lastDaemonOk = true
+ await evaluateRules('docker_daemon', { ok: true })
+ } catch (err) {
+ lastDaemonOk = false
+ await evaluateRules('docker_daemon', {
+ ok: false,
+ error: err.message || String(err),
+ })
+ return // skip list/resource if docker is down
}
- s.total += 1
- if (c.State === 'running') s.running += 1
- const st = String(c.Status || '')
- if (/\(unhealthy\)/i.test(st)) s.unhealthy += 1
}
- for (const s of stacks.values()) {
- const degraded = s.running < s.total || s.unhealthy > 0
- await evaluateRules('stack_health', {
- name: s.name,
- status: degraded ? 'degraded' : 'ok',
- running: s.running,
- desired: s.total,
- message: degraded
- ? `${s.running}/${s.total} running${s.unhealthy ? `, ${s.unhealthy} unhealthy` : ''}`
- : 'all services running',
- })
- }
- } catch (err) {
- logger.debug('alerts: stack poll failed', { error: err.message })
- }
- // Disk via system df
- try {
- if (typeof docker.df === 'function') {
- const df = await docker.df()
- // Docker df doesn't give host disk %; approximate via layers size vs nothing.
- // Prefer host free space from / if available via system info
+ /** @type {object[]} */
+ let containers = []
+ if (needs.list) {
+ try {
+ containers = (await docker.listContainers({ all: true })) || []
+ } catch (err) {
+ logger.debug('alerts: container list poll failed', { error: err.message })
+ }
}
- // Host disk from Node (server filesystem)
- try {
- const { statfsSync } = await import('fs')
- if (typeof statfsSync === 'function') {
- const st = statfsSync('/')
- const total = Number(st.blocks) * Number(st.bsize)
- const free = Number(st.bfree) * Number(st.bsize)
- if (total > 0) {
- const usedPct = ((total - free) / total) * 100
- await evaluateRules('resource', {
- diskPercent: usedPct,
- mount: '/',
+
+ if (needs.list && hasEnabledRuleKind('container_health')) {
+ try {
+ for (const c of containers) {
+ const health =
+ c.Status?.match(/\((healthy|unhealthy|health: starting)\)/i)?.[1] || c.State
+ const h =
+ c.Health?.Status ||
+ (typeof health === 'string'
+ ? health.toLowerCase().replace('health: ', '')
+ : null)
+ if (!h || h === 'none') continue
+ // Skip pure state strings that aren't healthcheck results
+ if (h === 'running' || h === 'exited' || h === 'created' || h === 'paused') {
+ continue
+ }
+ const name = (c.Names?.[0] || '').replace(/^\//, '') || c.Id?.slice(0, 12)
+ await evaluateRules('container_health', {
+ id: c.Id,
+ name,
+ labels: c.Labels,
+ health: h === 'starting' ? 'starting' : h,
})
}
+ } catch (err) {
+ logger.debug('alerts: container health poll failed', { error: err.message })
}
- } catch {
- // statfs not available on all platforms
}
- } catch (err) {
- logger.debug('alerts: resource poll failed', { error: err.message })
+
+ if (needs.list && hasEnabledRuleKind('stack_health')) {
+ try {
+ /** @type {Map} */
+ const stacks = new Map()
+ for (const c of containers) {
+ const project =
+ c.Labels?.['com.docker.compose.project'] ||
+ c.Labels?.['com.docker.stack.namespace']
+ if (!project) continue
+ let s = stacks.get(project)
+ if (!s) {
+ s = { name: project, running: 0, total: 0, unhealthy: 0 }
+ stacks.set(project, s)
+ }
+ s.total += 1
+ if (c.State === 'running') s.running += 1
+ const st = String(c.Status || '')
+ if (/\(unhealthy\)/i.test(st)) s.unhealthy += 1
+ }
+ for (const s of stacks.values()) {
+ const degraded = s.running < s.total || s.unhealthy > 0
+ await evaluateRules('stack_health', {
+ name: s.name,
+ status: degraded ? 'degraded' : 'ok',
+ running: s.running,
+ desired: s.total,
+ message: degraded
+ ? `${s.running}/${s.total} running${s.unhealthy ? `, ${s.unhealthy} unhealthy` : ''}`
+ : 'all services running',
+ })
+ }
+ } catch (err) {
+ logger.debug('alerts: stack poll failed', { error: err.message })
+ }
+ }
+
+ // Host disk only — never docker.df() (expensive Engine inventory scan).
+ if (needs.resource) {
+ try {
+ const { statfsSync } = await import('fs')
+ if (typeof statfsSync === 'function') {
+ const st = statfsSync('/')
+ const total = Number(st.blocks) * Number(st.bsize)
+ const free = Number(st.bfree) * Number(st.bsize)
+ if (total > 0) {
+ const usedPct = ((total - free) / total) * 100
+ await evaluateRules('resource', {
+ diskPercent: usedPct,
+ mount: '/',
+ })
+ }
+ }
+ } catch (err) {
+ logger.debug('alerts: resource poll failed', { error: err?.message })
+ }
+ }
+ } finally {
+ pollInFlight = false
}
}
@@ -1175,7 +1332,12 @@ function armPoll() {
pollTimer = null
}
if (!config.enabled) return
- const ms = config.pollIntervalMs || DEFAULT_POLL_MS
+ const needs = pollNeeds()
+ if (!needs.daemon && !needs.list && !needs.resource) {
+ logger.debug('alerts: poll timer off (no poll-backed rules enabled)')
+ return
+ }
+ const ms = Math.max(MIN_POLL_MS, config.pollIntervalMs || DEFAULT_POLL_MS)
pollTimer = setInterval(() => {
pollHealth().catch((err) => {
logger.debug('alerts: poll error', { error: err.message })
@@ -1228,6 +1390,9 @@ export function stopAlerts() {
clearInterval(pollTimer)
pollTimer = null
}
+ eventQueue = []
+ eventDraining = false
+ pollInFlight = false
if (saveTimer) {
clearTimeout(saveTimer)
saveTimer = null
@@ -1333,6 +1498,8 @@ export function upsertAlertRule(input) {
if (idx >= 0) config.rules[idx] = rule
else config.rules.push(rule)
flushSave()
+ // Rule kind/enable changes which Docker polls we need
+ armPoll()
logger.debug('alerts: rule upserted live', {
id: rule.id,
enabled: rule.enabled,
@@ -1345,6 +1512,7 @@ export function deleteAlertRule(id) {
const before = config.rules.length
config.rules = config.rules.filter((r) => r.id !== id)
flushSave()
+ armPoll()
return before !== config.rules.length
}
@@ -1354,6 +1522,7 @@ export function listAlertHistory(limit = 50) {
}
export function getAlertsStatus() {
+ const needs = pollNeeds()
return {
enabled: config.enabled,
channelCount: config.channels.filter((c) => c.enabled).length,
@@ -1361,6 +1530,10 @@ export function getAlertsStatus() {
lastDaemonOk,
historyCount: history.length,
pollIntervalMs: config.pollIntervalMs,
+ pollActive: Boolean(pollTimer),
+ pollInFlight,
+ pollNeeds: needs,
+ eventQueueDepth: eventQueue.length,
quietHoursActive: inQuietHours(),
rateLimitPerMinute: config.rateLimitPerMinute,
}
diff --git a/server/services/stats.js b/server/services/stats.js
index a312290..1c3ef3c 100644
--- a/server/services/stats.js
+++ b/server/services/stats.js
@@ -1,15 +1,18 @@
/**
* Container stats collection and broadcast.
*
- * Uses **one-shot** Docker stats (stream:false) with bounded concurrency —
- * not perpetual `stats({ stream: true })` attachments.
+ * Strategy (tuned against dockerd load, informed by Dozzle's approach):
*
- * Why: each live stats stream forces dockerd to sample cgroups ~1 Hz for that
- * container for as long as the stream is open. With N running containers that
- * is N concurrent streams and is a well-known host CPU / load-average killer.
- * One-shot polls only work when we ask, and we can throttle concurrency.
- *
- * Collection only runs while peers.size > 0 (paused when idle).
+ * - **One-shot** `stats({ stream: false })` — no perpetual stream:true attachments.
+ * Dozzle uses long-lived streams for all running containers and documents
+ * elevated dockerd CPU as expected; we intentionally avoid that.
+ * - **Round-robin batches** — each tick samples at most `samplesPerTick`
+ * containers (not the entire fleet). Full-fleet one-shot sweeps are often
+ * *worse* than streams when sweep time > interval (dockerd never idles).
+ * - **Post-sweep spacing** — next tick is scheduled only after the previous
+ * finishes + collectIntervalMs (not a fixed setInterval that overlaps work).
+ * - **Roster cache** — listContainers is not called every tick.
+ * - **Peer-idle pause** — no collection when peers.size === 0.
*/
import fs from 'fs'
import path from 'path'
@@ -28,6 +31,8 @@ export const STATS_LIMITS = Object.freeze({
collectIntervalMs: { min: 1000, max: 60_000 },
broadcastIntervalMs: { min: 1000, max: 60_000 },
concurrency: { min: 1, max: 32 },
+ samplesPerTick: { min: 1, max: 100 },
+ listIntervalMs: { min: 2000, max: 120_000 },
cacheTtlMs: { min: 0, max: 30_000 },
warnContainers: { min: 10, max: 10_000 },
})
@@ -45,10 +50,13 @@ function clampInt(value, min, max, fallback) {
/**
* Built-in defaults, then env overrides (bootstrap before disk / UI).
+ * Conservative defaults keep dockerd idle between batches on busy hosts.
* @returns {{
* collectIntervalMs: number,
* broadcastIntervalMs: number,
* concurrency: number,
+ * samplesPerTick: number,
+ * listIntervalMs: number,
* cacheTtlMs: number,
* warnContainers: number,
* }}
@@ -57,29 +65,41 @@ export function defaultStatsConfig() {
const collectIntervalMs = clampInt(
envInt(
'PEARDOCK_STATS_INTERVAL_MS',
- CONFIG.STATS?.ACTIVE_INTERVAL_MS ?? CONFIG.STATS?.INTERVAL_MS ?? 2000
+ CONFIG.STATS?.ACTIVE_INTERVAL_MS ?? CONFIG.STATS?.INTERVAL_MS ?? 5000
),
STATS_LIMITS.collectIntervalMs.min,
STATS_LIMITS.collectIntervalMs.max,
- 2000
+ 5000
)
const broadcastIntervalMs = clampInt(
- envInt('PEARDOCK_STATS_BROADCAST_MS', CONFIG.STATS?.INTERVAL_MS ?? 2000),
+ envInt('PEARDOCK_STATS_BROADCAST_MS', CONFIG.STATS?.INTERVAL_MS ?? 5000),
STATS_LIMITS.broadcastIntervalMs.min,
STATS_LIMITS.broadcastIntervalMs.max,
- 2000
+ 5000
)
const concurrency = clampInt(
- envInt('PEARDOCK_STATS_CONCURRENCY', CONFIG.STATS?.CONCURRENCY ?? 6),
+ envInt('PEARDOCK_STATS_CONCURRENCY', CONFIG.STATS?.CONCURRENCY ?? 2),
STATS_LIMITS.concurrency.min,
STATS_LIMITS.concurrency.max,
- 6
+ 2
+ )
+ const samplesPerTick = clampInt(
+ envInt('PEARDOCK_STATS_SAMPLES_PER_TICK', CONFIG.STATS?.SAMPLES_PER_TICK ?? 4),
+ STATS_LIMITS.samplesPerTick.min,
+ STATS_LIMITS.samplesPerTick.max,
+ 4
+ )
+ const listIntervalMs = clampInt(
+ envInt('PEARDOCK_STATS_LIST_INTERVAL_MS', CONFIG.STATS?.LIST_INTERVAL_MS ?? 15_000),
+ STATS_LIMITS.listIntervalMs.min,
+ STATS_LIMITS.listIntervalMs.max,
+ 15_000
)
const cacheTtlMs = clampInt(
- envInt('PEARDOCK_STATS_CACHE_TTL_MS', CONFIG.STATS?.CACHE_TTL_MS ?? 1000),
+ envInt('PEARDOCK_STATS_CACHE_TTL_MS', CONFIG.STATS?.CACHE_TTL_MS ?? 2000),
STATS_LIMITS.cacheTtlMs.min,
STATS_LIMITS.cacheTtlMs.max,
- 1000
+ 2000
)
const warnContainers = clampInt(
envInt('PEARDOCK_STATS_WARN_CONTAINERS', 80),
@@ -91,6 +111,8 @@ export function defaultStatsConfig() {
collectIntervalMs,
broadcastIntervalMs,
concurrency,
+ samplesPerTick,
+ listIntervalMs,
cacheTtlMs,
warnContainers,
}
@@ -120,6 +142,8 @@ function saveStatsDisk() {
collectIntervalMs: runtime.collectIntervalMs,
broadcastIntervalMs: runtime.broadcastIntervalMs,
concurrency: runtime.concurrency,
+ samplesPerTick: runtime.samplesPerTick,
+ listIntervalMs: runtime.listIntervalMs,
cacheTtlMs: runtime.cacheTtlMs,
warnContainers: runtime.warnContainers,
}
@@ -138,7 +162,7 @@ function saveStatsDisk() {
*/
export function normalizeStatsConfig(partial = {}, base = defaultStatsConfig()) {
const src = partial && typeof partial === 'object' ? partial : {}
- // Accept seconds from UI as collectIntervalSec / broadcastIntervalSec
+ // Accept seconds from UI as collectIntervalSec / broadcastIntervalSec / listIntervalSec
let collectMs = src.collectIntervalMs
if (collectMs == null && src.collectIntervalSec != null) {
collectMs = Number(src.collectIntervalSec) * 1000
@@ -147,6 +171,10 @@ export function normalizeStatsConfig(partial = {}, base = defaultStatsConfig())
if (broadcastMs == null && src.broadcastIntervalSec != null) {
broadcastMs = Number(src.broadcastIntervalSec) * 1000
}
+ let listMs = src.listIntervalMs
+ if (listMs == null && src.listIntervalSec != null) {
+ listMs = Number(src.listIntervalSec) * 1000
+ }
return {
collectIntervalMs: clampInt(
collectMs ?? base.collectIntervalMs,
@@ -166,6 +194,18 @@ export function normalizeStatsConfig(partial = {}, base = defaultStatsConfig())
STATS_LIMITS.concurrency.max,
base.concurrency
),
+ samplesPerTick: clampInt(
+ src.samplesPerTick ?? base.samplesPerTick,
+ STATS_LIMITS.samplesPerTick.min,
+ STATS_LIMITS.samplesPerTick.max,
+ base.samplesPerTick
+ ),
+ listIntervalMs: clampInt(
+ listMs ?? base.listIntervalMs,
+ STATS_LIMITS.listIntervalMs.min,
+ STATS_LIMITS.listIntervalMs.max,
+ base.listIntervalMs
+ ),
cacheTtlMs: clampInt(
src.cacheTtlMs ?? base.cacheTtlMs,
STATS_LIMITS.cacheTtlMs.min,
@@ -210,13 +250,17 @@ export function getStatsConfig() {
collectIntervalMs: runtime.collectIntervalMs,
broadcastIntervalMs: runtime.broadcastIntervalMs,
concurrency: runtime.concurrency,
+ samplesPerTick: runtime.samplesPerTick,
+ listIntervalMs: runtime.listIntervalMs,
cacheTtlMs: runtime.cacheTtlMs,
warnContainers: runtime.warnContainers,
// Convenience for UI
collectIntervalSec: Math.round(runtime.collectIntervalMs / 1000),
broadcastIntervalSec: Math.round(runtime.broadcastIntervalMs / 1000),
- active: Boolean(intervalHandle),
+ listIntervalSec: Math.round(runtime.listIntervalMs / 1000),
+ active: collectionLoopActive,
peers: peers.size,
+ runningTracked: rosterRunning.length,
limits: { ...STATS_LIMITS },
path: STATS_FILE,
}
@@ -232,35 +276,21 @@ export function updateStatsConfig(partial = {}, opts = {}) {
ensureStatsConfigLoaded()
const base = opts.replace ? defaultStatsConfig() : { ...runtime }
const next = normalizeStatsConfig(partial, base)
- const intervalChanged = next.collectIntervalMs !== runtime.collectIntervalMs
runtime = next
if (opts.persist !== false) saveStatsDisk()
- if (intervalChanged && intervalHandle) {
- rearmCollectInterval()
- }
+ // Interval is applied on the next schedule after a tick completes
logger.info('Stats config updated', {
collectIntervalMs: runtime.collectIntervalMs,
broadcastIntervalMs: runtime.broadcastIntervalMs,
concurrency: runtime.concurrency,
+ samplesPerTick: runtime.samplesPerTick,
+ listIntervalMs: runtime.listIntervalMs,
cacheTtlMs: runtime.cacheTtlMs,
persist: opts.persist !== false,
})
return getStatsConfig()
}
-/** Rearm setInterval after collect interval change (while peers connected). */
-function rearmCollectInterval() {
- if (!intervalHandle) return
- clearInterval(intervalHandle)
- intervalHandle = setInterval(() => {
- statsTick().catch(() => {})
- }, runtime.collectIntervalMs)
- if (typeof intervalHandle.unref === 'function') intervalHandle.unref()
- logger.debug('Stats collect interval rearmed', {
- intervalMs: runtime.collectIntervalMs,
- })
-}
-
export function getStatsFilePath() {
return STATS_FILE
}
@@ -270,7 +300,10 @@ const containerStats = {}
const statsCache = new Map()
const containerActivity = new Map()
-let intervalHandle = null
+/** @type {ReturnType|null} */
+let loopTimer = null
+/** True while peer-connected collection loop is armed */
+let collectionLoopActive = false
/** Peer-registry unsubscribe; set while stats service is armed */
let unsubPeers = null
/** True after startStatsBroadcast(); false after stopStatsBroadcast() */
@@ -282,6 +315,14 @@ let dockerBackoffUntil = 0
let tickInFlight = false
let largeFleetWarned = false
+/** Cached running containers from last listContainers */
+let rosterRunning = []
+/** @type {Set} */
+let rosterAllIds = new Set()
+let rosterFetchedAt = 0
+/** Round-robin cursor into rosterRunning */
+let rrIndex = 0
+
/**
* Docker Engine CPU % (same idea as `docker stats`).
* One-shot stats populate precpu_stats after ~1s wait inside the engine.
@@ -580,30 +621,41 @@ export function destroyStatsForContainer(id) {
}
/**
- * List once, sample running containers with bounded one-shot stats.
- * No perpetual streams — dockerd only does work while we request samples.
+ * Refresh container roster from Docker (throttled).
+ * @param {boolean} [force]
*/
-async function collectContainerStats() {
- // Single list (all) — derive running from State; was 2× listContainers/tick
+async function refreshRoster(force = false) {
+ const now = Date.now()
+ if (!force && rosterFetchedAt && now - rosterFetchedAt < runtime.listIntervalMs) {
+ return
+ }
+
const all = await docker.listContainers({ all: true })
- const running = all.filter((c) => String(c.State || '').toLowerCase() === 'running')
+ const running = all
+ .filter((c) => String(c.State || '').toLowerCase() === 'running')
+ .sort((a, b) => String(a.Id).localeCompare(String(b.Id)))
const allIds = new Set(all.map((c) => c.Id))
const runningIds = new Set(running.map((c) => c.Id))
+ rosterRunning = running
+ rosterAllIds = allIds
+ rosterFetchedAt = now
+ if (rrIndex >= running.length) rrIndex = 0
+
if (running.length >= runtime.warnContainers && !largeFleetWarned) {
largeFleetWarned = true
logger.warn(
- 'Large running fleet for stats collection — dockerd load scales with container count',
+ 'Large running fleet for stats collection — dockerd load scales with samples/tick',
{
running: running.length,
+ samplesPerTick: runtime.samplesPerTick,
concurrency: runtime.concurrency,
intervalMs: runtime.collectIntervalMs,
- hint: 'Settings → Performance, or PEARDOCK_STATS_INTERVAL_MS / PEARDOCK_STATS_CONCURRENCY',
+ hint: 'Settings → Performance: raise interval, lower samples/tick & concurrency',
}
)
}
- // Ensure entries + drop gone / zero-out stopped
for (const containerInfo of running) {
ensureStatsEntry(containerInfo)
}
@@ -625,9 +677,35 @@ async function collectContainerStats() {
}
}
}
+}
- // Bounded parallel one-shot samples (only running)
- await mapPool(running, runtime.concurrency, async (containerInfo) => {
+/**
+ * Pick next round-robin batch of running containers (at most samplesPerTick).
+ * @param {object[]} running
+ * @returns {object[]}
+ */
+function nextSampleBatch(running) {
+ if (!running.length) return []
+ const n = Math.min(runtime.samplesPerTick, running.length)
+ const batch = []
+ for (let i = 0; i < n; i++) {
+ batch.push(running[(rrIndex + i) % running.length])
+ }
+ rrIndex = (rrIndex + n) % running.length
+ return batch
+}
+
+/**
+ * Throttled roster + round-robin one-shot samples.
+ * Dockerd cost ≈ samplesPerTick × ~1s / concurrency per tick, then idle for interval.
+ */
+async function collectContainerStats() {
+ await refreshRoster(false)
+
+ const batch = nextSampleBatch(rosterRunning)
+ if (!batch.length) return
+
+ await mapPool(batch, runtime.concurrency, async (containerInfo) => {
if (peers.size === 0) return
const statsData = containerStats[containerInfo.Id]
if (!statsData) return
@@ -654,17 +732,26 @@ function destroyAllStatsEntries() {
statsCache.clear()
containerActivity.clear()
largeFleetWarned = false
+ rosterRunning = []
+ rosterAllIds = new Set()
+ rosterFetchedAt = 0
+ rrIndex = 0
+}
+
+function clearLoopTimer() {
+ if (loopTimer) {
+ clearTimeout(loopTimer)
+ loopTimer = null
+ }
}
/**
- * Pause collection: stop interval and drop in-memory state.
+ * Pause collection: stop loop and drop in-memory state.
* Safe to call when already paused.
*/
export function pauseStatsCollection() {
- if (intervalHandle) {
- clearInterval(intervalHandle)
- intervalHandle = null
- }
+ collectionLoopActive = false
+ clearLoopTimer()
destroyAllStatsEntries()
tickInFlight = false
logger.debug('Stats collection paused (no peers)')
@@ -769,6 +856,33 @@ async function statsTick() {
}
}
+/**
+ * Run one stats tick, then schedule the next after collectIntervalMs.
+ * Spacing is measured from tick *completion* so long sweeps cannot overlap.
+ */
+function runLoopTick() {
+ if (!collectionLoopActive || !serviceArmed || peers.size === 0) {
+ collectionLoopActive = false
+ clearLoopTimer()
+ return
+ }
+ statsTick()
+ .catch(() => {})
+ .finally(() => {
+ if (!collectionLoopActive || !serviceArmed || peers.size === 0) {
+ collectionLoopActive = false
+ clearLoopTimer()
+ return
+ }
+ clearLoopTimer()
+ loopTimer = setTimeout(() => {
+ loopTimer = null
+ runLoopTick()
+ }, runtime.collectIntervalMs)
+ if (typeof loopTimer.unref === 'function') loopTimer.unref()
+ })
+}
+
/**
* Resume collection when at least one peer is connected.
* Idempotent; kicks an immediate tick so first client is not waiting a full interval.
@@ -776,20 +890,19 @@ async function statsTick() {
export function resumeStatsCollection() {
if (!serviceArmed) return
if (peers.size === 0) return
- if (!intervalHandle) {
- intervalHandle = setInterval(() => {
- statsTick().catch(() => {})
- }, runtime.collectIntervalMs)
- if (typeof intervalHandle.unref === 'function') intervalHandle.unref()
- logger.debug('Stats collection resumed', {
- peers: peers.size,
- intervalMs: runtime.collectIntervalMs,
- concurrency: runtime.concurrency,
- })
- }
- // Immediate kick so first connected client gets samples ASAP
+ if (collectionLoopActive) return
+ collectionLoopActive = true
lastBroadcast = 0
- statsTick().catch(() => {})
+ logger.debug('Stats collection resumed', {
+ peers: peers.size,
+ intervalMs: runtime.collectIntervalMs,
+ concurrency: runtime.concurrency,
+ samplesPerTick: runtime.samplesPerTick,
+ listIntervalMs: runtime.listIntervalMs,
+ })
+ // Force fresh roster on first tick after resume
+ rosterFetchedAt = 0
+ runLoopTick()
}
function onPeerCountChange(size, prevSize) {
@@ -842,9 +955,9 @@ export function stopStatsBroadcast() {
pauseStatsCollection()
}
-/** @returns {boolean} whether the collect interval is currently running */
+/** @returns {boolean} whether the collect loop is currently armed */
export function isStatsCollectionActive() {
- return Boolean(intervalHandle)
+ return collectionLoopActive
}
/** Test/ops helpers — same shape as getStatsConfig subset */
@@ -854,6 +967,8 @@ export function getStatsRuntimeConfig() {
collectIntervalMs: runtime.collectIntervalMs,
broadcastIntervalMs: runtime.broadcastIntervalMs,
concurrency: runtime.concurrency,
+ samplesPerTick: runtime.samplesPerTick,
+ listIntervalMs: runtime.listIntervalMs,
cacheTtlMs: runtime.cacheTtlMs,
warnContainers: runtime.warnContainers,
}
diff --git a/test/alerts.test.js b/test/alerts.test.js
index 0e98058..eebaa93 100644
--- a/test/alerts.test.js
+++ b/test/alerts.test.js
@@ -90,6 +90,49 @@ test('alerts update global settings persists', async (t) => {
mod2.stopAlerts()
})
+test('alerts poll interval clamps to min 60s', async (t) => {
+ withTempAlerts(t)
+ const mod = await import('../server/services/alerts.js?' + Date.now() + 'pollclamp')
+ mod.startAlerts()
+ mod.updateAlertsConfig({ pollIntervalMs: 5_000 })
+ t.is(mod.getAlertsConfig().pollIntervalMs, 60_000, 'floor is 60s')
+ mod.updateAlertsConfig({ pollIntervalMs: 180_000 })
+ t.is(mod.getAlertsConfig().pollIntervalMs, 180_000)
+ mod.stopAlerts()
+})
+
+test('alerts ignore noisy docker events (exec/attach)', async (t) => {
+ withTempAlerts(t)
+ const mod = await import('../server/services/alerts.js?' + Date.now() + 'noisy')
+ mod.startAlerts()
+ // Fire noisy events — should not throw and should not enqueue work
+ mod.onDockerEvent({
+ Type: 'container',
+ Action: 'exec_create',
+ id: 'abc',
+ Actor: { Attributes: { name: 'x' } },
+ })
+ mod.onDockerEvent({
+ Type: 'container',
+ Action: 'attach',
+ id: 'abc',
+ Actor: { Attributes: { name: 'x' } },
+ })
+ const st = mod.getAlertsStatus()
+ t.is(st.eventQueueDepth, 0, 'noisy events dropped before queue')
+ // Relevant event should queue / process
+ mod.onDockerEvent({
+ Type: 'container',
+ Action: 'die',
+ id: 'dead1',
+ Actor: { Attributes: { name: 'boom' } },
+ })
+ // Allow microtask drain
+ await new Promise((r) => setTimeout(r, 30))
+ t.ok(mod.listAlertHistory(5).length >= 1, 'die event fires alert history')
+ mod.stopAlerts()
+})
+
test('protocol exposes alert methods and push', async (t) => {
const { Methods, MethodRoles, Pushes, Roles } = await import('../shared/protocol.js')
t.is(Methods.getAlertsConfig, 'getAlertsConfig')
diff --git a/test/stats-config.test.js b/test/stats-config.test.js
index 868cb46..d92689c 100644
--- a/test/stats-config.test.js
+++ b/test/stats-config.test.js
@@ -17,12 +17,16 @@ test('normalizeStatsConfig clamps out-of-range values', (t) => {
collectIntervalMs: 50,
broadcastIntervalMs: 999_999,
concurrency: 100,
+ samplesPerTick: 0,
+ listIntervalMs: 100,
cacheTtlMs: -5,
warnContainers: 1,
})
t.is(n.collectIntervalMs, STATS_LIMITS.collectIntervalMs.min)
t.is(n.broadcastIntervalMs, STATS_LIMITS.broadcastIntervalMs.max)
t.is(n.concurrency, STATS_LIMITS.concurrency.max)
+ t.is(n.samplesPerTick, STATS_LIMITS.samplesPerTick.min)
+ t.is(n.listIntervalMs, STATS_LIMITS.listIntervalMs.min)
t.is(n.cacheTtlMs, STATS_LIMITS.cacheTtlMs.min)
t.is(n.warnContainers, STATS_LIMITS.warnContainers.min)
})
@@ -31,11 +35,15 @@ test('normalizeStatsConfig accepts seconds aliases from UI', (t) => {
const n = normalizeStatsConfig({
collectIntervalSec: 5,
broadcastIntervalSec: 3,
+ listIntervalSec: 20,
concurrency: 4,
+ samplesPerTick: 8,
})
t.is(n.collectIntervalMs, 5000)
t.is(n.broadcastIntervalMs, 3000)
+ t.is(n.listIntervalMs, 20000)
t.is(n.concurrency, 4)
+ t.is(n.samplesPerTick, 8)
})
test('defaultStatsConfig is within limits', (t) => {
diff --git a/ui/alerts-settings.js b/ui/alerts-settings.js
index da0df69..7a2aec1 100644
--- a/ui/alerts-settings.js
+++ b/ui/alerts-settings.js
@@ -100,7 +100,7 @@ function renderAlertsGlobal(config, status) {
const min = document.getElementById('alerts-min-severity')
if (min) min.value = config.minSeverity || 'info'
const poll = document.getElementById('alerts-poll-ms')
- if (poll) poll.value = String(Math.round((config.pollIntervalMs || 60000) / 1000))
+ if (poll) poll.value = String(Math.round((config.pollIntervalMs || 120000) / 1000))
const rate = document.getElementById('alerts-rate-limit')
if (rate) rate.value = String(config.rateLimitPerMinute || 40)
const qh = document.getElementById('alerts-quiet-enabled')
@@ -120,10 +120,14 @@ function renderAlertsGlobal(config, status) {
const statusEl = document.getElementById('alerts-engine-status')
if (statusEl && status) {
+ const pollLabel = status.pollActive
+ ? `poll ${Math.round((status.pollIntervalMs || 0) / 1000)}s`
+ : 'poll off'
const parts = [
status.enabled ? 'Enabled' : 'Disabled',
`${status.channelCount} channel(s)`,
`${status.ruleCount} rule(s)`,
+ pollLabel,
status.lastDaemonOk ? 'Docker OK' : 'Docker DOWN',
status.quietHoursActive ? 'Quiet hours' : null,
].filter(Boolean)
@@ -373,7 +377,7 @@ function readGlobalConfigFromUi() {
enabled: document.getElementById('alerts-enabled')?.value !== '0',
minSeverity: document.getElementById('alerts-min-severity')?.value || 'info',
pollIntervalMs:
- Math.max(15, Number(document.getElementById('alerts-poll-ms')?.value) || 60) * 1000,
+ Math.max(60, Number(document.getElementById('alerts-poll-ms')?.value) || 120) * 1000,
rateLimitPerMinute: Number(document.getElementById('alerts-rate-limit')?.value) || 40,
includeHostname: document.getElementById('alerts-include-hostname')?.value !== '0',
quietHours: {
diff --git a/ui/ops-app.js b/ui/ops-app.js
index bfefd8b..14be7b4 100644
--- a/ui/ops-app.js
+++ b/ui/ops-app.js
@@ -2518,7 +2518,9 @@ function isServerStatsLiveField(el) {
return (
id === 'stats-collect-sec' ||
id === 'stats-broadcast-sec' ||
+ id === 'stats-samples-per-tick' ||
id === 'stats-concurrency' ||
+ id === 'stats-list-sec' ||
id === 'stats-cache-ttl-ms' ||
id === 'stats-warn-containers'
)
diff --git a/ui/stats-settings.js b/ui/stats-settings.js
index 1746761..52f611e 100644
--- a/ui/stats-settings.js
+++ b/ui/stats-settings.js
@@ -75,34 +75,44 @@ function renderStatsForm(config) {
const collect = document.getElementById('stats-collect-sec')
if (collect) {
collect.value = String(
- config.collectIntervalSec ?? Math.round((config.collectIntervalMs || 2000) / 1000)
+ config.collectIntervalSec ?? Math.round((config.collectIntervalMs || 5000) / 1000)
)
}
const broadcast = document.getElementById('stats-broadcast-sec')
if (broadcast) {
broadcast.value = String(
- config.broadcastIntervalSec ?? Math.round((config.broadcastIntervalMs || 2000) / 1000)
+ config.broadcastIntervalSec ?? Math.round((config.broadcastIntervalMs || 5000) / 1000)
)
}
+ const samples = document.getElementById('stats-samples-per-tick')
+ if (samples) samples.value = String(config.samplesPerTick ?? 4)
const conc = document.getElementById('stats-concurrency')
- if (conc) conc.value = String(config.concurrency ?? 6)
+ if (conc) conc.value = String(config.concurrency ?? 2)
+ const listSec = document.getElementById('stats-list-sec')
+ if (listSec) {
+ listSec.value = String(
+ config.listIntervalSec ?? Math.round((config.listIntervalMs || 15000) / 1000)
+ )
+ }
const cache = document.getElementById('stats-cache-ttl-ms')
- if (cache) cache.value = String(config.cacheTtlMs ?? 1000)
+ if (cache) cache.value = String(config.cacheTtlMs ?? 2000)
const warn = document.getElementById('stats-warn-containers')
if (warn) warn.value = String(config.warnContainers ?? 80)
const active = config.active ? 'collecting' : 'idle (no peers or paused)'
setEngineStatus(
- `${active} · every ${config.collectIntervalSec ?? '?'}s · ×${config.concurrency ?? '?'}`
+ `${active} · gap ${config.collectIntervalSec ?? '?'}s · ${config.samplesPerTick ?? '?'} samples · ×${config.concurrency ?? '?'}`
)
const meta = document.getElementById('stats-runtime-meta')
if (meta) {
const parts = [
`peers=${config.peers ?? 0}`,
- `collect=${config.collectIntervalMs}ms`,
- `broadcast=${config.broadcastIntervalMs}ms`,
+ `running=${config.runningTracked ?? '?'}`,
+ `gap=${config.collectIntervalMs}ms`,
+ `samples/tick=${config.samplesPerTick}`,
`concurrency=${config.concurrency}`,
- `cacheTtl=${config.cacheTtlMs}ms`,
+ `list=${config.listIntervalMs}ms`,
+ `broadcast=${config.broadcastIntervalMs}ms`,
]
if (config.path) parts.push(`file=${config.path}`)
meta.textContent = parts.join(' · ')
@@ -117,13 +127,17 @@ function renderStatsForm(config) {
function readFormPartial() {
const collectSec = Number(document.getElementById('stats-collect-sec')?.value)
const broadcastSec = Number(document.getElementById('stats-broadcast-sec')?.value)
+ const samplesPerTick = Number(document.getElementById('stats-samples-per-tick')?.value)
const concurrency = Number(document.getElementById('stats-concurrency')?.value)
+ const listSec = Number(document.getElementById('stats-list-sec')?.value)
const cacheTtlMs = Number(document.getElementById('stats-cache-ttl-ms')?.value)
const warnContainers = Number(document.getElementById('stats-warn-containers')?.value)
return {
collectIntervalSec: Number.isFinite(collectSec) ? collectSec : undefined,
broadcastIntervalSec: Number.isFinite(broadcastSec) ? broadcastSec : undefined,
+ samplesPerTick: Number.isFinite(samplesPerTick) ? samplesPerTick : undefined,
concurrency: Number.isFinite(concurrency) ? concurrency : undefined,
+ listIntervalSec: Number.isFinite(listSec) ? listSec : undefined,
cacheTtlMs: Number.isFinite(cacheTtlMs) ? cacheTtlMs : undefined,
warnContainers: Number.isFinite(warnContainers) ? warnContainers : undefined,
}