Alert System Changes: Fixes applied (feature kept, load bounded)

1. Removed docker.df() entirely
2. Poll only what rules need
   • docker_daemon → cheap ping
   • container_health / stack_health → listContainers
   • resource → host statfs only (no Docker)
   • Event-only rules → no poll timer
3. Poll floor 60s, default 120s (was 15s min / 60s default)
4. No overlapping polls (pollInFlight)
5. Event path: drop noisy actions (exec_*, attach, …), match only relevant rules, serialize work (bounded queue)
6. UI status shows whether poll is active

server/services/stats.js

@@ -1,15 +1,18 @@
 /**
  * Container stats collection and broadcast.
  *
- * Uses **one-shot** Docker stats (stream:false) with bounded concurrency —
- * not perpetual `stats({ stream: true })` attachments.
+ * Strategy (tuned against dockerd load, informed by Dozzle's approach):
  *
- * Why: each live stats stream forces dockerd to sample cgroups ~1 Hz for that
- * container for as long as the stream is open. With N running containers that
- * is N concurrent streams and is a well-known host CPU / load-average killer.
- * One-shot polls only work when we ask, and we can throttle concurrency.
- *
- * Collection only runs while peers.size > 0 (paused when idle).
+ * - **One-shot** `stats({ stream: false })` — no perpetual stream:true attachments.
+ *   Dozzle uses long-lived streams for all running containers and documents
+ *   elevated dockerd CPU as expected; we intentionally avoid that.
+ * - **Round-robin batches** — each tick samples at most `samplesPerTick`
+ *   containers (not the entire fleet). Full-fleet one-shot sweeps are often
+ *   *worse* than streams when sweep time > interval (dockerd never idles).
+ * - **Post-sweep spacing** — next tick is scheduled only after the previous
+ *   finishes + collectIntervalMs (not a fixed setInterval that overlaps work).
+ * - **Roster cache** — listContainers is not called every tick.
+ * - **Peer-idle pause** — no collection when peers.size === 0.
  */
 import fs from 'fs'
 import path from 'path'
@@ -28,6 +31,8 @@ export const STATS_LIMITS = Object.freeze({
   collectIntervalMs: { min: 1000, max: 60_000 },
   broadcastIntervalMs: { min: 1000, max: 60_000 },
   concurrency: { min: 1, max: 32 },
+  samplesPerTick: { min: 1, max: 100 },
+  listIntervalMs: { min: 2000, max: 120_000 },
   cacheTtlMs: { min: 0, max: 30_000 },
   warnContainers: { min: 10, max: 10_000 },
 })
@@ -45,10 +50,13 @@ function clampInt(value, min, max, fallback) {
 
 /**
  * Built-in defaults, then env overrides (bootstrap before disk / UI).
+ * Conservative defaults keep dockerd idle between batches on busy hosts.
  * @returns {{
  *   collectIntervalMs: number,
  *   broadcastIntervalMs: number,
  *   concurrency: number,
+ *   samplesPerTick: number,
+ *   listIntervalMs: number,
  *   cacheTtlMs: number,
  *   warnContainers: number,
  * }}
@@ -57,29 +65,41 @@ export function defaultStatsConfig() {
   const collectIntervalMs = clampInt(
     envInt(
       'PEARDOCK_STATS_INTERVAL_MS',
-      CONFIG.STATS?.ACTIVE_INTERVAL_MS ?? CONFIG.STATS?.INTERVAL_MS ?? 2000
+      CONFIG.STATS?.ACTIVE_INTERVAL_MS ?? CONFIG.STATS?.INTERVAL_MS ?? 5000
     ),
     STATS_LIMITS.collectIntervalMs.min,
     STATS_LIMITS.collectIntervalMs.max,
-    2000
+    5000
   )
   const broadcastIntervalMs = clampInt(
-    envInt('PEARDOCK_STATS_BROADCAST_MS', CONFIG.STATS?.INTERVAL_MS ?? 2000),
+    envInt('PEARDOCK_STATS_BROADCAST_MS', CONFIG.STATS?.INTERVAL_MS ?? 5000),
     STATS_LIMITS.broadcastIntervalMs.min,
     STATS_LIMITS.broadcastIntervalMs.max,
-    2000
+    5000
   )
   const concurrency = clampInt(
-    envInt('PEARDOCK_STATS_CONCURRENCY', CONFIG.STATS?.CONCURRENCY ?? 6),
+    envInt('PEARDOCK_STATS_CONCURRENCY', CONFIG.STATS?.CONCURRENCY ?? 2),
     STATS_LIMITS.concurrency.min,
     STATS_LIMITS.concurrency.max,
-    6
+    2
+  )
+  const samplesPerTick = clampInt(
+    envInt('PEARDOCK_STATS_SAMPLES_PER_TICK', CONFIG.STATS?.SAMPLES_PER_TICK ?? 4),
+    STATS_LIMITS.samplesPerTick.min,
+    STATS_LIMITS.samplesPerTick.max,
+    4
+  )
+  const listIntervalMs = clampInt(
+    envInt('PEARDOCK_STATS_LIST_INTERVAL_MS', CONFIG.STATS?.LIST_INTERVAL_MS ?? 15_000),
+    STATS_LIMITS.listIntervalMs.min,
+    STATS_LIMITS.listIntervalMs.max,
+    15_000
   )
   const cacheTtlMs = clampInt(
-    envInt('PEARDOCK_STATS_CACHE_TTL_MS', CONFIG.STATS?.CACHE_TTL_MS ?? 1000),
+    envInt('PEARDOCK_STATS_CACHE_TTL_MS', CONFIG.STATS?.CACHE_TTL_MS ?? 2000),
     STATS_LIMITS.cacheTtlMs.min,
     STATS_LIMITS.cacheTtlMs.max,
-    1000
+    2000
   )
   const warnContainers = clampInt(
     envInt('PEARDOCK_STATS_WARN_CONTAINERS', 80),
@@ -91,6 +111,8 @@ export function defaultStatsConfig() {
     collectIntervalMs,
     broadcastIntervalMs,
     concurrency,
+    samplesPerTick,
+    listIntervalMs,
     cacheTtlMs,
     warnContainers,
   }
@@ -120,6 +142,8 @@ function saveStatsDisk() {
       collectIntervalMs: runtime.collectIntervalMs,
       broadcastIntervalMs: runtime.broadcastIntervalMs,
       concurrency: runtime.concurrency,
+      samplesPerTick: runtime.samplesPerTick,
+      listIntervalMs: runtime.listIntervalMs,
       cacheTtlMs: runtime.cacheTtlMs,
       warnContainers: runtime.warnContainers,
     }
@@ -138,7 +162,7 @@ function saveStatsDisk() {
  */
 export function normalizeStatsConfig(partial = {}, base = defaultStatsConfig()) {
   const src = partial && typeof partial === 'object' ? partial : {}
-  // Accept seconds from UI as collectIntervalSec / broadcastIntervalSec
+  // Accept seconds from UI as collectIntervalSec / broadcastIntervalSec / listIntervalSec
   let collectMs = src.collectIntervalMs
   if (collectMs == null && src.collectIntervalSec != null) {
     collectMs = Number(src.collectIntervalSec) * 1000
@@ -147,6 +171,10 @@ export function normalizeStatsConfig(partial = {}, base = defaultStatsConfig())
   if (broadcastMs == null && src.broadcastIntervalSec != null) {
     broadcastMs = Number(src.broadcastIntervalSec) * 1000
   }
+  let listMs = src.listIntervalMs
+  if (listMs == null && src.listIntervalSec != null) {
+    listMs = Number(src.listIntervalSec) * 1000
+  }
   return {
     collectIntervalMs: clampInt(
       collectMs ?? base.collectIntervalMs,
@@ -166,6 +194,18 @@ export function normalizeStatsConfig(partial = {}, base = defaultStatsConfig())
       STATS_LIMITS.concurrency.max,
       base.concurrency
     ),
+    samplesPerTick: clampInt(
+      src.samplesPerTick ?? base.samplesPerTick,
+      STATS_LIMITS.samplesPerTick.min,
+      STATS_LIMITS.samplesPerTick.max,
+      base.samplesPerTick
+    ),
+    listIntervalMs: clampInt(
+      listMs ?? base.listIntervalMs,
+      STATS_LIMITS.listIntervalMs.min,
+      STATS_LIMITS.listIntervalMs.max,
+      base.listIntervalMs
+    ),
     cacheTtlMs: clampInt(
       src.cacheTtlMs ?? base.cacheTtlMs,
       STATS_LIMITS.cacheTtlMs.min,
@@ -210,13 +250,17 @@ export function getStatsConfig() {
     collectIntervalMs: runtime.collectIntervalMs,
     broadcastIntervalMs: runtime.broadcastIntervalMs,
     concurrency: runtime.concurrency,
+    samplesPerTick: runtime.samplesPerTick,
+    listIntervalMs: runtime.listIntervalMs,
     cacheTtlMs: runtime.cacheTtlMs,
     warnContainers: runtime.warnContainers,
     // Convenience for UI
     collectIntervalSec: Math.round(runtime.collectIntervalMs / 1000),
     broadcastIntervalSec: Math.round(runtime.broadcastIntervalMs / 1000),
-    active: Boolean(intervalHandle),
+    listIntervalSec: Math.round(runtime.listIntervalMs / 1000),
+    active: collectionLoopActive,
     peers: peers.size,
+    runningTracked: rosterRunning.length,
     limits: { ...STATS_LIMITS },
     path: STATS_FILE,
   }
@@ -232,35 +276,21 @@ export function updateStatsConfig(partial = {}, opts = {}) {
   ensureStatsConfigLoaded()
   const base = opts.replace ? defaultStatsConfig() : { ...runtime }
   const next = normalizeStatsConfig(partial, base)
-  const intervalChanged = next.collectIntervalMs !== runtime.collectIntervalMs
   runtime = next
   if (opts.persist !== false) saveStatsDisk()
-  if (intervalChanged && intervalHandle) {
-    rearmCollectInterval()
-  }
+  // Interval is applied on the next schedule after a tick completes
   logger.info('Stats config updated', {
     collectIntervalMs: runtime.collectIntervalMs,
     broadcastIntervalMs: runtime.broadcastIntervalMs,
     concurrency: runtime.concurrency,
+    samplesPerTick: runtime.samplesPerTick,
+    listIntervalMs: runtime.listIntervalMs,
     cacheTtlMs: runtime.cacheTtlMs,
     persist: opts.persist !== false,
   })
   return getStatsConfig()
 }
 
-/** Rearm setInterval after collect interval change (while peers connected). */
-function rearmCollectInterval() {
-  if (!intervalHandle) return
-  clearInterval(intervalHandle)
-  intervalHandle = setInterval(() => {
-    statsTick().catch(() => {})
-  }, runtime.collectIntervalMs)
-  if (typeof intervalHandle.unref === 'function') intervalHandle.unref()
-  logger.debug('Stats collect interval rearmed', {
-    intervalMs: runtime.collectIntervalMs,
-  })
-}
-
 export function getStatsFilePath() {
   return STATS_FILE
 }
@@ -270,7 +300,10 @@ const containerStats = {}
 const statsCache = new Map()
 const containerActivity = new Map()
 
-let intervalHandle = null
+/** @type {ReturnType<typeof setTimeout>|null} */
+let loopTimer = null
+/** True while peer-connected collection loop is armed */
+let collectionLoopActive = false
 /** Peer-registry unsubscribe; set while stats service is armed */
 let unsubPeers = null
 /** True after startStatsBroadcast(); false after stopStatsBroadcast() */
@@ -282,6 +315,14 @@ let dockerBackoffUntil = 0
 let tickInFlight = false
 let largeFleetWarned = false
 
+/** Cached running containers from last listContainers */
+let rosterRunning = []
+/** @type {Set<string>} */
+let rosterAllIds = new Set()
+let rosterFetchedAt = 0
+/** Round-robin cursor into rosterRunning */
+let rrIndex = 0
+
 /**
  * Docker Engine CPU % (same idea as `docker stats`).
  * One-shot stats populate precpu_stats after ~1s wait inside the engine.
@@ -580,30 +621,41 @@ export function destroyStatsForContainer(id) {
 }
 
 /**
- * List once, sample running containers with bounded one-shot stats.
- * No perpetual streams — dockerd only does work while we request samples.
+ * Refresh container roster from Docker (throttled).
+ * @param {boolean} [force]
  */
-async function collectContainerStats() {
-  // Single list (all) — derive running from State; was 2× listContainers/tick
+async function refreshRoster(force = false) {
+  const now = Date.now()
+  if (!force && rosterFetchedAt && now - rosterFetchedAt < runtime.listIntervalMs) {
+    return
+  }
+
   const all = await docker.listContainers({ all: true })
-  const running = all.filter((c) => String(c.State || '').toLowerCase() === 'running')
+  const running = all
+    .filter((c) => String(c.State || '').toLowerCase() === 'running')
+    .sort((a, b) => String(a.Id).localeCompare(String(b.Id)))
   const allIds = new Set(all.map((c) => c.Id))
   const runningIds = new Set(running.map((c) => c.Id))
 
+  rosterRunning = running
+  rosterAllIds = allIds
+  rosterFetchedAt = now
+  if (rrIndex >= running.length) rrIndex = 0
+
   if (running.length >= runtime.warnContainers && !largeFleetWarned) {
     largeFleetWarned = true
     logger.warn(
-      'Large running fleet for stats collection — dockerd load scales with container count',
+      'Large running fleet for stats collection — dockerd load scales with samples/tick',
       {
         running: running.length,
+        samplesPerTick: runtime.samplesPerTick,
         concurrency: runtime.concurrency,
         intervalMs: runtime.collectIntervalMs,
-        hint: 'Settings → Performance, or PEARDOCK_STATS_INTERVAL_MS / PEARDOCK_STATS_CONCURRENCY',
+        hint: 'Settings → Performance: raise interval, lower samples/tick & concurrency',
       }
     )
   }
 
-  // Ensure entries + drop gone / zero-out stopped
   for (const containerInfo of running) {
     ensureStatsEntry(containerInfo)
   }
@@ -625,9 +677,35 @@ async function collectContainerStats() {
       }
     }
   }
+}
 
-  // Bounded parallel one-shot samples (only running)
-  await mapPool(running, runtime.concurrency, async (containerInfo) => {
+/**
+ * Pick next round-robin batch of running containers (at most samplesPerTick).
+ * @param {object[]} running
+ * @returns {object[]}
+ */
+function nextSampleBatch(running) {
+  if (!running.length) return []
+  const n = Math.min(runtime.samplesPerTick, running.length)
+  const batch = []
+  for (let i = 0; i < n; i++) {
+    batch.push(running[(rrIndex + i) % running.length])
+  }
+  rrIndex = (rrIndex + n) % running.length
+  return batch
+}
+
+/**
+ * Throttled roster + round-robin one-shot samples.
+ * Dockerd cost ≈ samplesPerTick × ~1s / concurrency per tick, then idle for interval.
+ */
+async function collectContainerStats() {
+  await refreshRoster(false)
+
+  const batch = nextSampleBatch(rosterRunning)
+  if (!batch.length) return
+
+  await mapPool(batch, runtime.concurrency, async (containerInfo) => {
     if (peers.size === 0) return
     const statsData = containerStats[containerInfo.Id]
     if (!statsData) return
@@ -654,17 +732,26 @@ function destroyAllStatsEntries() {
   statsCache.clear()
   containerActivity.clear()
   largeFleetWarned = false
+  rosterRunning = []
+  rosterAllIds = new Set()
+  rosterFetchedAt = 0
+  rrIndex = 0
+}
+
+function clearLoopTimer() {
+  if (loopTimer) {
+    clearTimeout(loopTimer)
+    loopTimer = null
+  }
 }
 
 /**
- * Pause collection: stop interval and drop in-memory state.
+ * Pause collection: stop loop and drop in-memory state.
  * Safe to call when already paused.
  */
 export function pauseStatsCollection() {
-  if (intervalHandle) {
-    clearInterval(intervalHandle)
-    intervalHandle = null
-  }
+  collectionLoopActive = false
+  clearLoopTimer()
   destroyAllStatsEntries()
   tickInFlight = false
   logger.debug('Stats collection paused (no peers)')
@@ -769,6 +856,33 @@ async function statsTick() {
   }
 }
 
+/**
+ * Run one stats tick, then schedule the next after collectIntervalMs.
+ * Spacing is measured from tick *completion* so long sweeps cannot overlap.
+ */
+function runLoopTick() {
+  if (!collectionLoopActive || !serviceArmed || peers.size === 0) {
+    collectionLoopActive = false
+    clearLoopTimer()
+    return
+  }
+  statsTick()
+    .catch(() => {})
+    .finally(() => {
+      if (!collectionLoopActive || !serviceArmed || peers.size === 0) {
+        collectionLoopActive = false
+        clearLoopTimer()
+        return
+      }
+      clearLoopTimer()
+      loopTimer = setTimeout(() => {
+        loopTimer = null
+        runLoopTick()
+      }, runtime.collectIntervalMs)
+      if (typeof loopTimer.unref === 'function') loopTimer.unref()
+    })
+}
+
 /**
  * Resume collection when at least one peer is connected.
  * Idempotent; kicks an immediate tick so first client is not waiting a full interval.
@@ -776,20 +890,19 @@ async function statsTick() {
 export function resumeStatsCollection() {
   if (!serviceArmed) return
   if (peers.size === 0) return
-  if (!intervalHandle) {
-    intervalHandle = setInterval(() => {
-      statsTick().catch(() => {})
-    }, runtime.collectIntervalMs)
-    if (typeof intervalHandle.unref === 'function') intervalHandle.unref()
-    logger.debug('Stats collection resumed', {
-      peers: peers.size,
-      intervalMs: runtime.collectIntervalMs,
-      concurrency: runtime.concurrency,
-    })
-  }
-  // Immediate kick so first connected client gets samples ASAP
+  if (collectionLoopActive) return
+  collectionLoopActive = true
   lastBroadcast = 0
-  statsTick().catch(() => {})
+  logger.debug('Stats collection resumed', {
+    peers: peers.size,
+    intervalMs: runtime.collectIntervalMs,
+    concurrency: runtime.concurrency,
+    samplesPerTick: runtime.samplesPerTick,
+    listIntervalMs: runtime.listIntervalMs,
+  })
+  // Force fresh roster on first tick after resume
+  rosterFetchedAt = 0
+  runLoopTick()
 }
 
 function onPeerCountChange(size, prevSize) {
@@ -842,9 +955,9 @@ export function stopStatsBroadcast() {
   pauseStatsCollection()
 }
 
-/** @returns {boolean} whether the collect interval is currently running */
+/** @returns {boolean} whether the collect loop is currently armed */
 export function isStatsCollectionActive() {
-  return Boolean(intervalHandle)
+  return collectionLoopActive
 }
 
 /** Test/ops helpers — same shape as getStatsConfig subset */
@@ -854,6 +967,8 @@ export function getStatsRuntimeConfig() {
     collectIntervalMs: runtime.collectIntervalMs,
     broadcastIntervalMs: runtime.broadcastIntervalMs,
     concurrency: runtime.concurrency,
+    samplesPerTick: runtime.samplesPerTick,
+    listIntervalMs: runtime.listIntervalMs,
     cacheTtlMs: runtime.cacheTtlMs,
     warnContainers: runtime.warnContainers,
   }