From c5015a66e8a996456921a7e14be9a933e695da23 Mon Sep 17 00:00:00 2001 From: Raven Scott Date: Tue, 4 Aug 2026 22:13:20 -0400 Subject: [PATCH] Alert System Changes: Fixes applied (feature kept, load bounded) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Removed docker.df() entirely 2. Poll only what rules need • docker_daemon → cheap ping • container_health / stack_health → listContainers • resource → host statfs only (no Docker) • Event-only rules → no poll timer 3. Poll floor 60s, default 120s (was 15s min / 60s default) 4. No overlapping polls (pollInFlight) 5. Event path: drop noisy actions (exec_*, attach, …), match only relevant rules, serialize work (bounded queue) 6. UI status shows whether poll is active --- config.js | 15 +- docs/SERVER.md | 18 +- index.html | 39 ++-- server/services/alerts.js | 419 +++++++++++++++++++++++++++----------- server/services/stats.js | 247 ++++++++++++++++------ test/alerts.test.js | 43 ++++ test/stats-config.test.js | 8 + ui/alerts-settings.js | 8 +- ui/ops-app.js | 2 + ui/stats-settings.js | 30 ++- 10 files changed, 603 insertions(+), 226 deletions(-) diff --git a/config.js b/config.js index 4f24ad5..46c6732 100644 --- a/config.js +++ b/config.js @@ -4,15 +4,16 @@ export const CONFIG = { // Stats collection (server/services/stats.js) - // Prefer one-shot polls over perpetual stream:true attachments — each live - // stream forces dockerd to sample that container ~1Hz and is a host CPU hog. - // Override at runtime: PEARDOCK_STATS_INTERVAL_MS, PEARDOCK_STATS_CONCURRENCY. + // Round-robin one-shot samples — not full-fleet stream:true (Dozzle-style streams + // keep dockerd busy ~1Hz × N containers). See Settings → Performance. STATS: { - INTERVAL_MS: 2000, // Min gap between allStats broadcasts - ACTIVE_INTERVAL_MS: 2000, // Collection tick while peers are connected + INTERVAL_MS: 5000, // Min gap between allStats broadcasts + ACTIVE_INTERVAL_MS: 5000, // Idle gap after each sample batch completes IDLE_INTERVAL_MS: 5000, // Reserved (peer-idle uses full pause) - CACHE_TTL_MS: 1000, // Per-container broadcast cache TTL - CONCURRENCY: 6, // Max parallel one-shot stats to dockerd + CACHE_TTL_MS: 2000, // Per-container broadcast cache TTL + CONCURRENCY: 2, // Max parallel one-shot stats to dockerd + SAMPLES_PER_TICK: 4, // Max containers sampled per tick (round-robin) + LIST_INTERVAL_MS: 15000, // How often to re-list containers SMOOTHING_FACTOR: 0.2, // Client-side smoothing (if used) }, diff --git a/docs/SERVER.md b/docs/SERVER.md index 702aa26..9db24ef 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -105,16 +105,22 @@ Subscribes to Engine events; fans out `push:dockerEvent` and may trigger list re Periodic container stats → `push:allStats`; history buffer for charts. -**Important:** collection uses **one-shot** `stats({ stream: false })` with bounded concurrency (default 6), not perpetual per-container streams. Live `stream: true` attachments force dockerd to sample every running container ~1 Hz and routinely pin host load average. Stats only run while `peers.size > 0`. Operators can tune intervals and concurrency from the client under **Settings → Performance** (admin) without restarting the server. +**Important:** peardock does **not** open perpetual per-container `stream: true` stats (Dozzle does, and [documents elevated dockerd CPU](https://dozzle.dev/guide/faq) as expected). Instead: + +1. **Round-robin one-shot** samples (`stream: false`) — at most `samplesPerTick` containers per batch +2. **Post-batch idle gap** — next batch waits `collectIntervalMs` after the previous finishes (no overlapping sweeps) +3. **Roster throttle** — `listContainers` only every `listIntervalMs` +4. **Peer-idle pause** — no sampling when `peers.size === 0` | Env / UI | Default | Meaning | |----------|---------|---------| -| `PEARDOCK_STATS_INTERVAL_MS` / Settings → Performance | 2000 | Collection tick while peers connected | -| `PEARDOCK_STATS_BROADCAST_MS` | 2000 | Min gap between `push:allStats` | -| `PEARDOCK_STATS_CONCURRENCY` | 6 | Max parallel one-shot stats requests | -| `PEARDOCK_STATS_CACHE_TTL_MS` | 1000 | Idle-container broadcast cache TTL | +| Collect gap / `PEARDOCK_STATS_INTERVAL_MS` | 5000 | Idle after each sample batch | +| Samples/tick / `PEARDOCK_STATS_SAMPLES_PER_TICK` | 4 | Max containers sampled per batch (main load knob) | +| Concurrency / `PEARDOCK_STATS_CONCURRENCY` | 2 | Parallel one-shots within a batch | +| List interval / `PEARDOCK_STATS_LIST_INTERVAL_MS` | 15000 | Roster refresh | +| Broadcast / `PEARDOCK_STATS_BROADCAST_MS` | 5000 | Min gap between `push:allStats` | -Runtime values persist in `peardock-stats.json` (cwd or `PEARDOCK_STATS_PATH`). RPC: `getStatsConfig` (viewer), `updateStatsConfig` (admin). The Settings → **Performance** tab applies changes live with no server restart. +Runtime values persist in `peardock-stats.json` (cwd or `PEARDOCK_STATS_PATH`). RPC: `getStatsConfig` (viewer), `updateStatsConfig` (admin). Settings → **Performance** applies live. On hot hosts: raise collect gap, keep samples/tick at 2–4 and concurrency at 1–2. ### Metrics (`services/metrics.js`) diff --git a/index.html b/index.html index e978ab7..35c1677 100644 --- a/index.html +++ b/index.html @@ -2906,33 +2906,43 @@ services:

Changes apply live on the server — no restart and no Save button.

- - -
How often the server starts a stats sweep while a client is connected. Higher = less dockerd CPU.
+ + +
Wait this long after a sample batch finishes before the next. Higher = less dockerd CPU.
- - -
Min gap between push:allStats updates to clients.
+ + +
Containers sampled each tick (round-robin). Main load control — do not set to fleet size.
- -
Max parallel one-shot stats requests to dockerd. Keep modest on busy hosts.
+ +
Max parallel one-shot stats to dockerd within a tick. Keep at 1–2 on busy hosts.
- - -
Reuse last sample for idle containers when broadcasting (0 = always fresh).
+ + +
Min gap between push:allStats to clients (can reuse last samples).
+
+ + +
Roster refresh rate (start/stop detection for the sampler).
+
+
+ + +
Reuse last sample for idle containers when broadcasting.
+
-
Log a one-time warning when this many containers are running.
+
One-time server log when this many containers are running.
-
+
@@ -3098,7 +3108,8 @@ services:
- + +
Min 60s. Only runs for daemon / health / stack / disk rules (not pure event rules).
diff --git a/server/services/alerts.js b/server/services/alerts.js index 507ea05..6de7ba2 100644 --- a/server/services/alerts.js +++ b/server/services/alerts.js @@ -3,7 +3,14 @@ * * Fully customizable rules + webhook channels (Discord, Slack, Teams, * generic HTTP, ntfy, Gotify, Telegram). Evaluates Docker events and - * periodic health polls even when no client is connected. + * light periodic health polls even when no client is connected. + * + * Host-load rules (keep this cheap on dockerd): + * - Never call docker.df() (expensive, unused). + * - Poll only kinds that have enabled rules (skip listContainers when unused). + * - Min poll interval 60s (default 120s); never overlap polls. + * - Docker event path filters noise (exec_*, attach, …) and serializes work. + * - Event-only rules need no Docker polling. * * Config (mode 0600), first match wins: * PEARDOCK_ALERTS_PATH @@ -45,10 +52,36 @@ const FILE = resolveAlertsFilePath() const LEGACY_CWD_FILE = path.join(process.cwd(), 'peardock-alerts.json') const HISTORY_MAX = 200 -const DEFAULT_POLL_MS = 60_000 +/** Default health poll — 2 minutes (was 60s). */ +const DEFAULT_POLL_MS = 120_000 +/** Hard floor — sub-minute listContainers/ping loops thrash dockerd. */ +const MIN_POLL_MS = 60_000 +const MAX_POLL_MS = 3_600_000 +/** Drop oldest under event flood rather than unbounded memory. */ +const EVENT_QUEUE_MAX = 200 const SEVERITIES = ['info', 'warning', 'critical'] const SEV_RANK = { info: 1, warning: 2, critical: 3 } +/** + * Docker event actions that never match default peardock alert rules and are + * extremely chatty (terminals, attach, top). Skip before evaluateRules. + */ +const NOISY_DOCKER_ACTIONS = new Set([ + 'exec_create', + 'exec_start', + 'exec_die', + 'exec_detach', + 'attach', + 'detach', + 'resize', + 'top', + 'export', + 'commit', + 'copy', + 'archive-path', + 'extract-to-dir', +]) + const CHANNEL_TYPES = new Set([ 'discord', 'slack', @@ -85,6 +118,11 @@ let deliveriesThisMinute = 0 let deliveriesMinuteStart = Date.now() let lastDaemonOk = true let started = false +/** Prevent overlapping pollHealth (setInterval + slow listContainers). */ +let pollInFlight = false +/** Serialize docker-event alert evaluation under noisy fleets. */ +let eventQueue = [] +let eventDraining = false function newId(prefix) { return `${prefix}_${Date.now().toString(36)}_${randomBytes(3).toString('hex')}` @@ -316,8 +354,8 @@ function normalizeConfig(raw) { version: 1, enabled: r.enabled !== false, pollIntervalMs: Math.max( - 15_000, - Math.min(3_600_000, Number(r.pollIntervalMs) || DEFAULT_POLL_MS) + MIN_POLL_MS, + Math.min(MAX_POLL_MS, Number(r.pollIntervalMs) || DEFAULT_POLL_MS) ), minSeverity: normalizeSeverity(r.minSeverity, 'info'), rateLimitPerMinute: Math.max(1, Math.min(300, Number(r.rateLimitPerMinute) || 40)), @@ -1031,141 +1069,260 @@ async function evaluateRules(kind, ctx) { } /** - * Called from Docker event stream. - * @param {object} event + * @param {string} kind + * @returns {boolean} */ -export async function onDockerEvent(event) { - if (!event || !config.enabled) return - try { - await evaluateRules('docker_event', event) - // Health status often arrives as container health_status event - const action = String(event.Action || event.status || '') - if (event.Type === 'container' && action.startsWith('health_status')) { - const health = action.includes(':') - ? action.split(':').slice(1).join(':').trim() - : event.status - await evaluateRules('container_health', { - id: event.id || event.Actor?.ID, - name: event.Actor?.Attributes?.name, - labels: event.Actor?.Attributes, - health: health || event.Actor?.Attributes?.health_status, - }) - } - } catch (err) { - logger.warn('alerts: onDockerEvent failed', { error: err.message }) +function hasEnabledRuleKind(kind) { + return config.rules.some( + (r) => + r.enabled && + r.kind === kind && + severityAtLeast(r.severity, config.minSeverity) + ) +} + +/** + * Kinds that require a periodic Docker / host poll (not pure events). + */ +function pollNeeds() { + return { + daemon: hasEnabledRuleKind('docker_daemon'), + list: hasEnabledRuleKind('container_health') || hasEnabledRuleKind('stack_health'), + resource: hasEnabledRuleKind('resource'), } } +/** + * Whether this Docker event could fire any enabled alert rule. + * Drops terminal/exec noise that was previously evaluated on every event. + * @param {object} event + */ +function isAlertRelevantEvent(event) { + if (!event) return false + const type = String(event.Type || event.type || '') + const actionRaw = String(event.Action || event.status || '') + const action = actionRaw.split(':')[0] + + if (NOISY_DOCKER_ACTIONS.has(action)) return false + + // Healthcheck transitions → container_health rules + if ( + type === 'container' && + actionRaw.startsWith('health_status') && + hasEnabledRuleKind('container_health') + ) { + return true + } + + if (!hasEnabledRuleKind('docker_event')) return false + + for (const rule of config.rules) { + if (!rule.enabled || rule.kind !== 'docker_event') continue + if (!severityAtLeast(rule.severity, config.minSeverity)) continue + const match = rule.match || {} + if (Array.isArray(match.types) && match.types.length && !match.types.includes(type)) { + continue + } + if ( + Array.isArray(match.actions) && + match.actions.length && + !match.actions.includes(action) + ) { + continue + } + return true + } + return false +} + +/** + * Process one Docker event for alert rules (awaited). + * @param {object} event + */ +async function processDockerEvent(event) { + await evaluateRules('docker_event', event) + const action = String(event.Action || event.status || '') + if (event.Type === 'container' && action.startsWith('health_status')) { + const health = action.includes(':') + ? action.split(':').slice(1).join(':').trim() + : event.status + await evaluateRules('container_health', { + id: event.id || event.Actor?.ID, + name: event.Actor?.Attributes?.name, + labels: event.Actor?.Attributes, + health: health || event.Actor?.Attributes?.health_status, + }) + } +} + +async function drainEventQueue() { + if (eventDraining) return + eventDraining = true + try { + while (eventQueue.length) { + const ev = eventQueue.shift() + if (!ev || !config.enabled) continue + try { + await processDockerEvent(ev) + } catch (err) { + logger.warn('alerts: onDockerEvent failed', { error: err.message }) + } + } + } finally { + eventDraining = false + // Work may have been enqueued while draining + if (eventQueue.length) { + drainEventQueue().catch(() => {}) + } + } +} + +/** + * Called from Docker event stream (fire-and-forget from events.js). + * Cheap sync filter + bounded queue; never piles concurrent evaluate/fire work. + * @param {object} event + */ +export function onDockerEvent(event) { + if (!event || !config.enabled) return + if (!isAlertRelevantEvent(event)) return + if (eventQueue.length >= EVENT_QUEUE_MAX) { + eventQueue.shift() + } + eventQueue.push(event) + drainEventQueue().catch((err) => { + logger.debug('alerts: event drain error', { error: err?.message }) + }) +} + +/** + * Periodic health poll. Skips Docker work for disabled rule kinds. + * Never calls docker.df (expensive and unused). + */ async function pollHealth() { if (!config.enabled) return - - // Docker daemon - try { - await docker.ping() - lastDaemonOk = true - await evaluateRules('docker_daemon', { ok: true }) - } catch (err) { - lastDaemonOk = false - await evaluateRules('docker_daemon', { - ok: false, - error: err.message || String(err), - }) - return // skip further polls if docker is down + if (pollInFlight) { + logger.debug('alerts: poll skipped (previous still running)') + return + } + const needs = pollNeeds() + if (!needs.daemon && !needs.list && !needs.resource) { + // Only docker_event / peardock rules — pure event path, no polling work + return } - // One listContainers for both container_health and stack_health (half the socket work) - /** @type {object[]} */ - let containers = [] + pollInFlight = true try { - containers = (await docker.listContainers({ all: true })) || [] - } catch (err) { - logger.debug('alerts: container list poll failed', { error: err.message }) - } - - // Container health - try { - for (const c of containers) { - const health = c.Status?.match(/\((healthy|unhealthy|health: starting)\)/i)?.[1] - || c.State // running/exited — Health may be in inspect only - // Prefer Health from inspect-lite if present - const h = - c.Health?.Status || - (typeof health === 'string' ? health.toLowerCase().replace('health: ', '') : null) - if (!h || h === 'none') continue - const name = (c.Names?.[0] || '').replace(/^\//, '') || c.Id?.slice(0, 12) - await evaluateRules('container_health', { - id: c.Id, - name, - labels: c.Labels, - health: h === 'starting' ? 'starting' : h, - }) - } - } catch (err) { - logger.debug('alerts: container health poll failed', { error: err.message }) - } - - // Stack / compose project health (group by com.docker.compose.project) - try { - /** @type {Map} */ - const stacks = new Map() - for (const c of containers) { - const project = - c.Labels?.['com.docker.compose.project'] || - c.Labels?.['com.docker.stack.namespace'] - if (!project) continue - let s = stacks.get(project) - if (!s) { - s = { name: project, running: 0, total: 0, unhealthy: 0 } - stacks.set(project, s) + if (needs.daemon) { + try { + await docker.ping() + lastDaemonOk = true + await evaluateRules('docker_daemon', { ok: true }) + } catch (err) { + lastDaemonOk = false + await evaluateRules('docker_daemon', { + ok: false, + error: err.message || String(err), + }) + return // skip list/resource if docker is down } - s.total += 1 - if (c.State === 'running') s.running += 1 - const st = String(c.Status || '') - if (/\(unhealthy\)/i.test(st)) s.unhealthy += 1 } - for (const s of stacks.values()) { - const degraded = s.running < s.total || s.unhealthy > 0 - await evaluateRules('stack_health', { - name: s.name, - status: degraded ? 'degraded' : 'ok', - running: s.running, - desired: s.total, - message: degraded - ? `${s.running}/${s.total} running${s.unhealthy ? `, ${s.unhealthy} unhealthy` : ''}` - : 'all services running', - }) - } - } catch (err) { - logger.debug('alerts: stack poll failed', { error: err.message }) - } - // Disk via system df - try { - if (typeof docker.df === 'function') { - const df = await docker.df() - // Docker df doesn't give host disk %; approximate via layers size vs nothing. - // Prefer host free space from / if available via system info + /** @type {object[]} */ + let containers = [] + if (needs.list) { + try { + containers = (await docker.listContainers({ all: true })) || [] + } catch (err) { + logger.debug('alerts: container list poll failed', { error: err.message }) + } } - // Host disk from Node (server filesystem) - try { - const { statfsSync } = await import('fs') - if (typeof statfsSync === 'function') { - const st = statfsSync('/') - const total = Number(st.blocks) * Number(st.bsize) - const free = Number(st.bfree) * Number(st.bsize) - if (total > 0) { - const usedPct = ((total - free) / total) * 100 - await evaluateRules('resource', { - diskPercent: usedPct, - mount: '/', + + if (needs.list && hasEnabledRuleKind('container_health')) { + try { + for (const c of containers) { + const health = + c.Status?.match(/\((healthy|unhealthy|health: starting)\)/i)?.[1] || c.State + const h = + c.Health?.Status || + (typeof health === 'string' + ? health.toLowerCase().replace('health: ', '') + : null) + if (!h || h === 'none') continue + // Skip pure state strings that aren't healthcheck results + if (h === 'running' || h === 'exited' || h === 'created' || h === 'paused') { + continue + } + const name = (c.Names?.[0] || '').replace(/^\//, '') || c.Id?.slice(0, 12) + await evaluateRules('container_health', { + id: c.Id, + name, + labels: c.Labels, + health: h === 'starting' ? 'starting' : h, }) } + } catch (err) { + logger.debug('alerts: container health poll failed', { error: err.message }) } - } catch { - // statfs not available on all platforms } - } catch (err) { - logger.debug('alerts: resource poll failed', { error: err.message }) + + if (needs.list && hasEnabledRuleKind('stack_health')) { + try { + /** @type {Map} */ + const stacks = new Map() + for (const c of containers) { + const project = + c.Labels?.['com.docker.compose.project'] || + c.Labels?.['com.docker.stack.namespace'] + if (!project) continue + let s = stacks.get(project) + if (!s) { + s = { name: project, running: 0, total: 0, unhealthy: 0 } + stacks.set(project, s) + } + s.total += 1 + if (c.State === 'running') s.running += 1 + const st = String(c.Status || '') + if (/\(unhealthy\)/i.test(st)) s.unhealthy += 1 + } + for (const s of stacks.values()) { + const degraded = s.running < s.total || s.unhealthy > 0 + await evaluateRules('stack_health', { + name: s.name, + status: degraded ? 'degraded' : 'ok', + running: s.running, + desired: s.total, + message: degraded + ? `${s.running}/${s.total} running${s.unhealthy ? `, ${s.unhealthy} unhealthy` : ''}` + : 'all services running', + }) + } + } catch (err) { + logger.debug('alerts: stack poll failed', { error: err.message }) + } + } + + // Host disk only — never docker.df() (expensive Engine inventory scan). + if (needs.resource) { + try { + const { statfsSync } = await import('fs') + if (typeof statfsSync === 'function') { + const st = statfsSync('/') + const total = Number(st.blocks) * Number(st.bsize) + const free = Number(st.bfree) * Number(st.bsize) + if (total > 0) { + const usedPct = ((total - free) / total) * 100 + await evaluateRules('resource', { + diskPercent: usedPct, + mount: '/', + }) + } + } + } catch (err) { + logger.debug('alerts: resource poll failed', { error: err?.message }) + } + } + } finally { + pollInFlight = false } } @@ -1175,7 +1332,12 @@ function armPoll() { pollTimer = null } if (!config.enabled) return - const ms = config.pollIntervalMs || DEFAULT_POLL_MS + const needs = pollNeeds() + if (!needs.daemon && !needs.list && !needs.resource) { + logger.debug('alerts: poll timer off (no poll-backed rules enabled)') + return + } + const ms = Math.max(MIN_POLL_MS, config.pollIntervalMs || DEFAULT_POLL_MS) pollTimer = setInterval(() => { pollHealth().catch((err) => { logger.debug('alerts: poll error', { error: err.message }) @@ -1228,6 +1390,9 @@ export function stopAlerts() { clearInterval(pollTimer) pollTimer = null } + eventQueue = [] + eventDraining = false + pollInFlight = false if (saveTimer) { clearTimeout(saveTimer) saveTimer = null @@ -1333,6 +1498,8 @@ export function upsertAlertRule(input) { if (idx >= 0) config.rules[idx] = rule else config.rules.push(rule) flushSave() + // Rule kind/enable changes which Docker polls we need + armPoll() logger.debug('alerts: rule upserted live', { id: rule.id, enabled: rule.enabled, @@ -1345,6 +1512,7 @@ export function deleteAlertRule(id) { const before = config.rules.length config.rules = config.rules.filter((r) => r.id !== id) flushSave() + armPoll() return before !== config.rules.length } @@ -1354,6 +1522,7 @@ export function listAlertHistory(limit = 50) { } export function getAlertsStatus() { + const needs = pollNeeds() return { enabled: config.enabled, channelCount: config.channels.filter((c) => c.enabled).length, @@ -1361,6 +1530,10 @@ export function getAlertsStatus() { lastDaemonOk, historyCount: history.length, pollIntervalMs: config.pollIntervalMs, + pollActive: Boolean(pollTimer), + pollInFlight, + pollNeeds: needs, + eventQueueDepth: eventQueue.length, quietHoursActive: inQuietHours(), rateLimitPerMinute: config.rateLimitPerMinute, } diff --git a/server/services/stats.js b/server/services/stats.js index a312290..1c3ef3c 100644 --- a/server/services/stats.js +++ b/server/services/stats.js @@ -1,15 +1,18 @@ /** * Container stats collection and broadcast. * - * Uses **one-shot** Docker stats (stream:false) with bounded concurrency — - * not perpetual `stats({ stream: true })` attachments. + * Strategy (tuned against dockerd load, informed by Dozzle's approach): * - * Why: each live stats stream forces dockerd to sample cgroups ~1 Hz for that - * container for as long as the stream is open. With N running containers that - * is N concurrent streams and is a well-known host CPU / load-average killer. - * One-shot polls only work when we ask, and we can throttle concurrency. - * - * Collection only runs while peers.size > 0 (paused when idle). + * - **One-shot** `stats({ stream: false })` — no perpetual stream:true attachments. + * Dozzle uses long-lived streams for all running containers and documents + * elevated dockerd CPU as expected; we intentionally avoid that. + * - **Round-robin batches** — each tick samples at most `samplesPerTick` + * containers (not the entire fleet). Full-fleet one-shot sweeps are often + * *worse* than streams when sweep time > interval (dockerd never idles). + * - **Post-sweep spacing** — next tick is scheduled only after the previous + * finishes + collectIntervalMs (not a fixed setInterval that overlaps work). + * - **Roster cache** — listContainers is not called every tick. + * - **Peer-idle pause** — no collection when peers.size === 0. */ import fs from 'fs' import path from 'path' @@ -28,6 +31,8 @@ export const STATS_LIMITS = Object.freeze({ collectIntervalMs: { min: 1000, max: 60_000 }, broadcastIntervalMs: { min: 1000, max: 60_000 }, concurrency: { min: 1, max: 32 }, + samplesPerTick: { min: 1, max: 100 }, + listIntervalMs: { min: 2000, max: 120_000 }, cacheTtlMs: { min: 0, max: 30_000 }, warnContainers: { min: 10, max: 10_000 }, }) @@ -45,10 +50,13 @@ function clampInt(value, min, max, fallback) { /** * Built-in defaults, then env overrides (bootstrap before disk / UI). + * Conservative defaults keep dockerd idle between batches on busy hosts. * @returns {{ * collectIntervalMs: number, * broadcastIntervalMs: number, * concurrency: number, + * samplesPerTick: number, + * listIntervalMs: number, * cacheTtlMs: number, * warnContainers: number, * }} @@ -57,29 +65,41 @@ export function defaultStatsConfig() { const collectIntervalMs = clampInt( envInt( 'PEARDOCK_STATS_INTERVAL_MS', - CONFIG.STATS?.ACTIVE_INTERVAL_MS ?? CONFIG.STATS?.INTERVAL_MS ?? 2000 + CONFIG.STATS?.ACTIVE_INTERVAL_MS ?? CONFIG.STATS?.INTERVAL_MS ?? 5000 ), STATS_LIMITS.collectIntervalMs.min, STATS_LIMITS.collectIntervalMs.max, - 2000 + 5000 ) const broadcastIntervalMs = clampInt( - envInt('PEARDOCK_STATS_BROADCAST_MS', CONFIG.STATS?.INTERVAL_MS ?? 2000), + envInt('PEARDOCK_STATS_BROADCAST_MS', CONFIG.STATS?.INTERVAL_MS ?? 5000), STATS_LIMITS.broadcastIntervalMs.min, STATS_LIMITS.broadcastIntervalMs.max, - 2000 + 5000 ) const concurrency = clampInt( - envInt('PEARDOCK_STATS_CONCURRENCY', CONFIG.STATS?.CONCURRENCY ?? 6), + envInt('PEARDOCK_STATS_CONCURRENCY', CONFIG.STATS?.CONCURRENCY ?? 2), STATS_LIMITS.concurrency.min, STATS_LIMITS.concurrency.max, - 6 + 2 + ) + const samplesPerTick = clampInt( + envInt('PEARDOCK_STATS_SAMPLES_PER_TICK', CONFIG.STATS?.SAMPLES_PER_TICK ?? 4), + STATS_LIMITS.samplesPerTick.min, + STATS_LIMITS.samplesPerTick.max, + 4 + ) + const listIntervalMs = clampInt( + envInt('PEARDOCK_STATS_LIST_INTERVAL_MS', CONFIG.STATS?.LIST_INTERVAL_MS ?? 15_000), + STATS_LIMITS.listIntervalMs.min, + STATS_LIMITS.listIntervalMs.max, + 15_000 ) const cacheTtlMs = clampInt( - envInt('PEARDOCK_STATS_CACHE_TTL_MS', CONFIG.STATS?.CACHE_TTL_MS ?? 1000), + envInt('PEARDOCK_STATS_CACHE_TTL_MS', CONFIG.STATS?.CACHE_TTL_MS ?? 2000), STATS_LIMITS.cacheTtlMs.min, STATS_LIMITS.cacheTtlMs.max, - 1000 + 2000 ) const warnContainers = clampInt( envInt('PEARDOCK_STATS_WARN_CONTAINERS', 80), @@ -91,6 +111,8 @@ export function defaultStatsConfig() { collectIntervalMs, broadcastIntervalMs, concurrency, + samplesPerTick, + listIntervalMs, cacheTtlMs, warnContainers, } @@ -120,6 +142,8 @@ function saveStatsDisk() { collectIntervalMs: runtime.collectIntervalMs, broadcastIntervalMs: runtime.broadcastIntervalMs, concurrency: runtime.concurrency, + samplesPerTick: runtime.samplesPerTick, + listIntervalMs: runtime.listIntervalMs, cacheTtlMs: runtime.cacheTtlMs, warnContainers: runtime.warnContainers, } @@ -138,7 +162,7 @@ function saveStatsDisk() { */ export function normalizeStatsConfig(partial = {}, base = defaultStatsConfig()) { const src = partial && typeof partial === 'object' ? partial : {} - // Accept seconds from UI as collectIntervalSec / broadcastIntervalSec + // Accept seconds from UI as collectIntervalSec / broadcastIntervalSec / listIntervalSec let collectMs = src.collectIntervalMs if (collectMs == null && src.collectIntervalSec != null) { collectMs = Number(src.collectIntervalSec) * 1000 @@ -147,6 +171,10 @@ export function normalizeStatsConfig(partial = {}, base = defaultStatsConfig()) if (broadcastMs == null && src.broadcastIntervalSec != null) { broadcastMs = Number(src.broadcastIntervalSec) * 1000 } + let listMs = src.listIntervalMs + if (listMs == null && src.listIntervalSec != null) { + listMs = Number(src.listIntervalSec) * 1000 + } return { collectIntervalMs: clampInt( collectMs ?? base.collectIntervalMs, @@ -166,6 +194,18 @@ export function normalizeStatsConfig(partial = {}, base = defaultStatsConfig()) STATS_LIMITS.concurrency.max, base.concurrency ), + samplesPerTick: clampInt( + src.samplesPerTick ?? base.samplesPerTick, + STATS_LIMITS.samplesPerTick.min, + STATS_LIMITS.samplesPerTick.max, + base.samplesPerTick + ), + listIntervalMs: clampInt( + listMs ?? base.listIntervalMs, + STATS_LIMITS.listIntervalMs.min, + STATS_LIMITS.listIntervalMs.max, + base.listIntervalMs + ), cacheTtlMs: clampInt( src.cacheTtlMs ?? base.cacheTtlMs, STATS_LIMITS.cacheTtlMs.min, @@ -210,13 +250,17 @@ export function getStatsConfig() { collectIntervalMs: runtime.collectIntervalMs, broadcastIntervalMs: runtime.broadcastIntervalMs, concurrency: runtime.concurrency, + samplesPerTick: runtime.samplesPerTick, + listIntervalMs: runtime.listIntervalMs, cacheTtlMs: runtime.cacheTtlMs, warnContainers: runtime.warnContainers, // Convenience for UI collectIntervalSec: Math.round(runtime.collectIntervalMs / 1000), broadcastIntervalSec: Math.round(runtime.broadcastIntervalMs / 1000), - active: Boolean(intervalHandle), + listIntervalSec: Math.round(runtime.listIntervalMs / 1000), + active: collectionLoopActive, peers: peers.size, + runningTracked: rosterRunning.length, limits: { ...STATS_LIMITS }, path: STATS_FILE, } @@ -232,35 +276,21 @@ export function updateStatsConfig(partial = {}, opts = {}) { ensureStatsConfigLoaded() const base = opts.replace ? defaultStatsConfig() : { ...runtime } const next = normalizeStatsConfig(partial, base) - const intervalChanged = next.collectIntervalMs !== runtime.collectIntervalMs runtime = next if (opts.persist !== false) saveStatsDisk() - if (intervalChanged && intervalHandle) { - rearmCollectInterval() - } + // Interval is applied on the next schedule after a tick completes logger.info('Stats config updated', { collectIntervalMs: runtime.collectIntervalMs, broadcastIntervalMs: runtime.broadcastIntervalMs, concurrency: runtime.concurrency, + samplesPerTick: runtime.samplesPerTick, + listIntervalMs: runtime.listIntervalMs, cacheTtlMs: runtime.cacheTtlMs, persist: opts.persist !== false, }) return getStatsConfig() } -/** Rearm setInterval after collect interval change (while peers connected). */ -function rearmCollectInterval() { - if (!intervalHandle) return - clearInterval(intervalHandle) - intervalHandle = setInterval(() => { - statsTick().catch(() => {}) - }, runtime.collectIntervalMs) - if (typeof intervalHandle.unref === 'function') intervalHandle.unref() - logger.debug('Stats collect interval rearmed', { - intervalMs: runtime.collectIntervalMs, - }) -} - export function getStatsFilePath() { return STATS_FILE } @@ -270,7 +300,10 @@ const containerStats = {} const statsCache = new Map() const containerActivity = new Map() -let intervalHandle = null +/** @type {ReturnType|null} */ +let loopTimer = null +/** True while peer-connected collection loop is armed */ +let collectionLoopActive = false /** Peer-registry unsubscribe; set while stats service is armed */ let unsubPeers = null /** True after startStatsBroadcast(); false after stopStatsBroadcast() */ @@ -282,6 +315,14 @@ let dockerBackoffUntil = 0 let tickInFlight = false let largeFleetWarned = false +/** Cached running containers from last listContainers */ +let rosterRunning = [] +/** @type {Set} */ +let rosterAllIds = new Set() +let rosterFetchedAt = 0 +/** Round-robin cursor into rosterRunning */ +let rrIndex = 0 + /** * Docker Engine CPU % (same idea as `docker stats`). * One-shot stats populate precpu_stats after ~1s wait inside the engine. @@ -580,30 +621,41 @@ export function destroyStatsForContainer(id) { } /** - * List once, sample running containers with bounded one-shot stats. - * No perpetual streams — dockerd only does work while we request samples. + * Refresh container roster from Docker (throttled). + * @param {boolean} [force] */ -async function collectContainerStats() { - // Single list (all) — derive running from State; was 2× listContainers/tick +async function refreshRoster(force = false) { + const now = Date.now() + if (!force && rosterFetchedAt && now - rosterFetchedAt < runtime.listIntervalMs) { + return + } + const all = await docker.listContainers({ all: true }) - const running = all.filter((c) => String(c.State || '').toLowerCase() === 'running') + const running = all + .filter((c) => String(c.State || '').toLowerCase() === 'running') + .sort((a, b) => String(a.Id).localeCompare(String(b.Id))) const allIds = new Set(all.map((c) => c.Id)) const runningIds = new Set(running.map((c) => c.Id)) + rosterRunning = running + rosterAllIds = allIds + rosterFetchedAt = now + if (rrIndex >= running.length) rrIndex = 0 + if (running.length >= runtime.warnContainers && !largeFleetWarned) { largeFleetWarned = true logger.warn( - 'Large running fleet for stats collection — dockerd load scales with container count', + 'Large running fleet for stats collection — dockerd load scales with samples/tick', { running: running.length, + samplesPerTick: runtime.samplesPerTick, concurrency: runtime.concurrency, intervalMs: runtime.collectIntervalMs, - hint: 'Settings → Performance, or PEARDOCK_STATS_INTERVAL_MS / PEARDOCK_STATS_CONCURRENCY', + hint: 'Settings → Performance: raise interval, lower samples/tick & concurrency', } ) } - // Ensure entries + drop gone / zero-out stopped for (const containerInfo of running) { ensureStatsEntry(containerInfo) } @@ -625,9 +677,35 @@ async function collectContainerStats() { } } } +} - // Bounded parallel one-shot samples (only running) - await mapPool(running, runtime.concurrency, async (containerInfo) => { +/** + * Pick next round-robin batch of running containers (at most samplesPerTick). + * @param {object[]} running + * @returns {object[]} + */ +function nextSampleBatch(running) { + if (!running.length) return [] + const n = Math.min(runtime.samplesPerTick, running.length) + const batch = [] + for (let i = 0; i < n; i++) { + batch.push(running[(rrIndex + i) % running.length]) + } + rrIndex = (rrIndex + n) % running.length + return batch +} + +/** + * Throttled roster + round-robin one-shot samples. + * Dockerd cost ≈ samplesPerTick × ~1s / concurrency per tick, then idle for interval. + */ +async function collectContainerStats() { + await refreshRoster(false) + + const batch = nextSampleBatch(rosterRunning) + if (!batch.length) return + + await mapPool(batch, runtime.concurrency, async (containerInfo) => { if (peers.size === 0) return const statsData = containerStats[containerInfo.Id] if (!statsData) return @@ -654,17 +732,26 @@ function destroyAllStatsEntries() { statsCache.clear() containerActivity.clear() largeFleetWarned = false + rosterRunning = [] + rosterAllIds = new Set() + rosterFetchedAt = 0 + rrIndex = 0 +} + +function clearLoopTimer() { + if (loopTimer) { + clearTimeout(loopTimer) + loopTimer = null + } } /** - * Pause collection: stop interval and drop in-memory state. + * Pause collection: stop loop and drop in-memory state. * Safe to call when already paused. */ export function pauseStatsCollection() { - if (intervalHandle) { - clearInterval(intervalHandle) - intervalHandle = null - } + collectionLoopActive = false + clearLoopTimer() destroyAllStatsEntries() tickInFlight = false logger.debug('Stats collection paused (no peers)') @@ -769,6 +856,33 @@ async function statsTick() { } } +/** + * Run one stats tick, then schedule the next after collectIntervalMs. + * Spacing is measured from tick *completion* so long sweeps cannot overlap. + */ +function runLoopTick() { + if (!collectionLoopActive || !serviceArmed || peers.size === 0) { + collectionLoopActive = false + clearLoopTimer() + return + } + statsTick() + .catch(() => {}) + .finally(() => { + if (!collectionLoopActive || !serviceArmed || peers.size === 0) { + collectionLoopActive = false + clearLoopTimer() + return + } + clearLoopTimer() + loopTimer = setTimeout(() => { + loopTimer = null + runLoopTick() + }, runtime.collectIntervalMs) + if (typeof loopTimer.unref === 'function') loopTimer.unref() + }) +} + /** * Resume collection when at least one peer is connected. * Idempotent; kicks an immediate tick so first client is not waiting a full interval. @@ -776,20 +890,19 @@ async function statsTick() { export function resumeStatsCollection() { if (!serviceArmed) return if (peers.size === 0) return - if (!intervalHandle) { - intervalHandle = setInterval(() => { - statsTick().catch(() => {}) - }, runtime.collectIntervalMs) - if (typeof intervalHandle.unref === 'function') intervalHandle.unref() - logger.debug('Stats collection resumed', { - peers: peers.size, - intervalMs: runtime.collectIntervalMs, - concurrency: runtime.concurrency, - }) - } - // Immediate kick so first connected client gets samples ASAP + if (collectionLoopActive) return + collectionLoopActive = true lastBroadcast = 0 - statsTick().catch(() => {}) + logger.debug('Stats collection resumed', { + peers: peers.size, + intervalMs: runtime.collectIntervalMs, + concurrency: runtime.concurrency, + samplesPerTick: runtime.samplesPerTick, + listIntervalMs: runtime.listIntervalMs, + }) + // Force fresh roster on first tick after resume + rosterFetchedAt = 0 + runLoopTick() } function onPeerCountChange(size, prevSize) { @@ -842,9 +955,9 @@ export function stopStatsBroadcast() { pauseStatsCollection() } -/** @returns {boolean} whether the collect interval is currently running */ +/** @returns {boolean} whether the collect loop is currently armed */ export function isStatsCollectionActive() { - return Boolean(intervalHandle) + return collectionLoopActive } /** Test/ops helpers — same shape as getStatsConfig subset */ @@ -854,6 +967,8 @@ export function getStatsRuntimeConfig() { collectIntervalMs: runtime.collectIntervalMs, broadcastIntervalMs: runtime.broadcastIntervalMs, concurrency: runtime.concurrency, + samplesPerTick: runtime.samplesPerTick, + listIntervalMs: runtime.listIntervalMs, cacheTtlMs: runtime.cacheTtlMs, warnContainers: runtime.warnContainers, } diff --git a/test/alerts.test.js b/test/alerts.test.js index 0e98058..eebaa93 100644 --- a/test/alerts.test.js +++ b/test/alerts.test.js @@ -90,6 +90,49 @@ test('alerts update global settings persists', async (t) => { mod2.stopAlerts() }) +test('alerts poll interval clamps to min 60s', async (t) => { + withTempAlerts(t) + const mod = await import('../server/services/alerts.js?' + Date.now() + 'pollclamp') + mod.startAlerts() + mod.updateAlertsConfig({ pollIntervalMs: 5_000 }) + t.is(mod.getAlertsConfig().pollIntervalMs, 60_000, 'floor is 60s') + mod.updateAlertsConfig({ pollIntervalMs: 180_000 }) + t.is(mod.getAlertsConfig().pollIntervalMs, 180_000) + mod.stopAlerts() +}) + +test('alerts ignore noisy docker events (exec/attach)', async (t) => { + withTempAlerts(t) + const mod = await import('../server/services/alerts.js?' + Date.now() + 'noisy') + mod.startAlerts() + // Fire noisy events — should not throw and should not enqueue work + mod.onDockerEvent({ + Type: 'container', + Action: 'exec_create', + id: 'abc', + Actor: { Attributes: { name: 'x' } }, + }) + mod.onDockerEvent({ + Type: 'container', + Action: 'attach', + id: 'abc', + Actor: { Attributes: { name: 'x' } }, + }) + const st = mod.getAlertsStatus() + t.is(st.eventQueueDepth, 0, 'noisy events dropped before queue') + // Relevant event should queue / process + mod.onDockerEvent({ + Type: 'container', + Action: 'die', + id: 'dead1', + Actor: { Attributes: { name: 'boom' } }, + }) + // Allow microtask drain + await new Promise((r) => setTimeout(r, 30)) + t.ok(mod.listAlertHistory(5).length >= 1, 'die event fires alert history') + mod.stopAlerts() +}) + test('protocol exposes alert methods and push', async (t) => { const { Methods, MethodRoles, Pushes, Roles } = await import('../shared/protocol.js') t.is(Methods.getAlertsConfig, 'getAlertsConfig') diff --git a/test/stats-config.test.js b/test/stats-config.test.js index 868cb46..d92689c 100644 --- a/test/stats-config.test.js +++ b/test/stats-config.test.js @@ -17,12 +17,16 @@ test('normalizeStatsConfig clamps out-of-range values', (t) => { collectIntervalMs: 50, broadcastIntervalMs: 999_999, concurrency: 100, + samplesPerTick: 0, + listIntervalMs: 100, cacheTtlMs: -5, warnContainers: 1, }) t.is(n.collectIntervalMs, STATS_LIMITS.collectIntervalMs.min) t.is(n.broadcastIntervalMs, STATS_LIMITS.broadcastIntervalMs.max) t.is(n.concurrency, STATS_LIMITS.concurrency.max) + t.is(n.samplesPerTick, STATS_LIMITS.samplesPerTick.min) + t.is(n.listIntervalMs, STATS_LIMITS.listIntervalMs.min) t.is(n.cacheTtlMs, STATS_LIMITS.cacheTtlMs.min) t.is(n.warnContainers, STATS_LIMITS.warnContainers.min) }) @@ -31,11 +35,15 @@ test('normalizeStatsConfig accepts seconds aliases from UI', (t) => { const n = normalizeStatsConfig({ collectIntervalSec: 5, broadcastIntervalSec: 3, + listIntervalSec: 20, concurrency: 4, + samplesPerTick: 8, }) t.is(n.collectIntervalMs, 5000) t.is(n.broadcastIntervalMs, 3000) + t.is(n.listIntervalMs, 20000) t.is(n.concurrency, 4) + t.is(n.samplesPerTick, 8) }) test('defaultStatsConfig is within limits', (t) => { diff --git a/ui/alerts-settings.js b/ui/alerts-settings.js index da0df69..7a2aec1 100644 --- a/ui/alerts-settings.js +++ b/ui/alerts-settings.js @@ -100,7 +100,7 @@ function renderAlertsGlobal(config, status) { const min = document.getElementById('alerts-min-severity') if (min) min.value = config.minSeverity || 'info' const poll = document.getElementById('alerts-poll-ms') - if (poll) poll.value = String(Math.round((config.pollIntervalMs || 60000) / 1000)) + if (poll) poll.value = String(Math.round((config.pollIntervalMs || 120000) / 1000)) const rate = document.getElementById('alerts-rate-limit') if (rate) rate.value = String(config.rateLimitPerMinute || 40) const qh = document.getElementById('alerts-quiet-enabled') @@ -120,10 +120,14 @@ function renderAlertsGlobal(config, status) { const statusEl = document.getElementById('alerts-engine-status') if (statusEl && status) { + const pollLabel = status.pollActive + ? `poll ${Math.round((status.pollIntervalMs || 0) / 1000)}s` + : 'poll off' const parts = [ status.enabled ? 'Enabled' : 'Disabled', `${status.channelCount} channel(s)`, `${status.ruleCount} rule(s)`, + pollLabel, status.lastDaemonOk ? 'Docker OK' : 'Docker DOWN', status.quietHoursActive ? 'Quiet hours' : null, ].filter(Boolean) @@ -373,7 +377,7 @@ function readGlobalConfigFromUi() { enabled: document.getElementById('alerts-enabled')?.value !== '0', minSeverity: document.getElementById('alerts-min-severity')?.value || 'info', pollIntervalMs: - Math.max(15, Number(document.getElementById('alerts-poll-ms')?.value) || 60) * 1000, + Math.max(60, Number(document.getElementById('alerts-poll-ms')?.value) || 120) * 1000, rateLimitPerMinute: Number(document.getElementById('alerts-rate-limit')?.value) || 40, includeHostname: document.getElementById('alerts-include-hostname')?.value !== '0', quietHours: { diff --git a/ui/ops-app.js b/ui/ops-app.js index bfefd8b..14be7b4 100644 --- a/ui/ops-app.js +++ b/ui/ops-app.js @@ -2518,7 +2518,9 @@ function isServerStatsLiveField(el) { return ( id === 'stats-collect-sec' || id === 'stats-broadcast-sec' || + id === 'stats-samples-per-tick' || id === 'stats-concurrency' || + id === 'stats-list-sec' || id === 'stats-cache-ttl-ms' || id === 'stats-warn-containers' ) diff --git a/ui/stats-settings.js b/ui/stats-settings.js index 1746761..52f611e 100644 --- a/ui/stats-settings.js +++ b/ui/stats-settings.js @@ -75,34 +75,44 @@ function renderStatsForm(config) { const collect = document.getElementById('stats-collect-sec') if (collect) { collect.value = String( - config.collectIntervalSec ?? Math.round((config.collectIntervalMs || 2000) / 1000) + config.collectIntervalSec ?? Math.round((config.collectIntervalMs || 5000) / 1000) ) } const broadcast = document.getElementById('stats-broadcast-sec') if (broadcast) { broadcast.value = String( - config.broadcastIntervalSec ?? Math.round((config.broadcastIntervalMs || 2000) / 1000) + config.broadcastIntervalSec ?? Math.round((config.broadcastIntervalMs || 5000) / 1000) ) } + const samples = document.getElementById('stats-samples-per-tick') + if (samples) samples.value = String(config.samplesPerTick ?? 4) const conc = document.getElementById('stats-concurrency') - if (conc) conc.value = String(config.concurrency ?? 6) + if (conc) conc.value = String(config.concurrency ?? 2) + const listSec = document.getElementById('stats-list-sec') + if (listSec) { + listSec.value = String( + config.listIntervalSec ?? Math.round((config.listIntervalMs || 15000) / 1000) + ) + } const cache = document.getElementById('stats-cache-ttl-ms') - if (cache) cache.value = String(config.cacheTtlMs ?? 1000) + if (cache) cache.value = String(config.cacheTtlMs ?? 2000) const warn = document.getElementById('stats-warn-containers') if (warn) warn.value = String(config.warnContainers ?? 80) const active = config.active ? 'collecting' : 'idle (no peers or paused)' setEngineStatus( - `${active} · every ${config.collectIntervalSec ?? '?'}s · ×${config.concurrency ?? '?'}` + `${active} · gap ${config.collectIntervalSec ?? '?'}s · ${config.samplesPerTick ?? '?'} samples · ×${config.concurrency ?? '?'}` ) const meta = document.getElementById('stats-runtime-meta') if (meta) { const parts = [ `peers=${config.peers ?? 0}`, - `collect=${config.collectIntervalMs}ms`, - `broadcast=${config.broadcastIntervalMs}ms`, + `running=${config.runningTracked ?? '?'}`, + `gap=${config.collectIntervalMs}ms`, + `samples/tick=${config.samplesPerTick}`, `concurrency=${config.concurrency}`, - `cacheTtl=${config.cacheTtlMs}ms`, + `list=${config.listIntervalMs}ms`, + `broadcast=${config.broadcastIntervalMs}ms`, ] if (config.path) parts.push(`file=${config.path}`) meta.textContent = parts.join(' · ') @@ -117,13 +127,17 @@ function renderStatsForm(config) { function readFormPartial() { const collectSec = Number(document.getElementById('stats-collect-sec')?.value) const broadcastSec = Number(document.getElementById('stats-broadcast-sec')?.value) + const samplesPerTick = Number(document.getElementById('stats-samples-per-tick')?.value) const concurrency = Number(document.getElementById('stats-concurrency')?.value) + const listSec = Number(document.getElementById('stats-list-sec')?.value) const cacheTtlMs = Number(document.getElementById('stats-cache-ttl-ms')?.value) const warnContainers = Number(document.getElementById('stats-warn-containers')?.value) return { collectIntervalSec: Number.isFinite(collectSec) ? collectSec : undefined, broadcastIntervalSec: Number.isFinite(broadcastSec) ? broadcastSec : undefined, + samplesPerTick: Number.isFinite(samplesPerTick) ? samplesPerTick : undefined, concurrency: Number.isFinite(concurrency) ? concurrency : undefined, + listIntervalSec: Number.isFinite(listSec) ? listSec : undefined, cacheTtlMs: Number.isFinite(cacheTtlMs) ? cacheTtlMs : undefined, warnContainers: Number.isFinite(warnContainers) ? warnContainers : undefined, }