/** * cgroup v2 collector โ€” per-container / slice CPU, memory, I/O. * * Enable: PEARDATA_CGROUPS=1 (default on Linux when unset) * Disable: PEARDATA_CGROUPS=0 * Limit: PEARDATA_CGROUPS_MAX=32 */ import fs from 'fs' import path from 'path' import os from 'os' import { EventEmitter } from 'events' import { SAMPLE_INTERVAL_MS, registerChart, makeCgroupCpuChart, makeCgroupMemChart, makeCgroupIoChart, makeCgroupMemDetailChart, makeCgroupThrottleChart, } from '../../../shared/metrics.js' import { resolveContainerLabel } from '../../../shared/container-names.js' import { loadContainerNameMap, resolveDockerSocket } from './docker.js' import { readFileBuf } from '../../utils/fd-cache.js' import logger from '../../utils/logger.js' const log = logger.child('cgroups') const HOST_CPUS = (() => { try { return os.cpus().length || 1 } catch { return 1 } })() export function isCgroupsEnabled() { const v = process.env.PEARDATA_CGROUPS if (v === '0' || v === 'off' || v === 'false') return false if (v === '1' || v === 'on' || v === 'true') return true return os.platform() === 'linux' } function maxCgroups() { const n = Number(process.env.PEARDATA_CGROUPS_MAX) return Number.isFinite(n) && n > 0 ? Math.min(128, Math.floor(n)) : 32 } function readFile(p) { return readFileBuf(p) } function cgroupRoot() { if (fs.existsSync('/sys/fs/cgroup/cgroup.controllers')) return '/sys/fs/cgroup' return null } /** * Discover interesting leaf cgroups (docker, podman, systemd system.slice children). * @returns {Array<{ id: string, title: string, path: string }>} */ export function discoverCgroups() { const root = cgroupRoot() if (!root) return [] /** @type {Array<{ id: string, title: string, path: string }>} */ const found = [] const max = maxCgroups() const candidates = [ path.join(root, 'system.slice'), path.join(root, 'user.slice'), path.join(root, 'docker'), path.join(root, 'kubepods.slice'), path.join(root, 'kubepods'), ] function walk(dir, depth) { if (found.length >= max || depth > 4) return let ents try { ents = fs.readdirSync(dir, { withFileTypes: true }) } catch { return } for (const ent of ents) { if (!ent.isDirectory()) continue if (ent.name === 'init.scope' || ent.name.startsWith('.')) continue const full = path.join(dir, ent.name) const rel = full.slice(root.length + 1) const interesting = /docker|containerd|podman|libpod|kubepods|crio|\.service$|\.scope$/.test(rel) const hasCpu = fs.existsSync(path.join(full, 'cpu.stat')) const hasMem = fs.existsSync(path.join(full, 'memory.current')) if (interesting && (hasCpu || hasMem)) { const id = rel.replace(/[^\w.+-]+/g, '_').slice(0, 96) found.push({ id, title: ent.name.replace(/\.(service|scope)$/, ''), path: full }) if (found.length >= max) return } walk(full, depth + 1) if (found.length >= max) return } } for (const c of candidates) { if (fs.existsSync(c)) walk(c, 0) } // Also sample a few top-level slices if still empty if (!found.length) { try { for (const ent of fs.readdirSync(root, { withFileTypes: true })) { if (!ent.isDirectory()) continue if (!ent.name.endsWith('.slice') && ent.name !== 'docker') continue const full = path.join(root, ent.name) found.push({ id: ent.name, title: ent.name, path: full }) if (found.length >= Math.min(8, max)) break } } catch { // ignore } } return found } function parseCpuStat(dir) { const raw = readFile(path.join(dir, 'cpu.stat')) if (!raw) return null /** @type {Record} */ const o = {} for (const line of raw.split('\n')) { const [k, v] = line.trim().split(/\s+/) if (k) o[k] = Number(v) || 0 } return o } function parseIoStat(dir) { const raw = readFile(path.join(dir, 'io.stat')) if (!raw) return { rbytes: 0, wbytes: 0 } let rbytes = 0 let wbytes = 0 for (const line of raw.split('\n')) { if (!line.trim()) continue const parts = line.trim().split(/\s+/) for (const p of parts.slice(1)) { const [k, v] = p.split('=') if (k === 'rbytes') rbytes += Number(v) || 0 if (k === 'wbytes') wbytes += Number(v) || 0 } } return { rbytes, wbytes } } function parseMemoryStat(dir) { const raw = readFile(path.join(dir, 'memory.stat')) if (!raw) return null /** @type {Record} */ const o = {} for (const line of raw.split('\n')) { const [k, v] = line.trim().split(/\s+/) if (k) o[k] = Number(v) || 0 } return o } function parsePressureSome10(dir, kind) { const raw = readFile(path.join(dir, `${kind}.pressure`)) if (!raw) return null const m = raw.match(/some avg10=([\d.]+)/) return m ? Number(m[1]) : null } function makeCgroupPressureChart(id, title, kind) { const safe = String(id).replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 96) const display = title || safe return { id: `cgroup.pressure.${kind}.${safe}`, name: `cgroup.pressure.${kind}.${safe}`, context: 'cgroup.pressure', title: `${display} ยท ${kind} pressure`, units: 'percentage', family: display, chartType: 'line', priority: 6120, plugin: 'cgroups', dimensions: [{ id: 'some10', name: 'some10', algorithm: 'absolute' }], } } export class CgroupsCollector extends EventEmitter { constructor(opts = {}) { super() this.intervalMs = opts.intervalMs ?? (Number(process.env.PEARDATA_SAMPLE_MS) || SAMPLE_INTERVAL_MS) this.socketPath = resolveDockerSocket(opts.socketPath) this.timer = null this.running = false /** @type {Map} */ this.prev = new Map() this.lastTs = 0 /** @type {Map} */ this._names = new Map() this._nameRefreshAt = 0 /** @type {Array<{ id: string, title: string, path: string }>|null} */ this._cgCache = null this._cgTs = 0 } start() { if (this.running) return if (!cgroupRoot()) { log.info('cgroup v2 not available โ€” collector idle') return } this.running = true this._tick() this.timer = setInterval(() => this._tick(), this.intervalMs) if (this.timer.unref) this.timer.unref() log.info('Cgroups collector started', { max: maxCgroups() }) } stop() { this.running = false if (this.timer) clearInterval(this.timer) this.timer = null } async _refreshNames() { const now = Date.now() const interval = this._names.size ? 30_000 : 5_000 if (now - this._nameRefreshAt < interval) return this._nameRefreshAt = now try { this.socketPath = resolveDockerSocket(this.socketPath) const { map, source } = await loadContainerNameMap(this.socketPath) if (map.size) { const prev = this._names.size this._names = map if (prev !== map.size) { log.info('Container names loaded for cgroups', { count: map.size, source }) } } } catch { // ignore โ€” titles fall back to humanized hashes } } async _tick() { const t0 = performance.now() try { await this._refreshNames() } catch { // ignore } const ts = Date.now() const dtSec = this.lastTs ? (ts - this.lastTs) / 1000 : this.intervalMs / 1000 this.lastTs = ts const ncpu = HOST_CPUS /** @type {Array<{ chart: string, context: string, ts: number, values: object }>} */ const batch = [] const now = Date.now() if (!this._cgCache || now - this._cgTs > 30000) { this._cgCache = discoverCgroups() this._cgTs = now for (const cg of this._cgCache) { const title = resolveContainerLabel(cg.title, this._names) registerChart(makeCgroupCpuChart(cg.id, title)) registerChart(makeCgroupMemChart(cg.id, title)) registerChart(makeCgroupIoChart(cg.id, title)) registerChart(makeCgroupMemDetailChart(cg.id, title)) registerChart(makeCgroupThrottleChart(cg.id, title)) } } const cgList = this._cgCache for (const cg of cgList) { const title = resolveContainerLabel(cg.title, this._names) const cpuDef = makeCgroupCpuChart(cg.id, title) const memDef = makeCgroupMemChart(cg.id, title) const ioDef = makeCgroupIoChart(cg.id, title) const memDetailDef = makeCgroupMemDetailChart(cg.id, title) const throttleDef = makeCgroupThrottleChart(cg.id, title) const cpu = parseCpuStat(cg.path) let usageUsec = cpu?.usage_usec ?? 0 let userUsec = cpu?.user_usec ?? 0 let systemUsec = cpu?.system_usec ?? 0 const throttledUsec = cpu?.throttled_usec ?? 0 const memCur = Number(readFile(path.join(cg.path, 'memory.current')) || 0) const memMaxRaw = readFile(path.join(cg.path, 'memory.max')) const memMax = memMaxRaw && memMaxRaw.trim() !== 'max' ? Number(memMaxRaw.trim()) || 0 : 0 const io = parseIoStat(cg.path) const prev = this.prev.get(cg.id) let userPct = 0 let sysPct = 0 if (prev && dtSec > 0) { const dUser = Math.max(0, userUsec - prev.user) const dSys = Math.max(0, systemUsec - prev.system) const dUsage = Math.max(0, usageUsec - prev.usage) if (userUsec || systemUsec) { userPct = (dUser / (dtSec * 1e6 * ncpu)) * 100 sysPct = (dSys / (dtSec * 1e6 * ncpu)) * 100 } else { userPct = (dUsage / (dtSec * 1e6 * ncpu)) * 100 } } batch.push({ chart: cpuDef.id, context: 'cgroup.cpu', ts, values: { user: userPct, system: sysPct }, }) batch.push({ chart: memDef.id, context: 'cgroup.mem', ts, values: { usage: memCur / (1024 * 1024), limit: memMax / (1024 * 1024), }, }) batch.push({ chart: ioDef.id, context: 'cgroup.io', ts, values: { read: prev && dtSec > 0 ? Math.max(0, io.rbytes - prev.rbytes) / dtSec / 1024 : 0, write: prev && dtSec > 0 ? Math.max(0, io.wbytes - prev.wbytes) / dtSec / 1024 : 0, }, }) const memStat = parseMemoryStat(cg.path) if (memStat) { batch.push({ chart: memDetailDef.id, context: 'cgroup.mem_detail', ts, values: { anon: (memStat.anon || 0) / (1024 * 1024), file: (memStat.file || 0) / (1024 * 1024), kernel: (memStat.kernel || 0) / (1024 * 1024), sock: (memStat.sock || 0) / (1024 * 1024), }, }) } let throttlePct = 0 if (prev && dtSec > 0) { const dThrottled = Math.max(0, throttledUsec - prev.throttledUsec) throttlePct = Math.min(100, (dThrottled / (dtSec * 1e6)) * 100) } batch.push({ chart: throttleDef.id, context: 'cgroup.cpu_throttle', ts, values: { throttled: throttlePct }, }) for (const kind of ['cpu', 'memory', 'io']) { const some10 = parsePressureSome10(cg.path, kind) if (some10 == null) continue const pressureDef = makeCgroupPressureChart(cg.id, title, kind) registerChart(pressureDef) batch.push({ chart: pressureDef.id, context: 'cgroup.pressure', ts, values: { some10 }, }) } this.prev.set(cg.id, { usage: usageUsec, user: userUsec, system: systemUsec, rbytes: io.rbytes, wbytes: io.wbytes, throttledUsec, }) } if (batch.length) this.emit('samples', batch) const dur = performance.now() - t0 if (dur > 10) log.debug('tick', { collector: 'cgroups', durMs: Math.round(dur * 10) / 10 }) } } /** @type {CgroupsCollector|null} */ let singleton = null export function getCgroupsCollector() { if (!singleton) singleton = new CgroupsCollector() return singleton }