345 lines
10 KiB
JavaScript
345 lines
10 KiB
JavaScript
/**
|
|
* cgroup v2 collector — per-container / slice CPU, memory, I/O.
|
|
*
|
|
* Enable: PEARDATA_CGROUPS=1 (default on Linux when unset)
|
|
* Disable: PEARDATA_CGROUPS=0
|
|
* Limit: PEARDATA_CGROUPS_MAX=32
|
|
*/
|
|
import fs from 'fs'
|
|
import path from 'path'
|
|
import os from 'os'
|
|
import { EventEmitter } from 'events'
|
|
import {
|
|
SAMPLE_INTERVAL_MS,
|
|
registerChart,
|
|
makeCgroupCpuChart,
|
|
makeCgroupMemChart,
|
|
makeCgroupIoChart,
|
|
makeCgroupMemDetailChart,
|
|
makeCgroupThrottleChart,
|
|
} from '../../../shared/metrics.js'
|
|
import logger from '../../utils/logger.js'
|
|
|
|
const log = logger.child('cgroups')
|
|
|
|
export function isCgroupsEnabled() {
|
|
const v = process.env.PEARDATA_CGROUPS
|
|
if (v === '0' || v === 'off' || v === 'false') return false
|
|
if (v === '1' || v === 'on' || v === 'true') return true
|
|
return os.platform() === 'linux'
|
|
}
|
|
|
|
function maxCgroups() {
|
|
const n = Number(process.env.PEARDATA_CGROUPS_MAX)
|
|
return Number.isFinite(n) && n > 0 ? Math.min(128, Math.floor(n)) : 32
|
|
}
|
|
|
|
function readFile(p) {
|
|
try {
|
|
return fs.readFileSync(p, 'utf8')
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|
|
|
|
function cgroupRoot() {
|
|
if (fs.existsSync('/sys/fs/cgroup/cgroup.controllers')) return '/sys/fs/cgroup'
|
|
return null
|
|
}
|
|
|
|
/**
|
|
* Discover interesting leaf cgroups (docker, podman, systemd system.slice children).
|
|
* @returns {Array<{ id: string, title: string, path: string }>}
|
|
*/
|
|
export function discoverCgroups() {
|
|
const root = cgroupRoot()
|
|
if (!root) return []
|
|
/** @type {Array<{ id: string, title: string, path: string }>} */
|
|
const found = []
|
|
const max = maxCgroups()
|
|
|
|
const candidates = [
|
|
path.join(root, 'system.slice'),
|
|
path.join(root, 'user.slice'),
|
|
path.join(root, 'docker'),
|
|
path.join(root, 'kubepods.slice'),
|
|
path.join(root, 'kubepods'),
|
|
]
|
|
|
|
function walk(dir, depth) {
|
|
if (found.length >= max || depth > 4) return
|
|
let ents
|
|
try {
|
|
ents = fs.readdirSync(dir, { withFileTypes: true })
|
|
} catch {
|
|
return
|
|
}
|
|
for (const ent of ents) {
|
|
if (!ent.isDirectory()) continue
|
|
if (ent.name === 'init.scope' || ent.name.startsWith('.')) continue
|
|
const full = path.join(dir, ent.name)
|
|
const rel = full.slice(root.length + 1)
|
|
const interesting =
|
|
/docker|containerd|podman|libpod|kubepods|crio|\.service$|\.scope$/.test(rel)
|
|
const hasCpu = fs.existsSync(path.join(full, 'cpu.stat'))
|
|
const hasMem = fs.existsSync(path.join(full, 'memory.current'))
|
|
if (interesting && (hasCpu || hasMem)) {
|
|
const id = rel.replace(/[^\w.+-]+/g, '_').slice(0, 96)
|
|
found.push({ id, title: ent.name.replace(/\.(service|scope)$/, ''), path: full })
|
|
if (found.length >= max) return
|
|
}
|
|
walk(full, depth + 1)
|
|
if (found.length >= max) return
|
|
}
|
|
}
|
|
|
|
for (const c of candidates) {
|
|
if (fs.existsSync(c)) walk(c, 0)
|
|
}
|
|
|
|
// Also sample a few top-level slices if still empty
|
|
if (!found.length) {
|
|
try {
|
|
for (const ent of fs.readdirSync(root, { withFileTypes: true })) {
|
|
if (!ent.isDirectory()) continue
|
|
if (!ent.name.endsWith('.slice') && ent.name !== 'docker') continue
|
|
const full = path.join(root, ent.name)
|
|
found.push({ id: ent.name, title: ent.name, path: full })
|
|
if (found.length >= Math.min(8, max)) break
|
|
}
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
return found
|
|
}
|
|
|
|
function parseCpuStat(dir) {
|
|
const raw = readFile(path.join(dir, 'cpu.stat'))
|
|
if (!raw) return null
|
|
/** @type {Record<string, number>} */
|
|
const o = {}
|
|
for (const line of raw.split('\n')) {
|
|
const [k, v] = line.trim().split(/\s+/)
|
|
if (k) o[k] = Number(v) || 0
|
|
}
|
|
return o
|
|
}
|
|
|
|
function parseIoStat(dir) {
|
|
const raw = readFile(path.join(dir, 'io.stat'))
|
|
if (!raw) return { rbytes: 0, wbytes: 0 }
|
|
let rbytes = 0
|
|
let wbytes = 0
|
|
for (const line of raw.split('\n')) {
|
|
if (!line.trim()) continue
|
|
const parts = line.trim().split(/\s+/)
|
|
for (const p of parts.slice(1)) {
|
|
const [k, v] = p.split('=')
|
|
if (k === 'rbytes') rbytes += Number(v) || 0
|
|
if (k === 'wbytes') wbytes += Number(v) || 0
|
|
}
|
|
}
|
|
return { rbytes, wbytes }
|
|
}
|
|
|
|
function parseMemoryStat(dir) {
|
|
const raw = readFile(path.join(dir, 'memory.stat'))
|
|
if (!raw) return null
|
|
/** @type {Record<string, number>} */
|
|
const o = {}
|
|
for (const line of raw.split('\n')) {
|
|
const [k, v] = line.trim().split(/\s+/)
|
|
if (k) o[k] = Number(v) || 0
|
|
}
|
|
return o
|
|
}
|
|
|
|
function parsePressureSome10(dir, kind) {
|
|
const raw = readFile(path.join(dir, `${kind}.pressure`))
|
|
if (!raw) return null
|
|
const m = raw.match(/some avg10=([\d.]+)/)
|
|
return m ? Number(m[1]) : null
|
|
}
|
|
|
|
function makeCgroupPressureChart(id, title, kind) {
|
|
const safe = String(id).replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 96)
|
|
return {
|
|
id: `cgroup.pressure.${kind}.${safe}`,
|
|
name: `cgroup.pressure.${kind}.${safe}`,
|
|
context: 'cgroup.pressure',
|
|
title: `Cgroup ${kind} pressure ${title || safe}`,
|
|
units: 'percentage',
|
|
family: title || safe,
|
|
chartType: 'line',
|
|
priority: 6120,
|
|
plugin: 'cgroups',
|
|
dimensions: [{ id: 'some10', name: 'some10', algorithm: 'absolute' }],
|
|
}
|
|
}
|
|
|
|
export class CgroupsCollector extends EventEmitter {
|
|
constructor(opts = {}) {
|
|
super()
|
|
this.intervalMs = opts.intervalMs ?? (Number(process.env.PEARDATA_SAMPLE_MS) || SAMPLE_INTERVAL_MS)
|
|
this.timer = null
|
|
this.running = false
|
|
/** @type {Map<string, { usage: number, user: number, system: number, rbytes: number, wbytes: number, throttledUsec: number }>} */
|
|
this.prev = new Map()
|
|
this.lastTs = 0
|
|
}
|
|
|
|
start() {
|
|
if (this.running) return
|
|
if (!cgroupRoot()) {
|
|
log.info('cgroup v2 not available — collector idle')
|
|
return
|
|
}
|
|
this.running = true
|
|
this._tick()
|
|
this.timer = setInterval(() => this._tick(), this.intervalMs)
|
|
if (this.timer.unref) this.timer.unref()
|
|
log.info('Cgroups collector started', { max: maxCgroups() })
|
|
}
|
|
|
|
stop() {
|
|
this.running = false
|
|
if (this.timer) clearInterval(this.timer)
|
|
this.timer = null
|
|
}
|
|
|
|
_tick() {
|
|
const ts = Date.now()
|
|
const dtSec = this.lastTs ? (ts - this.lastTs) / 1000 : this.intervalMs / 1000
|
|
this.lastTs = ts
|
|
const ncpu = os.cpus().length || 1
|
|
/** @type {Array<{ chart: string, context: string, ts: number, values: object }>} */
|
|
const batch = []
|
|
|
|
for (const cg of discoverCgroups()) {
|
|
const cpuDef = makeCgroupCpuChart(cg.id, cg.title)
|
|
const memDef = makeCgroupMemChart(cg.id, cg.title)
|
|
const ioDef = makeCgroupIoChart(cg.id, cg.title)
|
|
const memDetailDef = makeCgroupMemDetailChart(cg.id, cg.title)
|
|
const throttleDef = makeCgroupThrottleChart(cg.id, cg.title)
|
|
registerChart(cpuDef)
|
|
registerChart(memDef)
|
|
registerChart(ioDef)
|
|
registerChart(memDetailDef)
|
|
registerChart(throttleDef)
|
|
|
|
const cpu = parseCpuStat(cg.path)
|
|
let usageUsec = cpu?.usage_usec ?? 0
|
|
let userUsec = cpu?.user_usec ?? 0
|
|
let systemUsec = cpu?.system_usec ?? 0
|
|
const throttledUsec = cpu?.throttled_usec ?? 0
|
|
|
|
const memCur = Number(readFile(path.join(cg.path, 'memory.current')) || 0)
|
|
const memMaxRaw = readFile(path.join(cg.path, 'memory.max'))
|
|
const memMax =
|
|
memMaxRaw && memMaxRaw.trim() !== 'max' ? Number(memMaxRaw.trim()) || 0 : 0
|
|
|
|
const io = parseIoStat(cg.path)
|
|
const prev = this.prev.get(cg.id)
|
|
|
|
let userPct = 0
|
|
let sysPct = 0
|
|
if (prev && dtSec > 0) {
|
|
const dUser = Math.max(0, userUsec - prev.user)
|
|
const dSys = Math.max(0, systemUsec - prev.system)
|
|
const dUsage = Math.max(0, usageUsec - prev.usage)
|
|
if (userUsec || systemUsec) {
|
|
userPct = (dUser / (dtSec * 1e6 * ncpu)) * 100
|
|
sysPct = (dSys / (dtSec * 1e6 * ncpu)) * 100
|
|
} else {
|
|
userPct = (dUsage / (dtSec * 1e6 * ncpu)) * 100
|
|
}
|
|
}
|
|
|
|
batch.push({
|
|
chart: cpuDef.id,
|
|
context: 'cgroup.cpu',
|
|
ts,
|
|
values: { user: userPct, system: sysPct },
|
|
})
|
|
batch.push({
|
|
chart: memDef.id,
|
|
context: 'cgroup.mem',
|
|
ts,
|
|
values: {
|
|
usage: memCur / (1024 * 1024),
|
|
limit: memMax / (1024 * 1024),
|
|
},
|
|
})
|
|
batch.push({
|
|
chart: ioDef.id,
|
|
context: 'cgroup.io',
|
|
ts,
|
|
values: {
|
|
read: prev && dtSec > 0 ? Math.max(0, io.rbytes - prev.rbytes) / dtSec / 1024 : 0,
|
|
write: prev && dtSec > 0 ? Math.max(0, io.wbytes - prev.wbytes) / dtSec / 1024 : 0,
|
|
},
|
|
})
|
|
|
|
const memStat = parseMemoryStat(cg.path)
|
|
if (memStat) {
|
|
batch.push({
|
|
chart: memDetailDef.id,
|
|
context: 'cgroup.mem_detail',
|
|
ts,
|
|
values: {
|
|
anon: (memStat.anon || 0) / (1024 * 1024),
|
|
file: (memStat.file || 0) / (1024 * 1024),
|
|
kernel: (memStat.kernel || 0) / (1024 * 1024),
|
|
sock: (memStat.sock || 0) / (1024 * 1024),
|
|
},
|
|
})
|
|
}
|
|
|
|
let throttlePct = 0
|
|
if (prev && dtSec > 0) {
|
|
const dThrottled = Math.max(0, throttledUsec - prev.throttledUsec)
|
|
throttlePct = Math.min(100, (dThrottled / (dtSec * 1e6)) * 100)
|
|
}
|
|
batch.push({
|
|
chart: throttleDef.id,
|
|
context: 'cgroup.cpu_throttle',
|
|
ts,
|
|
values: { throttled: throttlePct },
|
|
})
|
|
|
|
for (const kind of ['cpu', 'memory', 'io']) {
|
|
const some10 = parsePressureSome10(cg.path, kind)
|
|
if (some10 == null) continue
|
|
const pressureDef = makeCgroupPressureChart(cg.id, cg.title, kind)
|
|
registerChart(pressureDef)
|
|
batch.push({
|
|
chart: pressureDef.id,
|
|
context: 'cgroup.pressure',
|
|
ts,
|
|
values: { some10 },
|
|
})
|
|
}
|
|
|
|
this.prev.set(cg.id, {
|
|
usage: usageUsec,
|
|
user: userUsec,
|
|
system: systemUsec,
|
|
rbytes: io.rbytes,
|
|
wbytes: io.wbytes,
|
|
throttledUsec,
|
|
})
|
|
}
|
|
|
|
if (batch.length) this.emit('samples', batch)
|
|
}
|
|
}
|
|
|
|
/** @type {CgroupsCollector|null} */
|
|
let singleton = null
|
|
|
|
export function getCgroupsCollector() {
|
|
if (!singleton) singleton = new CgroupsCollector()
|
|
return singleton
|
|
}
|