/** * Additional host collectors: softirqs, IRQ, cpuidle, NUMA nodes, pagetype, * conntrack, inotify, icmpmsg, ipv6, powercap, edac, debugfs extras. */ import fs from 'fs' import path from 'path' import os from 'os' import { registerChart, makeIrqChart, makeCpuIdleChart, makeNumaNodeMemChart, } from '../../../shared/metrics.js' function readFile(p) { try { return fs.readFileSync(p, 'utf8') } catch { return null } } function rate(prev, cur, dtSec) { if (dtSec <= 0 || cur < prev) return 0 return (cur - prev) / dtSec } function bytesToMiB(n) { return n / (1024 * 1024) } /** @returns {Record} */ export function parseSoftirqs() { const raw = readFile('/proc/softirqs') /** @type {Record} */ const out = {} if (!raw) return out const lines = raw.trim().split('\n') for (const line of lines.slice(1)) { const p = line.trim().split(/\s+/) const name = p[0]?.replace(':', '') if (!name) continue let sum = 0 for (let i = 1; i < p.length; i++) sum += Number(p[i]) || 0 out[name] = sum } return out } /** @returns {Array<{ irq: string, name: string, count: number }>} */ export function parseInterrupts() { const raw = readFile('/proc/interrupts') if (!raw) return [] const lines = raw.trim().split('\n') const out = [] for (const line of lines.slice(1)) { const m = line.match(/^\s*(\d+):\s+(.+)$/) if (!m) continue const irq = m[1] const rest = m[2].trim().split(/\s+/) // last non-numeric tokens are name let count = 0 let i = 0 while (i < rest.length && /^\d+$/.test(rest[i])) { count += Number(rest[i]) || 0 i++ } const name = rest.slice(i).join(' ').slice(0, 48) || irq out.push({ irq, name, count }) } // keep top IRQs by count to limit cardinality out.sort((a, b) => b.count - a.count) return out.slice(0, Number(process.env.PEARDATA_IRQ_MAX) || 24) } export function parseSnmp6() { const raw = readFile('/proc/net/snmp6') /** @type {Record} */ const out = {} if (!raw) return out for (const line of raw.split('\n')) { const [k, v] = line.trim().split(/\s+/) if (k) out[k] = Number(v) || 0 } return out } export function parseIcmpMsg() { const raw = readFile('/proc/net/snmp') if (!raw) return {} const lines = raw.split('\n') for (let i = 0; i + 1 < lines.length; i += 2) { if (!lines[i].startsWith('IcmpMsg:')) continue const header = lines[i].trim().split(/\s+/) const values = lines[i + 1].trim().split(/\s+/) /** @type {Record} */ const row = {} for (let j = 1; j < header.length; j++) row[header[j]] = Number(values[j]) || 0 return row } return {} } export function parseConntrack(state) { const entries = Number(readFile('/proc/sys/net/netfilter/nf_conntrack_count') || 0) const max = Number(readFile('/proc/sys/net/netfilter/nf_conntrack_max') || 0) const raw = readFile('/proc/net/stat/nf_conntrack') /** @type {Record} */ const sums = { searched: 0, found: 0, new: 0, invalid: 0, drop: 0, insert_failed: 0, } if (raw) { const lines = raw.trim().split('\n') const header = lines[0]?.trim().split(/\s+/) || [] const idx = (name) => header.indexOf(name) for (const line of lines.slice(1)) { const cols = line.trim().split(/\s+/) const get = (name) => { const i = idx(name) return i >= 0 ? parseInt(cols[i], 16) || 0 : 0 } sums.searched += get('searched') sums.found += get('found') sums.new += get('new') sums.invalid += get('invalid') sums.drop += get('drop') sums.insert_failed += get('insert_failed') } } return { entries, max, ...sums } } export function parsePagetypeGlobal() { const raw = readFile('/proc/pagetypeinfo') /** @type {Record} */ const orders = {} for (let i = 0; i <= 10; i++) orders[`order${i}`] = 0 if (!raw) return orders for (const line of raw.split('\n')) { if (!line.includes('Node') || !line.includes('free')) continue const nums = line.match( /\s(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s*$/ ) if (!nums) continue for (let i = 0; i <= 10; i++) orders[`order${i}`] += Number(nums[i + 1]) || 0 } return orders } export function collectNumaNodes(batch, ts) { const root = '/sys/devices/system/node' let nodes try { nodes = fs.readdirSync(root).filter((d) => /^node\d+$/.test(d)) } catch { return } if (nodes.length < 1) return for (const n of nodes) { const id = n.slice(4) const meminfo = readFile(path.join(root, n, 'meminfo')) if (!meminfo) continue /** @type {Record} */ const vals = {} for (const line of meminfo.split('\n')) { const m = line.match(/Node \d+ (\w+):\s+(\d+)/) if (m) vals[m[1]] = (Number(m[2]) || 0) * 1024 } const def = makeNumaNodeMemChart(id) registerChart(def) const total = vals.MemTotal || 0 const free = vals.MemFree || 0 batch.push({ chart: def.id, context: def.context, ts, values: { MemTotal: bytesToMiB(total), MemFree: bytesToMiB(free), MemUsed: bytesToMiB(Math.max(0, total - free)), }, }) } } /** * C-state residency % from cpuidle time counters. * @param {Array} batch * @param {number} ts * @param {number} dtSec * @param {{ lastIdle?: Record> }} state */ export function collectCpuidle(batch, ts, dtSec, state) { if (os.platform() !== 'linux') return const cpuRoot = '/sys/devices/system/cpu' let dirs try { dirs = fs.readdirSync(cpuRoot).filter((d) => /^cpu\d+$/.test(d)) } catch { return } const prevAll = state.lastIdle || {} /** @type {Record>} */ const nextAll = {} for (const d of dirs) { const coreId = d.slice(3) const idleDir = path.join(cpuRoot, d, 'cpuidle') let states try { states = fs.readdirSync(idleDir).filter((s) => /^state\d+$/.test(s)) } catch { continue } /** @type {Record} */ const times = {} for (const s of states) { const name = (readFile(path.join(idleDir, s, 'name')) || s).trim().toLowerCase().replace(/\s+/g, '_') const t = Number(readFile(path.join(idleDir, s, 'time')) || 0) // µs times[name || s] = t } nextAll[coreId] = times const prev = prevAll[coreId] if (!prev || dtSec <= 0) continue /** @type {Record} */ const deltas = {} let sum = 0 for (const [k, v] of Object.entries(times)) { const dlt = Math.max(0, v - (prev[k] || 0)) deltas[k] = dlt sum += dlt } // active ≈ wall time − idle (µs); wall ≈ dtSec * 1e6 const wall = dtSec * 1e6 const active = Math.max(0, wall - sum) sum += active if (sum <= 0) continue /** @type {Record} */ const pcts = { active: (active / sum) * 100 } for (const [k, v] of Object.entries(deltas)) pcts[k] = (v / sum) * 100 const def = makeCpuIdleChart(coreId) def.dimensions = Object.keys(pcts).map((id) => ({ id, name: id, algorithm: 'absolute', })) registerChart(def) batch.push({ chart: def.id, context: 'cpuidle.cpu_cstate_residency_time', ts, values: pcts, }) } state.lastIdle = nextAll } export function collectPowercap(batch, ts, state) { const base = '/sys/class/powercap' let ents try { ents = fs.readdirSync(base).filter((e) => e.includes('intel-rapl')) } catch { return } let energy = 0 for (const e of ents) { if (e.includes(':')) continue // only package-level intel-rapl:0 const raw = readFile(path.join(base, e, 'energy_uj')) if (raw == null) continue energy += Number(raw.trim()) || 0 } // also try intel-rapl:0 specifically const pkg = readFile(path.join(base, 'intel-rapl:0', 'energy_uj')) if (pkg != null) energy = Number(pkg.trim()) || energy const prev = state.lastEnergyUj const prevTs = state.lastEnergyTs if (prev != null && prevTs && ts > prevTs) { const dt = (ts - prevTs) / 1000 const watts = Math.max(0, (energy - prev) / 1e6 / dt) batch.push({ chart: 'system.power', context: 'system.power', ts, values: { package: watts }, }) } state.lastEnergyUj = energy state.lastEnergyTs = ts } export function collectEdac(batch, ts) { const root = '/sys/devices/system/edac/mc' let mcs try { mcs = fs.readdirSync(root).filter((d) => d.startsWith('mc')) } catch { return } let ce = 0 let ue = 0 for (const mc of mcs) { ce += Number(readFile(path.join(root, mc, 'ce_count')) || 0) ue += Number(readFile(path.join(root, mc, 'ue_count')) || 0) } batch.push({ chart: 'mem.edac', context: 'mem.edac', ts, values: { corrected: ce, uncorrected: ue }, }) } export function collectDebugfsExtras(batch, ts) { // zswap pool (optional) const pool = readFile('/sys/kernel/debug/zswap/pool_total_size') const stored = readFile('/sys/kernel/debug/zswap/stored_pages') if (pool != null || stored != null) { registerChart({ id: 'mem.zswap_pool', name: 'mem.zswap_pool', context: 'mem.zswap_pool', title: 'Zswap pool (debugfs)', units: 'MiB', family: 'zswap', chartType: 'line', priority: 2485, plugin: 'debugfs', dimensions: [ { id: 'pool', name: 'pool', algorithm: 'absolute' }, { id: 'stored', name: 'stored', algorithm: 'absolute' }, ], }) batch.push({ chart: 'mem.zswap_pool', context: 'mem.zswap_pool', ts, values: { pool: bytesToMiB(Number(pool || 0)), stored: bytesToMiB((Number(stored || 0) || 0) * 4096), }, }) } // extfrag index — sample first node/zone order0 as signal const frag = readFile('/sys/kernel/debug/extfrag/extfrag_index') if (frag) { const nums = frag.match(/-?[\d.]+/g) if (nums?.length) { registerChart({ id: 'mem.fragmentation_index', name: 'mem.fragmentation_index', context: 'mem.fragmentation_index', title: 'Memory fragmentation index (debugfs)', units: 'index', family: 'fragmentation', chartType: 'line', priority: 277, plugin: 'debugfs', dimensions: [{ id: 'index', name: 'index', algorithm: 'absolute' }], }) batch.push({ chart: 'mem.fragmentation_index', context: 'mem.fragmentation_index', ts, values: { index: Number(nums[0]) || 0 }, }) } } } /** * @param {Record} mem */ export function collectMemExtras(batch, ts, mem) { if (!mem) return const mib = (k) => bytesToMiB(mem[k] || 0) if (mem.HighTotal != null || mem.LowTotal != null) { batch.push({ chart: 'mem.high_low', context: 'mem.high_low', ts, values: { high: mib('HighTotal'), low: mib('LowTotal') }, }) } if (mem.CmaTotal != null) { batch.push({ chart: 'mem.cma', context: 'mem.cma', ts, values: { total: mib('CmaTotal'), free: mib('CmaFree') }, }) } batch.push({ chart: 'mem.directmaps', context: 'mem.directmaps', ts, values: { '4k': mib('DirectMap4k'), '2M': mib('DirectMap2M') || mib('DirectMap4M'), '1G': mib('DirectMap1G'), }, }) } export function collectThpMore(batch, ts, dtSec, vm, prevVm) { if (!vm) return const r = (k) => (prevVm ? rate(prevVm[k] || 0, vm[k] || 0, dtSec) : 0) batch.push({ chart: 'mem.thp_file', context: 'mem.thp_file', ts, values: { alloc: r('thp_file_alloc'), fallback: r('thp_file_fallback'), mapped: r('thp_file_mapped'), }, }) batch.push({ chart: 'mem.thp_zero', context: 'mem.thp_zero', ts, values: { alloc: r('thp_zero_page_alloc'), failed: r('thp_zero_page_alloc_failed') }, }) batch.push({ chart: 'mem.thp_collapse', context: 'mem.thp_collapse', ts, values: { alloc: r('thp_collapse_alloc'), failed: r('thp_collapse_alloc_failed') }, }) batch.push({ chart: 'mem.thp_swapout', context: 'mem.thp_swapout', ts, values: { swapout: r('thp_swpout'), fallback: r('thp_swpout_fallback') }, }) batch.push({ chart: 'mem.thp_compact', context: 'mem.thp_compact', ts, values: { success: r('compact_success'), fail: r('compact_fail'), stall: r('compact_stall'), }, }) } export function collectKsmExtras(batch, ts) { const keys = ['pages_shared', 'pages_sharing', 'pages_unshared', 'pages_volatile'] /** @type {Record} */ const ksm = {} for (const k of keys) { const raw = readFile(path.join('/sys/kernel/mm/ksm', k)) if (raw == null) return ksm[k] = Number(raw.trim()) || 0 } const page = 4096 const offered = ((ksm.pages_sharing || 0) + (ksm.pages_shared || 0) + (ksm.pages_unshared || 0) + (ksm.pages_volatile || 0)) * page const saved = (ksm.pages_sharing || 0) * page batch.push({ chart: 'mem.ksm_savings', context: 'mem.ksm_savings', ts, values: { offered: bytesToMiB(offered), saved: bytesToMiB(saved) }, }) batch.push({ chart: 'mem.ksm_ratios', context: 'mem.ksm_ratios', ts, values: { savings: offered > 0 ? (saved / offered) * 100 : 0 }, }) } /** * Master tick for host-more extras. */ export function collectHostMore(batch, ts, dtSec, state, { mem, vm, prevVm } = {}) { // softirqs const soft = parseSoftirqs() const prevSoft = state.lastSoftirqs if (Object.keys(soft).length) { /** @type {Record} */ const values = {} for (const k of Object.keys(soft)) { values[k] = prevSoft ? rate(prevSoft[k] || 0, soft[k] || 0, dtSec) : 0 } batch.push({ chart: 'system.softirqs', context: 'system.softirqs', ts, values }) state.lastSoftirqs = soft } // IRQs const irqs = parseInterrupts() const prevIrq = state.lastIrqs || {} /** @type {Record} */ const nextIrq = {} for (const { irq, name, count } of irqs) { nextIrq[irq] = count const def = makeIrqChart(irq, name) registerChart(def) batch.push({ chart: def.id, context: 'system.intr_irq', ts, values: { interrupts: prevIrq[irq] != null ? rate(prevIrq[irq], count, dtSec) : 0, }, }) } state.lastIrqs = nextIrq // icmpmsg const icmpMsg = parseIcmpMsg() const prevMsg = state.lastIcmpMsg if (Object.keys(icmpMsg).length) { const pick = (k) => (prevMsg ? rate(prevMsg[k] || 0, icmpMsg[k] || 0, dtSec) : 0) batch.push({ chart: 'ipv4.icmpmsg', context: 'ipv4.icmpmsg', ts, values: { InEchoes: pick('InType8') || pick('InEchoes'), OutEchoes: pick('OutType8') || pick('OutEchoes'), InDestUnreachs: pick('InType3') || pick('InDestUnreachs'), OutDestUnreachs: pick('OutType3') || pick('OutDestUnreachs'), InTimeExcds: pick('InType11') || pick('InTimeExcds'), OutTimeExcds: pick('OutType11') || pick('OutTimeExcds'), InRedirects: pick('InType5') || pick('InRedirects'), OutRedirects: pick('OutType5') || pick('OutRedirects'), }, }) state.lastIcmpMsg = icmpMsg } // ipv6 const s6 = parseSnmp6() const prev6 = state.lastSnmp6 if (Object.keys(s6).length) { const r = (k) => (prev6 ? rate(prev6[k] || 0, s6[k] || 0, dtSec) : 0) batch.push({ chart: 'ipv6.packets', context: 'ipv6.packets', ts, values: { received: r('Ip6InReceives'), sent: r('Ip6OutRequests'), forwarded: r('Ip6OutForwDatagrams') || r('Ip6InForwDatagrams'), delivered: r('Ip6InDelivers'), }, }) batch.push({ chart: 'ipv6.errors', context: 'ipv6.errors', ts, values: { InHdrErrors: r('Ip6InHdrErrors'), InAddrErrors: r('Ip6InAddrErrors'), InDiscards: r('Ip6InDiscards'), OutDiscards: r('Ip6OutDiscards'), InTruncatedPkts: r('Ip6InTruncatedPkts'), }, }) batch.push({ chart: 'ipv6.icmp', context: 'ipv6.icmp', ts, values: { received: r('Icmp6InMsgs'), sent: r('Icmp6OutMsgs'), InErrors: r('Icmp6InErrors'), OutErrors: r('Icmp6OutErrors'), }, }) state.lastSnmp6 = s6 } // conntrack const ct = parseConntrack() const prevCt = state.lastCt batch.push({ chart: 'net.conntrack', context: 'net.conntrack', ts, values: { entries: ct.entries, max: ct.max, searched: prevCt ? rate(prevCt.searched, ct.searched, dtSec) : 0, found: prevCt ? rate(prevCt.found, ct.found, dtSec) : 0, new: prevCt ? rate(prevCt.new, ct.new, dtSec) : 0, invalid: prevCt ? rate(prevCt.invalid, ct.invalid, dtSec) : 0, drop: prevCt ? rate(prevCt.drop, ct.drop, dtSec) : 0, insert_failed: prevCt ? rate(prevCt.insert_failed, ct.insert_failed, dtSec) : 0, }, }) state.lastCt = ct // inotify limits batch.push({ chart: 'system.inotify', context: 'system.inotify', ts, values: { max_user_watches: Number(readFile('/proc/sys/fs/inotify/max_user_watches') || 0), max_user_instances: Number(readFile('/proc/sys/fs/inotify/max_user_instances') || 0), }, }) batch.push({ chart: 'mem.pagetype_global', context: 'mem.pagetype_global', ts, values: parsePagetypeGlobal(), }) collectNumaNodes(batch, ts) collectCpuidle(batch, ts, dtSec, state) collectPowercap(batch, ts, state) collectEdac(batch, ts) collectDebugfsExtras(batch, ts) if (mem) collectMemExtras(batch, ts, mem) collectThpMore(batch, ts, dtSec, vm, prevVm) collectKsmExtras(batch, ts) }