Files
peardata/server/services/collectors/host-more.js
T
Raven Scott 2e1a3e9b06
CI / test (push) Successful in 1m24s
Release rolling / release (push) Has been cancelled
Updates
2026-07-18 19:44:32 -04:00

637 lines
18 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* Additional host collectors: softirqs, IRQ, cpuidle, NUMA nodes, pagetype,
* conntrack, inotify, icmpmsg, ipv6, powercap, edac, debugfs extras.
*/
import fs from 'fs'
import path from 'path'
import os from 'os'
import {
registerChart,
makeIrqChart,
makeCpuIdleChart,
makeNumaNodeMemChart,
} from '../../../shared/metrics.js'
function readFile(p) {
try {
return fs.readFileSync(p, 'utf8')
} catch {
return null
}
}
function rate(prev, cur, dtSec) {
if (dtSec <= 0 || cur < prev) return 0
return (cur - prev) / dtSec
}
function bytesToMiB(n) {
return n / (1024 * 1024)
}
/** @returns {Record<string, number>} */
export function parseSoftirqs() {
const raw = readFile('/proc/softirqs')
/** @type {Record<string, number>} */
const out = {}
if (!raw) return out
const lines = raw.trim().split('\n')
for (const line of lines.slice(1)) {
const p = line.trim().split(/\s+/)
const name = p[0]?.replace(':', '')
if (!name) continue
let sum = 0
for (let i = 1; i < p.length; i++) sum += Number(p[i]) || 0
out[name] = sum
}
return out
}
/** @returns {Array<{ irq: string, name: string, count: number }>} */
export function parseInterrupts() {
const raw = readFile('/proc/interrupts')
if (!raw) return []
const lines = raw.trim().split('\n')
const out = []
for (const line of lines.slice(1)) {
const m = line.match(/^\s*(\d+):\s+(.+)$/)
if (!m) continue
const irq = m[1]
const rest = m[2].trim().split(/\s+/)
// last non-numeric tokens are name
let count = 0
let i = 0
while (i < rest.length && /^\d+$/.test(rest[i])) {
count += Number(rest[i]) || 0
i++
}
const name = rest.slice(i).join(' ').slice(0, 48) || irq
out.push({ irq, name, count })
}
// keep top IRQs by count to limit cardinality
out.sort((a, b) => b.count - a.count)
return out.slice(0, Number(process.env.PEARDATA_IRQ_MAX) || 24)
}
export function parseSnmp6() {
const raw = readFile('/proc/net/snmp6')
/** @type {Record<string, number>} */
const out = {}
if (!raw) return out
for (const line of raw.split('\n')) {
const [k, v] = line.trim().split(/\s+/)
if (k) out[k] = Number(v) || 0
}
return out
}
export function parseIcmpMsg() {
const raw = readFile('/proc/net/snmp')
if (!raw) return {}
const lines = raw.split('\n')
for (let i = 0; i + 1 < lines.length; i += 2) {
if (!lines[i].startsWith('IcmpMsg:')) continue
const header = lines[i].trim().split(/\s+/)
const values = lines[i + 1].trim().split(/\s+/)
/** @type {Record<string, number>} */
const row = {}
for (let j = 1; j < header.length; j++) row[header[j]] = Number(values[j]) || 0
return row
}
return {}
}
export function parseConntrack(state) {
const entries = Number(readFile('/proc/sys/net/netfilter/nf_conntrack_count') || 0)
const max = Number(readFile('/proc/sys/net/netfilter/nf_conntrack_max') || 0)
const raw = readFile('/proc/net/stat/nf_conntrack')
/** @type {Record<string, number>} */
const sums = {
searched: 0,
found: 0,
new: 0,
invalid: 0,
drop: 0,
insert_failed: 0,
}
if (raw) {
const lines = raw.trim().split('\n')
const header = lines[0]?.trim().split(/\s+/) || []
const idx = (name) => header.indexOf(name)
for (const line of lines.slice(1)) {
const cols = line.trim().split(/\s+/)
const get = (name) => {
const i = idx(name)
return i >= 0 ? parseInt(cols[i], 16) || 0 : 0
}
sums.searched += get('searched')
sums.found += get('found')
sums.new += get('new')
sums.invalid += get('invalid')
sums.drop += get('drop')
sums.insert_failed += get('insert_failed')
}
}
return { entries, max, ...sums }
}
export function parsePagetypeGlobal() {
const raw = readFile('/proc/pagetypeinfo')
/** @type {Record<string, number>} */
const orders = {}
for (let i = 0; i <= 10; i++) orders[`order${i}`] = 0
if (!raw) return orders
for (const line of raw.split('\n')) {
if (!line.includes('Node') || !line.includes('free')) continue
const nums = line.match(
/\s(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s*$/
)
if (!nums) continue
for (let i = 0; i <= 10; i++) orders[`order${i}`] += Number(nums[i + 1]) || 0
}
return orders
}
export function collectNumaNodes(batch, ts) {
const root = '/sys/devices/system/node'
let nodes
try {
nodes = fs.readdirSync(root).filter((d) => /^node\d+$/.test(d))
} catch {
return
}
if (nodes.length < 1) return
for (const n of nodes) {
const id = n.slice(4)
const meminfo = readFile(path.join(root, n, 'meminfo'))
if (!meminfo) continue
/** @type {Record<string, number>} */
const vals = {}
for (const line of meminfo.split('\n')) {
const m = line.match(/Node \d+ (\w+):\s+(\d+)/)
if (m) vals[m[1]] = (Number(m[2]) || 0) * 1024
}
const def = makeNumaNodeMemChart(id)
registerChart(def)
const total = vals.MemTotal || 0
const free = vals.MemFree || 0
batch.push({
chart: def.id,
context: def.context,
ts,
values: {
MemTotal: bytesToMiB(total),
MemFree: bytesToMiB(free),
MemUsed: bytesToMiB(Math.max(0, total - free)),
},
})
}
}
/**
* C-state residency % from cpuidle time counters.
* @param {Array} batch
* @param {number} ts
* @param {number} dtSec
* @param {{ lastIdle?: Record<string, Record<string, number>> }} state
*/
export function collectCpuidle(batch, ts, dtSec, state) {
if (os.platform() !== 'linux') return
const cpuRoot = '/sys/devices/system/cpu'
let dirs
try {
dirs = fs.readdirSync(cpuRoot).filter((d) => /^cpu\d+$/.test(d))
} catch {
return
}
const prevAll = state.lastIdle || {}
/** @type {Record<string, Record<string, number>>} */
const nextAll = {}
for (const d of dirs) {
const coreId = d.slice(3)
const idleDir = path.join(cpuRoot, d, 'cpuidle')
let states
try {
states = fs.readdirSync(idleDir).filter((s) => /^state\d+$/.test(s))
} catch {
continue
}
/** @type {Record<string, number>} */
const times = {}
for (const s of states) {
const name = (readFile(path.join(idleDir, s, 'name')) || s).trim().toLowerCase().replace(/\s+/g, '_')
const t = Number(readFile(path.join(idleDir, s, 'time')) || 0) // µs
times[name || s] = t
}
nextAll[coreId] = times
const prev = prevAll[coreId]
if (!prev || dtSec <= 0) continue
/** @type {Record<string, number>} */
const deltas = {}
let sum = 0
for (const [k, v] of Object.entries(times)) {
const dlt = Math.max(0, v - (prev[k] || 0))
deltas[k] = dlt
sum += dlt
}
// active ≈ wall time idle (µs); wall ≈ dtSec * 1e6
const wall = dtSec * 1e6
const active = Math.max(0, wall - sum)
sum += active
if (sum <= 0) continue
/** @type {Record<string, number>} */
const pcts = { active: (active / sum) * 100 }
for (const [k, v] of Object.entries(deltas)) pcts[k] = (v / sum) * 100
const def = makeCpuIdleChart(coreId)
def.dimensions = Object.keys(pcts).map((id) => ({
id,
name: id,
algorithm: 'absolute',
}))
registerChart(def)
batch.push({
chart: def.id,
context: 'cpuidle.cpu_cstate_residency_time',
ts,
values: pcts,
})
}
state.lastIdle = nextAll
}
export function collectPowercap(batch, ts, state) {
const base = '/sys/class/powercap'
let ents
try {
ents = fs.readdirSync(base).filter((e) => e.includes('intel-rapl'))
} catch {
return
}
let energy = 0
for (const e of ents) {
if (e.includes(':')) continue // only package-level intel-rapl:0
const raw = readFile(path.join(base, e, 'energy_uj'))
if (raw == null) continue
energy += Number(raw.trim()) || 0
}
// also try intel-rapl:0 specifically
const pkg = readFile(path.join(base, 'intel-rapl:0', 'energy_uj'))
if (pkg != null) energy = Number(pkg.trim()) || energy
const prev = state.lastEnergyUj
const prevTs = state.lastEnergyTs
if (prev != null && prevTs && ts > prevTs) {
const dt = (ts - prevTs) / 1000
const watts = Math.max(0, (energy - prev) / 1e6 / dt)
batch.push({
chart: 'system.power',
context: 'system.power',
ts,
values: { package: watts },
})
}
state.lastEnergyUj = energy
state.lastEnergyTs = ts
}
export function collectEdac(batch, ts) {
const root = '/sys/devices/system/edac/mc'
let mcs
try {
mcs = fs.readdirSync(root).filter((d) => d.startsWith('mc'))
} catch {
return
}
let ce = 0
let ue = 0
for (const mc of mcs) {
ce += Number(readFile(path.join(root, mc, 'ce_count')) || 0)
ue += Number(readFile(path.join(root, mc, 'ue_count')) || 0)
}
batch.push({
chart: 'mem.edac',
context: 'mem.edac',
ts,
values: { corrected: ce, uncorrected: ue },
})
}
export function collectDebugfsExtras(batch, ts) {
// zswap pool (optional)
const pool = readFile('/sys/kernel/debug/zswap/pool_total_size')
const stored = readFile('/sys/kernel/debug/zswap/stored_pages')
if (pool != null || stored != null) {
registerChart({
id: 'mem.zswap_pool',
name: 'mem.zswap_pool',
context: 'mem.zswap_pool',
title: 'Zswap pool (debugfs)',
units: 'MiB',
family: 'zswap',
chartType: 'line',
priority: 2485,
plugin: 'debugfs',
dimensions: [
{ id: 'pool', name: 'pool', algorithm: 'absolute' },
{ id: 'stored', name: 'stored', algorithm: 'absolute' },
],
})
batch.push({
chart: 'mem.zswap_pool',
context: 'mem.zswap_pool',
ts,
values: {
pool: bytesToMiB(Number(pool || 0)),
stored: bytesToMiB((Number(stored || 0) || 0) * 4096),
},
})
}
// extfrag index — sample first node/zone order0 as signal
const frag = readFile('/sys/kernel/debug/extfrag/extfrag_index')
if (frag) {
const nums = frag.match(/-?[\d.]+/g)
if (nums?.length) {
registerChart({
id: 'mem.fragmentation_index',
name: 'mem.fragmentation_index',
context: 'mem.fragmentation_index',
title: 'Memory fragmentation index (debugfs)',
units: 'index',
family: 'fragmentation',
chartType: 'line',
priority: 277,
plugin: 'debugfs',
dimensions: [{ id: 'index', name: 'index', algorithm: 'absolute' }],
})
batch.push({
chart: 'mem.fragmentation_index',
context: 'mem.fragmentation_index',
ts,
values: { index: Number(nums[0]) || 0 },
})
}
}
}
/**
* @param {Record<string, number>} mem
*/
export function collectMemExtras(batch, ts, mem) {
if (!mem) return
const mib = (k) => bytesToMiB(mem[k] || 0)
if (mem.HighTotal != null || mem.LowTotal != null) {
batch.push({
chart: 'mem.high_low',
context: 'mem.high_low',
ts,
values: { high: mib('HighTotal'), low: mib('LowTotal') },
})
}
if (mem.CmaTotal != null) {
batch.push({
chart: 'mem.cma',
context: 'mem.cma',
ts,
values: { total: mib('CmaTotal'), free: mib('CmaFree') },
})
}
batch.push({
chart: 'mem.directmaps',
context: 'mem.directmaps',
ts,
values: {
'4k': mib('DirectMap4k'),
'2M': mib('DirectMap2M') || mib('DirectMap4M'),
'1G': mib('DirectMap1G'),
},
})
}
export function collectThpMore(batch, ts, dtSec, vm, prevVm) {
if (!vm) return
const r = (k) => (prevVm ? rate(prevVm[k] || 0, vm[k] || 0, dtSec) : 0)
batch.push({
chart: 'mem.thp_file',
context: 'mem.thp_file',
ts,
values: {
alloc: r('thp_file_alloc'),
fallback: r('thp_file_fallback'),
mapped: r('thp_file_mapped'),
},
})
batch.push({
chart: 'mem.thp_zero',
context: 'mem.thp_zero',
ts,
values: { alloc: r('thp_zero_page_alloc'), failed: r('thp_zero_page_alloc_failed') },
})
batch.push({
chart: 'mem.thp_collapse',
context: 'mem.thp_collapse',
ts,
values: { alloc: r('thp_collapse_alloc'), failed: r('thp_collapse_alloc_failed') },
})
batch.push({
chart: 'mem.thp_swapout',
context: 'mem.thp_swapout',
ts,
values: { swapout: r('thp_swpout'), fallback: r('thp_swpout_fallback') },
})
batch.push({
chart: 'mem.thp_compact',
context: 'mem.thp_compact',
ts,
values: {
success: r('compact_success'),
fail: r('compact_fail'),
stall: r('compact_stall'),
},
})
}
export function collectKsmExtras(batch, ts) {
const keys = ['pages_shared', 'pages_sharing', 'pages_unshared', 'pages_volatile']
/** @type {Record<string, number>} */
const ksm = {}
for (const k of keys) {
const raw = readFile(path.join('/sys/kernel/mm/ksm', k))
if (raw == null) return
ksm[k] = Number(raw.trim()) || 0
}
const page = 4096
const offered =
((ksm.pages_sharing || 0) +
(ksm.pages_shared || 0) +
(ksm.pages_unshared || 0) +
(ksm.pages_volatile || 0)) *
page
const saved = (ksm.pages_sharing || 0) * page
batch.push({
chart: 'mem.ksm_savings',
context: 'mem.ksm_savings',
ts,
values: { offered: bytesToMiB(offered), saved: bytesToMiB(saved) },
})
batch.push({
chart: 'mem.ksm_ratios',
context: 'mem.ksm_ratios',
ts,
values: { savings: offered > 0 ? (saved / offered) * 100 : 0 },
})
}
/**
* Master tick for host-more extras.
*/
export function collectHostMore(batch, ts, dtSec, state, { mem, vm, prevVm } = {}) {
// softirqs
const soft = parseSoftirqs()
const prevSoft = state.lastSoftirqs
if (Object.keys(soft).length) {
/** @type {Record<string, number>} */
const values = {}
for (const k of Object.keys(soft)) {
values[k] = prevSoft ? rate(prevSoft[k] || 0, soft[k] || 0, dtSec) : 0
}
batch.push({ chart: 'system.softirqs', context: 'system.softirqs', ts, values })
state.lastSoftirqs = soft
}
// IRQs
const irqs = parseInterrupts()
const prevIrq = state.lastIrqs || {}
/** @type {Record<string, number>} */
const nextIrq = {}
for (const { irq, name, count } of irqs) {
nextIrq[irq] = count
const def = makeIrqChart(irq, name)
registerChart(def)
batch.push({
chart: def.id,
context: 'system.intr_irq',
ts,
values: {
interrupts: prevIrq[irq] != null ? rate(prevIrq[irq], count, dtSec) : 0,
},
})
}
state.lastIrqs = nextIrq
// icmpmsg
const icmpMsg = parseIcmpMsg()
const prevMsg = state.lastIcmpMsg
if (Object.keys(icmpMsg).length) {
const pick = (k) => (prevMsg ? rate(prevMsg[k] || 0, icmpMsg[k] || 0, dtSec) : 0)
batch.push({
chart: 'ipv4.icmpmsg',
context: 'ipv4.icmpmsg',
ts,
values: {
InEchoes: pick('InType8') || pick('InEchoes'),
OutEchoes: pick('OutType8') || pick('OutEchoes'),
InDestUnreachs: pick('InType3') || pick('InDestUnreachs'),
OutDestUnreachs: pick('OutType3') || pick('OutDestUnreachs'),
InTimeExcds: pick('InType11') || pick('InTimeExcds'),
OutTimeExcds: pick('OutType11') || pick('OutTimeExcds'),
InRedirects: pick('InType5') || pick('InRedirects'),
OutRedirects: pick('OutType5') || pick('OutRedirects'),
},
})
state.lastIcmpMsg = icmpMsg
}
// ipv6
const s6 = parseSnmp6()
const prev6 = state.lastSnmp6
if (Object.keys(s6).length) {
const r = (k) => (prev6 ? rate(prev6[k] || 0, s6[k] || 0, dtSec) : 0)
batch.push({
chart: 'ipv6.packets',
context: 'ipv6.packets',
ts,
values: {
received: r('Ip6InReceives'),
sent: r('Ip6OutRequests'),
forwarded: r('Ip6OutForwDatagrams') || r('Ip6InForwDatagrams'),
delivered: r('Ip6InDelivers'),
},
})
batch.push({
chart: 'ipv6.errors',
context: 'ipv6.errors',
ts,
values: {
InHdrErrors: r('Ip6InHdrErrors'),
InAddrErrors: r('Ip6InAddrErrors'),
InDiscards: r('Ip6InDiscards'),
OutDiscards: r('Ip6OutDiscards'),
InTruncatedPkts: r('Ip6InTruncatedPkts'),
},
})
batch.push({
chart: 'ipv6.icmp',
context: 'ipv6.icmp',
ts,
values: {
received: r('Icmp6InMsgs'),
sent: r('Icmp6OutMsgs'),
InErrors: r('Icmp6InErrors'),
OutErrors: r('Icmp6OutErrors'),
},
})
state.lastSnmp6 = s6
}
// conntrack
const ct = parseConntrack()
const prevCt = state.lastCt
batch.push({
chart: 'net.conntrack',
context: 'net.conntrack',
ts,
values: {
entries: ct.entries,
max: ct.max,
searched: prevCt ? rate(prevCt.searched, ct.searched, dtSec) : 0,
found: prevCt ? rate(prevCt.found, ct.found, dtSec) : 0,
new: prevCt ? rate(prevCt.new, ct.new, dtSec) : 0,
invalid: prevCt ? rate(prevCt.invalid, ct.invalid, dtSec) : 0,
drop: prevCt ? rate(prevCt.drop, ct.drop, dtSec) : 0,
insert_failed: prevCt ? rate(prevCt.insert_failed, ct.insert_failed, dtSec) : 0,
},
})
state.lastCt = ct
// inotify limits
batch.push({
chart: 'system.inotify',
context: 'system.inotify',
ts,
values: {
max_user_watches: Number(readFile('/proc/sys/fs/inotify/max_user_watches') || 0),
max_user_instances: Number(readFile('/proc/sys/fs/inotify/max_user_instances') || 0),
},
})
batch.push({
chart: 'mem.pagetype_global',
context: 'mem.pagetype_global',
ts,
values: parsePagetypeGlobal(),
})
collectNumaNodes(batch, ts)
collectCpuidle(batch, ts, dtSec, state)
collectPowercap(batch, ts, state)
collectEdac(batch, ts)
collectDebugfsExtras(batch, ts)
if (mem) collectMemExtras(batch, ts, mem)
collectThpMore(batch, ts, dtSec, vm, prevVm)
collectKsmExtras(batch, ts)
}