Files
peardata/server/services/collectors/host-deep.js
T
Raven Scott af6609c2d3
CI / test (push) Successful in 1m11s
Release rolling / release (push) Successful in 7m35s
CPU reductions (same charts/features)
Cache cpuidle names (was re-reading name sysfs every tick × cores)
Cache CPU sysfs dir list + register cpufreq/throttle once
Map for per-core prev lookups (was O(n²) .find)
Dedupe meminfo/vmstat once per tick
Cache iface speed/duplex/mtu/qlen (30s)
Cache inotify limits (30s)
Gate registerChart for IRQs, cpuidle, peardock mapped charts
Remove leftover profile() stderr timers
Cache os.cpus().length in docker collector
Grow fd-cache buffer past 64KB for /proc/interrupts etc.
2026-07-22 13:40:39 -04:00

594 lines
17 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* Deep host metric helpers — CPU freq/throttle, KSM, softnet, TcpExt, PSI full.
* Called from the main MetricsCollector (Linux /proc + /sys).
*/
import fs from 'fs'
import os from 'os'
import path from 'path'
import { registerChart } from '../../../shared/metrics.js'
import { readFileCached } from '../../utils/fd-cache.js'
function readFile(p) {
return readFileCached(p)
}
function rate(prev, cur, dtSec) {
if (dtSec <= 0 || cur < prev) return 0
return (cur - prev) / dtSec
}
function bytesToMiB(n) {
return n / (1024 * 1024)
}
function pageKiB() {
try {
return (os.constants?.dlopen ? 4 : 4) // pages typically 4 KiB
} catch {
return 4
}
}
/** @returns {Record<string, number>} */
export function parseNetstatTables() {
const raw = readFile('/proc/net/netstat')
/** @type {Record<string, Record<string, number>>} */
const tables = {}
if (!raw) return { TcpExt: {}, IpExt: {} }
const lines = raw.split('\n')
for (let i = 0; i + 1 < lines.length; i += 2) {
const header = lines[i].trim().split(/\s+/)
const values = lines[i + 1].trim().split(/\s+/)
const name = header[0].replace(':', '')
/** @type {Record<string, number>} */
const row = {}
for (let j = 1; j < header.length; j++) row[header[j]] = Number(values[j]) || 0
tables[name] = row
}
return { TcpExt: tables.TcpExt || {}, IpExt: tables.IpExt || {} }
}
/** @returns {{ processed: number, dropped: number, squeezed: number, received_rps: number, flow_limit: number }} */
export function parseSoftnet() {
const raw = readFile('/proc/net/softnet_stat')
const out = { processed: 0, dropped: 0, squeezed: 0, received_rps: 0, flow_limit: 0 }
if (!raw) return out
for (const line of raw.split('\n')) {
const cols = line.trim().split(/\s+/)
if (cols.length < 3) continue
out.processed += parseInt(cols[0], 16) || 0
out.dropped += parseInt(cols[1], 16) || 0
out.squeezed += parseInt(cols[2], 16) || 0
if (cols[9]) out.received_rps += parseInt(cols[9], 16) || 0
if (cols[10]) out.flow_limit += parseInt(cols[10], 16) || 0
}
return out
}
/**
* @param {string} kind
* @returns {{ some10:number, some60:number, some300:number, full10?:number, full60?:number, full300?:number }|null}
*/
export function parsePressureFull(kind) {
const raw = readFile(`/proc/pressure/${kind}`)
if (!raw) return null
const some = raw.match(/some avg10=([\d.]+) avg60=([\d.]+) avg300=([\d.]+)/)
if (!some) return null
const full = raw.match(/full avg10=([\d.]+) avg60=([\d.]+) avg300=([\d.]+)/)
/** @type {any} */
const out = {
some10: Number(some[1]),
some60: Number(some[2]),
some300: Number(some[3]),
}
if (full) {
out.full10 = Number(full[1])
out.full60 = Number(full[2])
out.full300 = Number(full[3])
}
return out
}
/** @returns {Record<string, number>|null} */
export function readKsm() {
const base = '/sys/kernel/mm/ksm'
const keys = ['pages_shared', 'pages_sharing', 'pages_unshared', 'pages_volatile']
/** @type {Record<string, number>} */
const out = {}
let any = false
for (const k of keys) {
const raw = readFile(path.join(base, k))
if (raw == null) continue
out[k] = Number(raw.trim()) || 0
any = true
}
return any ? out : null
}
/**
* Per-core MHz + throttle counters.
* @param {Array<{ chart: string, context: string, ts: number, values: object }>} batch
* @param {number} ts
* @param {number} dtSec
* @param {{ lastThrottle?: Record<string, { core: number, pkg: number }> }} state
*/
export function collectCpuSysfs(batch, ts, dtSec, state) {
if (os.platform() !== 'linux') return
const cpuRoot = '/sys/devices/system/cpu'
const now = Date.now()
let dirs = state._cpuSysfsDirs
if (!dirs || now - (state._cpuSysfsDirsTs || 0) > 30000) {
try {
dirs = fs.readdirSync(cpuRoot).filter((d) => /^cpu\d+$/.test(d))
} catch {
return
}
state._cpuSysfsDirs = dirs
state._cpuSysfsDirsTs = now
}
/** @type {Record<string, number>} */
const freqs = {}
const throttle = state.lastThrottle || {}
/** @type {Record<string, { core: number, pkg: number }>} */
const nextThrottle = {}
/** @type {Record<string, number>} */
const coreThrottleRates = {}
/** @type {Record<string, number>} */
const pkgThrottleRates = {}
for (const d of dirs) {
const id = d.slice(3)
const freqRaw =
readFile(path.join(cpuRoot, d, 'cpufreq/scaling_cur_freq')) ||
readFile(path.join(cpuRoot, d, 'cpufreq/cpuinfo_cur_freq'))
if (freqRaw != null) {
const khz = Number(freqRaw.trim()) || 0
freqs[`cpu${id}`] = khz / 1000
}
const coreRaw = readFile(path.join(cpuRoot, d, 'thermal_throttle/core_throttle_count'))
const pkgRaw = readFile(path.join(cpuRoot, d, 'thermal_throttle/package_throttle_count'))
if (coreRaw != null || pkgRaw != null) {
const core = Number(coreRaw?.trim()) || 0
const pkg = Number(pkgRaw?.trim()) || 0
nextThrottle[id] = { core, pkg }
const prev = throttle[id]
if (prev) {
coreThrottleRates[`cpu${id}`] = rate(prev.core, core, dtSec)
pkgThrottleRates[`cpu${id}`] = rate(prev.pkg, pkg, dtSec)
}
}
}
if (Object.keys(freqs).length) {
if (!state._cpufreqRegistered) {
registerChart({
id: 'cpu.cpufreq',
name: 'cpu.cpufreq',
context: 'cpu.cpufreq',
title: 'CPU frequency',
units: 'MHz',
family: 'cpufreq',
chartType: 'line',
priority: 180,
plugin: 'proc',
dimensions: Object.keys(freqs).map((id) => ({
id,
name: id,
algorithm: 'absolute',
})),
})
state._cpufreqRegistered = true
}
batch.push({ chart: 'cpu.cpufreq', context: 'cpu.cpufreq', ts, values: freqs })
}
if (Object.keys(coreThrottleRates).length || Object.keys(nextThrottle).length) {
if (!state._throttleChartsRegistered) {
registerChart({
id: 'cpu.core_throttling',
name: 'cpu.core_throttling',
context: 'cpu.core_throttling',
title: 'CPU core thermal throttling',
units: 'events/s',
family: 'throttling',
chartType: 'line',
priority: 185,
plugin: 'proc',
dimensions: Object.keys(nextThrottle).map((id) => ({
id: `cpu${id}`,
name: `cpu${id}`,
algorithm: 'incremental',
})),
})
registerChart({
id: 'cpu.package_throttling',
name: 'cpu.package_throttling',
context: 'cpu.package_throttling',
title: 'CPU package thermal throttling',
units: 'events/s',
family: 'throttling',
chartType: 'line',
priority: 186,
plugin: 'proc',
dimensions: Object.keys(nextThrottle).map((id) => ({
id: `cpu${id}`,
name: `cpu${id}`,
algorithm: 'incremental',
})),
})
state._throttleChartsRegistered = true
}
if (Object.keys(coreThrottleRates).length) {
batch.push({
chart: 'cpu.core_throttling',
context: 'cpu.core_throttling',
ts,
values: coreThrottleRates,
})
batch.push({
chart: 'cpu.package_throttling',
context: 'cpu.package_throttling',
ts,
values: pkgThrottleRates,
})
}
}
state.lastThrottle = nextThrottle
}
/**
* @param {Record<string, number>} mem bytes from parseMeminfo
* @param {Record<string, number>|null} vm
* @param {Array} batch
* @param {number} ts
* @param {number} dtSec
* @param {Record<string, number>|null} prevVm
*/
export function collectMemDeep(batch, ts, dtSec, mem, vm, prevVm) {
if (!mem) return prevVm
const kib = (k) => bytesToMiB(mem[k] || 0)
if (mem.Zswap != null || mem.Zswapped != null) {
batch.push({
chart: 'mem.zswap',
context: 'mem.zswap',
ts,
values: { in_ram: kib('Zswap'), on_disk: kib('Zswapped') },
})
}
if (mem.HardwareCorrupted != null) {
batch.push({
chart: 'mem.hwcorrupt',
context: 'mem.hwcorrupt',
ts,
values: { HardwareCorrupted: kib('HardwareCorrupted') },
})
}
// HugePages_* in meminfo are page counts (not KiB) — re-parse
const rawMem = readFile('/proc/meminfo')
let hugeTotal = 0
let hugeFree = 0
let hugeRsvd = 0
let hugeSurp = 0
let hugeSizeKiB = 2048
if (rawMem) {
for (const line of rawMem.split('\n')) {
const m = line.match(/^([\w()]+):\s+(\d+)/)
if (!m) continue
if (m[1] === 'HugePages_Total') hugeTotal = Number(m[2])
if (m[1] === 'HugePages_Free') hugeFree = Number(m[2])
if (m[1] === 'HugePages_Rsvd') hugeRsvd = Number(m[2])
if (m[1] === 'HugePages_Surp') hugeSurp = Number(m[2])
if (m[1] === 'Hugepagesize') hugeSizeKiB = Number(m[2])
}
}
if (hugeTotal > 0) {
const scale = hugeSizeKiB / 1024 // MiB per page
// HugePages_Rsvd ⊆ HugePages_Free — used is simply total free
const used = Math.max(0, hugeTotal - hugeFree) * scale
batch.push({
chart: 'mem.hugepages',
context: 'mem.hugepages',
ts,
values: {
free: hugeFree * scale,
used,
surplus: hugeSurp * scale,
reserved: hugeRsvd * scale,
},
})
}
batch.push({
chart: 'mem.thp',
context: 'mem.thp',
ts,
values: { anonymous: kib('AnonHugePages'), shmem: kib('ShmemHugePages') },
})
batch.push({
chart: 'mem.thp_details',
context: 'mem.thp_details',
ts,
values: {
ShmemPmdMapped: kib('ShmemPmdMapped'),
FileHugePages: kib('FileHugePages'),
FilePmdMapped: kib('FilePmdMapped'),
},
})
batch.push({
chart: 'mem.reclaiming',
context: 'mem.reclaiming',
ts,
values: {
Active: kib('Active'),
Inactive: kib('Inactive'),
Active_anon: kib('Active(anon)'),
Inactive_anon: kib('Inactive(anon)'),
Active_file: kib('Active(file)'),
Inactive_file: kib('Inactive(file)'),
Unevictable: kib('Unevictable'),
Mlocked: kib('Mlocked'),
},
})
const ksm = readKsm()
if (ksm) {
const pageBytes = 4096
batch.push({
chart: 'mem.ksm',
context: 'mem.ksm',
ts,
values: {
shared: bytesToMiB((ksm.pages_shared || 0) * pageBytes),
sharing: bytesToMiB((ksm.pages_sharing || 0) * pageBytes),
unshared: bytesToMiB((ksm.pages_unshared || 0) * pageBytes),
volatile: bytesToMiB((ksm.pages_volatile || 0) * pageBytes),
},
})
}
if (vm) {
const page = pageKiB()
const r = (key) => (prevVm ? rate(prevVm[key] || 0, vm[key] || 0, dtSec) : 0)
batch.push({
chart: 'mem.oom_kill',
context: 'mem.oom_kill',
ts,
values: { kills: r('oom_kill') },
})
batch.push({
chart: 'mem.balloon',
context: 'mem.balloon',
ts,
values: {
inflate: r('balloon_inflate') * page,
deflate: r('balloon_deflate') * page,
migrate: r('balloon_migrate') * page,
},
})
batch.push({
chart: 'mem.zswapio',
context: 'mem.zswapio',
ts,
values: { in: r('zswpin') * page, out: r('zswpout') * page },
})
batch.push({
chart: 'mem.numa',
context: 'mem.numa',
ts,
values: {
local: r('numa_hit') || r('numa_local'),
foreign: r('numa_foreign'),
interleave: r('numa_interleave'),
other: r('numa_other'),
pte_updates: r('numa_pte_updates'),
pages_migrated: r('numa_pages_migrated'),
},
})
batch.push({
chart: 'mem.thp_faults',
context: 'mem.thp_faults',
ts,
values: {
alloc: r('thp_fault_alloc'),
fallback: r('thp_fault_fallback'),
fallback_charge: r('thp_fault_fallback_charge'),
},
})
batch.push({
chart: 'mem.thp_split',
context: 'mem.thp_split',
ts,
values: {
split: r('thp_split_page') || r('thp_split'),
failed: r('thp_split_page_failed'),
split_pmd: r('thp_split_pmd'),
},
})
batch.push({
chart: 'mem.ksm_cow',
context: 'mem.ksm_cow',
ts,
values: {
swapin: r('ksm_swpin_copy') * page,
write: r('cow_ksm') * page,
},
})
}
return vm
}
/**
* @param {Array} batch
* @param {number} ts
* @param {number} dtSec
* @param {{ lastSoftnet?: object, lastTcpExt?: object, lastIcmp?: object }} state
* @param {{ tcp: object, icmp?: object }} snmp
*/
export function collectNetDeep(batch, ts, dtSec, state, snmp) {
const soft = parseSoftnet()
const prevS = state.lastSoftnet
batch.push({
chart: 'system.softnet_stat',
context: 'system.softnet_stat',
ts,
values: {
processed: prevS ? rate(prevS.processed, soft.processed, dtSec) : 0,
dropped: prevS ? rate(prevS.dropped, soft.dropped, dtSec) : 0,
squeezed: prevS ? rate(prevS.squeezed, soft.squeezed, dtSec) : 0,
received_rps: prevS ? rate(prevS.received_rps, soft.received_rps, dtSec) : 0,
flow_limit: prevS ? rate(prevS.flow_limit, soft.flow_limit, dtSec) : 0,
},
})
state.lastSoftnet = soft
const icmp = snmp.icmp || {}
const prevI = state.lastIcmp
if (icmp.InMsgs != null || icmp.OutMsgs != null) {
batch.push({
chart: 'ipv4.icmp',
context: 'ipv4.icmp',
ts,
values: {
received: prevI ? rate(prevI.InMsgs || 0, icmp.InMsgs || 0, dtSec) : 0,
sent: prevI ? rate(prevI.OutMsgs || 0, icmp.OutMsgs || 0, dtSec) : 0,
},
})
batch.push({
chart: 'ipv4.icmp_errors',
context: 'ipv4.icmp_errors',
ts,
values: {
InErrors: prevI ? rate(prevI.InErrors || 0, icmp.InErrors || 0, dtSec) : 0,
OutErrors: prevI ? rate(prevI.OutErrors || 0, icmp.OutErrors || 0, dtSec) : 0,
InCsumErrors: prevI ? rate(prevI.InCsumErrors || 0, icmp.InCsumErrors || 0, dtSec) : 0,
},
})
state.lastIcmp = icmp
}
const { TcpExt } = parseNetstatTables()
const prev = state.lastTcpExt
const tcp = snmp.tcp || {}
const r = (key) => (prev ? rate(prev[key] || 0, TcpExt[key] || 0, dtSec) : 0)
const rt = (key) => (prev ? rate(prev[`tcp_${key}`] || 0, tcp[key] || 0, dtSec) : 0)
batch.push({
chart: 'ip.tcpsyncookies',
context: 'ip.tcpsyncookies',
ts,
values: {
received: r('SyncookiesRecv'),
sent: r('SyncookiesSent'),
failed: r('SyncookiesFailed'),
},
})
batch.push({
chart: 'ip.tcp_syn_queue',
context: 'ip.tcp_syn_queue',
ts,
values: { drops: r('TCPReqQFullDrop'), cookies: r('TCPReqQFullDoCookies') },
})
batch.push({
chart: 'ip.tcp_accept_queue',
context: 'ip.tcp_accept_queue',
ts,
values: { overflows: r('ListenOverflows'), drops: r('ListenDrops') },
})
batch.push({
chart: 'ip.tcpconnaborts',
context: 'ip.tcpconnaborts',
ts,
values: {
baddata: r('TCPAbortOnData'),
userclosed: r('TCPAbortOnClose'),
nomemory: r('TCPAbortOnMemory'),
timeout: r('TCPAbortOnTimeout'),
linger: r('TCPAbortOnLinger'),
failed: r('TCPAbortFailed'),
},
})
batch.push({
chart: 'ip.tcpofo',
context: 'ip.tcpofo',
ts,
values: {
inqueue: r('TCPOFOQueue'),
dropped: r('TCPOFODrop'),
merged: r('TCPOFOMerge'),
pruned: r('OfoPruned'),
},
})
batch.push({
chart: 'ip.tcpreorders',
context: 'ip.tcpreorders',
ts,
values: {
timestamp: r('TCPTSReorder'),
sack: r('TCPSACKReorder'),
fack: r('TCPFACKReorder'),
reno: r('TCPRenoReorder'),
},
})
batch.push({
chart: 'ip.tcphandshake',
context: 'ip.tcphandshake',
ts,
values: {
EstabResets: rt('EstabResets'),
OutRsts: rt('OutRsts'),
AttemptFails: rt('AttemptFails'),
SynRetrans: r('TCPSynRetrans'),
},
})
batch.push({
chart: 'ip.tcpmemorypressures',
context: 'ip.tcpmemorypressures',
ts,
values: { pressures: r('TCPMemoryPressures') },
})
state.lastTcpExt = {
...TcpExt,
tcp_EstabResets: tcp.EstabResets || 0,
tcp_OutRsts: tcp.OutRsts || 0,
tcp_AttemptFails: tcp.AttemptFails || 0,
}
}
export function ifaceSpeedKbps(iface, state) {
const now = Date.now()
const cache = state?._ifaceMeta || (state && (state._ifaceMeta = {}))
const entry = cache?.[iface]
if (entry && now - entry.ts < 30000 && 'speed' in entry) return entry.speed
const raw = readFile(`/sys/class/net/${iface}/speed`)
let speed = null
if (raw != null) {
const mbps = Number(raw.trim())
if (Number.isFinite(mbps) && mbps > 0) speed = mbps * 1000
}
if (cache) {
cache[iface] = { ...(entry || {}), ts: now, speed }
}
return speed
}
/** Cached duplex/mtu/qlen — topology rarely changes */
export function ifaceLinkMeta(iface, state) {
const now = Date.now()
const cache = state._ifaceMeta || (state._ifaceMeta = {})
const entry = cache[iface]
if (entry && now - entry.ts < 30000 && entry.duplex != null) {
return { duplex: entry.duplex, mtu: entry.mtu, qlen: entry.qlen }
}
const duplex = (readFile(`/sys/class/net/${iface}/duplex`) || '').trim().toLowerCase() || null
const mtu = Number(readFile(`/sys/class/net/${iface}/mtu`) || 0) || 0
const qlen = Number(readFile(`/sys/class/net/${iface}/tx_queue_len`) || 0) || 0
cache[iface] = { ...(entry || {}), ts: now, duplex, mtu, qlen, speed: entry?.speed }
return { duplex, mtu, qlen }
}