558 lines
16 KiB
JavaScript
558 lines
16 KiB
JavaScript
/**
|
|
* Deep host metric helpers — CPU freq/throttle, KSM, softnet, TcpExt, PSI full.
|
|
* Called from the main MetricsCollector (Linux /proc + /sys).
|
|
*/
|
|
import fs from 'fs'
|
|
import os from 'os'
|
|
import path from 'path'
|
|
import { registerChart } from '../../../shared/metrics.js'
|
|
import { readFileCached } from '../../utils/fd-cache.js'
|
|
|
|
function readFile(p) {
|
|
return readFileCached(p)
|
|
}
|
|
|
|
function rate(prev, cur, dtSec) {
|
|
if (dtSec <= 0 || cur < prev) return 0
|
|
return (cur - prev) / dtSec
|
|
}
|
|
|
|
function bytesToMiB(n) {
|
|
return n / (1024 * 1024)
|
|
}
|
|
|
|
function pageKiB() {
|
|
try {
|
|
return (os.constants?.dlopen ? 4 : 4) // pages typically 4 KiB
|
|
} catch {
|
|
return 4
|
|
}
|
|
}
|
|
|
|
/** @returns {Record<string, number>} */
|
|
export function parseNetstatTables() {
|
|
const raw = readFile('/proc/net/netstat')
|
|
/** @type {Record<string, Record<string, number>>} */
|
|
const tables = {}
|
|
if (!raw) return { TcpExt: {}, IpExt: {} }
|
|
const lines = raw.split('\n')
|
|
for (let i = 0; i + 1 < lines.length; i += 2) {
|
|
const header = lines[i].trim().split(/\s+/)
|
|
const values = lines[i + 1].trim().split(/\s+/)
|
|
const name = header[0].replace(':', '')
|
|
/** @type {Record<string, number>} */
|
|
const row = {}
|
|
for (let j = 1; j < header.length; j++) row[header[j]] = Number(values[j]) || 0
|
|
tables[name] = row
|
|
}
|
|
return { TcpExt: tables.TcpExt || {}, IpExt: tables.IpExt || {} }
|
|
}
|
|
|
|
/** @returns {{ processed: number, dropped: number, squeezed: number, received_rps: number, flow_limit: number }} */
|
|
export function parseSoftnet() {
|
|
const raw = readFile('/proc/net/softnet_stat')
|
|
const out = { processed: 0, dropped: 0, squeezed: 0, received_rps: 0, flow_limit: 0 }
|
|
if (!raw) return out
|
|
for (const line of raw.split('\n')) {
|
|
const cols = line.trim().split(/\s+/)
|
|
if (cols.length < 3) continue
|
|
out.processed += parseInt(cols[0], 16) || 0
|
|
out.dropped += parseInt(cols[1], 16) || 0
|
|
out.squeezed += parseInt(cols[2], 16) || 0
|
|
if (cols[9]) out.received_rps += parseInt(cols[9], 16) || 0
|
|
if (cols[10]) out.flow_limit += parseInt(cols[10], 16) || 0
|
|
}
|
|
return out
|
|
}
|
|
|
|
/**
|
|
* @param {string} kind
|
|
* @returns {{ some10:number, some60:number, some300:number, full10?:number, full60?:number, full300?:number }|null}
|
|
*/
|
|
export function parsePressureFull(kind) {
|
|
const raw = readFile(`/proc/pressure/${kind}`)
|
|
if (!raw) return null
|
|
const some = raw.match(/some avg10=([\d.]+) avg60=([\d.]+) avg300=([\d.]+)/)
|
|
if (!some) return null
|
|
const full = raw.match(/full avg10=([\d.]+) avg60=([\d.]+) avg300=([\d.]+)/)
|
|
/** @type {any} */
|
|
const out = {
|
|
some10: Number(some[1]),
|
|
some60: Number(some[2]),
|
|
some300: Number(some[3]),
|
|
}
|
|
if (full) {
|
|
out.full10 = Number(full[1])
|
|
out.full60 = Number(full[2])
|
|
out.full300 = Number(full[3])
|
|
}
|
|
return out
|
|
}
|
|
|
|
/** @returns {Record<string, number>|null} */
|
|
export function readKsm() {
|
|
const base = '/sys/kernel/mm/ksm'
|
|
const keys = ['pages_shared', 'pages_sharing', 'pages_unshared', 'pages_volatile']
|
|
/** @type {Record<string, number>} */
|
|
const out = {}
|
|
let any = false
|
|
for (const k of keys) {
|
|
const raw = readFile(path.join(base, k))
|
|
if (raw == null) continue
|
|
out[k] = Number(raw.trim()) || 0
|
|
any = true
|
|
}
|
|
return any ? out : null
|
|
}
|
|
|
|
/**
|
|
* Per-core MHz + throttle counters.
|
|
* @param {Array<{ chart: string, context: string, ts: number, values: object }>} batch
|
|
* @param {number} ts
|
|
* @param {number} dtSec
|
|
* @param {{ lastThrottle?: Record<string, { core: number, pkg: number }> }} state
|
|
*/
|
|
export function collectCpuSysfs(batch, ts, dtSec, state) {
|
|
if (os.platform() !== 'linux') return
|
|
const cpuRoot = '/sys/devices/system/cpu'
|
|
let dirs
|
|
try {
|
|
dirs = fs.readdirSync(cpuRoot).filter((d) => /^cpu\d+$/.test(d))
|
|
} catch {
|
|
return
|
|
}
|
|
|
|
/** @type {Record<string, number>} */
|
|
const freqs = {}
|
|
const throttle = state.lastThrottle || {}
|
|
/** @type {Record<string, { core: number, pkg: number }>} */
|
|
const nextThrottle = {}
|
|
/** @type {Record<string, number>} */
|
|
const coreThrottleRates = {}
|
|
/** @type {Record<string, number>} */
|
|
const pkgThrottleRates = {}
|
|
|
|
for (const d of dirs) {
|
|
const id = d.slice(3)
|
|
const freqRaw =
|
|
readFile(path.join(cpuRoot, d, 'cpufreq/scaling_cur_freq')) ||
|
|
readFile(path.join(cpuRoot, d, 'cpufreq/cpuinfo_cur_freq'))
|
|
if (freqRaw != null) {
|
|
const khz = Number(freqRaw.trim()) || 0
|
|
freqs[`cpu${id}`] = khz / 1000
|
|
}
|
|
|
|
const coreRaw = readFile(path.join(cpuRoot, d, 'thermal_throttle/core_throttle_count'))
|
|
const pkgRaw = readFile(path.join(cpuRoot, d, 'thermal_throttle/package_throttle_count'))
|
|
if (coreRaw != null || pkgRaw != null) {
|
|
const core = Number(coreRaw?.trim()) || 0
|
|
const pkg = Number(pkgRaw?.trim()) || 0
|
|
nextThrottle[id] = { core, pkg }
|
|
const prev = throttle[id]
|
|
if (prev) {
|
|
coreThrottleRates[`cpu${id}`] = rate(prev.core, core, dtSec)
|
|
pkgThrottleRates[`cpu${id}`] = rate(prev.pkg, pkg, dtSec)
|
|
}
|
|
}
|
|
}
|
|
|
|
if (Object.keys(freqs).length) {
|
|
registerChart({
|
|
id: 'cpu.cpufreq',
|
|
name: 'cpu.cpufreq',
|
|
context: 'cpu.cpufreq',
|
|
title: 'CPU frequency',
|
|
units: 'MHz',
|
|
family: 'cpufreq',
|
|
chartType: 'line',
|
|
priority: 180,
|
|
plugin: 'proc',
|
|
dimensions: Object.keys(freqs).map((id) => ({
|
|
id,
|
|
name: id,
|
|
algorithm: 'absolute',
|
|
})),
|
|
})
|
|
batch.push({ chart: 'cpu.cpufreq', context: 'cpu.cpufreq', ts, values: freqs })
|
|
}
|
|
|
|
if (Object.keys(coreThrottleRates).length || Object.keys(nextThrottle).length) {
|
|
registerChart({
|
|
id: 'cpu.core_throttling',
|
|
name: 'cpu.core_throttling',
|
|
context: 'cpu.core_throttling',
|
|
title: 'CPU core thermal throttling',
|
|
units: 'events/s',
|
|
family: 'throttling',
|
|
chartType: 'line',
|
|
priority: 185,
|
|
plugin: 'proc',
|
|
dimensions: Object.keys(nextThrottle).map((id) => ({
|
|
id: `cpu${id}`,
|
|
name: `cpu${id}`,
|
|
algorithm: 'incremental',
|
|
})),
|
|
})
|
|
registerChart({
|
|
id: 'cpu.package_throttling',
|
|
name: 'cpu.package_throttling',
|
|
context: 'cpu.package_throttling',
|
|
title: 'CPU package thermal throttling',
|
|
units: 'events/s',
|
|
family: 'throttling',
|
|
chartType: 'line',
|
|
priority: 186,
|
|
plugin: 'proc',
|
|
dimensions: Object.keys(nextThrottle).map((id) => ({
|
|
id: `cpu${id}`,
|
|
name: `cpu${id}`,
|
|
algorithm: 'incremental',
|
|
})),
|
|
})
|
|
if (Object.keys(coreThrottleRates).length) {
|
|
batch.push({
|
|
chart: 'cpu.core_throttling',
|
|
context: 'cpu.core_throttling',
|
|
ts,
|
|
values: coreThrottleRates,
|
|
})
|
|
batch.push({
|
|
chart: 'cpu.package_throttling',
|
|
context: 'cpu.package_throttling',
|
|
ts,
|
|
values: pkgThrottleRates,
|
|
})
|
|
}
|
|
}
|
|
state.lastThrottle = nextThrottle
|
|
}
|
|
|
|
/**
|
|
* @param {Record<string, number>} mem bytes from parseMeminfo
|
|
* @param {Record<string, number>|null} vm
|
|
* @param {Array} batch
|
|
* @param {number} ts
|
|
* @param {number} dtSec
|
|
* @param {Record<string, number>|null} prevVm
|
|
*/
|
|
export function collectMemDeep(batch, ts, dtSec, mem, vm, prevVm) {
|
|
if (!mem) return prevVm
|
|
|
|
const kib = (k) => bytesToMiB(mem[k] || 0)
|
|
|
|
if (mem.Zswap != null || mem.Zswapped != null) {
|
|
batch.push({
|
|
chart: 'mem.zswap',
|
|
context: 'mem.zswap',
|
|
ts,
|
|
values: { in_ram: kib('Zswap'), on_disk: kib('Zswapped') },
|
|
})
|
|
}
|
|
if (mem.HardwareCorrupted != null) {
|
|
batch.push({
|
|
chart: 'mem.hwcorrupt',
|
|
context: 'mem.hwcorrupt',
|
|
ts,
|
|
values: { HardwareCorrupted: kib('HardwareCorrupted') },
|
|
})
|
|
}
|
|
|
|
// HugePages_* in meminfo are page counts (not KiB) — re-parse
|
|
const rawMem = readFile('/proc/meminfo')
|
|
let hugeTotal = 0
|
|
let hugeFree = 0
|
|
let hugeRsvd = 0
|
|
let hugeSurp = 0
|
|
let hugeSizeKiB = 2048
|
|
if (rawMem) {
|
|
for (const line of rawMem.split('\n')) {
|
|
const m = line.match(/^([\w()]+):\s+(\d+)/)
|
|
if (!m) continue
|
|
if (m[1] === 'HugePages_Total') hugeTotal = Number(m[2])
|
|
if (m[1] === 'HugePages_Free') hugeFree = Number(m[2])
|
|
if (m[1] === 'HugePages_Rsvd') hugeRsvd = Number(m[2])
|
|
if (m[1] === 'HugePages_Surp') hugeSurp = Number(m[2])
|
|
if (m[1] === 'Hugepagesize') hugeSizeKiB = Number(m[2])
|
|
}
|
|
}
|
|
if (hugeTotal > 0) {
|
|
const scale = hugeSizeKiB / 1024 // MiB per page
|
|
const used = Math.max(0, hugeTotal - hugeFree - hugeRsvd) * scale
|
|
batch.push({
|
|
chart: 'mem.hugepages',
|
|
context: 'mem.hugepages',
|
|
ts,
|
|
values: {
|
|
free: hugeFree * scale,
|
|
used,
|
|
surplus: hugeSurp * scale,
|
|
reserved: hugeRsvd * scale,
|
|
},
|
|
})
|
|
}
|
|
|
|
batch.push({
|
|
chart: 'mem.thp',
|
|
context: 'mem.thp',
|
|
ts,
|
|
values: { anonymous: kib('AnonHugePages'), shmem: kib('ShmemHugePages') },
|
|
})
|
|
batch.push({
|
|
chart: 'mem.thp_details',
|
|
context: 'mem.thp_details',
|
|
ts,
|
|
values: {
|
|
ShmemPmdMapped: kib('ShmemPmdMapped'),
|
|
FileHugePages: kib('FileHugePages'),
|
|
FilePmdMapped: kib('FilePmdMapped'),
|
|
},
|
|
})
|
|
batch.push({
|
|
chart: 'mem.reclaiming',
|
|
context: 'mem.reclaiming',
|
|
ts,
|
|
values: {
|
|
Active: kib('Active'),
|
|
Inactive: kib('Inactive'),
|
|
Active_anon: kib('Active(anon)'),
|
|
Inactive_anon: kib('Inactive(anon)'),
|
|
Active_file: kib('Active(file)'),
|
|
Inactive_file: kib('Inactive(file)'),
|
|
Unevictable: kib('Unevictable'),
|
|
Mlocked: kib('Mlocked'),
|
|
},
|
|
})
|
|
|
|
const ksm = readKsm()
|
|
if (ksm) {
|
|
const pageBytes = 4096
|
|
batch.push({
|
|
chart: 'mem.ksm',
|
|
context: 'mem.ksm',
|
|
ts,
|
|
values: {
|
|
shared: bytesToMiB((ksm.pages_shared || 0) * pageBytes),
|
|
sharing: bytesToMiB((ksm.pages_sharing || 0) * pageBytes),
|
|
unshared: bytesToMiB((ksm.pages_unshared || 0) * pageBytes),
|
|
volatile: bytesToMiB((ksm.pages_volatile || 0) * pageBytes),
|
|
},
|
|
})
|
|
}
|
|
|
|
if (vm) {
|
|
const page = pageKiB()
|
|
const r = (key) => (prevVm ? rate(prevVm[key] || 0, vm[key] || 0, dtSec) : 0)
|
|
batch.push({
|
|
chart: 'mem.oom_kill',
|
|
context: 'mem.oom_kill',
|
|
ts,
|
|
values: { kills: r('oom_kill') },
|
|
})
|
|
batch.push({
|
|
chart: 'mem.balloon',
|
|
context: 'mem.balloon',
|
|
ts,
|
|
values: {
|
|
inflate: r('balloon_inflate') * page,
|
|
deflate: r('balloon_deflate') * page,
|
|
migrate: r('balloon_migrate') * page,
|
|
},
|
|
})
|
|
batch.push({
|
|
chart: 'mem.zswapio',
|
|
context: 'mem.zswapio',
|
|
ts,
|
|
values: { in: r('zswpin') * page, out: r('zswpout') * page },
|
|
})
|
|
batch.push({
|
|
chart: 'mem.numa',
|
|
context: 'mem.numa',
|
|
ts,
|
|
values: {
|
|
local: r('numa_hit') || r('numa_local'),
|
|
foreign: r('numa_foreign'),
|
|
interleave: r('numa_interleave'),
|
|
other: r('numa_other'),
|
|
pte_updates: r('numa_pte_updates'),
|
|
pages_migrated: r('numa_pages_migrated'),
|
|
},
|
|
})
|
|
batch.push({
|
|
chart: 'mem.thp_faults',
|
|
context: 'mem.thp_faults',
|
|
ts,
|
|
values: {
|
|
alloc: r('thp_fault_alloc'),
|
|
fallback: r('thp_fault_fallback'),
|
|
fallback_charge: r('thp_fault_fallback_charge'),
|
|
},
|
|
})
|
|
batch.push({
|
|
chart: 'mem.thp_split',
|
|
context: 'mem.thp_split',
|
|
ts,
|
|
values: {
|
|
split: r('thp_split_page') || r('thp_split'),
|
|
failed: r('thp_split_page_failed'),
|
|
split_pmd: r('thp_split_pmd'),
|
|
},
|
|
})
|
|
batch.push({
|
|
chart: 'mem.ksm_cow',
|
|
context: 'mem.ksm_cow',
|
|
ts,
|
|
values: {
|
|
swapin: r('ksm_swpin_copy') * page,
|
|
write: r('cow_ksm') * page,
|
|
},
|
|
})
|
|
}
|
|
return vm
|
|
}
|
|
|
|
/**
|
|
* @param {Array} batch
|
|
* @param {number} ts
|
|
* @param {number} dtSec
|
|
* @param {{ lastSoftnet?: object, lastTcpExt?: object, lastIcmp?: object }} state
|
|
* @param {{ tcp: object, icmp?: object }} snmp
|
|
*/
|
|
export function collectNetDeep(batch, ts, dtSec, state, snmp) {
|
|
const soft = parseSoftnet()
|
|
const prevS = state.lastSoftnet
|
|
batch.push({
|
|
chart: 'system.softnet_stat',
|
|
context: 'system.softnet_stat',
|
|
ts,
|
|
values: {
|
|
processed: prevS ? rate(prevS.processed, soft.processed, dtSec) : 0,
|
|
dropped: prevS ? rate(prevS.dropped, soft.dropped, dtSec) : 0,
|
|
squeezed: prevS ? rate(prevS.squeezed, soft.squeezed, dtSec) : 0,
|
|
received_rps: prevS ? rate(prevS.received_rps, soft.received_rps, dtSec) : 0,
|
|
flow_limit: prevS ? rate(prevS.flow_limit, soft.flow_limit, dtSec) : 0,
|
|
},
|
|
})
|
|
state.lastSoftnet = soft
|
|
|
|
const icmp = snmp.icmp || {}
|
|
const prevI = state.lastIcmp
|
|
if (icmp.InMsgs != null || icmp.OutMsgs != null) {
|
|
batch.push({
|
|
chart: 'ipv4.icmp',
|
|
context: 'ipv4.icmp',
|
|
ts,
|
|
values: {
|
|
received: prevI ? rate(prevI.InMsgs || 0, icmp.InMsgs || 0, dtSec) : 0,
|
|
sent: prevI ? rate(prevI.OutMsgs || 0, icmp.OutMsgs || 0, dtSec) : 0,
|
|
},
|
|
})
|
|
batch.push({
|
|
chart: 'ipv4.icmp_errors',
|
|
context: 'ipv4.icmp_errors',
|
|
ts,
|
|
values: {
|
|
InErrors: prevI ? rate(prevI.InErrors || 0, icmp.InErrors || 0, dtSec) : 0,
|
|
OutErrors: prevI ? rate(prevI.OutErrors || 0, icmp.OutErrors || 0, dtSec) : 0,
|
|
InCsumErrors: prevI ? rate(prevI.InCsumErrors || 0, icmp.InCsumErrors || 0, dtSec) : 0,
|
|
},
|
|
})
|
|
state.lastIcmp = icmp
|
|
}
|
|
|
|
const { TcpExt } = parseNetstatTables()
|
|
const prev = state.lastTcpExt
|
|
const tcp = snmp.tcp || {}
|
|
const r = (key) => (prev ? rate(prev[key] || 0, TcpExt[key] || 0, dtSec) : 0)
|
|
const rt = (key) => (prev ? rate(prev[`tcp_${key}`] || 0, tcp[key] || 0, dtSec) : 0)
|
|
|
|
batch.push({
|
|
chart: 'ip.tcpsyncookies',
|
|
context: 'ip.tcpsyncookies',
|
|
ts,
|
|
values: {
|
|
received: r('SyncookiesRecv'),
|
|
sent: r('SyncookiesSent'),
|
|
failed: r('SyncookiesFailed'),
|
|
},
|
|
})
|
|
batch.push({
|
|
chart: 'ip.tcp_syn_queue',
|
|
context: 'ip.tcp_syn_queue',
|
|
ts,
|
|
values: { drops: r('TCPReqQFullDrop'), cookies: r('TCPReqQFullDoCookies') },
|
|
})
|
|
batch.push({
|
|
chart: 'ip.tcp_accept_queue',
|
|
context: 'ip.tcp_accept_queue',
|
|
ts,
|
|
values: { overflows: r('ListenOverflows'), drops: r('ListenDrops') },
|
|
})
|
|
batch.push({
|
|
chart: 'ip.tcpconnaborts',
|
|
context: 'ip.tcpconnaborts',
|
|
ts,
|
|
values: {
|
|
baddata: r('TCPAbortOnData'),
|
|
userclosed: r('TCPAbortOnClose'),
|
|
nomemory: r('TCPAbortOnMemory'),
|
|
timeout: r('TCPAbortOnTimeout'),
|
|
linger: r('TCPAbortOnLinger'),
|
|
failed: r('TCPAbortFailed'),
|
|
},
|
|
})
|
|
batch.push({
|
|
chart: 'ip.tcpofo',
|
|
context: 'ip.tcpofo',
|
|
ts,
|
|
values: {
|
|
inqueue: r('TCPOFOQueue'),
|
|
dropped: r('TCPOFODrop'),
|
|
merged: r('TCPOFOMerge'),
|
|
pruned: r('OfoPruned'),
|
|
},
|
|
})
|
|
batch.push({
|
|
chart: 'ip.tcpreorders',
|
|
context: 'ip.tcpreorders',
|
|
ts,
|
|
values: {
|
|
timestamp: r('TCPTSReorder'),
|
|
sack: r('TCPSACKReorder'),
|
|
fack: r('TCPFACKReorder'),
|
|
reno: r('TCPRenoReorder'),
|
|
},
|
|
})
|
|
batch.push({
|
|
chart: 'ip.tcphandshake',
|
|
context: 'ip.tcphandshake',
|
|
ts,
|
|
values: {
|
|
EstabResets: rt('EstabResets'),
|
|
OutRsts: rt('OutRsts'),
|
|
AttemptFails: rt('AttemptFails'),
|
|
SynRetrans: r('TCPSynRetrans'),
|
|
},
|
|
})
|
|
batch.push({
|
|
chart: 'ip.tcpmemorypressures',
|
|
context: 'ip.tcpmemorypressures',
|
|
ts,
|
|
values: { pressures: r('TCPMemoryPressures') },
|
|
})
|
|
|
|
state.lastTcpExt = {
|
|
...TcpExt,
|
|
tcp_EstabResets: tcp.EstabResets || 0,
|
|
tcp_OutRsts: tcp.OutRsts || 0,
|
|
tcp_AttemptFails: tcp.AttemptFails || 0,
|
|
}
|
|
}
|
|
|
|
export function ifaceSpeedKbps(iface) {
|
|
const raw = readFile(`/sys/class/net/${iface}/speed`)
|
|
if (raw == null) return null
|
|
const mbps = Number(raw.trim())
|
|
if (!Number.isFinite(mbps) || mbps <= 0) return null
|
|
return mbps * 1000
|
|
}
|