In /proc/stat, guest is already inside user (and guest_nice inside nice). Peardata was adding guest into the total again, which shrinks every other slice — including idle.
CI / test (push) Successful in 1m0s
Release rolling / release (push) Failing after 1m50s

readFileCached capped at 64KB. On 56-core hosts /proc/interrupts (and similar) often exceeds that — reads were silently truncated. Buffer now grows (up to 1MB). That didn’t break system.cpu idle (aggregate line is first), but it was a real high-core regression.
This commit is contained in:
Raven Scott
2026-07-22 12:36:30 -04:00
parent 6570276179
commit da9541528e
5 changed files with 114 additions and 44 deletions
+43 -19
View File
@@ -161,6 +161,11 @@ function parseProcStat() {
return { aggregate, cores, intr, ctxt, processes, procsRunning, procsBlocked }
}
/**
* Convert /proc/stat jiffy counters to percentages.
* Linux accounts guest in user and guest_nice in nice — subtract before totaling
* (same approach as htop/netdata) so idle isn't understated on hypervisor hosts.
*/
function cpuDeltaPct(prev, cur) {
if (!prev || !cur) {
return {
@@ -176,25 +181,26 @@ function cpuDeltaPct(prev, cur) {
idle: 100,
}
}
const keys = [
'user',
'nice',
'system',
'idle',
'iowait',
'irq',
'softirq',
'steal',
'guest',
'guest_nice',
]
/** @type {Record<string, number>} */
const d = {}
let total = 0
for (const k of keys) {
d[k] = Math.max(0, (cur[k] || 0) - (prev[k] || 0))
total += d[k]
const delta = (k) => Math.max(0, (cur[k] || 0) - (prev[k] || 0))
const guest = delta('guest')
const guestNice = delta('guest_nice')
// guest ⊆ user, guest_nice ⊆ nice in kernel counters
const user = Math.max(0, delta('user') - guest)
const nice = Math.max(0, delta('nice') - guestNice)
const d = {
guest_nice: guestNice,
guest,
steal: delta('steal'),
softirq: delta('softirq'),
irq: delta('irq'),
user,
system: delta('system'),
nice,
iowait: delta('iowait'),
idle: delta('idle'),
}
let total = 0
for (const v of Object.values(d)) total += v
if (total <= 0) total = 1
return {
guest_nice: pct(d.guest_nice, total),
@@ -423,6 +429,8 @@ export class MetricsCollector extends EventEmitter {
/** @type {Map<string, Record<string, number|null>>} */
this.latest = new Map()
this._chartsRegistered = false
this._lastCpuUsage = process.cpuUsage()
this._lastCpuTs = Date.now()
}
start() {
@@ -509,7 +517,20 @@ export class MetricsCollector extends EventEmitter {
log.error('Collector tick failed', { error: err.message })
}
const dur = performance.now() - t0
if (dur > 15) log.debug('tick', { collector: 'main', durMs: Math.round(dur * 10) / 10 })
if (dur > 15) {
const now = Date.now()
const cpuDelta = process.cpuUsage(this._lastCpuUsage)
const wallMs = now - this._lastCpuTs
this._lastCpuUsage = process.cpuUsage()
this._lastCpuTs = now
const totalMs = (cpuDelta.user + cpuDelta.system) / 1000
const pct = wallMs > 0 ? Math.round((totalMs / wallMs) * 100) : 0
if (pct > 30) {
log.debug('tick', { collector: 'main', durMs: Math.round(dur * 10) / 10, cpuPct: pct, cpuUserMs: Math.round(cpuDelta.user / 1000), cpuSysMs: Math.round(cpuDelta.system / 1000), wallMs })
} else {
log.debug('tick', { collector: 'main', durMs: Math.round(dur * 10) / 10 })
}
}
}
_safeCollect(name, fn) {
@@ -1290,3 +1311,6 @@ export function getCollector() {
if (!singleton) singleton = new MetricsCollector()
return singleton
}
/** @internal exported for unit tests */
export { cpuDeltaPct }
+4 -18
View File
@@ -49,20 +49,6 @@ function readFile(p) {
return readFileBuf(p)
}
let _profileIdx = 0
function profile(label, fn) {
const t0 = performance.now()
const r = fn()
const dur = performance.now() - t0
if (dur > 2) {
_profileIdx++
if (_profileIdx <= 40) {
process.stderr.write(`cgroups:${label} ${Math.round(dur * 10) / 10}ms\n`)
}
}
return r
}
function cgroupRoot() {
if (fs.existsSync('/sys/fs/cgroup/cgroup.controllers')) return '/sys/fs/cgroup'
return null
@@ -294,18 +280,18 @@ export class CgroupsCollector extends EventEmitter {
const memDetailDef = makeCgroupMemDetailChart(cg.id, title)
const throttleDef = makeCgroupThrottleChart(cg.id, title)
const cpu = profile('cpuStat:' + cg.id.slice(0,20), () => parseCpuStat(cg.path))
const cpu = parseCpuStat(cg.path)
let usageUsec = cpu?.usage_usec ?? 0
let userUsec = cpu?.user_usec ?? 0
let systemUsec = cpu?.system_usec ?? 0
const throttledUsec = cpu?.throttled_usec ?? 0
const memCur = profile('memCur:' + cg.id.slice(0,20), () => Number(readFile(path.join(cg.path, 'memory.current')) || 0))
const memCur = Number(readFile(path.join(cg.path, 'memory.current')) || 0)
const memMaxRaw = readFile(path.join(cg.path, 'memory.max'))
const memMax =
memMaxRaw && memMaxRaw.trim() !== 'max' ? Number(memMaxRaw.trim()) || 0 : 0
const io = profile('ioStat:' + cg.id.slice(0,20), () => parseIoStat(cg.path))
const io = parseIoStat(cg.path)
const prev = this.prev.get(cg.id)
let userPct = 0
@@ -347,7 +333,7 @@ export class CgroupsCollector extends EventEmitter {
},
})
const memStat = profile('memStat:' + cg.id.slice(0,20), () => parseMemoryStat(cg.path))
const memStat = parseMemoryStat(cg.path)
if (memStat) {
batch.push({
chart: memDetailDef.id,
+5 -4
View File
@@ -219,7 +219,7 @@ export function collectCpuidle(batch, ts, dtSec, state) {
} catch {
return
}
/** @type {Record<string, Array<{ nameFile: string, timeFile: string }>>} */
/** @type {Record<string, Array<{ id: string, nameFile: string, timeFile: string }>>} */
const entries = {}
for (const d of dirs) {
const coreId = d.slice(3)
@@ -233,6 +233,7 @@ export function collectCpuidle(batch, ts, dtSec, state) {
const files = []
for (const s of states) {
files.push({
id: s,
nameFile: path.join(idleDir, s, 'name'),
timeFile: path.join(idleDir, s, 'time'),
})
@@ -251,10 +252,10 @@ export function collectCpuidle(batch, ts, dtSec, state) {
const files = entries[coreId]
/** @type {Record<string, number>} */
const times = {}
for (const { nameFile, timeFile } of files) {
const name = (readFile(nameFile) || '').trim().toLowerCase().replace(/\s+/g, '_')
for (const { id, nameFile, timeFile } of files) {
const name = (readFile(nameFile) || id).trim().toLowerCase().replace(/\s+/g, '_')
const t = Number(readFile(timeFile) || 0) // µs
times[name || 'state'] = t
times[name || id] = t
}
nextAll[coreId] = times
const prev = prevAll[coreId]