updates
CI / test (push) Successful in 51s
Release rolling / release (push) Has been cancelled

This commit is contained in:
Raven Scott
2026-07-18 19:04:39 -04:00
parent b56d6b4c0e
commit 9bc7a67020
22 changed files with 1704 additions and 190 deletions
+4
View File
@@ -25,6 +25,10 @@ PEARDATA_DEFAULT_ROLE=viewer
# ── HyperDB (warm history + metadata + linked sync) ──────────
# PEARDATA_HYPERDB=1
# PEARDATA_SWARM=0
# PEARDATA_DOCKER=0
# PEARDATA_DOCKER_SOCKET=/var/run/docker.sock
# PEARDATA_PROCESSES=0
# PEARDATA_PROCESSES_TOP=8
# See docs/STORAGE-HYPERDB.md
# ── agent-style REST API ───────────────────────────────────
+288 -61
View File
@@ -4,16 +4,28 @@
import { manager } from './client/manager.js'
import { Methods, Pushes } from './shared/protocol.js'
import { getClientIdentity } from './client/identity.js'
import {
loadBookmarks,
upsertBookmark,
removeBookmark,
setBookmarkAlias,
} from './client/bookmarks.js'
import { classifyConnectionInput } from './shared/crypto-auth.js'
const $ = (id) => document.getElementById(id)
const els = {
connectInput: $('connect-input'),
peerAlias: $('peer-alias'),
adminSeed: $('admin-seed'),
btnConnect: $('btn-connect'),
btnDisconnect: $('btn-disconnect'),
btnInvite: $('btn-invite'),
peerList: $('peer-list'),
bookmarkList: $('bookmark-list'),
compareToggle: $('compare-toggle'),
exploreChart: $('explore-chart'),
chartCpuLabel: $('chart-cpu-label'),
serverInfo: $('server-info'),
log: $('log'),
status: $('status-chip'),
@@ -21,6 +33,7 @@ const els = {
roleBadge: $('role-badge'),
inviteOut: $('invite-out'),
anomalyList: $('anomaly-list'),
offlineBanner: $('offline-banner'),
statCpu: $('stat-cpu'),
statRam: $('stat-ram'),
statLoad: $('stat-load'),
@@ -33,22 +46,41 @@ const series = {
cpu: [],
ram: [],
net: [],
io: [],
explore: [],
}
/** Per-peer CPU series for compare mode @type {Map<string, number[]>} */
const peerCpu = new Map()
const SERIES_MAX = 60
const COMPARE_COLORS = ['#5b8cff', '#3dd6c6', '#f0b429', '#ff6b7a', '#c084fc']
let exploreChartId = 'system.io'
function log(line) {
const ts = new Date().toLocaleTimeString()
els.log.textContent = `[${ts}] ${line}\n` + els.log.textContent
}
function setOnline(online) {
function setOnline(online, opts = {}) {
els.status.textContent = online ? 'live' : 'offline'
els.status.classList.toggle('online', online)
els.status.classList.toggle('offline', !online)
els.btnDisconnect.disabled = !online
els.btnInvite.disabled = !online
els.btnConnect.disabled = online
// Allow adding more peers while online
els.btnConnect.disabled = false
const hadSamples =
series.cpu.length > 0 || series.ram.length > 0 || series.net.length > 0
const showBanner = !online && (hadSamples || opts.reconnecting)
if (els.offlineBanner) {
els.offlineBanner.classList.toggle('hidden', !showBanner)
if (showBanner) {
els.offlineBanner.textContent = opts.reconnecting
? 'Agent offline — showing last-known samples. Reconnecting…'
: 'Agent offline — showing last-known samples.'
}
}
document.body.classList.toggle('is-offline', showBanner)
}
function escapeHtml(s) {
@@ -61,11 +93,31 @@ function escapeHtml(s) {
function pushPoint(key, value) {
const arr = series[key]
if (!arr) return
arr.push(Number(value) || 0)
while (arr.length > SERIES_MAX) arr.shift()
}
function pushPeerCpu(peerId, value) {
const id = String(peerId).toLowerCase()
let arr = peerCpu.get(id)
if (!arr) {
arr = []
peerCpu.set(id, arr)
}
arr.push(Number(value) || 0)
while (arr.length > SERIES_MAX) arr.shift()
}
function drawChart(canvasId, values, color = '#5b8cff') {
drawMultiChart(canvasId, [{ values, color }])
}
/**
* @param {string} canvasId
* @param {Array<{ values: number[], color: string, label?: string }>} lines
*/
function drawMultiChart(canvasId, lines) {
const canvas = $(canvasId)
if (!canvas) return
const ctx = canvas.getContext('2d')
@@ -87,12 +139,16 @@ function drawChart(canvasId, values, color = '#5b8cff') {
ctx.stroke()
}
if (values.length < 2) return
const max = Math.max(...values, 1)
const min = Math.min(...values, 0)
const all = lines.flatMap((l) => l.values)
if (all.length < 2) return
const max = Math.max(...all, 1)
const min = Math.min(...all, 0)
const span = max - min || 1
ctx.strokeStyle = color
lines.forEach((line, li) => {
const values = line.values
if (values.length < 2) return
ctx.strokeStyle = line.color
ctx.lineWidth = 2
ctx.beginPath()
values.forEach((v, i) => {
@@ -102,13 +158,74 @@ function drawChart(canvasId, values, color = '#5b8cff') {
else ctx.lineTo(x, y)
})
ctx.stroke()
// fill
if (li === 0 && lines.length === 1) {
ctx.lineTo(w, h)
ctx.lineTo(0, h)
ctx.closePath()
ctx.fillStyle = color + '22'
ctx.fillStyle = line.color + '22'
ctx.fill()
}
})
}
function redrawCpu() {
if (els.compareToggle.checked && peerCpu.size > 0) {
const lines = [...peerCpu.entries()].slice(0, 5).map(([id, values], i) => ({
values,
color: COMPARE_COLORS[i % COMPARE_COLORS.length],
label: id.slice(0, 8),
}))
drawMultiChart('chart-cpu', lines)
els.chartCpuLabel.textContent = `compare · ${lines.length} peers`
} else {
drawChart('chart-cpu', series.cpu, '#5b8cff')
els.chartCpuLabel.textContent = 'system.cpu'
}
}
function redrawAll() {
redrawCpu()
drawChart('chart-ram', series.ram, '#3dd6c6')
drawChart('chart-net', series.net, '#f0b429')
drawChart('chart-explore', series.explore, '#ff6b7a')
}
function renderBookmarks() {
const list = loadBookmarks()
els.bookmarkList.innerHTML = ''
if (!list.length) {
const li = document.createElement('li')
li.className = 'muted'
li.textContent = 'No saved agents'
els.bookmarkList.appendChild(li)
return
}
for (const b of list) {
const li = document.createElement('li')
const label = b.alias || `${b.publicKeyHex.slice(0, 12)}`
li.innerHTML = `<span title="${escapeHtml(b.publicKeyHex)}">${escapeHtml(label)}</span>
<span class="bookmark-actions">
<button type="button" class="linkish" data-act="connect">↗</button>
<button type="button" class="linkish" data-act="forget">×</button>
</span>`
li.querySelector('[data-act="connect"]').addEventListener('click', (e) => {
e.stopPropagation()
els.connectInput.value = b.invite || b.publicKeyHex
if (b.alias) els.peerAlias.value = b.alias
els.btnConnect.click()
})
li.querySelector('[data-act="forget"]').addEventListener('click', (e) => {
e.stopPropagation()
removeBookmark(b.publicKeyHex)
renderBookmarks()
log(`Forgot ${label}`)
})
li.addEventListener('click', () => {
els.connectInput.value = b.invite || b.publicKeyHex
if (b.alias) els.peerAlias.value = b.alias
})
els.bookmarkList.appendChild(li)
}
}
function renderPeers() {
@@ -121,27 +238,64 @@ function renderPeers() {
els.peerList.appendChild(li)
return
}
const bookmarks = loadBookmarks()
for (const p of list) {
const li = document.createElement('li')
const id = p.publicKeyHex || p.id
const bm = bookmarks.find((b) => b.publicKeyHex === id)
const active = manager.active?.publicKeyHex === id
const li = document.createElement('li')
li.className = active ? 'active' : ''
li.innerHTML = `<span>${escapeHtml(String(id).slice(0, 12))}</span><span class="muted">${p.connected ? 'live' : '…'}</span>`
const label = bm?.alias || `${String(id).slice(0, 12)}`
li.innerHTML = `<span>${escapeHtml(label)}</span><span class="muted">${p.connected ? 'live' : '…'}</span>`
li.addEventListener('click', () => {
manager.setActive(id)
renderPeers()
refreshMeta().catch(() => {})
})
li.addEventListener('dblclick', () => {
const alias = prompt('Alias for this agent', bm?.alias || '')
if (alias != null) {
setBookmarkAlias(id, alias)
renderBookmarks()
renderPeers()
}
})
els.peerList.appendChild(li)
}
}
function onSamples(samples) {
function exploreValue(chart, values) {
if (!values) return 0
if (chart === 'system.io' || chart.startsWith('disk_io.')) {
return (values.reads || values.in || 0) + (values.writes || values.out || 0)
}
if (chart.startsWith('cpu.cpu') || chart === 'system.cpu') {
return 100 - (values.idle ?? 100)
}
if (chart.startsWith('net.') || chart === 'system.net') {
return values.received ?? 0
}
if (chart.startsWith('disk_util.')) return values.utilization ?? 0
if (chart.startsWith('disk_space.')) return values.used ?? 0
const first = Object.values(values).find((v) => typeof v === 'number')
return first ?? 0
}
function onSamples(samples, conn) {
const peerId = conn?.publicKeyHex || manager.active?.publicKeyHex || 'active'
const isActive = !manager.active || manager.active.publicKeyHex === peerId
for (const s of samples || []) {
if (s.chart === 'system.cpu') {
const used = 100 - (s.values.idle ?? 100)
pushPeerCpu(peerId, used)
if (isActive) {
pushPoint('cpu', used)
els.statCpu.textContent = `${used.toFixed(1)}%`
}
}
if (!isActive) continue
if (s.chart === 'system.ram') {
const used = s.values.used ?? 0
pushPoint('ram', used)
@@ -155,14 +309,11 @@ function onSamples(samples) {
pushPoint('net', rx)
els.statNet.textContent = `${rx.toFixed(1)} kb/s`
}
if (s.chart === 'system.io') {
pushPoint('io', (s.values.reads || 0) + (s.values.writes || 0))
if (s.chart === exploreChartId) {
pushPoint('explore', exploreValue(exploreChartId, s.values))
}
}
drawChart('chart-cpu', series.cpu, '#5b8cff')
drawChart('chart-ram', series.ram, '#3dd6c6')
drawChart('chart-net', series.net, '#f0b429')
drawChart('chart-io', series.io, '#ff6b7a')
redrawAll()
}
function prependAnomaly(ev) {
@@ -188,61 +339,118 @@ async function refreshMeta() {
const id = getClientIdentity()
els.connMeta.textContent = `you ${id.publicKeyHex.slice(0, 12)}… · ${auth.role} · ${auth.authMode}`
renderPeers()
await populateExploreCharts()
}
async function populateExploreCharts() {
try {
const res = await manager.request(Methods.listCharts, {})
const charts = res.charts || {}
const ids = Object.keys(charts).sort()
const prefer = ids.filter(
(id) =>
id.startsWith('cpu.cpu') ||
id.startsWith('disk_io.') ||
id.startsWith('disk_ops.') ||
id.startsWith('net.') ||
id.startsWith('disk_space.') ||
id.startsWith('docker.') ||
id.startsWith('processes.') ||
id === 'system.io' ||
id === 'mem.available' ||
id === 'system.load'
)
const options = prefer.length ? prefer : ids.slice(0, 40)
const prev = exploreChartId
els.exploreChart.innerHTML = ''
for (const id of options) {
const opt = document.createElement('option')
opt.value = id
opt.textContent = id
els.exploreChart.appendChild(opt)
}
if (options.includes(prev)) {
els.exploreChart.value = prev
exploreChartId = prev
} else if (options.length) {
exploreChartId = options[0]
els.exploreChart.value = exploreChartId
}
} catch {
// ignore
}
}
async function seedHistory() {
const charts = [
['system.cpu', 'cpu'],
['system.ram', 'ram'],
['system.net', 'net'],
[exploreChartId, 'explore'],
]
for (const [chart, key] of charts) {
try {
const q = await manager.request(Methods.queryData, { chart, after: -60, points: 60 })
series[key] = []
for (const row of q.data || []) {
if (chart === 'system.cpu') {
const idleIdx = q.labels.indexOf('idle')
const idle = idleIdx >= 0 ? row[idleIdx] : 100
pushPoint('cpu', 100 - (idle ?? 100))
} else {
// first numeric dim after time
pushPoint(key, row[1] ?? 0)
}
}
if (q.source === 'hyperdb-warm') log(`History ${chart} from HyperDB warm`)
} catch {
// ignore
}
}
redrawAll()
}
els.btnConnect.addEventListener('click', async () => {
const raw = els.connectInput.value.trim()
const adminSeed = els.adminSeed.value.trim() || null
const alias = els.peerAlias.value.trim()
if (!raw) {
log('Enter a public key or pd1 invite')
return
}
els.btnConnect.disabled = true
try {
log(`Dialing…`)
log('Dialing…')
const parsed = classifyConnectionInput(raw)
const conn = await manager.connect(raw, { adminSeed })
log(`Connected ${conn.publicKeyHex.slice(0, 16)}`)
setOnline(true)
upsertBookmark({
publicKeyHex: conn.publicKeyHex,
alias,
invite: parsed.kind === 'invite' ? raw : null,
})
renderBookmarks()
await manager.request(Methods.subscribeMetrics, { charts: ['*'], intervalMs: 1000 })
await manager.request(Methods.subscribeAnomalies, {})
await refreshMeta()
// seed charts from history
for (const chart of ['system.cpu', 'system.ram', 'system.net', 'system.io']) {
try {
const q = await manager.request(Methods.queryData, { chart, after: -60, points: 60 })
const dimIdx = chart === 'system.cpu' ? q.labels.indexOf('user') : 1
for (const row of q.data || []) {
if (chart === 'system.cpu') {
const idleIdx = q.labels.indexOf('idle')
const idle = idleIdx >= 0 ? row[idleIdx] : 100
pushPoint('cpu', 100 - (idle ?? 100))
} else if (dimIdx >= 0) {
const key = chart === 'system.ram' ? 'ram' : chart === 'system.net' ? 'net' : 'io'
pushPoint(key, row[dimIdx] ?? 0)
}
}
} catch {
// ignore
}
}
drawChart('chart-cpu', series.cpu, '#5b8cff')
drawChart('chart-ram', series.ram, '#3dd6c6')
drawChart('chart-net', series.net, '#f0b429')
drawChart('chart-io', series.io, '#ff6b7a')
await seedHistory()
log('Subscribed to live metrics')
} catch (err) {
log(`Connect failed: ${err.message}`)
setOnline(false)
if (!manager.list().some((c) => c.connected)) setOnline(false)
} finally {
els.btnConnect.disabled = false
}
})
els.btnDisconnect.addEventListener('click', async () => {
await manager.disconnect()
setOnline(false)
const active = manager.active?.publicKeyHex
if (active) await manager.disconnect(active)
else await manager.disconnect()
if (!manager.list().some((c) => c.connected)) setOnline(false)
renderPeers()
log('Disconnected')
log(active ? `Disconnected ${active.slice(0, 12)}` : 'Disconnected')
})
els.btnInvite.addEventListener('click', async () => {
@@ -256,8 +464,22 @@ els.btnInvite.addEventListener('click', async () => {
}
})
manager.on('push', (ev) => {
if (ev.type === Pushes.metrics) onSamples(ev.data?.samples)
els.compareToggle.addEventListener('change', () => {
redrawCpu()
})
els.exploreChart.addEventListener('change', async () => {
exploreChartId = els.exploreChart.value
series.explore = []
try {
await seedHistory()
} catch {
redrawAll()
}
})
manager.on('push', (ev, conn) => {
if (ev.type === Pushes.metrics) onSamples(ev.data?.samples, conn)
if (ev.type === Pushes.anomaly) {
prependAnomaly(ev.data)
log(`Anomaly: ${ev.data?.message || ''}`)
@@ -273,19 +495,24 @@ manager.on('connected', () => {
})
manager.on('disconnected', () => {
if (!manager.active?.connected) setOnline(false)
if (!manager.list().some((c) => c.connected)) setOnline(false, { reconnecting: true })
renderPeers()
log('Agent disconnected')
})
setOnline(false)
renderPeers()
log('PearData ready — connect an agent public key')
// redraw on resize
window.addEventListener('resize', () => {
drawChart('chart-cpu', series.cpu, '#5b8cff')
drawChart('chart-ram', series.ram, '#3dd6c6')
drawChart('chart-net', series.net, '#f0b429')
drawChart('chart-io', series.io, '#ff6b7a')
manager.on('reconnect-failed', ({ tries }) => {
setOnline(false, { reconnecting: true })
log(`Reconnect attempt failed (try ${tries})`)
})
manager.on('reconnect-exhausted', ({ publicKeyHex }) => {
setOnline(false, { reconnecting: false })
log(`Reconnect exhausted for ${String(publicKeyHex).slice(0, 12)}`)
})
setOnline(false)
renderBookmarks()
renderPeers()
log('PearData ready — connect an agent or pick a saved peer')
window.addEventListener('resize', () => redrawAll())
+137
View File
@@ -0,0 +1,137 @@
/**
* Persistent peer bookmarks (pubkey / invite / alias).
* Stored under Pear.config.storage or ~/.config/peardata/bookmarks.json.
*/
import fs from 'fs'
import path from 'path'
import os from 'os'
const FILE_NAME = 'bookmarks.json'
const VERSION = 1
function storageRoot() {
try {
const pear = globalThis.Pear?.config?.storage || globalThis.Pear?.app?.storage
if (pear) return String(pear)
} catch {
// ignore
}
try {
const home =
(typeof process !== 'undefined' && (process.env?.PEARDATA_HOME || process.env?.HOME)) ||
(typeof os.homedir === 'function' ? os.homedir() : '') ||
''
return path.join(home, '.config', 'peardata')
} catch {
return ''
}
}
export function bookmarksPath() {
const root = storageRoot()
return root ? path.join(root, FILE_NAME) : ''
}
/**
* @typedef {{
* id: string,
* publicKeyHex: string,
* alias?: string,
* invite?: string|null,
* lastConnectedAt?: number|null,
* createdAt: number,
* }} Bookmark
*/
/**
* @returns {Bookmark[]}
*/
export function loadBookmarks() {
const file = bookmarksPath()
if (!file) return []
try {
if (!fs.existsSync(file)) return []
const raw = JSON.parse(fs.readFileSync(file, 'utf8'))
const list = Array.isArray(raw?.bookmarks) ? raw.bookmarks : []
return list
.filter((b) => b && /^[0-9a-f]{64}$/i.test(String(b.publicKeyHex || '')))
.map((b) => ({
id: String(b.id || b.publicKeyHex).toLowerCase(),
publicKeyHex: String(b.publicKeyHex).toLowerCase(),
alias: b.alias ? String(b.alias).slice(0, 64) : '',
invite: b.invite || null,
lastConnectedAt: b.lastConnectedAt || null,
createdAt: b.createdAt || Date.now(),
}))
} catch {
return []
}
}
/**
* @param {Bookmark[]} bookmarks
*/
export function saveBookmarks(bookmarks) {
const file = bookmarksPath()
if (!file) return false
try {
const dir = path.dirname(file)
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true, mode: 0o700 })
fs.writeFileSync(
file,
JSON.stringify({ version: VERSION, bookmarks, updatedAt: new Date().toISOString() }, null, 2),
{ mode: 0o600 }
)
return true
} catch {
return false
}
}
/**
* Upsert a bookmark after a successful connect.
* @param {{ publicKeyHex: string, alias?: string, invite?: string|null }} peer
*/
export function upsertBookmark(peer) {
const publicKeyHex = String(peer.publicKeyHex || '').toLowerCase()
if (!/^[0-9a-f]{64}$/.test(publicKeyHex)) return loadBookmarks()
const list = loadBookmarks()
const idx = list.findIndex((b) => b.publicKeyHex === publicKeyHex)
const next = {
id: publicKeyHex,
publicKeyHex,
alias: peer.alias != null ? String(peer.alias).slice(0, 64) : list[idx]?.alias || '',
invite: peer.invite || list[idx]?.invite || null,
lastConnectedAt: Date.now(),
createdAt: list[idx]?.createdAt || Date.now(),
}
if (idx >= 0) list[idx] = next
else list.unshift(next)
// keep newest 50
saveBookmarks(list.slice(0, 50))
return list
}
/**
* @param {string} publicKeyHex
* @param {string} alias
*/
export function setBookmarkAlias(publicKeyHex, alias) {
const id = String(publicKeyHex || '').toLowerCase()
const list = loadBookmarks()
const b = list.find((x) => x.publicKeyHex === id)
if (!b) return list
b.alias = String(alias || '').slice(0, 64)
saveBookmarks(list)
return list
}
/**
* @param {string} publicKeyHex
*/
export function removeBookmark(publicKeyHex) {
const id = String(publicKeyHex || '').toLowerCase()
const list = loadBookmarks().filter((b) => b.publicKeyHex !== id)
saveBookmarks(list)
return list
}
+6 -1
View File
@@ -36,7 +36,8 @@ Treat `SERVER_SEED` like a root password. Prefer `pd1.` invites for operators.
| Variable | Default | Description |
|----------|---------|-------------|
| `PEARDATA_DATA_DIR` | `./data` | Peer policy JSON + `audit.log` |
| `PEARDATA_HOME` | OS home | Root for client identity (`~/.config/peardata/identity.json`) |
| `PEARDATA_HOME` | OS home | Root for client identity + bookmarks (`~/.config/peardata/`) |
| `PEARDATA_STORAGE` | — | Electron/Pear storage dir override (bookmarks prefer `Pear.config.storage`) |
| `PEARDATA_RATE_LIMIT_RPM` | `120` | Per-peer RPC requests per minute |
| `PEARDATA_MAX_RECONNECT` | `20` | Client manager reconnect attempts per peer |
@@ -67,6 +68,10 @@ Permissions: directory `0700`. Do **not** commit `data/` or `.env`.
|----------|---------|-------------|
| `PEARDATA_HYPERDB` | on | `0` / `off` disables HyperDB |
| `PEARDATA_SWARM` | off | `1` enables Hyperswarm Corestore replication |
| `PEARDATA_DOCKER` | off | `1` enables container collector (cgroup + optional Docker socket) |
| `PEARDATA_DOCKER_SOCKET` | `/var/run/docker.sock` | Unix socket for container name enrichment |
| `PEARDATA_PROCESSES` | off | `1` enables process top-N CPU/RSS charts (Linux `/proc`) |
| `PEARDATA_PROCESSES_TOP` | `8` | How many processes to keep in top charts (max 32) |
Storage: `$PEARDATA_DATA_DIR/corestore` (named core `peardata-meta`).
+15 -3
View File
@@ -56,13 +56,25 @@ Operators run via `runJob` or future REST function execute.
```
server/services/collectors/
system.js # default
docker.js # Phase 3
peardock.js # bridge
docker.js # PEARDATA_DOCKER=1 — cgroup + optional Docker socket
peardock.js # bridge (planned)
```
Have `pipeline.js` start each enabled collector; all emit `samples` batches into the same store.
### Docker collector (shipped spike)
1. Set `PEARDATA_DOCKER=1` on the agent.
2. Charts: `docker.containers`, `docker.cpu.<id>`, `docker.mem.<id>` (registered at runtime).
3. Discovery: cgroup v2 `docker-*.scope` / `libpod-*.scope`; names from `PEARDATA_DOCKER_SOCKET` when present.
4. Pick them in the desktop Explore chart dropdown (`docker.*`).
### Process top-N collector
1. Set `PEARDATA_PROCESSES=1` (optional `PEARDATA_PROCESSES_TOP=8`).
2. Charts: `processes.top_cpu`, `processes.top_rss` (dimensions = process comm names).
3. Linux `/proc` only; safe no-op on other platforms.
## Parent peer (fleet aggregator)
1. Parent dials child agents with `ConnectionManager`.
+104 -65
View File
@@ -8,88 +8,113 @@ Phased plan from MVP agent → real-time fleet observability on pure P2P.
2. **Agent efficiency** — stay in a tight ballpark for CPU/RAM overhead.
3. **PearDock patterns** — HyperDHT identity, protomux-rpc, roles, `pd1.` invites.
4. **Dual access** — P2P desktop + agent-style REST (`/api/v1|v2|v3`).
5. **Ecosystem glue** — ready for PearDock / PearVirt / HoneyPeer / BareOS later.
5. **Bare/Pear-ready** — no Node builtins at runtime; import maps → `bare-*`.
6. **Ecosystem glue** — ready for PearDock / PearVirt / HoneyPeer / BareOS later.
---
## Phase 0 — Foundation ✅ (this repo)
## Phase 0 — Foundation ✅
Template rebrand, protocol, collector, memory store, anomalies, REST, desktop MVP, docs.
---
## Phase 1 — Hardening, packaging & UX ✅
| Item | Status |
|------|--------|
| Copy pear-app-template → PearData rebrand (`pd1.`, `PEARDATA_*`, `peardata/rpc`) | Done |
| Shared protocol + schema + metrics catalog | Done |
| Agent collector (full system/OS charts + per-cpu/disk/iface/mount) @ ~1s | Done |
| In-memory tiered store (1s + downsample) | Done |
| Threshold anomaly engine + health | Done |
| P2P RPC surface (query, subscribe, alerts, jobs, ACL) | Done |
| agent-compatible REST v1/v2/v3 (local :19999) | Done |
| Pear desktop fleet overview + live canvas charts | Done |
| Docs: architecture, protocol, data model, REST, roadmap, tech choices | Done |
| systemd unit + CI skeletons | Done |
| Bare import maps + `bare-node-runtime` bootstrap | Done |
| Full system chart catalog + instance collectors | Done |
| One-line installer (`scripts/install.sh`) | Done |
| Bare Linux server + Electron clients (all arches) + CI | Done |
| brittle tests (protocol, crypto, store, REST, collector, HyperDB) | Done |
| Persistent peer bookmarks + aliases | Done |
| Multi-peer CPU compare overlay | Done |
| Instance / explore chart picker (`listCharts`) | Done |
| Alert silence TTL auto-reenable | Done |
| Reconnection (manager auto-reconnect) | Done |
**MVP acceptance criteria**
- [x] Agent exposes metrics over P2P by public key
- [x] Desktop connects to one+ agents and shows live CPU/RAM/net/disk
- [x] REST `/api/v3/data?chart=system.cpu&after=-60` returns series
- [x] Viewer vs admin (pubkey / seed / `pd1.` invite)
- [x] Simple threshold anomalies pushed to clients
**Phase 1 exit** — all criteria met (install path, dual archives, bookmarks, compare).
---
## Phase 1Hardening & UX (next)
## Phase 2Storage & history depth ✅ / 🔄
- Persistent client peer bookmarks + aliases in desktop
- Multi-peer compare mode (overlay 24 nodes on one chart)
- Reconnection / offline banners with last-known samples
- Agent process title / `peardata-agent` binary naming polish
- Alert silence TTL auto-reenable
- brittle tests for collector, store query, REST routes
- One-line install script (`curl | bash`) for Linux agents
| Item | Status |
|------|--------|
| HyperDB bee + Corestore (local warm) | Done |
| Hot path stays memory (no 1s → HyperDB) | Done |
| RPC peer-link + `getDbInfo` / REST `/api/v3/db` | Done |
| Query fallback: memory miss / long window → HyperDB warm | Done |
| Hyperswarm mesh (`PEARDATA_SWARM=1`) | Done (opt-in) |
| Configurable retention knobs (tier sizes) | Done (`PEARDATA_TIER*`) |
| Documented warm history across restart (operator M4) | Done (`store-hyperdb-fallback` + STORAGE doc) |
| Export snapshot job → JSON / Prometheus remote-write | Next |
| Autobase multi-writer parents | Later (Phase 2c) |
| Rocks engine for local-only desktop cache | Optional |
**Phase 2 exit criteria**
- [x] `queryData` / REST `/data` can return `source: "hyperdb-warm"`
- [x] Soak test: empty memory + reopen Corestore → `hyperdb-warm`
- [ ] Linked peer pulls warm points over swarm without re-scraping
---
## Phase 2Storage & history (HyperDB)
## Phase 3Discovery & app collectors ← **in progress**
See **[STORAGE-HYPERDB.md](./STORAGE-HYPERDB.md)** for the full design (from Holepunch `hyperdb` / workshop / Autobase patterns).
| Item | Status |
|------|--------|
| Docker / container discovery | Done (opt-in `PEARDATA_DOCKER=1`) |
| Offline banner (last-known samples) | Done |
| Process top-N | Done (opt-in `PEARDATA_PROCESSES=1`) |
| Service plugins | nginx, postgres, redis collectors |
| PearDock bridge | Container metrics from dock peers |
| PearVirt / BareOS adapters | Thin translators into shared contexts |
| Holesail optional REST expose | Tunneled agent API |
- HyperDB (bee + Corestore) for metadata, alerts, peer-links, **warm** downsampled points
- Keep memory ring for hot 1s path; do **not** tx every sample into HyperDB
- Hyperswarm `store.replicate` for linked-node / desktop seed sync
- Configurable retention (hours@1s memory, days@1m HyperDB, weeks@1h)
- Historical query: memory miss → HyperDB range
- Optional later: Autobase multi-writer parents; Rocks engine for local-only speed
- Export snapshot job → JSON / Prometheus remote write (optional)
**Phase 3 exit criteria**
---
## Phase 3 — Discovery & apps
- Docker / container auto-discovery (cgroup + Docker API)
- Common service collectors (nginx, postgres, redis) as plugins
- PearDock integration: container metrics from dock peers
- PearVirt / BareOS node metric adapters
- Holesail optional expose of REST UI per agent
- [x] One container host can emit per-container CPU/mem charts (`docker.cpu.*` / `docker.mem.*`)
- [x] Plugin docs + Docker collector in [EXTENDING.md](./EXTENDING.md)
- [x] Process top-N opt-in collector (`processes.top_cpu` / `processes.top_rss`)
---
## Phase 4 — Smarter anomalies & fleet
- Anomaly scoring + chart highlighting in UI
- Lightweight ML job (`runJob` retrain) — start with streaming z-score / k-means
- Fleet-wide composite views and correlation (`/api/v3/weights` depth)
- Parent peer aggregation (P2P “parent” without central SaaS)
- Push notifications (desktop + optional webhook)
| Item | Notes |
|------|--------|
| Anomaly scoring + UI highlight | Beyond binary warn/crit |
| Streaming z-score / k-means job | `runJob` retrain stub exists |
| Fleet composite views | Multi-node health strip |
| `/api/v3/weights` depth | Real metric weights |
| Parent peer aggregation | P2P parent without SaaS |
| Notifications | Desktop + optional webhook |
---
## Phase 5 — Product polish
- Signed release artifacts (agent + Pear desktop)
- Role templates (viewer / SRE operator / admin)
- Plugin SDK documentation
- Grafana datasource (REST) cookbook
- HoneyPeer presence for agent directory (opt-in)
| Item | Notes |
|------|--------|
| Signed / notarized macOS clients | Beyond ad-hoc `rcodesign` |
| Role templates | viewer / SRE / admin presets |
| Plugin SDK | Collector + chart registration API |
| Grafana cookbook | REST + Prometheus recipes |
| HoneyPeer directory | Opt-in agent presence |
| Windows collector depth | Close `/proc`-only gaps |
---
## Suggested near-term sequence
1. ~~HyperDB soak~~
2. ~~Docker collector spike~~ ✅ (`PEARDATA_DOCKER=1`)
3. ~~Offline banner~~
4. ~~Process top-N~~ ✅ (`PEARDATA_PROCESSES=1`)
5. **Parent peer prototype** — aggregate health from N agents
6. **Swarm pull validation** — linked peer reads warm without re-scrape
---
@@ -97,17 +122,31 @@ See **[STORAGE-HYPERDB.md](./STORAGE-HYPERDB.md)** for the full design (from Hol
- Replacing centralized multi-tenant SaaS monitoring products
- Third-party proprietary on-disk metric DB formats
- Shipping a browser-only public dashboard without auth by default (REST stays localhost unless explicitly bound)
- Browser-only public dashboard without auth (REST stays localhost unless bound)
- Byte-identical third-party plugin surface (system stats first; apps as plugins)
---
## Milestone checklist (operators)
## Milestone checklist
| Milestone | You can… |
|-----------|----------|
| M0 | `npm run start:server` + `curl localhost:19999/api/v3/info` |
| M1 | Pear UI live charts from agent pubkey |
| M2 | Mint `pd1.` operator invite; revoke peer |
| M3 | Historical scrub 1h@1s via REST + RPC |
| M4 | Container charts from Docker hosts |
| M5 | Parent peer rolling up a homelab fleet |
| Milestone | You can… | Status |
|-----------|----------|--------|
| M0 | Agent + REST info | ✅ |
| M1 | Desktop live charts from pubkey | ✅ |
| M2 | Mint `pd1.` invite; revoke peer | ✅ |
| M3 | `curl \| bash` install; rolling server+client | ✅ |
| M4 | Historical scrub across restart (HyperDB warm) | ✅ |
| M5 | Container charts from Docker hosts | ✅ opt-in spike |
| M6 | Parent peer rolling up a fleet | Planned |
---
## Related docs
| Doc | Role |
|-----|------|
| [STORAGE-HYPERDB.md](./STORAGE-HYPERDB.md) | Warm store + swarm + Autobase |
| [CI.md](./CI.md) / [RELEASE.md](./RELEASE.md) | Binary matrix + installer |
| [TECH-CHOICES.md](./TECH-CHOICES.md) | Bare maps, collectors |
| [EXTENDING.md](./EXTENDING.md) | New charts / RPCs / plugins |
| [DESKTOP.md](./DESKTOP.md) | Pear / Electron shell |
+6 -1
View File
@@ -273,9 +273,13 @@ Future: `autobase`, `hyperdispatch` for HA parents.
```bash
npm run build:db
SKIP_INTEGRATION=1 npm test
# includes test/hyperdb.test.js
# includes test/hyperdb.test.js + test/store-hyperdb-fallback.test.js (M4)
```
### M4 soak — history across restart
Automated: `store-hyperdb-fallback` writes warm points, clears the memory rings, and asserts `query()` returns `source: "hyperdb-warm"` after reopening Corestore (simulates agent restart).
Manual:
```bash
@@ -283,6 +287,7 @@ npm run start:server
curl -s http://127.0.0.1:19999/api/v3/db | jq
# wait ~60s for first warm bucket, then:
curl -s 'http://127.0.0.1:19999/api/v3/data?chart=system.cpu&after=-7200&tier=1&points=120' | jq '.source,.points'
# expect "hyperdb-warm" after restarting the agent with the same PEARDATA_DATA_DIR
```
---
+1 -1
View File
@@ -89,7 +89,7 @@ Target overhead: single timer, no child processes per tick, ring buffers only.
| Agent | Node 20+ or Bare (`bin/peardata-server.mjs`), systemd unit `deploy/peardata.service` |
| Desktop | Pear (`pear-electron` + `pear-bridge`) with Bare-ready imports |
| Invites | `pd1.` tokens (PearDock-style) |
| Installer | Phase 1 one-liner script |
| Installer | `scripts/install.sh` (rolling binaries; see [RELEASE.md](./RELEASE.md)) |
## Reusable library extraction (recommended later)
+27 -3
View File
@@ -25,6 +25,9 @@
</div>
<div id="app">
<div id="offline-banner" class="offline-banner hidden" role="status">
Agent offline — showing last-known samples. Reconnecting…
</div>
<aside class="fleet-rail panel">
<h2>Fleet</h2>
<p class="hint">Connect agents by public key or <code>pd1.</code> invite.</p>
@@ -32,6 +35,10 @@
Agent / invite
<textarea id="connect-input" rows="2" placeholder="64-hex key or pd1.…"></textarea>
</label>
<label>
Alias (optional)
<input id="peer-alias" type="text" maxlength="64" placeholder="homelab-1" autocomplete="off" />
</label>
<label>
Admin seed (optional)
<input id="admin-seed" type="password" autocomplete="off" placeholder="SERVER_SEED" />
@@ -40,6 +47,13 @@
<button id="btn-connect" class="primary">Connect</button>
<button id="btn-disconnect" class="ghost" disabled>Disconnect</button>
</div>
<label class="check-row">
<input type="checkbox" id="compare-toggle" />
Compare peers (CPU overlay)
</label>
<h3 class="rail-sub">Saved</h3>
<ul id="bookmark-list" class="peer-list bookmark-list"></ul>
<h3 class="rail-sub">Connected</h3>
<ul id="peer-list" class="peer-list"></ul>
<div id="conn-meta" class="meta muted"></div>
<div class="admin-block">
@@ -74,7 +88,10 @@
<section class="charts-grid">
<article class="panel chart-panel">
<header><h3>CPU</h3><span class="muted">system.cpu</span></header>
<header>
<h3>CPU</h3>
<span class="muted" id="chart-cpu-label">system.cpu</span>
</header>
<canvas id="chart-cpu" height="140"></canvas>
</article>
<article class="panel chart-panel">
@@ -86,8 +103,15 @@
<canvas id="chart-net" height="140"></canvas>
</article>
<article class="panel chart-panel">
<header><h3>Disk I/O</h3><span class="muted">system.io</span></header>
<canvas id="chart-io" height="140"></canvas>
<header>
<h3>Explore</h3>
<label class="chart-picker">
<select id="explore-chart" aria-label="Instance chart">
<option value="system.io">system.io</option>
</select>
</label>
</header>
<canvas id="chart-explore" height="140"></canvas>
</article>
</section>
+46 -14
View File
@@ -1,9 +1,18 @@
/**
* Agent data pipeline: collector → memory store → anomaly → push
* ↘ warm HyperDB flush
* (+ optional Docker collector)
*/
import os from 'os'
import { getCollector } from './services/collector.js'
import {
getDockerCollector,
isDockerCollectorEnabled,
} from './services/collectors/docker.js'
import {
getProcessCollector,
isProcessCollectorEnabled,
} from './services/collectors/processes.js'
import { getStore } from './services/store.js'
import { getAnomalyEngine } from './services/anomaly.js'
import {
@@ -21,16 +30,12 @@ const log = logger.child('pipeline')
let healthEvery = 0
let warmFlushEvery = 0
export function startPipeline() {
const collector = getCollector()
const store = getStore()
const anomalies = getAnomalyEngine(os.cpus().length)
store.on('warm', (point) => {
enqueueWarmPoint(point)
})
collector.on('samples', (batch) => {
/**
* @param {import('./services/store.js').MetricStore} store
* @param {import('./services/anomaly.js').AnomalyEngine} anomalies
* @param {Array<{ chart: string, context: string, ts: number, values: object }>} batch
*/
function ingestBatch(store, anomalies, batch) {
store.ingest(batch)
broadcastMetrics(batch)
@@ -47,22 +52,49 @@ export function startPipeline() {
}
}
// periodic health push (~15s)
healthEvery++
if (healthEvery >= 15) {
healthEvery = 0
broadcastHealth(anomalies.getHealth())
}
// flush warm HyperDB batch every ~10s
warmFlushEvery++
if (warmFlushEvery >= 10) {
warmFlushEvery = 0
flushWarmPending().catch(() => {})
}
}
export function startPipeline() {
const collector = getCollector()
const store = getStore()
const anomalies = getAnomalyEngine(os.cpus().length)
store.on('warm', (point) => {
enqueueWarmPoint(point)
})
collector.on('samples', (batch) => ingestBatch(store, anomalies, batch))
collector.start()
log.info('Metrics pipeline started', { hyperdb: Boolean(getDb()) })
return { collector, store, anomalies }
let docker = null
if (isDockerCollectorEnabled()) {
docker = getDockerCollector()
docker.on('samples', (batch) => ingestBatch(store, anomalies, batch))
docker.start()
}
let processes = null
if (isProcessCollectorEnabled()) {
processes = getProcessCollector()
processes.on('samples', (batch) => ingestBatch(store, anomalies, batch))
processes.start()
}
log.info('Metrics pipeline started', {
hyperdb: Boolean(getDb()),
docker: Boolean(docker),
processes: Boolean(processes),
})
return { collector, store, anomalies, docker, processes }
}
+16
View File
@@ -139,6 +139,22 @@ async function shutdown() {
} catch {
// ignore
}
try {
const { getDockerCollector, isDockerCollectorEnabled } = await import(
'./services/collectors/docker.js'
)
if (isDockerCollectorEnabled()) getDockerCollector().stop()
} catch {
// ignore
}
try {
const { getProcessCollector, isProcessCollectorEnabled } = await import(
'./services/collectors/processes.js'
)
if (isProcessCollectorEnabled()) getProcessCollector().stop()
} catch {
// ignore
}
try {
await flushWarmPending()
} catch {
+11 -1
View File
@@ -119,7 +119,17 @@ export class AnomalyEngine extends EventEmitter {
const fired = []
const byChart = new Map(batch.map((s) => [s.chart, s]))
for (const cfg of this.configs.values()) {
for (const id of [...this.configs.keys()]) {
let cfg = this.configs.get(id)
// Auto-reenable after silence TTL
if (cfg.enabled === false && cfg._silencedUntil) {
if (Date.now() >= Number(cfg._silencedUntil)) {
this.setConfig({ ...cfg, enabled: true, _silencedUntil: null })
cfg = this.configs.get(id)
} else {
continue
}
}
if (cfg.enabled === false) continue
const sample = byChart.get(cfg.chart)
if (!sample) continue
+317
View File
@@ -0,0 +1,317 @@
/**
* Opt-in Docker / container collector (Phase 3 spike).
*
* Enable: PEARDATA_DOCKER=1
* Discovers containers via cgroup v2 scopes (docker-*.scope / libpod-*.scope)
* and optionally Docker Engine API over a unix socket for names.
*
* Emits:
* docker.containers — running / total counts
* docker.cpu.<shortId> — % of one host CPU
* docker.mem.<shortId> — usage / limit MiB
*/
import fs from 'fs'
import path from 'path'
import http from 'http'
import os from 'os'
import { EventEmitter } from 'events'
import {
SAMPLE_INTERVAL_MS,
registerChart,
DOCKER_CONTAINERS_CHART,
makeDockerCpuChart,
makeDockerMemChart,
} from '../../../shared/metrics.js'
import logger from '../../utils/logger.js'
const log = logger.child('docker')
export function isDockerCollectorEnabled() {
const v = process.env.PEARDATA_DOCKER
return v === '1' || v === 'on' || v === 'true'
}
function readFile(p) {
try {
return fs.readFileSync(p, 'utf8')
} catch {
return null
}
}
function bytesToMiB(n) {
return n / (1024 * 1024)
}
function hostCpus() {
try {
return os.cpus().length || 1
} catch {
return 1
}
}
/**
* @returns {Array<{ id: string, shortId: string, name: string, cgroupPath: string }>}
*/
export function discoverCgroupContainers() {
const roots = [
'/sys/fs/cgroup/system.slice',
'/sys/fs/cgroup/docker',
'/sys/fs/cgroup',
]
/** @type {Map<string, { id: string, shortId: string, name: string, cgroupPath: string }>} */
const found = new Map()
for (const root of roots) {
let entries
try {
entries = fs.readdirSync(root, { withFileTypes: true })
} catch {
continue
}
for (const ent of entries) {
if (!ent.isDirectory()) continue
const name = ent.name
let id = null
let label = name
const dockerScope = name.match(/^docker-([0-9a-f]{12,64})\.scope$/i)
const podmanScope = name.match(/^libpod-([0-9a-f]{12,64})\.scope$/i)
if (dockerScope) {
id = dockerScope[1].toLowerCase()
label = id.slice(0, 12)
} else if (podmanScope) {
id = podmanScope[1].toLowerCase()
label = id.slice(0, 12)
} else if (/^[0-9a-f]{64}$/i.test(name) && root.endsWith('/docker')) {
id = name.toLowerCase()
label = id.slice(0, 12)
}
if (!id || found.has(id)) continue
const cgroupPath = path.join(root, name)
if (
!readFile(path.join(cgroupPath, 'cpu.stat')) &&
!readFile(path.join(cgroupPath, 'cpuacct.usage'))
) {
continue
}
found.set(id, {
id,
shortId: id.slice(0, 12),
name: label,
cgroupPath,
})
}
}
return [...found.values()]
}
/**
* @param {string} cgroupPath
* @returns {{ usageNs: number }|null}
*/
export function readCgroupCpu(cgroupPath) {
const raw = readFile(path.join(cgroupPath, 'cpu.stat'))
if (raw) {
/** @type {Record<string, number>} */
const m = {}
for (const line of raw.split('\n')) {
const [k, v] = line.trim().split(/\s+/)
if (k && v != null) m[k] = Number(v)
}
if (m.usage_usec != null) {
return { usageNs: m.usage_usec * 1000 }
}
}
const acct = readFile(path.join(cgroupPath, 'cpuacct.usage'))
if (acct) {
const usageNs = Number(acct.trim())
if (Number.isFinite(usageNs)) return { usageNs }
}
return null
}
/**
* @param {string} cgroupPath
* @returns {{ usage: number, limit: number }|null} bytes
*/
export function readCgroupMemory(cgroupPath) {
const current = readFile(path.join(cgroupPath, 'memory.current'))
if (current) {
const usage = Number(current.trim())
let limit = Number(readFile(path.join(cgroupPath, 'memory.max'))?.trim())
if (!Number.isFinite(limit) || limit <= 0 || limit > 1e15) limit = 0
if (Number.isFinite(usage)) return { usage, limit }
}
const usageFile = readFile(path.join(cgroupPath, 'memory.usage_in_bytes'))
if (usageFile) {
const usage = Number(usageFile.trim())
const limitRaw = readFile(path.join(cgroupPath, 'memory.limit_in_bytes'))
let limit = limitRaw ? Number(limitRaw.trim()) : 0
if (limit > 1e15) limit = 0
if (Number.isFinite(usage)) return { usage, limit }
}
return null
}
/**
* @param {string} socketPath
* @returns {Promise<Map<string, string>>} id → name
*/
export function fetchDockerNames(socketPath) {
return new Promise((resolve) => {
const req = http.request(
{
socketPath,
path: '/containers/json?all=1',
method: 'GET',
timeout: 2000,
},
(res) => {
let body = ''
res.on('data', (c) => {
body += c
})
res.on('end', () => {
try {
const list = JSON.parse(body)
/** @type {Map<string, string>} */
const map = new Map()
for (const c of list) {
const id = String(c.Id || '').toLowerCase()
const name = String((c.Names && c.Names[0]) || id)
.replace(/^\//, '')
.slice(0, 64)
if (id) map.set(id, name)
if (id.length >= 12) map.set(id.slice(0, 12), name)
}
resolve(map)
} catch {
resolve(new Map())
}
})
}
)
req.on('error', () => resolve(new Map()))
req.on('timeout', () => {
req.destroy()
resolve(new Map())
})
req.end()
})
}
export class DockerCollector extends EventEmitter {
constructor(opts = {}) {
super()
this.intervalMs = opts.intervalMs || Number(process.env.PEARDATA_SAMPLE_MS) || SAMPLE_INTERVAL_MS
this.socketPath = opts.socketPath || process.env.PEARDATA_DOCKER_SOCKET || '/var/run/docker.sock'
this._timer = null
/** @type {Map<string, { usageNs: number, wallMs: number }>} */
this._prevCpu = new Map()
/** @type {Map<string, string>} */
this._names = new Map()
this._nameRefreshAt = 0
}
start() {
if (this._timer) return
registerChart(DOCKER_CONTAINERS_CHART)
log.info('Docker collector started', { socket: this.socketPath })
this._tick()
this._timer = setInterval(() => this._tick(), this.intervalMs)
if (typeof this._timer.unref === 'function') this._timer.unref()
}
stop() {
if (this._timer) {
clearInterval(this._timer)
this._timer = null
}
}
async _refreshNames() {
const now = Date.now()
if (now - this._nameRefreshAt < 30_000) return
this._nameRefreshAt = now
try {
if (fs.existsSync(this.socketPath)) {
this._names = await fetchDockerNames(this.socketPath)
}
} catch {
// ignore
}
}
async _tick() {
try {
await this._refreshNames()
const containers = discoverCgroupContainers()
const ts = Date.now()
const wallMs = ts
const ncpu = hostCpus()
/** @type {Array<{ chart: string, context: string, ts: number, values: Record<string, number|null> }>} */
const batch = [
{
chart: 'docker.containers',
context: 'docker.containers',
ts,
values: { running: containers.length, total: containers.length },
},
]
for (const c of containers) {
const display =
this._names.get(c.id) || this._names.get(c.shortId) || c.name || c.shortId
const cpuDef = makeDockerCpuChart(c.shortId, display)
const memDef = makeDockerMemChart(c.shortId, display)
registerChart(cpuDef)
registerChart(memDef)
const cpu = readCgroupCpu(c.cgroupPath)
let usagePct = 0
if (cpu) {
const prev = this._prevCpu.get(c.id)
if (prev && wallMs > prev.wallMs) {
const dNs = cpu.usageNs - prev.usageNs
const dWallNs = (wallMs - prev.wallMs) * 1e6
if (dNs >= 0 && dWallNs > 0) {
usagePct = Math.min(100 * ncpu, (dNs / dWallNs) * 100)
}
}
this._prevCpu.set(c.id, { usageNs: cpu.usageNs, wallMs })
}
batch.push({
chart: cpuDef.id,
context: 'docker.cpu',
ts,
values: { usage: usagePct },
})
const mem = readCgroupMemory(c.cgroupPath)
batch.push({
chart: memDef.id,
context: 'docker.mem',
ts,
values: {
usage: mem ? bytesToMiB(mem.usage) : 0,
limit: mem && mem.limit ? bytesToMiB(mem.limit) : null,
},
})
}
this.emit('samples', batch)
} catch (err) {
log.warn('Docker tick failed', { error: err.message })
}
}
}
/** @type {DockerCollector|null} */
let singleton = null
export function getDockerCollector() {
if (!singleton) singleton = new DockerCollector()
return singleton
}
+259
View File
@@ -0,0 +1,259 @@
/**
* Opt-in process top-N collector (Phase 3).
*
* Enable: PEARDATA_PROCESSES=1
* Limit: PEARDATA_PROCESSES_TOP=8 (default)
*
* Emits:
* processes.top_cpu — % of one host CPU for top processes (by name)
* processes.top_rss — RSS MiB for top processes (by RSS)
*
* Linux /proc only; no-op elsewhere.
*/
import fs from 'fs'
import path from 'path'
import os from 'os'
import { EventEmitter } from 'events'
import { SAMPLE_INTERVAL_MS, registerChart } from '../../../shared/metrics.js'
import logger from '../../utils/logger.js'
const log = logger.child('processes')
export function isProcessCollectorEnabled() {
const v = process.env.PEARDATA_PROCESSES
return v === '1' || v === 'on' || v === 'true'
}
function topN() {
const n = Number(process.env.PEARDATA_PROCESSES_TOP)
return Number.isFinite(n) && n > 0 ? Math.min(32, Math.floor(n)) : 8
}
function hostCpus() {
try {
return os.cpus().length || 1
} catch {
return 1
}
}
function sanitizeDim(name) {
const s = String(name || 'unknown')
.replace(/[^\w.+-]/g, '_')
.replace(/^_+|_+$/g, '')
.slice(0, 48)
return s || 'unknown'
}
/**
* @returns {Array<{ pid: number, name: string, utime: number, stime: number, rssPages: number }>|null}
*/
export function listProcStats() {
if (os.platform() !== 'linux') return null
let dirs
try {
dirs = fs.readdirSync('/proc')
} catch {
return null
}
/** @type {Array<{ pid: number, name: string, utime: number, stime: number, rssPages: number }>} */
const out = []
for (const ent of dirs) {
if (!/^\d+$/.test(ent)) continue
const pid = Number(ent)
let raw
try {
raw = fs.readFileSync(path.join('/proc', ent, 'stat'), 'utf8')
} catch {
continue
}
const open = raw.indexOf('(')
const close = raw.lastIndexOf(')')
if (open < 0 || close < open) continue
const name = raw.slice(open + 1, close)
const rest = raw.slice(close + 2).split(/\s+/)
// fields after comm: state(0) … utime(11) stime(12) … rss(21)
const utime = Number(rest[11])
const stime = Number(rest[12])
const rssPages = Number(rest[21])
if (!Number.isFinite(utime) || !Number.isFinite(stime)) continue
out.push({
pid,
name: sanitizeDim(name),
utime,
stime,
rssPages: Number.isFinite(rssPages) ? rssPages : 0,
})
}
return out
}
function pageSize() {
try {
return os.constants?.os?.PAGE_SIZE || 4096
} catch {
return 4096
}
}
/**
* @param {string[]} names
* @param {'cpu'|'rss'} kind
*/
function registerTopChart(names, kind) {
const dims = [...new Set(names)].slice(0, topN()).map((id) => ({
id,
name: id,
algorithm: 'absolute',
}))
if (!dims.length) {
dims.push({ id: '_idle', name: '_idle', algorithm: 'absolute' })
}
const def =
kind === 'cpu'
? {
id: 'processes.top_cpu',
name: 'processes.top_cpu',
context: 'processes.top_cpu',
title: 'Top processes CPU',
units: 'percentage',
family: 'processes',
chartType: 'stacked',
priority: 6000,
plugin: 'processes',
dimensions: dims,
}
: {
id: 'processes.top_rss',
name: 'processes.top_rss',
context: 'processes.top_rss',
title: 'Top processes RSS',
units: 'MiB',
family: 'processes',
chartType: 'stacked',
priority: 6010,
plugin: 'processes',
dimensions: dims,
}
registerChart(def)
return def
}
export class ProcessCollector extends EventEmitter {
constructor(opts = {}) {
super()
this.intervalMs = opts.intervalMs || Number(process.env.PEARDATA_SAMPLE_MS) || SAMPLE_INTERVAL_MS
this._timer = null
/** @type {Map<number, { ticks: number, wallMs: number, name: string }>|null} */
this._prev = null
this._pageBytes = 4096
}
start() {
if (this._timer) return
this._pageBytes = pageSize()
log.info('Process top-N collector started', { top: topN() })
this._tick()
this._timer = setInterval(() => this._tick(), this.intervalMs)
if (typeof this._timer.unref === 'function') this._timer.unref()
}
stop() {
if (this._timer) {
clearInterval(this._timer)
this._timer = null
}
}
_tick() {
try {
const procs = listProcStats()
if (!procs) return
const ts = Date.now()
const wallMs = ts
const ncpu = hostCpus()
const limit = topN()
/** @type {Map<string, number>} */
const cpuByName = new Map()
if (this._prev) {
for (const p of procs) {
const prev = this._prev.get(p.pid)
if (!prev || wallMs <= prev.wallMs) continue
const ticks = p.utime + p.stime
const dTicks = ticks - prev.ticks
const dSec = (wallMs - prev.wallMs) / 1000
if (dTicks < 0 || dSec <= 0) continue
// Linux USER_HZ typically 100
const pct = (dTicks / 100 / dSec) * 100
const key = p.name
cpuByName.set(key, (cpuByName.get(key) || 0) + Math.min(100 * ncpu, pct))
}
}
/** @type {Map<number, { ticks: number, wallMs: number, name: string }>} */
const next = new Map()
for (const p of procs) {
next.set(p.pid, { ticks: p.utime + p.stime, wallMs, name: p.name })
}
this._prev = next
const cpuRanked = [...cpuByName.entries()].sort((a, b) => b[1] - a[1]).slice(0, limit)
const rssRanked = [...procs]
.sort((a, b) => b.rssPages - a.rssPages)
.slice(0, limit)
/** @type {Map<string, number>} */
const rssByName = new Map()
for (const p of rssRanked) {
const mib = (p.rssPages * this._pageBytes) / (1024 * 1024)
rssByName.set(p.name, (rssByName.get(p.name) || 0) + mib)
}
const rssTop = [...rssByName.entries()].sort((a, b) => b[1] - a[1]).slice(0, limit)
const cpuDef = registerTopChart(
cpuRanked.map(([n]) => n),
'cpu'
)
const rssDef = registerTopChart(
rssTop.map(([n]) => n),
'rss'
)
/** @type {Record<string, number|null>} */
const cpuValues = {}
for (const d of cpuDef.dimensions) cpuValues[d.id] = 0
for (const [name, pct] of cpuRanked) cpuValues[name] = pct
/** @type {Record<string, number|null>} */
const rssValues = {}
for (const d of rssDef.dimensions) rssValues[d.id] = 0
for (const [name, mib] of rssTop) rssValues[name] = mib
this.emit('samples', [
{
chart: 'processes.top_cpu',
context: 'processes.top_cpu',
ts,
values: cpuValues,
},
{
chart: 'processes.top_rss',
context: 'processes.top_rss',
ts,
values: rssValues,
},
])
} catch (err) {
log.warn('Process tick failed', { error: err.message })
}
}
}
/** @type {ProcessCollector|null} */
let singleton = null
export function getProcessCollector() {
if (!singleton) singleton = new ProcessCollector()
return singleton
}
+13 -3
View File
@@ -130,8 +130,16 @@ export class MetricStore extends EventEmitter {
let windowed = src.filter((p) => p.ts >= afterMs && p.ts <= beforeMs)
let source = useTier1 ? 'memory-tier1' : 'memory-tier0'
// HyperDB warm fallback when memory misses (or explicit tier>=1 with sparse memory)
if (!windowed.length || (opts.tier >= 1 && windowed.length < (opts.points || 60) / 2)) {
const oldestMem = src.length ? src[0].ts : null
const windowExceedsMemory =
oldestMem != null && afterMs < oldestMem - 1000
const sparse =
!windowed.length ||
(opts.tier >= 1 && windowed.length < (opts.points || 60) / 2) ||
windowExceedsMemory
// HyperDB warm fallback when memory misses, is sparse, or cannot cover the after window
if (sparse) {
const db = getDb()
if (db) {
try {
@@ -139,13 +147,15 @@ export class MetricStore extends EventEmitter {
chart,
afterMs,
beforeMs,
limit: opts.points || 10_000,
limit: Math.max(opts.points || 60, 10_000),
tier: 1,
})
if (warm.length) {
if (!windowed.length || warm.length >= windowed.length || windowExceedsMemory) {
windowed = warm
source = 'hyperdb-warm'
}
}
} catch {
// keep memory result
}
+52
View File
@@ -802,6 +802,58 @@ export function makeDiskInodesChart(mountId, mountPath) {
}
}
/** Static aggregate when Docker collector is enabled */
export const DOCKER_CONTAINERS_CHART = {
id: 'docker.containers',
name: 'docker.containers',
context: 'docker.containers',
title: 'Running containers',
units: 'containers',
family: 'containers',
chartType: 'line',
priority: 5000,
plugin: 'docker',
dimensions: [
{ id: 'running', name: 'running', algorithm: 'absolute' },
{ id: 'total', name: 'total', algorithm: 'absolute' },
],
}
export function makeDockerCpuChart(shortId, name) {
const id = shortId.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 64)
return {
id: `docker.cpu.${id}`,
name: `docker.cpu.${id}`,
context: 'docker.cpu',
title: `Container CPU ${name || id}`,
units: 'percentage',
family: name || id,
chartType: 'area',
priority: 5100,
plugin: 'docker',
dimensions: [{ id: 'usage', name: 'usage', algorithm: 'absolute' }],
}
}
export function makeDockerMemChart(shortId, name) {
const id = shortId.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 64)
return {
id: `docker.mem.${id}`,
name: `docker.mem.${id}`,
context: 'docker.mem',
title: `Container memory ${name || id}`,
units: 'MiB',
family: name || id,
chartType: 'area',
priority: 5200,
plugin: 'docker',
dimensions: [
{ id: 'usage', name: 'usage', algorithm: 'absolute' },
{ id: 'limit', name: 'limit', algorithm: 'absolute' },
],
}
}
/**
* Build a chart summary object for discovery APIs.
* @param {ChartDef} def
+76
View File
@@ -0,0 +1,76 @@
import test from 'brittle'
import { AnomalyEngine } from '../server/services/anomaly.js'
import { silenceAlert } from '../server/services/alerts.js'
test('silence TTL auto-reenables alert', (t) => {
const engine = new AnomalyEngine({ cpuCount: 2 })
// Use singleton path via silenceAlert — it mutates getAnomalyEngine().
// Drive a local engine for isolation instead:
const cfg = {
id: 'cpu_user_high',
chart: 'system.cpu',
dimension: 'user',
warn: 10,
crit: 50,
comparator: '>',
enabled: true,
info: 'test',
}
engine.setConfig(cfg)
engine.setConfig({
...cfg,
enabled: false,
_silencedUntil: Date.now() - 1000,
})
const fired = engine.evaluate([
{
chart: 'system.cpu',
context: 'system.cpu',
ts: Date.now(),
values: {
user: 80,
system: 0,
nice: 0,
iowait: 0,
irq: 0,
softirq: 0,
idle: 20,
steal: 0,
guest: 0,
guest_nice: 0,
},
},
])
t.ok(engine.configs.get('cpu_user_high').enabled === true)
t.ok(fired.length >= 1)
t.ok(silenceAlert) // keep import used / API present
})
test('silence still active before TTL', (t) => {
const engine = new AnomalyEngine({ cpuCount: 2 })
engine.setConfig({
id: 'cpu_user_high',
chart: 'system.cpu',
dimension: 'user',
warn: 10,
crit: 50,
comparator: '>',
enabled: false,
_silencedUntil: Date.now() + 60_000,
info: 'test',
})
const fired = engine.evaluate([
{
chart: 'system.cpu',
context: 'system.cpu',
ts: Date.now(),
values: { user: 90, idle: 10 },
},
])
t.is(fired.length, 0)
t.is(engine.configs.get('cpu_user_high').enabled, false)
})
+39
View File
@@ -0,0 +1,39 @@
import test from 'brittle'
import fs from 'fs'
import path from 'path'
import os from 'os'
import {
loadBookmarks,
upsertBookmark,
removeBookmark,
setBookmarkAlias,
bookmarksPath,
} from '../client/bookmarks.js'
const tmp = path.join(os.tmpdir(), `peardata-bm-${Date.now()}`)
test('bookmarks upsert alias remove', (t) => {
fs.mkdirSync(tmp, { recursive: true })
process.env.PEARDATA_HOME = tmp
t.ok(bookmarksPath().includes('bookmarks.json'))
const pk = 'a'.repeat(64)
upsertBookmark({ publicKeyHex: pk, alias: 'box-1' })
let list = loadBookmarks()
t.is(list.length, 1)
t.is(list[0].alias, 'box-1')
setBookmarkAlias(pk, 'box-2')
list = loadBookmarks()
t.is(list[0].alias, 'box-2')
removeBookmark(pk)
t.is(loadBookmarks().length, 0)
try {
fs.rmSync(tmp, { recursive: true, force: true })
} catch {
// ignore
}
delete process.env.PEARDATA_HOME
})
+68
View File
@@ -0,0 +1,68 @@
import test from 'brittle'
import fs from 'fs'
import path from 'path'
import os from 'os'
import {
discoverCgroupContainers,
readCgroupCpu,
readCgroupMemory,
isDockerCollectorEnabled,
} from '../server/services/collectors/docker.js'
import {
makeDockerCpuChart,
makeDockerMemChart,
DOCKER_CONTAINERS_CHART,
} from '../shared/metrics.js'
function writeFixtureCgroup(root, id) {
const dir = path.join(root, `docker-${id}.scope`)
fs.mkdirSync(dir, { recursive: true })
fs.writeFileSync(
path.join(dir, 'cpu.stat'),
'usage_usec 5000000\nuser_usec 4000000\nsystem_usec 1000000\n'
)
fs.writeFileSync(path.join(dir, 'memory.current'), String(64 * 1024 * 1024))
fs.writeFileSync(path.join(dir, 'memory.max'), String(256 * 1024 * 1024))
return dir
}
test('docker chart helpers', (t) => {
t.is(DOCKER_CONTAINERS_CHART.id, 'docker.containers')
const cpu = makeDockerCpuChart('abc123def456', 'nginx')
t.is(cpu.id, 'docker.cpu.abc123def456')
t.is(cpu.context, 'docker.cpu')
const mem = makeDockerMemChart('abc123def456', 'nginx')
t.is(mem.id, 'docker.mem.abc123def456')
t.ok(mem.dimensions.find((d) => d.id === 'usage'))
})
test('isDockerCollectorEnabled', (t) => {
const prev = process.env.PEARDATA_DOCKER
delete process.env.PEARDATA_DOCKER
t.absent(isDockerCollectorEnabled())
process.env.PEARDATA_DOCKER = '1'
t.ok(isDockerCollectorEnabled())
if (prev == null) delete process.env.PEARDATA_DOCKER
else process.env.PEARDATA_DOCKER = prev
})
test('readCgroupCpu / memory from fixture', (t) => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'peardata-cgroup-'))
const slice = path.join(tmp, 'system.slice')
fs.mkdirSync(slice, { recursive: true })
const id = 'a'.repeat(64)
const cgroupPath = writeFixtureCgroup(slice, id)
const cpu = readCgroupCpu(cgroupPath)
t.ok(cpu)
t.is(cpu.usageNs, 5000000 * 1000)
const mem = readCgroupMemory(cgroupPath)
t.ok(mem)
t.is(mem.usage, 64 * 1024 * 1024)
t.is(mem.limit, 256 * 1024 * 1024)
t.ok(typeof discoverCgroupContainers === 'function')
fs.rmSync(tmp, { recursive: true, force: true })
})
+25
View File
@@ -0,0 +1,25 @@
import test from 'brittle'
import { isProcessCollectorEnabled, listProcStats } from '../server/services/collectors/processes.js'
test('isProcessCollectorEnabled', (t) => {
const prev = process.env.PEARDATA_PROCESSES
delete process.env.PEARDATA_PROCESSES
t.absent(isProcessCollectorEnabled())
process.env.PEARDATA_PROCESSES = '1'
t.ok(isProcessCollectorEnabled())
if (prev == null) delete process.env.PEARDATA_PROCESSES
else process.env.PEARDATA_PROCESSES = prev
})
test('listProcStats on linux returns pids', (t) => {
const rows = listProcStats()
if (process.platform !== 'linux') {
t.is(rows, null)
return
}
t.ok(Array.isArray(rows))
t.ok(rows.length > 0)
t.ok(rows[0].pid > 0)
t.ok(rows[0].name)
t.ok(Number.isFinite(rows[0].utime))
})
+74
View File
@@ -0,0 +1,74 @@
/**
* M4 soak: query falls back to HyperDB warm after memory miss / "restart".
*/
import test from 'brittle'
import fs from 'fs'
import path from 'path'
import { fileURLToPath } from 'url'
import { MetricStore } from '../server/services/store.js'
import { openDb, closeDb, getDb } from '../server/db/index.js'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const TMP = path.join(__dirname, '..', 'tmp-store-hyperdb-test')
test('query uses hyperdb-warm when memory is empty', async (t) => {
fs.rmSync(TMP, { recursive: true, force: true })
process.env.PEARDATA_DATA_DIR = TMP
delete process.env.PEARDATA_HYPERDB
await closeDb().catch(() => {})
const db = await openDb()
t.ok(db)
const now = Date.now()
await db.putMetricPoints([
{
chart: 'system.cpu',
context: 'system.cpu',
ts: now - 120_000,
values: { user: 11, system: 4, nice: 0, iowait: 0, irq: 0, softirq: 0, idle: 85 },
tier: 1,
},
{
chart: 'system.cpu',
context: 'system.cpu',
ts: now - 60_000,
values: { user: 22, system: 5, nice: 0, iowait: 0, irq: 0, softirq: 0, idle: 73 },
tier: 1,
},
])
// Fresh memory store (simulates process restart with empty rings)
const store = new MetricStore()
const q = await store.query({
chart: 'system.cpu',
after: Math.floor((now - 180_000) / 1000),
before: Math.floor(now / 1000),
points: 60,
tier: 1,
})
t.absent(q.error)
t.is(q.source, 'hyperdb-warm')
t.ok(q.data.length >= 2)
await closeDb()
// Re-open same Corestore — warm points must still be queryable
const db2 = await openDb()
t.ok(db2)
t.is(getDb(), db2)
const store2 = new MetricStore()
const q2 = await store2.query({
chart: 'system.cpu',
after: Math.floor((now - 180_000) / 1000),
before: Math.floor(now / 1000),
points: 60,
tier: 1,
})
t.is(q2.source, 'hyperdb-warm')
t.ok(q2.data.length >= 2)
await closeDb()
fs.rmSync(TMP, { recursive: true, force: true })
delete process.env.PEARDATA_DATA_DIR
})
+83
View File
@@ -216,6 +216,31 @@ pear-ctrl[data-platform='darwin'] {
border-color: rgba(62, 207, 142, 0.35);
}
.offline-banner {
flex: 1 0 100%;
order: -1;
margin: 12px 12px 0;
padding: 10px 14px;
font-size: 13px;
color: #f0d9a8;
background: linear-gradient(90deg, rgba(240, 180, 41, 0.14), rgba(240, 180, 41, 0.05));
border: 1px solid rgba(240, 180, 41, 0.35);
border-radius: 8px;
}
.offline-banner.hidden {
display: none;
}
#app:has(.offline-banner:not(.hidden)) {
flex-wrap: wrap;
align-content: flex-start;
}
body.is-offline .dash-main .chart-panel canvas {
opacity: 0.72;
}
.layout {
display: grid;
grid-template-columns: 300px 1fr 280px;
@@ -298,6 +323,64 @@ textarea {
margin-top: 10px;
}
.check-row {
display: flex;
align-items: center;
gap: 8px;
margin-top: 12px;
font-size: 13px;
color: var(--muted);
}
.check-row input {
width: auto;
margin: 0;
}
.rail-sub {
margin: 16px 0 6px;
font-size: 12px;
text-transform: uppercase;
letter-spacing: 0.04em;
color: var(--muted);
font-weight: 600;
}
.bookmark-list li {
justify-content: space-between;
gap: 8px;
}
.bookmark-actions {
display: flex;
gap: 4px;
flex-shrink: 0;
}
button.linkish {
border: none;
background: transparent;
color: var(--muted);
padding: 2px 6px;
font-size: 14px;
line-height: 1;
}
button.linkish:hover {
color: var(--accent);
}
.chart-picker select {
max-width: 160px;
font-size: 11px;
font-family: var(--mono);
padding: 4px 6px;
background: var(--panel-2);
color: var(--muted);
border: 1px solid var(--border);
border-radius: 6px;
}
button {
border: 1px solid var(--border);
background: #1a2540;