544 lines
16 KiB
JavaScript
544 lines
16 KiB
JavaScript
/**
|
|
* PearDataModel — HyperDB facade for metadata, peer links, alerts, warm metrics.
|
|
*
|
|
* Pattern mirrors holepunch hyperdb-workshop Registry:
|
|
* HyperDB.bee(core, spec, { autoUpdate: true })
|
|
* exclusiveTransaction / transaction + flush
|
|
*/
|
|
import ReadyResource from 'ready-resource'
|
|
import HyperDB from 'hyperdb'
|
|
import b4a from 'b4a'
|
|
import spec from '../../spec/hyperdb/index.js'
|
|
|
|
export class PearDataModel extends ReadyResource {
|
|
/**
|
|
* @param {import('hypercore')} core
|
|
* @param {{ autoUpdate?: boolean, writable?: boolean, extension?: boolean }} [opts]
|
|
*/
|
|
constructor(core, opts = {}) {
|
|
super()
|
|
this.db = HyperDB.bee(core, spec, {
|
|
autoUpdate: opts.autoUpdate !== false,
|
|
writable: opts.writable !== false,
|
|
extension: opts.extension !== false,
|
|
})
|
|
}
|
|
|
|
get publicKey() {
|
|
return this.db.core.key
|
|
}
|
|
|
|
get publicKeyHex() {
|
|
return b4a.toString(this.db.core.key, 'hex')
|
|
}
|
|
|
|
get discoveryKey() {
|
|
return this.db.core.discoveryKey
|
|
}
|
|
|
|
get discoveryKeyHex() {
|
|
return b4a.toString(this.db.core.discoveryKey, 'hex')
|
|
}
|
|
|
|
async _open() {
|
|
await this.db.ready()
|
|
}
|
|
|
|
async _close() {
|
|
await this.db.close()
|
|
}
|
|
|
|
// ── nodes ─────────────────────────────────────────────────
|
|
|
|
/**
|
|
* @param {object} node
|
|
*/
|
|
async putNode(node) {
|
|
await this.ready()
|
|
const tx = await this.db.exclusiveTransaction()
|
|
try {
|
|
await tx.insert('@peardata/node', {
|
|
nodeId: String(node.nodeId),
|
|
hostname: String(node.hostname || ''),
|
|
publicKeyHex: node.publicKeyHex || null,
|
|
platform: node.platform || null,
|
|
arch: node.arch || null,
|
|
cpus: node.cpus ?? null,
|
|
totalMemMiB: node.totalMemMiB != null ? Math.floor(node.totalMemMiB) : null,
|
|
agentVersion: node.agentVersion || null,
|
|
labelsJson: node.labels ? JSON.stringify(node.labels) : node.labelsJson || null,
|
|
updatedAt: node.updatedAt || Date.now(),
|
|
})
|
|
await tx.flush()
|
|
} catch (err) {
|
|
await tx.close().catch(() => {})
|
|
throw err
|
|
}
|
|
}
|
|
|
|
async getNode(nodeId) {
|
|
await this.ready()
|
|
return this.db.get('@peardata/node', { nodeId: String(nodeId) })
|
|
}
|
|
|
|
async listNodes() {
|
|
await this.ready()
|
|
return this.db.find('@peardata/node').toArray()
|
|
}
|
|
|
|
// ── peer links ────────────────────────────────────────────
|
|
|
|
/**
|
|
* @param {object} link
|
|
*/
|
|
async putPeerLink(link) {
|
|
await this.ready()
|
|
const tx = await this.db.exclusiveTransaction()
|
|
try {
|
|
await tx.insert('@peardata/peer-link', {
|
|
localNodeId: String(link.localNodeId),
|
|
remotePublicKey: String(link.remotePublicKey).toLowerCase(),
|
|
role: link.role || 'viewer',
|
|
alias: link.alias || null,
|
|
dbKeyHex: link.dbKeyHex || null,
|
|
discoveryKeyHex: link.discoveryKeyHex || null,
|
|
syncMode: link.syncMode || 'both',
|
|
linkedAt: link.linkedAt || Date.now(),
|
|
lastSeen: link.lastSeen || null,
|
|
})
|
|
await tx.flush()
|
|
} catch (err) {
|
|
await tx.close().catch(() => {})
|
|
throw err
|
|
}
|
|
}
|
|
|
|
async deletePeerLink(localNodeId, remotePublicKey) {
|
|
await this.ready()
|
|
const tx = await this.db.exclusiveTransaction()
|
|
try {
|
|
await tx.delete('@peardata/peer-link', {
|
|
localNodeId: String(localNodeId),
|
|
remotePublicKey: String(remotePublicKey).toLowerCase(),
|
|
})
|
|
await tx.flush()
|
|
} catch (err) {
|
|
await tx.close().catch(() => {})
|
|
throw err
|
|
}
|
|
}
|
|
|
|
async listPeerLinks(localNodeId = null) {
|
|
await this.ready()
|
|
if (!localNodeId) {
|
|
return this.db.find('@peardata/peer-link').toArray()
|
|
}
|
|
return this.db
|
|
.find('@peardata/peer-link', {
|
|
gte: { localNodeId: String(localNodeId), remotePublicKey: '' },
|
|
lte: { localNodeId: String(localNodeId), remotePublicKey: '\uffff' },
|
|
})
|
|
.toArray()
|
|
}
|
|
|
|
// ── alert configs ─────────────────────────────────────────
|
|
|
|
async putAlertConfig(cfg) {
|
|
await this.ready()
|
|
const tx = await this.db.exclusiveTransaction()
|
|
try {
|
|
await tx.insert('@peardata/alert-config', {
|
|
id: String(cfg.id),
|
|
chart: String(cfg.chart),
|
|
dimension: String(cfg.dimension),
|
|
warn: cfg.warn ?? null,
|
|
crit: cfg.crit ?? null,
|
|
comparator: cfg.comparator || '>',
|
|
enabled: cfg.enabled !== false,
|
|
info: cfg.info || null,
|
|
updatedAt: Date.now(),
|
|
})
|
|
await tx.flush()
|
|
} catch (err) {
|
|
await tx.close().catch(() => {})
|
|
throw err
|
|
}
|
|
}
|
|
|
|
async getAlertConfig(id) {
|
|
await this.ready()
|
|
return this.db.get('@peardata/alert-config', { id: String(id) })
|
|
}
|
|
|
|
async listAlertConfigs() {
|
|
await this.ready()
|
|
return this.db.find('@peardata/alert-config').toArray()
|
|
}
|
|
|
|
// ── alert events ──────────────────────────────────────────
|
|
|
|
async putAlertEvent(ev) {
|
|
await this.ready()
|
|
const tx = await this.db.exclusiveTransaction()
|
|
try {
|
|
await tx.insert('@peardata/alert-event', {
|
|
id: String(ev.id),
|
|
ts: Number(ev.ts) || Date.now(),
|
|
chart: String(ev.chart || ''),
|
|
dimension: ev.dimension || null,
|
|
severity: String(ev.severity || 'warning'),
|
|
value: ev.value ?? null,
|
|
threshold: ev.threshold ?? null,
|
|
message: ev.message || null,
|
|
cleared: Boolean(ev.cleared),
|
|
})
|
|
await tx.flush()
|
|
} catch (err) {
|
|
await tx.close().catch(() => {})
|
|
throw err
|
|
}
|
|
}
|
|
|
|
async listAlertEvents({ chart = null, limit = 100 } = {}) {
|
|
await this.ready()
|
|
let rows
|
|
if (chart) {
|
|
rows = await this.db
|
|
.find(
|
|
'@peardata/alert-event-by-chart',
|
|
{
|
|
gte: { chart: String(chart), ts: 0 },
|
|
lte: { chart: String(chart), ts: Number.MAX_SAFE_INTEGER },
|
|
},
|
|
{ reverse: true, limit }
|
|
)
|
|
.toArray()
|
|
} else {
|
|
rows = await this.db
|
|
.find('@peardata/alert-event', {}, { reverse: true, limit })
|
|
.toArray()
|
|
}
|
|
return rows
|
|
}
|
|
|
|
// ── warm metric points ────────────────────────────────────
|
|
|
|
/**
|
|
* Batch-insert warm (downsampled) metric points.
|
|
* @param {Array<{ chart: string, context: string, ts: number, tier?: number, values: object }>} points
|
|
*/
|
|
async putMetricPoints(points) {
|
|
if (!points?.length) return { inserted: 0 }
|
|
await this.ready()
|
|
const tx = await this.db.exclusiveTransaction()
|
|
try {
|
|
for (const p of points) {
|
|
await tx.insert('@peardata/metric-point', {
|
|
chart: String(p.chart),
|
|
ts: Number(p.ts),
|
|
context: String(p.context || p.chart),
|
|
tier: p.tier ?? 1,
|
|
valuesJson: JSON.stringify(p.values || {}),
|
|
})
|
|
}
|
|
await tx.flush()
|
|
return { inserted: points.length }
|
|
} catch (err) {
|
|
await tx.close().catch(() => {})
|
|
throw err
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Query warm points for a chart in [afterMs, beforeMs].
|
|
* @param {{ chart: string, afterMs: number, beforeMs: number, limit?: number, tier?: number }} opts
|
|
*/
|
|
async queryMetricPoints(opts) {
|
|
await this.ready()
|
|
const chart = String(opts.chart)
|
|
const afterMs = Number(opts.afterMs) || 0
|
|
const beforeMs = Number(opts.beforeMs) || Date.now()
|
|
const limit = Math.min(opts.limit || 10_000, 50_000)
|
|
|
|
const rows = await this.db
|
|
.find(
|
|
'@peardata/metric-point',
|
|
{
|
|
gte: { chart, ts: afterMs },
|
|
lte: { chart, ts: beforeMs },
|
|
},
|
|
{ limit }
|
|
)
|
|
.toArray()
|
|
|
|
return rows
|
|
.filter((r) => (opts.tier == null ? true : r.tier === opts.tier))
|
|
.map((r) => ({
|
|
chart: r.chart,
|
|
context: r.context,
|
|
ts: r.ts,
|
|
tier: r.tier,
|
|
values: safeJson(r.valuesJson),
|
|
}))
|
|
}
|
|
|
|
/**
|
|
* Sample warm-point stats for Data Manager (bounded scan).
|
|
* @param {{ limit?: number }} [opts]
|
|
*/
|
|
async sampleMetricPointsStats(opts = {}) {
|
|
await this.ready()
|
|
const limit = Math.min(opts.limit || 50_000, 100_000)
|
|
const rows = await this.db
|
|
.find(
|
|
'@peardata/metric-point',
|
|
{
|
|
gte: { chart: '', ts: 0 },
|
|
lte: { chart: '\uffff', ts: Number.MAX_SAFE_INTEGER },
|
|
},
|
|
{ limit }
|
|
)
|
|
.toArray()
|
|
const charts = new Set()
|
|
let oldestTs = null
|
|
let newestTs = null
|
|
for (const r of rows) {
|
|
charts.add(r.chart)
|
|
if (oldestTs == null || r.ts < oldestTs) oldestTs = r.ts
|
|
if (newestTs == null || r.ts > newestTs) newestTs = r.ts
|
|
}
|
|
return {
|
|
pointsSampled: rows.length,
|
|
chartsSampled: charts.size,
|
|
oldestTs,
|
|
newestTs,
|
|
sampleCapped: rows.length >= limit,
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Distinct chart ids present in warm storage (bounded sample).
|
|
* @param {{ limit?: number }} [opts]
|
|
* @returns {Promise<string[]>}
|
|
*/
|
|
async listMetricPointCharts(opts = {}) {
|
|
await this.ready()
|
|
const limit = Math.min(opts.limit || 100_000, 200_000)
|
|
const rows = await this.db
|
|
.find(
|
|
'@peardata/metric-point',
|
|
{
|
|
gte: { chart: '', ts: 0 },
|
|
lte: { chart: '\uffff', ts: Number.MAX_SAFE_INTEGER },
|
|
},
|
|
{ limit }
|
|
)
|
|
.toArray()
|
|
return [...new Set(rows.map((r) => r.chart))]
|
|
}
|
|
|
|
/**
|
|
* Delete warm points with ts < beforeMs for the given charts.
|
|
* Multi-pass until no more rows match (or maxPasses), so large backlogs drain fully.
|
|
* @param {number} beforeMs
|
|
* @param {{ charts?: string[], dryRun?: boolean, limitPerChart?: number, maxPasses?: number }} [opts]
|
|
*/
|
|
async deleteMetricPointsBefore(beforeMs, opts = {}) {
|
|
await this.ready()
|
|
const cutoff = Number(beforeMs)
|
|
if (!Number.isFinite(cutoff) || cutoff <= 0) {
|
|
return { deleted: 0, scanned: 0, passes: 0 }
|
|
}
|
|
const dryRun = Boolean(opts.dryRun)
|
|
const limitPerChart = Math.min(opts.limitPerChart || 20_000, 50_000)
|
|
const maxPasses = Math.min(Math.max(opts.maxPasses || 50, 1), 200)
|
|
const charts = opts.charts?.length ? opts.charts : null
|
|
let deleted = 0
|
|
let scanned = 0
|
|
let passes = 0
|
|
|
|
/** @type {string[]} */
|
|
let chartList = charts
|
|
if (!chartList) {
|
|
chartList = await this.listMetricPointCharts({ limit: 100_000 })
|
|
}
|
|
|
|
for (let pass = 0; pass < maxPasses; pass++) {
|
|
passes++
|
|
let passDeleted = 0
|
|
for (const chart of chartList) {
|
|
const rows = await this.db
|
|
.find(
|
|
'@peardata/metric-point',
|
|
{
|
|
gte: { chart: String(chart), ts: 0 },
|
|
lte: { chart: String(chart), ts: cutoff - 1 },
|
|
},
|
|
{ limit: limitPerChart }
|
|
)
|
|
.toArray()
|
|
scanned += rows.length
|
|
if (!rows.length) continue
|
|
if (dryRun) {
|
|
// Dry-run: one pass counts matching rows (full count would need multi-pass without delete)
|
|
deleted += rows.length
|
|
passDeleted += rows.length
|
|
if (rows.length >= limitPerChart) {
|
|
// still more possible — keep scanning charts, then continue passes with estimate floor
|
|
}
|
|
continue
|
|
}
|
|
const tx = await this.db.exclusiveTransaction()
|
|
try {
|
|
for (const r of rows) {
|
|
await tx.delete('@peardata/metric-point', { chart: r.chart, ts: r.ts })
|
|
deleted++
|
|
passDeleted++
|
|
}
|
|
await tx.flush()
|
|
} catch (err) {
|
|
await tx.close().catch(() => {})
|
|
throw err
|
|
}
|
|
}
|
|
if (dryRun) break
|
|
if (passDeleted === 0) break
|
|
}
|
|
return { deleted, scanned, passes }
|
|
}
|
|
|
|
/**
|
|
* If total sampled points exceed maxPoints, delete oldest first (multi-pass for large overages).
|
|
* @param {number} maxPoints
|
|
* @param {{ dryRun?: boolean, charts?: string[], maxPasses?: number }} [opts]
|
|
*/
|
|
async enforceMetricPointCap(maxPoints, opts = {}) {
|
|
await this.ready()
|
|
const max = Math.floor(Number(maxPoints))
|
|
if (!Number.isFinite(max) || max <= 0) return { deleted: 0, scanned: 0, passes: 0 }
|
|
|
|
const limit = 100_000
|
|
const maxPasses = Math.min(Math.max(opts.maxPasses || 20, 1), 100)
|
|
let deleted = 0
|
|
let scanned = 0
|
|
let passes = 0
|
|
|
|
for (let pass = 0; pass < maxPasses; pass++) {
|
|
passes++
|
|
const rows = await this.db
|
|
.find(
|
|
'@peardata/metric-point',
|
|
{
|
|
gte: { chart: '', ts: 0 },
|
|
lte: { chart: '\uffff', ts: Number.MAX_SAFE_INTEGER },
|
|
},
|
|
{ limit }
|
|
)
|
|
.toArray()
|
|
scanned += rows.length
|
|
if (rows.length <= max) break
|
|
|
|
rows.sort((a, b) => a.ts - b.ts)
|
|
const over = rows.length - max
|
|
const victims = rows.slice(0, over)
|
|
if (opts.dryRun) {
|
|
deleted += victims.length
|
|
break
|
|
}
|
|
|
|
const tx = await this.db.exclusiveTransaction()
|
|
try {
|
|
for (const r of victims) {
|
|
await tx.delete('@peardata/metric-point', { chart: r.chart, ts: r.ts })
|
|
}
|
|
await tx.flush()
|
|
deleted += victims.length
|
|
} catch (err) {
|
|
await tx.close().catch(() => {})
|
|
throw err
|
|
}
|
|
|
|
// If we deleted the full sample overage but sample was capped, loop again
|
|
if (rows.length < limit) break
|
|
}
|
|
return { deleted, scanned, passes }
|
|
}
|
|
|
|
/**
|
|
* Reclaim physical Hypercore/RocksDB space after logical HyperDB deletes.
|
|
* Hyperbee is append-only: delete appends tombstones. clearUnlinked drops
|
|
* unlinked blocks; compact runs RocksDB compactRange so free space is real.
|
|
*
|
|
* @param {{ batchSize?: number }} [opts]
|
|
*/
|
|
async reclaimStorage(opts = {}) {
|
|
await this.ready()
|
|
const bee = this.db.db
|
|
const core = this.db.core
|
|
const batchSize = Math.min(Math.max(opts.batchSize || 4096, 256), 65_536)
|
|
let clearMs = 0
|
|
let compactMs = 0
|
|
let cleared = false
|
|
let compacted = false
|
|
|
|
if (bee && typeof bee.clearUnlinked === 'function') {
|
|
const t0 = Date.now()
|
|
await bee.clearUnlinked({ batchSize })
|
|
clearMs = Date.now() - t0
|
|
cleared = true
|
|
}
|
|
|
|
if (core && typeof core.compact === 'function') {
|
|
const t1 = Date.now()
|
|
await core.compact()
|
|
compactMs = Date.now() - t1
|
|
compacted = true
|
|
}
|
|
|
|
return {
|
|
cleared,
|
|
compacted,
|
|
clearMs,
|
|
compactMs,
|
|
coreLength: core?.length ?? null,
|
|
batchSize,
|
|
}
|
|
}
|
|
|
|
// ── jobs ──────────────────────────────────────────────────
|
|
|
|
async putJob(job) {
|
|
await this.ready()
|
|
const tx = await this.db.exclusiveTransaction()
|
|
try {
|
|
await tx.insert('@peardata/job', {
|
|
id: String(job.id),
|
|
name: String(job.name),
|
|
status: String(job.status),
|
|
startedAt: job.startedAt ?? null,
|
|
finishedAt: job.finishedAt ?? null,
|
|
resultJson: job.result != null ? JSON.stringify(job.result) : job.resultJson || null,
|
|
error: job.error || null,
|
|
})
|
|
await tx.flush()
|
|
} catch (err) {
|
|
await tx.close().catch(() => {})
|
|
throw err
|
|
}
|
|
}
|
|
|
|
async listJobs(limit = 50) {
|
|
await this.ready()
|
|
const rows = await this.db.find('@peardata/job', {}, { reverse: true, limit }).toArray()
|
|
return rows
|
|
}
|
|
}
|
|
|
|
function safeJson(s) {
|
|
try {
|
|
return JSON.parse(s || '{}')
|
|
} catch {
|
|
return {}
|
|
}
|
|
}
|