Add core.status RPC, RPC invite diagnostics, and WebSocket stats refresh

- core.status handler and coreStatusRequest; diagnoseInviteIssuesAsync
  clears stale failed-invite flags when peers report canProvideInvite
- Admin invite diagnostics and Core stats UI: per-peer remote state,
  text status marks, recommendations aligned with invite.deliver RPC
- Extract stats-collector; push stats/health/status via subscribe-stats
  WebSocket; HTTP only on first Stats tab load
- Document core.status in PLUGIN_SDK; proxy registers onCoreStatus
This commit is contained in:
Raven Scott
2026-05-28 11:35:01 -04:00
parent 1d309aa82d
commit 8854365add
18 changed files with 1243 additions and 665 deletions
@@ -573,7 +573,9 @@ async function handleDiagnosticsRoutes(req, res) {
return true;
}
const diagnostics = state.diagnoseInviteIssues();
const diagnostics = typeof state.diagnoseInviteIssuesAsync === 'function'
? await state.diagnoseInviteIssuesAsync()
: state.diagnoseInviteIssues();
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify(diagnostics));
trackRequest(urlPath, true);
+5 -557
View File
@@ -1,37 +1,7 @@
const dgram = require('dgram');
const state = require('../../../infrastructure/state');
const { getMetrics, getHistoricalData, trackRequestWithTiming, trackRequest } = require('../../../maintenance/metrics');
const { getHashForDomain } = require('../../../core/core');
const { logDebug, logError } = require('../../../infrastructure/logger');
const { trackRequestWithTiming, trackRequest } = require('../../../maintenance/metrics');
const { logError } = require('../../../infrastructure/logger');
const { createErrorResponse } = require('../../../infrastructure/error_handler');
const { parseMinutesToMs } = require('../../../infrastructure/utils');
const pidusage = require('pidusage');
// Import db-manager and replication-manager for HyperDB stats
let dbManager = null;
let replicationManager = null;
let driveReplicationManager = null;
let driveManager = null;
try {
dbManager = require('../../../plugins/db-manager');
} catch (e) {
logDebug('Admin', 'db-manager not available for stats');
}
try {
replicationManager = require('../../../plugins/replication-manager');
} catch (e) {
logDebug('Admin', 'replication-manager not available for stats');
}
try {
driveReplicationManager = require('../../../plugins/drive-replication-manager');
} catch (e) {
logDebug('Admin', 'drive-replication-manager not available for stats');
}
try {
driveManager = require('../../../plugins/drive-manager');
} catch (e) {
logDebug('Admin', 'drive-manager not available for stats');
}
const { collectAdminStats, collectHistoricalMinutes } = require('../stats-collector');
async function handleStatsRoutes(req, res) {
const urlPath = req.urlPath || new URL(req.url, `https://${req.headers.host}`).pathname;
@@ -40,282 +10,7 @@ async function handleStatsRoutes(req, res) {
if (method === 'GET' && urlPath === '/api/stats') {
try {
const startTime = Date.now();
const stats = getMetrics();
if (state.hypercoreStats && typeof state.hypercoreStats.toJson === 'function') {
try {
const { normalizeHolepunchStats } = require('../../../infrastructure/holepunch-stats-schema');
stats.holepunch = normalizeHolepunchStats(state.hypercoreStats.toJson());
} catch (err) {
logDebug('Admin', `Failed to collect hypercore stats: ${err.message}`);
}
}
const holesailChildren = [];
const pidStatsPromises = [];
const pidToChildMap = new Map();
for (const [id, child] of state.holesailChildren.entries()) {
try {
const opts = state.holesailOpts.get(id) || {};
const info = state.holesailInfos.get(id) || {};
const startTime = state.holesailChildStartTimes.get(id);
const uptime = startTime ? Date.now() - startTime : 0;
const status = child && !child.killed ? 'running' : 'stopped';
const pid = child ? child.pid : null;
if (pid && child && !child.killed) {
pidStatsPromises.push(
pidusage(pid).then(stats => ({ id, type: 'server', stats })).catch(err => {
logDebug('Admin', `Failed to get stats for server child ${id} (PID ${pid}): ${err.message}`);
return { id, type: 'server', stats: null };
})
);
}
pidToChildMap.set(id, {
id,
type: 'server',
status,
pid,
uptime,
cpuUsage: null,
memoryUsage: null,
opts: {
port: opts.port,
host: opts.host || '0.0.0.0',
protocol: opts.udp ? 'udp' : 'tcp',
secure: opts.secure || false,
domain: opts.domain || null
},
info: info
});
} catch (err) {
logError('Admin', `Error collecting stats for server child ${id}: ${err.message}`);
}
}
for (const [id, child] of state.holesailClientChildren.entries()) {
try {
const opts = state.holesailClientOpts.get(id) || {};
const info = state.holesailClientInfos.get(id) || {};
const startTime = state.holesailChildStartTimes.get(id);
const uptime = startTime ? Date.now() - startTime : 0;
const status = child && !child.killed ? 'running' : 'stopped';
const pid = child ? child.pid : null;
if (pid && child && !child.killed) {
pidStatsPromises.push(
pidusage(pid).then(stats => ({ id, type: 'client', stats })).catch(err => {
logDebug('Admin', `Failed to get stats for client child ${id} (PID ${pid}): ${err.message}`);
return { id, type: 'client', stats: null };
})
);
}
pidToChildMap.set(id, {
id,
type: 'client',
status,
pid,
uptime,
cpuUsage: null,
memoryUsage: null,
opts: {
domain: opts.domain || null,
port: opts.port,
host: opts.host || null,
protocol: opts.protocol || 'tcp'
},
info: info
});
} catch (err) {
logError('Admin', `Error collecting stats for client child ${id}: ${err.message}`);
}
}
const pidStatsResults = await Promise.all(pidStatsPromises);
for (const result of pidStatsResults) {
if (result.stats) {
const childData = pidToChildMap.get(result.id);
if (childData) {
childData.cpuUsage = {
user: result.stats.cpu / 2,
system: result.stats.cpu / 2,
percentage: result.stats.cpu
};
childData.memoryUsage = {
rss: result.stats.memory,
heapUsed: result.stats.memory * 0.8,
heapTotal: result.stats.memory,
external: 0,
arrayBuffers: 0
};
}
}
}
for (const childData of pidToChildMap.values()) {
holesailChildren.push(childData);
}
const managedConnections = new Set();
for (const child of holesailChildren) {
if (child.opts && child.opts.domain && child.opts.port) {
managedConnections.add(`${child.opts.domain}:${child.opts.port}`);
}
}
const p2pDomainEntries = [];
const hashPromises = [];
for (const [key, holesail] of state.holesails.entries()) {
try {
if (managedConnections.has(key)) {
continue;
}
const keyParts = key.split(':');
if (keyParts.length !== 2) {
logDebug('Admin', `Skipping invalid holesail key format: ${key}`);
continue;
}
const domain = keyParts[0];
const port = parseInt(keyParts[1], 10);
if (isNaN(port)) {
logDebug('Admin', `Skipping holesail key with invalid port: ${key}`);
continue;
}
const ip = state.domainToIPMap.get ? state.domainToIPMap.get(domain) : state.domainToIPMap[domain];
const isPersistent = state.persistentConnections && state.persistentConnections.has(key);
const status = holesail && typeof holesail === 'object' ? 'running' : 'stopped';
let holesailInfo = null;
try {
if (holesail && typeof holesail === 'object' && holesail.info) {
holesailInfo = holesail.info;
}
} catch (e) {
// Info not available
}
p2pDomainEntries.push({
key,
domain,
port,
ip,
isPersistent,
status,
holesailInfo
});
hashPromises.push(
getHashForDomain(domain).catch(err => {
logDebug('Admin', `Could not get hash for domain ${domain}: ${err.message}`);
return null;
})
);
} catch (err) {
logError('Admin', `Error processing p2p domain connection ${key}: ${err.message}`);
}
}
const hashResults = await Promise.all(hashPromises);
let mainProcessStats = null;
try {
mainProcessStats = await pidusage(process.pid);
} catch (err) {
logDebug('Admin', `Failed to get main process stats for p2p connections: ${err.message}`);
}
const p2pConnectionCount = p2pDomainEntries.length || 1;
let perConnectionCpu = null;
let perConnectionMemory = null;
if (mainProcessStats) {
perConnectionCpu = {
user: mainProcessStats.cpu / (p2pConnectionCount * 2),
system: mainProcessStats.cpu / (p2pConnectionCount * 2),
percentage: mainProcessStats.cpu / p2pConnectionCount
};
perConnectionMemory = {
rss: Math.floor(mainProcessStats.memory / p2pConnectionCount),
heapUsed: Math.floor((mainProcessStats.memory * 0.8) / p2pConnectionCount),
heapTotal: Math.floor(mainProcessStats.memory / p2pConnectionCount),
external: 0,
arrayBuffers: 0
};
}
for (let i = 0; i < p2pDomainEntries.length; i++) {
try {
const entry = p2pDomainEntries[i];
const hash = hashResults[i];
const startTime = state.holesailStartTimes && state.holesailStartTimes.get(entry.key);
const uptime = startTime ? Date.now() - startTime : 0;
let timeRemaining = null;
if (!entry.isPersistent && process.env.FULL_PERSISTENCE !== 'true' && startTime) {
const timeoutDuration = parseMinutesToMs(process.env.HOLESAIL_TIMEOUT || '5');
const elapsed = Date.now() - startTime;
const remaining = Math.max(0, timeoutDuration - elapsed);
timeRemaining = remaining;
}
holesailChildren.push({
id: entry.key,
type: 'p2p-domain',
status: entry.status,
pid: process.pid,
uptime: uptime,
timeRemaining: timeRemaining,
cpuUsage: perConnectionCpu,
memoryUsage: perConnectionMemory,
opts: {
domain: entry.domain,
port: entry.port,
host: entry.ip || null,
protocol: 'tcp'
},
persistent: entry.isPersistent,
hash: hash || null,
info: entry.holesailInfo || null
});
} catch (err) {
logError('Admin', `Error creating stats entry for p2p domain connection ${p2pDomainEntries[i].key}: ${err.message}`);
}
}
stats.holesailChildren = holesailChildren;
// Collect Peer Channels stats
const peerChannelsStats = collectPeerChannelsStats();
stats.peerChannels = peerChannelsStats;
// Collect HyperDB stats
const hyperdbStats = collectHyperDBStats();
stats.hyperdb = hyperdbStats;
// Collect Hyperdrive stats
const hyperdriveStats = collectHyperdriveStats();
stats.hyperdrive = hyperdriveStats;
if (typeof state.diagnoseInviteIssues === 'function') {
try {
stats.core = state.diagnoseInviteIssues();
} catch (err) {
logDebug('Admin', `Core RPC stats unavailable: ${err.message}`);
stats.core = { initializing: true, error: err.message };
}
} else {
stats.core = { initializing: true };
}
const stats = await collectAdminStats();
const responseTime = Date.now() - startTime;
trackRequestWithTiming('/api/stats', true, responseTime);
res.writeHead(200, { 'Content-Type': 'application/json' });
@@ -339,12 +34,10 @@ async function handleStatsRoutes(req, res) {
const minutesParam = params.get('minutes');
if (minutesParam) {
minutes = parseInt(minutesParam, 10);
if (isNaN(minutes) || minutes < 1) minutes = 60;
if (minutes > 1440) minutes = 1440;
}
}
const startTime = Date.now();
const historical = getHistoricalData(minutes);
const historical = collectHistoricalMinutes(minutes);
const responseTime = Date.now() - startTime;
trackRequestWithTiming('/api/stats/historical', true, responseTime);
res.writeHead(200, { 'Content-Type': 'application/json' });
@@ -362,249 +55,4 @@ async function handleStatsRoutes(req, res) {
return false;
}
/**
* Collect comprehensive Peer Channels statistics
*/
function collectPeerChannelsStats() {
const stats = {
totalPlugins: 0,
totalProtocols: 0,
totalPeerConnections: 0,
openChannels: 0,
closedChannels: 0,
plugins: []
};
try {
if (!state.pluginChannels || state.pluginChannels.size === 0) {
return stats;
}
stats.totalPlugins = state.pluginChannels.size;
for (const [pluginDomain, protocolMap] of state.pluginChannels.entries()) {
const handlers = state.pluginChannelHandlers?.get(pluginDomain);
const pluginInfo = {
domain: pluginDomain,
protocols: []
};
for (const [protocol, channelInfo] of protocolMap.entries()) {
stats.totalProtocols++;
const handler = handlers?.get(protocol);
const protocolInfo = {
name: protocol,
fullName: `${pluginDomain}-${protocol}`,
encoding: handler?.encoding || 'unknown',
autoReconnect: handler?.autoReconnect !== false,
peerCount: 0,
openCount: 0,
closedCount: 0,
peers: []
};
if (channelInfo.peerChannels) {
protocolInfo.peerCount = channelInfo.peerChannels.size;
stats.totalPeerConnections += channelInfo.peerChannels.size;
for (const [peerId, peerChannel] of channelInfo.peerChannels.entries()) {
const isOpen = peerChannel.channel?.opened || false;
const isClosed = peerChannel.channel?.closed || false;
if (isOpen) {
protocolInfo.openCount++;
stats.openChannels++;
} else {
protocolInfo.closedCount++;
stats.closedChannels++;
}
const peerInfo = {
peerId: peerId.substring(0, 16) + '...',
fullPeerId: peerId,
status: isClosed ? 'closed' : (isOpen ? 'open' : 'connecting'),
localOpened: peerChannel.localOpened || false,
remoteOpened: peerChannel.remoteOpened || false,
openedAt: peerChannel.openedAt || null,
closedAt: peerChannel.closedAt || null,
lastRemoteOpen: peerChannel.lastRemoteOpen || null,
reopenAttempts: peerChannel.reopenAttempts || 0,
lastReopenAttempt: peerChannel.lastReopenAttempt || null,
connectionValid: peerChannel.conn && !peerChannel.conn.destroyed,
muxValid: !!peerChannel.mux
};
protocolInfo.peers.push(peerInfo);
}
}
pluginInfo.protocols.push(protocolInfo);
}
stats.plugins.push(pluginInfo);
}
} catch (err) {
logError('Admin', `Error collecting peer channels stats: ${err.message}`);
}
return stats;
}
/**
* Collect comprehensive HyperDB statistics
*/
function collectHyperDBStats() {
const stats = {
totalDatabases: 0,
totalStores: 0,
replicationActive: 0,
databases: []
};
try {
if (!dbManager) {
return stats;
}
// Get all plugin stores
const pluginStores = dbManager.getAllPluginStores ? dbManager.getAllPluginStores() : new Map();
stats.totalStores = pluginStores.size;
for (const [pluginDomain, store] of pluginStores.entries()) {
const dbInfo = {
pluginDomain,
pluginVersion: null,
storeReady: false,
coreKey: null,
coreKeyShort: null,
databaseOpen: false,
replicationActive: false,
writable: false,
length: 0
};
try {
// Get plugin version from config
try {
const pluginHandler = require('../../../plugins/plugin-handler');
const plugin = pluginHandler.getPlugin(pluginDomain);
if (plugin && plugin.config && plugin.config.version) {
dbInfo.pluginVersion = plugin.config.version;
}
} catch (err) {
// Ignore version lookup errors
}
// Check store status
if (store && !store.closed) {
dbInfo.storeReady = true;
}
// Get core key
const coreKey = dbManager.getPluginCoreKey ? dbManager.getPluginCoreKey(pluginDomain) : null;
if (coreKey) {
dbInfo.coreKey = coreKey.toString('hex');
dbInfo.coreKeyShort = coreKey.toString('hex').substring(0, 16) + '...';
}
// Get core instance for more details
const core = dbManager.getPluginCore ? dbManager.getPluginCore(pluginDomain) : null;
if (core) {
dbInfo.writable = core.writable || false;
dbInfo.length = core.length || 0;
}
// Get database instance
const db = dbManager.getDatabaseInstance ? dbManager.getDatabaseInstance(pluginDomain) : null;
if (db && !db.closed) {
dbInfo.databaseOpen = true;
stats.totalDatabases++;
}
// Check replication status
if (replicationManager && replicationManager.isReplicationActive) {
dbInfo.replicationActive = replicationManager.isReplicationActive(pluginDomain);
if (dbInfo.replicationActive) {
stats.replicationActive++;
}
}
} catch (err) {
logDebug('Admin', `Error getting HyperDB info for ${pluginDomain}: ${err.message}`);
}
stats.databases.push(dbInfo);
}
} catch (err) {
logError('Admin', `Error collecting HyperDB stats: ${err.message}`);
}
return stats;
}
/**
* Collect Hyperdrive statistics
*/
function collectHyperdriveStats() {
const stats = {
totalDrives: 0,
replicationActive: 0,
drives: []
};
try {
if (!driveManager) {
return stats;
}
// Get all drives from drive manager
const allDrives = driveManager.getAllDriveInstances ? driveManager.getAllDriveInstances() : new Map();
stats.totalDrives = allDrives.size;
for (const [driveKey, driveInfo] of allDrives.entries()) {
const { pluginDomain, driveName, drive } = driveInfo;
const info = {
pluginDomain,
driveName,
driveReady: false,
discoveryKey: null,
discoveryKeyShort: null,
replicationActive: false,
writable: false,
version: 0
};
try {
if (drive) {
info.driveReady = !drive.closed;
info.writable = drive.writable || false;
info.version = drive.version || 0;
if (drive.discoveryKey) {
info.discoveryKey = drive.discoveryKey.toString('hex');
info.discoveryKeyShort = drive.discoveryKey.toString('hex').substring(0, 16) + '...';
}
}
// Check replication status
if (driveReplicationManager && driveReplicationManager.isReplicationActive) {
info.replicationActive = driveReplicationManager.isReplicationActive(pluginDomain, driveName);
if (info.replicationActive) {
stats.replicationActive++;
}
}
} catch (err) {
logDebug('Admin', `Error getting Hyperdrive info for ${pluginDomain}/${driveName}: ${err.message}`);
}
stats.drives.push(info);
}
} catch (err) {
logError('Admin', `Error collecting Hyperdrive stats: ${err.message}`);
}
return stats;
}
module.exports = { handleStatsRoutes };