reorg
This commit is contained in:
Raven Scott
@@ -0,0 +1,750 @@
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const { exec } = require('child_process');
|
||||
const { promisify } = require('util');
|
||||
const { logInfo, logError, logWarn, logDebug } = require('../infrastructure/logger');
|
||||
const state = require('../infrastructure/state');
|
||||
const { parseMinutesToMs } = require('../infrastructure/utils');
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
const BACKUP_DIR = process.env.BACKUP_DIR || './backups';
|
||||
const BACKUP_INTERVAL = parseMinutesToMs(process.env.BACKUP_INTERVAL || '720'); // Default: 12 hours = 720 minutes
|
||||
|
||||
// Get backup retention from state (defaults to 25, or env var if state not initialized)
|
||||
function getBackupRetention() {
|
||||
return state.backupRetentionCount || parseInt(process.env.BACKUP_RETENTION || '25', 10);
|
||||
}
|
||||
|
||||
// Files and directories to backup
|
||||
const FILES_TO_BACKUP = [
|
||||
process.env.DOMAINS_FILE || './cache/domains.json',
|
||||
process.env.LOCAL_DNS_FILE || './cache/local_dns.json',
|
||||
process.env.SELECTOR_CACHE_FILE || './cache/selector_cache.json',
|
||||
process.env.HOLESAIL_SERVERS_FILE || './cache/holesail_servers.json',
|
||||
process.env.HOLESAIL_CLIENTS_FILE || './cache/holesail_clients.json',
|
||||
process.env.BLOCKED_PEERS_FILE || './cache/blocked_peers.json',
|
||||
process.env.PEER_METRICS_FILE || './cache/peer_metrics.json',
|
||||
process.env.PEER_HISTORY_FILE || './cache/peer_history.json',
|
||||
process.env.SUBSCRIPTIONS_FILE || './cache/subscriptions.json',
|
||||
'./cache/keypair.json', // Persistent hyperswarm keypair - critical for peer identity
|
||||
];
|
||||
|
||||
const DIRS_TO_BACKUP = [
|
||||
process.env.CERTS_DIR || './certs',
|
||||
];
|
||||
|
||||
const OPTIONAL_FILES = [
|
||||
'./p2ns.json',
|
||||
'./.env',
|
||||
];
|
||||
|
||||
let backupInterval = null;
|
||||
|
||||
/**
|
||||
* Get system state snapshot for metadata
|
||||
*/
|
||||
function getStateSnapshot() {
|
||||
try {
|
||||
const snapshot = {
|
||||
connectedPeers: state.connectedPeers ? state.connectedPeers.size : 0,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
// Try to get domain count from domains.json if accessible
|
||||
try {
|
||||
const domainsFile = process.env.DOMAINS_FILE || './cache/domains.json';
|
||||
const domains = require('fs').readFileSync(domainsFile, 'utf8');
|
||||
const domainsData = JSON.parse(domains);
|
||||
snapshot.domainsCount = Array.isArray(domainsData) ? domainsData.length : 0;
|
||||
} catch (err) {
|
||||
snapshot.domainsCount = 0;
|
||||
}
|
||||
|
||||
return snapshot;
|
||||
} catch (err) {
|
||||
logWarn('Backup', `Failed to capture state snapshot: ${err.message}`);
|
||||
return {
|
||||
timestamp: new Date().toISOString(),
|
||||
connectedPeers: 0,
|
||||
domainsCount: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure backup directory exists
|
||||
*/
|
||||
async function ensureBackupDir() {
|
||||
try {
|
||||
await fs.mkdir(BACKUP_DIR, { recursive: true });
|
||||
} catch (err) {
|
||||
logError('Backup', `Failed to create backup directory: ${err.message}`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up leftover extract directories
|
||||
*/
|
||||
async function cleanupExtractDirectories() {
|
||||
try {
|
||||
const entries = await fs.readdir(BACKUP_DIR, { withFileTypes: true });
|
||||
const extractDirs = entries.filter(
|
||||
entry => entry.isDirectory() && (entry.name.startsWith('extract-') || entry.name.startsWith('extract-metadata-'))
|
||||
);
|
||||
|
||||
if (extractDirs.length > 0) {
|
||||
logDebug('Backup', `Cleaning up ${extractDirs.length} leftover extract directory(ies)`);
|
||||
for (const dir of extractDirs) {
|
||||
try {
|
||||
await fs.rm(path.join(BACKUP_DIR, dir.name), { recursive: true, force: true });
|
||||
logDebug('Backup', `Removed leftover extract directory: ${dir.name}`);
|
||||
} catch (err) {
|
||||
logWarn('Backup', `Failed to remove extract directory ${dir.name}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logWarn('Backup', `Failed to cleanup extract directories: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a path exists
|
||||
*/
|
||||
async function pathExists(filePath) {
|
||||
try {
|
||||
await fs.access(filePath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a backup of all configured files and directories as tar.gz
|
||||
* @returns {Promise<string>} Path to backup archive
|
||||
*/
|
||||
async function createBackup() {
|
||||
try {
|
||||
await ensureBackupDir();
|
||||
|
||||
// Clean up any leftover extract directories before creating backup
|
||||
await cleanupExtractDirectories();
|
||||
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const backupName = `backup-${timestamp}.tar.gz`;
|
||||
const backupPath = path.join(BACKUP_DIR, backupName);
|
||||
|
||||
logInfo('Backup', `Creating backup: ${backupName}`);
|
||||
|
||||
// Create temporary directory for backup contents
|
||||
const tempDir = path.join(BACKUP_DIR, `temp-${timestamp}`);
|
||||
await fs.mkdir(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
const backedUpItems = [];
|
||||
|
||||
// Resolve paths relative to project root (where p2ns.js is located)
|
||||
// Find project root by going up from this file's location
|
||||
const projectRoot = path.resolve(__dirname, '../..');
|
||||
|
||||
// Backup individual files
|
||||
for (const filePath of FILES_TO_BACKUP) {
|
||||
try {
|
||||
// Resolve path relative to project root to ensure consistency
|
||||
let fullPath;
|
||||
if (path.isAbsolute(filePath)) {
|
||||
fullPath = filePath;
|
||||
} else {
|
||||
fullPath = path.resolve(projectRoot, filePath);
|
||||
}
|
||||
|
||||
if (await pathExists(fullPath)) {
|
||||
const fileName = path.basename(fullPath);
|
||||
const destPath = path.join(tempDir, 'cache', fileName);
|
||||
await fs.mkdir(path.dirname(destPath), { recursive: true });
|
||||
await fs.copyFile(fullPath, destPath);
|
||||
backedUpItems.push(filePath);
|
||||
logDebug('Backup', `Backed up ${filePath}`);
|
||||
} else {
|
||||
logDebug('Backup', `File ${filePath} (resolved: ${fullPath}) does not exist, skipping`);
|
||||
}
|
||||
} catch (err) {
|
||||
logWarn('Backup', `Failed to backup ${filePath}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Backup optional files
|
||||
for (const filePath of OPTIONAL_FILES) {
|
||||
try {
|
||||
// Resolve path relative to project root to ensure consistency
|
||||
let fullPath;
|
||||
if (path.isAbsolute(filePath)) {
|
||||
fullPath = filePath;
|
||||
} else {
|
||||
fullPath = path.resolve(projectRoot, filePath);
|
||||
}
|
||||
|
||||
if (await pathExists(fullPath)) {
|
||||
const fileName = path.basename(fullPath);
|
||||
// Handle .env specially - create sanitized copy
|
||||
if (fileName === '.env') {
|
||||
const envContent = await fs.readFile(fullPath, 'utf8');
|
||||
// Don't sanitize, just copy as-is (user can sanitize manually if needed)
|
||||
const destPath = path.join(tempDir, fileName);
|
||||
await fs.writeFile(destPath, envContent);
|
||||
backedUpItems.push(filePath);
|
||||
logDebug('Backup', `Backed up ${filePath}`);
|
||||
} else {
|
||||
const destPath = path.join(tempDir, fileName);
|
||||
await fs.copyFile(fullPath, destPath);
|
||||
backedUpItems.push(filePath);
|
||||
logDebug('Backup', `Backed up ${filePath}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logDebug('Backup', `Optional file ${filePath} not found or failed: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Backup directories
|
||||
// Use projectRoot already defined above
|
||||
const dirsToBackup = DIRS_TO_BACKUP;
|
||||
|
||||
logInfo('Backup', `Backing up ${dirsToBackup.length} directories: ${dirsToBackup.join(', ')}`);
|
||||
|
||||
for (const dirPath of dirsToBackup) {
|
||||
try {
|
||||
// Resolve path relative to project root to ensure consistency
|
||||
let fullPath;
|
||||
if (path.isAbsolute(dirPath)) {
|
||||
fullPath = dirPath;
|
||||
} else {
|
||||
// Resolve relative to project root
|
||||
fullPath = path.resolve(projectRoot, dirPath);
|
||||
}
|
||||
|
||||
if (await pathExists(fullPath)) {
|
||||
const dirName = path.basename(fullPath);
|
||||
const destDir = path.join(tempDir, dirName);
|
||||
|
||||
// Copy directory recursively
|
||||
await copyDirectory(fullPath, destDir);
|
||||
backedUpItems.push(dirPath);
|
||||
logInfo('Backup', `Successfully backed up directory: ${dirPath} (${fullPath}) -> ${destDir}`);
|
||||
} else {
|
||||
logWarn('Backup', `Directory ${dirPath} (resolved: ${fullPath}) does not exist, skipping`);
|
||||
}
|
||||
} catch (err) {
|
||||
logError('Backup', `Failed to backup directory ${dirPath}: ${err.message}`);
|
||||
// Don't throw - continue with other directories
|
||||
}
|
||||
}
|
||||
|
||||
// Get state snapshot
|
||||
const stateSnapshot = getStateSnapshot();
|
||||
|
||||
// Create metadata file - ensure all backed up items are properly categorized
|
||||
// Files are items from FILES_TO_BACKUP or OPTIONAL_FILES
|
||||
// Directories are items from DIRS_TO_BACKUP
|
||||
const backedUpFiles = backedUpItems.filter(item =>
|
||||
FILES_TO_BACKUP.includes(item) || OPTIONAL_FILES.includes(item)
|
||||
);
|
||||
|
||||
// Include all directories that were backed up
|
||||
const backedUpDirectories = backedUpItems.filter(item =>
|
||||
DIRS_TO_BACKUP.includes(item)
|
||||
);
|
||||
|
||||
// Create metadata file
|
||||
const metadata = {
|
||||
timestamp: new Date().toISOString(),
|
||||
files: backedUpFiles,
|
||||
directories: backedUpDirectories,
|
||||
version: require('../../package.json').version || 'unknown',
|
||||
format: 'tar.gz',
|
||||
state: stateSnapshot,
|
||||
};
|
||||
|
||||
// Log what was backed up for debugging
|
||||
logInfo('Backup', `Backup metadata - Files: ${backedUpFiles.length}, Directories: ${backedUpDirectories.length}`);
|
||||
if (backedUpDirectories.length > 0) {
|
||||
logInfo('Backup', `Backed up directories: ${backedUpDirectories.join(', ')}`);
|
||||
}
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(tempDir, 'metadata.json'),
|
||||
JSON.stringify(metadata, null, 2)
|
||||
);
|
||||
|
||||
// Create tar.gz archive
|
||||
const tempDirName = path.basename(tempDir);
|
||||
const parentDir = path.dirname(tempDir);
|
||||
|
||||
try {
|
||||
// Use tar command to create compressed archive
|
||||
// Use backupName (not backupPath) since we're cd'ing into parentDir
|
||||
await execAsync(`cd "${parentDir}" && tar -czf "${backupName}" "${tempDirName}"`);
|
||||
logDebug('Backup', `Created tar.gz archive: ${backupPath}`);
|
||||
|
||||
// Remove temporary directory
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
} catch (err) {
|
||||
// Fallback: try without cd if that fails
|
||||
logWarn('Backup', `Failed to create tar.gz with cd, trying alternative method: ${err.message}`);
|
||||
try {
|
||||
await execAsync(`tar -czf "${backupPath}" -C "${parentDir}" "${tempDirName}"`);
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
} catch (err2) {
|
||||
// Clean up temp dir even if tar fails
|
||||
await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {});
|
||||
throw new Error(`Failed to create tar.gz archive: ${err2.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
logInfo('Backup', `Backup completed: ${backupName}`);
|
||||
return backupPath;
|
||||
} catch (err) {
|
||||
// Clean up temp directory on error
|
||||
await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {});
|
||||
throw err;
|
||||
}
|
||||
} catch (err) {
|
||||
logError('Backup', `Backup failed: ${err.message}`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy directory recursively
|
||||
*/
|
||||
async function copyDirectory(src, dest) {
|
||||
await fs.mkdir(dest, { recursive: true });
|
||||
const entries = await fs.readdir(src, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const srcPath = path.join(src, entry.name);
|
||||
const destPath = path.join(dest, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await copyDirectory(srcPath, destPath);
|
||||
} else {
|
||||
await fs.copyFile(srcPath, destPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract tar.gz archive to temporary directory
|
||||
* @returns {Promise<{extractDir: string, contentDir: string}>} Object with extractDir (for cleanup) and contentDir (for access)
|
||||
*/
|
||||
async function extractArchive(archivePath) {
|
||||
const extractDir = path.join(BACKUP_DIR, `extract-${Date.now()}`);
|
||||
await fs.mkdir(extractDir, { recursive: true });
|
||||
|
||||
try {
|
||||
await execAsync(`tar -xzf "${archivePath}" -C "${extractDir}"`);
|
||||
|
||||
// Find the extracted directory (tar extracts with the directory name)
|
||||
const entries = await fs.readdir(extractDir, { withFileTypes: true });
|
||||
let contentDir = extractDir;
|
||||
if (entries.length === 1 && entries[0].isDirectory()) {
|
||||
contentDir = path.join(extractDir, entries[0].name);
|
||||
}
|
||||
// Return both paths: extractDir for cleanup, contentDir for accessing content
|
||||
return { extractDir, contentDir };
|
||||
} catch (err) {
|
||||
await fs.rm(extractDir, { recursive: true, force: true }).catch(() => {});
|
||||
throw new Error(`Failed to extract archive: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List all available backups (both directory and tar.gz formats)
|
||||
* @returns {Promise<Array>} Array of backup metadata
|
||||
*/
|
||||
async function listBackups() {
|
||||
try {
|
||||
await ensureBackupDir();
|
||||
// Clean up any leftover extract directories
|
||||
await cleanupExtractDirectories();
|
||||
const entries = await fs.readdir(BACKUP_DIR, { withFileTypes: true });
|
||||
const backups = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
// Skip temporary directories
|
||||
if (entry.name.startsWith('temp-') || entry.name.startsWith('extract-')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let backupPath, metadataPath, isTarGz;
|
||||
|
||||
if (entry.isDirectory() && entry.name.startsWith('backup-')) {
|
||||
// Old directory format
|
||||
backupPath = path.join(BACKUP_DIR, entry.name);
|
||||
metadataPath = path.join(backupPath, 'metadata.json');
|
||||
isTarGz = false;
|
||||
} else if (entry.isFile() && entry.name.startsWith('backup-') && entry.name.endsWith('.tar.gz')) {
|
||||
// New tar.gz format
|
||||
backupPath = path.join(BACKUP_DIR, entry.name);
|
||||
isTarGz = true;
|
||||
|
||||
// Extract metadata from tar.gz
|
||||
try {
|
||||
const { extractDir, contentDir } = await extractArchive(backupPath);
|
||||
metadataPath = path.join(contentDir, 'metadata.json');
|
||||
|
||||
// Read metadata
|
||||
const metadataContent = await fs.readFile(metadataPath, 'utf8');
|
||||
const metadata = JSON.parse(metadataContent);
|
||||
|
||||
// Get file size
|
||||
const stats = await fs.stat(backupPath);
|
||||
|
||||
// Clean up extraction (use extractDir to remove the entire extract directory)
|
||||
await fs.rm(extractDir, { recursive: true, force: true });
|
||||
|
||||
// Extract timestamp from filename if not in metadata
|
||||
let timestamp = metadata.timestamp;
|
||||
if (!timestamp) {
|
||||
const match = entry.name.match(/backup-(.+?)\.tar\.gz/);
|
||||
if (match) {
|
||||
timestamp = match[1].replace(/-/g, ':');
|
||||
}
|
||||
}
|
||||
|
||||
backups.push({
|
||||
name: entry.name,
|
||||
path: backupPath,
|
||||
timestamp: timestamp || new Date().toISOString(),
|
||||
files: metadata.files || [],
|
||||
directories: metadata.directories || [],
|
||||
version: metadata.version || 'unknown',
|
||||
format: 'tar.gz',
|
||||
size: stats.size,
|
||||
state: metadata.state || {},
|
||||
});
|
||||
} catch (err) {
|
||||
logWarn('Backup', `Failed to read metadata from ${entry.name}: ${err.message}`);
|
||||
// Fallback: use filename for timestamp
|
||||
const match = entry.name.match(/backup-(.+?)\.tar\.gz/);
|
||||
const timestamp = match ? match[1].replace(/-/g, ':') : new Date().toISOString();
|
||||
const stats = await fs.stat(backupPath).catch(() => ({ size: 0 }));
|
||||
backups.push({
|
||||
name: entry.name,
|
||||
path: backupPath,
|
||||
timestamp: timestamp,
|
||||
files: [],
|
||||
directories: [],
|
||||
version: 'unknown',
|
||||
format: 'tar.gz',
|
||||
size: stats.size || 0,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle directory format backups
|
||||
try {
|
||||
if (await pathExists(metadataPath)) {
|
||||
const metadata = JSON.parse(await fs.readFile(metadataPath, 'utf8'));
|
||||
|
||||
// Calculate size for directory backups
|
||||
let totalSize = 0;
|
||||
try {
|
||||
const dirStats = await getDirectorySize(backupPath);
|
||||
totalSize = dirStats;
|
||||
} catch (err) {
|
||||
logDebug('Backup', `Failed to calculate size for ${entry.name}: ${err.message}`);
|
||||
}
|
||||
|
||||
backups.push({
|
||||
name: entry.name,
|
||||
path: backupPath,
|
||||
timestamp: metadata.timestamp || entry.name.replace('backup-', '').replace(/-/g, ':'),
|
||||
files: metadata.files || [],
|
||||
directories: metadata.directories || [],
|
||||
version: metadata.version || 'unknown',
|
||||
format: 'directory',
|
||||
size: totalSize,
|
||||
state: metadata.state || {},
|
||||
});
|
||||
} else {
|
||||
// No metadata, use filename
|
||||
const timestamp = entry.name.replace('backup-', '').replace(/-/g, ':');
|
||||
let totalSize = 0;
|
||||
try {
|
||||
totalSize = await getDirectorySize(backupPath);
|
||||
} catch (err) {
|
||||
logDebug('Backup', `Failed to calculate size for ${entry.name}: ${err.message}`);
|
||||
}
|
||||
backups.push({
|
||||
name: entry.name,
|
||||
path: backupPath,
|
||||
timestamp: timestamp,
|
||||
files: [],
|
||||
directories: [],
|
||||
version: 'unknown',
|
||||
format: 'directory',
|
||||
size: totalSize,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
logWarn('Backup', `Failed to read metadata for ${entry.name}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by timestamp (newest first)
|
||||
backups.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
|
||||
return backups;
|
||||
} catch (err) {
|
||||
logError('Backup', `Failed to list backups: ${err.message}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total size of directory
|
||||
*/
|
||||
async function getDirectorySize(dirPath) {
|
||||
let totalSize = 0;
|
||||
const entries = await fs.readdir(dirPath, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const entryPath = path.join(dirPath, entry.name);
|
||||
try {
|
||||
if (entry.isDirectory()) {
|
||||
totalSize += await getDirectorySize(entryPath);
|
||||
} else {
|
||||
const stats = await fs.stat(entryPath);
|
||||
totalSize += stats.size;
|
||||
}
|
||||
} catch (err) {
|
||||
// Skip files we can't access
|
||||
}
|
||||
}
|
||||
|
||||
return totalSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore from a backup (supports both directory and tar.gz formats)
|
||||
* @param {string} backupName - Name of backup directory or tar.gz file
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function restoreBackup(backupName) {
|
||||
try {
|
||||
const backupPath = path.join(BACKUP_DIR, backupName);
|
||||
|
||||
// Verify backup exists
|
||||
if (!await pathExists(backupPath)) {
|
||||
throw new Error(`Backup ${backupName} does not exist`);
|
||||
}
|
||||
|
||||
logInfo('Backup', `Restoring from backup: ${backupName}`);
|
||||
|
||||
// Create a backup before restoring (safety measure)
|
||||
await createBackup();
|
||||
|
||||
let extractDir = null;
|
||||
let restoreSource = backupPath;
|
||||
let isTarGz = backupName.endsWith('.tar.gz');
|
||||
|
||||
try {
|
||||
// Extract if tar.gz
|
||||
if (isTarGz) {
|
||||
const extractResult = await extractArchive(backupPath);
|
||||
extractDir = extractResult.extractDir;
|
||||
restoreSource = extractResult.contentDir;
|
||||
}
|
||||
|
||||
// Read metadata
|
||||
const metadataPath = path.join(restoreSource, 'metadata.json');
|
||||
let metadata = {};
|
||||
if (await pathExists(metadataPath)) {
|
||||
metadata = JSON.parse(await fs.readFile(metadataPath, 'utf8'));
|
||||
}
|
||||
|
||||
const filesToRestore = metadata.files || FILES_TO_BACKUP.map(f => path.basename(f));
|
||||
const dirsToRestore = metadata.directories || DIRS_TO_BACKUP;
|
||||
|
||||
// Restore individual files
|
||||
for (const filePath of filesToRestore) {
|
||||
const fileName = path.basename(filePath);
|
||||
let sourcePath;
|
||||
|
||||
// Try cache subdirectory first (new format)
|
||||
sourcePath = path.join(restoreSource, 'cache', fileName);
|
||||
if (!await pathExists(sourcePath)) {
|
||||
// Try root (old format or direct placement)
|
||||
sourcePath = path.join(restoreSource, fileName);
|
||||
}
|
||||
|
||||
if (await pathExists(sourcePath)) {
|
||||
// Determine destination based on original file path
|
||||
let destPath;
|
||||
if (filePath.startsWith('./')) {
|
||||
destPath = path.resolve(filePath);
|
||||
} else {
|
||||
// Try to match to known files
|
||||
if (fileName === 'domains.json') {
|
||||
destPath = path.resolve(process.env.DOMAINS_FILE || './cache/domains.json');
|
||||
} else if (fileName === 'local_dns.json') {
|
||||
destPath = path.resolve(process.env.LOCAL_DNS_FILE || './cache/local_dns.json');
|
||||
} else if (fileName === 'selector_cache.json') {
|
||||
destPath = path.resolve(process.env.SELECTOR_CACHE_FILE || './cache/selector_cache.json');
|
||||
} else if (fileName === 'holesail_servers.json') {
|
||||
destPath = path.resolve(process.env.HOLESAIL_SERVERS_FILE || './cache/holesail_servers.json');
|
||||
} else if (fileName === 'holesail_clients.json') {
|
||||
destPath = path.resolve(process.env.HOLESAIL_CLIENTS_FILE || './cache/holesail_clients.json');
|
||||
} else if (fileName === 'blocked_peers.json') {
|
||||
destPath = path.resolve(process.env.BLOCKED_PEERS_FILE || './cache/blocked_peers.json');
|
||||
} else if (fileName === 'peer_metrics.json') {
|
||||
destPath = path.resolve(process.env.PEER_METRICS_FILE || './cache/peer_metrics.json');
|
||||
} else if (fileName === 'peer_history.json') {
|
||||
destPath = path.resolve(process.env.PEER_HISTORY_FILE || './cache/peer_history.json');
|
||||
} else if (fileName === 'subscriptions.json') {
|
||||
destPath = path.resolve(process.env.SUBSCRIPTIONS_FILE || './cache/subscriptions.json');
|
||||
} else if (fileName === 'keypair.json') {
|
||||
destPath = path.resolve('./cache/keypair.json');
|
||||
} else {
|
||||
destPath = path.resolve(filePath);
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure destination directory exists
|
||||
await fs.mkdir(path.dirname(destPath), { recursive: true });
|
||||
await fs.copyFile(sourcePath, destPath);
|
||||
logInfo('Backup', `Restored ${fileName} to ${destPath}`);
|
||||
} else {
|
||||
logWarn('Backup', `File ${fileName} not found in backup, skipping`);
|
||||
}
|
||||
}
|
||||
|
||||
// Restore optional files
|
||||
for (const fileName of ['p2ns.json', '.env']) {
|
||||
const sourcePath = path.join(restoreSource, fileName);
|
||||
if (await pathExists(sourcePath)) {
|
||||
const destPath = path.resolve(fileName);
|
||||
await fs.copyFile(sourcePath, destPath);
|
||||
logInfo('Backup', `Restored ${fileName}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Restore directories
|
||||
for (const dirPath of dirsToRestore) {
|
||||
const dirName = path.basename(dirPath);
|
||||
const sourceDir = path.join(restoreSource, dirName);
|
||||
|
||||
if (await pathExists(sourceDir)) {
|
||||
const destDir = path.resolve(dirPath);
|
||||
|
||||
// Remove existing directory if it exists
|
||||
if (await pathExists(destDir)) {
|
||||
await fs.rm(destDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
// Copy directory
|
||||
await fs.mkdir(path.dirname(destDir), { recursive: true });
|
||||
await copyDirectory(sourceDir, destDir);
|
||||
logInfo('Backup', `Restored directory ${dirPath}`);
|
||||
} else {
|
||||
logWarn('Backup', `Directory ${dirName} not found in backup, skipping`);
|
||||
}
|
||||
}
|
||||
|
||||
logInfo('Backup', `Restore completed from: ${backupName}`);
|
||||
} finally {
|
||||
// Clean up extraction directory
|
||||
if (extractDir) {
|
||||
await fs.rm(extractDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logError('Backup', `Restore failed: ${err.message}`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old backups (keep only last N)
|
||||
*/
|
||||
async function cleanupOldBackups() {
|
||||
try {
|
||||
const backups = await listBackups();
|
||||
const retention = getBackupRetention();
|
||||
|
||||
if (backups.length <= retention) {
|
||||
return;
|
||||
}
|
||||
|
||||
const toDelete = backups.slice(retention);
|
||||
logInfo('Backup', `Cleaning up ${toDelete.length} old backup(s) (retention: ${retention})`);
|
||||
|
||||
for (const backup of toDelete) {
|
||||
try {
|
||||
await fs.rm(backup.path, { recursive: true, force: true });
|
||||
logDebug('Backup', `Deleted old backup: ${backup.name}`);
|
||||
} catch (err) {
|
||||
logError('Backup', `Failed to delete backup ${backup.name}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logError('Backup', `Cleanup failed: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start automatic backups
|
||||
*/
|
||||
function startAutomaticBackups() {
|
||||
if (backupInterval) {
|
||||
clearInterval(backupInterval);
|
||||
}
|
||||
|
||||
// Create initial backup (after cleanup)
|
||||
(async () => {
|
||||
try {
|
||||
await cleanupOldBackups();
|
||||
await createBackup();
|
||||
} catch (err) {
|
||||
logError('Backup', `Initial backup failed: ${err.message}`);
|
||||
}
|
||||
})();
|
||||
|
||||
// Schedule periodic backups
|
||||
backupInterval = setInterval(async () => {
|
||||
try {
|
||||
// Cleanup BEFORE backup
|
||||
await cleanupOldBackups();
|
||||
await createBackup();
|
||||
} catch (err) {
|
||||
logError('Backup', `Scheduled backup failed: ${err.message}`);
|
||||
}
|
||||
}, BACKUP_INTERVAL);
|
||||
|
||||
logInfo('Backup', `Automatic backups started (interval: ${BACKUP_INTERVAL}ms, retention: ${getBackupRetention()})`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop automatic backups
|
||||
*/
|
||||
function stopAutomaticBackups() {
|
||||
if (backupInterval) {
|
||||
clearInterval(backupInterval);
|
||||
backupInterval = null;
|
||||
logInfo('Backup', 'Automatic backups stopped');
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
createBackup,
|
||||
listBackups,
|
||||
restoreBackup,
|
||||
cleanupOldBackups,
|
||||
startAutomaticBackups,
|
||||
stopAutomaticBackups,
|
||||
};
|
||||
@@ -0,0 +1,504 @@
|
||||
const { exec } = require('child_process');
|
||||
const os = require('os');
|
||||
const util = require('util');
|
||||
const net = require('net');
|
||||
const dgram = require('dgram');
|
||||
const state = require('../infrastructure/state');
|
||||
const { logDebug, logError, logInfo, logWarn } = require('../infrastructure/logger');
|
||||
|
||||
const execAsync = util.promisify(exec);
|
||||
|
||||
// Helper function to check if a port is available (TCP and UDP)
|
||||
async function checkPortAvailability(host, port) {
|
||||
const tcpPromise = new Promise((resolve, reject) => {
|
||||
const server = net.createServer();
|
||||
server.once('error', (err) => {
|
||||
server.close();
|
||||
if (err.code === 'EADDRINUSE') {
|
||||
reject(new Error(`TCP port ${port} on ${host} is already in use`));
|
||||
} else {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
server.once('listening', () => {
|
||||
server.close();
|
||||
resolve(true);
|
||||
});
|
||||
server.listen(port, host);
|
||||
});
|
||||
|
||||
const udpPromise = new Promise((resolve, reject) => {
|
||||
const socket = dgram.createSocket('udp4');
|
||||
socket.once('error', (err) => {
|
||||
socket.close();
|
||||
if (err.code === 'EADDRINUSE') {
|
||||
reject(new Error(`UDP port ${port} on ${host} is already in use`));
|
||||
} else {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
socket.once('listening', () => {
|
||||
socket.close();
|
||||
resolve(true);
|
||||
});
|
||||
socket.bind(port, host);
|
||||
});
|
||||
|
||||
try {
|
||||
await Promise.all([tcpPromise, udpPromise]);
|
||||
return true;
|
||||
} catch (err) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to wait for a port to be released (TCP and UDP)
|
||||
async function waitForPortRelease(host, port, maxAttempts = 60, delayMs = 1000) {
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
await checkPortAvailability(host, port);
|
||||
logDebug('Cleanup', `Port ${port} on ${host} is now free (attempt ${attempt})`);
|
||||
return true;
|
||||
} catch (err) {
|
||||
logDebug('Cleanup', `Port ${port} on ${host} still in use (attempt ${attempt}): ${err.message}`);
|
||||
if (attempt === maxAttempts) {
|
||||
logWarn('Cleanup', `Port ${port} on ${host} still in use after ${maxAttempts} attempts`);
|
||||
return false;
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, delayMs));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Improved Linux-compatible function to find PIDs using a port
|
||||
async function findPIDsUsingPort(host, port) {
|
||||
const pids = new Set();
|
||||
const isLinux = os.platform() === 'linux';
|
||||
|
||||
try {
|
||||
// Method 1: Use lsof (works on both platforms)
|
||||
const protocols = ['TCP', 'UDP'];
|
||||
for (const proto of protocols) {
|
||||
try {
|
||||
const { stdout } = await execAsync(`sudo lsof -i ${proto}@${host}:${port} -t`);
|
||||
stdout.trim().split('\n').filter(pid => pid).forEach(pid => pids.add(pid));
|
||||
} catch (err) {
|
||||
logDebug('Cleanup', `No ${proto} processes found with lsof for ${host}:${port}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Method 2: Linux-specific netstat + ps approach (more reliable on Linux)
|
||||
if (isLinux) {
|
||||
try {
|
||||
// Find TCP connections
|
||||
const { stdout: netstatTcp } = await execAsync(`netstat -tlnp 2>/dev/null | grep :${port} || true`);
|
||||
const tcpMatches = netstatTcp.match(/(\d+)\/\w+/g);
|
||||
if (tcpMatches) {
|
||||
tcpMatches.forEach(match => {
|
||||
const pid = match.split('/')[0];
|
||||
if (pid && pid !== '0' && pid !== '-') pids.add(pid);
|
||||
});
|
||||
}
|
||||
|
||||
// Find UDP connections
|
||||
const { stdout: netstatUdp } = await execAsync(`netstat -ulnp 2>/dev/null | grep :${port} || true`);
|
||||
const udpMatches = netstatUdp.match(/(\d+)\/\w+/g);
|
||||
if (udpMatches) {
|
||||
udpMatches.forEach(match => {
|
||||
const pid = match.split('/')[0];
|
||||
if (pid && pid !== '0' && pid !== '-') pids.add(pid);
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
logDebug('Cleanup', `netstat method failed for ${host}:${port}: ${err.message}`);
|
||||
}
|
||||
|
||||
// Method 3: Use ss command (modern Linux)
|
||||
try {
|
||||
const { stdout: ssTcp } = await execAsync(`ss -tlnp sport = :${port} 2>/dev/null || true`);
|
||||
const { stdout: ssUdp } = await execAsync(`ss -ulnp sport = :${port} 2>/dev/null || true`);
|
||||
|
||||
const extractPidsFromSs = (output) => {
|
||||
const matches = output.match(/pid=(\d+)/g);
|
||||
if (matches) {
|
||||
matches.forEach(match => {
|
||||
const pid = match.split('=')[1];
|
||||
if (pid && pid !== '0') pids.add(pid);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
extractPidsFromSs(ssTcp);
|
||||
extractPidsFromSs(ssUdp);
|
||||
} catch (err) {
|
||||
logDebug('Cleanup', `ss method failed for ${host}:${port}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(pids);
|
||||
} catch (err) {
|
||||
logError('Cleanup', `Error finding PIDs for ${host}:${port}: ${err.message}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Improved function to kill a process with better Linux support
|
||||
async function killProcess(pid, maxWaitTime = 5000) {
|
||||
try {
|
||||
// Check if process exists
|
||||
try {
|
||||
await execAsync(`ps -p ${pid}`);
|
||||
} catch (err) {
|
||||
logDebug('Cleanup', `PID ${pid} already dead`);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Send SIGTERM
|
||||
await execAsync(`sudo kill -15 ${pid}`);
|
||||
logInfo('Cleanup', `Sent SIGTERM to PID ${pid}`);
|
||||
|
||||
// Wait for graceful termination
|
||||
const startTime = Date.now();
|
||||
while (Date.now() - startTime < maxWaitTime) {
|
||||
try {
|
||||
await execAsync(`ps -p ${pid}`);
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
} catch (err) {
|
||||
logDebug('Cleanup', `PID ${pid} terminated gracefully`);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// If still running, force kill
|
||||
try {
|
||||
await execAsync(`ps -p ${pid}`);
|
||||
await execAsync(`sudo kill -9 ${pid}`);
|
||||
logWarn('Cleanup', `Forced SIGKILL for PID ${pid}`);
|
||||
|
||||
// Give it a moment to die
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
|
||||
try {
|
||||
await execAsync(`ps -p ${pid}`);
|
||||
logError('Cleanup', `PID ${pid} still alive after SIGKILL`);
|
||||
return false;
|
||||
} catch (err) {
|
||||
logDebug('Cleanup', `PID ${pid} confirmed dead after SIGKILL`);
|
||||
return true;
|
||||
}
|
||||
} catch (err) {
|
||||
logDebug('Cleanup', `PID ${pid} already dead before SIGKILL`);
|
||||
return true;
|
||||
}
|
||||
} catch (err) {
|
||||
logError('Cleanup', `Failed to kill PID ${pid}: ${err.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to free a port by killing processes using it
|
||||
async function freePort(host, port, maxRetries = 3) {
|
||||
for (let retry = 1; retry <= maxRetries; retry++) {
|
||||
logDebug('Cleanup', `Attempting to free port ${host}:${port} (attempt ${retry}/${maxRetries})`);
|
||||
|
||||
const pids = await findPIDsUsingPort(host, port);
|
||||
|
||||
if (pids.length === 0) {
|
||||
logDebug('Cleanup', `No PIDs found for ${host}:${port} (attempt ${retry})`);
|
||||
|
||||
// On Linux, check if port is actually free or just in TIME_WAIT
|
||||
const isReleased = await waitForPortRelease(host, port, 5, 500);
|
||||
if (isReleased) {
|
||||
logInfo('Cleanup', `Port ${host}:${port} is now free`);
|
||||
return true;
|
||||
} else {
|
||||
logWarn('Cleanup', `Port ${host}:${port} still appears busy despite no PIDs found`);
|
||||
}
|
||||
} else {
|
||||
logInfo('Cleanup', `Found ${pids.length} PID(s) using ${host}:${port}: ${pids.join(', ')}`);
|
||||
|
||||
// Kill all processes
|
||||
const killPromises = pids.map(pid => killProcess(pid));
|
||||
await Promise.allSettled(killPromises);
|
||||
|
||||
// Wait a bit for the port to be released
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
|
||||
// Check if port is now free
|
||||
const isReleased = await waitForPortRelease(host, port, 10, 500);
|
||||
if (isReleased) {
|
||||
logInfo('Cleanup', `Successfully freed port ${host}:${port} on attempt ${retry}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (retry < maxRetries) {
|
||||
logDebug('Cleanup', `Port ${host}:${port} still busy, retrying in 2 seconds...`);
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
}
|
||||
}
|
||||
|
||||
logError('Cleanup', `Failed to free port ${host}:${port} after ${maxRetries} attempts`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Helper function to get all configured IPs on the interface
|
||||
async function getConfiguredIPs() {
|
||||
// If virtual interfaces are disabled, return empty array
|
||||
if (process.env.DISABLE_VIRTUAL_INTERFACES === 'true') {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
let command;
|
||||
if (os.platform() === 'darwin') {
|
||||
command = `ifconfig ${state.subnetName} | grep 'inet ' | awk '{print $2}'`;
|
||||
} else if (os.platform() === 'linux') {
|
||||
command = `ip addr show ${state.subnetName} | grep 'inet ' | awk '{print $2}' | cut -d'/' -f1`;
|
||||
} else if (os.platform() === 'win32') {
|
||||
command = `netsh interface ip show addresses "${state.subnetName || 'Loopback Pseudo-Interface 1'}" | findstr "IP Address"`;
|
||||
} else {
|
||||
logWarn('Cleanup', `Unsupported platform: ${os.platform()}, assuming no configured IPs`);
|
||||
return [];
|
||||
}
|
||||
const { stdout } = await execAsync(command);
|
||||
if (os.platform() === 'win32') {
|
||||
// Parse Windows output: "IP Address: 192.168.3.2"
|
||||
return stdout.split('\n')
|
||||
.map(line => {
|
||||
const match = line.match(/IP Address:\s*(\d+\.\d+\.\d+\.\d+)/);
|
||||
return match ? match[1] : null;
|
||||
})
|
||||
.filter(ip => ip);
|
||||
}
|
||||
return stdout.split('\n').map(ip => ip.trim()).filter(ip => ip);
|
||||
} catch (err) {
|
||||
logError('Cleanup', `Error listing configured IPs: ${err.message}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to remove virtual interface
|
||||
async function removeVirtualInterface(ip) {
|
||||
// Check if virtual interfaces are disabled
|
||||
if (process.env.DISABLE_VIRTUAL_INTERFACES === 'true') {
|
||||
logDebug('Cleanup', 'Virtual interfaces disabled, skipping interface removal');
|
||||
return;
|
||||
}
|
||||
if (!state.subnetName) {
|
||||
logWarn('Cleanup', 'SUBNET_NAME not defined, skipping interface removal');
|
||||
return;
|
||||
}
|
||||
|
||||
const ipParts = ip.split('.');
|
||||
if (ipParts.length !== 4 || ipParts.some(part => isNaN(part) || part < 0 || part > 255)) {
|
||||
logWarn('Cleanup', `Skipping removal of invalid IP ${ip}`);
|
||||
return;
|
||||
}
|
||||
|
||||
let configuredIPs;
|
||||
try {
|
||||
configuredIPs = await getConfiguredIPs();
|
||||
} catch (err) {
|
||||
logError('Cleanup', `Error checking if IP ${ip} exists: ${err.message}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!configuredIPs.includes(ip)) {
|
||||
logDebug('Cleanup', `IP ${ip} not configured on system, skipping removal`);
|
||||
return;
|
||||
}
|
||||
|
||||
const cidrIp = `${ip}/24`;
|
||||
let command;
|
||||
if (os.platform() === 'darwin') {
|
||||
command = `sudo ifconfig ${state.subnetName} -alias ${ip}`;
|
||||
} else if (os.platform() === 'linux') {
|
||||
command = `sudo ip addr del ${cidrIp} dev ${state.subnetName}`;
|
||||
} else if (os.platform() === 'win32') {
|
||||
const interfaceName = state.subnetName || 'Loopback Pseudo-Interface 1';
|
||||
command = `netsh interface ip delete address "${interfaceName}" ${ip}`;
|
||||
} else {
|
||||
logWarn('Cleanup', `Unsupported platform: ${os.platform()}, skipping interface removal`);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await execAsync(command);
|
||||
logInfo('Cleanup', `Removed virtual interface IP: ${ip}`);
|
||||
} catch (err) {
|
||||
if (err.stderr && (err.stderr.includes("Can't assign requested address") || err.stderr.includes("No such device"))) {
|
||||
logDebug('Cleanup', `IP ${ip} already removed or not configured: ${err.stderr}`);
|
||||
} else {
|
||||
logError('Cleanup', `Error removing virtual interface ${ip}: ${err.stderr || err.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to close a server with a timeout
|
||||
async function closeServer(server, type, key, isUdp = false) {
|
||||
return new Promise((resolve) => {
|
||||
if (!server) {
|
||||
logDebug('Cleanup', `No ${type} server to close for ${key}`);
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
logWarn('Cleanup', `Timeout closing ${type} server for ${key}, forcing closure`);
|
||||
if (isUdp && server.close) {
|
||||
server.close();
|
||||
} else if (!isUdp && server.destroy) {
|
||||
server.destroy();
|
||||
} else if (!isUdp) {
|
||||
server.close();
|
||||
}
|
||||
resolve();
|
||||
}, 10000);
|
||||
|
||||
if (isUdp && server.close) {
|
||||
server.close(() => {
|
||||
clearTimeout(timeout);
|
||||
logDebug('Cleanup', `Closed UDP ${type} server for ${key}`);
|
||||
resolve();
|
||||
});
|
||||
} else if (!isUdp) {
|
||||
server.close((err) => {
|
||||
clearTimeout(timeout);
|
||||
if (err) {
|
||||
logError('Cleanup', `Error closing ${type} server for ${key}: ${err.message}`);
|
||||
} else {
|
||||
logDebug('Cleanup', `Closed ${type} server for ${key}`);
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
} else {
|
||||
clearTimeout(timeout);
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Cleanup all interfaces
|
||||
async function cleanupInterfaces() {
|
||||
try {
|
||||
const configuredIPs = await getConfiguredIPs();
|
||||
logDebug('Cleanup', `Configured IPs on ${state.subnetName}: ${configuredIPs.join(', ')}`);
|
||||
|
||||
const portsToCheck = new Map();
|
||||
for (const key of state.holesails.keys()) {
|
||||
const [domain, port] = key.split(':');
|
||||
const ip = state.domainToIPMap.get(domain);
|
||||
if (ip && port) {
|
||||
portsToCheck.set(`${ip}:${port}`, { ip, port: parseInt(port), domain });
|
||||
}
|
||||
}
|
||||
|
||||
// Close all holesail connections first
|
||||
for (const [key, holesail] of state.holesails) {
|
||||
try {
|
||||
if (holesail instanceof dgram.Socket) {
|
||||
await closeServer(holesail, 'Holesail UDP', key, true);
|
||||
} else {
|
||||
await holesail.close();
|
||||
logDebug('Cleanup', `Closed Holesail TCP client for ${key}`);
|
||||
}
|
||||
} catch (err) {
|
||||
logError('Cleanup', `Error closing Holesail client for ${key}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
state.holesails.clear();
|
||||
logDebug('Cleanup', 'Cleared Holesail clients state');
|
||||
|
||||
// Close TLS servers
|
||||
for (const [key, server] of state.tlsServers) {
|
||||
await closeServer(server, 'TLS', key);
|
||||
}
|
||||
state.tlsServers.clear();
|
||||
logDebug('Cleanup', 'Cleared TLS servers state');
|
||||
|
||||
// Close HTTP servers
|
||||
for (const [key, server] of state.httpServers) {
|
||||
await closeServer(server, 'HTTP', key);
|
||||
}
|
||||
state.httpServers.clear();
|
||||
logDebug('Cleanup', 'Cleared HTTP servers state');
|
||||
|
||||
// Kill holesail server children
|
||||
for (const [id, child] of state.holesailChildren) {
|
||||
try {
|
||||
await killProcess(child.pid, 10000);
|
||||
} catch (err) {
|
||||
logError('Cleanup', `Error killing Holesail child for server ${id}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
state.holesailChildren.clear();
|
||||
state.holesailOpts.clear();
|
||||
state.holesailInfos.clear();
|
||||
logDebug('Cleanup', 'Cleared Holesail server child processes state');
|
||||
|
||||
// Kill holesail client children
|
||||
for (const [id, child] of state.holesailClientChildren) {
|
||||
try {
|
||||
await killProcess(child.pid, 10000);
|
||||
} catch (err) {
|
||||
logError('Cleanup', `Error killing Holesail client child for ${id}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
state.holesailClientChildren.clear();
|
||||
state.holesailClientOpts.clear();
|
||||
state.holesailClientInfos.clear();
|
||||
logDebug('Cleanup', 'Cleared Holesail client child processes state');
|
||||
|
||||
// Free all ports
|
||||
for (const [key, { ip, port, domain }] of portsToCheck) {
|
||||
const isPortFree = await freePort(ip, port);
|
||||
if (!isPortFree) {
|
||||
logError('Cleanup', `Port ${port} on ${ip} for ${domain} still in use after exhaustive cleanup attempts`);
|
||||
}
|
||||
}
|
||||
|
||||
const reservedIPs = ['127.0.0.1'];
|
||||
|
||||
// Remove virtual interfaces (skip if disabled)
|
||||
if (process.env.DISABLE_VIRTUAL_INTERFACES !== 'true') {
|
||||
const removedIPs = new Set();
|
||||
for (const [domain, ip] of state.domainToIPMap) {
|
||||
if (reservedIPs.includes(ip)) {
|
||||
logDebug('Cleanup', `Skipping removal of reserved IP ${ip} for domain ${domain}`);
|
||||
continue;
|
||||
}
|
||||
if (!configuredIPs.includes(ip)) {
|
||||
logDebug('Cleanup', `IP ${ip} for domain ${domain} not configured on system, skipping removal`);
|
||||
removedIPs.add(ip);
|
||||
continue;
|
||||
}
|
||||
await removeVirtualInterface(ip);
|
||||
removedIPs.add(ip);
|
||||
}
|
||||
} else {
|
||||
logDebug('Cleanup', 'Virtual interfaces disabled, skipping interface removal');
|
||||
}
|
||||
|
||||
// Update state
|
||||
const newDomainToIPMap = new Map();
|
||||
for (const [domain, ip] of state.domainToIPMap) {
|
||||
if (reservedIPs.includes(ip) && !removedIPs.has(ip)) {
|
||||
newDomainToIPMap.set(domain, ip);
|
||||
}
|
||||
}
|
||||
state.domainToIPMap = newDomainToIPMap;
|
||||
state.currentIP = parseInt(process.env.INITIAL_IP_INDEX || 2);
|
||||
|
||||
logInfo('Cleanup', 'Interfaces cleaned up successfully');
|
||||
} catch (err) {
|
||||
logError('Cleanup', `Failed to cleanup interfaces: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
cleanupInterfaces,
|
||||
removeVirtualInterface,
|
||||
checkPortAvailability,
|
||||
waitForPortRelease,
|
||||
freePort
|
||||
};
|
||||
@@ -0,0 +1,996 @@
|
||||
const state = require('../infrastructure/state');
|
||||
const os = require('os');
|
||||
const { performance } = require('perf_hooks');
|
||||
const { parseMinutesToMs, parseSecondsToMs } = require('../infrastructure/utils');
|
||||
|
||||
// Circular buffer for time-series data
|
||||
class CircularBuffer {
|
||||
constructor(maxSize = 1000) {
|
||||
this.maxSize = maxSize;
|
||||
this.buffer = [];
|
||||
this.head = 0;
|
||||
}
|
||||
|
||||
push(value) {
|
||||
if (this.buffer.length < this.maxSize) {
|
||||
this.buffer.push(value);
|
||||
} else {
|
||||
this.buffer[this.head] = value;
|
||||
this.head = (this.head + 1) % this.maxSize;
|
||||
}
|
||||
}
|
||||
|
||||
getData() {
|
||||
if (this.buffer.length < this.maxSize) {
|
||||
return this.buffer;
|
||||
}
|
||||
return [...this.buffer.slice(this.head), ...this.buffer.slice(0, this.head)];
|
||||
}
|
||||
|
||||
getLatest(count = 100) {
|
||||
const data = this.getData();
|
||||
return data.slice(-count);
|
||||
}
|
||||
|
||||
length() {
|
||||
return this.buffer.length;
|
||||
}
|
||||
|
||||
clear() {
|
||||
this.buffer = [];
|
||||
this.head = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Metrics configuration
|
||||
const METRICS_CONFIG = {
|
||||
retentionPeriod: parseMinutesToMs(process.env.METRICS_RETENTION_MS || '60'), // 1 hour = 60 minutes default
|
||||
samplingRate: parseFloat(process.env.METRICS_SAMPLING_RATE || '1.0'), // 1.0 = 100%, 0.5 = 50%
|
||||
aggregationInterval: parseSecondsToMs(process.env.METRICS_AGGREGATION_INTERVAL || '60'), // 1 minute = 60 seconds
|
||||
maxCircularBufferSize: parseInt(process.env.METRICS_MAX_BUFFER_SIZE || '1000', 10)
|
||||
};
|
||||
|
||||
// Aggregated metrics cache
|
||||
let aggregatedMetrics = {
|
||||
requests: {
|
||||
total: 0,
|
||||
successful: 0,
|
||||
failed: 0,
|
||||
avgResponseTime: 0,
|
||||
minResponseTime: Infinity,
|
||||
maxResponseTime: 0,
|
||||
lastUpdated: Date.now()
|
||||
},
|
||||
dns: {
|
||||
queries: 0,
|
||||
p2pResolutions: 0,
|
||||
publicResolutions: 0,
|
||||
failures: 0,
|
||||
avgResponseTime: 0,
|
||||
lastUpdated: Date.now()
|
||||
}
|
||||
};
|
||||
|
||||
let aggregationInterval = null;
|
||||
|
||||
// Simple metrics tracking
|
||||
const metrics = {
|
||||
requests: {
|
||||
total: 0,
|
||||
successful: 0,
|
||||
failed: 0,
|
||||
byEndpoint: new Map(),
|
||||
maxEndpoints: 500, // Limit to 500 endpoints
|
||||
responseTimes: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
timestamp: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize)
|
||||
},
|
||||
dns: {
|
||||
queries: 0,
|
||||
p2pResolutions: 0,
|
||||
publicResolutions: 0,
|
||||
failures: 0,
|
||||
responseTimes: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
queryTimestamps: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
queryTypes: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
domainQueries: new Map(), // domain -> count
|
||||
maxDomainQueries: 1000 // Limit to 1000 domains
|
||||
},
|
||||
peers: {
|
||||
connected: 0,
|
||||
disconnected: 0,
|
||||
current: 0,
|
||||
connectionEvents: new CircularBuffer(Math.floor(METRICS_CONFIG.maxCircularBufferSize / 2)),
|
||||
connectionDurations: new CircularBuffer(Math.floor(METRICS_CONFIG.maxCircularBufferSize / 2))
|
||||
},
|
||||
domains: {
|
||||
added: 0,
|
||||
removed: 0,
|
||||
current: 0,
|
||||
addEvents: new CircularBuffer(Math.floor(METRICS_CONFIG.maxCircularBufferSize / 2)),
|
||||
removeEvents: new CircularBuffer(Math.floor(METRICS_CONFIG.maxCircularBufferSize / 2))
|
||||
},
|
||||
consensus: {
|
||||
resolutions: 0,
|
||||
quorumFailures: 0,
|
||||
ties: 0,
|
||||
validationFailures: 0,
|
||||
totalVotes: 0,
|
||||
avgVotesPerDomain: 0,
|
||||
events: new CircularBuffer(Math.floor(METRICS_CONFIG.maxCircularBufferSize / 2))
|
||||
},
|
||||
holesail: {
|
||||
clientsStarted: 0,
|
||||
clientsStopped: 0,
|
||||
serversStarted: 0,
|
||||
serversStopped: 0,
|
||||
activeConnections: 0,
|
||||
connectionEvents: new CircularBuffer(Math.floor(METRICS_CONFIG.maxCircularBufferSize / 2)),
|
||||
connectionDurations: new CircularBuffer(Math.floor(METRICS_CONFIG.maxCircularBufferSize / 2)),
|
||||
protocolCounts: { tcp: 0, udp: 0 }
|
||||
},
|
||||
network: {
|
||||
throughput: {
|
||||
bytesIn: 0,
|
||||
bytesOut: 0,
|
||||
dataPoints: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize)
|
||||
}
|
||||
},
|
||||
resources: {
|
||||
sockets: 0,
|
||||
timers: 0,
|
||||
intervals: 0,
|
||||
servers: 0,
|
||||
dataPoints: new CircularBuffer(Math.floor(METRICS_CONFIG.maxCircularBufferSize / 2))
|
||||
},
|
||||
process: {
|
||||
memory: {
|
||||
heapUsed: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
heapTotal: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
rss: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
external: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
arrayBuffers: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
timestamp: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize)
|
||||
},
|
||||
cpu: {
|
||||
user: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
system: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
percentage: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
timestamp: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize)
|
||||
},
|
||||
eventLoop: {
|
||||
lag: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
timestamp: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
lastCheck: performance.now()
|
||||
},
|
||||
system: {
|
||||
totalMemory: os.totalmem(),
|
||||
freeMemory: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
usedMemoryPercentage: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
loadAverage: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
timestamp: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize)
|
||||
},
|
||||
handles: {
|
||||
active: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize),
|
||||
timestamp: new CircularBuffer(METRICS_CONFIG.maxCircularBufferSize)
|
||||
},
|
||||
lastCpuUsage: process.cpuUsage(),
|
||||
lastCpuTime: Date.now(),
|
||||
collectionInterval: null
|
||||
},
|
||||
startTime: Date.now(),
|
||||
lastUpdate: Date.now()
|
||||
};
|
||||
|
||||
/**
|
||||
* Increment a metric counter
|
||||
* @param {string} category - Metric category
|
||||
* @param {string} metric - Metric name
|
||||
* @param {number} value - Value to increment by (default: 1)
|
||||
*/
|
||||
function increment(category, metric, value = 1) {
|
||||
if (metrics[category] && typeof metrics[category][metric] === 'number') {
|
||||
metrics[category][metric] += value;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track DNS query with timing (with sampling)
|
||||
* @param {string} type - Query type ('p2p', 'public', 'failure')
|
||||
* @param {string} domain - Domain name
|
||||
* @param {number} responseTime - Response time in ms
|
||||
*/
|
||||
function trackDNSQueryWithTiming(type, domain, responseTime = null) {
|
||||
// Apply sampling rate
|
||||
if (Math.random() > METRICS_CONFIG.samplingRate) {
|
||||
return; // Skip this metric due to sampling
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
increment('dns', 'queries');
|
||||
if (type === 'p2p') {
|
||||
increment('dns', 'p2pResolutions');
|
||||
} else if (type === 'public') {
|
||||
increment('dns', 'publicResolutions');
|
||||
} else if (type === 'failure') {
|
||||
increment('dns', 'failures');
|
||||
}
|
||||
|
||||
metrics.dns.queryTimestamps.push(now);
|
||||
metrics.dns.queryTypes.push(type);
|
||||
if (responseTime !== null) {
|
||||
metrics.dns.responseTimes.push(responseTime);
|
||||
}
|
||||
|
||||
// Track per-domain queries with size limit
|
||||
if (domain) {
|
||||
// If map is at limit, remove oldest entries (simple FIFO by removing first key)
|
||||
if (metrics.dns.domainQueries.size >= metrics.dns.maxDomainQueries) {
|
||||
const firstKey = metrics.dns.domainQueries.keys().next().value;
|
||||
if (firstKey) {
|
||||
metrics.dns.domainQueries.delete(firstKey);
|
||||
}
|
||||
}
|
||||
const count = metrics.dns.domainQueries.get(domain) || 0;
|
||||
metrics.dns.domainQueries.set(domain, count + 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track request with response time (with sampling)
|
||||
* @param {string} endpoint - API endpoint
|
||||
* @param {boolean} success - Whether request was successful
|
||||
* @param {number} responseTime - Response time in ms
|
||||
*/
|
||||
function trackRequestWithTiming(endpoint, success, responseTime = null) {
|
||||
// Apply sampling rate
|
||||
if (Math.random() > METRICS_CONFIG.samplingRate) {
|
||||
// Still track basic request count, but skip detailed metrics
|
||||
trackRequest(endpoint, success);
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
trackRequest(endpoint, success);
|
||||
|
||||
if (responseTime !== null) {
|
||||
metrics.requests.responseTimes.push(responseTime);
|
||||
metrics.requests.timestamp.push(now);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track peer connection event
|
||||
* @param {string} event - 'connect' or 'disconnect'
|
||||
* @param {string} peerId - Peer ID
|
||||
*/
|
||||
function trackPeerEvent(event, peerId) {
|
||||
const now = Date.now();
|
||||
if (event === 'connect') {
|
||||
increment('peers', 'connected');
|
||||
metrics.peers.connectionEvents.push({ type: 'connect', peerId, timestamp: now });
|
||||
} else if (event === 'disconnect') {
|
||||
increment('peers', 'disconnected');
|
||||
const lastConnect = metrics.peers.connectionEvents.getData().slice().reverse()
|
||||
.find(e => e.type === 'connect' && e.peerId === peerId);
|
||||
if (lastConnect) {
|
||||
const duration = now - lastConnect.timestamp;
|
||||
metrics.peers.connectionDurations.push(duration);
|
||||
}
|
||||
metrics.peers.connectionEvents.push({ type: 'disconnect', peerId, timestamp: now });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track domain event
|
||||
* @param {string} event - 'add' or 'remove'
|
||||
* @param {string} domain - Domain name
|
||||
*/
|
||||
function trackDomainEvent(event, domain) {
|
||||
const now = Date.now();
|
||||
if (event === 'add') {
|
||||
increment('domains', 'added');
|
||||
metrics.domains.addEvents.push({ domain, timestamp: now });
|
||||
} else if (event === 'remove') {
|
||||
increment('domains', 'removed');
|
||||
metrics.domains.removeEvents.push({ domain, timestamp: now });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track consensus event
|
||||
* @param {string} event - Event type ('quorum_failure', 'tie', 'validation_failure', etc.)
|
||||
* @param {object} data - Event data
|
||||
*/
|
||||
function trackConsensusEvent(event, data = {}) {
|
||||
const now = Date.now();
|
||||
metrics.consensus.events.push({ event, data, timestamp: now });
|
||||
|
||||
if (event === 'quorum_failure') {
|
||||
increment('consensus', 'quorumFailures');
|
||||
} else if (event === 'tie') {
|
||||
increment('consensus', 'ties');
|
||||
} else if (event === 'validation_failure') {
|
||||
increment('consensus', 'validationFailures');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track Holesail connection event
|
||||
* @param {string} type - 'client' or 'server'
|
||||
* @param {string} event - 'start' or 'stop'
|
||||
* @param {string} protocol - 'tcp' or 'udp'
|
||||
* @param {number} duration - Connection duration in ms (for stop events)
|
||||
*/
|
||||
function trackHolesailEvent(type, event, protocol = 'tcp', duration = null) {
|
||||
const now = Date.now();
|
||||
if (type === 'client') {
|
||||
if (event === 'start') {
|
||||
increment('holesail', 'clientsStarted');
|
||||
metrics.holesail.activeConnections++;
|
||||
} else if (event === 'stop') {
|
||||
increment('holesail', 'clientsStopped');
|
||||
metrics.holesail.activeConnections = Math.max(0, metrics.holesail.activeConnections - 1);
|
||||
}
|
||||
} else if (type === 'server') {
|
||||
if (event === 'start') {
|
||||
increment('holesail', 'serversStarted');
|
||||
} else if (event === 'stop') {
|
||||
increment('holesail', 'serversStopped');
|
||||
}
|
||||
}
|
||||
|
||||
metrics.holesail.connectionEvents.push({ type, event, protocol, timestamp: now });
|
||||
if (protocol === 'tcp') {
|
||||
metrics.holesail.protocolCounts.tcp++;
|
||||
} else if (protocol === 'udp') {
|
||||
metrics.holesail.protocolCounts.udp++;
|
||||
}
|
||||
|
||||
if (duration !== null) {
|
||||
metrics.holesail.connectionDurations.push(duration);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update resource metrics
|
||||
*/
|
||||
function updateResourceMetrics() {
|
||||
const now = Date.now();
|
||||
const resourceCounts = {
|
||||
sockets: metrics.requests.responseTimes.length(),
|
||||
timers: 0, // Would need external tracking
|
||||
intervals: 0, // Would need external tracking
|
||||
servers: (state.tlsServers?.size || 0) + (state.httpServers?.size || 0),
|
||||
holesails: state.holesails?.size || 0,
|
||||
timestamp: now
|
||||
};
|
||||
|
||||
metrics.resources.dataPoints.push(resourceCounts);
|
||||
metrics.resources.sockets = resourceCounts.sockets;
|
||||
metrics.resources.servers = resourceCounts.servers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Track process memory usage
|
||||
*/
|
||||
function trackProcessMemory() {
|
||||
const memUsage = process.memoryUsage();
|
||||
const now = Date.now();
|
||||
|
||||
metrics.process.memory.heapUsed.push(memUsage.heapUsed);
|
||||
metrics.process.memory.heapTotal.push(memUsage.heapTotal);
|
||||
metrics.process.memory.rss.push(memUsage.rss);
|
||||
metrics.process.memory.external.push(memUsage.external || 0);
|
||||
metrics.process.memory.arrayBuffers.push(memUsage.arrayBuffers || 0);
|
||||
metrics.process.memory.timestamp.push(now);
|
||||
}
|
||||
|
||||
/**
|
||||
* Track CPU usage
|
||||
*/
|
||||
function trackCPUUsage() {
|
||||
const now = Date.now();
|
||||
|
||||
// Initialize if this is the first call
|
||||
if (!metrics.process.lastCpuTime || !metrics.process.lastCpuUsage) {
|
||||
metrics.process.lastCpuUsage = process.cpuUsage();
|
||||
metrics.process.lastCpuTime = now;
|
||||
return; // Skip first measurement
|
||||
}
|
||||
|
||||
const currentCpuUsage = process.cpuUsage(metrics.process.lastCpuUsage);
|
||||
const elapsed = (now - metrics.process.lastCpuTime) * 1000; // Convert to microseconds
|
||||
|
||||
// Only calculate if enough time has passed (avoid division by very small numbers)
|
||||
if (elapsed > 1000) { // At least 1ms in microseconds
|
||||
const userPercent = (currentCpuUsage.user / elapsed) * 100;
|
||||
const systemPercent = (currentCpuUsage.system / elapsed) * 100;
|
||||
const totalPercent = userPercent + systemPercent;
|
||||
|
||||
metrics.process.cpu.user.push(currentCpuUsage.user);
|
||||
metrics.process.cpu.system.push(currentCpuUsage.system);
|
||||
metrics.process.cpu.percentage.push(Math.min(totalPercent, 100)); // Cap at 100%
|
||||
metrics.process.cpu.timestamp.push(now);
|
||||
}
|
||||
|
||||
// Always update for next calculation
|
||||
metrics.process.lastCpuUsage = process.cpuUsage();
|
||||
metrics.process.lastCpuTime = now;
|
||||
}
|
||||
|
||||
/**
|
||||
* Track event loop lag - approximate by measuring time between collection intervals
|
||||
*/
|
||||
function trackEventLoopLag() {
|
||||
const now = performance.now();
|
||||
const expectedInterval = 5000; // Expected interval in ms
|
||||
const actualElapsed = now - (metrics.process.eventLoop.lastCheck || now);
|
||||
const lag = Math.max(0, actualElapsed - expectedInterval);
|
||||
|
||||
metrics.process.eventLoop.lag.push(lag);
|
||||
metrics.process.eventLoop.timestamp.push(Date.now());
|
||||
metrics.process.eventLoop.lastCheck = now;
|
||||
}
|
||||
|
||||
/**
|
||||
* Track system resources
|
||||
*/
|
||||
function trackSystemResources() {
|
||||
const now = Date.now();
|
||||
const freeMem = os.freemem();
|
||||
const totalMem = metrics.process.system.totalMemory;
|
||||
const usedMem = totalMem - freeMem;
|
||||
const usedMemPercent = (usedMem / totalMem) * 100;
|
||||
|
||||
// Get load average (1 minute average)
|
||||
const loadAvg = os.loadavg();
|
||||
|
||||
metrics.process.system.freeMemory.push(freeMem);
|
||||
metrics.process.system.usedMemoryPercentage.push(usedMemPercent);
|
||||
metrics.process.system.loadAverage.push(loadAvg[0] || 0);
|
||||
metrics.process.system.timestamp.push(now);
|
||||
|
||||
// Track active handles (approximate via _getActiveHandles if available)
|
||||
try {
|
||||
const activeHandles = process._getActiveHandles ? process._getActiveHandles().length : 0;
|
||||
metrics.process.handles.active.push(activeHandles);
|
||||
metrics.process.handles.timestamp.push(now);
|
||||
} catch (e) {
|
||||
// _getActiveHandles might not be available in all contexts
|
||||
metrics.process.handles.active.push(0);
|
||||
metrics.process.handles.timestamp.push(now);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect all process metrics
|
||||
*/
|
||||
function collectProcessMetrics() {
|
||||
trackProcessMemory();
|
||||
trackCPUUsage();
|
||||
trackEventLoopLag();
|
||||
trackSystemResources();
|
||||
}
|
||||
|
||||
/**
|
||||
* Start periodic collection of process metrics
|
||||
* @param {number} intervalMs - Collection interval in milliseconds (default: 5000)
|
||||
*/
|
||||
function startProcessMetricsCollection(intervalMs = 5000) {
|
||||
// Clear existing interval if any
|
||||
if (metrics.process.collectionInterval) {
|
||||
clearInterval(metrics.process.collectionInterval);
|
||||
}
|
||||
|
||||
// Initial collection
|
||||
collectProcessMetrics();
|
||||
|
||||
// Set up periodic collection
|
||||
metrics.process.collectionInterval = setInterval(() => {
|
||||
collectProcessMetrics();
|
||||
}, intervalMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop periodic collection of process metrics
|
||||
*/
|
||||
function stopProcessMetricsCollection() {
|
||||
if (metrics.process.collectionInterval) {
|
||||
clearInterval(metrics.process.collectionInterval);
|
||||
metrics.process.collectionInterval = null;
|
||||
}
|
||||
if (aggregationInterval) {
|
||||
clearInterval(aggregationInterval);
|
||||
aggregationInterval = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate metrics to reduce memory usage
|
||||
*/
|
||||
function aggregateMetrics() {
|
||||
try {
|
||||
const now = Date.now();
|
||||
|
||||
// Aggregate request metrics
|
||||
const requestTimes = metrics.requests.responseTimes.getData();
|
||||
if (requestTimes.length > 0) {
|
||||
aggregatedMetrics.requests.avgResponseTime = requestTimes.reduce((a, b) => a + b, 0) / requestTimes.length;
|
||||
aggregatedMetrics.requests.minResponseTime = Math.min(...requestTimes);
|
||||
aggregatedMetrics.requests.maxResponseTime = Math.max(...requestTimes);
|
||||
}
|
||||
aggregatedMetrics.requests.total = metrics.requests.total;
|
||||
aggregatedMetrics.requests.successful = metrics.requests.successful;
|
||||
aggregatedMetrics.requests.failed = metrics.requests.failed;
|
||||
aggregatedMetrics.requests.lastUpdated = now;
|
||||
|
||||
// Aggregate DNS metrics
|
||||
const dnsTimes = metrics.dns.responseTimes.getData();
|
||||
if (dnsTimes.length > 0) {
|
||||
aggregatedMetrics.dns.avgResponseTime = dnsTimes.reduce((a, b) => a + b, 0) / dnsTimes.length;
|
||||
}
|
||||
aggregatedMetrics.dns.queries = metrics.dns.queries;
|
||||
aggregatedMetrics.dns.p2pResolutions = metrics.dns.p2pResolutions;
|
||||
aggregatedMetrics.dns.publicResolutions = metrics.dns.publicResolutions;
|
||||
aggregatedMetrics.dns.failures = metrics.dns.failures;
|
||||
aggregatedMetrics.dns.lastUpdated = now;
|
||||
|
||||
// Clean up old data based on retention period
|
||||
const cutoff = now - METRICS_CONFIG.retentionPeriod;
|
||||
cleanOldMetrics(cutoff);
|
||||
} catch (err) {
|
||||
const { logError } = require('../infrastructure/logger');
|
||||
logError('Metrics', `Error aggregating metrics: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up metrics older than cutoff timestamp
|
||||
*/
|
||||
function cleanOldMetrics(cutoff) {
|
||||
// Clean up timestamps and corresponding data
|
||||
const cleanBuffer = (buffer, timestampBuffer) => {
|
||||
const timestamps = timestampBuffer.getData();
|
||||
const data = buffer.getData();
|
||||
const filtered = timestamps.map((ts, idx) => ts >= cutoff ? data[idx] : null).filter(v => v !== null);
|
||||
buffer.buffer = filtered;
|
||||
buffer.head = 0;
|
||||
};
|
||||
|
||||
// Clean request metrics
|
||||
cleanBuffer(metrics.requests.responseTimes, metrics.requests.timestamp);
|
||||
|
||||
// Clean DNS metrics
|
||||
cleanBuffer(metrics.dns.responseTimes, metrics.dns.queryTimestamps);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start metrics aggregation
|
||||
*/
|
||||
function startMetricsAggregation() {
|
||||
if (aggregationInterval) {
|
||||
clearInterval(aggregationInterval);
|
||||
}
|
||||
|
||||
// Initial aggregation
|
||||
aggregateMetrics();
|
||||
|
||||
// Schedule periodic aggregation
|
||||
aggregationInterval = setInterval(() => {
|
||||
aggregateMetrics();
|
||||
}, METRICS_CONFIG.aggregationInterval);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all metrics
|
||||
* @returns {object} - Metrics object
|
||||
*/
|
||||
function getMetrics() {
|
||||
try {
|
||||
// Update current counts from state
|
||||
metrics.peers.current = state.connectedPeers?.size || 0;
|
||||
metrics.domains.current = state.domainToIPMap?.size || 0;
|
||||
metrics.holesail.activeConnections = state.holesails?.size || 0;
|
||||
|
||||
// Update resource metrics
|
||||
updateResourceMetrics();
|
||||
|
||||
// Always collect fresh process metrics when stats are requested
|
||||
// This ensures real-time data regardless of collection interval
|
||||
collectProcessMetrics();
|
||||
|
||||
const uptime = Date.now() - metrics.startTime;
|
||||
metrics.lastUpdate = Date.now();
|
||||
|
||||
// Calculate statistics
|
||||
const dnsResponseTimes = metrics.dns.responseTimes ? metrics.dns.responseTimes.getData() : [];
|
||||
const requestResponseTimes = metrics.requests.responseTimes ? metrics.requests.responseTimes.getData() : [];
|
||||
|
||||
return {
|
||||
system: {
|
||||
uptime: {
|
||||
ms: uptime,
|
||||
seconds: Math.floor(uptime / 1000),
|
||||
minutes: Math.floor(uptime / 60000),
|
||||
hours: Math.floor(uptime / 3600000),
|
||||
days: Math.floor(uptime / 86400000)
|
||||
},
|
||||
isMaster: state.isMaster || false,
|
||||
isConnected: !!state.dnsPass,
|
||||
startTime: metrics.startTime,
|
||||
lastUpdate: metrics.lastUpdate
|
||||
},
|
||||
requests: {
|
||||
total: metrics.requests.total,
|
||||
successful: metrics.requests.successful,
|
||||
failed: metrics.requests.failed,
|
||||
successRate: metrics.requests.total > 0
|
||||
? (metrics.requests.successful / metrics.requests.total * 100).toFixed(2)
|
||||
: 0,
|
||||
avgResponseTime: requestResponseTimes.length > 0
|
||||
? (requestResponseTimes.reduce((a, b) => a + b, 0) / requestResponseTimes.length).toFixed(2)
|
||||
: 0,
|
||||
minResponseTime: requestResponseTimes.length > 0 ? Math.min(...requestResponseTimes) : 0,
|
||||
maxResponseTime: requestResponseTimes.length > 0 ? Math.max(...requestResponseTimes) : 0,
|
||||
byEndpoint: metrics.requests.byEndpoint && metrics.requests.byEndpoint.size > 0
|
||||
? Object.fromEntries(
|
||||
Array.from(metrics.requests.byEndpoint.entries()).map(([k, v]) => [
|
||||
k,
|
||||
{ ...v, successRate: v.total > 0 ? (v.successful / v.total * 100).toFixed(2) : 0 }
|
||||
])
|
||||
)
|
||||
: {}
|
||||
},
|
||||
dns: {
|
||||
queries: metrics.dns.queries,
|
||||
p2pResolutions: metrics.dns.p2pResolutions,
|
||||
publicResolutions: metrics.dns.publicResolutions,
|
||||
failures: metrics.dns.failures,
|
||||
successRate: metrics.dns.queries > 0
|
||||
? ((metrics.dns.queries - metrics.dns.failures) / metrics.dns.queries * 100).toFixed(2)
|
||||
: 0,
|
||||
p2pRate: metrics.dns.queries > 0
|
||||
? (metrics.dns.p2pResolutions / metrics.dns.queries * 100).toFixed(2)
|
||||
: 0,
|
||||
avgResponseTime: dnsResponseTimes.length > 0
|
||||
? (dnsResponseTimes.reduce((a, b) => a + b, 0) / dnsResponseTimes.length).toFixed(2)
|
||||
: 0,
|
||||
minResponseTime: dnsResponseTimes.length > 0 ? Math.min(...dnsResponseTimes) : 0,
|
||||
maxResponseTime: dnsResponseTimes.length > 0 ? Math.max(...dnsResponseTimes) : 0,
|
||||
topDomains: metrics.dns.domainQueries && metrics.dns.domainQueries.size > 0
|
||||
? Array.from(metrics.dns.domainQueries.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 10)
|
||||
.map(([domain, count]) => ({ domain, count }))
|
||||
: []
|
||||
},
|
||||
peers: {
|
||||
connected: metrics.peers.connected,
|
||||
disconnected: metrics.peers.disconnected,
|
||||
current: metrics.peers.current,
|
||||
avgConnectionDuration: metrics.peers.connectionDurations && metrics.peers.connectionDurations.length() > 0
|
||||
? (metrics.peers.connectionDurations.getData().reduce((a, b) => a + b, 0) / metrics.peers.connectionDurations.length()).toFixed(2)
|
||||
: 0
|
||||
},
|
||||
domains: {
|
||||
added: metrics.domains.added,
|
||||
removed: metrics.domains.removed,
|
||||
current: metrics.domains.current
|
||||
},
|
||||
consensus: {
|
||||
resolutions: metrics.consensus.resolutions,
|
||||
quorumFailures: metrics.consensus.quorumFailures,
|
||||
ties: metrics.consensus.ties,
|
||||
validationFailures: metrics.consensus.validationFailures,
|
||||
totalVotes: metrics.consensus.totalVotes,
|
||||
avgVotesPerDomain: metrics.consensus.avgVotesPerDomain > 0
|
||||
? metrics.consensus.avgVotesPerDomain.toFixed(2)
|
||||
: 0
|
||||
},
|
||||
holesail: {
|
||||
clientsStarted: metrics.holesail.clientsStarted,
|
||||
clientsStopped: metrics.holesail.clientsStopped,
|
||||
serversStarted: metrics.holesail.serversStarted,
|
||||
serversStopped: metrics.holesail.serversStopped,
|
||||
activeConnections: metrics.holesail.activeConnections,
|
||||
protocolCounts: metrics.holesail.protocolCounts,
|
||||
avgConnectionDuration: metrics.holesail.connectionDurations && metrics.holesail.connectionDurations.length() > 0
|
||||
? (metrics.holesail.connectionDurations.getData().reduce((a, b) => a + b, 0) / metrics.holesail.connectionDurations.length()).toFixed(2)
|
||||
: 0
|
||||
},
|
||||
resources: {
|
||||
sockets: metrics.resources.sockets,
|
||||
timers: metrics.resources.timers,
|
||||
intervals: metrics.resources.intervals,
|
||||
servers: metrics.resources.servers
|
||||
},
|
||||
process: (() => {
|
||||
// Always get fresh process metrics for current values
|
||||
const memUsage = process.memoryUsage();
|
||||
const freeMem = os.freemem();
|
||||
const totalMem = metrics.process.system.totalMemory;
|
||||
const usedMem = totalMem - freeMem;
|
||||
const usedMemPercent = (usedMem / totalMem) * 100;
|
||||
|
||||
// Calculate current CPU usage - need a meaningful time window
|
||||
const now = Date.now();
|
||||
let currentCpuPercent = '0.00';
|
||||
if (metrics.process.lastCpuTime && metrics.process.lastCpuUsage) {
|
||||
const elapsedMs = now - metrics.process.lastCpuTime;
|
||||
if (elapsedMs > 100) { // Only calculate if enough time has passed (>100ms)
|
||||
const currentCpuUsage = process.cpuUsage(metrics.process.lastCpuUsage);
|
||||
const elapsed = elapsedMs * 1000; // Convert to microseconds
|
||||
const userPercent = elapsed > 0 ? (currentCpuUsage.user / elapsed) * 100 : 0;
|
||||
const systemPercent = elapsed > 0 ? (currentCpuUsage.system / elapsed) * 100 : 0;
|
||||
currentCpuPercent = Math.min(userPercent + systemPercent, 100).toFixed(2);
|
||||
} else {
|
||||
// Use the latest tracked percentage if available
|
||||
if (metrics.process.cpu.percentage.length() > 0) {
|
||||
currentCpuPercent = metrics.process.cpu.percentage.getLatest(1)[0]?.toFixed(2) || '0.00';
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// First time - use latest if available
|
||||
if (metrics.process.cpu.percentage.length() > 0) {
|
||||
currentCpuPercent = metrics.process.cpu.percentage.getLatest(1)[0]?.toFixed(2) || '0.00';
|
||||
}
|
||||
}
|
||||
|
||||
// Get fresh active handles count
|
||||
let activeHandles = 0;
|
||||
try {
|
||||
activeHandles = process._getActiveHandles ? process._getActiveHandles().length : 0;
|
||||
} catch (e) {
|
||||
// Ignore if not available
|
||||
}
|
||||
|
||||
return {
|
||||
memory: {
|
||||
current: memUsage,
|
||||
heapUsedMB: (memUsage.heapUsed / 1024 / 1024).toFixed(2),
|
||||
heapTotalMB: (memUsage.heapTotal / 1024 / 1024).toFixed(2),
|
||||
rssMB: (memUsage.rss / 1024 / 1024).toFixed(2),
|
||||
externalMB: ((memUsage.external || 0) / 1024 / 1024).toFixed(2),
|
||||
arrayBuffersMB: ((memUsage.arrayBuffers || 0) / 1024 / 1024).toFixed(2)
|
||||
},
|
||||
cpu: {
|
||||
current: process.cpuUsage(),
|
||||
percentage: currentCpuPercent, // Already a string from .toFixed(2) above
|
||||
avgPercentage: metrics.process.cpu.percentage.length() > 0
|
||||
? (metrics.process.cpu.percentage.getData().reduce((a, b) => a + b, 0) / metrics.process.cpu.percentage.length()).toFixed(2)
|
||||
: '0.00'
|
||||
},
|
||||
eventLoop: {
|
||||
currentLag: metrics.process.eventLoop.lag.length() > 0
|
||||
? metrics.process.eventLoop.lag.getLatest(1)[0]?.toFixed(2) || '0.00'
|
||||
: '0.00',
|
||||
avgLag: metrics.process.eventLoop.lag.length() > 0
|
||||
? (metrics.process.eventLoop.lag.getData().reduce((a, b) => a + b, 0) / metrics.process.eventLoop.lag.length()).toFixed(2)
|
||||
: '0.00',
|
||||
maxLag: metrics.process.eventLoop.lag.length() > 0
|
||||
? Math.max(...metrics.process.eventLoop.lag.getData()).toFixed(2)
|
||||
: '0.00'
|
||||
},
|
||||
system: {
|
||||
totalMemoryMB: (totalMem / 1024 / 1024).toFixed(2),
|
||||
freeMemoryMB: (freeMem / 1024 / 1024).toFixed(2),
|
||||
usedMemoryPercent: usedMemPercent.toFixed(2),
|
||||
loadAverage: os.loadavg()[0]?.toFixed(2) || '0.00'
|
||||
},
|
||||
handles: {
|
||||
active: activeHandles,
|
||||
avgActive: metrics.process.handles.active.length() > 0
|
||||
? Math.round(metrics.process.handles.active.getData().reduce((a, b) => a + b, 0) / metrics.process.handles.active.length())
|
||||
: 0
|
||||
},
|
||||
uptime: process.uptime(),
|
||||
nodeVersion: process.version,
|
||||
platform: process.platform,
|
||||
arch: process.arch
|
||||
};
|
||||
})()
|
||||
};
|
||||
} catch (err) {
|
||||
// Return minimal error response if metrics collection fails
|
||||
const { logError } = require('./logger');
|
||||
logError('Metrics', `Error in getMetrics: ${err.message}`, err);
|
||||
return {
|
||||
system: {
|
||||
uptime: { ms: 0, seconds: 0, minutes: 0, hours: 0, days: 0 },
|
||||
isMaster: false,
|
||||
isConnected: false,
|
||||
startTime: metrics.startTime || Date.now(),
|
||||
lastUpdate: Date.now(),
|
||||
error: err.message
|
||||
},
|
||||
requests: { total: 0, successful: 0, failed: 0, successRate: 0, byEndpoint: {} },
|
||||
dns: { queries: 0, p2pResolutions: 0, publicResolutions: 0, failures: 0, successRate: 0, topDomains: [] },
|
||||
peers: { connected: 0, disconnected: 0, current: 0, avgConnectionDuration: 0 },
|
||||
domains: { added: 0, removed: 0, current: 0 },
|
||||
holesail: { clientsStarted: 0, clientsStopped: 0, serversStarted: 0, serversStopped: 0, activeConnections: 0, protocolCounts: { tcp: 0, udp: 0 }, avgConnectionDuration: 0 },
|
||||
resources: { sockets: 0, timers: 0, intervals: 0, servers: 0 }
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get historical time-series data
|
||||
* @param {number} minutes - Number of minutes of history to return
|
||||
* @returns {object} - Time-series data
|
||||
*/
|
||||
function getHistoricalData(minutes = 60) {
|
||||
const now = Date.now();
|
||||
const cutoff = now - (minutes * 60 * 1000);
|
||||
|
||||
try {
|
||||
const dnsTimestamps = metrics.dns.queryTimestamps.getData();
|
||||
const dnsTypes = metrics.dns.queryTypes.getData();
|
||||
const dnsResponseTimes = metrics.dns.responseTimes.getData();
|
||||
const filteredDnsTimestamps = dnsTimestamps.filter(ts => ts >= cutoff);
|
||||
const filteredCount = filteredDnsTimestamps.length;
|
||||
|
||||
// Filter process metrics by timestamp
|
||||
const memTimestamps = metrics.process.memory.timestamp.getData();
|
||||
const memFilteredIndices = memTimestamps.map((ts, idx) => ts >= cutoff ? idx : -1).filter(idx => idx !== -1);
|
||||
|
||||
return {
|
||||
dns: {
|
||||
queryTimestamps: filteredDnsTimestamps,
|
||||
queryTypes: dnsTypes.slice(-filteredCount),
|
||||
responseTimes: dnsResponseTimes.slice(-filteredCount)
|
||||
},
|
||||
requests: {
|
||||
timestamps: metrics.requests.timestamp.getData().filter(ts => ts >= cutoff),
|
||||
responseTimes: metrics.requests.responseTimes.getData().slice(-metrics.requests.timestamp.length())
|
||||
},
|
||||
peers: {
|
||||
events: metrics.peers.connectionEvents.getData().filter(e => e && e.timestamp && e.timestamp >= cutoff)
|
||||
},
|
||||
domains: {
|
||||
addEvents: metrics.domains.addEvents.getData().filter(e => e && e.timestamp && e.timestamp >= cutoff),
|
||||
removeEvents: metrics.domains.removeEvents.getData().filter(e => e && e.timestamp && e.timestamp >= cutoff)
|
||||
},
|
||||
holesail: {
|
||||
events: metrics.holesail.connectionEvents.getData().filter(e => e && e.timestamp && e.timestamp >= cutoff)
|
||||
},
|
||||
resources: {
|
||||
dataPoints: metrics.resources.dataPoints.getData().filter(dp => dp && dp.timestamp && dp.timestamp >= cutoff)
|
||||
},
|
||||
process: {
|
||||
memory: {
|
||||
heapUsed: metrics.process.memory.heapUsed.getData().filter((_, idx) => memFilteredIndices.includes(idx)),
|
||||
heapTotal: metrics.process.memory.heapTotal.getData().filter((_, idx) => memFilteredIndices.includes(idx)),
|
||||
rss: metrics.process.memory.rss.getData().filter((_, idx) => memFilteredIndices.includes(idx)),
|
||||
timestamps: memTimestamps.filter(ts => ts >= cutoff)
|
||||
},
|
||||
cpu: {
|
||||
percentage: metrics.process.cpu.percentage.getData().filter((_, idx) => {
|
||||
const cpuTimestamps = metrics.process.cpu.timestamp.getData();
|
||||
return cpuTimestamps[idx] && cpuTimestamps[idx] >= cutoff;
|
||||
}),
|
||||
timestamps: metrics.process.cpu.timestamp.getData().filter(ts => ts >= cutoff)
|
||||
},
|
||||
eventLoop: {
|
||||
lag: metrics.process.eventLoop.lag.getData().filter((_, idx) => {
|
||||
const loopTimestamps = metrics.process.eventLoop.timestamp.getData();
|
||||
return loopTimestamps[idx] && loopTimestamps[idx] >= cutoff;
|
||||
}),
|
||||
timestamps: metrics.process.eventLoop.timestamp.getData().filter(ts => ts >= cutoff)
|
||||
},
|
||||
system: {
|
||||
freeMemory: metrics.process.system.freeMemory.getData().filter((_, idx) => {
|
||||
const sysTimestamps = metrics.process.system.timestamp.getData();
|
||||
return sysTimestamps[idx] && sysTimestamps[idx] >= cutoff;
|
||||
}),
|
||||
usedMemoryPercent: metrics.process.system.usedMemoryPercentage.getData().filter((_, idx) => {
|
||||
const sysTimestamps = metrics.process.system.timestamp.getData();
|
||||
return sysTimestamps[idx] && sysTimestamps[idx] >= cutoff;
|
||||
}),
|
||||
loadAverage: metrics.process.system.loadAverage.getData().filter((_, idx) => {
|
||||
const sysTimestamps = metrics.process.system.timestamp.getData();
|
||||
return sysTimestamps[idx] && sysTimestamps[idx] >= cutoff;
|
||||
}),
|
||||
timestamps: metrics.process.system.timestamp.getData().filter(ts => ts >= cutoff)
|
||||
},
|
||||
handles: {
|
||||
active: metrics.process.handles.active.getData().filter((_, idx) => {
|
||||
const handleTimestamps = metrics.process.handles.timestamp.getData();
|
||||
return handleTimestamps[idx] && handleTimestamps[idx] >= cutoff;
|
||||
}),
|
||||
timestamps: metrics.process.handles.timestamp.getData().filter(ts => ts >= cutoff)
|
||||
}
|
||||
}
|
||||
};
|
||||
} catch (err) {
|
||||
// Return empty data structure if there's an error
|
||||
return {
|
||||
dns: { queryTimestamps: [], queryTypes: [], responseTimes: [] },
|
||||
requests: { timestamps: [], responseTimes: [] },
|
||||
peers: { events: [] },
|
||||
domains: { addEvents: [], removeEvents: [] },
|
||||
holesail: { events: [] },
|
||||
resources: { dataPoints: [] },
|
||||
process: {
|
||||
memory: { heapUsed: [], heapTotal: [], rss: [], timestamps: [] },
|
||||
cpu: { percentage: [], timestamps: [] },
|
||||
eventLoop: { lag: [], timestamps: [] },
|
||||
system: { freeMemory: [], usedMemoryPercent: [], loadAverage: [], timestamps: [] },
|
||||
handles: { active: [], timestamps: [] }
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track a request
|
||||
* @param {string} endpoint - API endpoint
|
||||
* @param {boolean} success - Whether request was successful
|
||||
*/
|
||||
function trackRequest(endpoint, success) {
|
||||
increment('requests', 'total');
|
||||
if (success) {
|
||||
increment('requests', 'successful');
|
||||
} else {
|
||||
increment('requests', 'failed');
|
||||
}
|
||||
|
||||
// Limit byEndpoint map size
|
||||
if (!metrics.requests.byEndpoint.has(endpoint)) {
|
||||
// If map is at limit, remove oldest entries (simple FIFO by removing first key)
|
||||
if (metrics.requests.byEndpoint.size >= metrics.requests.maxEndpoints) {
|
||||
const firstKey = metrics.requests.byEndpoint.keys().next().value;
|
||||
if (firstKey) {
|
||||
metrics.requests.byEndpoint.delete(firstKey);
|
||||
}
|
||||
}
|
||||
metrics.requests.byEndpoint.set(endpoint, { total: 0, successful: 0, failed: 0 });
|
||||
}
|
||||
const endpointMetrics = metrics.requests.byEndpoint.get(endpoint);
|
||||
endpointMetrics.total++;
|
||||
if (success) {
|
||||
endpointMetrics.successful++;
|
||||
} else {
|
||||
endpointMetrics.failed++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track DNS query
|
||||
* @param {string} type - Query type ('p2p', 'public', 'failure')
|
||||
*/
|
||||
function trackDNSQuery(type) {
|
||||
increment('dns', 'queries');
|
||||
if (type === 'p2p') {
|
||||
increment('dns', 'p2pResolutions');
|
||||
} else if (type === 'public') {
|
||||
increment('dns', 'publicResolutions');
|
||||
} else if (type === 'failure') {
|
||||
increment('dns', 'failures');
|
||||
}
|
||||
}
|
||||
|
||||
// Start aggregation on module load
|
||||
startMetricsAggregation();
|
||||
|
||||
module.exports = {
|
||||
increment,
|
||||
getMetrics,
|
||||
getHistoricalData,
|
||||
trackRequest,
|
||||
trackRequestWithTiming,
|
||||
trackDNSQuery,
|
||||
trackDNSQueryWithTiming,
|
||||
trackPeerEvent,
|
||||
trackDomainEvent,
|
||||
trackConsensusEvent,
|
||||
trackHolesailEvent,
|
||||
updateResourceMetrics,
|
||||
collectProcessMetrics,
|
||||
startProcessMetricsCollection,
|
||||
stopProcessMetricsCollection,
|
||||
aggregateMetrics,
|
||||
startMetricsAggregation,
|
||||
metrics,
|
||||
aggregatedMetrics
|
||||
};
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
const { logDebug, logWarn } = require('../infrastructure/logger');
|
||||
|
||||
// Resource tracking for proper cleanup
|
||||
class ResourceTracker {
|
||||
constructor() {
|
||||
this.resources = new Map();
|
||||
this.resourceId = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a resource for tracking
|
||||
* @param {string} type - Resource type (socket, timer, interval, server, etc.)
|
||||
* @param {object} resource - Resource object
|
||||
* @param {function} cleanupFn - Cleanup function
|
||||
* @returns {number} - Resource ID
|
||||
*/
|
||||
register(type, resource, cleanupFn) {
|
||||
const id = ++this.resourceId;
|
||||
this.resources.set(id, {
|
||||
type,
|
||||
resource,
|
||||
cleanupFn,
|
||||
registered: Date.now()
|
||||
});
|
||||
logDebug('ResourceTracker', `Registered ${type} resource (ID: ${id})`);
|
||||
return id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unregister a resource
|
||||
* @param {number} id - Resource ID
|
||||
*/
|
||||
unregister(id) {
|
||||
if (this.resources.has(id)) {
|
||||
const info = this.resources.get(id);
|
||||
logDebug('ResourceTracker', `Unregistered ${info.type} resource (ID: ${id})`);
|
||||
this.resources.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup a specific resource
|
||||
* @param {number} id - Resource ID
|
||||
* @returns {Promise<boolean>} - Success status
|
||||
*/
|
||||
async cleanup(id) {
|
||||
if (!this.resources.has(id)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const info = this.resources.get(id);
|
||||
try {
|
||||
if (info.cleanupFn) {
|
||||
await info.cleanupFn();
|
||||
} else if (info.resource) {
|
||||
// Default cleanup based on type
|
||||
if (info.type === 'timer' || info.type === 'interval') {
|
||||
clearTimeout(info.resource);
|
||||
clearInterval(info.resource);
|
||||
} else if (typeof info.resource.close === 'function') {
|
||||
await new Promise(resolve => {
|
||||
info.resource.close(resolve);
|
||||
setTimeout(resolve, 5000); // Timeout
|
||||
});
|
||||
} else if (typeof info.resource.destroy === 'function') {
|
||||
info.resource.destroy();
|
||||
}
|
||||
}
|
||||
this.resources.delete(id);
|
||||
logDebug('ResourceTracker', `Cleaned up ${info.type} resource (ID: ${id})`);
|
||||
return true;
|
||||
} catch (err) {
|
||||
logWarn('ResourceTracker', `Error cleaning up ${info.type} resource (ID: ${id}): ${err.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup all resources
|
||||
* @returns {Promise<number>} - Number of resources cleaned up
|
||||
*/
|
||||
async cleanupAll() {
|
||||
const ids = Array.from(this.resources.keys());
|
||||
let cleaned = 0;
|
||||
|
||||
for (const id of ids) {
|
||||
if (await this.cleanup(id)) {
|
||||
cleaned++;
|
||||
}
|
||||
}
|
||||
|
||||
logDebug('ResourceTracker', `Cleaned up ${cleaned}/${ids.length} resources`);
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all registered resources
|
||||
* @returns {Array} - Array of resource info
|
||||
*/
|
||||
list() {
|
||||
return Array.from(this.resources.entries()).map(([id, info]) => ({
|
||||
id,
|
||||
...info,
|
||||
age: Date.now() - info.registered
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get count of resources by type
|
||||
* @returns {object} - Counts by type
|
||||
*/
|
||||
getCounts() {
|
||||
const counts = {};
|
||||
for (const info of this.resources.values()) {
|
||||
counts[info.type] = (counts[info.type] || 0) + 1;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
}
|
||||
|
||||
// Global resource tracker instance
|
||||
const resourceTracker = new ResourceTracker();
|
||||
|
||||
module.exports = { resourceTracker, ResourceTracker };
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
const state = require('../infrastructure/state');
|
||||
const { logDebug, logWarn, logInfo } = require('../infrastructure/logger');
|
||||
const dgram = require('dgram');
|
||||
const { parseMinutesToMs, secondsToMs } = require('../infrastructure/utils');
|
||||
|
||||
let validationInterval = null;
|
||||
const VALIDATION_INTERVAL_MS = parseMinutesToMs(process.env.RESOURCE_VALIDATION_INTERVAL || '5'); // Default 5 minutes
|
||||
|
||||
/**
|
||||
* Validate state maps for stale entries and clean them up
|
||||
*/
|
||||
function validateStateMaps() {
|
||||
let cleanedCount = 0;
|
||||
|
||||
try {
|
||||
// Validate holesails map - check if connections are still valid
|
||||
const holesailKeysToRemove = [];
|
||||
for (const [key, holesail] of state.holesails.entries()) {
|
||||
try {
|
||||
// Check if it's a UDP socket
|
||||
if (holesail instanceof dgram.Socket) {
|
||||
// UDP sockets don't have a destroyed property, check if they're closed
|
||||
if (holesail.closed) {
|
||||
holesailKeysToRemove.push(key);
|
||||
}
|
||||
} else {
|
||||
// For TCP connections, check if destroyed or closed
|
||||
if (holesail.destroyed || (holesail.readyState && holesail.readyState === 'closed')) {
|
||||
holesailKeysToRemove.push(key);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
// If we can't check the connection, assume it's stale
|
||||
logWarn('ResourceValidation', `Error checking holesail connection ${key}: ${err.message}`);
|
||||
holesailKeysToRemove.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
// Remove stale holesail connections
|
||||
for (const key of holesailKeysToRemove) {
|
||||
logDebug('ResourceValidation', `Removing stale holesail connection: ${key}`);
|
||||
state.holesails.delete(key);
|
||||
if (state.holesailStartTimes) {
|
||||
state.holesailStartTimes.delete(key);
|
||||
}
|
||||
cleanedCount++;
|
||||
|
||||
// Also clean up associated resources
|
||||
const [domain, port] = key.split(':');
|
||||
if (state.tlsServers.has(key)) {
|
||||
try {
|
||||
const tlsServer = state.tlsServers.get(key);
|
||||
if (tlsServer && typeof tlsServer.close === 'function') {
|
||||
tlsServer.close();
|
||||
}
|
||||
} catch (err) {
|
||||
logDebug('ResourceValidation', `Error closing TLS server for ${key}: ${err.message}`);
|
||||
}
|
||||
state.tlsServers.delete(key);
|
||||
}
|
||||
|
||||
if (state.httpServers.has(key)) {
|
||||
try {
|
||||
const httpServer = state.httpServers.get(key);
|
||||
if (httpServer && typeof httpServer.close === 'function') {
|
||||
httpServer.close();
|
||||
}
|
||||
} catch (err) {
|
||||
logDebug('ResourceValidation', `Error closing HTTP server for ${key}: ${err.message}`);
|
||||
}
|
||||
state.httpServers.delete(key);
|
||||
}
|
||||
|
||||
// Clear timeout if exists
|
||||
if (state.holesailClientTimeouts.has(key)) {
|
||||
const timeout = state.holesailClientTimeouts.get(key);
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
state.holesailClientTimeouts.delete(key);
|
||||
}
|
||||
|
||||
// Remove from persistent connections
|
||||
if (state.persistentConnections) {
|
||||
state.persistentConnections.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
// Validate TLS servers - check if they're still listening
|
||||
const tlsKeysToRemove = [];
|
||||
for (const [key, tlsServer] of state.tlsServers.entries()) {
|
||||
try {
|
||||
// Check if server is closed or destroyed
|
||||
if (tlsServer && (tlsServer.destroyed || (tlsServer.listening === false && !tlsServer.pending))) {
|
||||
tlsKeysToRemove.push(key);
|
||||
}
|
||||
} catch (err) {
|
||||
logWarn('ResourceValidation', `Error checking TLS server ${key}: ${err.message}`);
|
||||
tlsKeysToRemove.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of tlsKeysToRemove) {
|
||||
logDebug('ResourceValidation', `Removing stale TLS server: ${key}`);
|
||||
state.tlsServers.delete(key);
|
||||
cleanedCount++;
|
||||
}
|
||||
|
||||
// Validate HTTP servers - check if they're still listening
|
||||
const httpKeysToRemove = [];
|
||||
for (const [key, httpServer] of state.httpServers.entries()) {
|
||||
try {
|
||||
// Check if server is closed or destroyed
|
||||
if (httpServer && (httpServer.destroyed || (httpServer.listening === false && !httpServer.pending))) {
|
||||
httpKeysToRemove.push(key);
|
||||
}
|
||||
} catch (err) {
|
||||
logWarn('ResourceValidation', `Error checking HTTP server ${key}: ${err.message}`);
|
||||
httpKeysToRemove.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of httpKeysToRemove) {
|
||||
logDebug('ResourceValidation', `Removing stale HTTP server: ${key}`);
|
||||
state.httpServers.delete(key);
|
||||
cleanedCount++;
|
||||
}
|
||||
|
||||
// Validate child processes - check if they're still alive
|
||||
const childKeysToRemove = [];
|
||||
for (const [id, child] of state.holesailChildren.entries()) {
|
||||
try {
|
||||
// Check if process has exited
|
||||
if (child.killed || child.exitCode !== null) {
|
||||
childKeysToRemove.push(id);
|
||||
}
|
||||
} catch (err) {
|
||||
logWarn('ResourceValidation', `Error checking child process ${id}: ${err.message}`);
|
||||
childKeysToRemove.push(id);
|
||||
}
|
||||
}
|
||||
|
||||
for (const id of childKeysToRemove) {
|
||||
logDebug('ResourceValidation', `Removing stale child process entry: ${id}`);
|
||||
state.holesailChildren.delete(id);
|
||||
if (state.holesailOpts.has(id)) {
|
||||
state.holesailOpts.delete(id);
|
||||
}
|
||||
if (state.holesailInfos.has(id)) {
|
||||
state.holesailInfos.delete(id);
|
||||
}
|
||||
if (state.holesailChildStartTimes && state.holesailChildStartTimes.has(id)) {
|
||||
state.holesailChildStartTimes.delete(id);
|
||||
}
|
||||
cleanedCount++;
|
||||
}
|
||||
|
||||
// Validate client child processes
|
||||
const clientChildKeysToRemove = [];
|
||||
for (const [id, child] of state.holesailClientChildren.entries()) {
|
||||
try {
|
||||
if (child.killed || child.exitCode !== null) {
|
||||
clientChildKeysToRemove.push(id);
|
||||
}
|
||||
} catch (err) {
|
||||
logWarn('ResourceValidation', `Error checking client child process ${id}: ${err.message}`);
|
||||
clientChildKeysToRemove.push(id);
|
||||
}
|
||||
}
|
||||
|
||||
for (const id of clientChildKeysToRemove) {
|
||||
logDebug('ResourceValidation', `Removing stale client child process entry: ${id}`);
|
||||
state.holesailClientChildren.delete(id);
|
||||
if (state.holesailClientOpts.has(id)) {
|
||||
state.holesailClientOpts.delete(id);
|
||||
}
|
||||
if (state.holesailClientInfos.has(id)) {
|
||||
state.holesailClientInfos.delete(id);
|
||||
}
|
||||
cleanedCount++;
|
||||
}
|
||||
|
||||
// Validate timeout map - remove entries for connections that no longer exist
|
||||
const timeoutKeysToRemove = [];
|
||||
for (const [key, timeout] of state.holesailClientTimeouts.entries()) {
|
||||
if (!state.holesails.has(key)) {
|
||||
timeoutKeysToRemove.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of timeoutKeysToRemove) {
|
||||
logDebug('ResourceValidation', `Clearing orphaned timeout for: ${key}`);
|
||||
const timeout = state.holesailClientTimeouts.get(key);
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
state.holesailClientTimeouts.delete(key);
|
||||
cleanedCount++;
|
||||
}
|
||||
|
||||
// Log summary
|
||||
if (cleanedCount > 0) {
|
||||
logInfo('ResourceValidation', `Cleaned up ${cleanedCount} stale resource(s)`);
|
||||
logDebug('ResourceValidation', `State map sizes: holesails=${state.holesails.size}, tlsServers=${state.tlsServers.size}, httpServers=${state.httpServers.size}, children=${state.holesailChildren.size}, clientChildren=${state.holesailClientChildren.size}`);
|
||||
} else {
|
||||
logDebug('ResourceValidation', 'No stale resources found');
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
logWarn('ResourceValidation', `Error during resource validation: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start periodic resource validation
|
||||
*/
|
||||
function startResourceValidation() {
|
||||
if (validationInterval) {
|
||||
clearInterval(validationInterval);
|
||||
}
|
||||
|
||||
// Run initial validation after a short delay
|
||||
setTimeout(() => {
|
||||
validateStateMaps();
|
||||
}, secondsToMs(60)); // Wait 1 minute before first validation
|
||||
|
||||
// Schedule periodic validation
|
||||
validationInterval = setInterval(() => {
|
||||
validateStateMaps();
|
||||
}, VALIDATION_INTERVAL_MS);
|
||||
|
||||
logInfo('ResourceValidation', `Started periodic resource validation (interval: ${VALIDATION_INTERVAL_MS}ms)`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop periodic resource validation
|
||||
*/
|
||||
function stopResourceValidation() {
|
||||
if (validationInterval) {
|
||||
clearInterval(validationInterval);
|
||||
validationInterval = null;
|
||||
logInfo('ResourceValidation', 'Stopped periodic resource validation');
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
validateStateMaps,
|
||||
startResourceValidation,
|
||||
stopResourceValidation
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user