const http = require('http'); const net = require('net'); const Holesail = require('holesail'); const state = require('../infrastructure/state'); const { logDebug, logInfo, logError, logWarn } = require('../infrastructure/logger'); const ca = require('../security/certificate_authority'); const { createTlsProxy } = require('./p2p_domains_proxy'); const { checkPortResponsive, parseMinutesToMs, parseSecondsToMs, secondsToMs } = require('../infrastructure/utils'); const { checkPortAvailability, freePort } = require('../maintenance/cleanup'); const { trackHolesailEvent } = require('../maintenance/metrics'); const { retryWithBackoff } = require('../infrastructure/async_errors'); const { getCircuitBreaker } = require('../infrastructure/circuit_breaker'); // Cleanup connection resources async function cleanupConnection(key, domain, hash, ip, port, shouldRestart = false) { logInfo('Holesail', `Cleaning up connection for ${key}`); // Check if this was a persistent connection before cleanup const wasPersistent = state.persistentConnections && state.persistentConnections.has(key); // Remove from holesails map const holesail = state.holesails.get(key); if (holesail) { try { // Remove all event listeners before closing to prevent leaks holesail.removeAllListeners(); await holesail.close(); } catch (err) { logError('Holesail', `Error closing holesail client during cleanup for ${key}: ${err.message}`); } state.holesails.delete(key); // Clean up start time tracking if (state.holesailStartTimes) { state.holesailStartTimes.delete(key); } } // Clear timeout if it exists const timeout = state.holesailClientTimeouts.get(key); if (timeout) { clearTimeout(timeout); state.holesailClientTimeouts.delete(key); } // Remove from persistent connections if (state.persistentConnections) { state.persistentConnections.delete(key); } // Remove TLS server reference (but don't close - servers are shared across domains on same IP) // The servers are created by createTlsProxy and shared per bind IP, not per domain state.tlsServers.delete(key); // Remove HTTP server reference (but don't close - servers are shared across domains on same IP) // Note: The HTTP redirect server created separately for port 80 is also per-domain, // but we don't close it here to avoid breaking other domains using the same IP state.httpServers.delete(key); // If this was a persistent connection and we should restart, attempt to restart it // Skip restart if FULL_PERSISTENCE is enabled if (wasPersistent && shouldRestart && process.env.FULL_PERSISTENCE !== 'true') { logInfo('Holesail', `Attempting to restart persistent connection for ${key}`); // Use a small delay to avoid immediate restart loops setTimeout(async () => { try { await restartHolesailClient(domain, hash, ip, port); } catch (err) { logError('Holesail', `Failed to restart persistent connection for ${key}: ${err.message}`); } }, 2000); } } // Start Holesail client async function startHolesailClient(domain, hash, ip, port, persistent = false) { if (!ip) { logDebug('Holesail', `Invalid IP for domain: ${domain}`); return; } const key = `${domain}:${port}`; if (state.holesails.has(key)) { logDebug('Holesail', `Holesail client already exists for ${key}`); return; } let startPromise = state.starting.get(key); if (!startPromise) { startPromise = (async () => { logInfo('Holesail', `Starting Holesail client for domain: ${domain}, hash: ${hash}, IP: ${ip}, Port: ${port}`); try { // Check port availability try { await checkPortAvailability(ip, port); } catch (err) { logWarn('Holesail', `Port check failed initially for ${ip}:${port}: ${err.message}. Attempting to free the port.`); const freed = await freePort(ip, port); if (!freed) { throw new Error(`Unable to free port ${port} on ${ip} after attempt`); } logInfo('Holesail', `Successfully freed port ${port} on ${ip}`); } // Use circuit breaker for Holesail connections const circuitBreaker = getCircuitBreaker(`holesail-${domain}`, { failureThreshold: 3, resetTimeout: secondsToMs(30) // 30 seconds }); const holesail = new Holesail({ client: true, key: hash, port: port, host: ip, log: false }); try { const startTime = Date.now(); // Use retry with backoff and circuit breaker await circuitBreaker.execute(async () => { return await retryWithBackoff( async () => { await holesail.ready(); return holesail; }, { maxRetries: 3, initialDelay: 1000, maxDelay: 5000, shouldRetry: (err) => { // Retry on connection errors, not on auth errors return err.message && ( err.message.includes('ECONNREFUSED') || err.message.includes('ETIMEDOUT') || err.message.includes('ENOTFOUND') ); } } ); }, `holesail-${domain}`); state.holesails.set(key, holesail); // Track start time for uptime calculation if (!state.holesailStartTimes) { state.holesailStartTimes = new Map(); } state.holesailStartTimes.set(key, Date.now()); if (!state.persistentConnections) { state.persistentConnections = new Set(); } // If FULL_PERSISTENCE is enabled, make all connections persistent if (persistent || process.env.FULL_PERSISTENCE === 'true') { state.persistentConnections.add(key); } trackHolesailEvent('client', 'start', 'tcp', null); logInfo('Holesail', `Holesail client for ${key} connected on ${ip}:${port}`); // Add error and close event handlers to detect connection failures holesail.on('error', async (err) => { logError('Holesail', `Connection error for ${key}: ${err.message}`); // Only restart if it was a persistent connection and FULL_PERSISTENCE is not enabled const shouldRestart = state.persistentConnections && state.persistentConnections.has(key) && process.env.FULL_PERSISTENCE !== 'true'; await cleanupConnection(key, domain, hash, ip, port, shouldRestart); }); holesail.on('close', async () => { logWarn('Holesail', `Connection closed for ${key}`); // Only restart if it was a persistent connection and FULL_PERSISTENCE is not enabled const shouldRestart = state.persistentConnections && state.persistentConnections.has(key) && process.env.FULL_PERSISTENCE !== 'true'; await cleanupConnection(key, domain, hash, ip, port, shouldRestart); }); if (process.env.DISABLE_PROXY_SERVER !== 'true') { // Start TLS proxy const { tlsServer, httpServer: tlsHttpServer } = createTlsProxy(domain, ip, port, logDebug); state.tlsServers.set(key, tlsServer); // Start HTTP redirect server (only for port 80) if (port === parseInt(process.env.HTTP_PORT || 80)) { const httpServer = http.createServer((req, res) => { const host = req.headers.host || domain; res.writeHead(301, { 'Location': `https://${host}${req.url}` }); res.end(); }); try { await checkPortAvailability(ip, port); // Check again for HTTP server httpServer.listen(port, ip, () => { logInfo('Holesail', `HTTP redirect server for ${key} listening on ${ip}:${port}`); }); state.httpServers.set(key, httpServer); } catch (err) { logError('Holesail', `Failed to start HTTP redirect server for ${key}: ${err.message}`); } } } else { logInfo('Holesail', `Skipping TLS proxy and HTTP redirect server for ${key} due to DISABLE_PROXY_SERVER=true`); } // Skip timeout if FULL_PERSISTENCE is enabled or connection is persistent if (!persistent && process.env.FULL_PERSISTENCE !== 'true') { const timeout = setTimeout(async () => { // Check if connection still exists before cleaning up (may have been cleaned up already) if (!state.holesails.has(key)) { return; } const duration = Date.now() - startTime; logInfo('Holesail', `Closing Holesail client for ${key}`); try { // Remove listeners before closing if (holesail) { holesail.removeAllListeners(); await holesail.close(); } state.holesails.delete(key); if (state.holesailStartTimes) { state.holesailStartTimes.delete(key); } state.holesailClientTimeouts.delete(key); trackHolesailEvent('client', 'stop', 'tcp', duration); // Remove server references (but don't close - servers are shared across domains on same IP) // The servers are created by createTlsProxy and shared per bind IP, not per domain // Closing them would break all domains using that IP, including the internal proxy server state.tlsServers.delete(key); state.httpServers.delete(key); } catch (err) { logError('Holesail', `Error closing Holesail client for ${key}: ${err.message}`); } }, parseMinutesToMs(process.env.HOLESAIL_TIMEOUT || '5')); state.holesailClientTimeouts.set(key, timeout); } } catch (err) { logError('Holesail', `Error connecting Holesail client for ${key}: ${err.message}`); } } catch (err) { logError('Holesail', `Failed to start Holesail client for ${key}: ${err.message}`); } finally { state.starting.delete(key); } })(); state.starting.set(key, startPromise); } await startPromise; } // Restart Holesail client async function restartHolesailClient(domain, hash, ip, port) { if (!ip) { logDebug('Holesail', `Invalid IP for domain: ${domain}`); return; } const key = `${domain}:${port}`; const isResponsive = await checkPortResponsive(ip, port); if (isResponsive) { logDebug('Holesail', `Port ${port} on ${ip} is responsive, using the existing connection.`); // Ensure this connection is marked as persistent if (!state.persistentConnections) { state.persistentConnections = new Set(); } state.persistentConnections.add(key); // Clear any existing timeout to make it truly persistent const timeout = state.holesailClientTimeouts.get(key); if (timeout) { clearTimeout(timeout); state.holesailClientTimeouts.delete(key); logDebug('Holesail', `Cleared timeout for ${key} to make it persistent`); } return; } logInfo('Holesail', `Port ${port} on ${ip} is unresponsive, closing and recreating the Holesail client`); // Use the cleanup function to ensure all resources are properly closed // Don't auto-restart here since we'll restart manually below await cleanupConnection(key, domain, hash, ip, port, false); if (state.starting.has(key)) { await state.starting.get(key); } try { // In restart, also attempt to free if needed, but since we closed, it should be free, but to be safe try { await checkPortAvailability(ip, port); } catch (err) { logWarn('Holesail', `Port still in use after close for ${ip}:${port}: ${err.message}. Attempting to free.`); const freed = await freePort(ip, port); if (!freed) { throw new Error(`Unable to free port ${port} on ${ip} during restart`); } logInfo('Holesail', `Successfully freed port ${port} on ${ip} during restart`); } // Restart as persistent connection (no timeout) await startHolesailClient(domain, hash, ip, port, true); } catch (err) { logError('Holesail', `Failed to restart Holesail client for ${key}: ${err.message}`); } } // Periodic health check for all active connections let healthCheckInterval = null; function startHealthChecks() { const intervalMs = parseSecondsToMs(process.env.HOLESAIL_HEALTH_CHECK_INTERVAL || '30'); // Default 30 seconds if (healthCheckInterval) { clearInterval(healthCheckInterval); } healthCheckInterval = setInterval(async () => { const keysToCheck = Array.from(state.holesails.keys()); if (keysToCheck.length === 0) { return; } logDebug('Holesail', `Running health check on ${keysToCheck.length} connections`); for (const key of keysToCheck) { const [domain, portStr] = key.split(':'); const port = parseInt(portStr); const ip = state.domainToIPMap.get(domain); if (!ip) { logWarn('Holesail', `No IP found for domain ${domain} during health check`); continue; } // Get hash for domain // Skip health check if DNS service is not initialized if (!state.dnsPass) { logDebug('Holesail', `DNS service not initialized, skipping health check for ${key}`); continue; } const { getHashForDomain } = require('../core/core'); const hash = await getHashForDomain(domain); if (!hash) { logWarn('Holesail', `No hash found for domain ${domain} during health check`); continue; } // Check if port is responsive const isResponsive = await checkPortResponsive(ip, port); if (!isResponsive) { logWarn('Holesail', `Health check failed for ${key}, connection is unresponsive`); // Skip health check restarts if FULL_PERSISTENCE is enabled if (process.env.FULL_PERSISTENCE === 'true') { logDebug('Holesail', `FULL_PERSISTENCE is enabled, skipping health check restart for ${key}`); continue; } // If it's a persistent connection, restart it const isPersistent = state.persistentConnections && state.persistentConnections.has(key); if (isPersistent) { logInfo('Holesail', `Restarting unresponsive persistent connection ${key}`); try { await restartHolesailClient(domain, hash, ip, port); } catch (err) { logError('Holesail', `Failed to restart connection ${key} during health check: ${err.message}`); } } else { // For non-persistent connections, just clean up logInfo('Holesail', `Cleaning up unresponsive non-persistent connection ${key}`); await cleanupConnection(key, domain, hash, ip, port, false); } } else { logDebug('Holesail', `Health check passed for ${key}`); } } }, intervalMs); logInfo('Holesail', `Started periodic health checks (interval: ${intervalMs}ms)`); } function stopHealthChecks() { if (healthCheckInterval) { clearInterval(healthCheckInterval); healthCheckInterval = null; logInfo('Holesail', 'Stopped periodic health checks'); } } module.exports = { startHolesailClient, restartHolesailClient, startHealthChecks, stopHealthChecks };