380 lines
15 KiB
JavaScript
380 lines
15 KiB
JavaScript
const http = require('http');
|
|
const net = require('net');
|
|
const Holesail = require('holesail');
|
|
const state = require('../infrastructure/state');
|
|
const { logDebug, logInfo, logError, logWarn } = require('../infrastructure/logger');
|
|
const ca = require('../security/certificate_authority');
|
|
const { createTlsProxy } = require('./p2p_domains_proxy');
|
|
const { checkPortResponsive, parseMinutesToMs, parseSecondsToMs, secondsToMs } = require('../infrastructure/utils');
|
|
const { checkPortAvailability, freePort } = require('../maintenance/cleanup');
|
|
const { trackHolesailEvent } = require('../maintenance/metrics');
|
|
const { retryWithBackoff } = require('../infrastructure/async_errors');
|
|
const { getCircuitBreaker } = require('../infrastructure/circuit_breaker');
|
|
|
|
// Cleanup connection resources
|
|
async function cleanupConnection(key, domain, hash, ip, port, shouldRestart = false) {
|
|
logInfo('Holesail', `Cleaning up connection for ${key}`);
|
|
|
|
// Check if this was a persistent connection before cleanup
|
|
const wasPersistent = state.persistentConnections && state.persistentConnections.has(key);
|
|
|
|
// Remove from holesails map
|
|
const holesail = state.holesails.get(key);
|
|
if (holesail) {
|
|
try {
|
|
// Remove all event listeners before closing to prevent leaks
|
|
holesail.removeAllListeners();
|
|
await holesail.close();
|
|
} catch (err) {
|
|
logError('Holesail', `Error closing holesail client during cleanup for ${key}: ${err.message}`);
|
|
}
|
|
state.holesails.delete(key);
|
|
// Clean up start time tracking
|
|
if (state.holesailStartTimes) {
|
|
state.holesailStartTimes.delete(key);
|
|
}
|
|
}
|
|
|
|
// Clear timeout if it exists
|
|
const timeout = state.holesailClientTimeouts.get(key);
|
|
if (timeout) {
|
|
clearTimeout(timeout);
|
|
state.holesailClientTimeouts.delete(key);
|
|
}
|
|
|
|
// Remove from persistent connections
|
|
if (state.persistentConnections) {
|
|
state.persistentConnections.delete(key);
|
|
}
|
|
|
|
// Remove TLS server reference (but don't close - servers are shared across domains on same IP)
|
|
// The servers are created by createTlsProxy and shared per bind IP, not per domain
|
|
state.tlsServers.delete(key);
|
|
|
|
// Remove HTTP server reference (but don't close - servers are shared across domains on same IP)
|
|
// Note: The HTTP redirect server created separately for port 80 is also per-domain,
|
|
// but we don't close it here to avoid breaking other domains using the same IP
|
|
state.httpServers.delete(key);
|
|
|
|
// If this was a persistent connection and we should restart, attempt to restart it
|
|
// Skip restart if FULL_PERSISTENCE is enabled
|
|
if (wasPersistent && shouldRestart && process.env.FULL_PERSISTENCE !== 'true') {
|
|
logInfo('Holesail', `Attempting to restart persistent connection for ${key}`);
|
|
// Use a small delay to avoid immediate restart loops
|
|
setTimeout(async () => {
|
|
try {
|
|
await restartHolesailClient(domain, hash, ip, port);
|
|
} catch (err) {
|
|
logError('Holesail', `Failed to restart persistent connection for ${key}: ${err.message}`);
|
|
}
|
|
}, 2000);
|
|
}
|
|
}
|
|
|
|
// Start Holesail client
|
|
async function startHolesailClient(domain, hash, ip, port, persistent = false) {
|
|
if (!ip) {
|
|
logDebug('Holesail', `Invalid IP for domain: ${domain}`);
|
|
return;
|
|
}
|
|
const key = `${domain}:${port}`;
|
|
if (state.holesails.has(key)) {
|
|
logDebug('Holesail', `Holesail client already exists for ${key}`);
|
|
return;
|
|
}
|
|
let startPromise = state.starting.get(key);
|
|
if (!startPromise) {
|
|
startPromise = (async () => {
|
|
logInfo('Holesail', `Starting Holesail client for domain: ${domain}, hash: ${hash}, IP: ${ip}, Port: ${port}`);
|
|
try {
|
|
// Check port availability
|
|
try {
|
|
await checkPortAvailability(ip, port);
|
|
} catch (err) {
|
|
logWarn('Holesail', `Port check failed initially for ${ip}:${port}: ${err.message}. Attempting to free the port.`);
|
|
const freed = await freePort(ip, port);
|
|
if (!freed) {
|
|
throw new Error(`Unable to free port ${port} on ${ip} after attempt`);
|
|
}
|
|
logInfo('Holesail', `Successfully freed port ${port} on ${ip}`);
|
|
}
|
|
// Use circuit breaker for Holesail connections
|
|
const circuitBreaker = getCircuitBreaker(`holesail-${domain}`, {
|
|
failureThreshold: 3,
|
|
resetTimeout: secondsToMs(30) // 30 seconds
|
|
});
|
|
|
|
const holesail = new Holesail({
|
|
client: true,
|
|
key: hash,
|
|
port: port,
|
|
host: ip,
|
|
log: false
|
|
});
|
|
|
|
try {
|
|
const startTime = Date.now();
|
|
// Use retry with backoff and circuit breaker
|
|
await circuitBreaker.execute(async () => {
|
|
return await retryWithBackoff(
|
|
async () => {
|
|
await holesail.ready();
|
|
return holesail;
|
|
},
|
|
{
|
|
maxRetries: 3,
|
|
initialDelay: 1000,
|
|
maxDelay: 5000,
|
|
shouldRetry: (err) => {
|
|
// Retry on connection errors, not on auth errors
|
|
return err.message && (
|
|
err.message.includes('ECONNREFUSED') ||
|
|
err.message.includes('ETIMEDOUT') ||
|
|
err.message.includes('ENOTFOUND')
|
|
);
|
|
}
|
|
}
|
|
);
|
|
}, `holesail-${domain}`);
|
|
|
|
state.holesails.set(key, holesail);
|
|
// Track start time for uptime calculation
|
|
if (!state.holesailStartTimes) {
|
|
state.holesailStartTimes = new Map();
|
|
}
|
|
state.holesailStartTimes.set(key, Date.now());
|
|
if (!state.persistentConnections) {
|
|
state.persistentConnections = new Set();
|
|
}
|
|
// If FULL_PERSISTENCE is enabled, make all connections persistent
|
|
if (persistent || process.env.FULL_PERSISTENCE === 'true') {
|
|
state.persistentConnections.add(key);
|
|
}
|
|
trackHolesailEvent('client', 'start', 'tcp', null);
|
|
logInfo('Holesail', `Holesail client for ${key} connected on ${ip}:${port}`);
|
|
|
|
// Add error and close event handlers to detect connection failures
|
|
holesail.on('error', async (err) => {
|
|
logError('Holesail', `Connection error for ${key}: ${err.message}`);
|
|
// Only restart if it was a persistent connection and FULL_PERSISTENCE is not enabled
|
|
const shouldRestart = state.persistentConnections && state.persistentConnections.has(key) && process.env.FULL_PERSISTENCE !== 'true';
|
|
await cleanupConnection(key, domain, hash, ip, port, shouldRestart);
|
|
});
|
|
|
|
holesail.on('close', async () => {
|
|
logWarn('Holesail', `Connection closed for ${key}`);
|
|
// Only restart if it was a persistent connection and FULL_PERSISTENCE is not enabled
|
|
const shouldRestart = state.persistentConnections && state.persistentConnections.has(key) && process.env.FULL_PERSISTENCE !== 'true';
|
|
await cleanupConnection(key, domain, hash, ip, port, shouldRestart);
|
|
});
|
|
|
|
if (process.env.DISABLE_PROXY_SERVER !== 'true') {
|
|
// Start TLS proxy
|
|
const { tlsServer, httpServer: tlsHttpServer } = createTlsProxy(domain, ip, port, logDebug);
|
|
state.tlsServers.set(key, tlsServer);
|
|
// Start HTTP redirect server (only for port 80)
|
|
if (port === parseInt(process.env.HTTP_PORT || 80)) {
|
|
const httpServer = http.createServer((req, res) => {
|
|
const host = req.headers.host || domain;
|
|
res.writeHead(301, { 'Location': `https://${host}${req.url}` });
|
|
res.end();
|
|
});
|
|
try {
|
|
await checkPortAvailability(ip, port); // Check again for HTTP server
|
|
httpServer.listen(port, ip, () => {
|
|
logInfo('Holesail', `HTTP redirect server for ${key} listening on ${ip}:${port}`);
|
|
});
|
|
state.httpServers.set(key, httpServer);
|
|
} catch (err) {
|
|
logError('Holesail', `Failed to start HTTP redirect server for ${key}: ${err.message}`);
|
|
}
|
|
}
|
|
} else {
|
|
logInfo('Holesail', `Skipping TLS proxy and HTTP redirect server for ${key} due to DISABLE_PROXY_SERVER=true`);
|
|
}
|
|
// Skip timeout if FULL_PERSISTENCE is enabled or connection is persistent
|
|
if (!persistent && process.env.FULL_PERSISTENCE !== 'true') {
|
|
const timeout = setTimeout(async () => {
|
|
// Check if connection still exists before cleaning up (may have been cleaned up already)
|
|
if (!state.holesails.has(key)) {
|
|
return;
|
|
}
|
|
const duration = Date.now() - startTime;
|
|
logInfo('Holesail', `Closing Holesail client for ${key}`);
|
|
try {
|
|
// Remove listeners before closing
|
|
if (holesail) {
|
|
holesail.removeAllListeners();
|
|
await holesail.close();
|
|
}
|
|
state.holesails.delete(key);
|
|
if (state.holesailStartTimes) {
|
|
state.holesailStartTimes.delete(key);
|
|
}
|
|
state.holesailClientTimeouts.delete(key);
|
|
trackHolesailEvent('client', 'stop', 'tcp', duration);
|
|
// Remove server references (but don't close - servers are shared across domains on same IP)
|
|
// The servers are created by createTlsProxy and shared per bind IP, not per domain
|
|
// Closing them would break all domains using that IP, including the internal proxy server
|
|
state.tlsServers.delete(key);
|
|
state.httpServers.delete(key);
|
|
} catch (err) {
|
|
logError('Holesail', `Error closing Holesail client for ${key}: ${err.message}`);
|
|
}
|
|
}, parseMinutesToMs(process.env.HOLESAIL_TIMEOUT || '5'));
|
|
state.holesailClientTimeouts.set(key, timeout);
|
|
}
|
|
} catch (err) {
|
|
logError('Holesail', `Error connecting Holesail client for ${key}: ${err.message}`);
|
|
}
|
|
} catch (err) {
|
|
logError('Holesail', `Failed to start Holesail client for ${key}: ${err.message}`);
|
|
} finally {
|
|
state.starting.delete(key);
|
|
}
|
|
})();
|
|
state.starting.set(key, startPromise);
|
|
}
|
|
await startPromise;
|
|
}
|
|
|
|
// Restart Holesail client
|
|
async function restartHolesailClient(domain, hash, ip, port) {
|
|
if (!ip) {
|
|
logDebug('Holesail', `Invalid IP for domain: ${domain}`);
|
|
return;
|
|
}
|
|
const key = `${domain}:${port}`;
|
|
const isResponsive = await checkPortResponsive(ip, port);
|
|
if (isResponsive) {
|
|
logDebug('Holesail', `Port ${port} on ${ip} is responsive, using the existing connection.`);
|
|
// Ensure this connection is marked as persistent
|
|
if (!state.persistentConnections) {
|
|
state.persistentConnections = new Set();
|
|
}
|
|
state.persistentConnections.add(key);
|
|
// Clear any existing timeout to make it truly persistent
|
|
const timeout = state.holesailClientTimeouts.get(key);
|
|
if (timeout) {
|
|
clearTimeout(timeout);
|
|
state.holesailClientTimeouts.delete(key);
|
|
logDebug('Holesail', `Cleared timeout for ${key} to make it persistent`);
|
|
}
|
|
return;
|
|
}
|
|
logInfo('Holesail', `Port ${port} on ${ip} is unresponsive, closing and recreating the Holesail client`);
|
|
|
|
// Use the cleanup function to ensure all resources are properly closed
|
|
// Don't auto-restart here since we'll restart manually below
|
|
await cleanupConnection(key, domain, hash, ip, port, false);
|
|
|
|
if (state.starting.has(key)) {
|
|
await state.starting.get(key);
|
|
}
|
|
try {
|
|
// In restart, also attempt to free if needed, but since we closed, it should be free, but to be safe
|
|
try {
|
|
await checkPortAvailability(ip, port);
|
|
} catch (err) {
|
|
logWarn('Holesail', `Port still in use after close for ${ip}:${port}: ${err.message}. Attempting to free.`);
|
|
const freed = await freePort(ip, port);
|
|
if (!freed) {
|
|
throw new Error(`Unable to free port ${port} on ${ip} during restart`);
|
|
}
|
|
logInfo('Holesail', `Successfully freed port ${port} on ${ip} during restart`);
|
|
}
|
|
// Restart as persistent connection (no timeout)
|
|
await startHolesailClient(domain, hash, ip, port, true);
|
|
} catch (err) {
|
|
logError('Holesail', `Failed to restart Holesail client for ${key}: ${err.message}`);
|
|
}
|
|
}
|
|
|
|
// Periodic health check for all active connections
|
|
let healthCheckInterval = null;
|
|
|
|
function startHealthChecks() {
|
|
const intervalMs = parseSecondsToMs(process.env.HOLESAIL_HEALTH_CHECK_INTERVAL || '30'); // Default 30 seconds
|
|
|
|
if (healthCheckInterval) {
|
|
clearInterval(healthCheckInterval);
|
|
}
|
|
|
|
healthCheckInterval = setInterval(async () => {
|
|
const keysToCheck = Array.from(state.holesails.keys());
|
|
|
|
if (keysToCheck.length === 0) {
|
|
return;
|
|
}
|
|
|
|
logDebug('Holesail', `Running health check on ${keysToCheck.length} connections`);
|
|
|
|
for (const key of keysToCheck) {
|
|
const [domain, portStr] = key.split(':');
|
|
const port = parseInt(portStr);
|
|
const ip = state.domainToIPMap.get(domain);
|
|
|
|
if (!ip) {
|
|
logWarn('Holesail', `No IP found for domain ${domain} during health check`);
|
|
continue;
|
|
}
|
|
|
|
// Get hash for domain
|
|
// Skip health check if DNS service is not initialized
|
|
if (!state.dnsPass) {
|
|
logDebug('Holesail', `DNS service not initialized, skipping health check for ${key}`);
|
|
continue;
|
|
}
|
|
|
|
const { getHashForDomain } = require('../core/core');
|
|
const hash = await getHashForDomain(domain);
|
|
|
|
if (!hash) {
|
|
logWarn('Holesail', `No hash found for domain ${domain} during health check`);
|
|
continue;
|
|
}
|
|
|
|
// Check if port is responsive
|
|
const isResponsive = await checkPortResponsive(ip, port);
|
|
|
|
if (!isResponsive) {
|
|
logWarn('Holesail', `Health check failed for ${key}, connection is unresponsive`);
|
|
|
|
// Skip health check restarts if FULL_PERSISTENCE is enabled
|
|
if (process.env.FULL_PERSISTENCE === 'true') {
|
|
logDebug('Holesail', `FULL_PERSISTENCE is enabled, skipping health check restart for ${key}`);
|
|
continue;
|
|
}
|
|
|
|
// If it's a persistent connection, restart it
|
|
const isPersistent = state.persistentConnections && state.persistentConnections.has(key);
|
|
if (isPersistent) {
|
|
logInfo('Holesail', `Restarting unresponsive persistent connection ${key}`);
|
|
try {
|
|
await restartHolesailClient(domain, hash, ip, port);
|
|
} catch (err) {
|
|
logError('Holesail', `Failed to restart connection ${key} during health check: ${err.message}`);
|
|
}
|
|
} else {
|
|
// For non-persistent connections, just clean up
|
|
logInfo('Holesail', `Cleaning up unresponsive non-persistent connection ${key}`);
|
|
await cleanupConnection(key, domain, hash, ip, port, false);
|
|
}
|
|
} else {
|
|
logDebug('Holesail', `Health check passed for ${key}`);
|
|
}
|
|
}
|
|
}, intervalMs);
|
|
|
|
logInfo('Holesail', `Started periodic health checks (interval: ${intervalMs}ms)`);
|
|
}
|
|
|
|
function stopHealthChecks() {
|
|
if (healthCheckInterval) {
|
|
clearInterval(healthCheckInterval);
|
|
healthCheckInterval = null;
|
|
logInfo('Holesail', 'Stopped periodic health checks');
|
|
}
|
|
}
|
|
|
|
module.exports = { startHolesailClient, restartHolesailClient, startHealthChecks, stopHealthChecks }; |