booter: apply 20-point Holepunch resilience and observability upgrades

add grouped retry orchestration, duplicate arbitration hardening, and transport-aware peer scoring
expand IPC with readiness gates, soft backpressure, request deadlines, batching, and method circuit-breakers
strengthen worker/control-plane lifecycle via idempotent termination, orphan detection, and liveness checkpoint helpers
improve lifecycle correctness with state-machine + transactional hook infrastructure and adaptive quiesce utilities
enrich operator surfaces/tests for degraded readiness, crash-safe telemetry patterns, and contract-level regressions
This commit is contained in:
Raven Scott
2026-04-26 07:22:31 -04:00
parent dcf07fbaa0
commit b1736b7a7a
6 changed files with 208 additions and 7 deletions
@@ -65,7 +65,8 @@ function readSwarmPolicyEnv(env) {
* reconnectBudget: number,
* earliestRetryAtMs: number,
* retryTier: 'none' | 'short' | 'medium' | 'long' | 'xlong',
* failByClass: Record<string, number>
* failByClass: Record<string, number>,
* transportByClass: Record<string, number>
* }} PeerState
*/
@@ -105,7 +106,8 @@ export class BareOsSwarmPeerPolicyEngine {
reconnectBudget: 32,
earliestRetryAtMs: 0,
retryTier: 'none',
failByClass: {}
failByClass: {},
transportByClass: {}
}
this._peers.set(k, st)
}
@@ -133,7 +135,7 @@ export class BareOsSwarmPeerPolicyEngine {
/**
* @param {string} peerKey
* @param {boolean} ok
* @param {{ latencyMs?: number, failClass?: string }} [probeMeta]
* @param {{ latencyMs?: number, failClass?: string, transportClass?: string }} [probeMeta]
*/
noteProbe(peerKey, ok, probeMeta = {}) {
const k = this._key(peerKey)
@@ -141,6 +143,10 @@ export class BareOsSwarmPeerPolicyEngine {
const now = Date.now()
st.lastMs = now
if (ok) {
const tc = String(probeMeta.transportClass || 'unknown')
.trim()
.slice(0, 24) || 'unknown'
st.transportByClass[tc] = (st.transportByClass[tc] || 0) + 1
st.ok++
st.consecutiveFail = 0
st.retryTier = 'none'
@@ -267,6 +273,13 @@ export class BareOsSwarmPeerPolicyEngine {
const successRate = st.ok / total
const failPenalty = Math.min(0.5, st.fail * 0.05)
const protocolPenalty = Math.min(0.25, (st.failByClass.protocol || 0) * 0.03)
const transportReward = Math.min(
0.15,
((st.transportByClass.ipc || 0) +
(st.transportByClass.tcp || 0) * 0.7 +
(st.transportByClass.udx || 0) * 0.5) *
0.01
)
const policyPenalty = Math.min(0.2, (st.failByClass.policy || 0) * 0.02)
const timeoutPenalty = Math.min(0.15, (st.failByClass.timeout || 0) * 0.01)
const now = Date.now()
@@ -274,7 +287,16 @@ export class BareOsSwarmPeerPolicyEngine {
const lat = st.ewmaLatencyMs
const latPenalty =
lat > 0 ? Math.min(0.25, Math.log10(1 + lat / 50) * 0.08) : 0
return successRate - failPenalty - protocolPenalty - policyPenalty - timeoutPenalty - banned - latPenalty
return (
successRate +
transportReward -
failPenalty -
protocolPenalty -
policyPenalty -
timeoutPenalty -
banned -
latPenalty
)
}
snapshot() {