fix(mesh): evict stale guild peers with health metrics

Track per-peer lastSeenAt, sweep idle hyperswarm sockets on an interval and before peer-retry, and expose peerRetryAttempt, lastPeerSeenAt, and eviction totals on guildSyncHealthRail. Co-authored-by: Cursor <[email protected]>
2026-06-02 11:23:54 -04:00
parent eb9122b7ad
commit 47732f25db
2 changed files with 233 additions and 5 deletions
@@ -13,6 +13,7 @@ Application facade: one `PearcordPlatform` class that wires identity, database,
 **Phase 410 hardening (v0.8.643):** `guildSyncHealthRail` now carries bounded-retry metadata (`retryAttempt`, `retryDelayMs`, `retryMaxAttempts`, `nextRetryAt`, `retryCountdownMs`) so UI can show live “mesh bounded mode” retry countdown/tooltip state during background join retries.
 **Phase 410 hardening (v0.8.643):** Guild mesh peer retry scheduling now uses `_computeGuildMeshPeerRetryDelayMs` (last-peer-contact staleness + RTT bounds) instead of fixed attempt-index spacing; health rows track `lastPeerSeenAt` on peer join.
 **Phase 410 hardening (v0.8.643):** `_requestGuildMeshJoin` coalesces duplicate guild mesh joins during `guild.open`, peer retry, and startup/switch races; stale `loadGen` joins are skipped; `pearcord-guild#joinMesh` deduplicates in-flight `swarm.join`. Smoke: `npm run test:platform-guild-mesh-join-coalesce`.
+**Phase 410 hardening (v0.8.643):** `_evictStaleGuildMeshPeers` drops idle hyperswarm sockets (env `PEARCORD_GUILD_MESH_PEER_STALE_MS`, periodic sweep) and records `peerRetryAttempt`, `lastPeerSeenAt`, and eviction totals on `guildSyncHealthRail`. Smoke: `npm run test:platform-guild-mesh-stale-peer-evict`.

 **Phase 659 (v0.8.638):** DM scheduled messages slice — `sendMessage` parses `/schedule` and `/sendlater` in DM mode; queue APIs `createDmScheduledMessage`, `listDmScheduledMessages`, `cancelDmScheduledMessage`, `sendNowDmScheduledMessage`, plus due runner `runDueDmScheduledMessages()` and local timer flush. See [DM_SCHEDULED_MESSAGES.md](../../docs/DM_SCHEDULED_MESSAGES.md).

@@ -501,6 +501,9 @@ class PearcordPlatform extends EventEmitter {
    this._memberPageBurstTimestamps = []
    this._guildMeshBoundedByGuild = new Map()
    this._guildMeshPeerRetryMetaByGuild = new Map()
+    this._guildMeshPeerSeenByGuild = new Map()
+    this._guildMeshStalePeerSweepTimer = null
+    this._guildMeshStalePeerEvictedTotalByGuild = new Map()
    this._lastPartitionHeal = null
    this._lastPartitionHealByGuild = new Map()
    this._partitionHealInProgressByGuild = new Set()
@@ -654,6 +657,14 @@ class PearcordPlatform extends EventEmitter {
          (Number(this._guildMeshPeerRetryMetaByGuild.get(id)?.nextRetryAt) || 0) - Date.now()
        ),
        lastPeerSeenAt: Number(h?.lastPeerSeenAt) || 0,
+        peerRetryAttempt:
+          Number(h?.peerRetryAttempt) ||
+          Number(this._guildMeshPeerRetryMetaByGuild.get(id)?.attempt) ||
+          0,
+        stalePeersEvictedLast: Number(h?.stalePeersEvictedLast) || 0,
+        stalePeersEvictedTotal:
+          Number(this._guildMeshStalePeerEvictedTotalByGuild.get(id)) || 0,
+        lastStalePeerEvictAt: Number(h?.lastStalePeerEvictAt) || 0,
        retrySpacingSource:
          this._guildMeshPeerRetryMetaByGuild.get(id)?.spacingSource || null
      }
@@ -661,6 +672,188 @@ class PearcordPlatform extends EventEmitter {
    return out
  }

+  _guildMeshPeerSeenMap (guildId) {
+    const gid = guildId || this.guild?.guild?.id
+    if (!gid) return null
+    let map = this._guildMeshPeerSeenByGuild.get(gid)
+    if (!map) {
+      map = new Map()
+      this._guildMeshPeerSeenByGuild.set(gid, map)
+    }
+    return map
+  }
+
+  _touchGuildMeshPeerSeen (guildId, peerId, source = 'activity') {
+    const gid = guildId || this.guild?.guild?.id
+    if (!gid || !peerId) return null
+    const map = this._guildMeshPeerSeenMap(gid)
+    const now = Date.now()
+    const prev = map.get(peerId) || { peerId, joinedAt: now, lastSeenAt: 0 }
+    const row = {
+      ...prev,
+      peerId,
+      joinedAt: prev.joinedAt || now,
+      lastSeenAt: now,
+      lastSource: source
+    }
+    map.set(peerId, row)
+    const health = this._guildSyncHealthByGuild.get(gid)
+    const lastPeerSeenAt = Math.max(Number(health?.lastPeerSeenAt) || 0, now)
+    this._patchGuildSyncHealth({
+      guildId: gid,
+      lastPeerSeenAt,
+      lastSuccessfulPeerContactAt: lastPeerSeenAt
+    })
+    return row
+  }
+
+  _touchAllGuildMeshPeersSeen (guildId, source = 'mesh-activity') {
+    const gid = guildId || this.guild?.guild?.id
+    if (!gid || !this.guild?.peers?.size) return 0
+    let n = 0
+    for (const peerId of this.guild.peers.keys()) {
+      this._touchGuildMeshPeerSeen(gid, peerId, source)
+      n++
+    }
+    return n
+  }
+
+  _forgetGuildMeshPeerSeen (guildId, peerId) {
+    const gid = guildId || this.guild?.guild?.id
+    if (!gid || !peerId) return
+    this._guildMeshPeerSeenByGuild.get(gid)?.delete(peerId)
+  }
+
+  _getGuildMeshPeerRetryAttempt (guildId) {
+    const gid = guildId || this.guild?.guild?.id
+    if (!gid) return 0
+    const meta = this._guildMeshPeerRetryMetaByGuild.get(gid)
+    if (this.guild?.guild?.id === gid) {
+      return Math.max(
+        Number(meta?.attempt) || 0,
+        Number(this._guildMeshPeerRetryAttempts) || 0
+      )
+    }
+    return Number(meta?.attempt) || 0
+  }
+
+  /**
+   * Drop hyperswarm sockets that stayed connected without mesh activity.
+   * @returns {{ evicted: number, peerRetryAttempt: number, lastPeerSeenAt: number }}
+   */
+  _evictStaleGuildMeshPeers (guildId, opts = {}) {
+    const gid = guildId || this.guild?.guild?.id
+    const out = { evicted: 0, peerRetryAttempt: 0, lastPeerSeenAt: 0 }
+    if (!gid || this.guild?.guild?.id !== gid || !this.guild?.peers?.size) {
+      return out
+    }
+    const staleMs = Number(process.env.PEARCORD_GUILD_MESH_PEER_STALE_MS) || 180000
+    const minAgeMs = Number(process.env.PEARCORD_GUILD_MESH_PEER_STALE_MIN_AGE_MS) || 45000
+    const now = Date.now()
+    const peerMap = this._guildMeshPeerSeenMap(gid)
+    const peerRetryAttempt = this._getGuildMeshPeerRetryAttempt(gid)
+    out.peerRetryAttempt = peerRetryAttempt
+
+    for (const [peerId, conn] of [...this.guild.peers.entries()]) {
+      const row = peerMap.get(peerId) || {
+        peerId,
+        joinedAt: now,
+        lastSeenAt: 0
+      }
+      const joinedAt = Number(row.joinedAt) || now
+      const lastSeenAt = Number(row.lastSeenAt) || 0
+      const connAgeMs = now - joinedAt
+      const idleMs = lastSeenAt ? now - lastSeenAt : connAgeMs
+      const idleThreshold = opts.force === true ? minAgeMs : staleMs
+      const stale = connAgeMs >= minAgeMs && idleMs >= idleThreshold
+      if (!stale) continue
+
+      try {
+        conn.destroy?.()
+      } catch {
+        // ignore
+      }
+      this.guild.peers.delete(peerId)
+      this.guild._channels?.delete(peerId)
+      this.guild._attachChannels?.delete(peerId)
+      this.guild._voiceChannels?.delete(peerId)
+      this.guild._screenChannels?.delete(peerId)
+      peerMap.delete(peerId)
+      this._trackGuildTopicPeer(gid, peerId, 'leave')
+      out.evicted += 1
+      this.log.info('guild mesh stale peer evicted', {
+        spanKind: 'guild.mesh.stale-peer-evict',
+        guildId: gid,
+        peerId,
+        peerRetryAttempt,
+        lastPeerSeenAt: lastSeenAt || 0,
+        idleMs,
+        connAgeMs,
+        source: opts.source || 'stale-sweep'
+      })
+    }
+
+    if (out.evicted > 0) {
+      const total =
+        (this._guildMeshStalePeerEvictedTotalByGuild.get(gid) || 0) + out.evicted
+      this._guildMeshStalePeerEvictedTotalByGuild.set(gid, total)
+      const health = this._guildSyncHealthByGuild.get(gid)
+      out.lastPeerSeenAt = Number(health?.lastPeerSeenAt) || 0
+      let maxSeen = 0
+      for (const row of peerMap.values()) {
+        maxSeen = Math.max(maxSeen, Number(row.lastSeenAt) || 0)
+      }
+      const peers = this.guild.peers?.size ?? 0
+      this._patchGuildSyncHealth({
+        guildId: gid,
+        peers,
+        peerRetryAttempt,
+        lastPeerSeenAt: maxSeen || out.lastPeerSeenAt,
+        stalePeersEvictedLast: out.evicted,
+        lastStalePeerEvictAt: now
+      })
+      this._logMeshChurn('stale-peer-evict', {
+        guildId: gid,
+        evicted: out.evicted,
+        peerRetryAttempt,
+        lastPeerSeenAt: maxSeen || out.lastPeerSeenAt,
+        meshPeerCount: peers
+      })
+      if (peers === 0) this._scheduleGuildMeshPeerRetry(gid)
+    }
+    return out
+  }
+
+  _maybeEvictStaleGuildMeshPeers (guildId, source = 'sweep') {
+    if (!guildId || this.guild?.guild?.id !== guildId) return { evicted: 0 }
+    if (!(this.guild?.peers?.size ?? 0)) return { evicted: 0 }
+    return this._evictStaleGuildMeshPeers(guildId, { source })
+  }
+
+  _stopGuildMeshStalePeerSweep () {
+    if (this._guildMeshStalePeerSweepTimer) {
+      clearInterval(this._guildMeshStalePeerSweepTimer)
+      this._guildMeshStalePeerSweepTimer = null
+    }
+  }
+
+  _startGuildMeshStalePeerSweep (guildId) {
+    this._stopGuildMeshStalePeerSweep()
+    const gid = guildId || this.guild?.guild?.id
+    if (!gid) return
+    const intervalMs = Number(process.env.PEARCORD_GUILD_MESH_PEER_SWEEP_MS) || 60000
+    this._guildMeshStalePeerSweepTimer = setInterval(() => {
+      if (this.guild?.guild?.id !== gid) {
+        this._stopGuildMeshStalePeerSweep()
+        return
+      }
+      this._maybeEvictStaleGuildMeshPeers(gid, 'interval-sweep')
+    }, intervalMs)
+    if (typeof this._guildMeshStalePeerSweepTimer.unref === 'function') {
+      this._guildMeshStalePeerSweepTimer.unref()
+    }
+  }
+
  /**
   * Mesh peer retry delay from time since last successful peer contact (not fixed attempt cadence).
   * @param {string} guildId
@@ -1747,6 +1940,7 @@ class PearcordPlatform extends EventEmitter {

  async _leaveMeshes () {
    this._abortGuildMeshJoinCoalesce('leave-meshes')
+    this._stopGuildMeshStalePeerSweep()
    this._stopAuditExportScheduleTimer()
    this._stopDmScheduledMessageTimer()
    await this._leaveVoice({ gossip: true })
@@ -8497,6 +8691,9 @@ class PearcordPlatform extends EventEmitter {
              messages: msgCount,
              members: memCount
            })
+            if (payload?.guildId === this.guild?.guild?.id) {
+              this._touchAllGuildMeshPeersSeen(payload.guildId, 'guild-sync')
+            }
            void this._refreshGuildSyncHealthCounts(payload?.guildId).catch(() => {})
            this._patchGuildSyncHealth({
              guildId: payload?.guildId,
@@ -8531,7 +8728,10 @@ class PearcordPlatform extends EventEmitter {
      if (type === 'leave') {
        const gid = this.guild?.guild?.id
        const peers = this.guild?.peers?.size ?? 0
-        if (gid && peerId) this._trackGuildTopicPeer(gid, peerId, 'leave')
+        if (gid && peerId) {
+          this._trackGuildTopicPeer(gid, peerId, 'leave')
+          this._forgetGuildMeshPeerSeen(gid, peerId)
+        }
        if (gid) {
          this._logMeshChurn('peer-leave', {
            guildId: gid,
@@ -8571,15 +8771,20 @@ class PearcordPlatform extends EventEmitter {
        )
        void this._flushGuildGossipOutbox({ reason: 'peer-join' }).catch(() => {})
        const gid = this.guild?.guild?.id
-        if (gid && peerId) this._trackGuildTopicPeer(gid, peerId, 'join')
+        if (gid && peerId) {
+          this._trackGuildTopicPeer(gid, peerId, 'join')
+          this._touchGuildMeshPeerSeen(gid, peerId, 'peer-join')
+        }
        if (gid) {
          const seenAt = Date.now()
          this._patchGuildSyncHealth({
            guildId: gid,
            peers: this.guild?.peers?.size ?? 0,
            lastPeerSeenAt: seenAt,
-            lastSuccessfulPeerContactAt: seenAt
+            lastSuccessfulPeerContactAt: seenAt,
+            peerRetryAttempt: this._getGuildMeshPeerRetryAttempt(gid)
          })
+          this._maybeEvictStaleGuildMeshPeers(gid, 'peer-join-sweep')
          this._logMeshChurn('peer-join', {
            guildId: gid,
            peerId: peerId || null,
@@ -10991,6 +11196,8 @@ class PearcordPlatform extends EventEmitter {
    this._guildSyncAcks.delete(gid)
    this._guildSyncPushHalted.delete(gid)
    this._guildSyncHealthByGuild.delete(gid)
+    this._guildMeshPeerSeenByGuild.delete(gid)
+    this._guildMeshStalePeerEvictedTotalByGuild.delete(gid)
    const rep = this._guildReplicators.get(gid)
    if (rep) {
      await rep.pruneGuildStorage().catch(() => {})
@@ -12893,7 +13100,10 @@ class PearcordPlatform extends EventEmitter {
    this._scheduleProfileGossipBurst()
    if ((this.guild.peers?.size ?? 0) === 0) {
      this._scheduleGuildMeshPeerRetry(guildRecord.id)
+    } else {
+      this._maybeEvictStaleGuildMeshPeers(guildRecord.id, 'guild-open')
    }
+    this._startGuildMeshStalePeerSweep(guildRecord.id)
    const voiceSpan = this.log.time('guild.open.voice')
    await this._initGuildVoice(guildRecord.id)
    voiceSpan.end()
@@ -13102,6 +13312,9 @@ class PearcordPlatform extends EventEmitter {

  _scheduleGuildMeshPeerRetry (guildId) {
    if (this._guildMeshPeerRetryTimer) return
+    if (this.guild?.guild?.id === guildId) {
+      this._maybeEvictStaleGuildMeshPeers(guildId, 'peer-retry-schedule')
+    }
    if ((this.guild?.peers?.size ?? 0) > 0) return
    this._guildMeshPeerRetryAttempts = 0
    const maxAttempts = Number(process.env.PEARCORD_GUILD_MESH_PEER_RETRY_MAX) || 8
@@ -13139,8 +13352,21 @@ class PearcordPlatform extends EventEmitter {
        Number(health?.lastPeerSeenAt) || 0,
        Number(health?.lastSuccessfulPeerContactAt) || 0
      )
+      const peerRetryAttempt = this._guildMeshPeerRetryAttempts
+      this._patchGuildSyncHealth({
+        guildId,
+        peerRetryAttempt,
+        lastPeerSeenAt: Number(health?.lastPeerSeenAt) || 0
+      })
+      this._maybeEvictStaleGuildMeshPeers(guildId, 'peer-retry-attempt')
+      if ((this.guild.peers?.size ?? 0) > 0) {
+        this._clearGuildMeshPeerRetry()
+        this._scheduleProfileGossipBurst()
+        return
+      }
      this.log.debug('guild mesh peer retry', {
-        attempt: this._guildMeshPeerRetryAttempts,
+        attempt: peerRetryAttempt,
+        peerRetryAttempt,
        guildId,
        delayMs,
        sinceLastPeerContactMs: lastContact ? Date.now() - lastContact : null,
@@ -13178,7 +13404,8 @@ class PearcordPlatform extends EventEmitter {
        delayMs,
        nextRetryAt: Date.now() + delayMs,
        maxAttempts,
-        spacingSource: 'last-peer-contact'
+        spacingSource: 'last-peer-contact',
+        peerRetryAttempt: this._guildMeshPeerRetryAttempts
      })
      this._guildMeshPeerRetryTimer = setTimeout(attempt, delayMs)
    }