Phase 826: guild 3-peer single-peer churn rejoin mesh recovery (v0.8.793)

Add partial mesh drop/reconnect helpers and runGuildMeshTripleChurnRejoinJourney with meshChurnSurvivorReady and meshRecoveryBurstReady matchers. Co-authored-by: Cursor <[email protected]>
2026-06-04 03:31:23 -04:00
parent efe0151a35
commit 08e946e70e
4 changed files with 204 additions and 0 deletions
@@ -2,6 +2,8 @@

 Agent control plane for Pearcord — send the same IPC messages the UI sends, read `view` snapshots, and run scripted journeys against a **live worker** or an in-process **headless** session.

+**Phase 826 (v0.8.793):** `runGuildMeshTripleChurnRejoinJourney` (3-peer single-peer churn rejoin). Bundle: `npm run test:ci-phase826`.
+
 **Phase 825 (v0.8.792):** `runGuildMeshTripleRecoveryBurstJourney` (3-peer composite recovery burst). Bundle: `npm run test:ci-phase825`.

 **Phase 824 (v0.8.791):** `runGuildMeshTripleSyncBurstJourney` (3-peer sync burst). Bundle: `npm run test:ci-phase824`.
@@ -93,6 +93,15 @@ function matchView (view, spec) {
      if (outbox > 0) return false
      continue
    }
+    if (key === 'meshChurnSurvivorReady') {
+      if (!expected) return true
+      const peers = Number(view?.stats?.peers) || 0
+      if (peers < 1) return false
+      const gid = view?.guild?.id
+      const rail = gid ? view.guildSyncHealthRail?.[gid] : null
+      if (rail?.meshBounded) return false
+      continue
+    }
    if (key === 'presencePeersMin') {
      if ((view?.presence?.peers || []).length < Number(expected)) return false
      continue
@@ -7153,6 +7153,108 @@ class HeadlessSession {
    }
  }

+  /** Phase 826: 3-peer mesh — drop one guest, survivors stay synced, rejoin + recovery burst. */
+  async runGuildMeshTripleChurnRejoinJourney () {
+    const {
+      connectGuildMeshTriple,
+      disconnectGuildMeshTripleOnePeer,
+      reconnectGuildMeshTripleOnePeer
+    } = require('./mesh-helpers')
+    const token1 = `churn826a-${Date.now()}`
+    const token2 = `churn826b-${Date.now()}`
+    const base = path.join(os.tmpdir(), `pearcord-agentctl-churn826-${Date.now()}`)
+    const host = new HeadlessSession({ storagePath: path.join(base, 'host') })
+    const guest1 = new HeadlessSession({ storagePath: path.join(base, 'guest1') })
+    const guest2 = new HeadlessSession({ storagePath: path.join(base, 'guest2') })
+    await host.start()
+    await guest1.start()
+    await guest2.start()
+    try {
+      await host.registerUser({ username: 'churnhost826', displayName: 'Churn Host 826' })
+      await host.createGuild({ name: 'Churn Rejoin Triple 826', publicListing: false })
+      await host.wait({ mode: 'guild' }, 45000)
+      const invite = await host.platform.createInvite().catch(() => null)
+      const code = invite?.shareCode || invite?.code
+      if (!code) throw new Error('host invite missing')
+      await guest1.registerUser({ username: 'churnguest826a', displayName: 'Churn Guest 826a' })
+      await guest2.registerUser({ username: 'churnguest826b', displayName: 'Churn Guest 826b' })
+      await guest1.ipc({ type: 'join-invite', code })
+      await guest2.ipc({ type: 'join-invite', code })
+      await guest1.wait({ mode: 'guild' }, 45000)
+      await guest2.wait({ mode: 'guild' }, 45000)
+      const chId = (await host.refreshView()).activeChannelId
+      if (!chId) throw new Error('active channel missing')
+      await guest1.ipc({ type: 'select-channel', channelId: chId })
+      await guest2.ipc({ type: 'select-channel', channelId: chId })
+      await connectGuildMeshTriple(
+        host.platform,
+        guest1.platform,
+        guest2.platform,
+        50000
+      )
+      await host.wait({ meshPeersMin: 1 }, 30000)
+      await guest1.wait({ meshPeersMin: 1 }, 30000)
+      await guest2.wait({ meshPeersMin: 1 }, 30000)
+      await host.sendGuildMessage(token1)
+      const msg1Wait = { activeChannelMessageIncludes: token1 }
+      await guest1.wait(msg1Wait, 90000)
+      await guest2.wait(msg1Wait, 90000)
+      await disconnectGuildMeshTripleOnePeer(
+        host.platform,
+        guest1.platform,
+        guest2.platform,
+        1
+      )
+      await guest1.wait({ meshPeersExact: 0 }, 30000)
+      await host.wait({ meshChurnSurvivorReady: true }, 30000)
+      await guest2.wait({ meshChurnSurvivorReady: true }, 30000)
+      await host.sendGuildMessage(token2)
+      const msg2Wait = { activeChannelMessageIncludes: token2 }
+      await guest2.wait(msg2Wait, 90000)
+      await reconnectGuildMeshTripleOnePeer(
+        host.platform,
+        guest1.platform,
+        guest2.platform,
+        1
+      )
+      const gid = guest1.platform.guild?.guild?.id
+      if (gid && typeof guest1.platform._refreshGuildSyncHealthCounts === 'function') {
+        await guest1.platform._refreshGuildSyncHealthCounts(gid).catch(() => null)
+      }
+      if (typeof guest1.platform.runOfflineRecoverySync === 'function') {
+        await guest1.platform.runOfflineRecoverySync().catch(() => null)
+      }
+      if (typeof guest1.platform.requestGuildSyncBurst === 'function') {
+        guest1.platform.requestGuildSyncBurst({ clearPushHalt: true })
+      } else if (typeof guest1.platform.requestGuildSyncFanout === 'function') {
+        guest1.platform.requestGuildSyncFanout()
+      }
+      const rejoinWait = {
+        meshRecoveryBurstReady: true,
+        meshPeersMin: 1,
+        activeChannelMessageIncludes: token2
+      }
+      await guest1.wait(rejoinWait, 90000)
+      await guest1.wait({ activeChannelMessageIncludes: token1 }, 90000)
+      const g1v = await guest1.refreshView()
+      const hv = await host.refreshView()
+      const g2v = await guest2.refreshView()
+      return {
+        token1,
+        token2,
+        droppedPeers: g1v.stats?.peers,
+        hostPeers: hv.stats?.peers,
+        guest2Peers: g2v.stats?.peers,
+        guest1Outbox: g1v.guildSyncHealth?.pendingGossipCount,
+        guest1PushHalted: g1v.meshStabilityStats?.pushHalted
+      }
+    } finally {
+      await host.close()
+      await guest1.close()
+      await guest2.close()
+    }
+  }
+
  /** Phase 813: 3-peer mesh — both guests see full roster (host + 2 joiners). */
  async runGuildMeshTripleMemberRosterJourney () {
    const { connectGuildMeshTriple } = require('./mesh-helpers')
@@ -124,6 +124,95 @@ async function disconnectGuildMeshTriple (host, guest1, guest2, timeoutMs = 1500
  return { hostPeers: 0, guest1Peers: 0, guest2Peers: 0 }
 }

+/**
+ * Drop one peer from a 3-peer guild mesh; the other two should remain connected.
+ * @param {number} dropIndex 0=host, 1=guest1, 2=guest2
+ */
+async function disconnectGuildMeshTripleOnePeer (
+  host,
+  guest1,
+  guest2,
+  dropIndex = 1,
+  timeoutMs = 15000
+) {
+  const peers = [host, guest1, guest2]
+  for (const p of peers) {
+    if (!p?.guild?.swarm) throw new Error('all peers must be on an active guild mesh')
+  }
+  if (dropIndex < 0 || dropIndex > 2) throw new Error('dropIndex must be 0..2')
+  const drop = peers[dropIndex]
+  const keys = peers.map((p) => p.guild.swarm.keyPair.publicKey)
+  const dropKey = keys[dropIndex]
+  for (let j = 0; j < 3; j++) {
+    if (j === dropIndex) continue
+    drop.guild.swarm.leavePeer(keys[j])
+    peers[j].guild.swarm.leavePeer(dropKey)
+  }
+  for (const p of peers) {
+    await p.guild.swarm.flush()
+  }
+  const okDrop = await waitFor(async () => {
+    const v = await drop.view()
+    return (v.stats?.peers || 0) === 0
+  }, timeoutMs, 350)
+  if (!okDrop) throw new Error('dropped peer still connected to guild mesh')
+  const survivors = peers.filter((_, i) => i !== dropIndex)
+  const okSurv = await waitFor(async () => {
+    const views = await Promise.all(survivors.map((p) => p.view()))
+    return views.every((v) => (v.stats?.peers || 0) >= 1)
+  }, timeoutMs, 350)
+  if (!okSurv) throw new Error('survivor peers lost mesh after single-peer drop')
+  await sleep(800)
+  const [sv0, sv1] = await Promise.all(survivors.map((p) => p.view()))
+  return {
+    dropIndex,
+    droppedPeers: 0,
+    survivor0Peers: sv0.stats?.peers,
+    survivor1Peers: sv1.stats?.peers
+  }
+}
+
+/**
+ * Reconnect one dropped peer into an otherwise-connected 3-peer guild mesh.
+ * @param {number} joinIndex 0=host, 1=guest1, 2=guest2
+ */
+async function reconnectGuildMeshTripleOnePeer (
+  host,
+  guest1,
+  guest2,
+  joinIndex = 1,
+  timeoutMs = 28000
+) {
+  const peers = [host, guest1, guest2]
+  for (const p of peers) {
+    if (!p?.guild?.swarm) throw new Error('all peers must be on an active guild mesh')
+  }
+  if (joinIndex < 0 || joinIndex > 2) throw new Error('joinIndex must be 0..2')
+  const join = peers[joinIndex]
+  const keys = peers.map((p) => p.guild.swarm.keyPair.publicKey)
+  const joinKey = keys[joinIndex]
+  for (let j = 0; j < 3; j++) {
+    if (j === joinIndex) continue
+    join.guild.swarm.joinPeer(keys[j])
+    peers[j].guild.swarm.joinPeer(joinKey)
+  }
+  for (const p of peers) {
+    await p.guild.swarm.flush()
+  }
+  const ok = await waitFor(async () => {
+    const views = await Promise.all(peers.map((p) => p.view()))
+    return views.every((v) => (v.stats?.peers || 0) >= 1)
+  }, timeoutMs, 400)
+  if (!ok) throw new Error('3-peer mesh did not reconnect after single-peer rejoin')
+  await sleep(1200)
+  const [hv, g1v, g2v] = await Promise.all(peers.map((p) => p.view()))
+  return {
+    hostPeers: hv.stats?.peers,
+    guest1Peers: g1v.stats?.peers,
+    guest2Peers: g2v.stats?.peers
+  }
+}
+
 /** @returns {null} isolated DHT wiring is optional for headless IPC smokes */
 function createIsolatedMeshDht () {
  return null
@@ -136,5 +225,7 @@ module.exports = {
  disconnectGuildMeshPeers,
  connectGuildMeshTriple,
  disconnectGuildMeshTriple,
+  disconnectGuildMeshTripleOnePeer,
+  reconnectGuildMeshTripleOnePeer,
  createIsolatedMeshDht
 }