Phase 865 (v0.8.832): multi-guild production matrix + presence heal

Add runGuildMeshMultiGuildProductionPresenceJourney with presence vector ingest and partition heal on primary quad while secondary guilds stay isolated. Co-authored-by: Cursor <[email protected]>
2026-06-04 07:35:11 -04:00
parent 73c7197a9f
commit 5787798092
3 changed files with 293 additions and 0 deletions
@@ -2,6 +2,8 @@

 Agent control plane for Pearcord — send the same IPC messages the UI sends, read `view` snapshots, and run scripted journeys against a **live worker** or an in-process **headless** session.

+**Phase 865 (v0.8.832):** `runGuildMeshMultiGuildProductionPresenceJourney` (production matrix + presence heal). Bundle: `npm run test:ci-phase865`.
+
 **Phase 864 (v0.8.831):** `runGuildMeshMultiGuildProductionGossipJourney` (production matrix + gossip outbox). Bundle: `npm run test:ci-phase864`.

 **Phase 863 (v0.8.830):** `runGuildMeshMultiGuildProductionMatrixJourney` (multi-guild production matrix + replication lag). Bundle: `npm run test:ci-phase863`.
@@ -337,6 +337,20 @@ function matchView (view, spec) {
      if (outbox > 12) return false
      continue
    }
+    if (key === 'meshMultiGuildProductionPresenceReady') {
+      if (!expected) return true
+      if (!!view?.partitionHealInProgress) return false
+      if (!view?.lastPartitionHeal?.at) return false
+      if (!!view?.guildSyncHealth?.pending) return false
+      const rail = view?.guildSyncHealthRail || {}
+      if (!Object.keys(rail).length && view?.mode !== 'guild') return false
+      if ((view?.presence?.peers || []).length < 1 && view?.mode === 'guild') {
+        return false
+      }
+      const outbox = Number(view?.guildSyncHealth?.pendingGossipCount) || 0
+      if (outbox > 12) return false
+      continue
+    }
    if (key === 'meshQuadUltimateRosterScaleDmHealReady') {
      if (!expected) return true
      const peers = Number(view?.stats?.peers) || 0
@@ -8814,6 +8814,283 @@ class HeadlessSession {
    }
  }

+  /** Phase 865: multi-guild production matrix + presence vector heal (864 + Phase 831 / P654-30). */
+  async runGuildMeshMultiGuildProductionPresenceJourney () {
+    const {
+      connectGuildMeshQuad,
+      disconnectGuildMeshQuadOnePeer,
+      reconnectGuildMeshQuadOnePeer
+    } = require('./mesh-helpers')
+    const contractMode = process.env.PEARCORD_SKIP_MESH_ROUNDTRIP === '1'
+    const liveQuad =
+      process.env.PEARCORD_PHASE654_HEAL_FULL === '1' &&
+      process.env.PEARCORD_SKIP_MESH_ROUNDTRIP !== '1'
+    const guildCount = Math.max(
+      3,
+      Number(process.env.PEARCORD_PHASE865_GUILD_COUNT) ||
+        Number(process.env.PEARCORD_PHASE864_GUILD_COUNT) ||
+        4
+    )
+    const secondaryCount = guildCount - 1
+    const maxLagMs = Math.max(
+      0,
+      Number(process.env.PEARCORD_PHASE865_MAX_REPLICATION_LAG_MS) ||
+        Number(process.env.PEARCORD_PHASE864_MAX_REPLICATION_LAG_MS) ||
+        120000
+    )
+    const outboxMax = Math.max(
+      0,
+      Number(process.env.PEARCORD_PHASE865_GOSSIP_OUTBOX_MAX) ||
+        Number(process.env.PEARCORD_PHASE864_GOSSIP_OUTBOX_MAX) ||
+        12
+    )
+    const memberCount = contractMode
+      ? Math.max(
+          100,
+          Number(process.env.PEARCORD_PHASE865_MEMBER_COUNT) ||
+            Number(process.env.PEARCORD_PHASE864_MEMBER_COUNT) ||
+            120
+        )
+      : Math.max(
+          280,
+          Number(process.env.PEARCORD_PHASE865_MEMBER_COUNT) ||
+            Number(process.env.PEARCORD_PHASE864_MEMBER_COUNT) ||
+            300
+        )
+    const membersMin = contractMode
+      ? Math.max(80, memberCount - 20)
+      : Math.max(280, memberCount - 20)
+    const waitMs = memberCount >= 280 ? 150000 : 90000
+    const primaryToken = `mg865pri_${Date.now()}`
+    const presToken = `mg865pres_${Date.now()}`
+    const base = path.join(os.tmpdir(), `pearcord-agentctl-mg865-${Date.now()}`)
+    const host = new HeadlessSession({ storagePath: path.join(base, 'host') })
+    const guest1 = new HeadlessSession({ storagePath: path.join(base, 'guest1') })
+    const guest2 = new HeadlessSession({ storagePath: path.join(base, 'guest2') })
+    const guest3 = new HeadlessSession({ storagePath: path.join(base, 'guest3') })
+    const peerSessions = []
+    const secondaryIds = []
+    const allTokens = []
+    const flushGossipFor = async (sess, gid, label) => {
+      if (typeof sess.platform._flushGuildGossipOutbox === 'function') {
+        await sess.platform
+          ._flushGuildGossipOutbox({ reason: `mg865-${label}` })
+          .catch(() => null)
+      }
+      if (gid && typeof sess.platform._refreshGuildSyncHealthCounts === 'function') {
+        await sess.platform._refreshGuildSyncHealthCounts(gid).catch(() => null)
+      }
+      if (typeof sess.platform.requestGuildSyncFanout === 'function') {
+        sess.platform.requestGuildSyncFanout()
+      }
+      const v = await sess.refreshView()
+      const outbox = Number(v?.guildSyncHealth?.pendingGossipCount) || 0
+      if (outbox > outboxMax) {
+        throw new Error(`gossip outbox ${outbox} > ${outboxMax} on ${label}`)
+      }
+    }
+    const assertReplicationLag = (gid, label) => {
+      const meshDiag = host.platform.exportMeshReplicationDiagnostics(gid)
+      const lag = Number(meshDiag?.maxReplicationLagMs) || 0
+      if (lag > maxLagMs) {
+        throw new Error(`replication lag ${lag}ms > ${maxLagMs}ms on ${label}`)
+      }
+    }
+    const assertNoCrossLeak = async (gid) => {
+      await host.loadGuild(gid)
+      await host.wait({ mode: 'guild' }, 45000)
+      const v = await host.refreshView()
+      for (const row of allTokens) {
+        if (row.gid === gid) continue
+        if ((v.messages || []).some((m) => String(m?.content || '').includes(row.token))) {
+          throw new Error(`cross-topic leak on ${gid}`)
+        }
+      }
+    }
+    await host.start()
+    try {
+      await host.registerUser({ username: 'mg865host', displayName: 'MG865 Presence Host' })
+      await host.createGuild({ name: 'MG865 Primary Presence', publicListing: false })
+      await host.wait({ mode: 'guild' }, 45000)
+      const primaryGid = host.platform.guild?.guild?.id
+      if (!primaryGid) throw new Error('primary guild id missing')
+      allTokens.push({ gid: primaryGid, token: primaryToken })
+      await host.sendGuildMessage(primaryToken)
+      for (let i = 0; i < secondaryCount; i++) {
+        await host.createGuild({ name: `MG865 Secondary ${i}`, publicListing: false })
+        await host.wait({ mode: 'guild' }, 45000)
+        const sid = host.platform.guild?.guild?.id
+        if (!sid) throw new Error(`secondary ${i} id missing`)
+        secondaryIds.push(sid)
+        const tok = `mg865sec${i}_${Date.now()}`
+        allTokens.push({ gid: sid, token: tok })
+        await host.sendGuildMessage(tok)
+      }
+      for (const row of allTokens) {
+        await assertNoCrossLeak(row.gid)
+        assertReplicationLag(row.gid, `pre-${row.gid}`)
+        await flushGossipFor(host, row.gid, 'pre')
+      }
+      await host.loadGuild(primaryGid)
+      await host.runLargeGuildMemberJourney(primaryGid, memberCount)
+      await host.platform.runPartitionHealSync({
+        force: true,
+        source: 'agentctl-mg865-roster'
+      })
+      let guest2UserId = null
+      if (!contractMode) {
+        await guest1.start()
+        await guest2.start()
+        await guest3.start()
+        peerSessions.push(guest1, guest2, guest3)
+        const invite = await host.platform.createInvite().catch(() => null)
+        const code = invite?.shareCode || invite?.code
+        if (!code) throw new Error('invite missing')
+        for (const [sess, uname, dname] of [
+          [guest1, 'mg865a', 'Guest a'],
+          [guest2, 'mg865b', 'Guest b'],
+          [guest3, 'mg865c', 'Guest c']
+        ]) {
+          await sess.registerUser({ username: uname, displayName: dname })
+          await sess.ipc({ type: 'join-invite', code })
+          await sess.wait({ mode: 'guild' }, 45000)
+        }
+        const chId = (await host.refreshView()).activeChannelId
+        if (!chId) throw new Error('channel missing')
+        for (const sess of [guest1, guest2, guest3]) {
+          await sess.ipc({ type: 'select-channel', channelId: chId })
+        }
+        await connectGuildMeshQuad(
+          host.platform,
+          guest1.platform,
+          guest2.platform,
+          guest3.platform,
+          55000
+        )
+        const quadWait = { meshQuadReady: true, membersMin: 4 }
+        for (const sess of [host, guest1, guest2, guest3]) {
+          await sess.wait(quadWait, 30000)
+        }
+        guest2UserId = (await guest2.refreshView()).user?.id
+        if (!guest2UserId) throw new Error('guest2 user id missing')
+        await host.sendGuildMessage(presToken)
+        const msgWait = { activeChannelMessageIncludes: presToken }
+        await guest1.wait(msgWait, 90000)
+        await guest3.wait(msgWait, 90000)
+        for (const sess of [host, guest1, guest2, guest3]) {
+          await flushGossipFor(sess, primaryGid, 'quad')
+        }
+        if (liveQuad) {
+          await disconnectGuildMeshQuadOnePeer(
+            host.platform,
+            guest1.platform,
+            guest2.platform,
+            guest3.platform,
+            2
+          )
+          await guest2.wait({ meshPeersExact: 0 }, 30000)
+          await host.platform.runPartitionHealSync({
+            force: true,
+            source: 'agentctl-mg865-churn'
+          })
+          await reconnectGuildMeshQuadOnePeer(
+            host.platform,
+            guest1.platform,
+            guest2.platform,
+            guest3.platform,
+            2
+          )
+        }
+        const t0 = Date.now()
+        if (typeof host.platform._ingestPresenceVectorBundle === 'function') {
+          await host.platform._ingestPresenceVectorBundle(primaryGid, [
+            { userId: guest2UserId, status: 'dnd', at: t0 - 5000, vectorSeq: 24 }
+          ])
+          await host.platform._ingestPresenceVectorBundle(primaryGid, [
+            { userId: guest2UserId, status: 'online', at: t0, vectorSeq: 24 }
+          ])
+        }
+        await host.platform.runPartitionHealSync({
+          force: true,
+          source: 'agentctl-mg865-presence-heal'
+        })
+        for (const sid of secondaryIds) {
+          await assertNoCrossLeak(sid)
+          await flushGossipFor(host, sid, 'pres-rotate')
+        }
+        await host.loadGuild(primaryGid)
+        await host.ipc({ type: 'set-presence', status: 'idle' })
+        await new Promise((r) => setTimeout(r, 1500))
+        for (const p of [host.platform, guest1.platform, guest3.platform]) {
+          if (typeof p.requestGuildSyncFanout === 'function') p.requestGuildSyncFanout()
+        }
+        const presWait = { peerPresenceStatus: { userId: guest2UserId, status: 'online' } }
+        await guest1.wait(presWait, 90000)
+        await guest3.wait(presWait, 90000)
+        const hostIdleWait = {
+          peerPresenceStatus: { userId: (await host.refreshView()).user?.id, status: 'idle' }
+        }
+        if (hostIdleWait.peerPresenceStatus.userId) {
+          await guest1.wait(hostIdleWait, 60000).catch(() => null)
+        }
+      } else {
+        const hostUid = (await host.refreshView()).user?.id
+        if (
+          hostUid &&
+          typeof host.platform._ingestPresenceVectorBundle === 'function'
+        ) {
+          await host.platform._ingestPresenceVectorBundle(primaryGid, [
+            { userId: hostUid, status: 'online', at: Date.now(), vectorSeq: 1 }
+          ])
+        }
+      }
+      await host.platform.runPartitionHealSync({
+        force: true,
+        source: 'agentctl-mg865-final'
+      })
+      for (const row of allTokens) {
+        await assertNoCrossLeak(row.gid)
+        await flushGossipFor(host, row.gid, 'final')
+      }
+      await host.loadGuild(primaryGid)
+      const presMatrixWait = {
+        meshMultiGuildProductionPresenceReady: true,
+        meshMultiGuildProductionGossipReady: true,
+        meshMultiGuildProductionMatrixReady: true,
+        partitionHealComplete: true,
+        guildSyncNotPending: true,
+        guildSyncGossipOutboxMax: outboxMax,
+        membersMin,
+        presencePeersMin: contractMode ? 0 : 1
+      }
+      await host.wait(presMatrixWait, waitMs)
+      if (!contractMode && guest2UserId) {
+        for (const sess of [guest1, guest3]) {
+          await sess.wait(
+            {
+              meshMultiGuildProductionPresenceReady: true,
+              peerPresenceStatus: { userId: guest2UserId, status: 'online' }
+            },
+            90000
+          )
+        }
+      }
+      return {
+        guildCount,
+        primaryGid,
+        secondaryIds,
+        guest2UserId,
+        presToken,
+        multiFinal: host.exportMultiGuildSyncDiagnostics()
+      }
+    } finally {
+      for (const sess of peerSessions) {
+        await sess.close().catch(() => null)
+      }
+      await host.close()
+    }
+  }
+
  /** Phase 864: multi-guild production matrix + gossip outbox / sync pending (863 + Phase 840). */
  async runGuildMeshMultiGuildProductionGossipJourney () {
    const {