Phase 864 (v0.8.831): multi-guild production matrix + gossip outbox

Add runGuildMeshMultiGuildProductionGossipJourney with per-guild gossip flush,
outbox caps, and meshMultiGuildProductionGossipReady matcher (863 + Phase 840).

Co-authored-by: Cursor <[email protected]>

README.md

@@ -2,6 +2,8 @@
 
 Agent control plane for Pearcord — send the same IPC messages the UI sends, read `view` snapshots, and run scripted journeys against a **live worker** or an in-process **headless** session.
 
+**Phase 864 (v0.8.831):** `runGuildMeshMultiGuildProductionGossipJourney` (production matrix + gossip outbox). Bundle: `npm run test:ci-phase864`.
+
 **Phase 863 (v0.8.830):** `runGuildMeshMultiGuildProductionMatrixJourney` (multi-guild production matrix + replication lag). Bundle: `npm run test:ci-phase863`.
 
 **Phase 862 (v0.8.829):** `runGuildMeshMultiGuildScaleDmCompositeJourney` (multi-guild scale + DM-during-heal composite). Bundle: `npm run test:ci-phase862`.

assert.js

@@ -322,6 +322,21 @@ function matchView (view, spec) {
       if (outbox > 16) return false
       continue
     }
+    if (key === 'meshMultiGuildProductionGossipReady') {
+      if (!expected) return true
+      if (!!view?.partitionHealInProgress) return false
+      if (!view?.lastPartitionHeal?.at) return false
+      if (!!view?.guildSyncHealth?.pending) return false
+      const rail = view?.guildSyncHealthRail || {}
+      if (!Object.keys(rail).length && view?.mode !== 'guild') return false
+      const peers = Number(view?.stats?.peers) || 0
+      if (peers > 0 && peers < 2) return false
+      const rosterN = (view?.members || []).length
+      if (rosterN < 4 && view?.mode === 'guild') return false
+      const outbox = Number(view?.guildSyncHealth?.pendingGossipCount) || 0
+      if (outbox > 12) return false
+      continue
+    }
     if (key === 'meshQuadUltimateRosterScaleDmHealReady') {
       if (!expected) return true
       const peers = Number(view?.stats?.peers) || 0

headless.js

@@ -8814,6 +8814,320 @@ class HeadlessSession {
     }
   }
 
+  /** Phase 864: multi-guild production matrix + gossip outbox / sync pending (863 + Phase 840). */
+  async runGuildMeshMultiGuildProductionGossipJourney () {
+    const {
+      connectGuildMeshQuad,
+      disconnectGuildMeshQuadOnePeer,
+      reconnectGuildMeshQuadOnePeer
+    } = require('./mesh-helpers')
+    const contractMode = process.env.PEARCORD_SKIP_MESH_ROUNDTRIP === '1'
+    const liveQuad =
+      process.env.PEARCORD_PHASE654_HEAL_FULL === '1' &&
+      process.env.PEARCORD_SKIP_MESH_ROUNDTRIP !== '1'
+    const guildCount = Math.max(
+      3,
+      Number(process.env.PEARCORD_PHASE864_GUILD_COUNT) ||
+        Number(process.env.PEARCORD_PHASE863_GUILD_COUNT) ||
+        4
+    )
+    const secondaryCount = guildCount - 1
+    const maxLagMs = Math.max(
+      0,
+      Number(process.env.PEARCORD_PHASE864_MAX_REPLICATION_LAG_MS) ||
+        Number(process.env.PEARCORD_PHASE863_MAX_REPLICATION_LAG_MS) ||
+        120000
+    )
+    const outboxMax = Math.max(
+      0,
+      Number(process.env.PEARCORD_PHASE864_GOSSIP_OUTBOX_MAX) || 12
+    )
+    const memberCount = contractMode
+      ? Math.max(
+          100,
+          Number(process.env.PEARCORD_PHASE864_MEMBER_COUNT) ||
+            Number(process.env.PEARCORD_PHASE863_MEMBER_COUNT) ||
+            120
+        )
+      : Math.max(
+          280,
+          Number(process.env.PEARCORD_PHASE864_MEMBER_COUNT) ||
+            Number(process.env.PEARCORD_PHASE863_MEMBER_COUNT) ||
+            300
+        )
+    const membersMin = contractMode
+      ? Math.max(80, memberCount - 20)
+      : Math.max(280, memberCount - 20)
+    const waitMs = memberCount >= 280 ? 150000 : 90000
+    const hostLabel = 'MG Production Gossip Host 864'
+    const primaryToken = `mg864pri_${Date.now()}`
+    const dmToken = `mg864dm_${Date.now()}`
+    const churnToken = `mg864churn_${Date.now()}`
+    const gossipToken = `mg864gossip_${Date.now()}`
+    const base = path.join(os.tmpdir(), `pearcord-agentctl-mg864-${Date.now()}`)
+    const host = new HeadlessSession({ storagePath: path.join(base, 'host') })
+    const guest1 = new HeadlessSession({ storagePath: path.join(base, 'guest1') })
+    const guest2 = new HeadlessSession({ storagePath: path.join(base, 'guest2') })
+    const guest3 = new HeadlessSession({ storagePath: path.join(base, 'guest3') })
+    const peerSessions = []
+    const secondaryIds = []
+    const gossipByGuild = {}
+    const flushGossipFor = async (sess, gid, label) => {
+      if (typeof sess.platform._flushGuildGossipOutbox === 'function') {
+        await sess.platform
+          ._flushGuildGossipOutbox({ reason: `mg864-${label}` })
+          .catch(() => null)
+      }
+      if (gid && typeof sess.platform._refreshGuildSyncHealthCounts === 'function') {
+        await sess.platform._refreshGuildSyncHealthCounts(gid).catch(() => null)
+      }
+      if (typeof sess.platform.requestGuildSyncFanout === 'function') {
+        sess.platform.requestGuildSyncFanout()
+      }
+      const v = await sess.refreshView()
+      const outbox = Number(v?.guildSyncHealth?.pendingGossipCount) || 0
+      if (outbox > outboxMax) {
+        throw new Error(`gossip outbox ${outbox} > ${outboxMax} on ${label}`)
+      }
+      gossipByGuild[gid || label] = { outbox, pending: !!v?.guildSyncHealth?.pending }
+    }
+    const assertReplicationLag = (gid, label) => {
+      const meshDiag = host.platform.exportMeshReplicationDiagnostics(gid)
+      const lag = Number(meshDiag?.maxReplicationLagMs) || 0
+      if (lag > maxLagMs) {
+        throw new Error(`replication lag ${lag}ms > ${maxLagMs}ms on ${label}`)
+      }
+      const lanes = meshDiag?.gossipLaneCounts || null
+      gossipByGuild[gid] = { ...(gossipByGuild[gid] || {}), lag, lanes }
+      return meshDiag
+    }
+    const allTokens = []
+    const assertNoCrossLeak = async (gid) => {
+      await host.loadGuild(gid)
+      await host.wait({ mode: 'guild' }, 45000)
+      const v = await host.refreshView()
+      for (const row of allTokens) {
+        if (row.gid === gid) continue
+        const leak = (v.messages || []).some((m) =>
+          String(m?.content || m?.body || '').includes(row.token)
+        )
+        if (leak) {
+          throw new Error(`cross-topic leak: guild ${gid} saw ${row.token}`)
+        }
+      }
+    }
+    await host.start()
+    try {
+      await host.registerUser({ username: 'mg864host', displayName: hostLabel })
+      await host.createGuild({
+        name: 'MG864 Primary Gossip',
+        publicListing: false
+      })
+      await host.wait({ mode: 'guild' }, 45000)
+      const primaryGid = host.platform.guild?.guild?.id
+      if (!primaryGid) throw new Error('primary guild id missing')
+      allTokens.push({ gid: primaryGid, token: primaryToken })
+      await host.sendGuildMessage(primaryToken)
+      for (let i = 0; i < secondaryCount; i++) {
+        await host.createGuild({
+          name: `MG864 Secondary ${i}`,
+          publicListing: false
+        })
+        await host.wait({ mode: 'guild' }, 45000)
+        const sid = host.platform.guild?.guild?.id
+        if (!sid) throw new Error(`secondary guild ${i} id missing`)
+        secondaryIds.push(sid)
+        const tok = `mg864sec${i}_${Date.now()}`
+        allTokens.push({ gid: sid, token: tok })
+        await host.sendGuildMessage(tok)
+      }
+      const multi0 = host.exportMultiGuildSyncDiagnostics()
+      const listed = multi0?.guilds || multi0?.entries || []
+      if ((Array.isArray(listed) ? listed.length : 0) < guildCount) {
+        throw new Error('multi-guild diagnostics missing guild rows')
+      }
+      for (const row of allTokens) {
+        await assertNoCrossLeak(row.gid)
+        assertReplicationLag(row.gid, `pre-${row.gid}`)
+        await flushGossipFor(host, row.gid, `pre-${row.gid}`)
+      }
+      await host.loadGuild(primaryGid)
+      await host.wait({ mode: 'guild' }, 45000)
+      await host.runLargeGuildMemberJourney(primaryGid, memberCount)
+      const healRoster = await host.platform.runPartitionHealSync({
+        force: true,
+        source: 'agentctl-mg864-roster'
+      })
+      if (healRoster?.throttled) throw new Error('roster partition heal throttled')
+      await flushGossipFor(host, primaryGid, 'post-roster')
+      assertReplicationLag(primaryGid, 'post-roster-primary')
+      for (const sid of secondaryIds) {
+        await host.loadGuild(sid)
+        await flushGossipFor(host, sid, 'post-roster-sec')
+        assertReplicationLag(sid, `post-roster-${sid}`)
+      }
+      await host.loadGuild(primaryGid)
+      if (!contractMode) {
+        await guest1.start()
+        await guest2.start()
+        await guest3.start()
+        peerSessions.push(guest1, guest2, guest3)
+        const invite = await host.platform.createInvite().catch(() => null)
+        const code = invite?.shareCode || invite?.code
+        if (!code) throw new Error('primary invite missing')
+        await guest1.registerUser({
+          username: 'mg864guesta',
+          displayName: 'MG864 Guest a'
+        })
+        await guest2.registerUser({
+          username: 'mg864guestb',
+          displayName: 'MG864 Guest b'
+        })
+        await guest3.registerUser({
+          username: 'mg864guestc',
+          displayName: 'MG864 Guest c'
+        })
+        await guest1.ipc({ type: 'join-invite', code })
+        await guest2.ipc({ type: 'join-invite', code })
+        await guest3.ipc({ type: 'join-invite', code })
+        await guest1.wait({ mode: 'guild' }, 45000)
+        await guest2.wait({ mode: 'guild' }, 45000)
+        await guest3.wait({ mode: 'guild' }, 45000)
+        const chId = (await host.refreshView()).activeChannelId
+        if (!chId) throw new Error('active channel missing')
+        for (const sess of [guest1, guest2, guest3]) {
+          await sess.ipc({ type: 'select-channel', channelId: chId })
+        }
+        await connectGuildMeshQuad(
+          host.platform,
+          guest1.platform,
+          guest2.platform,
+          guest3.platform,
+          55000
+        )
+        const quadWait = { meshQuadReady: true, membersMin: 4 }
+        await host.wait(quadWait, 30000)
+        await guest1.wait(quadWait, 30000)
+        await guest2.wait(quadWait, 30000)
+        await guest3.wait(quadWait, 30000)
+        await host.sendGuildMessage(gossipToken)
+        const gossipWait = { activeChannelMessageIncludes: gossipToken }
+        await guest1.wait(gossipWait, 90000)
+        await guest2.wait(gossipWait, 90000)
+        await guest3.wait(gossipWait, 90000)
+        for (const sess of [host, guest1, guest2, guest3]) {
+          const g = (await sess.refreshView()).guild?.id
+          await flushGossipFor(sess, g, 'quad-flush')
+        }
+        const g2pre = await guest2.refreshView()
+        const guest2UserId = g2pre.user?.id
+        if (!guest2UserId) throw new Error('guest2 user id missing')
+        const healDmPromise = host.platform.runPartitionHealSync({
+          force: true,
+          source: 'agentctl-mg864-dm'
+        })
+        for (const sid of secondaryIds) {
+          await assertNoCrossLeak(sid)
+          await flushGossipFor(host, sid, 'dm-rotate')
+        }
+        await host.openDmWithPeer(guest2UserId, 'MG864 DM Peer')
+        const vDm = await host.refreshView()
+        if (vDm.mode !== 'dm') throw new Error('host not in dm during heal')
+        await host.sendDmMessage(dmToken)
+        const healDm = await healDmPromise
+        if (healDm?.throttled) throw new Error('dm partition heal throttled')
+        await host.ipc({ type: 'select-guild', guildId: primaryGid })
+        await host.wait({ mode: 'guild', activeChannelId: chId }, 45000)
+        await flushGossipFor(host, primaryGid, 'post-dm')
+        const dmSurvWait = { meshQuadDmDuringHealSurvivorReady: true }
+        await guest1.wait(dmSurvWait, 30000)
+        await guest3.wait(dmSurvWait, 30000)
+        if (liveQuad) {
+          await disconnectGuildMeshQuadOnePeer(
+            host.platform,
+            guest1.platform,
+            guest2.platform,
+            guest3.platform,
+            2
+          )
+          await guest2.wait({ meshPeersExact: 0 }, 30000)
+          await host.platform.runPartitionHealSync({
+            force: true,
+            source: 'agentctl-mg864-churn'
+          })
+          await reconnectGuildMeshQuadOnePeer(
+            host.platform,
+            guest1.platform,
+            guest2.platform,
+            guest3.platform,
+            2
+          )
+        }
+        await host.sendGuildMessage(churnToken)
+        const churnWait = { activeChannelMessageIncludes: churnToken }
+        await guest1.wait(churnWait, 90000)
+        await guest3.wait(churnWait, 90000)
+        for (const sess of [host, guest1, guest3]) {
+          const g = (await sess.refreshView()).guild?.id
+          await flushGossipFor(sess, g, 'post-churn')
+        }
+      } else {
+        for (const row of allTokens) {
+          await flushGossipFor(host, row.gid, 'contract-flush')
+        }
+      }
+      const healFinal = await host.platform.runPartitionHealSync({
+        force: true,
+        source: 'agentctl-mg864-final'
+      })
+      if (healFinal?.throttled) throw new Error('final partition heal throttled')
+      for (const row of allTokens) {
+        await assertNoCrossLeak(row.gid)
+        assertReplicationLag(row.gid, `final-${row.gid}`)
+        await flushGossipFor(host, row.gid, `final-${row.gid}`)
+      }
+      await host.loadGuild(primaryGid)
+      const gossipMatrixWait = {
+        meshMultiGuildProductionGossipReady: true,
+        meshMultiGuildProductionMatrixReady: true,
+        meshMultiGuildScaleDmCompositeReady: true,
+        partitionHealComplete: true,
+        guildSyncNotPending: true,
+        guildSyncGossipOutboxMax: outboxMax,
+        membersMin
+      }
+      await host.wait(gossipMatrixWait, waitMs)
+      if (!contractMode) {
+        for (const sess of [guest1, guest2, guest3]) {
+          await sess.wait(
+            {
+              meshMultiGuildProductionGossipReady: true,
+              guildSyncNotPending: true,
+              guildSyncGossipOutboxMax: outboxMax
+            },
+            90000
+          )
+        }
+      }
+      return {
+        guildCount,
+        primaryGid,
+        secondaryIds,
+        memberCount,
+        outboxMax,
+        maxLagMs,
+        gossipByGuild,
+        gossipToken,
+        multiFinal: host.exportMultiGuildSyncDiagnostics()
+      }
+    } finally {
+      for (const sess of peerSessions) {
+        await sess.close().catch(() => null)
+      }
+      await host.close()
+    }
+  }
+
   /** Phase 863: multi-guild production matrix — 862 composite + replication lag asserts per guild (P654-2). */
   async runGuildMeshMultiGuildProductionMatrixJourney () {
     const {