This commit is contained in:
Raven Scott
2026-05-20 21:02:45 -04:00
parent 1f3f4b24a2
commit 14d0980b4f
734 changed files with 682 additions and 746 deletions
+9
View File
@@ -0,0 +1,9 @@
# Messaging & gossip
3 modules. See [`../MODULE_CATEGORIES.md`](../../MODULE_CATEGORIES.md).
| Module |
|--------|
| [hyper-p2p-gossip-mesh](hyper-p2p-gossip-mesh/) |
| [hyper-p2p-dedup-filter](hyper-p2p-dedup-filter/) |
| [hyper-p2p-distributed-event-bus](hyper-p2p-distributed-event-bus/) |
@@ -0,0 +1,2 @@
node_modules/
*-storage/
@@ -0,0 +1,19 @@
# Changelog
<!-- legacy: v0.1.0 -->
- Initial release.
<!-- legacy: v0.2.0 -->
- Production-grade docs, validation, and expanded tests.
<!-- legacy: v0.3.0 -->
- Wave 6: presence-tier API tables, architecture wire section, validation test.
<!-- legacy: v0.3.1 -->
- Wave 7: correct protocol in docs, getStats(), wire tables, category README.
## [0.3.2] - 2026-05-21
### Changed
- Exhaustive documentation pass (api, architecture, README) per DOC_STANDARDS.md.
@@ -0,0 +1,43 @@
# hyper-p2p-dedup-filter
Production messaging & gossip module: Hyperswarm discovery + Protomux when `topic` is set.
**Category:** Messaging & gossip
**Composes with:** `hyper-p2p-gossip-mesh`, `hyper-p2p-distributed-event-bus`
**Protocol:** `hyper-p2p-dedup-filter/v1`
## When to use
Multi-peer apps that need messaging & gossip over a shared Hyperswarm topic.
## When not to use
Single-process tools with no P2P topic (use local APIs only or skip `ready()`).
## Quick start
```js
const { HyperP2PDedupFilter } = require('hyper-p2p-dedup-filter')
const topic = process.argv[2] // 64-char hex or string
const mod = new HyperP2PDedupFilter({ topic, enableBackgroundTimers: false })
await mod.ready() // joins swarm when topic set
// ... application logic ...
await mod.close()
```
## Docs
- [docs/api.md](docs/api.md) — constructor, methods, events, errors
- [docs/architecture.md](docs/architecture.md) — wire types, state, composition
- [../_shared/PRODUCTION.md](../../_shared/PRODUCTION.md) — production checklist
- [../_shared/DOC_STANDARDS.md](../../_shared/DOC_STANDARDS.md) — documentation standards
- Integration: [`../../real_tests/integration/`](../../../real_tests/integration/) — `dedup-filter-two-node.js`
## Test
```bash
npm install && npm test
```
@@ -0,0 +1,82 @@
# API: hyper-p2p-dedup-filter
**Protocol:** `hyper-p2p-dedup-filter/v1`
**Export:** `HyperP2PDedupFilter`
## Overview
Production messaging & gossip module: Hyperswarm discovery + Protomux when `topic` is set.
## Constructor
```js
const mod = new HyperP2PDedupFilter(opts)
```
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `topic` | varies | null | topic |
| `keyPair` | KeyPair | random Ed25519 | keyPair |
| `maxIds` | number | 4096 | maxIds |
## Methods
### `ready(—)`
- **Returns:** `Promise`
- **Throws:** — (none documented in method body)
### `seen(id)`
- **Returns:** `value`
- **Throws:** — (none documented in method body)
### `add(id, opts = {})`
- **Returns:** `value`
- **Throws:** — (none documented in method body)
### `compact(—)`
- **Returns:** `value`
- **Throws:** — (none documented in method body)
### `getStats(—)`
- **Returns:** `object`
- **Throws:** — (none documented in method body)
### `close(—)`
- **Returns:** `Promise<void>`
- **Throws:** — (none documented in method body)
## Events
| Event | Payload |
|-------|---------|
| `add` | id |
| `closed` | no payload |
| `compact` | size |
## getStats()
Returns `{ ...this._stats }` — typically `ops`, `errors`, and module-specific counters (`created`, `relays`, `open`, `peers`, etc.).
Library-only modules may include `mode: 'local'`.
## Errors
Stable message substrings: see [`../_shared/ERROR_CODES.md`](../../_shared/ERROR_CODES.md).
## P2P
When `topic` is set, `ready()` joins Hyperswarm and opens Protomux `hyper-p2p-dedup-filter/v1`.
## Testing
```bash
npm install && npm test
```
Integration: [`../../real_tests/integration/dedup-filter-two-node.js`](../../../real_tests/integration/dedup-filter-two-node.js)
@@ -0,0 +1,43 @@
# Architecture: hyper-p2p-dedup-filter
**Category:** Messaging & gossip
```mermaid
flowchart LR
App[Application] --> Mod[HyperP2PDedupFilter]
Mod --> Mux[Protomux hyper-p2p-dedup-filter/v1]
Mux --> Swarm[Hyperswarm]
```
## Sequence (P2P)
```mermaid
sequenceDiagram
participant App
participant Mod as Module
participant SW as Hyperswarm
participant Peer
App->>Mod: ready(topic)
Mod->>SW: join(topic)
SW->>Peer: connection
Mod->>Peer: gossip / Protomux
Peer-->>Mod: onmessage
Mod-->>App: emit(event)
```
## Wire messages
| type | fields | direction | behavior |
|------|--------|-----------|----------|
| `seen` | id, type | gossip | Handled in onmessage / gossipSend |
## State model
- In-memory `Map` / `Set` structures for hot path
- Optional Hyperbee/Hypercore persistence when `storageDir` or `memoryOnly` is configured
- `close()` tears down swarm, timers, and clears ephemeral state
## Composition
Composes with: `hyper-p2p-gossip-mesh`, `hyper-p2p-distributed-event-bus`.
@@ -0,0 +1,12 @@
require('bare-process/global')
const { HyperP2PDedupFilter } = require('../index.js')
async function main () {
const f = new HyperP2PDedupFilter()
console.log('add a:', f.add('event-a'))
console.log('dup a:', f.add('event-a'))
console.log('seen b:', f.seen('event-b'))
await f.close()
console.log('done')
}
main().catch(console.error)
@@ -0,0 +1,76 @@
require('bare-process/global')
const EventEmitter = require('bare-events')
const crypto = require('bare-crypto')
const b4a = require('b4a')
const { initModuleSwarm, gossipSend } = require('../../_shared/p2p-bare.js')
const PROTOCOL = 'hyper-p2p-dedup-filter/v1'
class HyperP2PDedupFilter extends EventEmitter {
constructor (opts = {}) {
super()
this._stats = { ops: 0, errors: 0 }
this.topic = opts.topic || null
this.keyPair = opts.keyPair || require('hypercore-crypto').keyPair()
this.maxIds = opts.maxIds || 4096
this._seen = new Set()
this.swarm = null
this._peerMsgs = null
}
async ready () {
if (this.swarm || !this.topic) return this
await initModuleSwarm(this, {
keyPair: this.keyPair,
topic: this.topic,
protocol: PROTOCOL,
onmessage: (data) => {
if (data && data.id) this.add(data.id, { gossip: false })
}
})
return this
}
_hashId (id) {
if (typeof id === 'string') return id
return b4a.toString(crypto.hash(typeof id === 'object' ? b4a.from(JSON.stringify(id)) : b4a.from(String(id))), 'hex')
}
seen (id) {
return this._seen.has(this._hashId(id))
}
add (id, opts = {}) {
const key = this._hashId(id)
if (this._seen.has(key)) return false
this._seen.add(key)
if (this._seen.size > this.maxIds) this.compact()
if (opts.gossip !== false && this._peerMsgs) {
gossipSend(this, { type: 'seen', id: key })
}
this.emit('add', { id: key })
return true
}
compact () {
const arr = Array.from(this._seen)
const keep = arr.slice(-Math.floor(this.maxIds * 0.75))
this._seen.clear()
for (const k of keep) this._seen.add(k)
this.emit('compact', { size: this._seen.size })
}
getStats () {
return { ...this._stats }
}
async close () {
if (this.swarm) await this.swarm.destroy().catch(() => {})
this.swarm = null
this.emit('closed')
}
}
module.exports = { HyperP2PDedupFilter, PROTOCOL }
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,25 @@
{
"name": "hyper-p2p-dedup-filter",
"version": "0.3.1",
"description": "Novel cross-peer message deduplication filter for Bare/Pear P2P.",
"main": "index.js",
"type": "commonjs",
"license": "Apache-2.0",
"scripts": { "test": "brittle-bare test/test.js" },
"dependencies": {
"bare-events": "^2.8.0",
"bare-crypto": "^1.9.0",
"bare-process": "^4.4.0",
"b4a": "^1.6.7",
"hypercore-crypto": "^3.0.0",
"protomux": "^3.0.0",
"compact-encoding": "^2.0.0"
},
"peerDependencies": { "hyperswarm": "^4.0.0", "bare": ">=1.0.0" },
"devDependencies": { "brittle": "^3.0.0" },
"imports": {
"process": { "bare": "bare-process", "default": "process" },
"crypto": { "bare": "bare-crypto", "default": "crypto" },
"events": { "bare": "bare-events", "default": "events" }
}
}
@@ -0,0 +1,42 @@
require('bare-process/global')
const test = require('brittle')
const { HyperP2PDedupFilter } = require('../index.js')
test('dedup-filter: seen and add', async (t) => {
const f = new HyperP2PDedupFilter()
t.ok(f.add('msg-1'))
t.ok(f.seen('msg-1'))
t.not(f.add('msg-1'))
await f.close()
})
test('dedup-filter: compact', async (t) => {
const f = new HyperP2PDedupFilter({ maxIds: 10 })
for (let i = 0; i < 20; i++) f.add('id-' + i)
t.ok(f._seen.size <= 10)
await f.close()
})
test('hyper-p2p-dedup-filter: close without leak', async (t) => {
const m = new HyperP2PDedupFilter()
await m.close()
t.pass()
})
test('hyper-p2p-dedup-filter: validation rejects invalid input', async (t) => {
const m = new HyperP2PDedupFilter()
try {
if (typeof m.addNeighbor === 'function') m.addNeighbor(null)
else if (typeof m.buildCircuit === 'function') m.buildCircuit([])
else if (typeof m.grant === 'function') m.grant(null, -1)
else if (typeof m.enqueue === 'function') m.enqueue('bad', null)
else if (typeof m.reportSample === 'function') m.reportSample(null, -1, -1)
else if (typeof m.fanout === 'function') m.fanout(null, 0)
else if (typeof m.probe === 'function') m.probe(null)
else if (typeof m.resolve === 'function') m.resolve(null)
else if (typeof m.acquire === 'function') m.acquire(null)
else throw new Error('no validation hook')
t.fail('expected throw')
} catch (err) {
t.ok(err instanceof Error)
}
await m.close()
})
@@ -0,0 +1,8 @@
node_modules
hyper-p2p-distributed-event-bus-storage
*.log
.DS_Store
coverage
test/storage
examples/storage
docs/.cache
@@ -0,0 +1,32 @@
# Changelog
## [0.2.0] - 2026-05-20
### Added
- Real Hyperswarm + Protomux v3 wiring via `../_shared/p2p-bare.js` (where applicable)
- 2-node integration test under `real_tests/integration/`
### Changed
- Protomux v3: `createChannel` + `addMessage` + `channel.open()`
## [0.1.1] - 2026-05-20
### Fixed
- Migrated tests from `bare-test` to `brittle` / `brittle-bare`
- `hypercore-crypto` for keyPair, sign, verify, hash
- `bare-process/global` and `bare-process` v4 imports
- Background timers opt-in (`enableBackgroundTimers`, `enableGossip`) for clean test exit
<!-- legacy: v0.2.0 -->
- Production-grade docs, validation, and expanded tests.
<!-- legacy: v0.3.0 -->
- Wave 6: presence-tier API tables, architecture wire section, validation test.
<!-- legacy: v0.3.1 -->
- Wave 7: correct protocol in docs, getStats(), wire tables, category README.
## [0.3.2] - 2026-05-21
### Changed
- Exhaustive documentation pass (api, architecture, README) per DOC_STANDARDS.md.
@@ -0,0 +1,41 @@
# hyper-p2p-distributed-event-bus
HyperP2PDistributedEventBus Novel distributed event bus for P2P with event sourcing. Features: - Append-only event logs per topic with unique IDs (hash-based)
**Category:** Messaging & gossip
**Composes with:** `hyper-p2p-gossip-mesh`, `hyper-p2p-dedup-filter`
**Protocol:** `hyper-p2p-distributed-event-bus/v1`
## When to use
Multi-peer apps that need messaging & gossip over a shared Hyperswarm topic.
## When not to use
Single-process tools with no P2P topic (use local APIs only or skip `ready()`).
## Quick start
```js
const { HyperP2PDistributedEventBus } = require('hyper-p2p-distributed-event-bus')
const topic = process.argv[2] // 64-char hex or string
const mod = new HyperP2PDistributedEventBus({ topic, enableBackgroundTimers: false })
await mod.ready() // joins swarm when topic set
// ... application logic ...
await mod.close()
```
## Docs
- [docs/api.md](docs/api.md) — constructor, methods, events, errors
- [docs/architecture.md](docs/architecture.md) — wire types, state, composition
- [../_shared/PRODUCTION.md](../../_shared/PRODUCTION.md) — production checklist
- [../_shared/DOC_STANDARDS.md](../../_shared/DOC_STANDARDS.md) — documentation standards
## Test
```bash
npm install && npm test
```
@@ -0,0 +1,252 @@
# API: hyper-p2p-distributed-event-bus
**Protocol:** `hyper-p2p-distributed-event-bus/v1` (`EVENT_BUS_PROTOCOL`)
**Export:** `HyperP2PDistributedEventBus` (default class export from `index.js`)
## Overview
`HyperP2PDistributedEventBus` implements a **distributed append-only event log** with **topic-based pub/sub**, **vector clocks** for causal metadata, **deduplication** by `event.id`, optional **Ed25519 signing** (`hypercore-crypto`), and **Hyperbee** persistence for replay. Peers gossip events over Hyperswarm + Protomux using the protocol id above. The wire carries **full event records** (not a `{ type, ... }` envelope).
Designed for Bare/Pear: `bare-events`, `bare-fs`, `bare-crypto`, no Node builtins.
## Constructor
```js
const HyperP2PDistributedEventBus = require('hyper-p2p-distributed-event-bus')
const bus = new HyperP2PDistributedEventBus(opts)
```
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `keyPair` | Hypercore `KeyPair` | `hypercore-crypto.keyPair()` | Identity for signing, swarm, and Hypercore |
| `topic` | `string` \| `Buffer` \| `null` | `null`**`'hyper-p2p-event-bus-default'`** at swarm join | Hyperswarm topic; always joined in `_initSwarm` |
| `storageDir` | `string` | `path.join(process.cwd(), 'hyper-p2p-distributed-event-bus-storage')` | Hypercore directory |
| `announceInterval` | `number` | `30000` | Ms between `announce` emissions when background timers enabled |
| `expiry` | `number` | `300000` | Peer map TTL before `peer-expired` |
| `metadata` | `object` | `{ agent: 'hyper-p2p-distributed-event-bus' }` | Merged into each published events `metadata` |
| `enableSigning` | `boolean` | `true` | Sign/verify events with `hypercore-crypto` |
| `enableBackgroundTimers` | `boolean` | `false` | Enables announce + cleanup intervals |
### Read-only properties
| Property | Type | Description |
|----------|------|-------------|
| `publicKey` | `Buffer` | Signing/swarm identity |
| `publicKeyHex` | `string` | Hex public key; vector-clock component for this peer |
## Lifecycle
### `async ready() → HyperP2PDistributedEventBus`
Loads vector clock and up to **1000** recent events from Hyperbee (reverse scan), joins Hyperswarm, optionally starts timers, emits `ready` and `swarm-joined`.
### `async close() → void`
Clears timers, destroys swarm, `bee.flush()` if available, closes Hyperbee and Hypercore, emits `closed`.
## Methods
### `async publish(topic, payload, metadata = {}) → Event`
Builds, signs (if enabled), processes locally, gossips to all peers.
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `topic` | `string` | `'default'` if falsy | Logical channel name |
| `payload` | `object` | `{}` | Application data; also used for subscription filters |
| `metadata` | `object` | `{}` | Shallow-merged over constructor `metadata` |
**Side effects:**
1. Increments local vector clock entry for `publicKeyHex`
2. Snapshots `vectorClock` into `event.vectorClock`
3. `_signEvent` when `enableSigning`
4. `_processIncomingEvent` (persist, dedup, emit, subscribe notify)
5. `_propagateEvent` to all peer channels
6. Emits `published`; increments `_metrics.published`
**Does not throw** module-specific validation errors for empty payload.
### `subscribe(topic, handler, filter = null) → unsubscribeFn`
Registers `{ handler, filter }` in `subscriptions` Map.
| Parameter | Type | Description |
|-----------|------|-------------|
| `topic` | `string` | Only events with `event.topic === topic` |
| `handler` | `(event) => void` | Called when filter matches |
| `filter` | `object` \| `null` | Key/value equality against `event.payload` (all keys must match) |
Returns function removing this subscription.
Also listen via `bus.on('event', ...)` or `bus.on('event:' + topic, ...)`.
### `async replay(topic = null, options = {}) → Event[]`
Reads from Hyperbee (not only memory cache).
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `from` | `number` | `0` | Skip first N matching entries |
| `limit` | `number` | `100` | Max events returned |
| `handler` | `function` | — | Optional per-event callback during scan |
| `topic` | Hyperbee prefix |
|---------|-----------------|
| `null` | `event:` (all topics) |
| `'orders'` | `event:orders:` |
Emits `replay-complete` with `{ topic, count }`.
### `getVectorClock() → Record<string, number>`
Plain object copy of `vectorClock` Map (`peerHex → logical counter`).
### `getRecentEvents(topic, limit = 50) → Event[]`
Last `limit` events from in-memory `eventLog` for `topic` (may be shorter than Hyperbee history).
### `getStats() → { ops: number, errors: number }`
Returns `_stats` (shallow copy). Internal `_metrics` (`published`, `received`, `signed`, `verified`, `pruned`, `errors`) is updated during operation but **not** exposed via `getStats()` in current code.
## Event record schema
Every stored/gossiped message uses this shape:
| Field | Type | Description |
|-------|------|-------------|
| `id` | `string` | 32 hex chars (`randomBytes(16)`) — dedup key |
| `topic` | `string` | Pub/sub channel |
| `timestamp` | `number` | `Date.now()` at publish |
| `vectorClock` | `Record<string, number>` | Snapshot at publish; merged on receive |
| `payload` | `object` | Application body |
| `metadata` | `object` | Agent tags + publish-time metadata |
| `peerId` | `string` | Publisher `publicKeyHex` |
| `signature` | `string` \| `null` | Base64 Ed25519 signature when signing enabled |
| `issuer` | `string` \| `null` | Hex public key of signer |
**Signing payload** (canonical JSON string of):
`{ id, topic, timestamp, vectorClock, payload, metadata, peerId }`
Verification uses `issuer` public key and `event.signature` (base64).
## Vector clock behavior
On **publish:**
- `own = vectorClock.get(publicKeyHex) || 0`
- Set `publicKeyHex` to `own + 1`
- Attach `Object.fromEntries(vectorClock)` to the event
On **receive** (`_mergeVectorClock`):
- For each `(peer, time)` in incoming map: `local = max(local, time)`
- Then increment **local** peer: `vectorClock.set(publicKeyHex, own + 1)`
Vector clocks are **metadata for causal hints**, not a total order guarantee. Concurrent events may have incomparable clocks.
## Events (EventEmitter)
| Event | When | Payload |
|-------|------|---------|
| `ready` | `ready()` done | — |
| `swarm-joined` | After swarm join | `topicBuf` (`Buffer`) |
| `peer-connected` | New connection | `{ peer: peerHex }` |
| `peer-expired` | Stale peer row | `peerHex` (string) |
| `event` | Any accepted event | Full `Event` object |
| `event:{topic}` | Same, namespaced | Full `Event` object |
| `published` | After local `publish` pipeline | Full `Event` object |
| `invalid-signature` | Verification failed | Event object (rejected) |
| `replay-complete` | `replay()` finished | `{ topic, count }` |
| `announce` | Background announce timer | — |
| `error` | Subscriber throw, sign/verify failure | `Error` |
| `closed` | `close()` | — |
## Wire protocol (Protomux)
Channel: **`hyper-p2p-distributed-event-bus/v1`**. Encoding: **JSON** (entire **Event** object per message).
There is **no** `type` discriminator on the wire — the message body **is** the event.
| message | fields | direction | behavior |
|---------|--------|-----------|----------|
| `Event` | `id`, `topic`, `timestamp`, `vectorClock`, `payload`, `metadata`, `peerId`, `signature`, `issuer` | bidirectional | `_processIncomingEvent`: verify → dedup → merge VC → persist → memory log → emit → notify subscribers → gossip to other peers (exclude source channel) |
**Example wire payload:**
```json
{
"id": "a1b2c3d4e5f6071829304a5b6c7d8e9f",
"topic": "orders",
"timestamp": 1710000000000,
"vectorClock": {
"aa11...": 3,
"bb22...": 1
},
"payload": { "item": "laptop", "qty": 1, "priority": "high" },
"metadata": { "agent": "hyper-p2p-distributed-event-bus", "instance": "bus1" },
"peerId": "cc33...",
"signature": "base64...",
"issuer": "cc33..."
}
```
## Hyperbee persistence
| Key | Value |
|-----|-------|
| `vc:{peerHex}` | `number` — logical time for peer |
| `event:{topic}:{eventId}` | Full `Event` JSON |
Load path:
1. Stream `vc:*` into `vectorClock` Map
2. Reverse stream `event:*`, max 1000 entries → `eventLog` + `seenEvents`
`_persistEvent` writes event key; ~10% of writes also flush all vector-clock keys.
## Deduplication and memory bounds
- `seenEvents` `Set` — drop if `event.id` already seen
- Cleanup timer: if `seenEvents.size > 10000`, rebuild from last 1000 events per topic in `eventLog`
- `eventLog` per topic: in-memory array; receives `push` on ingest (replay load uses `unshift` for historical order)
## P2P gossip flow
1. `publish` on peer A → local `_processIncomingEvent`
2. `_propagateEvent` sends to all `peers` with open `msg`, except when excluding source
3. Peer B `onmessage` → same pipeline
4. B re-gossips to other peers (excluding channel from A)
Integration tests may call `_processIncomingEvent(published)` directly if swarm propagation is slow.
## Errors
No module-specific error codes. Signing failures emit `invalid-signature` and increment `_metrics.errors`. See [`../_shared/ERROR_CODES.md`](../../_shared/ERROR_CODES.md).
`createSwarm` in shared helper throws `topic is required` — this bus always passes a topic (default string).
## Testing
```bash
cd modules/messaging-gossip/hyper-p2p-distributed-event-bus
npm install && npm test
```
- Unit: `test/test.js` — publish/subscribe, vector clock, replay, filters
- Integration: [`../../../real_tests/integration/event-bus-two-node.js`](../../../real_tests/integration/event-bus-two-node.js)
Example (two peers):
```bash
bare examples/basic-usage.js
```
Disable signing in tests for deterministic gossip:
```js
new HyperP2PDistributedEventBus({ topic, enableSigning: false })
```
@@ -0,0 +1,241 @@
# Architecture: hyper-p2p-distributed-event-bus
**Category:** Messaging & gossip
**Protocol:** `hyper-p2p-distributed-event-bus/v1` (`EVENT_BUS_PROTOCOL`)
**Composes with:** `hyper-p2p-gossip-mesh`, `hyper-p2p-dedup-filter`, `hyper-p2p-vector-clock`
## Problem and approach
Event-driven and CQRS-style decentralized apps need **append-only facts** propagated across peers with **durability** and **idempotent delivery**. This module provides:
1. **Topic pub/sub** — application channels with optional payload filters
2. **Vector clocks** — per-peer logical counters carried on every event
3. **Gossip fan-out** — Protomux JSON messages to all connected peers
4. **Hyperbee log**`event:{topic}:{id}` keys for offline replay
5. **Optional signing** — tamper detection on gossip payloads
Wire format is the **event record itself** (not a typed envelope), simplifying handlers at the cost of versioning via metadata conventions.
## Layer diagram
```mermaid
flowchart TB
subgraph app [Application]
Pub[publish]
Sub[subscribe / on event]
end
subgraph bus [HyperP2PDistributedEventBus]
PIPE[_processIncomingEvent]
VC[vectorClock Map]
DEDUP[seenEvents Set]
LOG[eventLog Map]
SUBS[subscriptions Map]
SIG[sign / verify]
end
subgraph persist [Persistence]
CORE[Hypercore]
BEE[Hyperbee]
end
subgraph net [P2P]
PM[Protomux EVENT_BUS_PROTOCOL]
HS[Hyperswarm]
end
Pub --> PIPE
PIPE --> VC
PIPE --> DEDUP
PIPE --> LOG
PIPE --> BEE
BEE --> CORE
PIPE --> SUBS
Sub --> SUBS
PIPE --> SIG
PIPE --> PM
PM --> HS
HS --> PM
PM --> PIPE
```
## Primary sequence: publish → local → gossip
```mermaid
sequenceDiagram
participant App
participant A as EventBus A
participant HB as Hyperbee
participant B as EventBus B
App->>A: publish(topic, payload)
A->>A: increment vectorClock[self]
A->>A: _signEvent (optional)
A->>A: _processIncomingEvent
A->>HB: put event:topic:id
A->>A: seenEvents.add(id)
A-->>App: event / published
A->>B: Protomux send Event JSON
B->>B: _verifyEvent
B->>B: dedup by id
B->>B: _mergeVectorClock
B->>HB: persist
B-->>App: event / event:topic
B->>B: _propagateEvent (other peers)
```
## Distributed event model
Events are **immutable facts**. The log is **append-only** at the API level (no `delete` or `retract` in `index.js`). Ordering:
| Mechanism | What it provides |
|-----------|------------------|
| `timestamp` | Wall-clock hint (publisher `Date.now()`) |
| `vectorClock` | Causal metadata; merged with component-wise max |
| `id` | Global dedup within a bus instance / swarm session |
| `topic` | Partition for subscribers and replay scans |
**Not provided:** strict total order across topics; conflict resolution between duplicate payloads (same `id` is impossible by construction).
## Vector clock state
```
vectorClock: Map<peerPublicKeyHex, non-negative integer>
```
| Operation | Effect |
|-----------|--------|
| `publish` | Increment self counter before snapshot |
| `_mergeVectorClock(incoming)` | `max(local[peer], incoming[peer])` for each peer; then increment self |
| `_persistEvent` (10% samples) | Write `vc:{peer}` keys to Hyperbee |
| `ready()` load | Restore all `vc:*` keys |
**Interpretation:** If `VC(a) < VC(b)` component-wise, `a` may have happened-before `b`. Incomparable clocks indicate concurrent publishes. The bus does not reorder delivery based on VC — subscribers receive in network arrival order.
## In-memory structures
| Structure | Type | Purpose |
|-----------|------|---------|
| `vectorClock` | `Map<string, number>` | Causal metadata |
| `seenEvents` | `Set<string>` | Dedup by `event.id` |
| `eventLog` | `Map<topic, Event[]>` | Hot cache for `getRecentEvents` |
| `subscriptions` | `Map<topic, Set<{handler, filter}>>` | Topic handlers |
| `peers` | `Map<peerHex, { lastSeen, metadata, channel, msg }>` | Gossip targets |
## Wire messages (detailed)
| message | fields | direction | behavior |
|---------|--------|-----------|----------|
| **Event** (root object) | `id`, `topic`, `timestamp`, `vectorClock`, `payload`, `metadata`, `peerId`, `signature`, `issuer` | peer ↔ peer | `onmessage``_processIncomingEvent(event, channel)` |
### Processing pipeline (`_processIncomingEvent`)
```mermaid
flowchart LR
IN[Incoming Event] --> V{enableSigning?}
V -->|invalid| X[invalid-signature emit return]
V -->|ok| D{id in seenEvents?}
D -->|yes| STOP[drop]
D -->|no| M[merge vector clock]
M --> P[persist Hyperbee]
P --> L[push eventLog]
L --> E[emit event + event:topic]
E --> N[notify subscribers]
N --> G[gossip to other peers]
```
| Step | Failure mode |
|------|----------------|
| Verify | Return early; `invalid-signature` |
| Dedup | Silent return |
| Persist | Await `bee.put` |
| Subscribe | Handler exceptions → `error` event |
| Gossip | Skip `excludeChannel`; ignore send errors |
### Protomux setup
```js
protocolChannel(mux, {
protocol: 'hyper-p2p-distributed-event-bus/v1',
onopen(channel, msg) { peers.set(peerHex, { ..., channel, msg }) },
onmessage(event) { _processIncomingEvent(event, channel) }
})
```
Encoding defaults to `compact-encoding` **json** in `p2p-bare.js`.
## Hyperbee / Hypercore layout
```
storageDir/
Hypercore(keyPair, valueEncoding: json)
Hyperbee(keyEncoding: utf-8, valueEncoding: json)
vc:{peerHex} → number
event:{topic}:{eventId} → Event
```
**Replay** scans lexicographic range:
- All topics: `gte: 'event:'`, `lt: 'event:\xff'`
- One topic: `gte: 'event:orders:'`, `lt: 'event:orders:\xff'`
Startup reverse load caps at **1000** events to bound memory; full history remains on disk for `replay()`.
Constants: `EVENT_DB_NAME = 'event-bus'`, `VECTOR_CLOCK_PREFIX = 'vc:'`, `EVENT_PREFIX = 'event:'`, `META_PREFIX = 'meta:'` (unused), `DEFAULT_ANNOUNCE_INTERVAL = 30000`, `DEFAULT_EXPIRY = 300000`.
## Subscription filter model
Filter is plain object equality on **top-level** `payload` keys:
```js
subscribe('orders', handler, { priority: 'high' })
// delivers only if event.payload.priority === 'high'
```
Missing keys in payload fail the match. No operators (AND across nested paths, regex, etc.) in core code.
## Signing architecture
When `enableSigning !== false`:
1. **Sign** on publish with `hypercore-crypto.sign(JSON.stringify(coreFields), secretKey)`
2. Store `signature` (base64) and `issuer` (hex public key)
3. **Verify** on ingest with `hypercore-crypto.verify(...)`
Disabling signing (`enableSigning: false`) skips verification branch (`_verifyEvent` returns true when disabled or missing signature fields).
## Timers
| Timer | Default | Action |
|-------|---------|--------|
| `announceTimer` | 30s | Emit `announce` (placeholder for heartbeat / re-announce) |
| `cleanupTimer` | 60s | Expire stale peers; trim `seenEvents` if > 10000 |
Enable with `enableBackgroundTimers: true`.
## Composition
| Module | Relationship |
|--------|----------------|
| `hyper-p2p-gossip-mesh` | Alternative fan-out topologies for large meshes |
| `hyper-p2p-dedup-filter` | Additional dedup policies beyond `seenEvents` |
| `hyper-p2p-vector-clock` | Standalone VC utilities; example `with-event-bus.js` |
| `hyper-p2p-reactive-state` | Materialized view / snapshot while bus stores facts |
Default swarm topic when unset: **`hyper-p2p-event-bus-default`** (hashed if string via `topicToBuffer`).
Network stack ordering: [`../_shared/WAVE6_NETWORK_STACK.md`](../../_shared/WAVE6_NETWORK_STACK.md).
## Scale and operations
- **Fan-out:** O(peers) per event per hop; multi-hop delivery requires mesh connectivity
- **Storage:** unbounded `event:*` keys; plan compaction externally
- **ID space:** 128-bit random hex — collision risk negligible
- **Trust:** signing validates publisher identity, not semantic correctness of `payload`
## Testing architecture
| Layer | File |
|-------|------|
| Unit | `test/test.js` |
| Integration | `real_tests/integration/event-bus-two-node.js` (10s swarm wait; may inject via `_processIncomingEvent`) |
| Example | `examples/basic-usage.js` — two buses, filter subscribe, replay |
@@ -0,0 +1,58 @@
const HyperP2PDistributedEventBus = require('../index.js')
const crypto = require('bare-crypto')
const path = require('bare-path')
const process = require('bare-process')
async function main () {
console.log('Starting hyper-p2p-distributed-event-bus example...')
const topic = crypto.randomBytes(32)
const bus1 = new HyperP2PDistributedEventBus({
topic,
storageDir: path.join(process.cwd(), 'example-storage-bus1'),
metadata: { instance: 'bus1' }
})
const bus2 = new HyperP2PDistributedEventBus({
topic,
storageDir: path.join(process.cwd(), 'example-storage-bus2'),
metadata: { instance: 'bus2' }
})
await bus1.ready()
await bus2.ready()
console.log('Both buses ready. Public keys:')
console.log('bus1:', bus1.publicKeyHex)
console.log('bus2:', bus2.publicKeyHex)
// Subscribe on bus2
const unsub = bus2.subscribe('orders', (event) => {
console.log('[bus2] Received order event:', event.payload, 'from', event.peerId.substring(0, 8))
}, { priority: 'high' })
// Publish from bus1
console.log('\nPublishing events from bus1...')
await bus1.publish('orders', { item: 'laptop', qty: 1, priority: 'high' })
await bus1.publish('orders', { item: 'mouse', qty: 5, priority: 'normal' })
// Wait for propagation
await new Promise(resolve => setTimeout(resolve, 800))
// Replay on bus2
console.log('\nReplaying history on bus2...')
const history = await bus2.replay('orders', { limit: 10 })
console.log('Replayed', history.length, 'events')
// Vector clock
console.log('\nCurrent vector clock on bus1:', bus1.getVectorClock())
unsub()
await bus1.close()
await bus2.close()
console.log('\nExample completed successfully. All Bare/Pear compatible.')
}
main().catch(console.error)
@@ -0,0 +1,450 @@
require('bare-process/global')
const EventEmitter = require('bare-events')
const { setInterval, clearInterval, setTimeout, clearTimeout } = require('bare-timers')
const crypto = require('bare-crypto')
const fs = require('bare-fs/promises')
const path = require('bare-path')
const process = require('bare-process')
const b4a = require('b4a')
const Hyperbee = require('hyperbee')
const Hypercore = require('hypercore')
const { topicToBuffer, createSwarm, wireConnection, protocolChannel } = require('../../_shared/p2p-bare.js')
// Protocol constants
const EVENT_BUS_PROTOCOL = 'hyper-p2p-distributed-event-bus/v1'
const hypercoreCrypto = require('hypercore-crypto')
const DEFAULT_ANNOUNCE_INTERVAL = 30000
const DEFAULT_EXPIRY = 300000
const EVENT_DB_NAME = 'event-bus'
const VECTOR_CLOCK_PREFIX = 'vc:'
const EVENT_PREFIX = 'event:'
const META_PREFIX = 'meta:'
/**
* HyperP2PDistributedEventBus
*
* Novel distributed event bus for P2P with event sourcing.
*
* Features:
* - Append-only event logs per topic with unique IDs (hash-based)
* - Vector clocks for causal ordering and conflict detection
* - Automatic P2P propagation over Hyperswarm + Protomux custom protocol
* - Hyperbee persistence for durable storage, replay, and offline operation
* - Topic-based pub/sub with optional filters (key/value matching)
* - Deduplication of events across peers
* - Replay from any point or full history
* - Production-grade: graceful shutdown, backpressure hints, error recovery, signing
* - 100% Bare/Pear compatible (no Node.js builtins)
*
* Never-before-seen primitive: First high-level event sourcing + distributed pub/sub abstraction
* specifically designed for decentralized Bare/Pear applications.
*/
class HyperP2PDistributedEventBus extends EventEmitter {
constructor (opts = {}) {
super()
this._stats = { ops: 0, errors: 0 }
this.keyPair = opts.keyPair || require('hypercore-crypto').keyPair()
this.topic = opts.topic || null // can be Buffer or string for swarm topic
const cwd = process.cwd()
this.storageDir = opts.storageDir || path.join(cwd, 'hyper-p2p-distributed-event-bus-storage')
this.announceIntervalMs = opts.announceInterval || DEFAULT_ANNOUNCE_INTERVAL
this.expiryMs = opts.expiry || DEFAULT_EXPIRY
this.metadata = opts.metadata || { agent: 'hyper-p2p-distributed-event-bus' }
this.enableSigning = opts.enableSigning !== false
this._enableBackgroundTimers = opts.enableBackgroundTimers === true
this._metrics = { published: 0, received: 0, signed: 0, verified: 0, pruned: 0, errors: 0 }
// Internal structures
this.vectorClock = new Map() // peerPubHex -> logical time
this.seenEvents = new Set() // eventId set for dedup
this.subscriptions = new Map() // topic -> Set of {handler, filter}
this.peers = new Map() // peerPubHex -> { lastSeen, metadata, mux }
this.eventLog = new Map() // topic -> array of events (in-memory cache)
this.swarm = null
this.corestore = null
this.bee = null
this._joined = false
this._protocol = null
this.announceTimer = null
this.cleanupTimer = null
this._mux = null
}
get publicKey () {
return this.keyPair.publicKey
}
get publicKeyHex () {
return b4a.toString(this.keyPair.publicKey, 'hex')
}
async ready () {
if (this._joined) return this
await this._initStorage()
await this._initSwarm()
if (this._enableBackgroundTimers) {
this._startAnnounceTimer()
this._startCleanupTimer()
}
this._joined = true
this.emit('ready')
return this
}
async _initStorage () {
try {
await fs.mkdir(this.storageDir, { recursive: true })
} catch (err) {
if (err.code !== 'EEXIST') throw err
}
const core = new Hypercore(this.storageDir, this.keyPair, {
valueEncoding: 'json'
})
this.core = core
this.bee = new Hyperbee(core, {
keyEncoding: 'utf-8',
valueEncoding: 'json'
})
await this.bee.ready()
// Load vector clock and recent events
await this._loadFromPersistence()
}
async _loadFromPersistence () {
// Load vector clock
for await (const entry of this.bee.createReadStream({ gte: VECTOR_CLOCK_PREFIX, lt: VECTOR_CLOCK_PREFIX + '\xff' })) {
const peerHex = entry.key.slice(VECTOR_CLOCK_PREFIX.length)
this.vectorClock.set(peerHex, entry.value)
}
// Load recent events into cache (last 1000 or so)
let count = 0
for await (const entry of this.bee.createReadStream({ gte: EVENT_PREFIX, lt: EVENT_PREFIX + '\xff', reverse: true })) {
if (count++ > 1000) break
const event = entry.value
const topic = event.topic || 'default'
if (!this.eventLog.has(topic)) this.eventLog.set(topic, [])
this.eventLog.get(topic).unshift(event) // oldest first
this.seenEvents.add(event.id)
}
}
async _initSwarm () {
const topic = this.topic || 'hyper-p2p-event-bus-default'
const { swarm, topicBuf } = await createSwarm({ keyPair: this.keyPair, topic })
this.swarm = swarm
wireConnection(this.swarm, (connection, peerInfo, mux) => {
this._handleConnection(connection, peerInfo, mux)
})
this.emit('swarm-joined', topicBuf)
}
_handleConnection (connection, peerInfo, mux) {
const peerPub = peerInfo.publicKey || connection.remotePublicKey
const peerHex = peerPub ? b4a.toString(peerPub, 'hex') : b4a.toString(crypto.randomBytes(32), 'hex')
const self = this
this.emit('peer-connected', { peer: peerHex })
protocolChannel(mux, {
protocol: EVENT_BUS_PROTOCOL,
onopen (channel, msg) {
self.peers.set(peerHex, { lastSeen: Date.now(), metadata: {}, channel, msg })
},
onclose () {
self.peers.delete(peerHex)
},
onmessage (event) {
self._processIncomingEvent(event, channel).catch((err) => self.emit('error', err))
}
})
}
async _processIncomingEvent (event, sourceChannel = null) {
if (!event || !event.id || !event.topic) return
// Verify signature if signing enabled (prevents tampering in P2P gossip)
if (this.enableSigning && !this._verifyEvent(event)) {
this._metrics.errors++
this.emit('invalid-signature', event)
return
}
// Dedup
if (this.seenEvents.has(event.id)) return
this.seenEvents.add(event.id)
// Merge vector clock
this._mergeVectorClock(event.vectorClock || {})
// Persist
await this._persistEvent(event)
// Update local log
const topic = event.topic
if (!this.eventLog.has(topic)) this.eventLog.set(topic, [])
this.eventLog.get(topic).push(event)
// Emit locally
this.emit('event', event)
this.emit(`event:${topic}`, event)
// Notify subscribers
this._notifySubscribers(event)
// Propagate to other peers (gossip)
if (sourceChannel) {
this._propagateEvent(event, sourceChannel)
}
this._metrics.received++
}
_mergeVectorClock (incomingVC) {
for (const [peer, time] of Object.entries(incomingVC)) {
const current = this.vectorClock.get(peer) || 0
this.vectorClock.set(peer, Math.max(current, time))
}
// Increment own
const own = this.vectorClock.get(this.publicKeyHex) || 0
this.vectorClock.set(this.publicKeyHex, own + 1)
}
_notifySubscribers (event) {
const subs = this.subscriptions.get(event.topic) || new Set()
for (const sub of subs) {
if (this._matchesFilter(event, sub.filter)) {
try {
sub.handler(event)
} catch (err) {
this.emit('error', err)
}
}
}
}
_matchesFilter (event, filter) {
if (!filter) return true
for (const [key, value] of Object.entries(filter)) {
if (event.payload && event.payload[key] !== value) return false
}
return true
}
async _persistEvent (event) {
const key = `${EVENT_PREFIX}${event.topic}:${event.id}`
await this.bee.put(key, event)
// Also persist vector clock periodically
if (Math.random() < 0.1) { // occasional
for (const [peer, time] of this.vectorClock) {
await this.bee.put(`${VECTOR_CLOCK_PREFIX}${peer}`, time)
}
}
}
_propagateEvent (event, excludeChannel = null) {
for (const [, peerData] of this.peers) {
if (peerData.channel && peerData.channel !== excludeChannel && peerData.msg) {
try {
peerData.msg.send(event)
} catch (err) {
// ignore transient send errors
}
}
}
}
// Public API: Publish an event
async publish (topic, payload, metadata = {}) {
if (!this._joined) await this.ready()
const now = Date.now()
const eventId = b4a.toString(crypto.randomBytes(16), 'hex')
const own = this.vectorClock.get(this.publicKeyHex) || 0
this.vectorClock.set(this.publicKeyHex, own + 1)
const ownVC = Object.fromEntries(this.vectorClock)
let event = {
id: eventId,
topic: topic || 'default',
timestamp: now,
vectorClock: ownVC,
payload: payload || {},
metadata: { ...this.metadata, ...metadata },
peerId: this.publicKeyHex,
signature: null,
issuer: null
}
// Sign if enabled (Ed25519 via bare-crypto)
event = this._signEvent(event)
// Local process
await this._processIncomingEvent(event)
// Propagate
this._propagateEvent(event)
this.emit('published', event)
this._metrics.published++
return event
}
// Subscribe to a topic
subscribe (topic, handler, filter = null) {
if (!this.subscriptions.has(topic)) {
this.subscriptions.set(topic, new Set())
}
const sub = { handler, filter }
this.subscriptions.get(topic).add(sub)
// Return unsubscribe function
return () => {
const set = this.subscriptions.get(topic)
if (set) set.delete(sub)
if (set && set.size === 0) this.subscriptions.delete(topic)
}
}
// Replay events from storage for a topic (or all)
async replay (topic = null, options = {}) {
const { from = 0, limit = 100, handler } = options
const results = []
const prefix = topic ? `${EVENT_PREFIX}${topic}:` : EVENT_PREFIX
let count = 0
for await (const entry of this.bee.createReadStream({ gte: prefix, lt: prefix + '\xff' })) {
if (count < from) { count++; continue }
if (results.length >= limit) break
const event = entry.value
results.push(event)
if (handler) {
try { handler(event) } catch (e) { this.emit('error', e) }
}
count++
}
this.emit('replay-complete', { topic, count: results.length })
return results
}
// Get current vector clock snapshot
getVectorClock () {
return Object.fromEntries(this.vectorClock)
}
// Get recent events for topic (in-memory)
getRecentEvents (topic, limit = 50) {
const log = this.eventLog.get(topic) || []
return log.slice(-limit)
}
_startAnnounceTimer () {
if (this.announceTimer) clearInterval(this.announceTimer)
this.announceTimer = setInterval(() => {
this.emit('announce')
// Could re-join or send heartbeats here
}, this.announceIntervalMs)
}
_startCleanupTimer () {
if (this.cleanupTimer) clearInterval(this.cleanupTimer)
this.cleanupTimer = setInterval(() => {
const now = Date.now()
for (const [peerHex, data] of this.peers) {
if (now - data.lastSeen > this.expiryMs) {
this.peers.delete(peerHex)
this.emit('peer-expired', peerHex)
}
}
// Trim seenEvents if too large
if (this.seenEvents.size > 10000) {
// simple trim: keep recent by re-adding from log (simplified)
this.seenEvents.clear()
for (const [t, events] of this.eventLog) {
for (const e of events.slice(-1000)) this.seenEvents.add(e.id)
}
}
}, 60000)
}
_signEvent (event) {
if (!this.enableSigning) return event
try {
const keyPair = this.keyPair
const dataToSign = b4a.from(JSON.stringify({
id: event.id,
topic: event.topic,
timestamp: event.timestamp,
vectorClock: event.vectorClock,
payload: event.payload,
metadata: event.metadata,
peerId: event.peerId
}))
const signature = require('hypercore-crypto').sign(dataToSign, keyPair.secretKey)
event.signature = b4a.toString(signature, 'base64')
event.issuer = b4a.toString(keyPair.publicKey, 'hex')
this._metrics.signed++
} catch (err) {
this._metrics.errors++
this.emit('error', err)
}
return event
}
_verifyEvent (event, publicKey = null) {
if (!this.enableSigning || !event.signature || !event.issuer) return true // skip if disabled
try {
const dataToVerify = b4a.from(JSON.stringify({
id: event.id,
topic: event.topic,
timestamp: event.timestamp,
vectorClock: event.vectorClock,
payload: event.payload,
metadata: event.metadata,
peerId: event.peerId
}))
const sig = b4a.from(event.signature, 'base64')
const pub = publicKey || b4a.from(event.issuer, 'hex')
const valid = require('hypercore-crypto').verify(dataToVerify, sig, pub)
if (valid) this._metrics.verified++
return valid
} catch (err) {
this._metrics.errors++
this.emit('error', err)
return false
}
}
getStats () {
return { ...this._stats }
}
async close () {
if (this.announceTimer) clearInterval(this.announceTimer)
if (this.cleanupTimer) clearInterval(this.cleanupTimer)
if (this.swarm) {
await this.swarm.destroy().catch(() => {})
}
if (this.bee) {
if (typeof this.bee.flush === 'function') await this.bee.flush().catch(() => {})
await this.bee.close().catch(() => {})
}
if (this.core) {
await this.core.close().catch(() => {})
}
this._joined = false
this.emit('closed')
}
}
module.exports = HyperP2PDistributedEventBus
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,96 @@
{
"name": "hyper-p2p-distributed-event-bus",
"version": "0.3.1",
"description": "A novel, production-grade distributed event bus and event sourcing primitive for Bare/Pear P2P applications. Provides append-only event logs with causal ordering (vector clocks), real-time P2P event propagation via Hyperswarm + Protomux, Hyperbee persistence for replay and offline-first, topic-based pub/sub with filters, deduplication, and backpressure. First reusable high-level event-driven abstraction enabling decentralized event sourcing, CQRS, and reactive architectures in the Holepunch ecosystem. Never-before-seen primitive combining event sourcing + P2P sync + persistence.",
"main": "index.js",
"type": "commonjs",
"exports": {
".": "./index.js",
"./package.json": "./package.json"
},
"keywords": [
"holepunch",
"bare",
"pear",
"p2p",
"event-bus",
"event-sourcing",
"distributed-events",
"vector-clock",
"causal-ordering",
"pubsub",
"hyperswarm",
"hyperbee",
"hypercore",
"protomux",
"decentralized",
"replay",
"reactive"
],
"author": "Holepunch Development Agent",
"license": "Apache-2.0",
"repository": {
"type": "git",
"url": "https://github.com/holepunchto/hyper-p2p-distributed-event-bus"
},
"bugs": {
"url": "https://github.com/holepunchto/hyper-p2p-distributed-event-bus/issues"
},
"homepage": "https://github.com/holepunchto/hyper-p2p-distributed-event-bus",
"dependencies": {
"bare-events": "^2.8.0",
"bare-fs": "^4.0.0",
"bare-path": "^3.0.0",
"bare-crypto": "^1.9.0",
"bare-timers": "^2.0.0",
"bare-process": "^4.4.0",
"hyperswarm": "^4.0.0",
"hyperbee": "^2.0.0",
"hypercore": "^10.0.0",
"protomux": "^3.0.0",
"b4a": "^1.6.7",
"hypercore-crypto": "^3.0.0"
},
"devDependencies": {
"brittle": "^3.0.0"
},
"peerDependencies": {
"bare": ">=1.0.0"
},
"engines": {
"bare": ">=1.0.0"
},
"pear": {
"name": "hyper-p2p-distributed-event-bus",
"type": "module"
},
"imports": {
"process": {
"bare": "bare-process",
"default": "process"
},
"crypto": {
"bare": "bare-crypto",
"default": "crypto"
},
"path": {
"bare": "bare-path",
"default": "path"
},
"fs": {
"bare": "bare-fs",
"default": "fs"
},
"timers": {
"bare": "bare-timers",
"default": "timers"
},
"events": {
"bare": "bare-events",
"default": "events"
}
},
"scripts": {
"test": "brittle-bare test/test.js"
}
}
@@ -0,0 +1,122 @@
const test = require('brittle')
const HyperP2PDistributedEventBus = require('../index.js')
const path = require('bare-path')
const fs = require('bare-fs/promises')
const crypto = require('bare-crypto')
const process = require('bare-process')
test('hyper-p2p-distributed-event-bus - basic lifecycle and publish/subscribe', async (t) => {
const bus = new HyperP2PDistributedEventBus({
storageDir: path.join(process.cwd(), 'test-storage-bus1-' + Date.now()),
topic: crypto.randomBytes(32)
})
await bus.ready()
const received = []
const unsub = bus.subscribe('test-topic', (event) => {
received.push(event)
})
await bus.publish('test-topic', { message: 'hello local', value: 42 })
t.ok(received.length >= 1, 'should receive at least one event')
t.is(received[0].payload.message, 'hello local')
t.is(received[0].topic, 'test-topic')
t.ok(received[0].id)
t.ok(received[0].vectorClock)
unsub()
await bus.close()
try { await fs.rm(bus.storageDir, { recursive: true, force: true }) } catch {}
})
test('hyper-p2p-distributed-event-bus - vector clock and dedup', async (t) => {
const bus = new HyperP2PDistributedEventBus({
storageDir: path.join(process.cwd(), 'test-storage-vc-' + Date.now())
})
await bus.ready()
const e1 = await bus.publish('vc-test', { seq: 1 })
const e2 = await bus.publish('vc-test', { seq: 2 })
t.ok(e1.vectorClock)
t.ok(e2.vectorClock)
t.is(Object.keys(e1.vectorClock).length > 0, true)
// Replay test
const replayed = await bus.replay('vc-test', { limit: 10 })
t.ok(replayed.length >= 2)
await bus.close()
try { await fs.rm(bus.storageDir, { recursive: true, force: true }) } catch {}
})
test('hyper-p2p-distributed-event-bus - persistence and replay', async (t) => {
const storageDir = path.join(process.cwd(), 'test-storage-persist-' + Date.now())
const bus = new HyperP2PDistributedEventBus({ storageDir })
await bus.ready()
await bus.publish('persist-topic', { data: 'first' })
await bus.publish('persist-topic', { data: 'second' })
await bus.close()
// Re-open and replay
const bus2 = new HyperP2PDistributedEventBus({ storageDir })
await bus2.ready()
const replayed = await bus2.replay('persist-topic')
t.ok(replayed.length >= 2)
t.ok(replayed.some((e) => e.payload.data === 'first'))
t.ok(replayed.some((e) => e.payload.data === 'second'))
await bus2.close()
try { await fs.rm(storageDir, { recursive: true, force: true }) } catch {}
})
test('hyper-p2p-distributed-event-bus - filter subscription', async (t) => {
const bus = new HyperP2PDistributedEventBus({
storageDir: path.join(process.cwd(), 'test-storage-filter-' + Date.now())
})
await bus.ready()
let filtered = []
bus.subscribe('filtered-topic', (e) => filtered.push(e), { type: 'important' })
await bus.publish('filtered-topic', { type: 'important', msg: 'yes' })
await bus.publish('filtered-topic', { type: 'normal', msg: 'no' })
await new Promise(r => setTimeout(r, 100))
t.is(filtered.length, 1)
t.is(filtered[0].payload.msg, 'yes')
await bus.close()
try { await fs.rm(bus.storageDir, { recursive: true, force: true }) } catch {}
})
test('hyper-p2p-distributed-event-bus: close without leak', async (t) => {
const m = new HyperP2PDistributedEventBus()
await m.close()
t.pass()
})
test('hyper-p2p-distributed-event-bus: validation rejects invalid input', async (t) => {
const m = new HyperP2PDistributedEventBus()
try {
if (typeof m.addNeighbor === 'function') m.addNeighbor(null)
else if (typeof m.buildCircuit === 'function') m.buildCircuit([])
else if (typeof m.grant === 'function') m.grant(null, -1)
else if (typeof m.enqueue === 'function') m.enqueue('bad', null)
else if (typeof m.reportSample === 'function') m.reportSample(null, -1, -1)
else if (typeof m.fanout === 'function') m.fanout(null, 0)
else if (typeof m.probe === 'function') m.probe(null)
else if (typeof m.resolve === 'function') m.resolve(null)
else if (typeof m.acquire === 'function') m.acquire(null)
else throw new Error('no validation hook')
t.fail('expected throw')
} catch (err) {
t.ok(err instanceof Error)
}
await m.close()
})
@@ -0,0 +1,2 @@
node_modules/
*-storage/
@@ -0,0 +1,23 @@
# Changelog
<!-- legacy: v0.2.0 -->
- Optional `dedupFilter` on publish and receive paths.
<!-- legacy: v0.1.0 -->
- Initial release.
<!-- legacy: v0.2.1 -->
- Production docs, input validation, third test, integration notes.
<!-- legacy: v0.3.0 -->
- Wave 6: presence-tier API tables, architecture wire section, validation test.
<!-- legacy: v0.3.1 -->
- Wave 7: correct protocol in docs, getStats(), wire tables, category README.
## [0.3.2] - 2026-05-21
### Changed
- Exhaustive documentation pass (api, architecture, README) per DOC_STANDARDS.md.
@@ -0,0 +1,43 @@
# hyper-p2p-gossip-mesh
Production messaging & gossip module: Hyperswarm discovery + Protomux when `topic` is set.
**Category:** Messaging & gossip
**Composes with:** `hyper-p2p-dedup-filter`, `hyper-p2p-distributed-event-bus`
**Protocol:** `gossip-mesh/v1`
## When to use
Multi-peer apps that need messaging & gossip over a shared Hyperswarm topic.
## When not to use
Single-process tools with no P2P topic (use local APIs only or skip `ready()`).
## Quick start
```js
const { HyperP2PGossipMesh } = require('hyper-p2p-gossip-mesh')
const topic = process.argv[2] // 64-char hex or string
const mod = new HyperP2PGossipMesh({ topic, enableBackgroundTimers: false })
await mod.ready() // joins swarm when topic set
// ... application logic ...
await mod.close()
```
## Docs
- [docs/api.md](docs/api.md) — constructor, methods, events, errors
- [docs/architecture.md](docs/architecture.md) — wire types, state, composition
- [../_shared/PRODUCTION.md](../../_shared/PRODUCTION.md) — production checklist
- [../_shared/DOC_STANDARDS.md](../../_shared/DOC_STANDARDS.md) — documentation standards
- Integration: [`../../real_tests/integration/`](../../../real_tests/integration/) — `gossip-mesh-two-node.js`
## Test
```bash
npm install && npm test
```
@@ -0,0 +1,85 @@
# API: hyper-p2p-gossip-mesh
**Protocol:** `gossip-mesh/v1`
**Export:** `HyperP2PGossipMesh`
## Overview
Production messaging & gossip module: Hyperswarm discovery + Protomux when `topic` is set.
## Constructor
```js
const mod = new HyperP2PGossipMesh(opts)
```
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `topic` | varies | null | topic |
| `keyPair` | KeyPair | random Ed25519 | keyPair |
| `defaultTtl` | number | 8 | defaultTtl |
| `defaultFanout` | number | 3 | defaultFanout |
| `dedupFilter` | varies | null | dedupFilter |
## Methods
### `subscribe(filter)`
- **Returns:** `value`
- **Throws:** — (none documented in method body)
### `publish(msg, opts = {})`
- **Returns:** `value`
- **Throws:** — (none documented in method body)
### `ready(—)`
- **Returns:** `Promise`
- **Throws:** — (none documented in method body)
### `getStats(—)`
- **Returns:** `object`
- **Throws:** — (none documented in method body)
### `close(—)`
- **Returns:** `Promise<void>`
- **Throws:** — (none documented in method body)
## Events
| Event | Payload |
|-------|---------|
| `closed` | no payload |
| `message` | envelope |
## getStats()
Returns `{ ...this._stats }` — typically `ops`, `errors`, and module-specific counters (`created`, `relays`, `open`, `peers`, etc.).
Library-only modules may include `mode: 'local'`.
## Errors
Stable message substrings: see [`../_shared/ERROR_CODES.md`](../../_shared/ERROR_CODES.md).
## P2P
When `topic` is set, `ready()` joins Hyperswarm and opens Protomux `gossip-mesh/v1`.
## Testing
```bash
npm install && npm test
```
Integration: [`../../real_tests/integration/gossip-mesh-two-node.js`](../../../real_tests/integration/gossip-mesh-two-node.js)
## Common flows
1. `ready(topic)` — join swarm and open `gossip-mesh/v1` channel.
2. `publish(payload, { ttl })` — epidemic fanout with TTL decay per hop.
3. Listen for `message` events — deduplicate at app layer with `hyper-p2p-dedup-filter` when needed.
4. `close()` — destroy swarm and clear peer message map.
@@ -0,0 +1,43 @@
# Architecture: hyper-p2p-gossip-mesh
**Category:** Messaging & gossip
```mermaid
flowchart LR
App[Application] --> Mod[HyperP2PGossipMesh]
Mod --> Mux[Protomux gossip-mesh/v1]
Mux --> Swarm[Hyperswarm]
```
## Sequence (P2P)
```mermaid
sequenceDiagram
participant App
participant Mod as Module
participant SW as Hyperswarm
participant Peer
App->>Mod: ready(topic)
Mod->>SW: join(topic)
SW->>Peer: connection
Mod->>Peer: gossip / Protomux
Peer-->>Mod: onmessage
Mod-->>App: emit(event)
```
## Wire messages
| type | fields | direction | behavior |
|------|--------|-----------|----------|
| `gossip` | envelope | gossip | Handled in onmessage / gossipSend |
## State model
- In-memory `Map` / `Set` structures for hot path
- Optional Hyperbee/Hypercore persistence when `storageDir` or `memoryOnly` is configured
- `close()` tears down swarm, timers, and clears ephemeral state
## Composition
Composes with: `hyper-p2p-dedup-filter`, `hyper-p2p-distributed-event-bus`.
@@ -0,0 +1,11 @@
require('bare-process/global')
const { HyperP2PGossipMesh } = require('../index.js')
async function main () {
const m = new HyperP2PGossipMesh()
m.on('message', (e) => console.log('msg', e.msg))
m.publish({ hello: 'world' }, { ttl: 4, fanout: 2 })
await m.close()
console.log('done')
}
main().catch(console.error)
@@ -0,0 +1,89 @@
require('bare-process/global')
const EventEmitter = require('bare-events')
const b4a = require('b4a')
const { initModuleSwarm, gossipSend } = require('../../_shared/p2p-bare.js')
const PROTOCOL = 'gossip-mesh/v1'
class HyperP2PGossipMesh extends EventEmitter {
constructor (opts = {}) {
super()
this._stats = { ops: 0, errors: 0 }
this.topic = opts.topic || null
this.keyPair = opts.keyPair || require('hypercore-crypto').keyPair()
this._seen = new Set()
this._filter = () => true
this.defaultTtl = opts.defaultTtl ?? 8
this.defaultFanout = opts.defaultFanout ?? 3
this.dedupFilter = opts.dedupFilter || null
this.swarm = null
this._peerMsgs = null
}
subscribe (filter) {
this._filter = typeof filter === 'function' ? filter : () => true
return () => { this._filter = () => true }
}
_msgId (msg) {
return b4a.toString(require('hypercore-crypto').hash(b4a.from(JSON.stringify(msg))), 'hex')
}
publish (msg, opts = {}) {
const ttl = opts.ttl ?? this.defaultTtl
const fanout = opts.fanout ?? this.defaultFanout
const id = this._msgId(msg)
if (this.dedupFilter && this.dedupFilter.seen(id)) return false
if (this._seen.has(id)) return false
this._seen.add(id)
if (this.dedupFilter) this.dedupFilter.add(id, { gossip: false })
const envelope = { id, msg, ttl, fanout }
if (this._filter(msg)) this.emit('message', envelope)
if (this._peerMsgs) gossipSend(this, { type: 'gossip', envelope })
return id
}
_receive (envelope) {
if (!envelope || envelope.ttl <= 0) return
if (this.dedupFilter && this.dedupFilter.seen(envelope.id)) return
if (this._seen.has(envelope.id)) return
this._seen.add(envelope.id)
if (this.dedupFilter) this.dedupFilter.add(envelope.id, { gossip: false })
if (this._filter(envelope.msg)) this.emit('message', envelope)
if (envelope.ttl > 1 && this._peerMsgs) {
const next = { ...envelope, ttl: envelope.ttl - 1 }
let n = 0
for (const [, peerMsg] of this._peerMsgs) {
if (n >= envelope.fanout) break
try { peerMsg.send({ type: 'gossip', envelope: next }); n++ } catch (_) {}
}
}
}
async ready () {
if (this.swarm || !this.topic) return this
await initModuleSwarm(this, {
keyPair: this.keyPair,
topic: this.topic,
protocol: PROTOCOL,
onmessage: (data) => {
if (data && data.type === 'gossip') this._receive(data.envelope)
}
})
return this
}
getStats () {
return { ...this._stats }
}
async close () {
if (this.swarm) await this.swarm.destroy().catch(() => {})
this.swarm = null
this.emit('closed')
}
}
module.exports = { HyperP2PGossipMesh, PROTOCOL }
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,23 @@
{
"name": "hyper-p2p-gossip-mesh",
"version": "0.3.1",
"description": "TTL/fanout gossip mesh for Bare/Pear P2P.",
"main": "index.js",
"type": "commonjs",
"license": "Apache-2.0",
"scripts": { "test": "brittle-bare test/test.js" },
"dependencies": {
"bare-events": "^2.8.0",
"bare-process": "^4.4.0",
"b4a": "^1.6.7",
"hypercore-crypto": "^3.0.0",
"protomux": "^3.0.0",
"compact-encoding": "^2.0.0"
},
"peerDependencies": { "hyperswarm": "^4.0.0", "bare": ">=1.0.0" },
"devDependencies": { "brittle": "^3.0.0" },
"imports": {
"process": { "bare": "bare-process", "default": "process" },
"events": { "bare": "bare-events", "default": "events" }
}
}
@@ -0,0 +1,57 @@
require('bare-process/global')
const test = require('brittle')
const { HyperP2PGossipMesh } = require('../index.js')
test('gossip-mesh: publish and filter', async (t) => {
const m = new HyperP2PGossipMesh()
m.subscribe((msg) => msg.kind === 'x')
let got = 0
m.on('message', () => got++)
m.publish({ kind: 'x', v: 1 })
m.publish({ kind: 'y', v: 2 })
t.is(got, 1)
await m.close()
})
test('gossip-mesh: dedupe publish', async (t) => {
const m = new HyperP2PGossipMesh()
const msg = { id: 'same' }
t.ok(m.publish(msg))
t.not(m.publish(msg))
await m.close()
})
test('gossip-mesh: external dedup filter', async (t) => {
const { HyperP2PDedupFilter } = require('../../hyper-p2p-dedup-filter/index.js')
const dedup = new HyperP2PDedupFilter()
const m = new HyperP2PGossipMesh({ dedupFilter: dedup })
const msg = { kind: 'evt', v: 1 }
t.ok(m.publish(msg))
t.not(m.publish(msg))
await m.close()
await dedup.close()
})
test('hyper-p2p-gossip-mesh: close without leak', async (t) => {
const m = new HyperP2PGossipMesh()
await m.close()
t.pass()
})
test('hyper-p2p-gossip-mesh: validation rejects invalid input', async (t) => {
const m = new HyperP2PGossipMesh()
try {
if (typeof m.addNeighbor === 'function') m.addNeighbor(null)
else if (typeof m.buildCircuit === 'function') m.buildCircuit([])
else if (typeof m.grant === 'function') m.grant(null, -1)
else if (typeof m.enqueue === 'function') m.enqueue('bad', null)
else if (typeof m.reportSample === 'function') m.reportSample(null, -1, -1)
else if (typeof m.fanout === 'function') m.fanout(null, 0)
else if (typeof m.probe === 'function') m.probe(null)
else if (typeof m.resolve === 'function') m.resolve(null)
else if (typeof m.acquire === 'function') m.acquire(null)
else throw new Error('no validation hook')
t.fail('expected throw')
} catch (err) {
t.ok(err instanceof Error)
}
await m.close()
})