Add Supercomputer category: 12 P2P resource-sharing modules (164 total)
Introduce a new production category for distributed system resource pooling over Hyperswarm — CPU, RAM, GPU, bandwidth, disk, jobs, cache, and egress. New modules (supercomputer/): - hyper-p2p-capacity-registry — gossip CPU/RAM/GPU/disk/bandwidth ads - hyper-p2p-cpu-share — CPU millisecond credit pool - hyper-p2p-ram-pool — RAM byte lend/claim/release - hyper-p2p-gpu-slot — GPU slot register/reserve - hyper-p2p-bandwidth-share — shared up/down Mbps - hyper-p2p-disk-stripe — striped block shards - hyper-p2p-job-dispatcher — submit/claim/complete jobs - hyper-p2p-thermal-guard — load/temperature throttle signals - hyper-p2p-cache-farm — distributed LRU cache - hyper-p2p-net-gateway — peer internet egress routing - hyper-p2p-cluster-affinity — hardware tag placement scoring - hyper-p2p-work-stealer — shard queues + work stealing Each package: index.js, tests, README, api/architecture docs, examples/basic.js. Shared layering guide: _shared/SUPERCOMPUTER_LAYERS.md. Registry, module-paths, MODULE_CATEGORIES, and workspace README updated to 164. Parent workspace also adds docs/supercomputer hub and demo-supercomputer-mesh (outside this repo). Co-authored-by: Cursor <[email protected]>
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
# Changelog
|
||||
|
||||
## 0.3.0
|
||||
|
||||
- Initial Supercomputer category release (disk-stripe/v1).
|
||||
@@ -0,0 +1,30 @@
|
||||
# hyper-p2p-disk-stripe
|
||||
|
||||
Production **Supercomputer** module: pool P2P system resources over Hyperswarm.
|
||||
|
||||
**Category:** Supercomputer · **Protocol:** `disk-stripe/v1` · **Export:** `HyperP2PDiskStripe`
|
||||
|
||||
## When to use
|
||||
|
||||
Striped block storage shards across the mesh.
|
||||
|
||||
## Quick start
|
||||
|
||||
```js
|
||||
const { HyperP2PDiskStripe } = require('hyper-p2p-disk-stripe')
|
||||
const mod = new HyperP2PDiskStripe({ topic: 'my-super-mesh' })
|
||||
await mod.ready()
|
||||
// ...
|
||||
await mod.close()
|
||||
```
|
||||
|
||||
## Docs
|
||||
|
||||
- [docs/api.md](docs/api.md)
|
||||
- [docs/architecture.md](docs/architecture.md)
|
||||
|
||||
## Test
|
||||
|
||||
```bash
|
||||
npm install && npm test
|
||||
```
|
||||
@@ -0,0 +1,36 @@
|
||||
# API: hyper-p2p-disk-stripe
|
||||
|
||||
**Protocol:** `disk-stripe/v1` · **Export:** `HyperP2PDiskStripe`
|
||||
|
||||
## Overview
|
||||
|
||||
Striped block storage shards across the mesh.
|
||||
|
||||
## Constructor
|
||||
|
||||
```js
|
||||
const mod = new HyperP2PDiskStripe(opts)
|
||||
```
|
||||
|
||||
| Option | Type | Default | Description |
|
||||
|--------|------|---------|-------------|
|
||||
| `topic` | Buffer \| string \| null | `null` | Hyperswarm topic |
|
||||
| `keyPair` | KeyPair | random | Discovery identity |
|
||||
|
||||
## Methods
|
||||
|
||||
See [`index.js`](../index.js) for the full method list. All modules implement `getStats()`, `async ready()`, and `async close()`.
|
||||
|
||||
## getStats()
|
||||
|
||||
Returns `{ ...stats, protocol: 'disk-stripe/v1' }` plus module-specific counters.
|
||||
|
||||
## P2P
|
||||
|
||||
When `topic` is set, `ready()` joins Hyperswarm and opens Protomux `disk-stripe/v1`.
|
||||
|
||||
## Testing
|
||||
|
||||
```bash
|
||||
npm install && npm test
|
||||
```
|
||||
@@ -0,0 +1,14 @@
|
||||
# Architecture: hyper-p2p-disk-stripe
|
||||
|
||||
**Category:** Supercomputer · **Protocol:** `disk-stripe/v1`
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
App[Application] --> Mod[HyperP2PDiskStripe]
|
||||
Mod --> Mux[Protomux disk-stripe/v1]
|
||||
Mux --> Swarm[Hyperswarm]
|
||||
```
|
||||
|
||||
## Composition
|
||||
|
||||
See [`../../_shared/SUPERCOMPUTER_LAYERS.md`](../../_shared/SUPERCOMPUTER_LAYERS.md) and [`../README.md`](../README.md).
|
||||
@@ -0,0 +1,10 @@
|
||||
require('bare-process/global')
|
||||
const { HyperP2PDiskStripe } = require('../index.js')
|
||||
|
||||
async function main () {
|
||||
const mod = new HyperP2PDiskStripe()
|
||||
console.log('stats', mod.getStats())
|
||||
await mod.close()
|
||||
console.log('done')
|
||||
}
|
||||
main().catch(console.error)
|
||||
@@ -0,0 +1,101 @@
|
||||
require('bare-process/global')
|
||||
const EventEmitter = require('bare-events')
|
||||
const b4a = require('b4a')
|
||||
const { initModuleSwarm, gossipSend } = require('../../_shared/p2p-bare.js')
|
||||
|
||||
const PROTOCOL = 'disk-stripe/v1'
|
||||
|
||||
class HyperP2PDiskStripe extends EventEmitter {
|
||||
constructor (opts = {}) {
|
||||
super()
|
||||
this._stats = { stripes: 0, shards: 0 }
|
||||
this.topic = opts.topic || null
|
||||
this.keyPair = opts.keyPair || require('hypercore-crypto').keyPair()
|
||||
this.peerId = b4a.toString(this.keyPair.publicKey, 'hex')
|
||||
this._stripes = new Map()
|
||||
this.swarm = null
|
||||
this._peerMsgs = null
|
||||
}
|
||||
|
||||
allocateStripe (sizeBytes, pathCount = 2, stripeId = null) {
|
||||
if (sizeBytes == null || sizeBytes < 1) throw new Error('sizeBytes required')
|
||||
if (pathCount < 1 || pathCount > 32) throw new Error('pathCount must be 1-32')
|
||||
const id = stripeId || b4a.toString(require('hypercore-crypto').hash(b4a.from(String(sizeBytes) + Date.now())), 'hex').slice(0, 16)
|
||||
const shardSize = Math.ceil(sizeBytes / pathCount)
|
||||
const stripe = {
|
||||
stripeId: id,
|
||||
sizeBytes,
|
||||
pathCount,
|
||||
shardSize,
|
||||
shards: new Map(),
|
||||
createdAt: Date.now()
|
||||
}
|
||||
this._stripes.set(id, stripe)
|
||||
this._stats.stripes++
|
||||
if (this._peerMsgs) gossipSend(this, { type: 'allocate', stripe: { ...stripe, shards: null } })
|
||||
this.emit('allocate', stripe)
|
||||
return id
|
||||
}
|
||||
|
||||
writeShard (stripeId, idx, data, peerId = null) {
|
||||
const stripe = this._stripes.get(stripeId)
|
||||
if (!stripe) throw new Error('unknown stripe')
|
||||
if (idx < 0 || idx >= stripe.pathCount) throw new Error('invalid shard index')
|
||||
const buf = b4a.isBuffer(data) ? data : b4a.from(data)
|
||||
const shard = {
|
||||
idx,
|
||||
data: b4a.toString(buf, 'hex'),
|
||||
peerId: peerId || this.peerId,
|
||||
at: Date.now()
|
||||
}
|
||||
stripe.shards.set(idx, shard)
|
||||
this._stats.shards++
|
||||
if (this._peerMsgs) gossipSend(this, { type: 'shard', stripeId, shard })
|
||||
this.emit('shard', { stripeId, shard })
|
||||
return shard
|
||||
}
|
||||
|
||||
readShard (stripeId, idx) {
|
||||
const stripe = this._stripes.get(stripeId)
|
||||
if (!stripe) return null
|
||||
const s = stripe.shards.get(idx)
|
||||
return s ? { ...s, data: b4a.from(s.data, 'hex') } : null
|
||||
}
|
||||
|
||||
isComplete (stripeId) {
|
||||
const stripe = this._stripes.get(stripeId)
|
||||
return stripe ? stripe.shards.size === stripe.pathCount : false
|
||||
}
|
||||
|
||||
listStripeIds () { return [...this._stripes.keys()] }
|
||||
|
||||
async ready () {
|
||||
if (this.swarm || !this.topic) return this
|
||||
await initModuleSwarm(this, {
|
||||
keyPair: this.keyPair, topic: this.topic, protocol: PROTOCOL,
|
||||
onmessage: (data) => {
|
||||
if (data?.type === 'allocate' && data.stripe) {
|
||||
const s = { ...data.stripe, shards: new Map() }
|
||||
this._stripes.set(s.stripeId, s)
|
||||
} else if (data?.type === 'shard') {
|
||||
const stripe = this._stripes.get(data.stripeId)
|
||||
if (stripe) stripe.shards.set(data.shard.idx, data.shard)
|
||||
}
|
||||
}
|
||||
})
|
||||
return this
|
||||
}
|
||||
|
||||
getStats () {
|
||||
return { ...this._stats, open: this._stripes.size, protocol: PROTOCOL }
|
||||
}
|
||||
|
||||
async close () {
|
||||
if (this.swarm) await this.swarm.destroy().catch(() => {})
|
||||
this.swarm = null
|
||||
this._stripes.clear()
|
||||
this.emit('closed')
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { HyperP2PDiskStripe, PROTOCOL }
|
||||
+1774
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "hyper-p2p-disk-stripe",
|
||||
"version": "0.3.0",
|
||||
"description": "Striped block storage across peers for Bare/Pear P2P supercomputer mesh.",
|
||||
"main": "index.js",
|
||||
"type": "commonjs",
|
||||
"license": "Apache-2.0",
|
||||
"scripts": { "test": "brittle-bare test/test.js" },
|
||||
"dependencies": {
|
||||
"bare-events": "^2.8.0",
|
||||
"bare-process": "^4.4.0",
|
||||
"b4a": "^1.6.7",
|
||||
"hypercore-crypto": "^3.0.0",
|
||||
"protomux": "^3.0.0",
|
||||
"compact-encoding": "^2.0.0"
|
||||
},
|
||||
"peerDependencies": { "hyperswarm": "^4.0.0", "bare": ">=1.0.0" },
|
||||
"devDependencies": { "brittle": "^3.0.0" },
|
||||
"imports": {
|
||||
"process": { "bare": "bare-process", "default": "process" },
|
||||
"events": { "bare": "bare-events", "default": "events" }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
require('bare-process/global')
|
||||
const test = require('brittle')
|
||||
const { HyperP2PDiskStripe } = require('../index.js')
|
||||
|
||||
test('disk-stripe: allocate and shard', async (t) => {
|
||||
const d = new HyperP2PDiskStripe()
|
||||
const id = d.allocateStripe(1024, 2)
|
||||
d.writeShard(id, 0, 'aa')
|
||||
d.writeShard(id, 1, 'bb')
|
||||
t.ok(d.isComplete(id))
|
||||
await d.close()
|
||||
})
|
||||
|
||||
test('disk-stripe: validation', async (t) => {
|
||||
const d = new HyperP2PDiskStripe()
|
||||
try {
|
||||
d.allocateStripe(0)
|
||||
t.fail('expected throw')
|
||||
} catch (e) {
|
||||
t.ok(e instanceof Error)
|
||||
}
|
||||
await d.close()
|
||||
})
|
||||
Reference in New Issue
Block a user