Updates

indexes-search/hyper-p2p-bloom-gossip/index.js

@@ -1,38 +1,137 @@
 require('bare-process/global')
 const EventEmitter = require('bare-events')
+const b4a = require('b4a')
 const { assertNonEmpty } = require('../../_shared/lib/errors.js')
-const { attachGossip, sendGossip } = require('../../_shared/storage-gossip-base.js')
+const { initModuleSwarm, gossipSend } = require('../../_shared/p2p-bare.js')
 
 const PROTOCOL = 'bloom-gossip/v1'
+const DEFAULT_BITS = 2048
+const HASH_COUNT = 4
+
+function fnvHash (str, seed) {
+  let h = seed >>> 0
+  for (let i = 0; i < str.length; i++) {
+    h ^= str.charCodeAt(i)
+    h = Math.imul(h, 16777619)
+  }
+  return h >>> 0
+}
+
+function positions (key, size, k) {
+  const out = []
+  for (let i = 0; i < k; i++) out.push(fnvHash(key, 0x811c9dc5 + i * 31) % size)
+  return out
+}
 
 class HyperP2PBloomGossip extends EventEmitter {
   constructor (opts = {}) {
     super()
-    this._index = new Map()
-    this._stats = { indexed: 0, queries: 0 }
+    this.topic = opts.topic || null
+    this.keyPair = opts.keyPair || require('hypercore-crypto').keyPair()
+    this.peerHex = b4a.toString(this.keyPair.publicKey, 'hex')
+    this._size = opts.bits || DEFAULT_BITS
+    this._bits = new Uint8Array(Math.ceil(this._size / 8))
+    this._keys = new Set()
+    this._stats = { added: 0, queries: 0, gossipIn: 0, gossipOut: 0 }
+    this.swarm = null
+    this._peerMsgs = null
   }
 
-  index (docId, terms) {
-    assertNonEmpty(docId, 'docId')
-    for (const term of terms || []) {
-      const set = this._index.get(term) || new Set()
-      set.add(docId)
-      this._index.set(term, set)
+  add (key) {
+    assertNonEmpty(key, 'key')
+    const k = String(key)
+    if (this._keys.has(k)) return false
+    for (const pos of positions(k, this._size, HASH_COUNT)) {
+      const byte = pos >> 3
+      const bit = pos & 7
+      this._bits[byte] |= 1 << bit
     }
-    this._stats.indexed++
+    this._keys.add(k)
+    this._stats.added++
+    this._gossip({ type: 'bloom-add', key: k, peer: this.peerHex, at: Date.now() })
+    this.emit('add', { key: k })
     return true
   }
 
-  search (term) {
+  mightContain (key) {
+    assertNonEmpty(key, 'key')
     this._stats.queries++
-    const set = this._index.get(term)
-    return set ? [...set] : []
+    const k = String(key)
+    for (const pos of positions(k, this._size, HASH_COUNT)) {
+      const byte = pos >> 3
+      const bit = pos & 7
+      if (!(this._bits[byte] & (1 << bit))) return false
+    }
+    return true
   }
 
-  getStats () { return { ...this._stats, terms: this._index.size, protocol: PROTOCOL } }
+  exportBits () {
+    return {
+      size: this._size,
+      hashCount: HASH_COUNT,
+      bits: b4a.toString(this._bits, 'hex'),
+      count: this._keys.size
+    }
+  }
 
-  async ready () { return this }
-  async close () { this._index.clear() }
+  importBits (payload) {
+    if (!payload || !payload.bits) return false
+    this._bits = b4a.from(payload.bits, 'hex')
+    if (payload.size) this._size = payload.size
+    return true
+  }
+
+  size () {
+    return this._keys.size
+  }
+
+  _gossip (data) {
+    if (!this._peerMsgs) return
+    gossipSend(this, data)
+    this._stats.gossipOut++
+  }
+
+  _onGossip (data) {
+    if (!data || data.type !== 'bloom-add' || !data.key) return
+    this._stats.gossipIn++
+    if (!this._keys.has(data.key)) {
+      for (const pos of positions(data.key, this._size, HASH_COUNT)) {
+        const byte = pos >> 3
+        const bit = pos & 7
+        this._bits[byte] |= 1 << bit
+      }
+      this._keys.add(data.key)
+      this.emit('remote-add', { key: data.key, peer: data.peer })
+    }
+  }
+
+  getStats () {
+    return {
+      ...this._stats,
+      bits: this._size,
+      keys: this._keys.size,
+      protocol: PROTOCOL
+    }
+  }
+
+  async ready () {
+    if (this.swarm || !this.topic) return this
+    await initModuleSwarm(this, {
+      keyPair: this.keyPair,
+      topic: this.topic,
+      protocol: PROTOCOL,
+      onmessage: (data) => this._onGossip(data)
+    })
+    return this
+  }
+
+  async close () {
+    this._bits.fill(0)
+    this._keys.clear()
+    if (this.swarm) await this.swarm.destroy().catch(() => {})
+    this.swarm = null
+    this.emit('closed')
+  }
 }
 
 module.exports = { HyperP2PBloomGossip, PROTOCOL }