Deepen indexes-search: APIs, tests, and category docs

Extend all eight index/search packages with practical helpers for app
development and fix semantic-vector getStats() to expose real metrics.

Per-module code:
- bloom-gossip: listKeys(), clear()
- inverted-index: searchAll(and|or), getDocTerms()
- fulltext-lite: search(and|or), suggest(prefix)
- trie-prefix: countPrefix(), autocomplete(limit)
- graph-index: removeEdge(), outDegree(), edgeCount()
- similarity-lsh: nearVector(), bucketCount()
- semantic-vector-index: getStats() delegates to getMetrics() + PROTOCOL export
- spatial-index: listLocalPoints(), pointCount(), protocol in getStats()

Tests added for new APIs across six packages; all indexes-search npm test
suites pass (8/8).

Docs:
- Rewrite indexes-search category README (module table, composition)
- Expand bloom-gossip and semantic-vector architecture notes
- modules/README.md: link indexes-search doc hub

Parent workspace docs/indexes-search/README.md and MODULE_DOC_PASS.md
updated separately (outside this git root).

Co-authored-by: Cursor <[email protected]>
This commit is contained in:
Raven Scott
2026-05-21 00:20:56 -04:00
co-authored by Cursor
parent a11e22badc
commit e43e2e83f1
19 changed files with 247 additions and 23 deletions
+29 -10
View File
@@ -1,14 +1,33 @@
# Indexes & search
**Path:** `modules/indexes-search/` · **Modules:** 8 (8 production, 0 scaffold)
**Path:** `modules/indexes-search/` · **Modules:** 8 (all production)
See [MODULE_CATEGORIES.md](../MODULE_CATEGORIES.md#indexes-search).
Doc hub: [`docs/indexes-search/README.md`](../../docs/indexes-search/README.md) (parent workspace) · Index: [MODULE_CATEGORIES.md](../MODULE_CATEGORIES.md#indexes-search)
- [hyper-p2p-bloom-gossip](./hyper-p2p-bloom-gossip/) — production
- [hyper-p2p-fulltext-lite](./hyper-p2p-fulltext-lite/) — production
- [hyper-p2p-graph-index](./hyper-p2p-graph-index/) — production
- [hyper-p2p-inverted-index](./hyper-p2p-inverted-index/) — production
- [hyper-p2p-semantic-vector-index](./hyper-p2p-semantic-vector-index/) — production
- [hyper-p2p-similarity-lsh](./hyper-p2p-similarity-lsh/) — production
- [hyper-p2p-trie-prefix](./hyper-p2p-trie-prefix/) — production
- [hyper-spatial-index](./hyper-spatial-index/) — production
Search and indexing helpers for Bare/Pear apps — from gossip Bloom filters to semantic embedding search.
## Packages
| Module | P2P | Summary |
|--------|:---:|---------|
| [hyper-p2p-bloom-gossip](./hyper-p2p-bloom-gossip/) | yes | `add` / `mightContain`, `exportBits` / `importBits`, `listKeys` |
| [hyper-p2p-inverted-index](./hyper-p2p-inverted-index/) | yes | `index`, `search`, `searchAll` (and/or), `getDocTerms` |
| [hyper-p2p-fulltext-lite](./hyper-p2p-fulltext-lite/) | no | `addDocument`, `search` (and/or), `suggest` |
| [hyper-p2p-trie-prefix](./hyper-p2p-trie-prefix/) | no | `insert`, `prefixSearch`, `autocomplete`, `remove` |
| [hyper-p2p-graph-index](./hyper-p2p-graph-index/) | no | `addEdge`, `bfs`, `removeEdge`, `outDegree` |
| [hyper-p2p-similarity-lsh](./hyper-p2p-similarity-lsh/) | no | `embed`, `near`, `nearVector`, LSH signatures |
| [hyper-p2p-semantic-vector-index](./hyper-p2p-semantic-vector-index/) | optional | Full embedding index: insert/search/tags/sign/TTL |
| [hyper-spatial-index](./hyper-spatial-index/) | yes | Hyperbee grid, `insert`, `rangeQuery`, `findInRadius` |
## When to compose
- **Discovery:** bloom-gossip before expensive inverted lookups across peers.
- **RAG / agents:** semantic-vector-index + fulltext-lite for hybrid retrieval.
- **Maps / games:** spatial-index + graph-index for proximity and connectivity.
## Test
```bash
cd hyper-p2p-inverted-index && npm test
cd hyper-p2p-semantic-vector-index && npm test
```
@@ -36,6 +36,12 @@ Ignored: messages with missing `type`, wrong `type`, or missing `key`.
| `_stats` | Counters: `added`, `queries`, `gossipIn`, `gossipOut` |
| `peerHex` | Local public key hex on gossip payloads |
## Introspection
- `listKeys()` — exact key set (for debugging; Bloom `mightContain` remains probabilistic for unknown keys)
- `clear()` — zero bits and keys
- `exportBits()` / `importBits()` — hand off filter state to new peers
## Lifecycle
1. Construct with optional `topic` and `bits`
@@ -85,6 +85,19 @@ class HyperP2PBloomGossip extends EventEmitter {
return this._keys.size
}
listKeys () {
return [...this._keys].sort()
}
clear () {
this._bits.fill(0)
const n = this._keys.size
this._keys.clear()
this._stats.added = 0
this.emit('clear', { count: n })
return n
}
_gossip (data) {
if (!this._peerMsgs) return
gossipSend(this, data)
@@ -35,3 +35,13 @@ test('getStats', async (t) => {
t.is(m.getStats().added, 1)
await m.close()
})
test('listKeys clear', async (t) => {
const m = new HyperP2PBloomGossip()
m.add('a')
m.add('b')
t.is(m.listKeys().length, 2)
t.is(m.clear(), 2)
t.is(m.size(), 0)
await m.close()
})
@@ -47,21 +47,30 @@ class HyperP2PFulltextLite extends EventEmitter {
return entry
}
search (query) {
search (query, mode = 'and') {
this._stats.queries++
const terms = this.tokenize(query)
if (!terms.length) return []
let hits = null
for (const term of terms) {
const set = this._index.get(term)
const ids = set ? [...set] : []
if (hits === null) hits = new Set(ids)
else {
const next = new Set()
for (const id of ids) if (hits.has(id)) next.add(id)
hits = next
if (mode === 'or') {
hits = new Set()
for (const term of terms) {
const set = this._index.get(term)
if (set) for (const id of set) hits.add(id)
}
} else {
for (const term of terms) {
const set = this._index.get(term)
const ids = set ? [...set] : []
if (hits === null) hits = new Set(ids)
else {
const next = new Set()
for (const id of ids) if (hits.has(id)) next.add(id)
hits = next
}
}
}
if (!hits || !hits.size) return []
const results = [...hits].map((id) => {
const doc = this._docs.get(id)
const score = terms.filter((t) => doc.tokens.includes(t)).length
@@ -71,6 +80,17 @@ class HyperP2PFulltextLite extends EventEmitter {
return results
}
suggest (prefix, limit = 8) {
const p = String(prefix || '').toLowerCase()
if (!p) return []
const out = []
for (const term of this._index.keys()) {
if (term.startsWith(p)) out.push(term)
if (out.length >= limit) break
}
return out.sort()
}
getDocument (id) {
return this._docs.get(id) || null
}
@@ -36,3 +36,12 @@ test('getStats', async (t) => {
t.is(m.getStats().indexed, 1)
await m.close()
})
test('search or mode suggest', async (t) => {
const m = new HyperP2PFulltextLite()
m.addDocument('1', 'hello world')
m.addDocument('2', 'goodbye moon')
t.is(m.search('hello moon', 'or').length, 2)
t.ok(m.suggest('hel').includes('hello'))
await m.close()
})
@@ -64,6 +64,29 @@ class HyperP2PGraphIndex extends EventEmitter {
return !!(set && set.has(to))
}
removeEdge (from, to, opts = {}) {
assertNonEmpty(from, 'from')
assertNonEmpty(to, 'to')
const directed = opts.directed !== false
const del = (a, b) => {
const set = this._adj.get(a)
if (set) set.delete(b)
}
del(from, to)
if (!directed) del(to, from)
return true
}
outDegree (node) {
return this.neighbors(node).length
}
edgeCount () {
let n = 0
for (const list of this._adj.values()) n += list.size
return n
}
nodes () {
const all = new Set()
for (const [n, list] of this._adj) {
@@ -36,3 +36,14 @@ test('getStats', async (t) => {
t.is(m.getStats().edges, 1)
await m.close()
})
test('removeEdge outDegree edgeCount', async (t) => {
const m = new HyperP2PGraphIndex()
m.addEdge('a', 'b')
m.addEdge('a', 'c')
t.is(m.outDegree('a'), 2)
t.is(m.edgeCount(), 2)
m.removeEdge('a', 'b')
t.is(m.outDegree('a'), 1)
await m.close()
})
@@ -47,6 +47,37 @@ class HyperP2PInvertedIndex extends EventEmitter {
return set ? [...set].sort() : []
}
searchAll (terms, mode = 'and') {
if (!Array.isArray(terms) || !terms.length) return []
const normalized = terms.map(normalizeTerm).filter(Boolean)
if (!normalized.length) return []
this._stats.queries++
if (mode === 'or') {
const out = new Set()
for (const term of normalized) {
for (const docId of this.search(term)) out.add(docId)
}
return [...out].sort()
}
let hits = null
for (const term of normalized) {
const set = this._terms.get(term)
const ids = set ? [...set] : []
if (hits === null) hits = new Set(ids)
else {
const next = new Set()
for (const id of ids) if (hits.has(id)) next.add(id)
hits = next
}
}
return hits ? [...hits].sort() : []
}
getDocTerms (docId) {
const terms = this._docs.get(docId)
return terms ? [...terms].sort() : []
}
removeDoc (docId) {
assertNonEmpty(docId, 'docId')
const terms = this._docs.get(docId)
@@ -38,3 +38,13 @@ test('getStats', async (t) => {
t.is(m.getStats().indexed, 1)
await m.close()
})
test('searchAll getDocTerms', async (t) => {
const m = new HyperP2PInvertedIndex()
m.index('d1', ['cat', 'dog'])
m.index('d2', ['cat'])
t.is(m.searchAll(['cat', 'dog'], 'and').length, 1)
t.is(m.searchAll(['cat', 'dog'], 'or').length, 2)
t.alike(m.getDocTerms('d1'), ['cat', 'dog'])
await m.close()
})
@@ -66,4 +66,4 @@ insert / search / pruneExpired → close()
## Limits
- Linear scan search (no HNSW)
- `getStats()` references `_stats` which is not populated — use `getMetrics()` for counters
- `getStats()` returns `getMetrics()` plus `protocol` (same counters as `getMetrics()`)
@@ -549,7 +549,10 @@ class HyperP2PSemanticVectorIndex extends EventEmitter {
*/
getStats () {
return { ...this._stats }
return {
...this.getMetrics(),
protocol: SEMANTIC_PROTOCOL
}
}
async close () {
@@ -598,3 +601,5 @@ class HyperP2PSemanticVectorIndex extends EventEmitter {
}
module.exports = HyperP2PSemanticVectorIndex
module.exports.HyperP2PSemanticVectorIndex = HyperP2PSemanticVectorIndex
module.exports.PROTOCOL = SEMANTIC_PROTOCOL
@@ -124,6 +124,10 @@ test('metrics and P2P topic', async (t) => {
t.ok(m.searches >= 1, 'search metric')
t.ok(m.totalVectors >= 1, 'count metric')
const stats = index.getStats()
t.is(stats.protocol, 'hyper-p2p-semantic-vector-index/v1')
t.ok(stats.totalVectors >= 1, 'getStats mirrors metrics')
const topic = index.createP2PTopic('test-ns')
t.ok(b4a.isBuffer(topic), 'topic is buffer')
t.is(topic.length, 32, 'hash length 32 bytes')
@@ -83,6 +83,24 @@ class HyperP2PSimilarityLsh extends EventEmitter {
return { id, sig }
}
nearVector (vector, k = 5) {
if (!Array.isArray(vector) || !vector.length) {
throw new Error('vector must be a non-empty number array')
}
const probe = this._resize(vector)
const sig = this._signature(probe)
const candidates = new Set()
const bucket = this._buckets.get(sig)
if (bucket) for (const cid of bucket) candidates.add(cid)
const ranked = []
for (const cid of candidates) {
const other = this._vectors.get(cid)
if (other) ranked.push({ id: cid, score: cosine(probe, other.vector) })
}
ranked.sort((a, b) => b.score - a.score)
return ranked.slice(0, Math.max(0, k | 0))
}
near (id, k = 5) {
assertNonEmpty(id, 'id')
this._stats.queries++
@@ -120,6 +138,10 @@ class HyperP2PSimilarityLsh extends EventEmitter {
return true
}
bucketCount () {
return this._buckets.size
}
getStats () {
return {
...this._stats,
@@ -38,3 +38,13 @@ test('getStats', async (t) => {
t.is(m.getStats().embedded, 1)
await m.close()
})
test('nearVector bucketCount', async (t) => {
const m = new HyperP2PSimilarityLsh({ dim: 4 })
m.embed('a', [1, 0, 0, 0])
m.embed('b', [0.9, 0.1, 0, 0])
const hits = m.nearVector([1, 0, 0, 0], 2)
t.ok(hits.length >= 1)
t.ok(m.bucketCount() >= 1)
await m.close()
})
@@ -85,6 +85,15 @@ class HyperP2PTriePrefix extends EventEmitter {
return [...this._words].sort()
}
countPrefix (prefix) {
return this.prefixSearch(prefix).length
}
autocomplete (prefix, limit = 10) {
const all = this.prefixSearch(prefix)
return all.slice(0, Math.max(0, limit | 0))
}
getStats () {
return {
...this._stats,
@@ -36,3 +36,13 @@ test('getStats', async (t) => {
t.is(m.getStats().inserted, 1)
await m.close()
})
test('autocomplete countPrefix', async (t) => {
const m = new HyperP2PTriePrefix()
m.insert('cat')
m.insert('car')
m.insert('dog')
t.is(m.countPrefix('ca'), 2)
t.is(m.autocomplete('ca', 1).length, 1)
await m.close()
})
+13 -1
View File
@@ -235,8 +235,20 @@ class SpatialIndex extends EventEmitter {
}
listLocalPoints () {
return [...this.localPoints.values()]
}
pointCount () {
return this.localPoints.size
}
getStats () {
return { ...this._stats }
return {
...this._stats,
points: this.localPoints.size,
protocol: SPATIAL_PROTOCOL
}
}
async close () {