fix: multi-repo local dev (file:../ deps, package exports, missing deps)
This commit is contained in:
Raven Scott
+33
@@ -0,0 +1,33 @@
|
||||
exports.isDefinition = isDefinition
|
||||
exports.compat = makeCompat
|
||||
|
||||
function makeCompat(definition) {
|
||||
if (definition.versions) return definition
|
||||
|
||||
for (const c of definition.collections) {
|
||||
if (typeof c.version === 'number') continue
|
||||
const { encodeValue } = c
|
||||
c.encodeValue = function (schemaVersion, collectionVersion, record) {
|
||||
return encodeValue(schemaVersion, record)
|
||||
}
|
||||
c.version = 0
|
||||
c.decodedVersion = 0
|
||||
}
|
||||
|
||||
for (const i of definition.indexes) {
|
||||
if (typeof i.version === 'number') continue
|
||||
i.version = 0
|
||||
}
|
||||
|
||||
return {
|
||||
versions: { schema: definition.version, db: 0 },
|
||||
collections: definition.collections,
|
||||
indexes: definition.indexes,
|
||||
resolveCollection: definition.resolveCollection,
|
||||
resolveIndex: definition.resolveIndex
|
||||
}
|
||||
}
|
||||
|
||||
function isDefinition(definition) {
|
||||
return !!(definition && typeof definition.resolveCollection === 'function')
|
||||
}
|
||||
+228
@@ -0,0 +1,228 @@
|
||||
const HyperBee = require('hyperbee')
|
||||
const ScopeLock = require('scope-lock')
|
||||
const { Readable, getStreamError } = require('streamx')
|
||||
const c = require('compact-encoding')
|
||||
const RefCounter = require('refcounter')
|
||||
const Sessions = require('../sessions.js')
|
||||
|
||||
class BeeSnapshot {
|
||||
constructor(snap, onfree) {
|
||||
this.refs = 1
|
||||
this.snapshot = snap
|
||||
this.opened = false
|
||||
this.onfree = onfree
|
||||
}
|
||||
|
||||
async ready() {
|
||||
await this.snapshot.ready()
|
||||
this.opened = true
|
||||
}
|
||||
|
||||
ref() {
|
||||
this.refs++
|
||||
return this
|
||||
}
|
||||
|
||||
unref() {
|
||||
if (--this.refs === 0) {
|
||||
this.snapshot.close().then(this.onfree, this.onfree)
|
||||
this.snapshot = null
|
||||
}
|
||||
}
|
||||
|
||||
cork() {}
|
||||
|
||||
uncork() {}
|
||||
|
||||
getIndirectRange(reconstruct, entries, checkout, reqs) {
|
||||
const promises = new Array(entries.length)
|
||||
|
||||
for (let i = 0; i < promises.length; i++) {
|
||||
const { key, value } = entries[i]
|
||||
promises[i] = getWrapped(this.snapshot, key, reconstruct(key, value), checkout, reqs)
|
||||
}
|
||||
|
||||
return promises
|
||||
}
|
||||
|
||||
getBatch(keys, checkout, reqs) {
|
||||
const promises = new Array(keys.length)
|
||||
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
promises[i] = getValue(this.snapshot, keys[i], checkout, reqs)
|
||||
}
|
||||
|
||||
return Promise.all(promises)
|
||||
}
|
||||
|
||||
get(key, checkout, reqs) {
|
||||
return getValue(this.snapshot, key, checkout, reqs)
|
||||
}
|
||||
|
||||
createReadStream(range, options) {
|
||||
return this.snapshot.createReadStream(range, options)
|
||||
}
|
||||
}
|
||||
|
||||
class ChangesStream extends Readable {
|
||||
constructor(db, versions, definition, range) {
|
||||
super()
|
||||
|
||||
this.db = db
|
||||
this.versions = versions
|
||||
this.definition = definition
|
||||
this.stream = null
|
||||
this.collectionsById = new Map()
|
||||
this.range = range
|
||||
|
||||
for (const c of this.definition.collections) this.collectionsById.set(c.id, c)
|
||||
}
|
||||
|
||||
_open(cb) {
|
||||
this.stream = this.db.createHistoryStream(this.range)
|
||||
this.stream.on('readable', this._ondrain.bind(this))
|
||||
this.stream.on('error', noop)
|
||||
this.stream.on('close', this._onclose.bind(this))
|
||||
cb(null)
|
||||
}
|
||||
|
||||
_ondrain() {
|
||||
while (Readable.isBackpressured(this) === false) {
|
||||
const data = this.stream.read()
|
||||
|
||||
if (data === null) break
|
||||
|
||||
const id = c.uint.decode({ start: 0, end: data.key.byteLength, buffer: data.key })
|
||||
const coll = this.collectionsById.get(id)
|
||||
if (coll === undefined) continue
|
||||
|
||||
if (data.type === 'put') {
|
||||
const doc = coll.reconstruct(this.versions.schema, data.key, data.value)
|
||||
this.push({ type: 'insert', seq: data.seq, collection: coll.name, value: doc })
|
||||
} else {
|
||||
const key = coll.reconstructKey(data.key)
|
||||
this.push({ type: 'delete', seq: data.seq, collection: coll.name, value: key })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_onclose() {
|
||||
const err = getStreamError(this.stream, { all: true })
|
||||
if (err === null) this.push(null)
|
||||
else this.destroy(err)
|
||||
}
|
||||
|
||||
_read(cb) {
|
||||
this._ondrain()
|
||||
cb(null)
|
||||
}
|
||||
|
||||
_destroy(cb) {
|
||||
this.stream.destroy()
|
||||
cb(null)
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = class BeeEngine {
|
||||
constructor(core, { extension, trace } = {}) {
|
||||
this.asap = true
|
||||
this.clock = 0
|
||||
this.core = core
|
||||
this.sessions = new Sessions()
|
||||
this.trace = trace
|
||||
this.snaps = new RefCounter()
|
||||
this.db = new HyperBee(core, {
|
||||
extension,
|
||||
keyEncoding: 'binary',
|
||||
valueEncoding: 'binary'
|
||||
})
|
||||
|
||||
this.tx = null
|
||||
this.lock = new ScopeLock()
|
||||
|
||||
this._freeSnapBound = this.snaps.dec.bind(this.snaps)
|
||||
}
|
||||
|
||||
get closed() {
|
||||
return this.db.closed
|
||||
}
|
||||
|
||||
ready() {
|
||||
return this.db.ready()
|
||||
}
|
||||
|
||||
async close() {
|
||||
while (!this.snaps.isIdle()) await this.snaps.idle()
|
||||
await this.db.close()
|
||||
}
|
||||
|
||||
enter() {
|
||||
return this.lock.lock()
|
||||
}
|
||||
|
||||
exit() {
|
||||
this.tx = null
|
||||
this.lock.unlock()
|
||||
}
|
||||
|
||||
finalize(collection, versions, checkout, tracing, key, value) {
|
||||
if (value === null) return null
|
||||
|
||||
const reconstructed = collection.reconstruct(versions.schema, key, value)
|
||||
if (this.trace && tracing) this.trace(collection.name, reconstructed, checkout)
|
||||
|
||||
return reconstructed
|
||||
}
|
||||
|
||||
changes(snapshot, versions, definition, range) {
|
||||
const db = snapshot === null ? this.db : snapshot.snapshot
|
||||
return new ChangesStream(db, versions, definition, range)
|
||||
}
|
||||
|
||||
snapshot() {
|
||||
this.snaps.inc()
|
||||
return new BeeSnapshot(this.db.snapshot(), this._freeSnapBound)
|
||||
}
|
||||
|
||||
outdated(snap) {
|
||||
return (
|
||||
snap === null ||
|
||||
this.core.length !== snap.snapshot.core.length ||
|
||||
this.core.fork !== snap.snapshot.core.fork
|
||||
)
|
||||
}
|
||||
|
||||
async commit(updates) {
|
||||
this.clock++
|
||||
|
||||
let i = 0
|
||||
|
||||
const entries = updates.batch()
|
||||
const batch = this.db.batch({ maxBlocksCached: entries.length * 2 })
|
||||
|
||||
for (; i < entries.length; i++) {
|
||||
const [key, value] = entries[i]
|
||||
|
||||
if (value !== null) await batch.put(key, value)
|
||||
else await batch.del(key)
|
||||
}
|
||||
|
||||
await batch.flush()
|
||||
}
|
||||
|
||||
clearRequests(reqs) {
|
||||
if (!reqs.length) return
|
||||
this.core.core.replicator.clearRequests(reqs, null)
|
||||
}
|
||||
}
|
||||
|
||||
async function getWrapped(db, key, value, checkout, reqs) {
|
||||
return { key, value: [value, await getValue(db, value, checkout, reqs)] }
|
||||
}
|
||||
|
||||
async function getValue(db, key, checkout, activeRequests) {
|
||||
const node = await db.get(key, { checkout, activeRequests })
|
||||
return node === null ? null : node.value
|
||||
}
|
||||
|
||||
function noop() {}
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
const RocksDB = require('rocksdb-native')
|
||||
const ScopeLock = require('scope-lock')
|
||||
const Sessions = require('../sessions.js')
|
||||
|
||||
class RocksSnapshot {
|
||||
constructor(snap, clock) {
|
||||
this.clock = clock
|
||||
this.corks = 0
|
||||
this.refs = 1
|
||||
this.snapshot = snap
|
||||
this.opened = true
|
||||
this.batch = null
|
||||
this.batches = []
|
||||
}
|
||||
|
||||
ready() {
|
||||
return this.snapshot.ready()
|
||||
}
|
||||
|
||||
ref() {
|
||||
this.refs++
|
||||
return this
|
||||
}
|
||||
|
||||
unref() {
|
||||
if (--this.refs === 0) {
|
||||
this.snapshot.close().catch(noop)
|
||||
for (const b of this.batches) b.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
cork() {
|
||||
this.corks++
|
||||
if (this.batch === null) this.batch = this.snapshot.read()
|
||||
}
|
||||
|
||||
uncork() {
|
||||
if (--this.corks !== 0) return
|
||||
if (this.batch !== null) this._flushBackground(this.batch)
|
||||
this.batch = null
|
||||
}
|
||||
|
||||
// checkout, reqs is just here for completeness, not supported in rocks
|
||||
getIndirectRange(reconstruct, entries, checkout, reqs) {
|
||||
// TODO: add prop api for this in rocks, ie snapshot.closing
|
||||
if (this.snapshot._state.closing) return rejectAll(entries)
|
||||
|
||||
const read = this.batches.length > 0 ? this.batches.pop() : this.snapshot.read()
|
||||
const promises = new Array(entries.length)
|
||||
|
||||
for (let i = 0; i < promises.length; i++) {
|
||||
const { key, value } = entries[i]
|
||||
promises[i] = getWrapped(read, key, reconstruct(key, value))
|
||||
}
|
||||
|
||||
this._flushBackground(read)
|
||||
return promises
|
||||
}
|
||||
|
||||
// checkout is just here for completeness, not supported in rocks
|
||||
getBatch(keys, checkout, reqs) {
|
||||
const read = this.batches.length > 0 ? this.batches.pop() : this.snapshot.read()
|
||||
const promises = new Array(keys.length)
|
||||
|
||||
for (let i = 0; i < promises.length; i++) {
|
||||
promises[i] = read.get(keys[i])
|
||||
}
|
||||
|
||||
this._flushBackground(read)
|
||||
return Promise.all(promises)
|
||||
}
|
||||
|
||||
// checkout is just here for completeness, not supported in rocks
|
||||
get(key, checkout, reqs) {
|
||||
return this.batch === null ? this.snapshot.get(key) : this.batch.get(key)
|
||||
}
|
||||
|
||||
createReadStream(range, options) {
|
||||
return this.snapshot.iterator({ ...range, ...options })
|
||||
}
|
||||
|
||||
async _flushBackground(batch) {
|
||||
try {
|
||||
await batch.flush()
|
||||
this.batches.push(batch)
|
||||
} catch (err) {
|
||||
batch.destroy()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = class RocksEngine {
|
||||
constructor(storage, options = {}) {
|
||||
this.asap = false
|
||||
this.clock = 0
|
||||
this.sessions = new Sessions()
|
||||
this.trace = options.trace || null
|
||||
this.core = null
|
||||
this.db = typeof storage === 'object' ? storage : new RocksDB(storage, options)
|
||||
this.db.ready().catch(noop)
|
||||
this.write = null
|
||||
this.tx = null
|
||||
this.lock = new ScopeLock()
|
||||
}
|
||||
|
||||
get closed() {
|
||||
return this.db.closed
|
||||
}
|
||||
|
||||
enter() {
|
||||
return this.lock.lock()
|
||||
}
|
||||
|
||||
exit() {
|
||||
this.tx = null
|
||||
this.lock.unlock()
|
||||
}
|
||||
|
||||
ready() {
|
||||
return this.db.ready()
|
||||
}
|
||||
|
||||
close() {
|
||||
if (this.write !== null) this.write.destroy()
|
||||
return this.db.close()
|
||||
}
|
||||
|
||||
finalize(collection, versions, checkout, tracing, key, value) {
|
||||
if (value === null) return null
|
||||
|
||||
const reconstructed = collection.reconstruct(versions.schema, key, value)
|
||||
if (this.trace && tracing) this.trace(collection.name, reconstructed, checkout)
|
||||
|
||||
return reconstructed
|
||||
}
|
||||
|
||||
changes() {
|
||||
throw new Error('Not supported in Rocks engine')
|
||||
}
|
||||
|
||||
snapshot() {
|
||||
return new RocksSnapshot(this.db.snapshot(), this.clock)
|
||||
}
|
||||
|
||||
outdated(snap) {
|
||||
return snap === null || snap.clock !== this.clock
|
||||
}
|
||||
|
||||
async commit(updates) {
|
||||
this.clock++
|
||||
|
||||
if (this.write === null) this.write = this.db.write()
|
||||
|
||||
const entries = updates.batch()
|
||||
|
||||
for (let i = 0; i < entries.length; i++) {
|
||||
const [key, value] = entries[i]
|
||||
|
||||
if (value !== null) this.write.tryPut(key, value)
|
||||
else this.write.tryDelete(key)
|
||||
}
|
||||
|
||||
await this.write.flush()
|
||||
}
|
||||
|
||||
clearRequests(reqs) {
|
||||
// api parity
|
||||
}
|
||||
}
|
||||
|
||||
async function getWrapped(read, key, value) {
|
||||
return { key, value: [value, await read.get(value)] }
|
||||
}
|
||||
|
||||
function noop() {}
|
||||
|
||||
function rejectAll(entries) {
|
||||
const promises = new Array(entries.length)
|
||||
for (let i = 0; i < entries.length; i++) {
|
||||
promises[i] = Promise.reject(new Error('Database is closed'))
|
||||
}
|
||||
return promises
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
module.exports = class SessionTracker {
|
||||
constructor() {
|
||||
this.sessions = []
|
||||
}
|
||||
|
||||
get size() {
|
||||
return this.sessions.length
|
||||
}
|
||||
|
||||
has(s) {
|
||||
return s.index < this.sessions.length && this.sessions[s.index] === s
|
||||
}
|
||||
|
||||
add(s) {
|
||||
if (this.has(s)) return
|
||||
s.index = this.sessions.push(s) - 1
|
||||
}
|
||||
|
||||
remove(s) {
|
||||
if (!this.has(s)) return
|
||||
|
||||
const head = this.sessions.pop()
|
||||
if (head === s) return
|
||||
this.sessions[(head.index = s.index)] = head
|
||||
}
|
||||
|
||||
close(skip) {
|
||||
const closing = []
|
||||
|
||||
for (let i = this.sessions.length - 1; i >= 0; i--) {
|
||||
const s = this.sessions[i]
|
||||
if (s === skip) continue
|
||||
closing.push(s.close())
|
||||
}
|
||||
|
||||
return Promise.all(closing)
|
||||
}
|
||||
}
|
||||
+262
@@ -0,0 +1,262 @@
|
||||
const { Readable, isEnded, getStreamError } = require('streamx')
|
||||
const b4a = require('b4a')
|
||||
|
||||
module.exports = class IndexStream extends Readable {
|
||||
constructor(
|
||||
db,
|
||||
range,
|
||||
{
|
||||
index = null,
|
||||
collection = index.collection,
|
||||
reverse = false,
|
||||
limit = -1,
|
||||
overlay = [],
|
||||
checkout = 0
|
||||
}
|
||||
) {
|
||||
super()
|
||||
|
||||
this.versions = db.versions
|
||||
this.engine = db.engine
|
||||
this.snapshot = db.engineSnapshot
|
||||
this.activeRequests = db.activeRequests
|
||||
this.traceable = db.traceable
|
||||
this.collection = collection
|
||||
this.index = index
|
||||
this.stream = this.snapshot.createReadStream(range, {
|
||||
reverse,
|
||||
limit,
|
||||
checkout,
|
||||
activeRequests: this.activeRequests
|
||||
})
|
||||
this.streamClosed = false
|
||||
this.overlay = overlay
|
||||
this.overlayIndex = 0
|
||||
this.continueDestroy = null
|
||||
this.limit = limit
|
||||
this.reverse = reverse
|
||||
this.mapping = false
|
||||
this.ending = false
|
||||
this.checkout = checkout
|
||||
|
||||
this.snapshot.ref()
|
||||
}
|
||||
|
||||
_open(cb) {
|
||||
const destroy = this.destroy.bind(this)
|
||||
const drain = this._drain.bind(this)
|
||||
const onclose = this._onclose.bind(this)
|
||||
|
||||
this.stream.on('readable', drain)
|
||||
this.stream.on('end', drain)
|
||||
|
||||
this.stream.on('error', destroy)
|
||||
this.stream.on('close', onclose)
|
||||
|
||||
cb(null)
|
||||
}
|
||||
|
||||
_onclose() {
|
||||
this.streamClosed = true
|
||||
if (isEnded(this.stream) === false) this.destroy()
|
||||
this._continueDestroyMaybe()
|
||||
}
|
||||
|
||||
_continueDestroyMaybe() {
|
||||
if (this.mapping === true) return
|
||||
if (this.streamClosed === false) return
|
||||
if (this.continueDestroy === null) return
|
||||
|
||||
const cb = this.continueDestroy
|
||||
this.continueDestroy = null
|
||||
this.snapshot.unref()
|
||||
cb(null)
|
||||
}
|
||||
|
||||
_push(value) {
|
||||
if (value === null) return
|
||||
if (this.limit > 0) this.limit--
|
||||
this.push(
|
||||
this.engine.finalize(
|
||||
this.collection,
|
||||
this.versions,
|
||||
this.checkout,
|
||||
this.traceable,
|
||||
value[0],
|
||||
value[1]
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
_process(batch, ended) {
|
||||
for (let i = 0; i < batch.length && this.limit !== 0; i++) {
|
||||
const data = batch[i]
|
||||
|
||||
while (this.limit !== 0) {
|
||||
if (this.overlayIndex >= this.overlay.length) {
|
||||
this._push(data.value)
|
||||
break
|
||||
}
|
||||
|
||||
const cmp = b4a.compare(data.key, this.overlay[this.overlayIndex].key)
|
||||
if (this.reverse === true ? cmp > 0 : cmp < 0) {
|
||||
this._push(data.value)
|
||||
break
|
||||
}
|
||||
|
||||
this._push(this.overlay[this.overlayIndex++].value)
|
||||
if (cmp === 0) break
|
||||
}
|
||||
}
|
||||
|
||||
if (ended === true) {
|
||||
while (this.overlayIndex < this.overlay.length && this.limit !== 0) {
|
||||
this._push(this.overlay[this.overlayIndex++].value)
|
||||
}
|
||||
|
||||
this.push(null)
|
||||
this.ending = true
|
||||
}
|
||||
}
|
||||
|
||||
async _processAsap(promises, ended) {
|
||||
for (let i = 1; i < promises.length; i++) promises[i].catch(noop) // promises are handled
|
||||
|
||||
for (let i = 0; i < promises.length; i++) {
|
||||
const batch = [await promises[i]]
|
||||
if (this.destroying === true) return
|
||||
this._process(batch, i === promises.length - 1 && ended)
|
||||
}
|
||||
}
|
||||
|
||||
async _mapAndProcess() {
|
||||
this.mapping = true
|
||||
|
||||
while (!Readable.isBackpressured(this) && this.destroying === false) {
|
||||
const entries = fullyDrain(this.stream)
|
||||
|
||||
const promises =
|
||||
entries.length === 0
|
||||
? entries
|
||||
: this.snapshot.getIndirectRange(
|
||||
this.index.reconstruct,
|
||||
entries,
|
||||
this.checkout,
|
||||
this.activeRequests
|
||||
)
|
||||
const ended = isEnded(this.stream)
|
||||
|
||||
if (promises.length === 0 && ended === false) break
|
||||
|
||||
try {
|
||||
if (this.engine.asap === true && promises.length > 1) {
|
||||
await this._processAsap(promises, ended)
|
||||
} else {
|
||||
const batch = await Promise.all(promises)
|
||||
if (this.destroying === false) this._process(batch, ended)
|
||||
}
|
||||
} catch (err) {
|
||||
await Promise.allSettled(promises)
|
||||
this.destroy(err)
|
||||
}
|
||||
}
|
||||
|
||||
this.mapping = false
|
||||
this._continueDestroyMaybe()
|
||||
}
|
||||
|
||||
_drain() {
|
||||
if (Readable.isBackpressured(this)) return
|
||||
|
||||
if (this.index === null) {
|
||||
this._process(fullyDrainMapped(this.stream), isEnded(this.stream))
|
||||
} else if (this.mapping === false) {
|
||||
this._mapAndProcess()
|
||||
}
|
||||
}
|
||||
|
||||
_read(cb) {
|
||||
this._drain()
|
||||
cb(null)
|
||||
}
|
||||
|
||||
_predestroy() {
|
||||
this.stream.destroy()
|
||||
}
|
||||
|
||||
_destroy(cb) {
|
||||
this.stream.destroy()
|
||||
this.continueDestroy = cb
|
||||
this._continueDestroyMaybe()
|
||||
}
|
||||
|
||||
one() {
|
||||
const stream = this
|
||||
let last = null
|
||||
|
||||
return new Promise(function (resolve, reject) {
|
||||
stream.on('error', noop)
|
||||
stream.on('readable', onreadable)
|
||||
stream.on('close', onclose)
|
||||
|
||||
function onreadable() {
|
||||
while (true) {
|
||||
const data = stream.read()
|
||||
if (data === null) return
|
||||
last = data
|
||||
}
|
||||
}
|
||||
|
||||
function onclose() {
|
||||
if (isEnded(stream)) resolve(last)
|
||||
else reject(getStreamError(stream, { all: true }))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async toArray() {
|
||||
const stream = this
|
||||
const list = []
|
||||
|
||||
return new Promise(function (resolve, reject) {
|
||||
stream.on('error', noop)
|
||||
stream.on('readable', onreadable)
|
||||
stream.on('close', onclose)
|
||||
|
||||
function onreadable() {
|
||||
while (true) {
|
||||
const data = stream.read()
|
||||
if (data === null) return
|
||||
list.push(data)
|
||||
}
|
||||
}
|
||||
|
||||
function onclose() {
|
||||
if (isEnded(stream)) resolve(list)
|
||||
else reject(getStreamError(stream, { all: true }))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function fullyDrainMapped(stream) {
|
||||
const batch = []
|
||||
|
||||
while (true) {
|
||||
const data = stream.read()
|
||||
if (data === null) return batch
|
||||
batch.push({ key: data.key, value: [data.key, data.value] })
|
||||
}
|
||||
}
|
||||
|
||||
function fullyDrain(stream) {
|
||||
const batch = []
|
||||
|
||||
while (true) {
|
||||
const data = stream.read()
|
||||
if (data === null) return batch
|
||||
batch.push(data)
|
||||
}
|
||||
}
|
||||
|
||||
function noop() {}
|
||||
Reference in New Issue
Block a user