fix: multi-repo local dev (file:../ deps, package exports, missing deps)
This commit is contained in:
Raven Scott
+1
@@ -0,0 +1 @@
|
||||
* text=auto eol=lf
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
name: Integrate
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
name: Lint
|
||||
steps:
|
||||
- uses: holepunchto/actions/node-base@v1
|
||||
- run: npm run lint
|
||||
test:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
platform: linux
|
||||
- os: macos-latest
|
||||
platform: darwin
|
||||
- os: windows-latest
|
||||
platform: win32
|
||||
runs-on: ${{ matrix.os }}
|
||||
name: Test / ${{ matrix.platform }}
|
||||
steps:
|
||||
- uses: holepunchto/actions/bare-base@v1
|
||||
- run: npm test
|
||||
+1
@@ -0,0 +1 @@
|
||||
"prettier-config-holepunch"
|
||||
+201
@@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
Copyright 2023 Holepunch Inc
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
# index-encoder
|
||||
|
||||
Encode multiple values into sorted keys
|
||||
|
||||
```
|
||||
npm install index-encoder
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```js
|
||||
const IndexEncoder = require('index-encoder')
|
||||
|
||||
const idx = new IndexEncoder([IndexEncoder.STRING, IndexEncoder.UINT])
|
||||
|
||||
const buf = idx.encode(['foo', 42])
|
||||
console.log(idx.decode(buf)) // ['foo', 42]
|
||||
|
||||
// pass any compact encoding name to map it to the relevant encoding
|
||||
const uintType = IndexEncoder.lookup('uint')
|
||||
const bufferType = IndexEncoder.lookup('buffer')
|
||||
const alsoBufferType = IndexEncoder.lookup('fixed32')
|
||||
```
|
||||
|
||||
## API
|
||||
|
||||
#### `const idx = new IndexEncoder(encodings, { prefix = -1 })`
|
||||
|
||||
Create an index encoder using the array of `encodings`. The index encoder will produce a buffer encoding in the same order as `encodings`. A `prefix` can be defined to differentiate encoded buffers with the same `encodings`.
|
||||
|
||||
#### `const buf = idx.encode(keys)`
|
||||
|
||||
Encode the provided `keys` as a buffer. `keys` must be in the same order as the `encodings` when constructing the index encoder. `keys` can have less elements than the defined `encodings` to create a prefix buffer. When used with [`hyperbee`](https://github.com/holepunchto/hyperbee), this can be used to search for only a subset of the total keys.
|
||||
|
||||
#### `const range = idx.encodeRange(query)`
|
||||
|
||||
Encodes the `query` object values into their buffer equivalent values. Useful for creating range queries for [`hyperbee`](https://github.com/holepunchto/hyperbee?tab=readme-ov-file#const-stream--dbcreatereadstreamrange-options) based on the index encoder.
|
||||
|
||||
The `query` specifies the range you want to read:
|
||||
|
||||
```
|
||||
{
|
||||
gt, // Return values only > than this
|
||||
gte, // Return values >= than this
|
||||
lt, // Return values < than this
|
||||
lte: // Return values <= than this
|
||||
}
|
||||
```
|
||||
|
||||
Returned query will be bounded by the `prefix` if defined and the query is unbounded. This ensures ranges only query ranges with the same `prefix`.
|
||||
|
||||
#### `const values = idx.decode(buffer)`
|
||||
|
||||
Decode the `buffer` into the original values. `values` will be in the same order as the defined `encodings`. Buffers encoded with a subset of keys will return the values encoded with remaining values being `0` filled if encoded with `UNIT` else `null`:
|
||||
|
||||
```
|
||||
const idx = new IndexEncoder([
|
||||
IndexEncoder.STRING,
|
||||
IndexEncoder.UINT,
|
||||
IndexEncoder.STRING
|
||||
])
|
||||
|
||||
const buf = idx.encode(['foo'])
|
||||
console.log(idx.decode(buf)) // ['foo', 0, null]
|
||||
```
|
||||
|
||||
## Encoding Types
|
||||
|
||||
The following are the types for `encodings` are supported provided as static properties on the `IndexEncoder` class:
|
||||
|
||||
- `IndexEncoder.BUFFER` : Encodes a buffer.
|
||||
- `IndexEncoder.STRING` : Encodes a string.
|
||||
- `IndexEncoder.UINT` : Encodes an unsigned integer.
|
||||
- `IndexEncoder.INT` : Encodes an signed integer.
|
||||
- `IndexEncoder.DATE` : Encodes a date.
|
||||
- `IndexEncoder.BOOL` : Encodes a boolean.
|
||||
|
||||
#### `const enc = IndexEncoder.lookup(c)`
|
||||
|
||||
Returns the encoding type for a given compact encoding name (`c`). If `c` is not supported, it will throw an `Unknown type` error.
|
||||
|
||||
## License
|
||||
|
||||
Apache 2.0
|
||||
+483
@@ -0,0 +1,483 @@
|
||||
const b4a = require('b4a')
|
||||
|
||||
const EMPTY = b4a.alloc(0)
|
||||
const MAX = b4a.from([0xff])
|
||||
const BUFFER = {}
|
||||
|
||||
BUFFER.preencode = function (state, buf) {
|
||||
if (buf === null) buf = EMPTY
|
||||
|
||||
let i = 0
|
||||
let extra = 3
|
||||
|
||||
while ((i = b4a.indexOf(buf, 0x00, i)) > -1) {
|
||||
i++
|
||||
extra++
|
||||
}
|
||||
|
||||
state.end += buf.byteLength + extra
|
||||
}
|
||||
|
||||
BUFFER.encode = function (state, buf) {
|
||||
if (buf === null) buf = EMPTY
|
||||
|
||||
state.buffer[state.start++] = 0x00
|
||||
|
||||
let prev = 0
|
||||
let i = 0
|
||||
|
||||
while ((i = b4a.indexOf(buf, 0x00, i)) > -1) {
|
||||
const slice = buf.subarray(prev, ++i)
|
||||
|
||||
state.buffer.set(slice, state.start)
|
||||
state.start += slice.byteLength
|
||||
state.buffer[state.start++] = 0x02
|
||||
prev = i
|
||||
}
|
||||
|
||||
const slice = buf.subarray(prev)
|
||||
|
||||
state.buffer.set(slice, state.start)
|
||||
state.start += slice.byteLength
|
||||
|
||||
state.buffer[state.start++] = 0x00
|
||||
state.buffer[state.start++] = 0x01
|
||||
}
|
||||
|
||||
BUFFER.decode = function (state) {
|
||||
if (state.start >= state.end) throw new Error('Out of bounds')
|
||||
if (state.buffer[state.start++] !== 0x00) {
|
||||
throw new Error('Invalid start of string')
|
||||
}
|
||||
|
||||
let escaped = null
|
||||
|
||||
let prev = state.start
|
||||
let i = state.start
|
||||
|
||||
while ((i = b4a.indexOf(state.buffer, 0x00, i)) > -1) {
|
||||
const next = ++i < state.end ? state.buffer[i] : 0x00
|
||||
|
||||
i++
|
||||
|
||||
if (next === 0x01) {
|
||||
break
|
||||
}
|
||||
|
||||
if (next === 0x02) {
|
||||
if (escaped === null) escaped = []
|
||||
escaped.push(state.buffer.subarray(prev, i - 1))
|
||||
prev = i
|
||||
continue
|
||||
}
|
||||
|
||||
throw new Error('Unknown value in terminator')
|
||||
}
|
||||
|
||||
if (i === -1) {
|
||||
throw new Error('No terminator found')
|
||||
}
|
||||
|
||||
state.start = i
|
||||
const last = state.buffer.subarray(prev, i - 2)
|
||||
if (escaped === null) return last
|
||||
|
||||
escaped.push(last)
|
||||
return b4a.concat(escaped)
|
||||
}
|
||||
|
||||
// TODO: can be optimised a lot
|
||||
|
||||
const STRING = {}
|
||||
|
||||
STRING.preencode = function (state, str) {
|
||||
BUFFER.preencode(state, b4a.from(str || ''))
|
||||
}
|
||||
|
||||
STRING.encode = function (state, str) {
|
||||
BUFFER.encode(state, b4a.from(str || ''))
|
||||
}
|
||||
|
||||
STRING.decode = function (state, str) {
|
||||
return b4a.toString(BUFFER.decode(state))
|
||||
}
|
||||
|
||||
const UINT = {}
|
||||
|
||||
UINT.preencode = function (state, n) {
|
||||
state.end += n <= 0xfb ? 1 : n <= 0xffff ? 3 : n <= 0xffffffff ? 5 : n === Infinity ? 1 : 9
|
||||
}
|
||||
|
||||
UINT.encode = function (state, n) {
|
||||
if (n === Infinity) {
|
||||
state.buffer[state.start++] = 0xff
|
||||
return
|
||||
}
|
||||
|
||||
if (n <= 0xfb) {
|
||||
state.buffer[state.start++] = n
|
||||
return
|
||||
}
|
||||
|
||||
if (n <= 0xffff) {
|
||||
state.buffer[state.start++] = 0xfc
|
||||
state.buffer[state.start++] = n >>> 8
|
||||
state.buffer[state.start++] = n
|
||||
return
|
||||
}
|
||||
|
||||
if (n <= 0xffffffff) {
|
||||
state.buffer[state.start++] = 0xfd
|
||||
encodeUint32(state, n)
|
||||
return
|
||||
}
|
||||
|
||||
if (Number.isSafeInteger(n)) {
|
||||
state.buffer[state.start++] = 0xfe
|
||||
|
||||
const r = Math.floor(n / 0x100000000)
|
||||
encodeUint32(state, r)
|
||||
encodeUint32(state, n)
|
||||
return
|
||||
}
|
||||
|
||||
throw new Error('Invalid number ' + n)
|
||||
}
|
||||
|
||||
UINT.decode = function (state) {
|
||||
if (state.start >= state.end) throw new Error('Out of bounds')
|
||||
|
||||
const a = state.buffer[state.start++]
|
||||
|
||||
if (a <= 0xfb) return a
|
||||
|
||||
if (a === 0xfc) {
|
||||
if (state.end - state.start < 2) throw new Error('Out of bounds')
|
||||
return state.buffer[state.start++] * 0x100 + state.buffer[state.start++]
|
||||
}
|
||||
|
||||
if (a === 0xfd) {
|
||||
return decodeUint32(state)
|
||||
}
|
||||
|
||||
if (a === 0xfe) {
|
||||
return decodeUint32(state) * 0x100000000 + decodeUint32(state)
|
||||
}
|
||||
|
||||
return Infinity
|
||||
}
|
||||
|
||||
const INT = {}
|
||||
|
||||
INT.preencode = function (state, n) {
|
||||
// One asymmetric condition
|
||||
if (n >= -0xff && n < 0) {
|
||||
state.end += 2
|
||||
return
|
||||
}
|
||||
n = Math.abs(n)
|
||||
state.end += n <= 0xf6 ? 1 : n <= 0xffff ? 3 : n <= 0xffffffff ? 5 : n === Infinity ? 1 : 9
|
||||
}
|
||||
|
||||
INT.encode = function (state, n) {
|
||||
if (n === Infinity) {
|
||||
state.buffer[state.start++] = 0xff
|
||||
return
|
||||
}
|
||||
|
||||
if (n === -Infinity) {
|
||||
state.buffer[state.start++] = 0x00
|
||||
return
|
||||
}
|
||||
|
||||
// Negative 64 bit
|
||||
if (n < -0xffffffff) {
|
||||
if (!Number.isSafeInteger(n)) throw new Error('Invalid number ' + n)
|
||||
state.buffer[state.start++] = 0x01
|
||||
|
||||
const r = Math.floor(-n / 0x100000000)
|
||||
encodeUint32(state, -r + 0xffffffff)
|
||||
encodeUint32(state, -(-n % 0x100000000) + 0xffffffff)
|
||||
return
|
||||
}
|
||||
|
||||
// Negative 32 bit
|
||||
if (n < -0xffff) {
|
||||
state.buffer[state.start++] = 0x02
|
||||
|
||||
n += 0xffffffff
|
||||
encodeUint32(state, n)
|
||||
return
|
||||
}
|
||||
|
||||
// Negative 16bit
|
||||
if (n < -0xff) {
|
||||
state.buffer[state.start++] = 0x03
|
||||
|
||||
n += 0xffff
|
||||
state.buffer[state.start++] = n >>> 8
|
||||
state.buffer[state.start++] = n
|
||||
return
|
||||
}
|
||||
|
||||
// Negative 8bit
|
||||
if (n < 0) {
|
||||
state.buffer[state.start++] = 0x04
|
||||
|
||||
n += 0xff
|
||||
state.buffer[state.start++] = n
|
||||
return
|
||||
}
|
||||
|
||||
// Remainder of 8bit space for positive int
|
||||
if (n <= 0xf6) {
|
||||
state.buffer[state.start++] = n + 0x05
|
||||
return
|
||||
}
|
||||
|
||||
// Positive 16bit
|
||||
if (n <= 0xffff) {
|
||||
state.buffer[state.start++] = 0xfc
|
||||
state.buffer[state.start++] = n >>> 8
|
||||
state.buffer[state.start++] = n
|
||||
return
|
||||
}
|
||||
|
||||
// Positive 32bit
|
||||
if (n <= 0xffffffff) {
|
||||
state.buffer[state.start++] = 0xfd
|
||||
encodeUint32(state, n)
|
||||
return
|
||||
}
|
||||
|
||||
// Positive 64bit
|
||||
if (Number.isSafeInteger(n)) {
|
||||
state.buffer[state.start++] = 0xfe
|
||||
|
||||
const r = Math.floor(n / 0x100000000)
|
||||
encodeUint32(state, r)
|
||||
encodeUint32(state, n)
|
||||
return
|
||||
}
|
||||
|
||||
throw new Error('Invalid number ' + n)
|
||||
}
|
||||
|
||||
INT.decode = function (state) {
|
||||
if (state.start >= state.end) throw new Error('Out of bounds')
|
||||
|
||||
const a = state.buffer[state.start++]
|
||||
|
||||
if (a === 0x00) return -Infinity
|
||||
|
||||
if (a === 0x01) {
|
||||
// Split (2^64 - 1) into its 32 bit components
|
||||
// r * 0x100000000 + n - (2^64 - 1)
|
||||
// r * 0x100000000 + n - (0xffffffff * 0x100000000) - 0xffffffff
|
||||
// (r - 0xffffffff) * 0x100000000 + n - 0xffffffff
|
||||
return (decodeUint32(state) - 0xffffffff) * 0x100000000 + decodeUint32(state) - 0xffffffff
|
||||
}
|
||||
|
||||
if (a === 0x02) {
|
||||
return decodeUint32(state) - 0xffffffff
|
||||
}
|
||||
|
||||
if (a === 0x03) {
|
||||
if (state.end - state.start < 2) throw new Error('Out of bounds')
|
||||
return state.buffer[state.start++] * 0x100 + state.buffer[state.start++] - 0xffff
|
||||
}
|
||||
|
||||
if (a === 0x04) return state.buffer[state.start++] - 0xff
|
||||
|
||||
if (a <= 0xfb) return a - 0x05
|
||||
|
||||
if (a === 0xfc) {
|
||||
if (state.end - state.start < 2) throw new Error('Out of bounds')
|
||||
return state.buffer[state.start++] * 0x100 + state.buffer[state.start++]
|
||||
}
|
||||
|
||||
if (a === 0xfd) {
|
||||
return decodeUint32(state)
|
||||
}
|
||||
|
||||
if (a === 0xfe) {
|
||||
return decodeUint32(state) * 0x100000000 + decodeUint32(state)
|
||||
}
|
||||
|
||||
return Infinity
|
||||
}
|
||||
|
||||
const DATE = {}
|
||||
|
||||
DATE.preencode = (state, d) => INT.preencode(state, d.getTime())
|
||||
DATE.encode = (state, d) => INT.encode(state, d.getTime())
|
||||
DATE.decode = (state) => new Date(INT.decode(state))
|
||||
|
||||
const BOOL = {}
|
||||
|
||||
BOOL.preencode = (state, b) => UINT.preencode(state, b ? 1 : 0)
|
||||
BOOL.encode = (state, b) => UINT.encode(state, b ? 1 : 0)
|
||||
BOOL.decode = (state) => !!UINT.decode(state)
|
||||
|
||||
module.exports = class IndexEncoder {
|
||||
constructor(encodings, { prefix = -1 } = {}) {
|
||||
this.encodings = encodings
|
||||
this.prefix = prefix
|
||||
}
|
||||
|
||||
static BUFFER = BUFFER
|
||||
static STRING = STRING
|
||||
static UINT = UINT
|
||||
static INT = INT
|
||||
static DATE = DATE
|
||||
static BOOL = BOOL
|
||||
|
||||
static lookup(c) {
|
||||
switch (c) {
|
||||
case 'uint':
|
||||
return UINT
|
||||
case 'uint8':
|
||||
return UINT
|
||||
case 'uint16':
|
||||
return UINT
|
||||
case 'uint24':
|
||||
return UINT
|
||||
case 'uint32':
|
||||
return UINT
|
||||
case 'uint40':
|
||||
return UINT
|
||||
case 'uint48':
|
||||
return UINT
|
||||
case 'uint56':
|
||||
return UINT
|
||||
case 'uint64':
|
||||
return UINT
|
||||
case 'int':
|
||||
return INT
|
||||
case 'int8':
|
||||
return INT
|
||||
case 'int16':
|
||||
return INT
|
||||
case 'int24':
|
||||
return INT
|
||||
case 'int32':
|
||||
return INT
|
||||
case 'int40':
|
||||
return INT
|
||||
case 'int48':
|
||||
return INT
|
||||
case 'int56':
|
||||
return INT
|
||||
case 'int64':
|
||||
return INT
|
||||
case 'string':
|
||||
return STRING
|
||||
case 'utf8':
|
||||
return STRING
|
||||
case 'ascii':
|
||||
return STRING
|
||||
case 'hex':
|
||||
return STRING
|
||||
case 'base64':
|
||||
return STRING
|
||||
case 'fixed32':
|
||||
return BUFFER
|
||||
case 'fixed64':
|
||||
return BUFFER
|
||||
case 'buffer':
|
||||
return BUFFER
|
||||
case 'date':
|
||||
return DATE
|
||||
case 'bool':
|
||||
return BOOL
|
||||
}
|
||||
|
||||
throw new Error('Unknown type')
|
||||
}
|
||||
|
||||
encode(keys) {
|
||||
return this._encode(keys, false)
|
||||
}
|
||||
|
||||
_encode(keys, terminate) {
|
||||
if (b4a.isBuffer(keys)) return keys
|
||||
|
||||
const state = { start: 0, end: 0, buffer: null }
|
||||
|
||||
if (this.prefix !== -1) UINT.preencode(state, this.prefix)
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
this.encodings[i].preencode(state, keys[i])
|
||||
}
|
||||
|
||||
if (terminate && keys.length < this.encodings.length) {
|
||||
state.end++
|
||||
}
|
||||
|
||||
state.buffer = b4a.allocUnsafe(state.end)
|
||||
|
||||
if (this.prefix !== -1) UINT.encode(state, this.prefix)
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
this.encodings[i].encode(state, keys[i])
|
||||
}
|
||||
|
||||
if (terminate && keys.length < this.encodings.length) {
|
||||
state.buffer[state.start++] = MAX[0]
|
||||
}
|
||||
|
||||
return state.buffer
|
||||
}
|
||||
|
||||
decode(buffer) {
|
||||
const state = { start: 0, end: buffer.byteLength, buffer }
|
||||
const result = []
|
||||
|
||||
if (this.prefix !== -1) UINT.decode(state)
|
||||
for (const enc of this.encodings) {
|
||||
const key = state.start < state.end ? enc.decode(state) : enc === UINT ? 0 : null
|
||||
result.push(key)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
encodeRange({ gt, gte, lt, lte }) {
|
||||
const range = {
|
||||
gt: gt && this._encode(gt, true),
|
||||
gte: gte && this._encode(gte, false),
|
||||
lt: lt && this._encode(lt, false),
|
||||
lte: lte && this._encode(lte, true)
|
||||
}
|
||||
|
||||
if (this.prefix !== -1) {
|
||||
if (!gt && !gte) range.gte = encodeUint(this.prefix)
|
||||
if (!lt && !lte) range.lt = encodeUint(this.prefix + 1)
|
||||
}
|
||||
|
||||
return range
|
||||
}
|
||||
}
|
||||
|
||||
function encodeUint(n) {
|
||||
const state = { start: 0, end: 0, buffer: null }
|
||||
UINT.preencode(state, n)
|
||||
state.buffer = b4a.allocUnsafe(state.end)
|
||||
UINT.encode(state, n)
|
||||
return state.buffer
|
||||
}
|
||||
|
||||
function encodeUint32(state, n) {
|
||||
state.buffer[state.start++] = n >>> 24
|
||||
state.buffer[state.start++] = n >>> 16
|
||||
state.buffer[state.start++] = n >>> 8
|
||||
state.buffer[state.start++] = n
|
||||
}
|
||||
|
||||
function decodeUint32(state) {
|
||||
if (state.end - state.start < 4) throw new Error('Out of bounds')
|
||||
return (
|
||||
state.buffer[state.start++] * 0x1000000 +
|
||||
state.buffer[state.start++] * 0x10000 +
|
||||
state.buffer[state.start++] * 0x100 +
|
||||
state.buffer[state.start++]
|
||||
)
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"name": "index-encoder",
|
||||
"version": "3.5.0",
|
||||
"description": "Encode multiple values into sorted keys",
|
||||
"main": "index.js",
|
||||
"dependencies": {
|
||||
"b4a": "^1.6.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"brittle": "^3.2.1",
|
||||
"hyperbee": "^2.10.5",
|
||||
"hypercore": "^11.0.0",
|
||||
"lunte": "^1.0.0",
|
||||
"prettier": "^3.6.2",
|
||||
"prettier-config-holepunch": "^2.0.0"
|
||||
},
|
||||
"scripts": {
|
||||
"format": "prettier --write .",
|
||||
"lint": "prettier --check . && lunte",
|
||||
"test": "npm run test:node && npm run test:bare",
|
||||
"test:node": "brittle-node test.js",
|
||||
"test:bare": "brittle-bare test.js"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/holepunchto/index-encoder.git"
|
||||
},
|
||||
"author": "Mathias Buus (@mafintosh)",
|
||||
"license": "Apache-2.0",
|
||||
"bugs": {
|
||||
"url": "https://github.com/holepunchto/index-encoder/issues"
|
||||
},
|
||||
"homepage": "https://github.com/holepunchto/index-encoder"
|
||||
}
|
||||
+319
@@ -0,0 +1,319 @@
|
||||
const b4a = require('b4a')
|
||||
const test = require('brittle')
|
||||
const Hyperbee = require('hyperbee')
|
||||
const Hypercore = require('hypercore')
|
||||
|
||||
const IndexEncoder = require('./')
|
||||
|
||||
test('a bunch of buffers', function (t) {
|
||||
const e = new IndexEncoder([IndexEncoder.BUFFER])
|
||||
|
||||
const all = []
|
||||
for (let i = 0; i < 65536; i++) {
|
||||
all.push({ i, buffer: e.encode([b4a.from([i >> 8, i & 255])]) })
|
||||
}
|
||||
|
||||
all.sort((a, b) => b4a.compare(a.buffer, b.buffer))
|
||||
|
||||
for (let i = 0; i < all.length; i++) {
|
||||
if (all[i].i !== i) {
|
||||
t.fail('sorted wrong:' + i)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
t.pass('all sorted correctly')
|
||||
})
|
||||
|
||||
test('some specific buffers', function (t) {
|
||||
const e = new IndexEncoder([IndexEncoder.BUFFER])
|
||||
|
||||
const all = [
|
||||
{ i: 0, buffer: e.encode([b4a.from([0])]) },
|
||||
{ i: 1, buffer: e.encode([b4a.from([0, 0])]) },
|
||||
{ i: 2, buffer: e.encode([b4a.from([0, 0, 0])]) },
|
||||
{ i: 3, buffer: e.encode([b4a.from([0, 1])]) },
|
||||
{ i: 4, buffer: e.encode([b4a.from([1])]) },
|
||||
{ i: 5, buffer: e.encode([b4a.from([1, 0])]) },
|
||||
{ i: 6, buffer: e.encode([b4a.from([1, 0, 0])]) },
|
||||
{ i: 7, buffer: e.encode([b4a.from([1, 1])]) },
|
||||
{ i: 8, buffer: e.encode([b4a.from([2])]) }
|
||||
]
|
||||
|
||||
all.sort((a, b) => b4a.compare(a.buffer, b.buffer))
|
||||
|
||||
for (let i = 0; i < all.length; i++) {
|
||||
if (all[i].i !== i) {
|
||||
t.fail('sorted wrong:' + i)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
t.pass('all sorted correctly')
|
||||
})
|
||||
|
||||
test('sliced', function (t) {
|
||||
const e = new IndexEncoder([IndexEncoder.STRING, IndexEncoder.UINT])
|
||||
|
||||
const all = [
|
||||
e.encode(['hello', 1]),
|
||||
e.encode(['hello', 2]),
|
||||
e.encode(['hallo', 1]),
|
||||
e.encode(['hollo', 1])
|
||||
]
|
||||
|
||||
all.sort(b4a.compare)
|
||||
|
||||
t.alike(sliceAndDecode(e, ['hello'], ['hello'], all), [
|
||||
['hello', 1],
|
||||
['hello', 2]
|
||||
])
|
||||
})
|
||||
|
||||
test('basic', function (t) {
|
||||
const i = new IndexEncoder([IndexEncoder.UINT, IndexEncoder.STRING])
|
||||
|
||||
const data = [
|
||||
[0, 'a'],
|
||||
[0, 'b'],
|
||||
[0, 'c'],
|
||||
[1, 'a'],
|
||||
[2, 'a'],
|
||||
[300, 'c'],
|
||||
[400, 'c']
|
||||
]
|
||||
|
||||
const keys = data.map((d) => i.encode(d))
|
||||
|
||||
t.alike(sliceAndDecode(i, [], [], keys), data)
|
||||
t.alike(sliceAndDecode(i, [0], [0], keys), [
|
||||
[0, 'a'],
|
||||
[0, 'b'],
|
||||
[0, 'c']
|
||||
])
|
||||
t.alike(sliceAndDecode(i, [0, 'b'], [0], keys), [
|
||||
[0, 'b'],
|
||||
[0, 'c']
|
||||
])
|
||||
t.alike(sliceAndDecodeNonInclusive(i, [0, 'a'], [0, 'c'], keys), [[0, 'b']])
|
||||
t.alike(sliceAndDecode(i, [1], [1], keys), [[1, 'a']])
|
||||
t.alike(sliceAndDecode(i, [2], [], keys), [
|
||||
[2, 'a'],
|
||||
[300, 'c'],
|
||||
[400, 'c']
|
||||
])
|
||||
})
|
||||
|
||||
test('int - encoder', function (t) {
|
||||
const enc = IndexEncoder.INT
|
||||
|
||||
const encode = (n) => {
|
||||
const state = { start: 0, end: 0, buffer: null }
|
||||
enc.preencode(state, n)
|
||||
state.buffer = b4a.allocUnsafe(state.end)
|
||||
enc.encode(state, n)
|
||||
return state
|
||||
}
|
||||
const encodeDecode = (n) => {
|
||||
const state = encode(n)
|
||||
state.start = 0
|
||||
return enc.decode(state)
|
||||
}
|
||||
|
||||
t.is(encodeDecode(0), 0, 'zero')
|
||||
t.alike(encode(-0).buffer, encode(0).buffer, 'neg zero ignore')
|
||||
t.is(encodeDecode(123), 123, '123')
|
||||
t.is(encodeDecode(-123), -123, '-123')
|
||||
t.is(encodeDecode(400), 400, '400')
|
||||
t.is(encodeDecode(-400), -400, '-400')
|
||||
t.is(encodeDecode(Infinity), Infinity, 'Infinity')
|
||||
t.is(encodeDecode(-Infinity), -Infinity, '-Infinity')
|
||||
t.is(encodeDecode(0xf6), 0xf6, '0xf6 - max 1 byte')
|
||||
t.is(encodeDecode(0xf7), 0xf7, '0xf7 - max 1 byte +1')
|
||||
t.is(encodeDecode(0xff), 0xff, '+(2^8-1)')
|
||||
t.is(encodeDecode(-0xff), -0xff, '-(2^8-1)')
|
||||
t.is(encodeDecode(0x100), 0x100, '+2^8')
|
||||
t.is(encodeDecode(-0x100), -0x100, '-2^8')
|
||||
t.is(encodeDecode(0xffff), 0xffff, '+(2^16-1)')
|
||||
t.is(encodeDecode(-0xffff), -0xffff, '-(2^16-1)')
|
||||
t.is(encodeDecode(0x10000), 0x10000, '+2^16')
|
||||
t.is(encodeDecode(-0x10000), -0x10000, '-2^16')
|
||||
t.is(encodeDecode(0xffffffff), 0xffffffff, '+(2^32-1)')
|
||||
t.is(encodeDecode(-0xffffffff), -0xffffffff, '-(2^32-1)')
|
||||
t.is(encodeDecode(0x100000000), 0x100000000, '+2^32')
|
||||
t.is(encodeDecode(-0x100000000), -0x100000000, '-2^32')
|
||||
t.is(encodeDecode(11491632000000), 11491632000000, '2^32 < x < 2^32')
|
||||
t.is(encodeDecode(-11491632000000), -11491632000000, '-2^32 > x > -2^32')
|
||||
t.is(encodeDecode(Number.MAX_SAFE_INTEGER), Number.MAX_SAFE_INTEGER, 'MAX_SAFE_INTEGER')
|
||||
t.is(encodeDecode(Number.MIN_SAFE_INTEGER), Number.MIN_SAFE_INTEGER, 'MIN_SAFE_INTEGER')
|
||||
|
||||
t.is(
|
||||
b4a.compare(encode(-100).buffer, encode(-1_000_000).buffer),
|
||||
1,
|
||||
'negative numbers order correctly'
|
||||
)
|
||||
|
||||
t.is(
|
||||
b4a.compare(encode(100).buffer, encode(-1_000_000).buffer),
|
||||
1,
|
||||
'positive vs negative numbers order correctly'
|
||||
)
|
||||
|
||||
t.exception(
|
||||
() => encode(Number.MIN_SAFE_INTEGER - 1),
|
||||
/Invalid number/,
|
||||
'throws on unsafe negative numbers'
|
||||
)
|
||||
t.exception(
|
||||
() => encode(Number.MAX_SAFE_INTEGER + 1),
|
||||
/Invalid number/,
|
||||
'throws on unsafe positive numbers'
|
||||
)
|
||||
})
|
||||
|
||||
test('int', function (t) {
|
||||
const i = new IndexEncoder([IndexEncoder.INT, IndexEncoder.STRING])
|
||||
|
||||
const data = [
|
||||
[0, 'a'],
|
||||
[-0, 'b'],
|
||||
[1, 'c'],
|
||||
[2, 'c'],
|
||||
[300, 'beep'],
|
||||
[-400, 'boop']
|
||||
]
|
||||
|
||||
const keys = data.map((d) => i.encode(d))
|
||||
|
||||
t.alike(sliceAndDecode(i, [], [], keys), [
|
||||
[0, 'a'],
|
||||
[0, 'b'], // Converts -0 to 0
|
||||
[1, 'c'],
|
||||
[2, 'c'],
|
||||
[300, 'beep'],
|
||||
[-400, 'boop']
|
||||
])
|
||||
t.alike(sliceAndDecodeNonInclusive(i, [0], [2], keys), [[1, 'c']])
|
||||
t.alike(sliceAndDecode(i, [], [-100], keys), [[-400, 'boop']])
|
||||
t.alike(
|
||||
sliceAndDecode(i, [300], [], keys),
|
||||
[[300, 'beep']],
|
||||
'ignores negative number of greater magnitude'
|
||||
)
|
||||
})
|
||||
|
||||
test('date', function (t) {
|
||||
const i = new IndexEncoder([IndexEncoder.DATE])
|
||||
|
||||
const data = [
|
||||
[new Date(0)],
|
||||
[new Date('1988-07-08')],
|
||||
[new Date('2016-09-08')],
|
||||
[new Date('2025-08-01')],
|
||||
[new Date('1605-11-05')]
|
||||
]
|
||||
|
||||
const keys = data.map((d) => i.encode(d))
|
||||
|
||||
t.alike(sliceAndDecode(i, [], [], keys), data)
|
||||
t.alike(sliceAndDecodeNonInclusive(i, [new Date('1987')], [new Date('2026')], keys), [
|
||||
[new Date('1988-07-08')],
|
||||
[new Date('2016-09-08')],
|
||||
[new Date('2025-08-01')]
|
||||
])
|
||||
t.alike(
|
||||
sliceAndDecode(i, [], [new Date('1979')], keys),
|
||||
[[new Date(0)], [new Date('1605-11-05')]],
|
||||
'range open start'
|
||||
)
|
||||
t.alike(
|
||||
sliceAndDecode(i, [new Date('2024')], [], keys),
|
||||
[[new Date('2025-08-01')]],
|
||||
'range open end'
|
||||
)
|
||||
})
|
||||
|
||||
test('bool indices', function (t) {
|
||||
const i = new IndexEncoder([IndexEncoder.BOOL, IndexEncoder.BOOL])
|
||||
|
||||
const data = [
|
||||
[true, true],
|
||||
[true, false],
|
||||
[false, true],
|
||||
[false, false]
|
||||
]
|
||||
|
||||
const keys = data.map((d) => i.encode(d))
|
||||
|
||||
t.alike(sliceAndDecode(i, [], [], keys), [
|
||||
[true, true],
|
||||
[true, false],
|
||||
[false, true],
|
||||
[false, false]
|
||||
])
|
||||
})
|
||||
|
||||
test('basic prefix', function (t) {
|
||||
const i = new IndexEncoder([IndexEncoder.UINT, IndexEncoder.STRING], {
|
||||
prefix: 4
|
||||
})
|
||||
|
||||
const buf = i.encode([])
|
||||
t.alike(buf, b4a.from([4]))
|
||||
|
||||
const range = i.encodeRange({})
|
||||
t.alike(range.gte, b4a.from([4]))
|
||||
t.alike(range.lt, b4a.from([5]))
|
||||
})
|
||||
|
||||
test('hyperbee bounded iteration', async function (t) {
|
||||
const keyEncoding = new IndexEncoder([IndexEncoder.UINT, IndexEncoder.STRING])
|
||||
const bee = new Hyperbee(new Hypercore(await t.tmp()), {
|
||||
keyEncoding,
|
||||
valueEncoding: 'utf-8'
|
||||
})
|
||||
t.teardown(() => bee.close())
|
||||
|
||||
await bee.put([1, 'a'], 'a')
|
||||
await bee.put([1, 'b'], 'b')
|
||||
await bee.put([2, 'aa'], 'aa')
|
||||
await bee.put([2, 'bb'], 'bb')
|
||||
await bee.put([3, 'aaa'], 'aaa')
|
||||
await bee.put([3, 'bbb'], 'bbb')
|
||||
|
||||
const expectedKeys = [
|
||||
[2, 'aa'],
|
||||
[2, 'bb']
|
||||
]
|
||||
|
||||
for await (const node of bee.createReadStream({ gt: [1], lt: [3] })) {
|
||||
t.alike(node.key, expectedKeys.shift())
|
||||
}
|
||||
t.is(expectedKeys.length, 0)
|
||||
})
|
||||
|
||||
function sliceAndDecodeNonInclusive(i, gt, lt, data) {
|
||||
const r = i.encodeRange({ gt, lt })
|
||||
const all = []
|
||||
|
||||
for (const key of data) {
|
||||
if (b4a.compare(r.gt, key) >= 0) continue
|
||||
if (b4a.compare(key, r.lt) >= 0) continue
|
||||
all.push(i.decode(key))
|
||||
}
|
||||
|
||||
return all
|
||||
}
|
||||
|
||||
function sliceAndDecode(i, gte, lte, data) {
|
||||
const r = i.encodeRange({ gte, lte })
|
||||
const all = []
|
||||
|
||||
for (const key of data) {
|
||||
if (b4a.compare(r.gte, key) > 0) continue
|
||||
if (b4a.compare(key, r.lte) > 0) continue
|
||||
all.push(i.decode(key))
|
||||
}
|
||||
|
||||
return all
|
||||
}
|
||||
Reference in New Issue
Block a user