fix tests

test/hyperdb.test.js

@@ -14,16 +14,73 @@ const HYPERDB_SOFT_MS = Number(process.env.PEARDATA_HYPERDB_TEST_MS) || 30_000
 const HYPERDB_HARD_MS = HYPERDB_SOFT_MS + 15_000
 
 /**
- * Race `fn` against a soft timeout. On timeout: pass + comment (skip), do not fail CI.
- * @param {import('brittle').Test} t
- * @param {() => Promise<void>} fn
+ * After one hang or RocksDB lock error, remaining cases skip.
+ * A timed-out case used to keep the shared Corestore LOCK; the next
+ * `withModel` then crashed Node with `EIO: No locks available`.
  */
-async function withSoftTimeout(t, fn) {
+let skipRest = false
+let tmpSeq = 0
+
+function isSoftTimeout(err) {
+  return err?.code === 'HYPERDB_SOFT_TIMEOUT' || err?.message === 'HYPERDB_SOFT_TIMEOUT'
+}
+
+function isLockError(err) {
+  if (!err) return false
+  if (err.code !== 'EIO' && err.code !== 'EBUSY') return false
+  return /lock hold|No locks available/i.test(String(err.message || ''))
+}
+
+function rmSafe(dir) {
+  try {
+    fs.rmSync(dir, { recursive: true, force: true })
+  } catch {
+    // RocksDB may still hold files; unique dirs make leftovers harmless
+  }
+}
+
+async function closeStore(model, store, tmp) {
+  if (model) await model.close().catch(() => {})
+  if (store) await store.close().catch(() => {})
+  if (tmp) rmSafe(tmp)
+}
+
+/**
+ * Open an isolated Corestore, run `fn`, race against a soft timeout.
+ * On timeout: pass + comment (skip), close best-effort, do not fail CI.
+ * Leftover work is caught so a late Corestore rejection cannot crash Node.
+ * @param {import('brittle').Test} t
+ * @param {(db: PearDataModel) => Promise<void>} fn
+ */
+async function withModel(t, fn) {
+  if (skipRest) {
+    t.comment('hyperdb remaining tests skipped after prior soft-timeout')
+    t.pass('soft-skip: prior hyperdb test timed out')
+    return
+  }
+
+  const tmp = path.join(TMP, `${process.pid}-${++tmpSeq}`)
+  rmSafe(tmp)
+  const store = new Corestore(path.join(tmp, 'corestore'))
+  store.on('error', () => {})
+  let model = null
   let timer
+  let timedOut = false
   const started = Date.now()
+
+  const work = (async () => {
+    await store.ready()
+    const core = store.get({ name: 'peardata-meta' })
+    model = new PearDataModel(core, { autoUpdate: true })
+    await model.ready()
+    await fn(model)
+  })()
+  // Separate branch: abandoning `work` after race timeout must not be unhandled
+  work.catch(() => {})
+
   try {
     await Promise.race([
-      fn(),
+      work,
       new Promise((_, reject) => {
         timer = setTimeout(() => {
           const err = new Error('HYPERDB_SOFT_TIMEOUT')
@@ -33,9 +90,13 @@ async function withSoftTimeout(t, fn) {
       }),
     ])
   } catch (err) {
-    if (err?.code === 'HYPERDB_SOFT_TIMEOUT' || err?.message === 'HYPERDB_SOFT_TIMEOUT') {
+    if (isSoftTimeout(err) || isLockError(err)) {
+      timedOut = true
+      skipRest = true
       t.comment(
-        `hyperdb test soft-skipped after ${HYPERDB_SOFT_MS}ms (elapsed ${Date.now() - started}ms) — environment slow or hung`
+        isSoftTimeout(err)
+          ? `hyperdb test soft-skipped after ${HYPERDB_SOFT_MS}ms (elapsed ${Date.now() - started}ms) — environment slow or hung`
+          : `hyperdb test soft-skipped after RocksDB lock error: ${err.message}`
       )
       t.pass('soft-skip: hyperdb timed out without failing CI')
       return
@@ -43,109 +104,89 @@ async function withSoftTimeout(t, fn) {
     throw err
   } finally {
     if (timer) clearTimeout(timer)
-  }
-}
-
-async function withModel(fn) {
-  fs.rmSync(TMP, { recursive: true, force: true })
-  const store = new Corestore(path.join(TMP, 'corestore'))
-  await store.ready()
-  const core = store.get({ name: 'peardata-meta' })
-  const model = new PearDataModel(core, { autoUpdate: true })
-  await model.ready()
-  try {
-    await fn(model)
-  } finally {
-    await model.close().catch(() => {})
-    await store.close().catch(() => {})
-    fs.rmSync(TMP, { recursive: true, force: true })
+    const closing = closeStore(model, store, tmp)
+    closing.catch(() => {})
+    // Don't await close after a hang — close itself can block until brittle's hard timeout
+    if (!timedOut) await closing
   }
 }
 
 test('hyperdb put/get node', { timeout: HYPERDB_HARD_MS }, async (t) => {
-  await withSoftTimeout(t, async () => {
-    await withModel(async (db) => {
-      await db.putNode({
-        nodeId: 'node-a',
-        hostname: 'lab',
-        publicKeyHex: 'ab'.repeat(32),
-        cpus: 4,
-        updatedAt: Date.now(),
-      })
-      const row = await db.getNode('node-a')
-      t.is(row.hostname, 'lab')
-      t.is(row.cpus, 4)
+  await withModel(t, async (db) => {
+    await db.putNode({
+      nodeId: 'node-a',
+      hostname: 'lab',
+      publicKeyHex: 'ab'.repeat(32),
+      cpus: 4,
+      updatedAt: Date.now(),
     })
+    const row = await db.getNode('node-a')
+    t.is(row.hostname, 'lab')
+    t.is(row.cpus, 4)
   })
 })
 
 test('hyperdb peer links', { timeout: HYPERDB_HARD_MS }, async (t) => {
-  await withSoftTimeout(t, async () => {
-    await withModel(async (db) => {
-      const remote = 'cd'.repeat(32)
-      await db.putPeerLink({
-        localNodeId: 'local',
-        remotePublicKey: remote,
-        syncMode: 'pull',
-        alias: 'nas',
-      })
-      const links = await db.listPeerLinks('local')
-      t.is(links.length, 1)
-      t.is(links[0].alias, 'nas')
-      await db.deletePeerLink('local', remote)
-      t.is((await db.listPeerLinks('local')).length, 0)
+  await withModel(t, async (db) => {
+    const remote = 'cd'.repeat(32)
+    await db.putPeerLink({
+      localNodeId: 'local',
+      remotePublicKey: remote,
+      syncMode: 'pull',
+      alias: 'nas',
     })
+    const links = await db.listPeerLinks('local')
+    t.is(links.length, 1)
+    t.is(links[0].alias, 'nas')
+    await db.deletePeerLink('local', remote)
+    t.is((await db.listPeerLinks('local')).length, 0)
   })
 })
 
 test('hyperdb warm metric points', { timeout: HYPERDB_HARD_MS }, async (t) => {
-  await withSoftTimeout(t, async () => {
-    await withModel(async (db) => {
-      const ts = Date.now()
-      await db.putMetricPoints([
-        {
-          chart: 'system.cpu',
-          context: 'system.cpu',
-          ts: ts - 1000,
-          values: { user: 10, idle: 90 },
-          tier: 1,
-        },
-        {
-          chart: 'system.cpu',
-          context: 'system.cpu',
-          ts,
-          values: { user: 20, idle: 80 },
-          tier: 1,
-        },
-      ])
-      const rows = await db.queryMetricPoints({
+  await withModel(t, async (db) => {
+    const ts = Date.now()
+    await db.putMetricPoints([
+      {
         chart: 'system.cpu',
-        afterMs: ts - 5000,
-        beforeMs: ts + 1000,
-      })
-      t.is(rows.length, 2)
-      t.is(rows[1].values.user, 20)
+        context: 'system.cpu',
+        ts: ts - 1000,
+        values: { user: 10, idle: 90 },
+        tier: 1,
+      },
+      {
+        chart: 'system.cpu',
+        context: 'system.cpu',
+        ts,
+        values: { user: 20, idle: 80 },
+        tier: 1,
+      },
+    ])
+    const rows = await db.queryMetricPoints({
+      chart: 'system.cpu',
+      afterMs: ts - 5000,
+      beforeMs: ts + 1000,
     })
+    t.is(rows.length, 2)
+    t.is(rows[1].values.user, 20)
   })
 })
 
 test('hyperdb alert event', { timeout: HYPERDB_HARD_MS }, async (t) => {
-  await withSoftTimeout(t, async () => {
-    await withModel(async (db) => {
-      const ts = Date.now()
-      await db.putAlertEvent({
-        id: 'cpu_user_high',
-        ts,
-        chart: 'system.cpu',
-        dimension: 'user',
-        severity: 'warning',
-        value: 90,
-        threshold: 80,
-        message: 'high',
-      })
-      const evs = await db.listAlertEvents({ chart: 'system.cpu', limit: 10 })
-      t.ok(evs.length >= 1)
-      t.is(evs[0].severity, 'warning')
+  await withModel(t, async (db) => {
+    const ts = Date.now()
+    await db.putAlertEvent({
+      id: 'cpu_user_high',
+      ts,
+      chart: 'system.cpu',
+      dimension: 'user',
+      severity: 'warning',
+      value: 90,
+      threshold: 80,
+      message: 'high',
     })
+    const evs = await db.listAlertEvents({ chart: 'system.cpu', limit: 10 })
+    t.ok(evs.length >= 1)
+    t.is(evs[0].severity, 'warning')
   })
 })