144 lines
4.5 KiB
JavaScript
144 lines
4.5 KiB
JavaScript
/**
|
|
* Packed Bare extracts the llama addon .bare + backend .so files under /tmp,
|
|
* but addon.js still sets backendsDir from bundled __dirname (`bare:/...`).
|
|
* C++ cannot dlopen that URL; llama.cpp then runs --fit against a broken
|
|
* backend path and init fails with "Failed to initialize model".
|
|
*
|
|
* Point backendsDir at the extracted prebuilds directory before createInstance.
|
|
*/
|
|
|
|
function posixDirname(p) {
|
|
const n = String(p || '').replace(/\\/g, '/').replace(/\/+$/, '');
|
|
const i = n.lastIndexOf('/');
|
|
if (i < 0) return n;
|
|
if (i === 0) return '/';
|
|
return n.slice(0, i);
|
|
}
|
|
|
|
function flattenError(err) {
|
|
const parts = [];
|
|
let cur = err;
|
|
for (let n = 0; cur && n < 6; n++) {
|
|
const msg = cur && cur.message ? String(cur.message) : cur ? String(cur) : '';
|
|
if (msg && parts.indexOf(msg) === -1) parts.push(msg);
|
|
cur = cur && cur.cause;
|
|
}
|
|
return parts.join(' | ') || 'unknown error';
|
|
}
|
|
|
|
function hrefOf(url) {
|
|
if (!url) return '';
|
|
if (typeof url === 'string') return url;
|
|
if (typeof url.href === 'string') return url.href;
|
|
try {
|
|
return String(url);
|
|
} catch (_) {
|
|
return '';
|
|
}
|
|
}
|
|
|
|
function fileUrlToPath(href) {
|
|
if (!href || href.indexOf('file:') !== 0) return null;
|
|
try {
|
|
const url = require('bare-url');
|
|
if (url && typeof url.fileURLToPath === 'function') return url.fileURLToPath(href);
|
|
} catch (_) {}
|
|
let rest = href.replace(/^file:\/\//, '');
|
|
if (/^\/[A-Za-z]:\//.test(rest)) rest = rest.slice(1);
|
|
try {
|
|
rest = decodeURIComponent(rest);
|
|
} catch (_) {}
|
|
return rest;
|
|
}
|
|
|
|
/**
|
|
* addon.js uses path.join(packageRoot, 'prebuilds').
|
|
* Extracted .bare lives at prebuilds/<host>/qvac__llm-llamacpp.bare
|
|
*/
|
|
function backendsDirFromAddonHref(href) {
|
|
const raw = hrefOf(href);
|
|
if (!raw || !/qvac__llm-llamacpp|llm-llamacpp/i.test(raw)) return null;
|
|
const file = raw.indexOf('file:') === 0 ? fileUrlToPath(raw) : raw.indexOf('bare:') === 0 ? null : raw;
|
|
if (!file) return null;
|
|
const base = file.replace(/\\/g, '/');
|
|
if (!/\.(bare|node)$/i.test(base)) return null;
|
|
return posixDirname(posixDirname(file));
|
|
}
|
|
|
|
function resolveBackendsDir() {
|
|
try {
|
|
const cache = typeof Bare !== 'undefined' && Bare.Addon && Bare.Addon.cache;
|
|
if (cache) {
|
|
const hrefs = Object.keys(cache);
|
|
for (let i = 0; i < hrefs.length; i++) {
|
|
const dir = backendsDirFromAddonHref(hrefs[i]);
|
|
if (dir) return dir;
|
|
const addon = cache[hrefs[i]];
|
|
const fromUrl = backendsDirFromAddonHref(addon && addon.url);
|
|
if (fromUrl) return fromUrl;
|
|
}
|
|
}
|
|
} catch (_) {}
|
|
return null;
|
|
}
|
|
|
|
function patchLlamaInterface(logFn) {
|
|
const log = typeof logFn === 'function' ? logFn : function () {};
|
|
let addon;
|
|
try {
|
|
addon = require('@qvac/llm-llamacpp/addon.js');
|
|
} catch (err) {
|
|
log('llm addon.js unavailable: ' + ((err && err.message) || err));
|
|
return false;
|
|
}
|
|
const Orig = addon && addon.LlamaInterface;
|
|
if (!Orig) return false;
|
|
if (Orig.__bsBackendsPatched) return true;
|
|
function Wrapped(binding, configurationParams, outputCb) {
|
|
if (!configurationParams) configurationParams = {};
|
|
if (!configurationParams.config) configurationParams.config = {};
|
|
const dir = resolveBackendsDir();
|
|
if (dir) {
|
|
configurationParams.config.backendsDir = dir;
|
|
log('llm backendsDir ' + dir);
|
|
}
|
|
return new Orig(binding, configurationParams, outputCb);
|
|
}
|
|
Wrapped.prototype = Orig.prototype;
|
|
try {
|
|
Object.setPrototypeOf(Wrapped, Orig);
|
|
} catch (_) {}
|
|
Wrapped.__bsBackendsPatched = true;
|
|
Orig.__bsBackendsPatched = true;
|
|
addon.LlamaInterface = Wrapped;
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Qwen3.5/3.6 default `remove_thinking_from_context: true`. That snapshots
|
|
* hybrid/SSM state at the prefill boundary and hard-fails with
|
|
* `ReasoningBlockCompactor::snapshotAtPrefillBoundary ... underflowed`.
|
|
* Keep thinking in KV (JS history compaction still drops old turns).
|
|
*/
|
|
function mergeGenerationParams(opts) {
|
|
const extra =
|
|
opts && opts.generationParams && typeof opts.generationParams === 'object' ? opts.generationParams : {};
|
|
let remove = false;
|
|
if (Object.prototype.hasOwnProperty.call(extra, 'remove_thinking_from_context')) {
|
|
remove = extra.remove_thinking_from_context === true;
|
|
} else if (opts && opts.removeThinkingFromContext !== undefined) {
|
|
remove = opts.removeThinkingFromContext === true;
|
|
}
|
|
const out = Object.assign({}, extra);
|
|
out.remove_thinking_from_context = remove;
|
|
return out;
|
|
}
|
|
|
|
module.exports = {
|
|
flattenError,
|
|
backendsDirFromAddonHref,
|
|
resolveBackendsDir,
|
|
patchLlamaInterface,
|
|
mergeGenerationParams,
|
|
};
|