Actually vendor harness
This commit is contained in:
Vendored
+461
@@ -0,0 +1,461 @@
|
||||
/**
|
||||
* Direct QVAC LLM wrapper via @qvac/sdk (Node worker). No BridgeSwarm.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const catalog = require('./catalog.js');
|
||||
const device = require('./device.js');
|
||||
const events = require('./events.js');
|
||||
const paths = require('./paths.js');
|
||||
|
||||
let sdk = null;
|
||||
let initError = null;
|
||||
let initPromise = null;
|
||||
let holdCount = 0;
|
||||
let loaded = emptyLoaded();
|
||||
const activeRequests = new Map();
|
||||
|
||||
function emptyLoaded() {
|
||||
return {
|
||||
modelId: null,
|
||||
friendlyId: null,
|
||||
constant: null,
|
||||
tools: false,
|
||||
vision: false,
|
||||
device: 'cpu',
|
||||
backend: null,
|
||||
backendId: null,
|
||||
deviceName: null,
|
||||
vram: null,
|
||||
ctxSize: null,
|
||||
requestId: null,
|
||||
};
|
||||
}
|
||||
|
||||
function log(msg) {
|
||||
try {
|
||||
process.stderr.write('[agent-harness] ' + msg + '\n');
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
function flattenError(err) {
|
||||
if (!err) return 'unknown error';
|
||||
if (typeof err === 'string') return err;
|
||||
const parts = [];
|
||||
let cur = err;
|
||||
for (let i = 0; i < 6 && cur; i++) {
|
||||
if (cur.message) parts.push(String(cur.message));
|
||||
else parts.push(String(cur));
|
||||
cur = cur.cause || cur.error;
|
||||
}
|
||||
return parts.join(' | ') || String(err);
|
||||
}
|
||||
|
||||
function lookupSrc(mod, name) {
|
||||
if (!name) return null;
|
||||
if (mod[name] != null) return mod[name];
|
||||
if (mod.models && mod.models[name] != null) return mod.models[name];
|
||||
if (mod.default && mod.default[name] != null) return mod.default[name];
|
||||
return null;
|
||||
}
|
||||
|
||||
async function ensureInit() {
|
||||
if (sdk) return sdk;
|
||||
if (initPromise) return initPromise;
|
||||
initPromise = (async () => {
|
||||
if (!process.env.QVAC_CONFIG_PATH) {
|
||||
const cfg = path.join(__dirname, '..', 'qvac.config.json');
|
||||
try {
|
||||
if (fs.existsSync(cfg)) process.env.QVAC_CONFIG_PATH = cfg;
|
||||
} catch (_) {}
|
||||
}
|
||||
paths.ensureQvacRoot();
|
||||
let mod;
|
||||
try {
|
||||
mod = await import('@qvac/sdk');
|
||||
} catch (err) {
|
||||
initError = flattenError(err);
|
||||
log('sdk import failed: ' + initError);
|
||||
throw new Error(initError);
|
||||
}
|
||||
if (typeof mod.loadModel !== 'function' && mod.default && typeof mod.default.loadModel === 'function') {
|
||||
mod = mod.default;
|
||||
}
|
||||
if (typeof mod.loadModel !== 'function' || typeof mod.completion !== 'function') {
|
||||
initError = '@qvac/sdk missing loadModel/completion';
|
||||
throw new Error(initError);
|
||||
}
|
||||
sdk = mod;
|
||||
return sdk;
|
||||
})();
|
||||
try {
|
||||
return await initPromise;
|
||||
} catch (err) {
|
||||
initPromise = null;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function probeResources() {
|
||||
return device.normalizeResources({
|
||||
totalRamBytes: os.totalmem(),
|
||||
vramBytes: 0,
|
||||
gpus: [],
|
||||
drivers: {},
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchSystemResources(s) {
|
||||
if (!s) return null;
|
||||
if (typeof s.getSystemResources === 'function') {
|
||||
try {
|
||||
return await s.getSystemResources();
|
||||
} catch (_) {}
|
||||
}
|
||||
if (typeof s.getSystemInfo === 'function') {
|
||||
try {
|
||||
return await s.getSystemInfo();
|
||||
} catch (_) {}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function resources() {
|
||||
const s = await ensureInit().catch(() => null);
|
||||
const info = await fetchSystemResources(s);
|
||||
if (info) return device.normalizeResources(info);
|
||||
return probeResources();
|
||||
}
|
||||
|
||||
function toTools(tools) {
|
||||
if (!tools || !Array.isArray(tools)) return undefined;
|
||||
return tools.map((t) => {
|
||||
if (t && t.type === 'function' && t.function) {
|
||||
return {
|
||||
type: 'function',
|
||||
name: t.function.name,
|
||||
description: t.function.description,
|
||||
parameters: t.function.parameters,
|
||||
};
|
||||
}
|
||||
return t;
|
||||
});
|
||||
}
|
||||
|
||||
function decodeDataUrl(dataUrl) {
|
||||
const m = String(dataUrl || '').match(/^data:([^;]+);base64,(.+)$/);
|
||||
if (!m) return null;
|
||||
return { mime: m[1], buf: Buffer.from(m[2], 'base64') };
|
||||
}
|
||||
|
||||
function extForMime(mime) {
|
||||
if (/png/i.test(mime)) return '.png';
|
||||
if (/webp/i.test(mime)) return '.webp';
|
||||
if (/gif/i.test(mime)) return '.gif';
|
||||
return '.jpg';
|
||||
}
|
||||
|
||||
function prepareVisionHistory(history) {
|
||||
const dir = paths.ensureDir(path.join(paths.ensureQvacRoot(), 'vision'));
|
||||
const list = Array.isArray(history) ? history : [];
|
||||
return list.map((msg) => {
|
||||
if (!msg || !Array.isArray(msg.images) || !msg.images.length) return msg;
|
||||
const attachments = [];
|
||||
for (let i = 0; i < Math.min(4, msg.images.length); i++) {
|
||||
const img = msg.images[i] || {};
|
||||
let buf = null;
|
||||
let mime = img.mime || 'image/jpeg';
|
||||
if (img.dataUrl) {
|
||||
const d = decodeDataUrl(img.dataUrl);
|
||||
if (d) {
|
||||
buf = d.buf;
|
||||
mime = d.mime;
|
||||
}
|
||||
} else if (img.dataBase64) {
|
||||
buf = Buffer.from(img.dataBase64, 'base64');
|
||||
} else if (img.path && fs.existsSync(img.path)) {
|
||||
attachments.push({ path: img.path });
|
||||
continue;
|
||||
}
|
||||
if (!buf) continue;
|
||||
const file = path.join(dir, 'img_' + Date.now() + '_' + i + extForMime(mime));
|
||||
fs.writeFileSync(file, buf);
|
||||
attachments.push({ path: file });
|
||||
}
|
||||
const copy = Object.assign({}, msg);
|
||||
delete copy.images;
|
||||
if (attachments.length) copy.attachments = (copy.attachments || []).concat(attachments);
|
||||
return copy;
|
||||
});
|
||||
}
|
||||
|
||||
async function resolveSrc(s, name) {
|
||||
const constant = catalog.resolveModelConstant(name);
|
||||
const fromSdk = lookupSrc(s, constant) || lookupSrc(s, name);
|
||||
if (fromSdk) return fromSdk;
|
||||
if (typeof s.lookupModelSrc === 'function') {
|
||||
try {
|
||||
const src = await s.lookupModelSrc(constant);
|
||||
if (src) return src;
|
||||
} catch (_) {}
|
||||
}
|
||||
throw new Error(
|
||||
'QVAC has no constant for ' +
|
||||
String(name) +
|
||||
' (' +
|
||||
constant +
|
||||
'). Only catalog GGUFs in @qvac/sdk load.'
|
||||
);
|
||||
}
|
||||
|
||||
async function resolveMmproj(s, entry) {
|
||||
const names = catalog.mmprojCandidates(entry);
|
||||
for (const n of names) {
|
||||
const src = lookupSrc(s, n);
|
||||
if (src) return src;
|
||||
if (typeof s.lookupModelSrc === 'function') {
|
||||
try {
|
||||
const found = await s.lookupModelSrc(n);
|
||||
if (found) return found;
|
||||
} catch (_) {}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function load(opts, onProgress) {
|
||||
const s = await ensureInit();
|
||||
opts = opts || {};
|
||||
onProgress = onProgress || opts.onProgress;
|
||||
if (loaded.modelId) {
|
||||
await unload().catch(() => {});
|
||||
}
|
||||
const entry = catalog.findCatalogEntry(opts.model || opts.modelSrc || opts.friendlyId);
|
||||
const modelSrc = await resolveSrc(s, opts.modelSrc || (entry && entry.constant) || opts.model);
|
||||
const res = await resources();
|
||||
let dev = device.pickDevice(opts.device || 'auto', res);
|
||||
const toolsOn = opts.tools !== false && (!entry || entry.tools !== false);
|
||||
const rawCtx = opts.ctxSize || opts.ctx_size || (entry && entry.ctxSize) || 8192;
|
||||
const ctxSize = device.capCtxSize(rawCtx, res, dev.device === 'gpu');
|
||||
const mmprojGpu = device.mmprojOnGpu(opts, res, dev.device === 'gpu');
|
||||
const wantVision =
|
||||
opts.vision !== false && (entry ? entry.vision === true : catalog.isVisionModel(opts.model || opts.modelSrc));
|
||||
const mmprojSrc = wantVision ? await resolveMmproj(s, entry) : null;
|
||||
const modelConfig = Object.assign(
|
||||
{
|
||||
device: dev.device,
|
||||
gpu_layers: device.gpuLayers(opts, dev),
|
||||
ctx_size: Number(ctxSize) || 8192,
|
||||
tools: !!toolsOn,
|
||||
'mmproj-use-gpu': !!mmprojGpu,
|
||||
},
|
||||
mmprojSrc ? { projectionModelSrc: mmprojSrc } : {}
|
||||
);
|
||||
const loadOpts = {
|
||||
modelSrc,
|
||||
modelType: 'llm',
|
||||
modelConfig,
|
||||
};
|
||||
if (typeof onProgress === 'function') {
|
||||
loadOpts.onProgress = (p) => {
|
||||
try {
|
||||
onProgress({
|
||||
percent: p.percentage != null ? p.percentage : p.percent,
|
||||
downloaded: p.downloaded,
|
||||
total: p.total,
|
||||
});
|
||||
} catch (_) {}
|
||||
};
|
||||
}
|
||||
log(
|
||||
'load ' +
|
||||
((entry && entry.id) || opts.model || 'model') +
|
||||
' device=' +
|
||||
modelConfig.device +
|
||||
' ngl=' +
|
||||
modelConfig.gpu_layers +
|
||||
' ctx=' +
|
||||
modelConfig.ctx_size +
|
||||
' vision=' +
|
||||
!!mmprojSrc
|
||||
);
|
||||
let modelId;
|
||||
try {
|
||||
modelId = await s.loadModel(loadOpts);
|
||||
} catch (err) {
|
||||
const msg = flattenError(err);
|
||||
log('load failed: ' + msg);
|
||||
if (dev.device === 'gpu' && String(opts.device || 'auto').toLowerCase() !== 'cpu') {
|
||||
log('retrying load on cpu');
|
||||
modelConfig.device = 'cpu';
|
||||
modelConfig.gpu_layers = 0;
|
||||
modelConfig['mmproj-use-gpu'] = false;
|
||||
try {
|
||||
modelId = await s.loadModel(loadOpts);
|
||||
dev = { device: 'cpu', gpu_layers: 0, fallback: 'cpu' };
|
||||
} catch (err2) {
|
||||
throw new Error(flattenError(err2));
|
||||
}
|
||||
} else {
|
||||
throw new Error(msg);
|
||||
}
|
||||
}
|
||||
const bl = device.backendLabel(res);
|
||||
loaded = {
|
||||
modelId,
|
||||
friendlyId: (entry && entry.id) || opts.model || null,
|
||||
constant: (entry && entry.constant) || catalog.resolveModelConstant(opts.model || opts.modelSrc),
|
||||
tools: !!toolsOn,
|
||||
vision: !!mmprojSrc,
|
||||
device: dev.device,
|
||||
backend: bl.backend,
|
||||
backendId: bl.backendId,
|
||||
deviceName: bl.deviceName,
|
||||
vram: res && (res.vram || res.vramBytes),
|
||||
ctxSize: modelConfig.ctx_size,
|
||||
requestId: null,
|
||||
};
|
||||
return Object.assign({}, loaded);
|
||||
}
|
||||
|
||||
async function unload() {
|
||||
if (!loaded.modelId || !sdk) {
|
||||
loaded = emptyLoaded();
|
||||
return;
|
||||
}
|
||||
const id = loaded.modelId;
|
||||
loaded = emptyLoaded();
|
||||
try {
|
||||
if (typeof sdk.unloadModel === 'function') await sdk.unloadModel({ modelId: id });
|
||||
} catch (err) {
|
||||
log('unload: ' + flattenError(err));
|
||||
}
|
||||
}
|
||||
|
||||
async function complete(opts, onEvent) {
|
||||
const s = await ensureInit();
|
||||
if (!loaded.modelId && !(opts && opts.modelId)) throw new Error('no model loaded');
|
||||
const rawHistory = (opts && opts.history) || (opts && opts.messages) || [];
|
||||
const history = prepareVisionHistory(rawHistory);
|
||||
const tools = toTools(opts && opts.tools);
|
||||
const dialect = (opts && opts.toolDialect) || catalog.toolDialectFor(loaded.friendlyId || loaded.constant);
|
||||
const params = {
|
||||
modelId: (opts && opts.modelId) || loaded.modelId,
|
||||
history,
|
||||
stream: opts && opts.stream === false ? false : true,
|
||||
captureThinking: true,
|
||||
};
|
||||
if (tools && tools.length) {
|
||||
params.tools = tools;
|
||||
params.toolDialect = dialect;
|
||||
}
|
||||
if (opts && opts.generationParams) params.generationParams = opts.generationParams;
|
||||
const run = s.completion(params);
|
||||
const requestId = run.requestId || (opts && opts.requestId) || null;
|
||||
loaded.requestId = requestId;
|
||||
if (requestId) activeRequests.set(requestId, run);
|
||||
|
||||
let text = '';
|
||||
let thinking = '';
|
||||
const toolCalls = [];
|
||||
try {
|
||||
if (run.events && typeof run.events[Symbol.asyncIterator] === 'function') {
|
||||
for await (const ev of run.events) {
|
||||
const n = events.normalizeCompletionEvent(ev);
|
||||
if (!n) continue;
|
||||
if (n.type === 'contentDelta') {
|
||||
text += n.delta;
|
||||
if (onEvent) onEvent(n);
|
||||
} else if (n.type === 'thinkingDelta') {
|
||||
thinking += n.delta;
|
||||
if (onEvent) onEvent(n);
|
||||
} else if (n.type === 'toolCall') {
|
||||
toolCalls.push(n.call);
|
||||
if (onEvent) onEvent(n);
|
||||
} else if (onEvent) {
|
||||
onEvent(n);
|
||||
}
|
||||
}
|
||||
} else if (run.tokenStream) {
|
||||
for await (const token of run.tokenStream) {
|
||||
text += token;
|
||||
if (onEvent) onEvent({ type: 'contentDelta', delta: token });
|
||||
}
|
||||
if (run.toolCallStream) {
|
||||
for await (const evt of run.toolCallStream) {
|
||||
const call = evt.call || evt;
|
||||
toolCalls.push(call);
|
||||
if (onEvent) onEvent({ type: 'toolCall', call });
|
||||
}
|
||||
}
|
||||
}
|
||||
let stats = null;
|
||||
try {
|
||||
if (run.final) {
|
||||
const fin = await run.final;
|
||||
if (fin) {
|
||||
if (fin.contentText && !text) text = fin.contentText;
|
||||
if (fin.thinkingText && !thinking) thinking = fin.thinkingText;
|
||||
if (Array.isArray(fin.toolCalls) && fin.toolCalls.length) {
|
||||
toolCalls.length = 0;
|
||||
for (const c of fin.toolCalls) toolCalls.push(c);
|
||||
}
|
||||
stats = fin.stats || null;
|
||||
}
|
||||
} else if (run.stats) {
|
||||
stats = await run.stats;
|
||||
}
|
||||
} catch (_) {}
|
||||
return { text, thinking, toolCalls, stats, requestId, stopReason: 'stop' };
|
||||
} finally {
|
||||
if (requestId) activeRequests.delete(requestId);
|
||||
}
|
||||
}
|
||||
|
||||
async function cancel() {
|
||||
const id = loaded.requestId;
|
||||
const run = id && activeRequests.get(id);
|
||||
try {
|
||||
if (run && typeof run.abort === 'function') run.abort();
|
||||
else if (sdk && typeof sdk.abortCompletion === 'function' && id) await sdk.abortCompletion({ requestId: id });
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
function getLoaded() {
|
||||
return Object.assign({}, loaded);
|
||||
}
|
||||
|
||||
function hold() {
|
||||
holdCount += 1;
|
||||
}
|
||||
|
||||
function release() {
|
||||
holdCount = Math.max(0, holdCount - 1);
|
||||
}
|
||||
|
||||
async function close() {
|
||||
await unload().catch(() => {});
|
||||
if (sdk && typeof sdk.close === 'function') {
|
||||
try {
|
||||
await sdk.close();
|
||||
} catch (_) {}
|
||||
}
|
||||
sdk = null;
|
||||
initPromise = null;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
ensureInit,
|
||||
load,
|
||||
unload,
|
||||
complete,
|
||||
cancel,
|
||||
getLoaded,
|
||||
resources,
|
||||
prepareVisionHistory,
|
||||
hold,
|
||||
release,
|
||||
close,
|
||||
};
|
||||
Reference in New Issue
Block a user