616 lines
18 KiB
JavaScript
616 lines
18 KiB
JavaScript
/**
|
|
* Direct QVAC LLM wrapper via @qvac/sdk (Node worker). No BridgeSwarm.
|
|
*/
|
|
|
|
const fs = require('fs');
|
|
const os = require('os');
|
|
const path = require('path');
|
|
const catalog = require('./catalog.js');
|
|
const device = require('./device.js');
|
|
const events = require('./events.js');
|
|
const paths = require('./paths.js');
|
|
const completeWatch = require('./complete-watch.js');
|
|
const toolParse = require('./tool-parse.js');
|
|
const repeatLoop = require('./repeat-loop.js');
|
|
|
|
let sdk = null;
|
|
let initError = null;
|
|
let initPromise = null;
|
|
let holdCount = 0;
|
|
let loaded = emptyLoaded();
|
|
const activeRequests = new Map();
|
|
|
|
function emptyLoaded() {
|
|
return {
|
|
modelId: null,
|
|
friendlyId: null,
|
|
constant: null,
|
|
tools: false,
|
|
vision: false,
|
|
device: 'cpu',
|
|
backend: null,
|
|
backendId: null,
|
|
deviceName: null,
|
|
vram: null,
|
|
ctxSize: null,
|
|
requestId: null,
|
|
};
|
|
}
|
|
|
|
function log(msg) {
|
|
try {
|
|
process.stderr.write('[agent-harness] ' + msg + '\n');
|
|
} catch (_) {}
|
|
}
|
|
|
|
function flattenError(err) {
|
|
if (!err) return 'unknown error';
|
|
if (typeof err === 'string') return err;
|
|
const parts = [];
|
|
let cur = err;
|
|
for (let i = 0; i < 6 && cur; i++) {
|
|
if (cur.message) parts.push(String(cur.message));
|
|
else parts.push(String(cur));
|
|
cur = cur.cause || cur.error;
|
|
}
|
|
return parts.join(' | ') || String(err);
|
|
}
|
|
|
|
function lookupSrc(mod, name) {
|
|
if (!name) return null;
|
|
if (mod[name] != null) return mod[name];
|
|
if (mod.models && mod.models[name] != null) return mod.models[name];
|
|
if (mod.default && mod.default[name] != null) return mod.default[name];
|
|
return null;
|
|
}
|
|
|
|
async function ensureInit() {
|
|
if (sdk) return sdk;
|
|
if (initPromise) return initPromise;
|
|
initPromise = (async () => {
|
|
if (!process.env.QVAC_CONFIG_PATH) {
|
|
const cfg = path.join(__dirname, '..', 'qvac.config.json');
|
|
try {
|
|
if (fs.existsSync(cfg)) process.env.QVAC_CONFIG_PATH = cfg;
|
|
} catch (_) {}
|
|
}
|
|
paths.ensureQvacRoot();
|
|
let mod;
|
|
try {
|
|
if (typeof Bare !== 'undefined') {
|
|
// @qvac/sdk's Bare RPC client is intentionally a stub. Bare hosts use
|
|
// the in-process inference surface and register the engines they own.
|
|
mod = await import('@qvac/inference');
|
|
const { llmPlugin } = await import('@qvac/inference/llamacpp-completion/plugin');
|
|
mod.registerPlugin(llmPlugin);
|
|
// Voice engines fail independently: an unavailable ASR addon must not
|
|
// prevent speech output or the text assistant from initializing.
|
|
for (const [modulePath, exportName] of [
|
|
['@qvac/inference/whispercpp-transcription/plugin', 'whisperPlugin'],
|
|
['@qvac/inference/tts-ggml/plugin', 'ttsPlugin'],
|
|
]) {
|
|
try { const plugin = await import(modulePath); mod.registerPlugin(plugin[exportName]); }
|
|
catch (error) { log(`${exportName} unavailable: ${flattenError(error)}`); }
|
|
}
|
|
} else {
|
|
mod = await import('@qvac/sdk');
|
|
}
|
|
} catch (err) {
|
|
initError = flattenError(err);
|
|
log('sdk import failed: ' + initError);
|
|
throw new Error(initError);
|
|
}
|
|
if (typeof mod.loadModel !== 'function' && mod.default && typeof mod.default.loadModel === 'function') {
|
|
mod = mod.default;
|
|
}
|
|
if (typeof mod.loadModel !== 'function' || typeof mod.completion !== 'function') {
|
|
initError = '@qvac/sdk missing loadModel/completion';
|
|
throw new Error(initError);
|
|
}
|
|
sdk = mod;
|
|
return sdk;
|
|
})();
|
|
try {
|
|
return await initPromise;
|
|
} catch (err) {
|
|
initPromise = null;
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
function probeResources() {
|
|
return device.normalizeResources({
|
|
totalRamBytes: os.totalmem(),
|
|
vramBytes: 0,
|
|
gpus: [],
|
|
drivers: {},
|
|
});
|
|
}
|
|
|
|
async function fetchSystemResources(s) {
|
|
if (!s) return null;
|
|
if (typeof s.getSystemResources === 'function') {
|
|
try {
|
|
return await s.getSystemResources();
|
|
} catch (_) {}
|
|
}
|
|
if (typeof s.getSystemInfo === 'function') {
|
|
try {
|
|
return await s.getSystemInfo();
|
|
} catch (_) {}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function resources() {
|
|
const s = await ensureInit().catch(() => null);
|
|
const info = await fetchSystemResources(s);
|
|
if (info) return device.normalizeResources(info);
|
|
return probeResources();
|
|
}
|
|
|
|
function toTools(tools) {
|
|
if (!tools || !Array.isArray(tools)) return undefined;
|
|
return tools.map((t) => {
|
|
if (t && t.type === 'function' && t.function) {
|
|
return {
|
|
type: 'function',
|
|
name: t.function.name,
|
|
description: t.function.description || t.function.name,
|
|
parameters: t.function.parameters && t.function.parameters.type === 'object'
|
|
? t.function.parameters
|
|
: { type: 'object', properties: {} },
|
|
};
|
|
}
|
|
if (!t || !t.name) return t;
|
|
const parameters = t.parameters && t.parameters.type === 'object'
|
|
? t.parameters
|
|
: { type: 'object', properties: (t.parameters && t.parameters.properties) || {} };
|
|
return {
|
|
type: 'function',
|
|
name: t.name,
|
|
description: t.description || t.name,
|
|
parameters,
|
|
};
|
|
});
|
|
}
|
|
|
|
function decodeDataUrl(dataUrl) {
|
|
const m = String(dataUrl || '').match(/^data:([^;]+);base64,(.+)$/);
|
|
if (!m) return null;
|
|
return { mime: m[1], buf: Buffer.from(m[2], 'base64') };
|
|
}
|
|
|
|
function extForMime(mime) {
|
|
if (/png/i.test(mime)) return '.png';
|
|
if (/webp/i.test(mime)) return '.webp';
|
|
if (/gif/i.test(mime)) return '.gif';
|
|
return '.jpg';
|
|
}
|
|
|
|
// Qwen VL / llama.cpp formatPrompt loads images then requires a user question.
|
|
const VISION_FOLLOWUP_QUESTION =
|
|
'Describe what you see in the attached still. Answer the user. Do not mention file paths.';
|
|
|
|
function attachmentsFromImages(msg, dir) {
|
|
const attachments = [];
|
|
for (let i = 0; i < Math.min(4, msg.images.length); i++) {
|
|
const img = msg.images[i] || {};
|
|
let buf = null;
|
|
let mime = img.mime || 'image/jpeg';
|
|
if (img.dataUrl) {
|
|
const d = decodeDataUrl(img.dataUrl);
|
|
if (d) {
|
|
buf = d.buf;
|
|
mime = d.mime;
|
|
}
|
|
} else if (img.dataBase64) {
|
|
buf = Buffer.from(img.dataBase64, 'base64');
|
|
} else if (img.path && fs.existsSync(img.path)) {
|
|
attachments.push({ path: img.path });
|
|
continue;
|
|
}
|
|
if (!buf) continue;
|
|
const file = path.join(dir, 'img_' + Date.now() + '_' + i + extForMime(mime));
|
|
fs.writeFileSync(file, buf);
|
|
attachments.push({ path: file });
|
|
}
|
|
return attachments;
|
|
}
|
|
|
|
function withVisionAttachments(msg, dir) {
|
|
if (!msg || !Array.isArray(msg.images) || !msg.images.length) return msg;
|
|
const attachments = attachmentsFromImages(msg, dir);
|
|
const copy = Object.assign({}, msg);
|
|
delete copy.images;
|
|
if (attachments.length) copy.attachments = (copy.attachments || []).concat(attachments);
|
|
return copy;
|
|
}
|
|
|
|
function ensureVisionQuestion(msg) {
|
|
if (!msg || !Array.isArray(msg.attachments) || !msg.attachments.length) return msg;
|
|
if (String(msg.content || '').trim()) return msg;
|
|
return Object.assign({}, msg, { content: VISION_FOLLOWUP_QUESTION });
|
|
}
|
|
|
|
function hoistToolVision(messages) {
|
|
const out = [];
|
|
for (const msg of messages) {
|
|
const role = msg && msg.role;
|
|
if (msg && (role === 'tool' || role === 'function') && Array.isArray(msg.attachments) && msg.attachments.length) {
|
|
const copy = Object.assign({}, msg);
|
|
const attachments = copy.attachments;
|
|
delete copy.attachments;
|
|
out.push(copy);
|
|
out.push({ role: 'user', content: VISION_FOLLOWUP_QUESTION, attachments });
|
|
continue;
|
|
}
|
|
out.push(ensureVisionQuestion(msg));
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function prepareVisionHistory(history) {
|
|
const dir = paths.ensureDir(path.join(paths.ensureQvacRoot(), 'vision'));
|
|
const list = Array.isArray(history) ? history : [];
|
|
return hoistToolVision(list.map((msg) => withVisionAttachments(msg, dir)));
|
|
}
|
|
|
|
async function resolveSrc(s, name) {
|
|
const constant = catalog.resolveModelConstant(name);
|
|
const fromSdk = lookupSrc(s, constant) || lookupSrc(s, name);
|
|
if (fromSdk) return fromSdk;
|
|
if (typeof s.lookupModelSrc === 'function') {
|
|
try {
|
|
const src = await s.lookupModelSrc(constant);
|
|
if (src) return src;
|
|
} catch (_) {}
|
|
}
|
|
throw new Error(
|
|
'QVAC has no constant for ' +
|
|
String(name) +
|
|
' (' +
|
|
constant +
|
|
'). Only catalog GGUFs in @qvac/sdk load.'
|
|
);
|
|
}
|
|
|
|
async function resolveMmproj(s, entry) {
|
|
const names = catalog.mmprojCandidates(entry);
|
|
for (const n of names) {
|
|
const src = lookupSrc(s, n);
|
|
if (src) return src;
|
|
if (typeof s.lookupModelSrc === 'function') {
|
|
try {
|
|
const found = await s.lookupModelSrc(n);
|
|
if (found) return found;
|
|
} catch (_) {}
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function load(opts, onProgress) {
|
|
const s = await ensureInit();
|
|
opts = opts || {};
|
|
onProgress = onProgress || opts.onProgress;
|
|
if (loaded.modelId) {
|
|
await unload().catch(() => {});
|
|
}
|
|
const entry = catalog.findCatalogEntry(opts.model || opts.modelSrc || opts.friendlyId);
|
|
const modelSrc = await resolveSrc(s, opts.modelSrc || (entry && entry.constant) || opts.model);
|
|
const res = await resources();
|
|
let dev = device.pickDevice(opts.device || 'auto', res);
|
|
const toolsOn = opts.tools !== false && (!entry || entry.tools !== false);
|
|
const rawCtx = opts.ctxSize || opts.ctx_size || (entry && entry.ctxSize) || 8192;
|
|
const ctxSize = device.capCtxSize(rawCtx, res, dev.device === 'gpu');
|
|
const mmprojGpu = device.mmprojOnGpu(opts, res, dev.device === 'gpu');
|
|
const wantVision =
|
|
opts.vision !== false && (entry ? entry.vision === true : catalog.isVisionModel(opts.model || opts.modelSrc));
|
|
const mmprojSrc = wantVision ? await resolveMmproj(s, entry) : null;
|
|
const loadParams = catalog.generationParamsForModel((entry && entry.id) || opts.model);
|
|
const modelConfig = Object.assign(
|
|
{
|
|
device: dev.device,
|
|
gpu_layers: device.gpuLayers(opts, dev),
|
|
ctx_size: Number(ctxSize) || 8192,
|
|
tools: !!toolsOn,
|
|
'mmproj-use-gpu': !!mmprojGpu,
|
|
},
|
|
loadParams.reasoning_budget != null ? { reasoning_budget: loadParams.reasoning_budget } : {},
|
|
mmprojSrc ? { projectionModelSrc: mmprojSrc } : {}
|
|
);
|
|
const loadOpts = {
|
|
modelSrc,
|
|
modelType: 'llm',
|
|
modelConfig,
|
|
};
|
|
if (typeof onProgress === 'function') {
|
|
loadOpts.onProgress = (p) => {
|
|
try {
|
|
onProgress({
|
|
percent: p.percentage != null ? p.percentage : p.percent,
|
|
downloaded: p.downloaded,
|
|
total: p.total,
|
|
});
|
|
} catch (_) {}
|
|
};
|
|
}
|
|
log(
|
|
'load ' +
|
|
((entry && entry.id) || opts.model || 'model') +
|
|
' device=' +
|
|
modelConfig.device +
|
|
' ngl=' +
|
|
modelConfig.gpu_layers +
|
|
' ctx=' +
|
|
modelConfig.ctx_size +
|
|
' vision=' +
|
|
!!mmprojSrc
|
|
);
|
|
let modelId;
|
|
try {
|
|
modelId = await s.loadModel(loadOpts);
|
|
} catch (err) {
|
|
const msg = flattenError(err);
|
|
log('load failed: ' + msg);
|
|
if (dev.device === 'gpu' && String(opts.device || 'auto').toLowerCase() === 'auto' && process.env.JARVIS_GPU_REQUIRED !== '1') {
|
|
log('retrying load on cpu');
|
|
modelConfig.device = 'cpu';
|
|
modelConfig.gpu_layers = 0;
|
|
modelConfig['mmproj-use-gpu'] = false;
|
|
try {
|
|
modelId = await s.loadModel(loadOpts);
|
|
dev = { device: 'cpu', gpu_layers: 0, fallback: 'cpu' };
|
|
} catch (err2) {
|
|
throw new Error(flattenError(err2));
|
|
}
|
|
} else {
|
|
throw new Error(msg);
|
|
}
|
|
}
|
|
const bl = device.backendLabel(res);
|
|
loaded = {
|
|
modelId,
|
|
friendlyId: (entry && entry.id) || opts.model || null,
|
|
constant: (entry && entry.constant) || catalog.resolveModelConstant(opts.model || opts.modelSrc),
|
|
tools: !!toolsOn,
|
|
vision: !!mmprojSrc,
|
|
device: dev.device,
|
|
backend: dev.device === 'gpu' ? bl.backend : 'cpu',
|
|
backendId: dev.device === 'gpu' ? bl.backendId : 0,
|
|
deviceName: bl.deviceName,
|
|
vram: res && (res.vram || res.vramBytes),
|
|
ctxSize: modelConfig.ctx_size,
|
|
requestId: null,
|
|
};
|
|
return Object.assign({}, loaded);
|
|
}
|
|
|
|
async function unload() {
|
|
if (!loaded.modelId || !sdk) {
|
|
loaded = emptyLoaded();
|
|
return;
|
|
}
|
|
const id = loaded.modelId;
|
|
loaded = emptyLoaded();
|
|
try {
|
|
if (typeof sdk.unloadModel === 'function') await sdk.unloadModel({ modelId: id });
|
|
} catch (err) {
|
|
log('unload: ' + flattenError(err));
|
|
}
|
|
}
|
|
|
|
async function complete(opts, onEvent) {
|
|
const s = await ensureInit();
|
|
if (!loaded.modelId && !(opts && opts.modelId)) throw new Error('no model loaded');
|
|
const rawHistory = (opts && opts.history) || (opts && opts.messages) || [];
|
|
const history = prepareVisionHistory(rawHistory);
|
|
const tools = toTools(opts && opts.tools);
|
|
const dialect = (opts && opts.toolDialect) || catalog.toolDialectFor(loaded.friendlyId || loaded.constant);
|
|
const generationParams = catalog.generationParamsForModel(
|
|
loaded.friendlyId || loaded.constant,
|
|
opts && opts.generationParams
|
|
);
|
|
const params = {
|
|
modelId: (opts && opts.modelId) || loaded.modelId,
|
|
history: toolParse.prepareToolHistory(history, dialect),
|
|
stream: opts && opts.stream === false ? false : true,
|
|
captureThinking: true,
|
|
};
|
|
if (tools && tools.length) {
|
|
params.tools = tools;
|
|
params.toolDialect = dialect;
|
|
}
|
|
if (Object.keys(generationParams).length) params.generationParams = generationParams;
|
|
const run = s.completion(params);
|
|
const requestId = run.requestId || (opts && opts.requestId) || null;
|
|
loaded.requestId = requestId;
|
|
if (requestId) activeRequests.set(requestId, run);
|
|
|
|
let text = '';
|
|
let thinking = '';
|
|
let thinkState = { inThink: false, carry: '' };
|
|
const toolCalls = [];
|
|
const abortRun = () => {
|
|
try {
|
|
if (run && typeof run.abort === 'function') run.abort();
|
|
else if (sdk && typeof sdk.abortCompletion === 'function' && requestId) sdk.abortCompletion({ requestId });
|
|
} catch (_) {}
|
|
};
|
|
let settleTimeout;
|
|
const timedOutGate = new Promise((resolve) => {
|
|
settleTimeout = () => resolve('timeout');
|
|
});
|
|
const watch = completeWatch.attachCompleteWatch({
|
|
timeoutMs: opts && opts.timeoutMs,
|
|
idleMs: opts && opts.idleMs,
|
|
abort: abortRun,
|
|
onTimeout: settleTimeout,
|
|
});
|
|
const consume = (async () => {
|
|
if (run.events && typeof run.events[Symbol.asyncIterator] === 'function') {
|
|
for await (const ev of run.events) {
|
|
if (watch.timedOut()) return;
|
|
const n = events.normalizeCompletionEvent(ev);
|
|
if (!n) continue;
|
|
watch.bump();
|
|
const split = events.expandThinkEvents(n, thinkState);
|
|
thinkState = split.state;
|
|
for (const p of split.events) {
|
|
if (p.type === 'contentDelta') {
|
|
text += p.delta;
|
|
if (onEvent) onEvent(p);
|
|
if (repeatLoop.isRepeating(text)) {
|
|
abortRun();
|
|
settleTimeout();
|
|
return;
|
|
}
|
|
} else if (p.type === 'thinkingDelta') {
|
|
thinking += p.delta;
|
|
if (onEvent) onEvent(p);
|
|
if (repeatLoop.isRepeating(thinking)) {
|
|
abortRun();
|
|
settleTimeout();
|
|
return;
|
|
}
|
|
} else if (p.type === 'toolCall') {
|
|
toolCalls.push(p.call);
|
|
if (onEvent) onEvent(p);
|
|
} else if (onEvent) {
|
|
onEvent(p);
|
|
}
|
|
}
|
|
}
|
|
} else if (run.tokenStream) {
|
|
for await (const token of run.tokenStream) {
|
|
if (watch.timedOut()) return;
|
|
watch.bump();
|
|
const split = events.expandThinkEvents({ type: 'contentDelta', delta: token }, thinkState);
|
|
thinkState = split.state;
|
|
for (const p of split.events) {
|
|
if (p.type === 'thinkingDelta') {
|
|
thinking += p.delta;
|
|
if (onEvent) onEvent(p);
|
|
if (repeatLoop.isRepeating(thinking)) {
|
|
abortRun();
|
|
settleTimeout();
|
|
return;
|
|
}
|
|
} else {
|
|
text += p.delta;
|
|
if (onEvent) onEvent({ type: 'contentDelta', delta: p.delta });
|
|
if (repeatLoop.isRepeating(text)) {
|
|
abortRun();
|
|
settleTimeout();
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (run.toolCallStream) {
|
|
for await (const evt of run.toolCallStream) {
|
|
if (watch.timedOut()) return;
|
|
watch.bump();
|
|
const call = evt.call || evt;
|
|
toolCalls.push(call);
|
|
if (onEvent) onEvent({ type: 'toolCall', call });
|
|
}
|
|
}
|
|
}
|
|
if (watch.timedOut()) return;
|
|
try {
|
|
if (run.final) {
|
|
const fin = await run.final;
|
|
if (fin) {
|
|
if (fin.contentText && !text) text = fin.contentText;
|
|
if (fin.thinkingText && !thinking) thinking = fin.thinkingText;
|
|
if (Array.isArray(fin.toolCalls) && fin.toolCalls.length) {
|
|
toolCalls.length = 0;
|
|
for (const c of fin.toolCalls) toolCalls.push(c);
|
|
}
|
|
}
|
|
}
|
|
} catch (_) {}
|
|
})();
|
|
consume.catch(() => {});
|
|
try {
|
|
await Promise.race([consume, timedOutGate]);
|
|
let stats = null;
|
|
if (!watch.timedOut()) {
|
|
try {
|
|
if (run.stats) stats = await run.stats;
|
|
} catch (_) {}
|
|
}
|
|
if (tools && tools.length) {
|
|
const recovered = toolParse.recover({ text, thinking, tools, existing: toolCalls });
|
|
if (!toolCalls.length && recovered.calls.length) {
|
|
for (const call of recovered.calls) toolCalls.push(call);
|
|
}
|
|
text = recovered.text;
|
|
} else {
|
|
text = toolParse.stripToolMarkup(text);
|
|
}
|
|
text = repeatLoop.collapseRepeats(text);
|
|
thinking = repeatLoop.collapseRepeats(thinking);
|
|
return {
|
|
text,
|
|
thinking,
|
|
toolCalls,
|
|
stats,
|
|
requestId,
|
|
stopReason: watch.timedOut() ? 'timeout' : 'stop',
|
|
};
|
|
} finally {
|
|
watch.clear();
|
|
if (requestId) activeRequests.delete(requestId);
|
|
}
|
|
}
|
|
|
|
async function cancel() {
|
|
const id = loaded.requestId;
|
|
const run = id && activeRequests.get(id);
|
|
try {
|
|
if (run && typeof run.abort === 'function') run.abort();
|
|
else if (sdk && typeof sdk.abortCompletion === 'function' && id) await sdk.abortCompletion({ requestId: id });
|
|
} catch (_) {}
|
|
}
|
|
|
|
function getLoaded() {
|
|
return Object.assign({}, loaded);
|
|
}
|
|
|
|
function hold() {
|
|
holdCount += 1;
|
|
}
|
|
|
|
function release() {
|
|
holdCount = Math.max(0, holdCount - 1);
|
|
}
|
|
|
|
async function close() {
|
|
await unload().catch(() => {});
|
|
if (sdk && typeof sdk.close === 'function') {
|
|
try {
|
|
await sdk.close();
|
|
} catch (_) {}
|
|
}
|
|
sdk = null;
|
|
initPromise = null;
|
|
}
|
|
|
|
module.exports = {
|
|
ensureInit,
|
|
load,
|
|
unload,
|
|
complete,
|
|
cancel,
|
|
getLoaded,
|
|
resources,
|
|
VISION_FOLLOWUP_QUESTION,
|
|
prepareVisionHistory,
|
|
hold,
|
|
release,
|
|
close,
|
|
};
|