/** * Direct QVAC LLM wrapper via @qvac/sdk (Node worker). No BridgeSwarm. */ const fs = require('fs'); const os = require('os'); const path = require('path'); const catalog = require('./catalog.js'); const device = require('./device.js'); const events = require('./events.js'); const paths = require('./paths.js'); const completeWatch = require('./complete-watch.js'); let sdk = null; let initError = null; let initPromise = null; let holdCount = 0; let loaded = emptyLoaded(); const activeRequests = new Map(); function emptyLoaded() { return { modelId: null, friendlyId: null, constant: null, tools: false, vision: false, device: 'cpu', backend: null, backendId: null, deviceName: null, vram: null, ctxSize: null, requestId: null, }; } function log(msg) { try { process.stderr.write('[agent-harness] ' + msg + '\n'); } catch (_) {} } function flattenError(err) { if (!err) return 'unknown error'; if (typeof err === 'string') return err; const parts = []; let cur = err; for (let i = 0; i < 6 && cur; i++) { if (cur.message) parts.push(String(cur.message)); else parts.push(String(cur)); cur = cur.cause || cur.error; } return parts.join(' | ') || String(err); } function lookupSrc(mod, name) { if (!name) return null; if (mod[name] != null) return mod[name]; if (mod.models && mod.models[name] != null) return mod.models[name]; if (mod.default && mod.default[name] != null) return mod.default[name]; return null; } async function ensureInit() { if (sdk) return sdk; if (initPromise) return initPromise; initPromise = (async () => { if (!process.env.QVAC_CONFIG_PATH) { const cfg = path.join(__dirname, '..', 'qvac.config.json'); try { if (fs.existsSync(cfg)) process.env.QVAC_CONFIG_PATH = cfg; } catch (_) {} } paths.ensureQvacRoot(); let mod; try { if (typeof Bare !== 'undefined') { // @qvac/sdk's Bare RPC client is intentionally a stub. Bare hosts use // the in-process inference surface and register the engines they own. mod = await import('@qvac/inference'); const { llmPlugin } = await import('@qvac/inference/llamacpp-completion/plugin'); mod.registerPlugin(llmPlugin); // Voice engines fail independently: an unavailable ASR addon must not // prevent speech output or the text assistant from initializing. for (const [modulePath, exportName] of [ ['@qvac/inference/whispercpp-transcription/plugin', 'whisperPlugin'], ['@qvac/inference/tts-ggml/plugin', 'ttsPlugin'], ]) { try { const plugin = await import(modulePath); mod.registerPlugin(plugin[exportName]); } catch (error) { log(`${exportName} unavailable: ${flattenError(error)}`); } } } else { mod = await import('@qvac/sdk'); } } catch (err) { initError = flattenError(err); log('sdk import failed: ' + initError); throw new Error(initError); } if (typeof mod.loadModel !== 'function' && mod.default && typeof mod.default.loadModel === 'function') { mod = mod.default; } if (typeof mod.loadModel !== 'function' || typeof mod.completion !== 'function') { initError = '@qvac/sdk missing loadModel/completion'; throw new Error(initError); } sdk = mod; return sdk; })(); try { return await initPromise; } catch (err) { initPromise = null; throw err; } } function probeResources() { return device.normalizeResources({ totalRamBytes: os.totalmem(), vramBytes: 0, gpus: [], drivers: {}, }); } async function fetchSystemResources(s) { if (!s) return null; if (typeof s.getSystemResources === 'function') { try { return await s.getSystemResources(); } catch (_) {} } if (typeof s.getSystemInfo === 'function') { try { return await s.getSystemInfo(); } catch (_) {} } return null; } async function resources() { const s = await ensureInit().catch(() => null); const info = await fetchSystemResources(s); if (info) return device.normalizeResources(info); return probeResources(); } function toTools(tools) { if (!tools || !Array.isArray(tools)) return undefined; return tools.map((t) => { if (t && t.type === 'function' && t.function) { return { type: 'function', name: t.function.name, description: t.function.description, parameters: t.function.parameters, }; } return t; }); } function decodeDataUrl(dataUrl) { const m = String(dataUrl || '').match(/^data:([^;]+);base64,(.+)$/); if (!m) return null; return { mime: m[1], buf: Buffer.from(m[2], 'base64') }; } function extForMime(mime) { if (/png/i.test(mime)) return '.png'; if (/webp/i.test(mime)) return '.webp'; if (/gif/i.test(mime)) return '.gif'; return '.jpg'; } function prepareVisionHistory(history) { const dir = paths.ensureDir(path.join(paths.ensureQvacRoot(), 'vision')); const list = Array.isArray(history) ? history : []; return list.map((msg) => { if (!msg || !Array.isArray(msg.images) || !msg.images.length) return msg; const attachments = []; for (let i = 0; i < Math.min(4, msg.images.length); i++) { const img = msg.images[i] || {}; let buf = null; let mime = img.mime || 'image/jpeg'; if (img.dataUrl) { const d = decodeDataUrl(img.dataUrl); if (d) { buf = d.buf; mime = d.mime; } } else if (img.dataBase64) { buf = Buffer.from(img.dataBase64, 'base64'); } else if (img.path && fs.existsSync(img.path)) { attachments.push({ path: img.path }); continue; } if (!buf) continue; const file = path.join(dir, 'img_' + Date.now() + '_' + i + extForMime(mime)); fs.writeFileSync(file, buf); attachments.push({ path: file }); } const copy = Object.assign({}, msg); delete copy.images; if (attachments.length) copy.attachments = (copy.attachments || []).concat(attachments); return copy; }); } async function resolveSrc(s, name) { const constant = catalog.resolveModelConstant(name); const fromSdk = lookupSrc(s, constant) || lookupSrc(s, name); if (fromSdk) return fromSdk; if (typeof s.lookupModelSrc === 'function') { try { const src = await s.lookupModelSrc(constant); if (src) return src; } catch (_) {} } throw new Error( 'QVAC has no constant for ' + String(name) + ' (' + constant + '). Only catalog GGUFs in @qvac/sdk load.' ); } async function resolveMmproj(s, entry) { const names = catalog.mmprojCandidates(entry); for (const n of names) { const src = lookupSrc(s, n); if (src) return src; if (typeof s.lookupModelSrc === 'function') { try { const found = await s.lookupModelSrc(n); if (found) return found; } catch (_) {} } } return null; } async function load(opts, onProgress) { const s = await ensureInit(); opts = opts || {}; onProgress = onProgress || opts.onProgress; if (loaded.modelId) { await unload().catch(() => {}); } const entry = catalog.findCatalogEntry(opts.model || opts.modelSrc || opts.friendlyId); const modelSrc = await resolveSrc(s, opts.modelSrc || (entry && entry.constant) || opts.model); const res = await resources(); let dev = device.pickDevice(opts.device || 'auto', res); const toolsOn = opts.tools !== false && (!entry || entry.tools !== false); const rawCtx = opts.ctxSize || opts.ctx_size || (entry && entry.ctxSize) || 8192; const ctxSize = device.capCtxSize(rawCtx, res, dev.device === 'gpu'); const mmprojGpu = device.mmprojOnGpu(opts, res, dev.device === 'gpu'); const wantVision = opts.vision !== false && (entry ? entry.vision === true : catalog.isVisionModel(opts.model || opts.modelSrc)); const mmprojSrc = wantVision ? await resolveMmproj(s, entry) : null; const modelConfig = Object.assign( { device: dev.device, gpu_layers: device.gpuLayers(opts, dev), ctx_size: Number(ctxSize) || 8192, tools: !!toolsOn, 'mmproj-use-gpu': !!mmprojGpu, }, mmprojSrc ? { projectionModelSrc: mmprojSrc } : {} ); const loadOpts = { modelSrc, modelType: 'llm', modelConfig, }; if (typeof onProgress === 'function') { loadOpts.onProgress = (p) => { try { onProgress({ percent: p.percentage != null ? p.percentage : p.percent, downloaded: p.downloaded, total: p.total, }); } catch (_) {} }; } log( 'load ' + ((entry && entry.id) || opts.model || 'model') + ' device=' + modelConfig.device + ' ngl=' + modelConfig.gpu_layers + ' ctx=' + modelConfig.ctx_size + ' vision=' + !!mmprojSrc ); let modelId; try { modelId = await s.loadModel(loadOpts); } catch (err) { const msg = flattenError(err); log('load failed: ' + msg); if (dev.device === 'gpu' && String(opts.device || 'auto').toLowerCase() !== 'cpu') { log('retrying load on cpu'); modelConfig.device = 'cpu'; modelConfig.gpu_layers = 0; modelConfig['mmproj-use-gpu'] = false; try { modelId = await s.loadModel(loadOpts); dev = { device: 'cpu', gpu_layers: 0, fallback: 'cpu' }; } catch (err2) { throw new Error(flattenError(err2)); } } else { throw new Error(msg); } } const bl = device.backendLabel(res); loaded = { modelId, friendlyId: (entry && entry.id) || opts.model || null, constant: (entry && entry.constant) || catalog.resolveModelConstant(opts.model || opts.modelSrc), tools: !!toolsOn, vision: !!mmprojSrc, device: dev.device, backend: bl.backend, backendId: bl.backendId, deviceName: bl.deviceName, vram: res && (res.vram || res.vramBytes), ctxSize: modelConfig.ctx_size, requestId: null, }; return Object.assign({}, loaded); } async function unload() { if (!loaded.modelId || !sdk) { loaded = emptyLoaded(); return; } const id = loaded.modelId; loaded = emptyLoaded(); try { if (typeof sdk.unloadModel === 'function') await sdk.unloadModel({ modelId: id }); } catch (err) { log('unload: ' + flattenError(err)); } } async function complete(opts, onEvent) { const s = await ensureInit(); if (!loaded.modelId && !(opts && opts.modelId)) throw new Error('no model loaded'); const rawHistory = (opts && opts.history) || (opts && opts.messages) || []; const history = prepareVisionHistory(rawHistory); const tools = toTools(opts && opts.tools); const dialect = (opts && opts.toolDialect) || catalog.toolDialectFor(loaded.friendlyId || loaded.constant); const params = { modelId: (opts && opts.modelId) || loaded.modelId, history, stream: opts && opts.stream === false ? false : true, captureThinking: true, }; if (tools && tools.length) { params.tools = tools; params.toolDialect = dialect; } if (opts && opts.generationParams) params.generationParams = opts.generationParams; const run = s.completion(params); const requestId = run.requestId || (opts && opts.requestId) || null; loaded.requestId = requestId; if (requestId) activeRequests.set(requestId, run); let text = ''; let thinking = ''; const toolCalls = []; const abortRun = () => { try { if (run && typeof run.abort === 'function') run.abort(); else if (sdk && typeof sdk.abortCompletion === 'function' && requestId) sdk.abortCompletion({ requestId }); } catch (_) {} }; let settleTimeout; const timedOutGate = new Promise((resolve) => { settleTimeout = () => resolve('timeout'); }); const watch = completeWatch.attachCompleteWatch({ timeoutMs: opts && opts.timeoutMs, idleMs: opts && opts.idleMs, abort: abortRun, onTimeout: settleTimeout, }); const consume = (async () => { if (run.events && typeof run.events[Symbol.asyncIterator] === 'function') { for await (const ev of run.events) { if (watch.timedOut()) return; const n = events.normalizeCompletionEvent(ev); if (!n) continue; watch.bump(); if (n.type === 'contentDelta') { text += n.delta; if (onEvent) onEvent(n); } else if (n.type === 'thinkingDelta') { thinking += n.delta; if (onEvent) onEvent(n); } else if (n.type === 'toolCall') { toolCalls.push(n.call); if (onEvent) onEvent(n); } else if (onEvent) { onEvent(n); } } } else if (run.tokenStream) { for await (const token of run.tokenStream) { if (watch.timedOut()) return; watch.bump(); text += token; if (onEvent) onEvent({ type: 'contentDelta', delta: token }); } if (run.toolCallStream) { for await (const evt of run.toolCallStream) { if (watch.timedOut()) return; watch.bump(); const call = evt.call || evt; toolCalls.push(call); if (onEvent) onEvent({ type: 'toolCall', call }); } } } if (watch.timedOut()) return; try { if (run.final) { const fin = await run.final; if (fin) { if (fin.contentText && !text) text = fin.contentText; if (fin.thinkingText && !thinking) thinking = fin.thinkingText; if (Array.isArray(fin.toolCalls) && fin.toolCalls.length) { toolCalls.length = 0; for (const c of fin.toolCalls) toolCalls.push(c); } } } } catch (_) {} })(); consume.catch(() => {}); try { await Promise.race([consume, timedOutGate]); let stats = null; if (!watch.timedOut()) { try { if (run.stats) stats = await run.stats; } catch (_) {} } return { text, thinking, toolCalls, stats, requestId, stopReason: watch.timedOut() ? 'timeout' : 'stop', }; } finally { watch.clear(); if (requestId) activeRequests.delete(requestId); } } async function cancel() { const id = loaded.requestId; const run = id && activeRequests.get(id); try { if (run && typeof run.abort === 'function') run.abort(); else if (sdk && typeof sdk.abortCompletion === 'function' && id) await sdk.abortCompletion({ requestId: id }); } catch (_) {} } function getLoaded() { return Object.assign({}, loaded); } function hold() { holdCount += 1; } function release() { holdCount = Math.max(0, holdCount - 1); } async function close() { await unload().catch(() => {}); if (sdk && typeof sdk.close === 'function') { try { await sdk.close(); } catch (_) {} } sdk = null; initPromise = null; } module.exports = { ensureInit, load, unload, complete, cancel, getLoaded, resources, prepareVisionHistory, hold, release, close, };