/** * Device selection: Metal (macOS), Vulkan (Linux/Windows NVIDIA+AMD), CPU fallback. * CUDA/ROCm LLM backends are not shipped by QVAC. * GPU is always attempted unless the caller explicitly requests CPU. */ function metricValue(metric, fallback) { if (metric == null) return fallback; if (typeof metric !== 'object') return metric; if (metric.status === 'supported') return metric.value; return fallback; } function flattenDrivers(drivers) { const out = {}; if (!drivers || typeof drivers !== 'object') return out; for (const key of Object.keys(drivers)) { const v = drivers[key]; if (v === true) out[key] = true; else if (v && v.status === 'supported' && v.value) out[key] = true; } return out; } function normalizeResources(raw) { if (!raw || typeof raw !== 'object') { return { totalRamBytes: 0, vramBytes: 0, gpus: [], drivers: {} }; } if (Array.isArray(raw.gpus) || raw.vramBytes != null || raw.vram != null) { const gpus = raw.gpus || []; const drivers = Object.keys(raw.drivers || {}).length ? flattenDrivers(raw.drivers) : {}; if (!Object.keys(drivers).length) { for (const g of gpus) Object.assign(drivers, flattenDrivers(g && g.drivers)); } return { totalRamBytes: Number(raw.totalRamBytes) || 0, vramBytes: vramBytes(raw), gpus, drivers, capabilities: raw.capabilities, gpu: raw.gpu, }; } const caps = raw.capabilities || {}; const gpuList = metricValue(caps.gpus, []) || []; const gpus = gpuList.map((g) => ({ id: g.id, name: metricValue(g.name, null), type: metricValue(g.type, null), memory: metricValue(g.memoryTotalBytes, 0), drivers: flattenDrivers(g.drivers), })); const drivers = {}; for (const g of gpus) Object.assign(drivers, g.drivers || {}); let maxVram = 0; for (const g of gpus) { const m = Number(g.memory) || 0; if (m > maxVram) maxVram = m; } return { totalRamBytes: Number(metricValue(caps.memory && caps.memory.totalBytes, 0)) || 0, vramBytes: maxVram, gpus, drivers, capabilities: { gpu: gpus.length > 0, vulkan: !!drivers.vulkan, metal: !!drivers.metal }, }; } function pickDevice(requested, resources) { const want = String(requested || 'auto').toLowerCase(); if (want === 'cpu') return { device: 'cpu', gpu_layers: 0 }; return { device: 'gpu', gpu_layers: 99 }; } function hasGpu(resources) { const res = resources && resources.capabilities && !Array.isArray(resources.gpus) ? normalizeResources(resources) : resources; if (!res) return false; const drivers = res.drivers || {}; if (drivers.metal || drivers.vulkan || drivers.cuda || drivers.opencl) return true; const gpus = res.gpus || []; if (gpus.length > 0) return true; const cap = res.capabilities || {}; if (cap.gpu || cap.vulkan || cap.metal) return true; if (res.gpu) return true; return false; } function backendLabel(resources) { const res = resources && resources.capabilities && !Array.isArray(resources.gpus) ? normalizeResources(resources) : resources; const gpus = (res && res.gpus) || []; const dedicated = gpus.find((g) => g && (g.type === 2 || String(g.type).toLowerCase() === 'dedicated')); const gpu = dedicated || gpus[0] || null; const gpuName = (gpu && gpu.name) || (res && res.gpu && (res.gpu.name || res.gpu.deviceName)) || null; const vram = (gpu && gpu.memory) || (res && (res.vram || res.vramBytes)) || null; const drivers = (res && res.drivers) || (gpu && gpu.drivers) || {}; if (process.platform === 'darwin' || drivers.metal) { return { backend: 'metal', backendId: 1, deviceName: gpuName, vram }; } if (drivers.vulkan || hasGpu(res)) { return { backend: 'vulkan', backendId: 3, deviceName: gpuName, vram }; } return { backend: 'cpu', backendId: 0, deviceName: gpuName, vram }; } function summarizeGpuProbe(info) { const res = normalizeResources(info); return { gpus: res.gpus || [], drivers: res.drivers || {}, length: (res.gpus && res.gpus.length) || 0, }; } /** * Normalize GPU memory reports to bytes. bare-gpu-info may yield bytes, MiB, or GiB. */ function vramBytes(resources) { if (!resources) return 0; const raw = resources.vramBytes != null ? resources.vramBytes : resources.vram != null ? resources.vram : resources.gpus && resources.gpus[0] && resources.gpus[0].memory; const n = Number(raw); if (!Number.isFinite(n) || n <= 0) return 0; if (n < 1024) return Math.round(n * 1024 * 1024 * 1024); if (n < 1024 * 1024) return Math.round(n * 1024 * 1024); return Math.round(n); } /** * Keep KV-cache context from dominating VRAM. Weights still use gpu_layers. */ function capCtxSize(want, resources, onGpu) { const requested = Math.max(512, Number(want) || 8192); if (!onGpu) return Math.min(requested, 8192); const gb = vramBytes(resources) / 1e9; let max = requested; if (gb > 0 && gb < 6) max = 4096; else if (gb > 0 && gb < 10) max = 8192; else if (gb > 0 && gb < 16) max = 16384; return Math.min(requested, max); } function mmprojOnGpu(opts, resources, onGpu) { if (!onGpu) return false; if (opts && opts.mmprojUseGpu === false) return false; return true; } function gpuLayers(opts, dev) { const raw = opts && opts.gpu_layers != null ? opts.gpu_layers : dev && dev.gpu_layers; const n = Number(raw); if (Number.isFinite(n) && n >= 0) return n; return dev && dev.device === 'gpu' ? 99 : 0; } module.exports = { pickDevice, hasGpu, backendLabel, summarizeGpuProbe, vramBytes, capCtxSize, mmprojOnGpu, normalizeResources, gpuLayers, };