/** * Probe host GPUs for QVAC load ordering (highest device-local memory first). * Uses vulkaninfo when available; nvidia-smi / sysfs as weak fallbacks. * * Also prefers hardware Vulkan ICDs (NVIDIA/Intel/AMD) and disables Mesa * lavapipe/llvmpipe so llama.cpp does not "succeed" on a CPU Vulkan device. * * Static bare-subprocess import: dynamic import() from this module was not * linked in the bare-pack resolution map (MODULE_NOT_FOUND → sdk worker crash). */ import bareSubprocess from 'bare-subprocess' import hostFs from '#host-fs' import path from '#host-path' import os from 'bare-os' /** Standard Vulkan ICD manifest directories (Linux). */ export const BARE_OS_QVAC_VULKAN_ICD_DIRS = [ '/usr/share/vulkan/icd.d', '/etc/vulkan/icd.d', '/usr/local/share/vulkan/icd.d' ] /** * @typedef {{ * index: number, * name: string, * memoryBytes: number, * deviceType: string, * source: string * }} BareOsQvacGpuInfo */ /** * Run a short host command; return stdout (utf8) or null. * @param {string} file * @param {string[]} args * @param {{ timeoutMs?: number }} [opts] * @returns {Promise} */ export async function bareOsQvacRunHostCommand(file, args, opts = {}) { const timeoutMs = Math.max(500, Math.min(30000, Number(opts.timeoutMs) || 8000)) const argv = Array.isArray(args) ? args.map((x) => String(x)) : [] /** * @param {{ spawn: Function }} sp */ async function viaSpawn(sp) { return await new Promise((resolve) => { let settled = false /** @type {string[]} */ const chunks = [] let child try { child = sp.spawn(file, argv, { stdio: ['ignore', 'pipe', 'pipe'] }) } catch { resolve(null) return } const finish = (out) => { if (settled) return settled = true try { if (child && typeof child.kill === 'function') child.kill() } catch { /* ignore */ } resolve(out) } const timer = setTimeout(() => finish(null), timeoutMs) try { if (child.stdout && typeof child.stdout.on === 'function') { child.stdout.on('data', (buf) => { chunks.push(Buffer.isBuffer(buf) ? buf.toString('utf8') : String(buf)) }) } } catch { /* ignore */ } const onDone = (code) => { clearTimeout(timer) if (code === 0 || chunks.length) finish(chunks.join('')) else finish(null) } try { child.on('exit', onDone) child.on('error', () => { clearTimeout(timer) finish(null) }) } catch { clearTimeout(timer) finish(null) } }) } try { const sp = bareSubprocess && (typeof bareSubprocess.spawn === 'function' ? bareSubprocess : bareSubprocess.default && typeof bareSubprocess.default.spawn === 'function' ? bareSubprocess.default : null) if (sp) { const out = await viaSpawn(sp) if (out != null) return out } } catch { /* try node next */ } try { const cp = await import('child_process') const execFile = cp.execFile || (cp.default && cp.default.execFile) if (typeof execFile !== 'function') return null return await new Promise((resolve) => { execFile( file, argv, { timeout: timeoutMs, maxBuffer: 8 * 1024 * 1024, encoding: 'utf8' }, (err, stdout) => { if (err && !stdout) resolve(null) else resolve(stdout != null ? String(stdout) : null) } ) }) } catch { return null } } /** * Parse `vulkaninfo` text for GPUs + device-local heap sizes. * @param {string} text * @returns {BareOsQvacGpuInfo[]} */ export function bareOsQvacParseVulkaninfo(text) { const src = String(text || '') if (!src.trim()) return [] /** @type {BareOsQvacGpuInfo[]} */ const out = [] // Split on GPU N: headers (vulkaninfo --summary and full dump). const parts = src.split(/(?=^GPU\d+:)/m) for (const part of parts) { const mIdx = /^GPU(\d+):/m.exec(part) if (!mIdx) continue const index = Number.parseInt(mIdx[1], 10) if (!Number.isFinite(index)) continue const nameMatch = /deviceName\s*=\s*(.+)$/m.exec(part) || /deviceName\s*=\s*(.+)$/im.exec(part) const name = nameMatch ? String(nameMatch[1]).trim() : 'GPU' + index const typeMatch = /deviceType\s*=\s*(\S+)/m.exec(part) const deviceType = typeMatch ? String(typeMatch[1]).trim() : '' let memoryBytes = 0 // Full vulkaninfo: memoryHeaps[i] blocks with DEVICE_LOCAL const heapBlocks = part.split(/(?=memoryHeaps\[\d+\]:)/i) for (const block of heapBlocks) { if (!/memoryHeaps\[\d+\]:/i.test(block)) continue const szMatch = /\bsize\s*=\s*(\d+)/i.exec(block) if (!szMatch) continue const sz = Number(szMatch[1]) if (!Number.isFinite(sz) || sz <= 0) continue if (/DEVICE_LOCAL/i.test(block) || sz > memoryBytes) { memoryBytes = Math.max(memoryBytes, sz) } } // Alternate: "heapSize = N" if (!memoryBytes) { const hs = [...part.matchAll(/heapSize\s*=\s*(\d+)/gi)] for (const x of hs) { const sz = Number(x[1]) if (Number.isFinite(sz) && sz > memoryBytes) memoryBytes = sz } } // Summary-only: no heaps — score by type so discrete still ranks above integrated. if (!memoryBytes) { if (/DISCRETE/i.test(deviceType)) memoryBytes = 8 * 1024 * 1024 * 1024 else if (/INTEGRATED/i.test(deviceType)) memoryBytes = 512 * 1024 * 1024 else memoryBytes = 256 * 1024 * 1024 } out.push({ index, name, memoryBytes, deviceType, source: 'vulkaninfo' }) } return out } /** * Parse nvidia-smi CSV: name, memory.total (MiB). * Indices are NVIDIA order, NOT Vulkan — used only as a presence / size hint. * @param {string} text * @returns {{ name: string, memoryBytes: number }[]} */ export function bareOsQvacParseNvidiaSmi(text) { /** @type {{ name: string, memoryBytes: number }[]} */ const out = [] for (const line of String(text || '').split(/\r?\n/)) { const t = line.trim() if (!t) continue const parts = t.split(',').map((x) => x.trim()) if (parts.length < 2) continue const name = parts[0] const mib = Number(parts[1]) if (!name || !Number.isFinite(mib) || mib <= 0) continue out.push({ name, memoryBytes: Math.floor(mib * 1024 * 1024) }) } return out } /** * Merge nvidia memory hints onto vulkan devices by fuzzy name match. * @param {BareOsQvacGpuInfo[]} vulkan * @param {{ name: string, memoryBytes: number }[]} nvidia */ export function bareOsQvacMergeNvidiaHints(vulkan, nvidia) { if (!vulkan.length || !nvidia.length) return vulkan return vulkan.map((g) => { const gName = g.name.toLowerCase() let best = null let bestScore = 0 for (const n of nvidia) { const nName = n.name.toLowerCase() if (gName.includes(nName) || nName.includes(gName) || gName.includes('nvidia')) { const score = n.memoryBytes if (score > bestScore) { bestScore = score best = n } } } if (best && best.memoryBytes > g.memoryBytes) { return { ...g, memoryBytes: best.memoryBytes, source: g.source + '+nvidia-smi' } } return g }) } /** * True for Mesa lavapipe / llvmpipe / SwiftShader / Vulkan CPU devices. * @param {{ name?: string, deviceType?: string } | null | undefined} g */ export function bareOsQvacIsSoftwareVulkanDevice(g) { if (!g || typeof g !== 'object') return false const name = String(g.name || '').toLowerCase() const type = String(g.deviceType || '').toLowerCase() if ( /llvmpipe|lavapipe|swiftshader|softpipe|cpu rasterizer|microsoft basic render/.test( name ) ) { return true } if ( type === 'cpu' || type.includes('physical_device_type_cpu') || /(^|_)cpu($|_)/.test(type) ) { return true } return false } /** * @param {BareOsQvacGpuInfo[]} gpus * @returns {BareOsQvacGpuInfo[]} */ export function bareOsQvacHardwareGpus(gpus) { return (Array.isArray(gpus) ? gpus : []).filter( (g) => !bareOsQvacIsSoftwareVulkanDevice(g) ) } /** * Score an ICD JSON filename — higher is preferred; software drivers are negative. * @param {string} filename */ export function bareOsQvacScoreVulkanIcdFilename(filename) { const n = String(filename || '').toLowerCase() if (/lvp|llvmpipe|lavapipe|swiftshader/.test(n)) return -100 if (/gfxstream|virtio/.test(n)) return -50 if (/nvidia/.test(n)) return 100 if (/radeon|amd|radv/.test(n)) return 80 if (/intel|anv|iris/.test(n)) return 50 if (/moltenvk|apple/.test(n)) return 40 if (/asahi/.test(n)) return 20 if (/nouveau/.test(n)) return 15 return 10 } /** * Vendor family for a Vulkan ICD manifest filename. * @param {string} filename * @returns {'software' | 'virtual' | 'nvidia' | 'amd' | 'intel' | 'apple' | 'asahi' | 'nouveau' | 'other'} */ export function bareOsQvacVulkanIcdVendor(filename) { const n = String(filename || '').toLowerCase() if (/lvp|llvmpipe|lavapipe|swiftshader/.test(n)) return 'software' if (/gfxstream|virtio/.test(n)) return 'virtual' if (/nvidia/.test(n)) return 'nvidia' if (/radeon|amd|radv/.test(n)) return 'amd' if (/intel|anv|hasvk|iris/.test(n)) return 'intel' if (/moltenvk/.test(n)) return 'apple' if (/asahi/.test(n)) return 'asahi' if (/nouveau/.test(n)) return 'nouveau' return 'other' } /** * VK_LOADER_DRIVERS_SELECT glob for a preferred vendor. * @param {string} vendor */ export function bareOsQvacVulkanVendorSelectGlob(vendor) { switch (vendor) { case 'nvidia': return '*nvidia*' case 'amd': return '*radeon*,*amd*,*radv*' case 'intel': return '*intel*' case 'apple': return '*moltenvk*' case 'asahi': return '*asahi*' case 'nouveau': return '*nouveau*' default: return '' } } /** * Keep ICDs for the highest-scored vendor so ggml does not enumerate unused * Mesa/emulator manifests (asahi/gfxstream/nouveau/virtio) next to RADV. * `icds` must already be score-sorted (best first), as from discover. * @param {string[]} icds * @returns {{ vendor: string, icds: string[], selectGlob: string }} */ export function bareOsQvacPreferredVulkanIcds(icds) { const list = (Array.isArray(icds) ? icds : []).filter( (p) => typeof p === 'string' && p ) if (!list.length) return { vendor: '', icds: [], selectGlob: '' } const vendor = bareOsQvacVulkanIcdVendor(path.basename(list[0])) if (vendor === 'software' || vendor === 'virtual' || vendor === 'other') { return { vendor, icds: list, selectGlob: '' } } const filtered = list.filter( (p) => bareOsQvacVulkanIcdVendor(path.basename(p)) === vendor ) return { vendor, icds: filtered.length ? filtered : list, selectGlob: bareOsQvacVulkanVendorSelectGlob(vendor) } } /** * List hardware Vulkan ICD manifest paths (software ICDs excluded). * @param {{ * dirs?: string[], * readdirSync?: (dir: string) => string[], * existsSync?: (p: string) => boolean * }} [opts] * @returns {string[]} */ export function bareOsQvacDiscoverHardwareVulkanIcds(opts = {}) { const dirs = Array.isArray(opts.dirs) ? opts.dirs : BARE_OS_QVAC_VULKAN_ICD_DIRS const exists = typeof opts.existsSync === 'function' ? opts.existsSync : (p) => { try { return hostFs.existsSync(p) } catch { return false } } const readdir = typeof opts.readdirSync === 'function' ? opts.readdirSync : (dir) => { try { return hostFs.readdirSync(dir) } catch { return [] } } /** @type {{ path: string, score: number }[]} */ const found = [] for (const dir of dirs) { if (!exists(dir)) continue let names = [] try { names = readdir(dir) || [] } catch { continue } for (const name of names) { const base = String(name) if (!base.toLowerCase().endsWith('.json')) continue const score = bareOsQvacScoreVulkanIcdFilename(base) if (score < 0) continue found.push({ path: path.join(dir, base), score }) } } found.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path)) return found.map((x) => x.path) } /** * Prefer real GPU Vulkan drivers before llama.cpp / vulkaninfo enumerate. * Disables lavapipe and, on Optimus laptops, nudges NVIDIA offload. * * @param {Record} [env] guest/host env overlay (BARE_OS_*) * @param {{ * procEnv?: Record | null, * icds?: string[], * hasNvidiaSmi?: boolean * }} [opts] * @returns {{ * note: string, * icds: string[], * disabledSoftware: boolean, * preferNvidia: boolean, * preferAmd: boolean, * vendor: string * }} */ export function bareOsQvacApplyHardwareVulkanEnv(env = {}, opts = {}) { const procEnv = opts.procEnv !== undefined ? opts.procEnv : globalThis.process && globalThis.process.env ? globalThis.process.env : null const empty = { note: '', icds: /** @type {string[]} */ ([]), disabledSoftware: false, preferNvidia: false, preferAmd: false, vendor: '' } if (!procEnv) return empty const keep = String( env.BARE_OS_QVAC_KEEP_VK_ENV ?? procEnv.BARE_OS_QVAC_KEEP_VK_ENV ?? '' ) .trim() .toLowerCase() if (keep === '1' || keep === 'true') { return { ...empty, note: 'BARE_OS_QVAC_KEEP_VK_ENV — left Vulkan env unchanged' } } const softwareDisableGlob = '*lvp*,*lavapipe*,*swiftshader*' const prevDisable = String(procEnv.VK_LOADER_DRIVERS_DISABLE || '').trim() if (!prevDisable) { procEnv.VK_LOADER_DRIVERS_DISABLE = softwareDisableGlob } else if (!/lvp|lavapipe|swiftshader/i.test(prevDisable)) { procEnv.VK_LOADER_DRIVERS_DISABLE = prevDisable + ',' + softwareDisableGlob } const discovered = Array.isArray(opts.icds) && opts.icds.length ? opts.icds : bareOsQvacDiscoverHardwareVulkanIcds() const preferred = bareOsQvacPreferredVulkanIcds(discovered) const preferNvidia = preferred.vendor === 'nvidia' || Boolean(opts.hasNvidiaSmi) const preferAmd = preferred.vendor === 'amd' && !preferNvidia const vendor = preferNvidia ? 'nvidia' : preferred.vendor // nvidia-smi without a listed NVIDIA ICD (unusual install): keep the full // hardware list and let VK_LOADER_DRIVERS_SELECT=*nvidia* filter at load. const icds = preferNvidia && preferred.vendor !== 'nvidia' ? discovered : preferred.icds const userDrivers = String( env.BARE_OS_QVAC_VK_DRIVER_FILES ?? procEnv.BARE_OS_QVAC_VK_DRIVER_FILES ?? '' ).trim() const alreadyForced = Boolean( String(procEnv.VK_DRIVER_FILES || '').trim() || String(procEnv.VK_ICD_FILENAMES || '').trim() || userDrivers ) if (userDrivers && !String(procEnv.VK_DRIVER_FILES || '').trim()) { procEnv.VK_DRIVER_FILES = userDrivers } else if (!alreadyForced && icds.length) { // One vendor's manifests only — AMD boxes often ship unused Mesa ICDs // (intel/asahi/nouveau/virtio/gfxstream) that confuse ggml enumeration. procEnv.VK_DRIVER_FILES = icds.join(':') } const selectMode = String( env.BARE_OS_QVAC_VK_SELECT ?? procEnv.BARE_OS_QVAC_VK_SELECT ?? 'auto' ) .trim() .toLowerCase() // Hybrid Intel+NVIDIA/AMD: bind the discrete vendor so ggml never lands on // iGPU/llvmpipe. Set BARE_OS_QVAC_VK_SELECT=off to keep every hardware ICD. const selectGlob = preferNvidia ? '*nvidia*' : preferred.selectGlob const wantSelect = selectGlob && (selectMode === 'auto' || selectMode === vendor || (preferNvidia && selectMode === 'nvidia')) if (wantSelect && !String(procEnv.VK_LOADER_DRIVERS_SELECT || '').trim()) { procEnv.VK_LOADER_DRIVERS_SELECT = selectGlob } if (preferNvidia) { if (!String(procEnv.__NV_PRIME_RENDER_OFFLOAD || '').trim()) { procEnv.__NV_PRIME_RENDER_OFFLOAD = '1' } if (!String(procEnv.__GLX_VENDOR_LIBRARY_NAME || '').trim()) { procEnv.__GLX_VENDOR_LIBRARY_NAME = 'nvidia' } if (!String(procEnv.__VK_LAYER_NV_optimus || '').trim()) { procEnv.__VK_LAYER_NV_optimus = 'NVIDIA_only' } } const noteParts = ['disabled software Vulkan ICDs (lavapipe/llvmpipe)'] if (icds.length) { noteParts.push( 'hardware ICDs=' + icds.map((p) => path.basename(p)).join(',') ) } if (preferNvidia) noteParts.push('NVIDIA Optimus offload env set') if (preferAmd) noteParts.push('RADV/AMD ICD select') const selectNow = String(procEnv.VK_LOADER_DRIVERS_SELECT || '') if (selectNow) { noteParts.push('VK_LOADER_DRIVERS_SELECT=' + selectNow) } return { note: noteParts.join('; '), icds, disabledSoftware: true, preferNvidia, preferAmd, vendor } } /** * True on macOS. ggml-metal is the GPU backend there (no Vulkan runtime). * @returns {boolean} */ export function bareOsQvacHostIsDarwin() { try { if (typeof os.platform === 'function') return os.platform() === 'darwin' } catch { /* fall back to process below */ } return Boolean( globalThis.process && globalThis.process.platform === 'darwin' ) } /** * Parse a Metal VRAM field ("18 GB", "Shared system memory: 18 GB", or a * nested `{ spdisplays_vram: "16 GB" }`) to bytes. Returns 0 when absent. * @param {unknown} v * @returns {number} */ export function bareOsQvacParseMetalMemory(v) { const raw = v && typeof v === 'object' && !Array.isArray(v) ? String( /** @type {Record} */ (v).spdisplays_vram || '' ) : String(v || '') const m = /(\d+(?:\.\d+)?)\s*(MB|GB|TB)/i.exec(raw) if (!m) return 0 const n = Number(m[1]) const unit = String(m[2]).toUpperCase() const mult = unit === 'GB' ? 1024 * 1024 * 1024 : unit === 'TB' ? 1024 * 1024 * 1024 * 1024 : 1024 * 1024 return Math.floor(n * mult) } /** * Parse `system_profiler SPDisplaysDataType -json` for Apple GPUs. * @param {string} text * @returns {BareOsQvacGpuInfo[]} */ export function bareOsQvacParseSystemProfilerDisplays(text) { /** @type {BareOsQvacGpuInfo[]} */ const out = [] let data try { data = JSON.parse(String(text || '')) } catch { return out } const list = Array.isArray(data && data.SPDisplaysDataType) ? /** @type {Array>} */ (data.SPDisplaysDataType) : [] const seen = new Set() for (const item of list) { if (!item || typeof item !== 'object') continue const name = String( item.chipset_model || item._name || '' ).trim() if (!name || seen.has(name)) continue seen.add(name) const vramRaw = item.spdisplays_vram ?? item.spdisplays_vram_shared ?? '' const memoryBytes = bareOsQvacParseMetalMemory(vramRaw) const eGpu = /egpu|external/i.test(String(item.sppci_device_type || '')) const discrete = eGpu || Boolean(item.spdisplays_vram) out.push({ index: out.length, name, memoryBytes, deviceType: discrete ? 'PHYSICAL_DEVICE_TYPE_DISCRETE_GPU' : 'PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU', source: 'metal' }) } return out } /** * Sort GPUs for load attempts: highest memory first; discrete before integrated on ties. * Software Vulkan devices are ranked last. * @param {BareOsQvacGpuInfo[]} gpus * @returns {BareOsQvacGpuInfo[]} */ export function bareOsQvacRankGpus(gpus) { return [...gpus].sort((a, b) => { const as = bareOsQvacIsSoftwareVulkanDevice(a) ? 1 : 0 const bs = bareOsQvacIsSoftwareVulkanDevice(b) ? 1 : 0 if (as !== bs) return as - bs if (b.memoryBytes !== a.memoryBytes) return b.memoryBytes - a.memoryBytes const ad = /DISCRETE/i.test(a.deviceType) ? 1 : 0 const bd = /DISCRETE/i.test(b.deviceType) ? 1 : 0 if (bd !== ad) return bd - ad return a.index - b.index }) } /** * Build ordered main-gpu candidates for auto mode. * @param {BareOsQvacGpuInfo[]} ranked * @returns {Array} */ export function bareOsQvacMainGpuCandidates(ranked) { /** @type {Array} */ const out = [] const seen = new Set() const push = (v) => { const k = String(v) if (seen.has(k)) return seen.add(k) out.push(v) } const hardware = bareOsQvacHardwareGpus(ranked) const vulkanBacked = hardware.filter((g) => String(g.source || '').includes('vulkaninfo') ) if (vulkanBacked.length) { for (const g of vulkanBacked) push(g.index) push('dedicated') return out } // nvidia-smi indices ≠ Vulkan indices on hybrid laptops — prefer class + common slots. if (hardware.length) { push('dedicated') push(1) push(0) push(2) push(3) return out } push('dedicated') push(1) push(0) push(2) push(3) return out } /** * Probe host GPUs (best-effort). Never throws. * @param {{ runCommand?: typeof bareOsQvacRunHostCommand }} [opts] * @returns {Promise} */ export async function bareOsQvacProbeGpus(opts = {}) { try { const run = opts.runCommand || bareOsQvacRunHostCommand // macOS: no vulkaninfo/nvidia-smi. Metal is the only GPU backend, so probe // the Apple GPU via system_profiler for name/VRAM reporting. if (bareOsQvacHostIsDarwin()) { const sp = await run('system_profiler', ['SPDisplaysDataType', '-json'], { timeoutMs: 15000 }) const metal = bareOsQvacParseSystemProfilerDisplays(sp || '') return bareOsQvacRankGpus(metal) } /** @type {BareOsQvacGpuInfo[]} */ let vulkan = [] const full = (await run('vulkaninfo', [], { timeoutMs: 12000 })) || (await run('vulkaninfo', ['--summary'], { timeoutMs: 8000 })) if (full) vulkan = bareOsQvacParseVulkaninfo(full) const smi = await run( 'nvidia-smi', ['--query-gpu=name,memory.total', '--format=csv,noheader,nounits'], { timeoutMs: 5000 } ) const nvidia = smi ? bareOsQvacParseNvidiaSmi(smi) : [] if (vulkan.length && nvidia.length) { vulkan = bareOsQvacMergeNvidiaHints(vulkan, nvidia) } else if (!vulkan.length && nvidia.length) { // No Vulkan enumeration — synthesize placeholders; load path still tries dedicated + indices. vulkan = nvidia.map((n, i) => ({ index: i, name: n.name, memoryBytes: n.memoryBytes, deviceType: 'PHYSICAL_DEVICE_TYPE_DISCRETE_GPU', source: 'nvidia-smi' })) } // Drop software adapters from the preferred list (keep rank helper for tests). return bareOsQvacRankGpus(vulkan) } catch { return [] } }