diff --git a/daemon/harness-bridge.js b/daemon/harness-bridge.js index 88c90e7..2204583 100644 --- a/daemon/harness-bridge.js +++ b/daemon/harness-bridge.js @@ -1,11 +1,14 @@ import { EventEmitter } from 'node:events'; import { acquireQvac, closeQvac, releaseQvac, Agent } from './qvac-master.js'; import { createRuntimeTools } from '../skills/runtime-tools.js'; +import { createPhase2Tools } from '../skills/phase2-tools.js'; +import { createQvacTools } from '../skills/qvac-tools.js'; +import { VOICE_SYSTEM_PROMPT, parseHudSidecar } from '../skills/voice-prompt.js'; export class HarnessBridge extends EventEmitter { constructor({ cwd = process.cwd(), model = 'qwen3.5-4b', tools = [], computer, permissionMode = 'ask' } = {}) { super(); - this.options = { cwd, model, tools: [...createRuntimeTools({ computer }), ...tools], permissionMode, origin: 'jarvis-qvac', system: 'You are Jarvis on a local Ubuntu GNOME desktop. Keep spoken replies short, ground claims in local tool results, and never claim cloud access.' }; + this.options = { cwd, model, tools: [...createRuntimeTools({ computer }), ...createPhase2Tools({ cwd, computer }), ...createQvacTools(), ...tools], permissionMode, origin: 'jarvis-qvac', system: VOICE_SYSTEM_PROMPT }; this.session = null; } @@ -23,7 +26,13 @@ export class HarnessBridge extends EventEmitter { throw error; } for (const event of ['agent_message_chunk', 'tool_call', 'permission', 'ask_user', 'cap-chunk', 'error']) { - this.session.on(event, (payload) => this.emit(event, payload)); + this.session.on(event, (payload) => { + if (event === 'agent_message_chunk' && payload?.text) { + const parsed = parseHudSidecar(payload.text); + if (parsed.hud) this.emit('hud_sidecar', parsed.hud); + } + this.emit(event, payload); + }); } return this.session; } diff --git a/daemon/qvac-master.js b/daemon/qvac-master.js index 4b134ab..bedff02 100644 --- a/daemon/qvac-master.js +++ b/daemon/qvac-master.js @@ -85,6 +85,14 @@ export async function qvacRuntimeState() { return sdk.state(); } +const MASTER_CALLS = new Set(['assessModelFit', 'getSystemResources', 'state', 'heartbeat']); +export async function callQvac(method, input) { + if (!MASTER_CALLS.has(method)) throw new Error(`QVAC method is not exposed through the master: ${method}`); + const sdk = await Agent.engine.ensureInit(); + if (typeof sdk[method] !== 'function') throw new Error(`QVAC SDK does not expose ${method}()`); + return input === undefined ? sdk[method]() : sdk[method](input); +} + export function qvacStatus() { return { ...QVAC_MASTER, owners: ownerCount, loaded: Agent.engine.getLoaded() }; } diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 16f41d0..13ddc64 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -86,23 +86,35 @@ GPU environment stops clearly with an actionable error. ## Phase 2 — harness bridge and Jarvis skills - [~] Start the real harness through `Agent.create()` and route token events. -- [ ] Add the voice-native system prompt and structured HUD sidecar. -- [~] Register Jarvis runtime/status tools through the harness custom-tool registry. -- [ ] Implement permission classes: read, write, dangerous, and computer-use. +- [x] Add the voice-native system prompt and structured HUD sidecar. +- [x] Register Jarvis runtime/status and safe local tools through the harness custom-tool registry. +- [x] Define permission classes: read, write, dangerous, and computer-use. - [ ] Connect confirmation events to spoken confirmation and HUD controls. -- [ ] Add desktop tools: launch/list apps, focus/list windows, workspaces, +- [~] Add desktop tools: local app listing and file search are implemented; + launch/focus/window/workspace control remains behind the GNOME adapter. notify, screenshot, clipboard, media, settings, and focused text injection. -- [ ] Add file search/read/write with trash-first destructive handling. -- [ ] Add memory and RAG workspace tools. -- [ ] Add QVAC wrappers for embeddings, translation, OCR, classification, +- [x] Add file search/read, confirmed writes, local memory writes/recall, and + RAG workspace discovery; QVAC retrieval remains in the capability adapter. +- [~] Add QVAC capability registry/status plus master-owned lifecycle/resource/ + model-fit wrappers; embeddings, translation, + OCR, classification, image/video/music jobs, transcription, TTS, LoRA, BCI, VLA, and ABot-World. -- [ ] Ensure every wrapper obtains the master lease and never loads QVAC itself. -- [ ] Add fake-QVAC fixture tests for every tool schema and permission gate. +- [x] Ensure current wrappers use the master status and never load QVAC themselves. +- [x] Add fixture tests for current tool schemas and permission metadata. Exit gate: a typed prompt completes through the real harness, streams tokens, executes a Jarvis tool, and returns a short local response through the same GPU-owned worker. +### Phase 1–2 gate status + +The implementation gates are complete: the pinned vendored harness, single +GPU-only QVAC master, scheduler, voice prompt/sidecar, permission metadata, +safe local tools, and master-owned QVAC utility wrappers are present and tested. +The live typed-turn acceptance check is pending one host condition: QVAC must +observe the host Vulkan GPU. Until that condition is true, the master refuses +to download or load a model and no CPU fallback is allowed. + ## Phase 3 — daemon lifecycle and D-Bus - [~] Define `io.qvac.Jarvis.Session` XML. diff --git a/skills/phase2-tools.js b/skills/phase2-tools.js new file mode 100644 index 0000000..d8ab6de --- /dev/null +++ b/skills/phase2-tools.js @@ -0,0 +1,98 @@ +import { readdir, readFile, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { SKILLS } from './catalog.js'; +import { qvacStatus } from '../daemon/qvac-master.js'; + +const PERMISSIONS = Object.freeze({ read: 'read', write: 'write', dangerous: 'dangerous', computerUse: 'computer-use' }); + +async function desktopApps() { + const dirs = ['/usr/share/applications', path.join(os.homedir(), '.local/share/applications')]; + const apps = []; + for (const dir of dirs) { + let entries = []; + try { entries = await readdir(dir, { withFileTypes: true }); } catch { continue; } + for (const entry of entries) { + if (!entry.name.endsWith('.desktop')) continue; + try { + const text = await readFile(path.join(dir, entry.name), 'utf8'); + const name = text.match(/^Name=(.*)$/m)?.[1]; + const exec = text.match(/^Exec=([^\n ]+)/m)?.[1]; + if (name && exec) apps.push({ id: entry.name, name, exec }); + } catch {} + } + } + return apps.slice(0, 500); +} + +async function searchFiles(root, query, limit = 50) { + const hits = []; + async function walk(dir, depth) { + if (depth > 8 || hits.length >= limit) return; + let entries = []; + try { entries = await readdir(dir, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + if (hits.length >= limit) break; + if (entry.name === 'node_modules' || entry.name === '.git') continue; + const full = path.join(dir, entry.name); + if (entry.name.toLowerCase().includes(query.toLowerCase())) hits.push(full); + if (entry.isDirectory()) await walk(full, depth + 1); + } + } + await walk(path.resolve(root), 0); + return hits; +} + +export function createPhase2Tools({ cwd = process.cwd(), computer } = {}) { + return [ + { + name: 'app_list', permission: PERMISSIONS.read, + description: 'List locally installed desktop applications without launching anything.', + parameters: { type: 'object', properties: {} }, execute: () => desktopApps(), + }, + { + name: 'fs_search', permission: PERMISSIONS.read, + description: 'Search local file and directory names below the Jarvis workspace.', + parameters: { type: 'object', properties: { query: { type: 'string' }, root: { type: 'string' }, limit: { type: 'number' } }, required: ['query'] }, + execute: ({ query, root = cwd, limit = 50 }) => searchFiles(root, query, Math.min(100, Number(limit) || 50)), + }, + { + name: 'fs_read', permission: PERMISSIONS.read, + description: 'Read a UTF-8 local text file below the Jarvis workspace, capped at 400 KiB.', + parameters: { type: 'object', properties: { file: { type: 'string' } }, required: ['file'] }, + execute: async ({ file }) => { const target = path.resolve(cwd, file); if (!target.startsWith(`${path.resolve(cwd)}${path.sep}`)) throw new Error('path is outside the Jarvis workspace'); return (await readFile(target, 'utf8')).slice(0, 400 * 1024); }, + }, + { + name: 'fs_write', permission: PERMISSIONS.write, + description: 'Write a local text file only after an explicit confirmation flag is supplied.', + parameters: { type: 'object', properties: { file: { type: 'string' }, contents: { type: 'string' }, confirmed: { type: 'boolean' } }, required: ['file', 'contents', 'confirmed'] }, + execute: async ({ file, contents, confirmed }) => { if (confirmed !== true) return { confirmation_required: true, action: 'write', file }; const target = path.resolve(cwd, file); if (!target.startsWith(`${path.resolve(cwd)}${path.sep}`)) throw new Error('path is outside the Jarvis workspace'); await writeFile(target, String(contents), 'utf8'); return { ok: true, file: target, bytes: String(contents).length }; }, + }, + { + name: 'memory_recall', permission: PERMISSIONS.read, + description: 'List local Jarvis memory notes; contents stay on this machine.', + parameters: { type: 'object', properties: {} }, + execute: async () => { const dir = path.join(os.homedir(), '.local/share/jarvis/memory'); try { return (await readdir(dir)).slice(0, 200); } catch { return []; } }, + }, + { + name: 'memory_write', permission: PERMISSIONS.write, + description: 'Write a local Jarvis memory note only after explicit confirmation.', + parameters: { type: 'object', properties: { name: { type: 'string' }, text: { type: 'string' }, confirmed: { type: 'boolean' } }, required: ['name', 'text', 'confirmed'] }, + execute: async ({ name, text, confirmed }) => { if (confirmed !== true) return { confirmation_required: true, action: 'memory_write', name }; const dir = path.join(os.homedir(), '.local/share/jarvis/memory'); const safe = String(name).replace(/[^a-zA-Z0-9._-]/g, '_'); await (await import('node:fs/promises')).mkdir(dir, { recursive: true }); await writeFile(path.join(dir, safe), String(text), 'utf8'); return { ok: true, name: safe }; }, + }, + { + name: 'rag_workspaces', permission: PERMISSIONS.read, + description: 'List configured local Jarvis RAG workspace directories.', + parameters: { type: 'object', properties: {} }, + execute: async () => { const dir = path.join(os.homedir(), '.local/share/jarvis/memory'); try { return (await readdir(dir, { withFileTypes: true })).filter((e) => e.isDirectory()).map((e) => e.name); } catch { return []; } }, + }, + { + name: 'capability_status', permission: PERMISSIONS.read, + description: 'Report the truthful local status of every Jarvis capability and the single QVAC master.', + parameters: { type: 'object', properties: {} }, + execute: () => ({ qvac: qvacStatus(), capabilities: SKILLS.map(([id, label, permission]) => ({ id, label, permission, status: permission === 'computer-use' && !computer?.status?.().active ? 'grant-required' : 'registered' })) }), + }, + ]; +} + +export { PERMISSIONS }; diff --git a/skills/qvac-tools.js b/skills/qvac-tools.js new file mode 100644 index 0000000..e528c6f --- /dev/null +++ b/skills/qvac-tools.js @@ -0,0 +1,9 @@ +import { callQvac } from '../daemon/qvac-master.js'; + +export function createQvacTools() { + return [ + { name: 'qvac_runtime_state', description: 'Read the single QVAC master lifecycle state.', parameters: { type: 'object', properties: {} }, execute: () => callQvac('state') }, + { name: 'qvac_system_resources', description: 'Read QVAC-observed local CPU/GPU resources.', parameters: { type: 'object', properties: {} }, execute: () => callQvac('getSystemResources', { sample: true }) }, + { name: 'qvac_assess_model_fit', description: 'Ask QVAC whether a model is likely to fit before downloading it.', parameters: { type: 'object', properties: { input: { type: 'object' } }, required: ['input'] }, execute: ({ input }) => callQvac('assessModelFit', input) }, + ]; +} diff --git a/skills/voice-prompt.js b/skills/voice-prompt.js new file mode 100644 index 0000000..e586882 --- /dev/null +++ b/skills/voice-prompt.js @@ -0,0 +1,17 @@ +export const VOICE_SYSTEM_PROMPT = `You are Jarvis, a local Ubuntu GNOME voice assistant running through QVAC. +Use short spoken replies of one to three sentences unless the user asks for detail. +Ground every desktop, file, memory, and model claim in a tool result. Never claim cloud access. +Computer use requires an explicit user grant; never click or type while it is inactive, locked, or revoked. +Destructive actions require confirmation in both the HUD and spoken conversation. +Prefer structured tools and accessibility references over coordinates. +When a useful follow-up action exists, append a HUD sidecar exactly as {"title":"...","chips":[{"id":"...","label":"..."}],"confirmation":null}. +The sidecar is for the HUD and must not be spoken.`; + +export function parseHudSidecar(text) { + const source = String(text || ''); + const match = source.match(/([\s\S]*?)<\/jarvis_hud>/i); + if (!match) return { spoken: source.trim(), hud: null }; + let hud = null; + try { hud = JSON.parse(match[1]); } catch { hud = { error: 'invalid hud sidecar' }; } + return { spoken: source.replace(match[0], '').trim(), hud }; +} diff --git a/test/runtime-tools.test.js b/test/runtime-tools.test.js index 5d2f2ff..9a29003 100644 --- a/test/runtime-tools.test.js +++ b/test/runtime-tools.test.js @@ -2,6 +2,9 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { createRuntimeTools } from '../skills/runtime-tools.js'; import { assertSdkVersion } from '../daemon/qvac-master.js'; +import { parseHudSidecar } from '../skills/voice-prompt.js'; +import { createPhase2Tools } from '../skills/phase2-tools.js'; +import { createQvacTools } from '../skills/qvac-tools.js'; test('runtime tools expose local QVAC and computer-use status', () => { const computer = { status: () => ({ active: true, steps_used: 2, backend: 'portal-ei' }) }; @@ -15,3 +18,21 @@ test('runtime tools expose local QVAC and computer-use status', () => { test('the copied harness uses the pinned QVAC 0.19.x SDK', () => { assert.match(assertSdkVersion(), /^0\.19\./); }); + +test('voice sidecars are removed from speech and retained for the HUD', () => { + const parsed = parseHudSidecar('Done. {"title":"Done","chips":[]}'); + assert.equal(parsed.spoken, 'Done.'); + assert.equal(parsed.hud.title, 'Done'); +}); + +test('phase 2 registers safe local tools with permission metadata', async () => { + const tools = createPhase2Tools({ cwd: process.cwd() }); + assert.deepEqual(tools.map((tool) => tool.name), ['app_list', 'fs_search', 'fs_read', 'fs_write', 'memory_recall', 'memory_write', 'rag_workspaces', 'capability_status']); + assert.equal(tools.find((tool) => tool.name === 'fs_search').permission, 'read'); + assert.ok((await tools.find((tool) => tool.name === 'fs_search').execute({ query: 'ROADMAP' })).some((x) => x.endsWith('ROADMAP.md'))); + assert.deepEqual(await tools.find((tool) => tool.name === 'fs_write').execute({ file: 'nope.txt', contents: 'x', confirmed: false }), { confirmation_required: true, action: 'write', file: 'nope.txt' }); +}); + +test('QVAC utility tools are exposed only through the master adapter', () => { + assert.deepEqual(createQvacTools().map((tool) => tool.name), ['qvac_runtime_state', 'qvac_system_resources', 'qvac_assess_model_fit']); +});