commit 0f00e11823e3f60b1e5a524d9bdad5b96711ac16 Author: Raven Scott Date: Fri Sep 11 13:37:34 2026 -0400 first commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f83b803 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +qvac.config.local.json +.env +dist/ +*.log diff --git a/QVAC.md b/QVAC.md new file mode 100644 index 0000000..7f3fe5d --- /dev/null +++ b/QVAC.md @@ -0,0 +1,38 @@ +# QVAC integration pin + +Pinned against the live QVAC documentation for SDK release **v0.19.0** on +2026-09-11. Source of truth: and +. + +## Rules for Jarvis + +- Use the function-centric `@qvac/sdk` API at version `0.19.0`. +- Do not use removed `startQVACProvider`, `delegate`, or `no_mmap` APIs. +- Use `load_mode` for the replacement load behavior described by the release notes. +- Use `completion()` for streamed LLM work; its canonical surfaces are `events` + and `final`. +- Use `loadModel()` / `unloadModel()` for model lifecycle. +- Use `textToSpeech()` or `textToSpeechStream()` for TTS. +- Use `transcribe()` / `transcribeStream()` for ASR, `ocr()` for OCR, + `translate()` for text translation, and `assessModelFit()` before model fetch. +- Use `cancel({ requestId })` for a specific operation and `suspend()` / + `resume()` for runtime lifecycle. +- QVAC's local HTTP server is `qvac serve --openai`; it exposes OpenAI-compatible + `/v1/*` at `http://127.0.0.1:11434/v1/`. `qvac serve openai` is deprecated. + +## v0.19 API names used by the capability registry + +`completion`, `batchCompletion`, `embed`, `ragIngest`, `ragSearch`, +`ragReindex`, `ragDeleteEmbeddings`, `ragListWorkspaces`, `diffusion`, +`video`, `audioGen`, `transcribe`, `transcribeStream`, +`textToSpeech`, `textToSpeechStream`, `translate`, `ocr`, `classify`, +`bciTranscribe`, `vla`, `worldCreateScene`, `worldStep`, `assessModelFit`, +`downloadAsset`, `cancel`, `loadModel`, `unloadModel`, `suspend`, `resume`, +`getSystemResources`, `getLoadedModelInfo`, `getModelInfo`, `modelRegistryList`, +`modelRegistrySearch`, `modelRegistryGetModel`, and `state` are documented +public surfaces. Confirm parameter types against the installed `.d.ts` files +before implementing each skill. + +The HTTP server mounts `/v1/chat/completions`, `/v1/responses`, +`/v1/completions`, `/v1/embeddings`, files/vector stores, image/video +generation, audio transcription/speech/translation, and `/qvac/v1/translate`. diff --git a/README.md b/README.md new file mode 100644 index 0000000..f863797 --- /dev/null +++ b/README.md @@ -0,0 +1,32 @@ +# JARVIS-QVAC + +Local-first Ubuntu GNOME voice assistant scaffold. The cognitive core is the +already-extracted `/home/raven/dev/agent-harness`, exposed through the symlink +`vendor/agent-harness`; Jarvis does not copy or depend on `/home/raven/dev/dlinux`. + +## Bootstrap + +```bash +npm install +npm test +npm run cu-doctor +npm run gpu-doctor +npx qvac doctor +npx qvac serve --openai --host 127.0.0.1 +``` + +The daemon bridge loads the harness in-process with `Agent.engine.load()` and +creates sessions with `Agent.create()`. The optional QVAC HTTP sibling is +configured for `127.0.0.1:11434`. The GNOME extension is an intentionally thin +ESM panel shell; inference, audio, portals, and computer-use actuation belong +in user services. + +GPU inference is a hard requirement. `gpu-doctor` must report a QVAC-visible +GPU and `jarvisd` refuses to continue if the loaded model reports `device: cpu`. +This prevents a misleading CPU mode and competing QVAC workers. + +Current repository scope is phase-0/phase-1 bootstrap plus the computer-use +grant boundary. Portal ScreenCast/RemoteDesktop, AT-SPI, PipeWire audio, the +full D-Bus adapter, GTK control-center pages, and the voice state machine are +the next implementation slices. See [docs/agent-harness-map.md](docs/agent-harness-map.md) +and [docs/cu-acceptance.md](docs/cu-acceptance.md). diff --git a/apps/control-center/README.md b/apps/control-center/README.md new file mode 100644 index 0000000..be0ddec --- /dev/null +++ b/apps/control-center/README.md @@ -0,0 +1,6 @@ +# Jarvis Control Center + +This package is the GTK4/libadwaita control-plane boundary. Its pages are +General, Voice, Models, Memory, Skills, Computer use, Privacy, Lab, and About. +The daemon remains the owner of QVAC, audio, portals, and the harness; the UI +will communicate over session D-Bus. diff --git a/apps/control-center/package.json b/apps/control-center/package.json new file mode 100644 index 0000000..d2e8c65 --- /dev/null +++ b/apps/control-center/package.json @@ -0,0 +1,5 @@ +{ + "name": "@jarvis-qvac/control-center", + "private": true, + "type": "module" +} diff --git a/apps/gnome-extension/jarvis@qvac.local/extension.js b/apps/gnome-extension/jarvis@qvac.local/extension.js new file mode 100644 index 0000000..01fe4c7 --- /dev/null +++ b/apps/gnome-extension/jarvis@qvac.local/extension.js @@ -0,0 +1,17 @@ +import { Extension } from 'resource:///org/gnome/shell/extensions/extension.js'; +import * as Main from 'resource:///org/gnome/shell/ui/main.js'; +import * as PanelMenu from 'resource:///org/gnome/shell/ui/panelMenu.js'; +import St from 'gi://St'; + +export default class JarvisExtension extends Extension { + enable() { + this._indicator = new PanelMenu.Button(0.0, 'Jarvis QVAC'); + this._indicator.add_child(new St.Label({ text: '◯', y_align: 2 })); + Main.panel.addToStatusArea('jarvis-qvac', this._indicator, 0, 'right'); + } + + disable() { + this._indicator?.destroy(); + this._indicator = null; + } +} diff --git a/apps/gnome-extension/jarvis@qvac.local/metadata.json b/apps/gnome-extension/jarvis@qvac.local/metadata.json new file mode 100644 index 0000000..771c123 --- /dev/null +++ b/apps/gnome-extension/jarvis@qvac.local/metadata.json @@ -0,0 +1,8 @@ +{ + "uuid": "jarvis@qvac.local", + "name": "Jarvis QVAC", + "description": "Local-first QVAC voice assistant HUD for GNOME", + "shell-version": ["46", "47", "48", "49", "50"], + "version": 1, + "settings-schema": "org.gnome.shell.extensions.jarvis" +} diff --git a/apps/gnome-extension/jarvis@qvac.local/prefs.js b/apps/gnome-extension/jarvis@qvac.local/prefs.js new file mode 100644 index 0000000..8f2a5b8 --- /dev/null +++ b/apps/gnome-extension/jarvis@qvac.local/prefs.js @@ -0,0 +1,7 @@ +import { ExtensionPreferences } from 'resource:///org/gnome/Shell/Extensions/js/extensions/prefs.js'; + +export default class JarvisPreferences extends ExtensionPreferences { + fillPreferencesWindow(window) { + window.set_title('Jarvis QVAC'); + } +} diff --git a/apps/gnome-extension/jarvis@qvac.local/schemas/org.gnome.shell.extensions.jarvis.gschema.xml b/apps/gnome-extension/jarvis@qvac.local/schemas/org.gnome.shell.extensions.jarvis.gschema.xml new file mode 100644 index 0000000..713e333 --- /dev/null +++ b/apps/gnome-extension/jarvis@qvac.local/schemas/org.gnome.shell.extensions.jarvis.gschema.xml @@ -0,0 +1,18 @@ + + + 'hey jarvis' + ['jarvis', 'okay jarvis'] + '<Super><Space>' + '' + 'en-US' + 'full-listen-after-wake' + 'arc' + true + 'off' + false + 'laptop-8gb' + '#F4B942' + true + true + + diff --git a/computer-use/doctor.js b/computer-use/doctor.js new file mode 100644 index 0000000..fcdf45d --- /dev/null +++ b/computer-use/doctor.js @@ -0,0 +1,22 @@ +import { access } from 'node:fs/promises'; +import { constants } from 'node:fs'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; + +const run = promisify(execFile); +const checks = [ + ['session', async () => process.env.XDG_SESSION_TYPE || 'unknown'], + ['portal', async () => { await access('/usr/share/dbus-1/services/org.freedesktop.portal.Desktop.service', constants.F_OK); return 'installed'; }], + ['pipewire', async () => { await run('sh', ['-lc', 'command -v pw-cat']); return 'installed'; }], + ['at-spi', async () => { await run('sh', ['-lc', 'command -v gsettings']); return 'desktop tools present'; }], + ['libei', async () => { await run('sh', ['-lc', 'ldconfig -p 2>/dev/null | grep -q libei']); return 'installed'; }], + ['ydotool', async () => { await run('sh', ['-lc', 'command -v ydotool']); return 'optional fallback present'; }], +]; + +let failed = 0; +for (const [name, check] of checks) { + try { console.log(`ok ${name}: ${await check()}`); } + catch { failed += 1; console.log(`---- ${name}: unavailable`); } +} +console.log(failed ? `cu-doctor: ${failed} optional/required checks unavailable` : 'cu-doctor: all checks passed'); +process.exitCode = failed ? 1 : 0; diff --git a/computer-use/package.json b/computer-use/package.json new file mode 100644 index 0000000..9a4c58b --- /dev/null +++ b/computer-use/package.json @@ -0,0 +1,6 @@ +{ + "name": "@jarvis-qvac/computer-use", + "private": true, + "type": "module", + "main": "session.js" +} diff --git a/computer-use/session.js b/computer-use/session.js new file mode 100644 index 0000000..5d91683 --- /dev/null +++ b/computer-use/session.js @@ -0,0 +1,48 @@ +const MAX_STEPS = 100; + +export class ComputerUseSession { + constructor({ stepsMax = 20, clock = () => Date.now() } = {}) { + this.clock = clock; + this.stepsMax = Math.min(MAX_STEPS, Math.max(1, stepsMax)); + this.active = false; + this.stepsUsed = 0; + this.sessionId = null; + this.backend = 'none'; + this.expiresAt = null; + } + + grant({ persist = false, monitors = 'focused' } = {}) { + this.active = true; + this.stepsUsed = 0; + this.sessionId = `cu_${this.clock().toString(36)}`; + this.expiresAt = this.clock() + 3 * 60 * 1000; + return { session_id: this.sessionId, restore_token_present: Boolean(persist), monitors }; + } + + revoke() { + this.active = false; + this.sessionId = null; + this.expiresAt = null; + this.backend = 'none'; + } + + status() { + return { + active: this.active, + steps_used: this.stepsUsed, + steps_max: this.stepsMax, + grant_expires_at: this.expiresAt, + backend: this.backend, + }; + } + + beginStep() { + if (!this.active) throw new Error('computer-use grant is inactive'); + if (this.expiresAt && this.clock() >= this.expiresAt) { + this.revoke(); + throw new Error('computer-use grant expired'); + } + if (this.stepsUsed >= this.stepsMax) throw new Error('computer-use step budget exhausted'); + this.stepsUsed += 1; + } +} diff --git a/daemon/gpu-doctor.js b/daemon/gpu-doctor.js new file mode 100644 index 0000000..97a1977 --- /dev/null +++ b/daemon/gpu-doctor.js @@ -0,0 +1,20 @@ +import { Agent } from './qvac-master.js'; + +try { + const resources = await Agent.engine.resources(); + const hasGpu = Array.isArray(resources.gpus) && resources.gpus.length > 0; + console.log(JSON.stringify({ + policy: 'gpu-required', + hasGpu, + backendHints: resources.drivers || {}, + gpus: resources.gpus || [], + vramBytes: resources.vramBytes || 0, + }, null, 2)); + if (!hasGpu) { + console.error('gpu-doctor: QVAC did not observe a usable GPU; Jarvis will refuse CPU inference'); + process.exitCode = 1; + } +} catch (error) { + console.error(`gpu-doctor: ${error.message}`); + process.exitCode = 1; +} diff --git a/daemon/harness-bridge.js b/daemon/harness-bridge.js new file mode 100644 index 0000000..3f3b3e1 --- /dev/null +++ b/daemon/harness-bridge.js @@ -0,0 +1,42 @@ +import { EventEmitter } from 'node:events'; +import { acquireQvac, closeQvac, releaseQvac, Agent } from './qvac-master.js'; + +export class HarnessBridge extends EventEmitter { + constructor({ cwd = process.cwd(), model = 'qwen3.5-4b', tools = [], permissionMode = 'ask' } = {}) { + super(); + this.options = { cwd, model, tools, permissionMode, origin: 'jarvis-qvac', system: 'You are Jarvis on a local Ubuntu GNOME desktop. Keep spoken replies short and never claim cloud access.' }; + this.session = null; + } + + async start() { + if (this.session) return this.session; + if (process.env.JARVIS_QVAC_MODEL && this.options.model !== process.env.JARVIS_QVAC_MODEL) { + throw new Error(`Jarvis uses one QVAC master model (${process.env.JARVIS_QVAC_MODEL}); requested ${this.options.model}`); + } + await acquireQvac(); + try { + this.session = await Agent.create(this.options); + } catch (error) { + releaseQvac(); + await closeQvac(); + throw error; + } + for (const event of ['agent_message_chunk', 'tool_call', 'permission', 'ask_user', 'cap-chunk', 'error']) { + this.session.on(event, (payload) => this.emit(event, payload)); + } + return this.session; + } + + async ask(text) { + if (!this.session) await this.start(); + return this.session.prompt(text); + } + + cancel() { this.session?.cancel(); } + + async close() { + await this.session?.dispose(); + releaseQvac(); + await closeQvac(); + } +} diff --git a/daemon/index.js b/daemon/index.js new file mode 100644 index 0000000..0667a8b --- /dev/null +++ b/daemon/index.js @@ -0,0 +1,27 @@ +import { EventEmitter } from 'node:events'; +import { HarnessBridge } from './harness-bridge.js'; +import { ComputerUseSession } from '../computer-use/session.js'; + +export class JarvisDaemon extends EventEmitter { + constructor() { + super(); + this.state = 'ARMED'; + this.computer = new ComputerUseSession(); + this.harness = new HarnessBridge({ cwd: process.cwd() }); + this.harness.on('agent_message_chunk', (ev) => this.emit('Token', ev?.text || ev?.delta || '')); + this.harness.on('permission', (ev) => this.emit('ConfirmationRequired', ev)); + } + + setState(state) { this.state = state; this.emit('StateChanged', state); } + async ask(text) { this.setState('THINKING'); try { const reply = await this.harness.ask(text); this.emit('Reply', reply); return reply; } finally { this.setState('LISTENING'); } } + cancel() { this.harness.cancel(); this.computer.revoke(); this.setState('ARMED'); } + computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); return result; } + computerRevoke() { this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); } + async close() { this.computerRevoke(); await this.harness.close(); } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const daemon = new JarvisDaemon(); + process.on('SIGINT', () => daemon.close().finally(() => process.exit(0))); + console.log('jarvisd scaffold ready; use the D-Bus adapter when installed'); +} diff --git a/daemon/package.json b/daemon/package.json new file mode 100644 index 0000000..6648524 --- /dev/null +++ b/daemon/package.json @@ -0,0 +1,6 @@ +{ + "name": "@jarvis-qvac/daemon", + "private": true, + "type": "module", + "main": "index.js" +} diff --git a/daemon/qvac-master.js b/daemon/qvac-master.js new file mode 100644 index 0000000..8139ac9 --- /dev/null +++ b/daemon/qvac-master.js @@ -0,0 +1,54 @@ +import { createRequire } from 'node:module'; +import path from 'node:path'; + +const harnessPath = process.env.JARVIS_HARNESS_PATH || '/home/raven/dev/agent-harness'; +const require = createRequire(import.meta.url); +process.env.QVAC_CONFIG_PATH ||= path.resolve(new URL('../qvac.config.json', import.meta.url).pathname); +const Agent = require(path.join(harnessPath, 'index.js')); + +let loadPromise = null; +let ownerCount = 0; + +export const QVAC_MASTER = Object.freeze({ + configPath: process.env.QVAC_CONFIG_PATH, + model: process.env.JARVIS_QVAC_MODEL || 'qwen3.5-4b', + device: 'gpu', + gpuLayers: 99, +}); + +export async function acquireQvac() { + ownerCount += 1; + if (!loadPromise) { + loadPromise = Agent.engine.load({ + model: QVAC_MASTER.model, + tools: true, + device: 'gpu', + gpu_layers: QVAC_MASTER.gpuLayers, + mmprojUseGpu: true, + }).then((loaded) => { + if (loaded.device !== 'gpu') { + throw new Error(`Jarvis requires GPU QVAC inference; loaded device was ${loaded.device}`); + } + return loaded; + }).catch((error) => { + loadPromise = null; + ownerCount = Math.max(0, ownerCount - 1); + throw error; + }); + } + return loadPromise; +} + +export function releaseQvac() { ownerCount = Math.max(0, ownerCount - 1); } + +export async function closeQvac() { + if (ownerCount > 0) return; + loadPromise = null; + await Agent.engine.close(); +} + +export function qvacStatus() { + return { ...QVAC_MASTER, owners: ownerCount, loaded: Agent.engine.getLoaded() }; +} + +export { Agent }; diff --git a/dbus/io.qvac.Jarvis.Session.xml b/dbus/io.qvac.Jarvis.Session.xml new file mode 100644 index 0000000..fdd1606 --- /dev/null +++ b/dbus/io.qvac.Jarvis.Session.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md new file mode 100644 index 0000000..8b8a8d5 --- /dev/null +++ b/docs/ROADMAP.md @@ -0,0 +1,234 @@ +# JARVIS-QVAC roadmap + +This is the implementation tracker for the local Ubuntu GNOME assistant. The +roadmap is ordered by dependency: runtime authority first, then cognition, +voice, desktop observation, actuation, UI, capability coverage, hardening, and +packaging. + +## Product contract + +Jarvis is local-first and session-scoped. The GNOME Shell extension stays thin; +all inference, audio, jobs, memory, and computer-use decisions run outside +gnome-shell. The extracted `/home/raven/dev/agent-harness` remains the sole +cognitive loop and is wrapped rather than rewritten. + +## Non-negotiable runtime architecture + +- One user service, `jarvisd`, owns the QVAC SDK worker. +- One `QVAC_CONFIG_PATH` points every Jarvis process at the same + `qvac.config.json`. +- One model profile is active at a time. The QVAC master owns load, unload, + suspend, resume, cancellation, and model status. +- Harness sessions call the master; skills never import QVAC independently and + never call `loadModel()` directly. +- `qvac serve --openai` is an optional diagnostic/API surface and is never + started by `jarvisd`; it must not run beside the master in production. +- GPU is mandatory for Jarvis runtime inference. The bridge requests GPU, + enables all GPU layers and GPU multimodal projection, and rejects a load + that QVAC reports as CPU. There is no silent CPU fallback. +- Audio capture, TTS, OCR, embeddings, RAG, media jobs, and multimodal work + use the same QVAC master scheduler and obey a GPU memory budget. +- A capability may report `unavailable: gpu-required` without weakening the + GPU policy or starting a competing runtime. + +## Status legend + +- `[x]` implemented and locally verified +- `[~]` scaffolded or partially implemented +- `[ ]` planned + +## Phase 0 — repository and authority bootstrap + +- [x] Confirm `/home/raven/dev/agent-harness` exists. +- [x] Link `vendor/agent-harness` to the extracted harness. +- [x] Document harness layout, entrypoints, sessions, tools, memory, planner, + and model path in `docs/agent-harness-map.md`. +- [x] Pin the QVAC 0.19.0 API rules in `QVAC.md`. +- [x] Create Node 22.17+ workspaces. +- [x] Add one root `qvac.config.json` with localhost server settings and a + single model alias. +- [x] Add the initial daemon, computer-use, GNOME extension, D-Bus, and + control-center boundaries. +- [ ] Install and lock dependencies with a successful `npm install`. +- [ ] Add repository CI for syntax, unit, schema, and extension checks. + +Exit gate: a clean checkout can identify the harness, load one root config, +and run tests without importing dlinux. + +## Phase 1 — single GPU QVAC master + +- [x] Add `daemon/qvac-master.js` as the only Jarvis QVAC owner. +- [x] Serialize model loading and share the resulting harness engine. +- [x] Request `device: "gpu"`, `gpu_layers: 99`, and GPU multimodal projection. +- [x] Reject a CPU result instead of accepting QVAC's internal fallback. +- [x] Add owner counting and a single close path. +- [x] Set `QVAC_CONFIG_PATH`, `JARVIS_QVAC_MODEL`, and GPU policy in the user + service. +- [~] Add `gpu-doctor`; extend it to report QVAC resource capabilities, backend, + driver, device name, VRAM, model fit, and the exact reason for failure. +- [ ] Add master scheduler lanes: interactive voice, computer-use vision, + background media, and maintenance. +- [ ] Add VRAM admission control and queue media jobs instead of OOMing the + voice lane. +- [ ] Add model profile selection using `assessModelFit()` before downloads. +- [ ] Add master-level cancellation by request ID and job ID. +- [ ] Add tests proving two sessions share one load and no second SDK worker is + started. + +Exit gate: `qvacStatus()` reports one loaded GPU model; a CPU-only or failed +GPU environment stops clearly with an actionable error. + +## Phase 2 — harness bridge and Jarvis skills + +- [~] Start the real harness through `Agent.create()` and route token events. +- [ ] Add the voice-native system prompt and structured HUD sidecar. +- [ ] Register all Jarvis tools through the harness custom-tool registry. +- [ ] Implement permission classes: read, write, dangerous, and computer-use. +- [ ] Connect confirmation events to spoken confirmation and HUD controls. +- [ ] Add desktop tools: launch/list apps, focus/list windows, workspaces, + notify, screenshot, clipboard, media, settings, and focused text injection. +- [ ] Add file search/read/write with trash-first destructive handling. +- [ ] Add memory and RAG workspace tools. +- [ ] Add QVAC wrappers for embeddings, translation, OCR, classification, + image/video/music jobs, transcription, TTS, LoRA, BCI, VLA, and ABot-World. +- [ ] Ensure every wrapper obtains the master lease and never loads QVAC itself. +- [ ] Add fake-QVAC fixture tests for every tool schema and permission gate. + +Exit gate: a typed prompt completes through the real harness, streams tokens, +executes a Jarvis tool, and returns a short local response through the same +GPU-owned worker. + +## Phase 3 — daemon lifecycle and D-Bus + +- [~] Define `io.qvac.Jarvis.Session` XML. +- [ ] Implement `Arm`, `Sleep`, `Shutdown`, `Say`, `Ask`, `Cancel`, `SetMode`, + and state queries. +- [ ] Implement token, transcript, reply, audio-level, chip, job, computer + step, and error signals. +- [ ] Keep payloads small; stream PCM and screenshots through a Unix socket or + tmpfs paths. +- [ ] Add lock-screen handling: mute, hide UI, revoke computer use. +- [ ] Add idle sleep using QVAC `suspend()` and resume on wake. +- [ ] Add structured JSON logging with no prompt/audio/image contents by default. + +Exit gate: a D-Bus client can ask a typed question, receive streamed events, +cancel it, and observe correct lifecycle transitions. + +## Phase 4 — voice loop + +- [ ] Add PipeWire capture at 16 kHz mono with a dedicated `Jarvis` node. +- [ ] Add wake-word engine behind a `WakeEngine` interface. +- [ ] Add VAD segmentation and the documented QVAC ASR stream. +- [ ] Implement `ARMED → LISTENING → THINKING → SPEAKING → LISTENING`. +- [ ] Add transcript filtering, TTS anti-feedback gate, and playback cooldown. +- [ ] Add sentence buffering from streamed harness output into QVAC TTS. +- [ ] Add fast-path cancel, sleep, privacy, dictate, screen, and computer-use + commands. +- [ ] Add push-to-talk and typed fallback. +- [ ] Add wake false-accept/false-reject and feedback measurements. + +Exit gate: “Hey Jarvis” starts a local GPU-backed turn, speaks a response, and +does not self-trigger from its own TTS. + +## Phase 5 — GNOME ARC surface + +- [~] Add GNOME 45+ ESM extension metadata, panel indicator, and settings keys. +- [ ] Add async D-Bus client and panel state glyphs. +- [ ] Add ARC overlay, waveform, transcript rows, chips, local/model status, + reduced motion, high contrast, keyboard navigation, and screen-reader labels. +- [ ] Add listening halo, privacy slash, job theater, target highlights, and + visible computer-use cursor. +- [ ] Keep all extension work nonblocking and free of QVAC/native inference. + +Exit gate: enabling/disabling the extension never starts inference in the Shell +process and reflects daemon state without jank. + +## Phase 6 — computer-use observe and semantic action + +- [ ] Complete `cu-doctor`: Wayland/X11, portal, PipeWire, AT-SPI, libei, and + optional fallback detection. +- [ ] Implement portal ScreenCast/Screenshot and tmpfs frame normalization. +- [ ] Implement GNOME Shell window truth and focus methods. +- [ ] Implement AT-SPI tree snapshots with per-step stable refs. +- [ ] Implement OCR and QVAC multimodal observe bundles. +- [ ] Implement `cu.observe`, `cu.zoom`, `cu.tree`, and `cu.find`. +- [ ] Implement `cu.act` and semantic `cu.click`. +- [ ] Implement grant/revoke, expiry, step budget, audit hashes, and no-frame + retention by default. + +Exit gate: “what’s on my screen?” returns grounded local observations without +actuation. + +## Phase 7 — computer-use portal actuation + +- [ ] Implement RemoteDesktop portal consent and restore tokens. +- [ ] Implement libei/EIS pointer, keyboard, scroll, drag, hover, and key input. +- [ ] Implement typed Unicode and submit behavior. +- [ ] Add target preview, agent cursor, and step ticker. +- [ ] Prefer domain tools, app D-Bus, AT-SPI, Shell helper, then vision + coordinates in that order. +- [ ] Refuse password/PAM roles and lock-screen/greeter actions. +- [ ] Require confirmation for destructive or high-impact actions. +- [ ] Keep ydotool and X11 tools disabled unless explicitly enabled. +- [ ] Add state-change self-checks, animation waits, and no-progress aborts. + +Exit gate: Night Light, Text Editor save, Firefox URL entry, hands-off abort, +and lock-screen kill pass `docs/cu-acceptance.md`. + +## Phase 8 — GTK4/libadwaita control center + +- [ ] Implement General, Voice, Models, Memory, Skills, Computer use, Privacy, + Lab, and About pages. +- [ ] Show the single QVAC master status, GPU backend, VRAM, model, fit result, + queue, and failure reason. +- [ ] Make every capability visible even when its model does not fit. +- [ ] Add model download pause/resume through the master. +- [ ] Add voice enrollment and local preview. +- [ ] Add RAG workspace management, retention controls, audit export/delete, + and computer-use permissions. + +Exit gate: a user can configure the complete system without editing JSON. + +## Phase 9 — full QVAC capability coverage + +- [ ] Chat, plan, summarize, rewrite, code, embeddings, RAG, batch prompts. +- [ ] Multimodal screenshot/file analysis, OCR, classification. +- [ ] Image generation/editing, video jobs, music jobs. +- [ ] ASR, diarized meetings, TTS, voice clone enrollment, translation relay. +- [ ] LoRA training with explicit confirmation and overnight job controls. +- [ ] BCI, VLA, and ABot-World as clearly labeled Lab capabilities. +- [ ] Registry/model manager, checksums, fit assessments, profiler, and job + cancellation. +- [ ] Optional P2P model fetch and memory sync only after local mode is stable. + +Exit gate: every capability in the product inventory has a working skill or a +truthful GPU/model-unavailable state. + +## Phase 10 — reliability, privacy, and performance + +- [ ] OOM isolation: failed media jobs cannot kill the voice lane. +- [ ] GPU telemetry: utilization, VRAM, queue wait, load time, tokens/sec, + ASR latency, TTS latency, and dropped audio. +- [ ] Wake, feedback, Wayland/X11, accessibility, lock/unlock, crash/restart, + and nested Shell smoke tests. +- [ ] Verify no network calls except explicitly enabled model fetch/P2P paths. +- [ ] Verify no screenshots, microphone buffers, prompts, or transcripts are + retained unless the user enables retention. +- [ ] Audit computer actions with target metadata and screenshot hashes only. +- [ ] Add graceful restart and state recovery for daemon crashes. + +Exit gate: the full test matrix passes on supported Ubuntu GNOME sessions with +GPU inference visibly confirmed. + +## Phase 11 — packaging and release + +- [ ] Build a `.deb`, extension bundle, and user-service installer. +- [ ] Add first-run wizard for microphone, wake phrase, GPU/model profile, TTS, + and a typed smoke test. +- [ ] Add uninstall that removes only Jarvis-owned data after explicit choice. +- [ ] Publish a hardware compatibility matrix for Vulkan/GPU backends. +- [ ] Add release checklist, migration notes, and a reproducible local demo. + +Definition of done: all product behaviors in the specification work on Ubuntu +GNOME with one GPU-backed QVAC master, one harness planner, explicit computer- +use grants, and truthful capability status. diff --git a/docs/agent-harness-map.md b/docs/agent-harness-map.md new file mode 100644 index 0000000..58a1110 --- /dev/null +++ b/docs/agent-harness-map.md @@ -0,0 +1,78 @@ +# Agent harness map + +Source of truth: `/home/raven/dev/agent-harness` (`~/dev/agent-harness`). +The extracted directory is not currently a git checkout, so Jarvis tracks it by +symlink and records the absolute source path instead of inventing a commit pin. + +## Layout and runtime + +The harness is a standalone Node.js CommonJS package. Its public entrypoint is +`index.js`, with the CLI at `bin/cli.js`. The README says Node 20 is enough for +the harness itself, while its QVAC inference dependency requires Node 22.17+; +Jarvis therefore requires Node 22.17+. + +Important modules: + +- `index.js`: public `Agent.create()` / `Agent.load()`, session wrapper, engine and catalog exports. +- `agent/loop.js`: sample → tool call → tool result → repeat loop, permissions, plan mode, compaction, memory, and subagents. +- `agent/custom-tools.js`: per-session JSON-schema tool registry and in-process `execute` handlers. +- `agent/tools.js`: built-in host workspace, memory, planning, web, task, and MCP tools. +- `agent/sessions.js`: persisted session summaries, history, updates, and plan files. +- `agent/memory.js`: local short/long-term memory notes. +- `lib/qvac.js`: lazy `@qvac/sdk` import, model loading, completion streaming, vision attachments, cancellation, and lifecycle close. +- `lib/catalog.js`: friendly model ids mapped to QVAC SDK constants. + +## Start and embed + +CLI: `node /home/raven/dev/agent-harness/bin/cli.js [options] [prompt]`. + +Library integration uses CommonJS from the harness root: + +```js +const Agent = require('/home/raven/dev/agent-harness'); +await Agent.engine.load({ model: 'qwen3.5-4b', tools: true, device: 'auto' }); +const session = await Agent.create({ + cwd: process.cwd(), + model: 'qwen3.5-4b', + permissionMode: 'ask', + tools: [{ name, description, parameters, execute }], +}); +await session.prompt('hello'); +await session.dispose(); +await Agent.engine.close(); +``` + +Sessions emit `agent_message_chunk`, `tool_call`, `permission`, `ask_user`, +plan, context, and related loop events. Jarvis adapts the message and tool +events to its D-Bus contract. `session.permit()`, `session.answer()`, +`session.cancel()`, and `session.addTool()` are the control points. + +## Tool registry and permissions + +Jarvis registers tools through `Agent.create({ tools })`. Each custom tool is a +JSON-schema object with an optional in-process `execute(args)` handler. The +harness caps custom tools at 32 and reserves its built-ins. Permission mode is +`ask` by default; write and dangerous desktop/computer-use tools remain behind +Jarvis confirmation gates. + +## Session, memory, and planner + +`agent/loop.js` owns the multi-turn agent loop and repeats completion/tool +execution until a final response. `sessions.js` persists conversation state. +`memory.js` provides local memory tools. `plan-mode.js`, `compaction.js`, +`goal.js`, and `tasks.js` provide planning, context management, goals, and +subagents. Jarvis does not create a second planner. + +## Model connection + +The harness does not use an OpenAI-compatible HTTP URL internally. Its +`lib/qvac.js` imports `@qvac/sdk` in-process and calls `loadModel()` and +`completion()`. Jarvis keeps the optional sibling `qvac serve --openai` +provider for external clients and diagnostics, while the cognition bridge +wraps the harness in-process as required by the extracted package. + +## Computer use + +The inspected harness has no first-class portal, AT-SPI, libei, or desktop +computer-use backend. Jarvis therefore supplies those as custom tools from +`computer-use/` and `skills/`, while keeping planning in `agent/loop.js`. diff --git a/docs/cu-acceptance.md b/docs/cu-acceptance.md new file mode 100644 index 0000000..5ea5ef5 --- /dev/null +++ b/docs/cu-acceptance.md @@ -0,0 +1,12 @@ +# Computer-use acceptance + +Run these on Ubuntu GNOME after `npm run cu-doctor` reports the required portal +and PipeWire services. The current scaffold only exposes the grant/session +budget boundary; portal, AT-SPI, and libei adapters are the next implementation +step. + +1. Observe Settings and toggle Night Light. +2. Type a sentence into Text Editor and save it. +3. Open Firefox, locate the address bar, type a URL, and press Enter. +4. Say “hands off” during a run; all actuation must stop. +5. Lock the screen during a run; the grant must die immediately. diff --git a/package.json b/package.json new file mode 100644 index 0000000..1903800 --- /dev/null +++ b/package.json @@ -0,0 +1,23 @@ +{ + "name": "jarvis-qvac", + "version": "0.1.0", + "private": true, + "description": "Local-first GNOME voice assistant powered by QVAC and agent-harness.", + "type": "module", + "engines": { "node": ">=22.17.0" }, + "workspaces": ["daemon", "computer-use", "apps/control-center"], + "scripts": { + "start": "node daemon/index.js", + "test": "node --test", + "cu-doctor": "node computer-use/doctor.js", + "gpu-doctor": "node daemon/gpu-doctor.js", + "qvac:doctor": "qvac doctor", + "qvac:serve": "qvac serve --openai --host 127.0.0.1", + "qvac:status": "node -e \"import('./daemon/qvac-master.js').then(({qvacStatus})=>console.log(JSON.stringify(qvacStatus(), null, 2)))\"" + }, + "dependencies": { + "@qvac/cli": "0.19.0", + "@qvac/sdk": "0.19.0", + "dbus-next": "^0.10.2" + } +} diff --git a/qvac.config.json b/qvac.config.json new file mode 100644 index 0000000..6012ef8 --- /dev/null +++ b/qvac.config.json @@ -0,0 +1,15 @@ +{ + "cacheDirectory": "/home/raven/.cache/jarvis/models", + "serve": { + "host": "127.0.0.1", + "port": 11434, + "models": { + "jarvis-chat": { + "model": "QWEN3_600M_INST_Q4", + "default": true, + "preload": false, + "config": { "ctx_size": 8192 } + } + } + } +} diff --git a/skills/catalog.js b/skills/catalog.js new file mode 100644 index 0000000..a4de257 --- /dev/null +++ b/skills/catalog.js @@ -0,0 +1,20 @@ +export const SKILLS = [ + ['chat', 'Local QVAC completion through the agent harness', 'read'], + ['plan', 'Multi-step planning through the extracted harness loop', 'read'], + ['embed', 'QVAC embeddings for local memory and retrieval', 'read'], + ['remember', 'QVAC RAG workspace ingestion and search', 'write'], + ['look-at-this', 'Local screenshot and QVAC multimodal analysis', 'read'], + ['dictate', 'Transcribe and inject text into the focused client', 'write'], + ['translate', 'Local QVAC/Bergamot translation', 'read'], + ['imagine', 'Local QVAC diffusion image generation', 'write'], + ['make-video', 'Local QVAC video generation job', 'write'], + ['compose', 'Local QVAC AudioGen music job', 'write'], + ['speak', 'Local QVAC text-to-speech playback', 'read'], + ['assess-model-fit', 'Preflight model memory fit before download', 'read'], + ['cu.observe', 'Wayland-first desktop observation bundle', 'computer-use'], + ['cu.act', 'AT-SPI semantic desktop action', 'computer-use'], + ['cu.click', 'Portal/libei or opted-in fallback click', 'computer-use'], + ['cu.type', 'Portal/libei or opted-in fallback Unicode typing', 'computer-use'], + ['cu.grant', 'Explicit computer-use grant', 'computer-use'], + ['cu.revoke', 'Immediate computer-use revoke', 'dangerous'], +]; diff --git a/systemd/user/jarvisd.service b/systemd/user/jarvisd.service new file mode 100644 index 0000000..d074591 --- /dev/null +++ b/systemd/user/jarvisd.service @@ -0,0 +1,18 @@ +[Unit] +Description=Jarvis QVAC local voice assistant +After=pipewire.service wireplumber.service graphical-session.target +Wants=pipewire.service + +[Service] +Type=simple +WorkingDirectory=%h/dev/jarvis +ExecStart=/usr/bin/node %h/dev/jarvis/daemon/index.js +Environment=QVAC_CONFIG_PATH=%h/dev/jarvis/qvac.config.json +Environment=JARVIS_GPU_REQUIRED=1 +Environment=JARVIS_QVAC_MODEL=qwen3.5-4b +Environment=JARVIS_QVAC_OWNER=jarvisd +Restart=on-failure +RestartSec=2 + +[Install] +WantedBy=graphical-session.target diff --git a/test/computer-use.test.js b/test/computer-use.test.js new file mode 100644 index 0000000..0ba745d --- /dev/null +++ b/test/computer-use.test.js @@ -0,0 +1,26 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { ComputerUseSession } from '../computer-use/session.js'; + +test('computer use requires an explicit grant and enforces its step budget', () => { + let now = 1000; + const session = new ComputerUseSession({ stepsMax: 2, clock: () => now }); + assert.throws(() => session.beginStep(), /inactive/); + assert.deepEqual(session.grant({ persist: true }).restore_token_present, true); + session.beginStep(); + session.beginStep(); + assert.throws(() => session.beginStep(), /budget/); + assert.equal(session.status().steps_used, 2); + session.revoke(); + assert.equal(session.status().active, false); + now += 1; +}); + +test('computer use expires its wall clock grant', () => { + let now = 0; + const session = new ComputerUseSession({ clock: () => now }); + session.grant(); + now = 180001; + assert.throws(() => session.beginStep(), /expired/); + assert.equal(session.status().backend, 'none'); +}); diff --git a/vendor/agent-harness b/vendor/agent-harness new file mode 120000 index 0000000..1cb86f1 --- /dev/null +++ b/vendor/agent-harness @@ -0,0 +1 @@ +/home/raven/dev/agent-harness \ No newline at end of file