From 25b2c4be2ea92e7aae4b7dde72e27c1091b9a9bf Mon Sep 17 00:00:00 2001 From: Raven Scott Date: Fri, 11 Sep 2026 14:55:34 -0400 Subject: [PATCH] R1 --- .gitea/workflows/ci.yml | 20 +++ .gitea/workflows/rolling-release.yml | 28 +++ README.md | 250 ++++++++++++++++++++++++--- docs/MIGRATIONS.md | 15 ++ docs/README.md | 56 ++++++ docs/RELEASE.md | 29 ++++ docs/ROADMAP.md | 14 +- docs/api.md | 43 +++++ docs/architecture.md | 80 +++++++++ docs/capabilities.md | 35 ++++ docs/ci-gitea.md | 46 +++++ docs/computer-use.md | 57 ++++++ docs/development.md | 47 +++++ docs/hardware-compatibility.md | 20 +++ docs/harness-integration.md | 48 +++++ docs/local-demo.md | 25 +++ docs/operations.md | 57 ++++++ docs/qvac-runtime.md | 68 ++++++++ docs/security-privacy.md | 44 +++++ docs/testing.md | 33 ++++ docs/troubleshooting.md | 46 +++++ docs/voice-pipeline.md | 44 +++++ package.json | 8 +- packaging/build-deb.sh | 25 +++ packaging/build-extension.sh | 9 + packaging/build-release.sh | 10 ++ packaging/first-run.sh | 46 +++++ packaging/install.sh | 25 +++ packaging/jarvisd.service | 17 ++ packaging/publish-gitea-release.sh | 39 +++++ packaging/qvac.config.template.json | 15 ++ packaging/test-package.sh | 14 ++ packaging/uninstall.sh | 14 ++ 33 files changed, 1293 insertions(+), 34 deletions(-) create mode 100644 .gitea/workflows/ci.yml create mode 100644 .gitea/workflows/rolling-release.yml create mode 100644 docs/MIGRATIONS.md create mode 100644 docs/README.md create mode 100644 docs/RELEASE.md create mode 100644 docs/api.md create mode 100644 docs/architecture.md create mode 100644 docs/capabilities.md create mode 100644 docs/ci-gitea.md create mode 100644 docs/computer-use.md create mode 100644 docs/development.md create mode 100644 docs/hardware-compatibility.md create mode 100644 docs/harness-integration.md create mode 100644 docs/local-demo.md create mode 100644 docs/operations.md create mode 100644 docs/qvac-runtime.md create mode 100644 docs/security-privacy.md create mode 100644 docs/testing.md create mode 100644 docs/troubleshooting.md create mode 100644 docs/voice-pipeline.md create mode 100755 packaging/build-deb.sh create mode 100755 packaging/build-extension.sh create mode 100755 packaging/build-release.sh create mode 100755 packaging/first-run.sh create mode 100755 packaging/install.sh create mode 100644 packaging/jarvisd.service create mode 100755 packaging/publish-gitea-release.sh create mode 100644 packaging/qvac.config.template.json create mode 100755 packaging/test-package.sh create mode 100755 packaging/uninstall.sh diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..81256fd --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,20 @@ +name: CI + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: npm test + - run: python3 -m py_compile apps/control-center/main.py computer-use/py/*.py + - run: npm run package:test diff --git a/.gitea/workflows/rolling-release.yml b/.gitea/workflows/rolling-release.yml new file mode 100644 index 0000000..9bb9fb2 --- /dev/null +++ b/.gitea/workflows/rolling-release.yml @@ -0,0 +1,28 @@ +name: Rolling release + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: npm test + - run: python3 -m py_compile apps/control-center/main.py computer-use/py/*.py + - run: npm run package + - name: Publish rolling prerelease to Gitea + env: + GITEA_SERVER_URL: ${{ gitea.server_url }} + GITEA_REPOSITORY: ${{ gitea.repository }} + GITEA_SHA: ${{ gitea.sha }} + RELEASE_TOKEN: ${{ secrets.RELEASE_TOKEN }} + run: bash packaging/publish-gitea-release.sh diff --git a/README.md b/README.md index ba18881..6f38fd3 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,232 @@ # JARVIS-QVAC -Local-first Ubuntu GNOME voice assistant. The cognitive core is the -already-extracted `/home/raven/dev/agent-harness`, copied into -`vendor/agent-harness`; Jarvis does not copy or depend on `/home/raven/dev/dlinux`. +JARVIS-QVAC is a local-first voice assistant and computer-use layer for Ubuntu +GNOME. It combines a wake-word loop, QVAC speech and language models, the +copied agent-harness planner, GNOME-native UI, local memory, and an explicit +Wayland computer-use grant. -## Bootstrap +The implementation covers Phases 0–11: daemon lifecycle, GPU-gated QVAC +ownership, voice, ARC UI, computer use, control center, capability adapters, +privacy hardening, packaging, and Gitea release automation. Full production +acceptance remains host-dependent on GPU visibility, GNOME portals, the local +wake bridge, and EIS input. -```bash -npm install +## Runtime contract + +JARVIS-QVAC has one cognitive loop and one QVAC authority. + +- jarvisd owns the QVAC SDK worker, model lifecycle, GPU policy, scheduler, + cancellation, voice loop, jobs, and D-Bus service. +- vendor/agent-harness is a copied repository dependency. It owns sessions, + planning, memory, and the tool loop. +- Skills call the QVAC master facade. They do not create workers or load models. +- The GNOME Shell extension is presentation-only: no inference, audio capture, + or input injection runs inside GNOME Shell. +- GPU inference is mandatory. If QVAC cannot see a usable GPU, Jarvis reports + an unavailable state and refuses CPU fallback. +- Computer use is off until explicitly granted and is revoked on Escape, lock + screen, expiry, or a spoken abort command. + +~~~mermaid +flowchart LR + U[User] --> G[GNOME Shell / ARC] + G <--> D[jarvisd / Node 22] + D --> H[agent-harness] + D --> M[Single QVAC master] + D --> V[Voice + PipeWire] + D --> C[Computer-use session] + M --> GPU[(GPU backend)] + C --> P[XDG portals + AT-SPI + EIS] +~~~ + +## Capabilities + +- Wake word, push-to-talk, typed fallback, VAD, local ASR, streamed replies, + TTS playback, anti-feedback gating, and cancellation. +- Chat, planning, code, summaries, rewriting, embeddings, RAG workspaces, + batch prompts, OCR, image classification, translation, and multimodal input. +- Image, video, and music generation through serialized local QVAC jobs. +- Voice enrollment, diarized transcription, LoRA training, and labeled BCI, + VLA, and ABot-World lab slots. +- GNOME app launch/focus/window actions, clipboard, notifications, media, + screenshot observation, AT-SPI actions, portal input, and vision fallback. +- ARC panel indicator and overlay, live waveform, transcripts, suggestions, + job progress, target highlights, and visible agent cursor. + +Unavailable capabilities remain visible with an honest reason such as missing +GPU, insufficient VRAM, missing model, unavailable hardware, or missing +desktop permission. + +## Requirements + +Full runtime support requires: + +- Ubuntu GNOME 24.04 or newer, GNOME Shell 46+, and a logged-in user session. +- Node.js 22.17 or newer. +- PipeWire and WirePlumber. +- xdg-desktop-portal with the GNOME backend. +- AT-SPI2 and, for primary Wayland actuation, libei/libeis plus a local EIS + injector. +- A QVAC-visible Vulkan, CUDA, or supported GPU backend. +- A local openWakeWord or sherpa-onnx bridge for always-on wake detection. + +ydotool is an optional legacy fallback and is disabled unless explicitly +enabled. X11 fallback is used only in an Xorg session. CPU-only systems can run +diagnostics and control surfaces, but Jarvis will not run inference on them. + +See [hardware compatibility](docs/hardware-compatibility.md) and +[troubleshooting](docs/troubleshooting.md). + +## Install from a checkout + +The harness is copied into vendor/agent-harness. It is not a symlink and the +dlinux platform is not vendored. + +~~~bash +git clone jarvis-qvac +cd jarvis-qvac +npm ci npm test -npm run cu-doctor npm run gpu-doctor -npx qvac doctor -npm run qvac:serve:diagnostic # optional; never run beside jarvisd -``` +npm run cu-doctor +npm run voice-doctor +~~~ -The daemon bridge loads the harness in-process with `Agent.engine.load()` and -creates sessions with `Agent.create()`. The optional QVAC HTTP sibling is -configured for `127.0.0.1:11434`. The GNOME extension is an intentionally thin -ESM panel shell; inference, audio, portals, and computer-use actuation belong -in user services. +Install the user service, extension, and per-user QVAC configuration: -GPU inference is a hard requirement. `gpu-doctor` must report a QVAC-visible -GPU and `jarvisd` refuses to continue if the loaded model reports `device: cpu`. -This prevents a misleading CPU mode and competing QVAC workers. +~~~bash +bash packaging/install.sh --enable +bash packaging/first-run.sh +gnome-extensions enable jarvis@qvac.local +~~~ -The repository contains the daemon, single GPU-only QVAC master, copied agent -harness, voice loop, GNOME ARC extension, GTK4/libadwaita control center, -computer-use observation and safety boundaries, and the full QVAC capability -skill registry. See [docs/ROADMAP.md](docs/ROADMAP.md), -[docs/agent-harness-map.md](docs/agent-harness-map.md), and -[docs/cu-acceptance.md](docs/cu-acceptance.md). +The first-run flow checks GPU/QVAC and PipeWire, records the wake phrase and +model profile, optionally previews TTS, and sends a typed smoke-test prompt +through the session bus. -Runtime inference refuses to start until the QVAC SDK reports a usable GPU. -The RemoteDesktop/libei input bridge also remains unavailable until the local -GNOME portal session and a real EIS injector are present; Jarvis never claims -that an action succeeded when the backend is missing. +Inspect the installed daemon with: + +~~~bash +systemctl --user status jarvisd.service +journalctl --user -u jarvisd.service -f +~~~ + +The service uses the application directory +~/.local/share/jarvis-qvac, configuration under ~/.config/jarvis, model cache +under the user's cache directory, and user data paths described in +[Operations](docs/operations.md). + +## Development commands + +~~~bash +npm test # Node tests, including the vendored harness test +npm run gpu-doctor # QVAC GPU, VRAM, and profile diagnostics +npm run cu-doctor # Wayland/X11, portals, AT-SPI, and libei +npm run voice-doctor # PipeWire, wake bridge, and playback diagnostics +npm run qvac:doctor # QVAC installation diagnostics +npm run qvac:status # Single-master runtime status +npm run package:test # Build and inspect release artifacts +npm run first-run # Interactive first-run setup +npm run start # Start jarvisd in the current session +~~~ + +The optional QVAC OpenAI-compatible server is diagnostic infrastructure only. +If used, it binds to 127.0.0.1 and must not become a second production +inference owner beside jarvisd. + +## Computer use + +Computer use is a mode, not default chat behavior. Say “take the wheel” or +select Computer in ARC to request a grant. Jarvis then prefers domain actions, +app D-Bus/GIO, AT-SPI references, GNOME Shell helpers, and finally vision +coordinates. RemoteDesktop plus EIS/libei is the primary Wayland path. + +Configure the local injector before live actuation: + +~~~bash +export JARVIS_LIBEI_BRIDGE="" +~~~ + +The helper will not claim input readiness without this bridge. Password fields, +lock screens, dangerous keys, destructive targets, and actions outside the +step budget are blocked or require confirmation. See the +[computer-use acceptance procedure](docs/cu-acceptance.md). + +## Privacy and safety + +- No cloud API is required and no telemetry is enabled. +- Microphone audio is ring-buffered in memory; raw audio is not saved by + default. +- Computer-use frames stay local in temporary storage and are wiped on revoke + unless trace retention is enabled. +- Audit records contain action metadata and screenshot hashes, not pixels. +- QVAC is localhost-only. Model fetch and optional P2P features are explicit + opt-ins. +- The daemon runs as the user and must never run as root. + +Read [Security and privacy](docs/security-privacy.md) before enabling legacy +input, retention, synchronization, or computer-use traces. + +## Build and release + +Build the Debian package, extension bundle, and checksums locally: + +~~~bash +npm ci +npm test +npm run package +cat dist/SHA256SUMS +~~~ + +Artifacts in dist are: + +- jarvis-qvac__.deb +- jarvis-qvac-extension.zip +- SHA256SUMS + +Gitea Actions workflows live under .gitea/workflows. CI runs tests, Python +checks, and package validation. A push to main additionally builds artifacts +and replaces the mutable rolling prerelease using the repository secret +RELEASE_TOKEN. Pull requests and non-main pushes do not publish a release. + +See [Gitea CI and rolling releases](docs/ci-gitea.md) and the +[release checklist](docs/RELEASE.md). + +## Uninstall and migration + +The uninstall command requires an explicit flag and preserves user data: + +~~~bash +bash packaging/uninstall.sh --yes +~~~ + +To remove Jarvis-owned memory, cache, and generated media as well: + +~~~bash +bash packaging/uninstall.sh --yes --purge-data +~~~ + +Read [Migration notes](docs/MIGRATIONS.md) before upgrading or moving an +installation. + +## Documentation + +The [documentation index](docs/README.md) is the authoritative guide. + +- [Architecture](docs/architecture.md) +- [QVAC runtime and GPU authority](docs/qvac-runtime.md) +- [Agent harness integration](docs/harness-integration.md) +- [Voice pipeline](docs/voice-pipeline.md) +- [Computer use](docs/computer-use.md) +- [API reference](docs/api.md) +- [Operations](docs/operations.md) +- [Testing](docs/testing.md) +- [Troubleshooting](docs/troubleshooting.md) +- [Roadmap](docs/ROADMAP.md) + +## Project status + +Automated checks and repository implementation cover Phases 0–11. Before +calling the deployment production-ready on a particular Ubuntu host, run the +doctor commands and confirm QVAC reports the intended GPU, the wake bridge is +configured, and the portal/EIS path is available. diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md new file mode 100644 index 0000000..d596fb9 --- /dev/null +++ b/docs/MIGRATIONS.md @@ -0,0 +1,15 @@ +# Migration notes + +The user installer keeps application files in +`~/.local/share/jarvis-qvac`, the service in +`~/.config/systemd/user/jarvisd.service`, and the generated QVAC config in +`~/.config/jarvis/qvac.config.json`. Memory, voice references, audit logs, and +generated media remain under their existing Jarvis data directories. + +Upgrades replace only the application and extension files. Existing config and +data are preserved. Run `systemctl --user daemon-reload` after installing a new +service file, then restart `jarvisd.service`. + +The uninstall command requires `--yes` and preserves data. Add +`--purge-data` only when the user explicitly wants Jarvis-owned memory, cache, +and generated media removed. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..ba19034 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,56 @@ +# JARVIS-QVAC documentation + +JARVIS-QVAC is a local-first voice and computer-use assistant for Ubuntu GNOME. +This directory is the maintained technical reference for the repository. + +## Start here + +| Need | Document | +| --- | --- | +| Understand the system | [Architecture](architecture.md) | +| Run it locally | [Operations](operations.md) and [Local demo](local-demo.md) | +| Understand the voice loop | [Voice pipeline](voice-pipeline.md) | +| Build desktop control | [Computer use](computer-use.md) | +| Integrate QVAC | [QVAC runtime](qvac-runtime.md) | +| Understand the agent | [Harness integration](harness-integration.md) | +| Use the D-Bus API | [API reference](api.md) | +| Build and release with Gitea | [CI and releases](ci-gitea.md) | +| Review privacy and safety | [Security and privacy](security-privacy.md) | +| Test changes | [Testing](testing.md) | +| Diagnose failures | [Troubleshooting](troubleshooting.md) | +| Track implementation | [Roadmap](ROADMAP.md) | + +## Runtime promise + +The project has one cognition loop and one QVAC authority. `jarvisd` owns the +QVAC SDK worker, model lifecycle, GPU policy, scheduler, and cancellation. The +vendored harness supplies planning, sessions, memory, and tool-loop behavior. +The GNOME Shell extension only renders state and forwards desktop facts. It +does not load models or perform inference. + +```mermaid +flowchart LR + U[User] --> G[GNOME Shell extension] + G <--> D[jarvisd] + D --> H[Vendored agent harness] + D --> Q[Single QVAC master] + D --> C[Computer-use backend] + D --> A[PipeWire audio] + Q --> GPU[GPU inference] +``` + +## Implementation truth + +The roadmap distinguishes implemented behavior (`[x]`), partial behavior +(``[~]``), and planned behavior (`[ ]`). A doctor command or acceptance test +is the source of truth for host-dependent capabilities. In particular, GPU +visibility, wake-word bridge availability, portal consent, and live EIS input +must be verified on the target GNOME session. + +## Documentation conventions + +- Commands are run from the repository root unless stated otherwise. +- Paths beginning with `~` are user-scoped paths. +- `jarvisd` means the user service and its Node process. +- “Master” means the singleton in `daemon/qvac-master.js`. +- Mermaid diagrams use Gitea-compatible fenced Markdown. diff --git a/docs/RELEASE.md b/docs/RELEASE.md new file mode 100644 index 0000000..e4a975a --- /dev/null +++ b/docs/RELEASE.md @@ -0,0 +1,29 @@ +# Release checklist + +## Local verification + +1. Confirm Node 22, GNOME dependencies, PipeWire, portal access, and a QVAC + GPU backend with `npm run gpu-doctor`, `npm run cu-doctor`, and + `npm run voice-doctor`. +2. Run `npm ci`, `npm test`, and `npm run package:test`. +3. Review `dist/SHA256SUMS` and test the user installer in a disposable user + session with `packaging/install.sh --enable`. +4. Run the typed first-run smoke test and the computer-use acceptance flow. + +## Gitea rolling release + +`.gitea/workflows/rolling-release.yml` runs on pushes to `main`. It tests the +tree, builds the `.deb`, extension zip, and checksums, then uses the repository +secret `RELEASE_TOKEN` to move the `rolling` tag and replace the `rolling` +prerelease through the Gitea REST API. The token is passed only as an Actions +secret and is never written to artifacts or logs. + +Release artifacts are intentionally marked prerelease. Create a permanent +release by downloading the rolling artifacts, validating them on the target +Ubuntu release, and using a separately reviewed tag. + +## Rollback + +Use the previous commit's artifacts or rerun the workflow for that commit. The +rolling tag is mutable by design; permanent tags remain immutable release +records. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 76a06c7..1408ba4 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -310,12 +310,16 @@ GPU inference visibly confirmed. ## Phase 11 — packaging and release -- [ ] Build a `.deb`, extension bundle, and user-service installer. -- [ ] Add first-run wizard for microphone, wake phrase, GPU/model profile, TTS, +- [x] Build a `.deb`, extension bundle, and user-service installer. +- [x] Add first-run wizard for microphone, wake phrase, GPU/model profile, TTS, and a typed smoke test. -- [ ] Add uninstall that removes only Jarvis-owned data after explicit choice. -- [ ] Publish a hardware compatibility matrix for Vulkan/GPU backends. -- [ ] Add release checklist, migration notes, and a reproducible local demo. +- [x] Add uninstall that removes only Jarvis-owned data after explicit choice. +- [x] Publish a hardware compatibility matrix for Vulkan/GPU backends. +- [x] Add release checklist, migration notes, and a reproducible local demo. + +Release automation lives in `.gitea/workflows/rolling-release.yml`. A push to +`main` runs the complete verification and package build, then replaces the +Gitea `rolling` prerelease using the repository's `RELEASE_TOKEN` secret. Definition of done: all product behaviors in the specification work on Ubuntu GNOME with one GPU-backed QVAC master, one harness planner, explicit computer- diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..e389074 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,43 @@ +# API reference + +## Session D-Bus + +Bus name: `io.qvac.Jarvis` +Object path: `/io/qvac/Jarvis` +Interface: `io.qvac.Jarvis.Session` + +The canonical XML contract is [io.qvac.Jarvis.Session.xml](../dbus/io.qvac.Jarvis.Session.xml). + +| Method | Purpose | +| --- | --- | +| `Arm()` / `Sleep()` / `Shutdown()` | Control daemon voice lifecycle | +| `PushToTalk(bool)` | Begin/end typed or microphone turn | +| `Say(string)` | Speak text through local TTS | +| `Ask(string)` | Submit typed text to the harness | +| `Cancel()` | Cancel active request/job | +| `SetMode(string)` | Set ARC capability mode | +| `GetState()` | Return serialized daemon state | +| `EnrollWake(string)` | Configure wake phrase | +| `IngestPath(string)` | Add a local path to RAG ingestion | +| `ComputerGrant(bool)` / `ComputerRevoke()` | Start/stop computer-use grant | +| `ComputerStatus()` | Return computer-use status JSON | +| `ShellListWindows()` / `ShellFocusWindow(uint64)` | Query/focus Shell windows | + +Signals include `StateChanged`, `WakeHeard`, `PartialTranscript`, +`FinalTranscript`, `Token`, `Reply`, `SpeakingLevel`, `ListeningLevel`, +`ChipOffered`, `JobProgress`, `ComputerStep`, `ComputerHighlight`, and `Error`. +Payloads stay small; PCM and image bytes do not travel over D-Bus. + +## Computer-use tools + +Tools are registered with the harness and exposed through the daemon. The +observable contract includes `cu.grant`, `cu.revoke`, `cu.status`, +`cu.observe`, `cu.zoom`, `cu.tree`, `cu.find`, `cu.click`, `cu.drag`, +`cu.scroll`, `cu.type`, `cu.key`, and `cu.act`. Every action is checked against +the active grant and safety policy. + +## Local HTTP + +QVAC's optional OpenAI-compatible server is localhost-only at port `11434`. +It is a diagnostic surface, not a second Jarvis runtime. Native QVAC-only +operations use `/qvac/v1/*` where supported by the installed QVAC version. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..67a7d89 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,80 @@ +# System architecture + +JARVIS-QVAC is split into a thin desktop surface, a user daemon, a cognitive +harness, and isolated capability adapters. This keeps GNOME Shell responsive, +prevents multiple model runtimes from competing for GPU memory, and makes +computer use revocable as a session capability. + +```mermaid +flowchart TB + subgraph Desktop[GNOME desktop session] + EXT[Shell extension\nARC HUD + panel state] + CC[Control Center\nGTK4/libadwaita] + PORTAL[XDG portals\nScreenCast + RemoteDesktop] + ATSPI[AT-SPI2\naccessible tree] + PW[PipeWire\nmicrophone + playback] + end + subgraph Service[User services] + D[ jarvisd\nNode 22 ESM ] + CU[Computer-use session\nobserve + act + audit] + end + subgraph Cognitive[Cognition] + H[agent-harness\nsessions + planner + memory] + S[Skills\nQVAC + desktop + files] + end + subgraph Inference[One inference authority] + M[QVAC master\nload / schedule / cancel] + Q[QVAC SDK 0.19.x] + GPU[(GPU backend)] + end + EXT <--> |session D-Bus| D + CC <--> |settings + diagnostics| D + D --> H --> S + D --> M --> Q --> GPU + D <--> CU + CU <--> PORTAL + CU <--> ATSPI + D <--> PW +``` + +## Process responsibilities + +| Process | Owns | Must not own | +| --- | --- | --- | +| GNOME Shell extension | panel, ARC overlay, highlights, Shell facts, D-Bus client | model loading, microphone capture, input injection, blocking work | +| `jarvisd` | voice state, QVAC master, harness bridge, jobs, D-Bus service | arbitrary Shell evaluation or a second planner | +| Control Center | settings, model fit, permissions, first-run controls | inference loop or hidden data deletion | +| Computer-use helpers | portal sessions, frames, AT-SPI, EIS/legacy backend | model planning or lock-screen bypass | +| Agent harness | session history, planning, memory, tool loop, permissions | direct QVAC ownership or direct desktop hacks | +| QVAC master | SDK worker, GPU admission, lifecycle, serialization, cancellation | cloud APIs or independent model instances | + +## Data and control boundaries + +Control messages and small state changes use session D-Bus. Audio streams and +large screenshots use PipeWire, temporary files, or the local IPC channel. The +daemon never sends screen frames or microphone data to a remote service. + +```mermaid +sequenceDiagram + participant Shell as GNOME Shell + participant Daemon as jarvisd + participant Harness as Harness + participant Master as QVAC master + participant GPU as GPU + Shell->>Daemon: Ask(text) / wake / PTT + Daemon->>Master: admit(interactive lane) + Daemon->>Harness: prompt(text, tools) + Harness->>Master: completion(request) + Master->>GPU: one serialized inference + GPU-->>Master: streamed tokens + Master-->>Harness: tokens + tool calls + Harness-->>Daemon: reply + sidecar + Daemon-->>Shell: Token / Reply / StateChanged +``` + +## Failure containment + +QVAC failures, unavailable models, or failed background jobs are reported as +capability errors and do not silently switch to CPU inference. A daemon restart +releases the master and restores the user service according to systemd policy. +Computer-use revoke is immediate and independent of the language model. diff --git a/docs/capabilities.md b/docs/capabilities.md new file mode 100644 index 0000000..98ddbfc --- /dev/null +++ b/docs/capabilities.md @@ -0,0 +1,35 @@ +# Capability catalog + +Capabilities are exposed through harness tools and report their real runtime +state. A capability can be registered while unavailable because its model is +not downloaded, does not fit the selected GPU profile, or requires hardware +that is not present. + +| Domain | Tools | Execution owner | Typical permission | +| --- | --- | --- | --- | +| Conversation | `chat`, `plan`, `summarize`, `rewrite`, `code` | Harness + QVAC master | read | +| Retrieval | `embed`, `remember`, `ask-my-files`, workspaces | QVAC master + local stores | read/write | +| Vision | screenshot, `look-at-this`, `ocr`, `what-is-this` | Portal + QVAC master | read | +| Speech | `dictate`, `transcribe-file`, `meeting`, `speak`, `clone-voice` | PipeWire + QVAC master | read/write | +| Language | `translate`, `relay-voice` | QVAC master | read | +| Media | `imagine`, `edit-image`, `make-video`, `compose` | Job runner + QVAC master | write | +| Training | `teach-me` | Isolated job + QVAC master | dangerous | +| Lab | `bci`, `robot`, `world` | Explicit lab adapters | read/write/dangerous | +| Desktop | launch, focus, window, clipboard, media, settings | GNOME/GIO/D-Bus | read/write | +| Computer use | `cu.observe`, `cu.find`, `cu.act`, `cu.click`, `cu.type`, `cu.key` | CU session | computer-use | + +```mermaid +flowchart TB + J((JARVIS-QVAC)) + J --> C[Conversation\nchat · plan · code · summarize] + J --> M[Memory\nembeddings · RAG · batch] + J --> P[Perception\nscreenshot · OCR · classification] + J --> V[Voice\nwake · ASR · TTS · translation] + J --> G[Media\nimage · video · music] + J --> D[Desktop\nGNOME · AT-SPI · computer use] + J --> L[Lab\nLoRA · BCI · VLA · ABot-World] +``` + +All QVAC-facing capabilities use the master allowlist and scheduler. The UI +must show unavailable reasons instead of hiding a feature or claiming that a +job ran. diff --git a/docs/ci-gitea.md b/docs/ci-gitea.md new file mode 100644 index 0000000..6ba1d24 --- /dev/null +++ b/docs/ci-gitea.md @@ -0,0 +1,46 @@ +# Gitea CI and rolling releases + +Gitea Actions reads workflows from `.gitea/workflows/`. This repository has two +workflows: + +```mermaid +flowchart LR + Push[Push / pull request] --> CI[.gitea/workflows/ci.yml] + CI --> Tests[npm test + Python syntax + package smoke] + Main[Push to main] --> RR[rolling-release.yml] + RR --> Build[Build deb + extension + checksums] + Build --> API[Gitea release API] + API --> Tag[Move rolling tag] + API --> Release[Replace rolling prerelease] +``` + +## Required repository configuration + +Create a repository secret named `RELEASE_TOKEN`. It must be able to read the +repository and create/update tags, releases, and release assets. The workflow +references it as `${{ secrets.RELEASE_TOKEN }}`; the value is never committed. + +## Rolling behavior + +Each push to `main` runs tests and packaging. The publish script: + +1. Deletes the existing release associated with `rolling`, if present. +2. Moves or creates the `rolling` tag at the pushed commit. +3. Creates a prerelease named with the commit prefix. +4. Uploads the `.deb`, extension ZIP, and `SHA256SUMS`. + +The moving tag is intentionally mutable. Permanent releases should use reviewed +immutable tags after validating a rolling artifact. + +## Local rehearsal + +```bash +npm ci +npm test +npm run package +cat dist/SHA256SUMS +``` + +Do not run the publish script locally unless `GITEA_SERVER_URL`, +`GITEA_REPOSITORY`, `GITEA_SHA`, and a write-capable `RELEASE_TOKEN` are +intentionally set. diff --git a/docs/computer-use.md b/docs/computer-use.md new file mode 100644 index 0000000..3ead1df --- /dev/null +++ b/docs/computer-use.md @@ -0,0 +1,57 @@ +# Computer use + +Computer use is a revocable mode that combines observation, semantic actions, +portal input, and a visible ARC overlay. It is never implicitly enabled by a +normal chat turn. + +```mermaid +sequenceDiagram + participant User + participant D as jarvisd + participant H as Harness + participant CU as CU session + participant P as Portal/EIS + participant A as AT-SPI + User->>D: “take the wheel” + D->>CU: grant(persist, monitors) + CU->>P: ScreenCast + RemoteDesktop consent + User-->>P: approve + loop step budget + CU->>P: capture frame + CU->>A: tree snapshot + CU->>H: observe bundle + H-->>CU: one action + CU->>CU: safety + target validation + CU->>P: EIS input or + CU->>A: named action + end + User->>D: “hands off” / Escape + D->>CU: revoke + CU->>P: close sessions immediately +``` + +## Actuator priority + +1. Domain skill or app API. +2. App D-Bus/GIO action. +3. AT-SPI named action or stable reference. +4. GNOME Shell window/focus helper. +5. Vision coordinates for broken accessibility. +6. Opt-in ydotool/X11 fallback. + +The primary Wayland input path is XDG RemoteDesktop plus EIS/libei. The +portal helper refuses to report input readiness without a configured local EIS +bridge (`JARVIS_LIBEI_BRIDGE`). Screenshot frames are downscaled for vision, +kept in temporary storage, and removed on revoke unless trace retention is +enabled. + +## Safety rules + +- Grant is off by default and dies on lock screen. +- Password/PAM roles are refused. +- Dangerous keys and destructive targets need confirmation. +- Default budget is 20 actions; hard cap is 100. +- Each action gets a visible target highlight and audit record. +- Three state-changing steps with no observed change abort the run. + +See [computer-use acceptance](cu-acceptance.md) for the live GNOME scenarios. diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..8f5319b --- /dev/null +++ b/docs/development.md @@ -0,0 +1,47 @@ +# Development guide + +## Repository map + +```text +daemon/ Node daemon, QVAC master, voice, D-Bus +computer-use/ observe/act session and host helpers +skills/ harness tool adapters +apps/gnome-extension/ GNOME Shell ESM UI and Shell bridge +apps/control-center/ GTK4/libadwaita settings application +vendor/agent-harness/ copied cognitive core +dbus/ introspection XML +packaging/ installer, service, artifacts, release API +systemd/user/ checkout development service +docs/ maintained technical documentation +test/ Node acceptance/unit fixtures +``` + +## Change workflow + +1. Read the relevant architecture and safety contract. +2. Keep QVAC calls behind the master. +3. Keep Shell code asynchronous and presentation-only. +4. Add or update a focused test for changed behavior. +5. Run the local checks and update docs/roadmap status. +6. Use a descriptive Gitea commit and let CI validate the push. + +## Adding a QVAC capability + +Add the capability to `skills/catalog.js`, implement its adapter in the +appropriate skill module, add the master operation to the allowlist if needed, +define its permission and unavailable states, and document the model/resource +requirements. Never import `@qvac/sdk` from a skill. + +## Adding a desktop action + +Prefer a structured GSettings, GIO, app D-Bus, or AT-SPI action. Only add a +coordinate path when semantic access is unavailable. Computer-use actuation +must emit a highlight and audit event and must pass the active grant and safety +checks. + +## Commit and review expectations + +Reviews should check single-master ownership, GPU enforcement, privacy +retention, permission gates, lock-screen behavior, and truthful unavailable +states. Release-affecting changes should include a migration note and a +package smoke-test result. diff --git a/docs/hardware-compatibility.md b/docs/hardware-compatibility.md new file mode 100644 index 0000000..2bddf46 --- /dev/null +++ b/docs/hardware-compatibility.md @@ -0,0 +1,20 @@ +# Hardware compatibility + +JARVIS-QVAC has one inference owner: `jarvisd` owns the QVAC master and every +model call goes through `daemon/qvac-master.js`. The harness, media jobs, and +computer-use tools do not start competing QVAC servers. The GPU gate is strict: +when QVAC cannot report a usable GPU backend, inference fails with a truthful +diagnostic instead of silently falling back to CPU. + +| Hardware path | Recommended profile | Expected backend | Notes | +| --- | --- | --- | --- | +| AMD Vulkan | laptop-16gb or desktop-gpu | Vulkan/llama.cpp | Verify the QVAC runtime sees the ICD, not only `vulkaninfo`. | +| NVIDIA CUDA | desktop-gpu | CUDA/cuBLAS | Install a driver compatible with the QVAC build and verify VRAM. | +| Intel Vulkan | laptop-16gb | Vulkan | Use a current Mesa stack; larger media jobs may remain queued. | +| CPU only | unsupported for inference | none | Wake and diagnostics can run, but the master refuses model inference. | + +Run `npm run gpu-doctor` before downloading models. `assessModelFit()` should +select the largest profile that fits reported VRAM; an unavailable model stays +visible as unavailable and never causes the voice loop to crash. The current +machine's GPU visibility is an installation concern when this document is +updated; record the exact doctor output in the release checklist. diff --git a/docs/harness-integration.md b/docs/harness-integration.md new file mode 100644 index 0000000..38c1eec --- /dev/null +++ b/docs/harness-integration.md @@ -0,0 +1,48 @@ +# Agent harness integration + +The copied harness at `vendor/agent-harness` is the cognitive core. It is kept +as a repository copy so releases are reproducible and do not depend on the +machine at `~/dev/agent-harness` after checkout. The original source remains +the upstream reference described in [agent-harness-map](agent-harness-map.md). + +```mermaid +flowchart LR + P[Prompt] --> B[harness-bridge.js] + B --> S[Agent session] + S --> L[agent/loop.js] + L --> T{Tool call?} + T -->|yes| R[Jarvis JSON-schema tool] + R --> X[master / desktop / CU / files] + X --> L + T -->|no| O[Reply + HUD sidecar] + S --> MEM[Local sessions + memory] +``` + +## Harness responsibilities + +- Multi-turn session state and compaction. +- Planner/tool-call loop and permission mode. +- Short- and long-term local memory. +- Tool schema validation and execution events. +- Streaming model messages adapted into daemon events. + +## Jarvis responsibilities + +Jarvis registers domain tools from `skills/`, supplies the system prompt, maps +tool permissions to confirmation gates, and translates harness events into the +D-Bus protocol. Computer-use tools are custom harness tools backed by +`computer-use/`; the computer-use layer never owns planning. + +## Adding a tool + +1. Put the implementation in the appropriate skill module. +2. Export a JSON-schema tool with a stable name, description, timeout, and + permission level. +3. Route QVAC work through the master facade. +4. Add a focused fixture test with a fake master or backend. +5. Update [API](api.md), the capability matrix, and the roadmap if the tool is + user-visible. + +Destructive tools require explicit confirmation. Computer-use tools additionally +require an active grant, a visible overlay, a live step budget, and a lock-screen +check. diff --git a/docs/local-demo.md b/docs/local-demo.md new file mode 100644 index 0000000..6e82d57 --- /dev/null +++ b/docs/local-demo.md @@ -0,0 +1,25 @@ +# Reproducible local demo + +From the repository root: + +```bash +npm ci +npm test +npm run gpu-doctor +npm run cu-doctor +npm run package:test +``` + +On a logged-in GNOME session with a visible QVAC GPU backend: + +```bash +bash packaging/install.sh --enable +bash packaging/first-run.sh +systemctl --user status jarvisd.service +gnome-extensions enable jarvis@qvac.local +``` + +Complete the typed prompt first, then say “Hey Jarvis, what’s on my screen?” +and approve a computer-use grant only for the acceptance actions in +`docs/cu-acceptance.md`. If the doctor reports no GPU, stop at diagnostics and +fix the QVAC runtime/driver path before attempting inference. diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..04b5f03 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,57 @@ +# Operations guide + +## Prerequisites + +Ubuntu GNOME 24.04 or newer, Node.js 22.17+, PipeWire/WirePlumber, GNOME +desktop portals, GTK4/libadwaita, AT-SPI2, and a QVAC-visible GPU are required +for the full runtime. See [hardware compatibility](hardware-compatibility.md). + +## Development start + +```bash +npm ci +npm test +npm run gpu-doctor +npm run cu-doctor +npm run voice-doctor +node daemon/index.js +``` + +Use `npm run qvac:serve:diagnostic` only when a separate local client needs the +OpenAI-compatible endpoint. Keep its host at `127.0.0.1`. + +## User service + +```bash +systemctl --user daemon-reload +systemctl --user enable --now jarvisd.service +systemctl --user status jarvisd.service +journalctl --user -u jarvisd.service -f +``` + +The packaged service uses `~/.local/share/jarvis-qvac` and the generated config +at `~/.config/jarvis/qvac.config.json`. The repository development unit uses +the checkout path; use the packaged unit when testing installation behavior. + +## Extension development + +```bash +gnome-extensions enable jarvis@qvac.local +dbus-run-session -- gnome-shell --nested --wayland +``` + +Keep inference and long-running work in `jarvisd`. Reloading the Shell is only +needed for extension changes; daemon changes can use `node --watch`. + +## State and storage + +| Location | Contents | Default retention | +| --- | --- | --- | +| `~/.local/share/jarvis/memory` | harness memory and RAG data | user-managed | +| `~/.local/share/jarvis/voices` | local voice references | user-managed | +| `~/.local/share/jarvis/audit` | CU action metadata and hashes | user-managed | +| `~/.cache/jarvis` | models, jobs, temporary CU state | disposable | +| `~/Jarvis` | generated media and transcripts | user-managed | + +Use the uninstall script with `--yes`; add `--purge-data` only to remove the +listed Jarvis-owned data locations. diff --git a/docs/qvac-runtime.md b/docs/qvac-runtime.md new file mode 100644 index 0000000..dce3c43 --- /dev/null +++ b/docs/qvac-runtime.md @@ -0,0 +1,68 @@ +# QVAC runtime and GPU authority + +## The single-master rule + +Only `daemon/qvac-master.js` may import and operate the QVAC SDK for Jarvis +runtime work. Skills receive a master facade; they do not call `loadModel()` or +create their own workers. The harness is connected through +`daemon/harness-bridge.js` and uses the same owner. + +```mermaid +flowchart LR + V[Voice] --> M[Master scheduler] + H[Harness] --> M + CU[Computer vision] --> M + J[Jobs: media / RAG] --> M + M --> L{GPU admission} + L -->|fit| SDK[@qvac/sdk 0.19.1] + L -->|no fit| E[truthful unavailable error] + SDK --> W[one QVAC worker] +``` + +The optional `qvac serve --openai` command is diagnostic/API infrastructure. +It is not started alongside `jarvisd` in production and must remain bound to +`127.0.0.1` when used. + +## Lifecycle + +1. Read `QVAC_CONFIG_PATH` and the selected profile. +2. Query GPU visibility and model fit before loading. +3. Admit work through interactive, computer-vision, background, or + maintenance lanes. +4. Load one model configuration at a time and serialize worker access. +5. Stream completion or media progress to the daemon. +6. Cancel by request ID/model ID when the user says stop or a grant is revoked. +7. Suspend or unload according to idle policy; close once during shutdown. + +```mermaid +stateDiagram-v2 + [*] --> Uninitialized + Uninitialized --> Preflight: start + Preflight --> Ready: GPU visible + fit accepted + Preflight --> Unavailable: GPU missing / fit rejected + Ready --> Loading: admitted request + Loading --> Loaded: loadModel + Loaded --> Busy: completion / media / vision + Busy --> Loaded: stream complete + Busy --> Loaded: cancellation + Loaded --> Suspended: idle policy + Suspended --> Loading: new request + Loaded --> Closing: shutdown + Suspended --> Closing: shutdown + Closing --> [*] +``` + +## Configuration + +The repository config is `qvac.config.json`. Installed user sessions use +`~/.config/jarvis/qvac.config.json`, generated from +`packaging/qvac.config.template.json`. The server is localhost-only and the +default alias is `jarvis-master`. Model cache files belong under the user's +cache directory, never under a system-wide writable location. + +## GPU policy + +`JARVIS_GPU_REQUIRED=1` is set by the service. The master requests GPU device +and maximum GPU layer offload, then rejects a result QVAC reports as CPU. Run +`npm run gpu-doctor` and consult [hardware compatibility](hardware-compatibility.md) +before changing a model profile. diff --git a/docs/security-privacy.md b/docs/security-privacy.md new file mode 100644 index 0000000..fbd08d2 --- /dev/null +++ b/docs/security-privacy.md @@ -0,0 +1,44 @@ +# Security and privacy + +The default posture is local-only, explicit, and fail-closed. No telemetry or +cloud inference is required. The network policy rejects unexpected outbound +model or tool traffic; optional model-fetch features remain user initiated. + +```mermaid +flowchart TD + R[User request] --> P{Permission gate} + P -->|read| X[Execute locally] + P -->|write / dangerous| C{Explicit confirmation?} + C -->|yes| X + C -->|no| B[Block and explain] + X --> A[Audit metadata] + X --> D[Return result] + L[Lock screen] --> K[Revoke CU + mute overlay] +``` + +## Protected assets + +- Microphone audio remains in a RAM ring buffer unless transcript retention is + explicitly enabled. +- Screen frames remain in memory or temporary storage and are wiped on + computer-use revoke by default. +- Computer-use audit logs store action metadata and screenshot hashes, not + screenshots. +- Voice references, memory, and model caches are user-owned files. +- QVAC binds to localhost; bearer tokens, when used, come from user-owned + configuration and are not logged. + +## Computer-use controls + +Computer use requires a spoken or HUD grant, shows a visible cursor/target, and +stops on revoke, Escape, lock screen, or grant expiry. It refuses password +fields, dangerous actions without confirmation, and actuation when the portal +or EIS backend is not ready. Legacy input is opt-in. + +## Threat model boundaries + +The daemon assumes the local user account and installed desktop libraries are +trusted. A malicious local process with access to the user's session bus or +filesystem is outside the protection boundary. Keep the repository and model +cache user-owned, use a restrictive token file mode, and avoid running the +daemon as root. diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..fd1477e --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,33 @@ +# Testing and acceptance + +## Automated checks + +```bash +npm test +npm run package:test +bash -n packaging/*.sh +python3 -m py_compile apps/control-center/main.py computer-use/py/*.py +git diff --check +``` + +The Node suite covers daemon infrastructure, QVAC ownership, network policy, +observer behavior, computer-use safety, runtime tools, and voice behavior. The +vendored harness test is included in the root test command. + +## Host acceptance + +| Area | Procedure | Expected result | +| --- | --- | --- | +| GPU | `npm run gpu-doctor` | QVAC reports a usable GPU and profile fit | +| Portals | `npm run cu-doctor` | portal, PipeWire, AT-SPI, and libei checks pass | +| Voice | [voice acceptance](voice-acceptance.md) | wake, VAD, TTS, anti-feedback, cancel pass | +| Computer use | [CU acceptance](cu-acceptance.md) | observe, act, abort, lock kill pass | +| Packaging | `npm run package:test` | `.deb`, zip, checksum all validate | +| Gitea | push a test commit to a branch/workflow | CI completes without release side effects | + +## Release gate + +Before a permanent tag, run the package smoke test, verify the rolling artifact +on the target Ubuntu GNOME release, execute the typed and voice demos, and +record GPU doctor output. Do not call a CPU diagnostic result a successful +inference acceptance. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..fdfaca4 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,46 @@ +# Troubleshooting + +## `gpu-doctor` reports no GPU + +Confirm the driver and Vulkan/CUDA runtime are installed, then run the doctor +under the same graphical user session that will run `jarvisd`. A host command +such as `vulkaninfo` is insufficient if QVAC itself reports no device. Jarvis +will remain available for diagnostics but refuses inference until QVAC sees a +GPU. + +## Voice doctor finds no wake engine + +Install a local openWakeWord or sherpa-onnx bridge and set +`JARVIS_WAKE_COMMAND`. The bridge must read raw 16 kHz mono PCM from stdin and +write detected phrases, one per line, to stdout. Push-to-talk remains useful +for typed or manually triggered testing. + +## Portal consent succeeds but input is unavailable + +ScreenCast consent does not prove that RemoteDesktop/EIS injection is ready. +Configure `JARVIS_LIBEI_BRIDGE`; the helper intentionally fails closed without +it. Use observe-only mode until the injector reports ready. `ydotool` is an +optional, explicitly enabled fallback. + +## The extension is missing + +Check that `metadata.json` has UUID `jarvis@qvac.local`, copy the extension into +`~/.local/share/gnome-shell/extensions/jarvis@qvac.local`, then run: + +```bash +gnome-extensions enable jarvis@qvac.local +journalctl --user -b | grep -i jarvis +``` + +## The daemon restarts + +Inspect `journalctl --user -u jarvisd.service`. Common causes are Node version, +invalid QVAC config, missing session D-Bus, missing GPU visibility, or a stale +portal grant. Run the three doctor commands before changing model settings. + +## Rolling release did not publish + +Check the Gitea Actions log and repository secret name. It must be exactly +`RELEASE_TOKEN` with repository write permission. The workflow only publishes +from `main`; pull requests and other branches run CI but do not replace the +`rolling` prerelease. diff --git a/docs/voice-pipeline.md b/docs/voice-pipeline.md new file mode 100644 index 0000000..3ddeff2 --- /dev/null +++ b/docs/voice-pipeline.md @@ -0,0 +1,44 @@ +# Voice pipeline + +Voice is input/output around the harness. The daemon owns capture, wake-word +processing, VAD, transcription, completion, TTS playback, barge-in policy, and +metrics. + +```mermaid +flowchart LR + MIC[PipeWire mic\n16 kHz mono] --> R[Wake ring buffer] + R --> W[Wake engine] + W -->|phrase| V[VAD + QVAC ASR] + V -->|final utterance| H[Harness bridge] + H -->|streamed reply| T[Sentence buffer] + T --> Q[QVAC TTS] + Q --> OUT[PipeWire playback] + OUT -. anti-feedback gate .-> V +``` + +## State machine + +```mermaid +stateDiagram-v2 + ARMED --> LISTENING: wake phrase / hotkey + LISTENING --> THINKING: VAD final utterance + THINKING --> SPEAKING: first reply audio + THINKING --> LISTENING: tool work / no audio yet + SPEAKING --> LISTENING: playback + cooldown + LISTENING --> ARMED: silence timeout + SPEAKING --> ARMED: cancel / Escape + ARMED --> SLEEPING: idle timeout + SLEEPING --> LISTENING: wake phrase +``` + +The wake stage is intentionally separate from QVAC ASR. Configure a local +openWakeWord or sherpa-onnx bridge through `JARVIS_WAKE_COMMAND`; no cloud wake +service is supported. During TTS, capture submission is gated and a 300–500 ms +cooldown prevents playback from becoming a new utterance. Trash transcripts and +very short fragments are discarded. + +## Diagnostics and acceptance + +Run `npm run voice-doctor`, then follow [voice acceptance](voice-acceptance.md). +The daemon metrics cover wake accepts/rejects, feedback drops, utterances, and +replies. Raw audio is not persisted by default. diff --git a/package.json b/package.json index a82e64c..8a6c63c 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,13 @@ "gpu-doctor": "node daemon/gpu-doctor.js", "qvac:doctor": "qvac doctor", "qvac:serve:diagnostic": "qvac serve --openai --host 127.0.0.1", - "qvac:status": "node -e \"import('./daemon/qvac-master.js').then(({qvacStatus})=>console.log(JSON.stringify(qvacStatus(), null, 2)))\"" + "qvac:status": "node -e \"import('./daemon/qvac-master.js').then(({qvacStatus})=>console.log(JSON.stringify(qvacStatus(), null, 2)))\"", + "package:extension": "bash packaging/build-extension.sh", + "package": "bash packaging/build-release.sh", + "package:test": "bash packaging/test-package.sh", + "first-run": "bash packaging/first-run.sh", + "install:user": "bash packaging/install.sh", + "uninstall:user": "bash packaging/uninstall.sh" }, "dependencies": { "@qvac/cli": "0.13.0", diff --git a/packaging/build-deb.sh b/packaging/build-deb.sh new file mode 100755 index 0000000..67c2515 --- /dev/null +++ b/packaging/build-deb.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VERSION="${JARVIS_VERSION:-$(node -p "require('${ROOT_DIR}/package.json').version")}" +ARCH="${JARVIS_DEB_ARCH:-$(dpkg --print-architecture)}" +OUT_DIR="${ROOT_DIR}/dist" +PKG_DIR="${OUT_DIR}/jarvis-qvac_${VERSION}_${ARCH}" +rm -rf "${PKG_DIR}" +mkdir -p "${PKG_DIR}/DEBIAN" "${PKG_DIR}/usr/lib/jarvis-qvac" "${PKG_DIR}/usr/share/doc/jarvis-qvac" +cat > "${PKG_DIR}/DEBIAN/control" </dev/null +rm -rf "${PKG_DIR}" +echo "${OUT_DIR}/jarvis-qvac_${VERSION}_${ARCH}.deb" diff --git a/packaging/build-extension.sh b/packaging/build-extension.sh new file mode 100755 index 0000000..7292562 --- /dev/null +++ b/packaging/build-extension.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUT_DIR="${ROOT_DIR}/dist" +mkdir -p "${OUT_DIR}" +rm -f "${OUT_DIR}/jarvis-qvac-extension.zip" +( cd "${ROOT_DIR}/apps/gnome-extension" && zip -q -r "${OUT_DIR}/jarvis-qvac-extension.zip" "jarvis@qvac.local" -x '*/__pycache__/*' -x '*.pyc' ) +test -s "${OUT_DIR}/jarvis-qvac-extension.zip" +echo "${OUT_DIR}/jarvis-qvac-extension.zip" diff --git a/packaging/build-release.sh b/packaging/build-release.sh new file mode 100755 index 0000000..6e15d30 --- /dev/null +++ b/packaging/build-release.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${ROOT_DIR}" +mkdir -p dist +rm -f dist/jarvis-qvac_*.deb dist/jarvis-qvac-extension.zip dist/SHA256SUMS +bash packaging/build-extension.sh +bash packaging/build-deb.sh +(cd dist && sha256sum jarvis-qvac_*.deb jarvis-qvac-extension.zip > SHA256SUMS) +echo "Release artifacts written to ${ROOT_DIR}/dist" diff --git a/packaging/first-run.sh b/packaging/first-run.sh new file mode 100755 index 0000000..c1ac0ac --- /dev/null +++ b/packaging/first-run.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${ROOT_DIR}" +echo "JARVIS-QVAC first-run setup" +echo "The daemon is GPU-gated and keeps QVAC behind one master owner." +echo +echo "[1/4] GPU and QVAC preflight" +npm run gpu-doctor || true +npm run qvac:doctor || true +echo +echo "[2/4] Microphone and PipeWire preflight" +npm run voice-doctor || true +echo +echo "[3/5] Wake phrase and model profile" +read -r -p "Wake phrase [hey jarvis]: " WAKE_PHRASE +WAKE_PHRASE="${WAKE_PHRASE:-hey jarvis}" +read -r -p "Model profile [laptop-8gb/laptop-16gb/desktop-gpu] (default: laptop-16gb): " MODEL_PROFILE +MODEL_PROFILE="${MODEL_PROFILE:-laptop-16gb}" +case "${MODEL_PROFILE}" in + laptop-8gb|laptop-16gb|desktop-gpu) ;; + *) echo "Unknown model profile: ${MODEL_PROFILE}" >&2; exit 2 ;; +esac +read -r -p "Enable TTS preview? [Y/n]: " TTS_CHOICE +TTS_ENABLED=true +[[ "${TTS_CHOICE:-Y}" =~ ^[Nn]$ ]] && TTS_ENABLED=false +mkdir -p "${HOME}/.config/jarvis" +node -e 'const fs=require("fs"); const p=process.argv[1], phrase=process.argv[2], profile=process.argv[3], tts=process.argv[4]==="true"; let c={}; try{c=JSON.parse(fs.readFileSync(p,"utf8"))}catch{}; Object.assign(c,{wakePhrase:phrase,modelProfile:profile,ttsEnabled:tts}); fs.writeFileSync(p,JSON.stringify(c,null,2)+"\n")' "${HOME}/.config/jarvis/config.json" "${WAKE_PHRASE}" "${MODEL_PROFILE}" "${TTS_ENABLED}" +echo "Saved wake phrase: ${WAKE_PHRASE}; model profile: ${MODEL_PROFILE}; TTS: ${TTS_ENABLED}" +echo +echo "[4/5] TTS preview" +if [[ "${TTS_ENABLED}" == true && -n "${DBUS_SESSION_BUS_ADDRESS:-}" ]]; then + busctl --user call io.qvac.Jarvis /io/qvac/Jarvis io.qvac.Jarvis.Session Say s "JARVIS local voice preview" || true +else + echo "Session D-Bus is unavailable or TTS is disabled; preview skipped." +fi +echo +echo "[5/5] Typed smoke test" +read -r -p "Type a test prompt (default: say hello): " PROMPT +PROMPT="${PROMPT:-say hello}" +if command -v busctl >/dev/null && [[ -n "${DBUS_SESSION_BUS_ADDRESS:-}" ]]; then + busctl --user call io.qvac.Jarvis /io/qvac/Jarvis io.qvac.Jarvis.Session Ask s "${PROMPT}" || true +else + echo "Session D-Bus is unavailable; saved setup and skipped live prompt." +fi +echo "First-run setup complete. Start with: systemctl --user enable --now jarvisd.service" diff --git a/packaging/install.sh b/packaging/install.sh new file mode 100755 index 0000000..814a166 --- /dev/null +++ b/packaging/install.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +APP_DIR="${HOME}/.local/share/jarvis-qvac" +EXT_DIR="${HOME}/.local/share/gnome-shell/extensions/jarvis@qvac.local" +CONFIG_DIR="${HOME}/.config/jarvis" +CACHE_DIR="${XDG_CACHE_HOME:-${HOME}/.cache}/jarvis/models" +mkdir -p "${APP_DIR}" "${EXT_DIR}" "${CONFIG_DIR}" "${HOME}/.config/systemd/user" "${CACHE_DIR}" +tar --exclude='.git' --exclude='./.agents' --exclude='./.codex' --exclude='./dist' --exclude='./node_modules' --exclude='*/node_modules' --exclude='*/__pycache__' --exclude='*.pyc' -cf - -C "${ROOT_DIR}" . | tar -xf - -C "${APP_DIR}" +if [[ ! -d "${APP_DIR}/node_modules" ]]; then + ( cd "${APP_DIR}" && npm ci --omit=dev --ignore-scripts ) +fi +rm -rf "${EXT_DIR}" +mkdir -p "${EXT_DIR}" +cp -a "${ROOT_DIR}/apps/gnome-extension/jarvis@qvac.local/." "${EXT_DIR}/" +cp "${ROOT_DIR}/packaging/jarvisd.service" "${HOME}/.config/systemd/user/jarvisd.service" +if [[ ! -f "${CONFIG_DIR}/qvac.config.json" ]]; then + sed "s#__JARVIS_CACHE__#${CACHE_DIR}#g" "${ROOT_DIR}/packaging/qvac.config.template.json" > "${CONFIG_DIR}/qvac.config.json" +fi +if [[ "${1:-}" == "--enable" ]]; then + systemctl --user daemon-reload + systemctl --user enable --now jarvisd.service +fi +echo "Installed JARVIS-QVAC to ${APP_DIR}" +echo "Run: ${APP_DIR}/packaging/first-run.sh" diff --git a/packaging/jarvisd.service b/packaging/jarvisd.service new file mode 100644 index 0000000..d52a159 --- /dev/null +++ b/packaging/jarvisd.service @@ -0,0 +1,17 @@ +[Unit] +Description=JARVIS-QVAC local voice assistant +After=pipewire.service wireplumber.service graphical-session.target +Wants=pipewire.service + +[Service] +Type=simple +WorkingDirectory=%h/.local/share/jarvis-qvac +ExecStart=/usr/bin/node %h/.local/share/jarvis-qvac/daemon/index.js +Environment=QVAC_CONFIG_PATH=%h/.config/jarvis/qvac.config.json +Environment=JARVIS_GPU_REQUIRED=1 +Environment=JARVIS_QVAC_OWNER=jarvisd +Restart=on-failure +RestartSec=2 + +[Install] +WantedBy=graphical-session.target diff --git a/packaging/publish-gitea-release.sh b/packaging/publish-gitea-release.sh new file mode 100755 index 0000000..b7cf704 --- /dev/null +++ b/packaging/publish-gitea-release.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${GITEA_SERVER_URL:?GITEA_SERVER_URL is required}" +: "${GITEA_REPOSITORY:?GITEA_REPOSITORY is required}" +: "${GITEA_SHA:?GITEA_SHA is required}" +: "${RELEASE_TOKEN:?RELEASE_TOKEN is required}" + +API="${GITEA_SERVER_URL%/}/api/v1/repos/${GITEA_REPOSITORY}" +AUTH=(-H "Authorization: token ${RELEASE_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json') +TAG="rolling" + +release_json="$(curl -fsS "${AUTH[@]}" "${API}/releases/tags/${TAG}" 2>/dev/null || true)" +if [[ -n "${release_json}" ]]; then + release_id="$(jq -r '.id // empty' <<<"${release_json}")" + if [[ -n "${release_id}" ]]; then + curl -fsS -X DELETE "${AUTH[@]}" "${API}/releases/${release_id}" >/dev/null + fi +fi + +tag_payload="$(jq -nc --arg sha "${GITEA_SHA}" '{new_target:$sha, force:true}')" +if ! curl -fsS -X PATCH "${AUTH[@]}" "${API}/git/refs/tags/${TAG}" --data "${tag_payload}" >/dev/null 2>&1; then + create_tag="$(jq -nc --arg sha "${GITEA_SHA}" --arg ref "refs/tags/${TAG}" '{ref:$ref, sha:$sha}')" + curl -fsS -X POST "${AUTH[@]}" "${API}/git/refs" --data "${create_tag}" >/dev/null +fi + +release_payload="$(jq -nc --arg tag "${TAG}" --arg sha "${GITEA_SHA}" \ + '{tag_name:$tag, target_commitish:$sha, name:("Rolling " + ($sha[0:12])), body:"Automated Gitea rolling release from main.", draft:false, prerelease:true}')" +created="$(curl -fsS -X POST "${AUTH[@]}" "${API}/releases" --data "${release_payload}")" +release_id="$(jq -r '.id // empty' <<<"${created}")" +test -n "${release_id}" + +for asset in dist/jarvis-qvac_*.deb dist/jarvis-qvac-extension.zip dist/SHA256SUMS; do + [[ -f "${asset}" ]] || continue + curl -fsS -X POST "${AUTH[@]}" \ + "${API}/releases/${release_id}/assets?name=$(basename "${asset}")" \ + -H 'Content-Type: application/octet-stream' --data-binary "@${asset}" >/dev/null +done +echo "Published Gitea rolling release ${TAG} for ${GITEA_SHA:0:12}" diff --git a/packaging/qvac.config.template.json b/packaging/qvac.config.template.json new file mode 100644 index 0000000..d912015 --- /dev/null +++ b/packaging/qvac.config.template.json @@ -0,0 +1,15 @@ +{ + "cacheDirectory": "__JARVIS_CACHE__", + "serve": { + "host": "127.0.0.1", + "port": 11434, + "models": { + "jarvis-master": { + "model": "QWEN3_5_4B_MULTIMODAL_Q4_K_M", + "default": true, + "preload": false, + "config": { "ctx_size": 8192 } + } + } + } +} diff --git a/packaging/test-package.sh b/packaging/test-package.sh new file mode 100755 index 0000000..84e4803 --- /dev/null +++ b/packaging/test-package.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${ROOT_DIR}" +command -v dpkg-deb >/dev/null +command -v zip >/dev/null +bash packaging/build-release.sh >/tmp/jarvis-package-test.log +DEB="$(find dist -maxdepth 1 -name 'jarvis-qvac_*.deb' -print -quit)" +test -n "${DEB}" +dpkg-deb --info "${DEB}" >/dev/null +dpkg-deb --contents "${DEB}" | grep 'usr/lib/jarvis-qvac/daemon/index.js' >/dev/null +unzip -t dist/jarvis-qvac-extension.zip >/dev/null +test -s dist/SHA256SUMS +echo "package smoke test passed" diff --git a/packaging/uninstall.sh b/packaging/uninstall.sh new file mode 100755 index 0000000..b60aac9 --- /dev/null +++ b/packaging/uninstall.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +if [[ "${1:-}" != "--yes" ]]; then + echo "Refusing to uninstall without --yes. Jarvis data is preserved by default." >&2 + exit 2 +fi +systemctl --user disable --now jarvisd.service 2>/dev/null || true +rm -f "${HOME}/.config/systemd/user/jarvisd.service" +rm -rf "${HOME}/.local/share/jarvis-qvac" "${HOME}/.local/share/gnome-shell/extensions/jarvis@qvac.local" +systemctl --user daemon-reload 2>/dev/null || true +if [[ "${2:-}" == "--purge-data" ]]; then + rm -rf "${HOME}/.local/share/jarvis" "${HOME}/.cache/jarvis" "${HOME}/Jarvis" +fi +echo "JARVIS-QVAC removed; user data was preserved unless --purge-data was supplied."