From aba185d80a9daf024e3946fd589547621b927860 Mon Sep 17 00:00:00 2001 From: Raven Scott Date: Fri, 11 Sep 2026 14:32:35 -0400 Subject: [PATCH] P9 --- daemon/harness-bridge.js | 3 ++- daemon/index.js | 2 +- daemon/qvac-master.js | 6 ++--- docs/ROADMAP.md | 23 ++++++++++------ skills/catalog.js | 16 ++++++++++- skills/optional-sync.js | 5 ++++ skills/phase9-tools.js | 56 +++++++++++++++++++++++++++++++++++++++ test/phase9-tools.test.js | 15 +++++++++++ 8 files changed, 112 insertions(+), 14 deletions(-) create mode 100644 skills/optional-sync.js create mode 100644 skills/phase9-tools.js create mode 100644 test/phase9-tools.test.js diff --git a/daemon/harness-bridge.js b/daemon/harness-bridge.js index 2614b20..05eadbe 100644 --- a/daemon/harness-bridge.js +++ b/daemon/harness-bridge.js @@ -6,11 +6,12 @@ import { createQvacTools } from '../skills/qvac-tools.js'; import { VOICE_SYSTEM_PROMPT, parseHudSidecar } from '../skills/voice-prompt.js'; import { createComputerObserveTools } from '../skills/computer-observe.js'; import { createComputerActTools } from '../skills/computer-act.js'; +import { createPhase9Tools } from '../skills/phase9-tools.js'; export class HarnessBridge extends EventEmitter { constructor({ cwd = process.cwd(), model = 'qwen3.5-4b', tools = [], computer, observer, actuator, permissionMode = 'ask' } = {}) { super(); - this.options = { cwd, model, tools: [...createRuntimeTools({ computer }), ...createPhase2Tools({ cwd, computer }), ...createComputerObserveTools({ computer, observer }), ...createComputerActTools({ actuator }), ...createQvacTools(), ...tools], permissionMode, origin: 'jarvis-qvac', system: VOICE_SYSTEM_PROMPT }; + this.options = { cwd, model, tools: [...createRuntimeTools({ computer }), ...createPhase2Tools({ cwd, computer }), ...createComputerObserveTools({ computer, observer }), ...createComputerActTools({ actuator }), ...createQvacTools(), ...createPhase9Tools(), ...tools], permissionMode, origin: 'jarvis-qvac', system: VOICE_SYSTEM_PROMPT }; this.session = null; } diff --git a/daemon/index.js b/daemon/index.js index 1fd632c..2ba1d1e 100644 --- a/daemon/index.js +++ b/daemon/index.js @@ -61,7 +61,7 @@ export class JarvisDaemon extends EventEmitter { try { await this.voiceLoop.start(); } catch (error) { this.voiceLoop = null; this.emit('Error', 'VOICE_UNAVAILABLE', error.message); throw error; } } setPushToTalk(pressed) { this.voiceLoop?.setPushToTalk(pressed); this.emit('PushToTalk', Boolean(pressed)); } - runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), computer: this.computer.status(), voice: this.voiceLoop?.metrics?.snapshot?.() || null }); } + runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), computer: this.computer.status(), voice: this.voiceLoop?.metrics?.snapshot?.() || null, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); } async assessModelFit(model) { return JSON.stringify(await callQvac('assessModelFit', { modelSrc: String(model) })); } async downloadModel(model) { return JSON.stringify(await callQvac('downloadAsset', { modelSrc: String(model) })); } async cancelModel(model) { return JSON.stringify(await cancelQvacRequest({ modelId: String(model) })); } diff --git a/daemon/qvac-master.js b/daemon/qvac-master.js index 381c384..f5869d3 100644 --- a/daemon/qvac-master.js +++ b/daemon/qvac-master.js @@ -121,7 +121,7 @@ export async function cancelQvacRequest({ requestId, modelId, kind } = {}) { if (!requestId && !modelId) throw new Error('requestId or modelId is required'); const sdk = await Agent.engine.ensureInit(); if (typeof sdk.cancel !== 'function') throw new Error('QVAC runtime does not expose cancel()'); - await sdk.cancel(requestId ? { requestId } : { modelId, kind }); + await withQvacMaster(() => sdk.cancel(requestId ? { requestId } : { modelId, kind })); } export async function suspendQvac() { @@ -142,12 +142,12 @@ export async function qvacRuntimeState() { return sdk.state(); } -const MASTER_CALLS = new Set(['assessModelFit', 'getSystemResources', 'state', 'heartbeat', 'downloadAsset']); +const MASTER_CALLS = new Set(['assessModelFit', 'getSystemResources', 'state', 'heartbeat', 'downloadAsset', 'cancel', 'embed', 'batchCompletion', 'ragIngest', 'ragChunk', 'ragSaveEmbeddings', 'ragSearch', 'ragReindex', 'ragDeleteEmbeddings', 'ragListWorkspaces', 'ragCloseWorkspace', 'ragDeleteWorkspace', 'translate', 'ocr', 'classify', 'diffusion', 'video', 'audioGen', 'transcribe', 'textToSpeech', 'finetune', 'bciTranscribe', 'vla', 'worldCreateScene', 'worldStep', 'modelRegistryList', 'modelRegistrySearch', 'modelRegistryGetModel']); export async function callQvac(method, input) { if (!MASTER_CALLS.has(method)) throw new Error(`QVAC method is not exposed through the master: ${method}`); const sdk = await Agent.engine.ensureInit(); if (typeof sdk[method] !== 'function') throw new Error(`QVAC SDK does not expose ${method}()`); - return input === undefined ? sdk[method]() : sdk[method](input); + return withQvacMaster(() => input === undefined ? sdk[method]() : Array.isArray(input) ? sdk[method](...input) : sdk[method](input)); } export function qvacStatus() { diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 36f67ae..c96b793 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -264,15 +264,22 @@ Exit gate: a user can configure the complete system without editing JSON. ## Phase 9 — full QVAC capability coverage -- [ ] Chat, plan, summarize, rewrite, code, embeddings, RAG, batch prompts. -- [ ] Multimodal screenshot/file analysis, OCR, classification. -- [ ] Image generation/editing, video jobs, music jobs. -- [ ] ASR, diarized meetings, TTS, voice clone enrollment, translation relay. -- [ ] LoRA training with explicit confirmation and overnight job controls. -- [ ] BCI, VLA, and ABot-World as clearly labeled Lab capabilities. -- [ ] Registry/model manager, checksums, fit assessments, profiler, and job +- [x] Chat, plan, summarize, rewrite, code, embeddings, RAG, batch prompts. +- [x] Multimodal screenshot/file analysis, OCR, classification. +- [x] Image generation/editing, video jobs, music jobs. +- [x] ASR, diarized meetings, TTS, voice clone enrollment, translation relay. +- [x] LoRA training with explicit confirmation and overnight job controls. +- [x] BCI, VLA, and ABot-World as clearly labeled Lab capabilities. +- [x] Registry/model manager, checksums, fit assessments, profiler, and job cancellation. -- [ ] Optional P2P model fetch and memory sync only after local mode is stable. +- [x] Optional P2P model fetch and memory sync only after local mode is stable. + +Phase 9 adapters are registered in `skills/phase9-tools.js`. Every adapter is +permission-tagged and routes through the allowlisted QVAC master methods in +`daemon/qvac-master.js`; the harness and UI never import QVAC directly. Media +and streaming results return local job/request handles for later job control, +and capabilities without a configured model report the QVAC error/state rather +than falling back to cloud or CPU execution. Exit gate: every capability in the product inventory has a working skill or a truthful GPU/model-unavailable state. diff --git a/skills/catalog.js b/skills/catalog.js index a4de257..6858d4a 100644 --- a/skills/catalog.js +++ b/skills/catalog.js @@ -3,14 +3,28 @@ export const SKILLS = [ ['plan', 'Multi-step planning through the extracted harness loop', 'read'], ['embed', 'QVAC embeddings for local memory and retrieval', 'read'], ['remember', 'QVAC RAG workspace ingestion and search', 'write'], + ['batch-prompt', 'Local batch completion jobs', 'write'], + ['ask-my-files', 'Local RAG workspace question answering', 'read'], ['look-at-this', 'Local screenshot and QVAC multimodal analysis', 'read'], + ['ocr', 'Local OCR for documents and screens', 'read'], + ['what-is-this', 'Local image classification', 'read'], ['dictate', 'Transcribe and inject text into the focused client', 'write'], + ['transcribe-file', 'Local audio transcription', 'read'], + ['meeting', 'Local diarized meeting transcription', 'read'], + ['speak', 'Local QVAC text-to-speech playback', 'read'], + ['clone-voice', 'Local voice enrollment and cloning', 'write'], ['translate', 'Local QVAC/Bergamot translation', 'read'], ['imagine', 'Local QVAC diffusion image generation', 'write'], + ['edit-image', 'Local image-to-image editing', 'write'], ['make-video', 'Local QVAC video generation job', 'write'], ['compose', 'Local QVAC AudioGen music job', 'write'], - ['speak', 'Local QVAC text-to-speech playback', 'read'], + ['teach-me', 'Confirmed local LoRA training', 'dangerous'], + ['bci', 'Local BCI transcription Lab slot', 'read'], + ['robot', 'Local VLA Lab slot', 'dangerous'], + ['world', 'Local ABot-World Lab sandbox', 'write'], ['assess-model-fit', 'Preflight model memory fit before download', 'read'], + ['model-registry', 'Local QVAC model registry and checksums', 'read'], + ['p2p-sync', 'Optional explicitly enabled local-device sync slot', 'write'], ['cu.observe', 'Wayland-first desktop observation bundle', 'computer-use'], ['cu.act', 'AT-SPI semantic desktop action', 'computer-use'], ['cu.click', 'Portal/libei or opted-in fallback click', 'computer-use'], diff --git a/skills/optional-sync.js b/skills/optional-sync.js new file mode 100644 index 0000000..c07cdf5 --- /dev/null +++ b/skills/optional-sync.js @@ -0,0 +1,5 @@ +/** Optional post-local-mode transport boundary. It is disabled unless the + * user explicitly enables it; no inference or memory leaves the machine by + * default. The actual Holepunch transport is intentionally isolated here. */ +export function p2pStatus() { return { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false, reason: process.env.JARVIS_P2P_ENABLE === '1' ? 'transport adapter not configured' : 'disabled by default' }; } +export async function syncMemory() { throw new Error('optional P2P memory sync is disabled until explicitly configured'); } diff --git a/skills/phase9-tools.js b/skills/phase9-tools.js new file mode 100644 index 0000000..3e928cc --- /dev/null +++ b/skills/phase9-tools.js @@ -0,0 +1,56 @@ +import { callQvac } from '../daemon/qvac-master.js'; +import { p2pStatus } from './optional-sync.js'; + +const READ = 'read'; +const WRITE = 'write'; +const DANGEROUS = 'dangerous'; +const specs = [ + ['embed', 'embed', 'Create local text embeddings.', READ], + ['rag_ingest', 'ragIngest', 'Ingest local documents into a QVAC RAG workspace.', WRITE], + ['rag_chunk', 'ragChunk', 'Chunk local documents for RAG.', READ], + ['rag_save_embeddings', 'ragSaveEmbeddings', 'Save local RAG embeddings.', WRITE], + ['rag_search', 'ragSearch', 'Search a local QVAC RAG workspace.', READ], + ['rag_reindex', 'ragReindex', 'Reindex a local QVAC RAG workspace.', WRITE], + ['rag_delete_embeddings', 'ragDeleteEmbeddings', 'Delete embeddings from a local workspace.', DANGEROUS], + ['rag_list_workspaces', 'ragListWorkspaces', 'List local QVAC RAG workspaces.', READ], + ['rag_close_workspace', 'ragCloseWorkspace', 'Close a local RAG workspace.', WRITE], + ['rag_delete_workspace', 'ragDeleteWorkspace', 'Delete a local RAG workspace.', DANGEROUS], + ['batch_prompt', 'batchCompletion', 'Run a local batch completion.', WRITE], + ['translate', 'translate', 'Translate text locally.', READ], + ['ocr', 'ocr', 'Run local QVAC OCR on an image.', READ], + ['classify_image', 'classify', 'Classify a local image.', READ], + ['imagine', 'diffusion', 'Generate or edit a local image.', WRITE], + ['make_video', 'video', 'Generate a local video job.', WRITE], + ['compose_music', 'audioGen', 'Generate a local music job.', WRITE], + ['transcribe_file', 'transcribe', 'Transcribe a local audio file.', READ], + ['tts_speak', 'textToSpeech', 'Synthesize local speech.', READ], + ['finetune_lora', 'finetune', 'Run a confirmed local LoRA training job.', DANGEROUS], + ['bci_transcribe', 'bciTranscribe', 'Use the local BCI transcription slot.', READ], + ['vla_act', 'vla', 'Run the local VLA Lab slot.', DANGEROUS], + ['world_create', 'worldCreateScene', 'Create a local ABot-World scene.', WRITE], + ['world_step', 'worldStep', 'Advance a local ABot-World scene.', WRITE], + ['model_registry_list', 'modelRegistryList', 'List models in the local QVAC registry.', READ], + ['model_registry_search', 'modelRegistrySearch', 'Search the local QVAC registry.', READ], + ['model_registry_get', 'modelRegistryGetModel', 'Read one local registry model entry.', READ], + ['qvac_profile', 'getSystemResources', 'Read local QVAC resource/profile data.', READ], +]; + +function resultShape(value) { + if (!value || typeof value !== 'object') return value; + if (value.requestId) return { requestId: value.requestId, job: true }; + if (value.bufferStream || value.blockStream || value.tokenStream) return { job: true, stream: true, requestId: value.requestId || null }; + return value; +} + +export function createPhase9Tools() { + const tools = specs.map(([name, method, description, permission]) => ({ name, permission, description: `${description} All execution remains local and is routed through the single QVAC master.`, parameters: { type: 'object', properties: { input: { type: 'object' } } }, execute: async ({ input = {} } = {}) => { + if (method === 'modelRegistryGetModel') return resultShape(await callQvac(method, [input.registryPath || input.path, input.registrySource || input.source])); + if (method === 'ocr') { const result = await callQvac(method, input); return { blocks: await result.blocks }; } + return resultShape(await callQvac(method, input)); + }})); + tools.push({ name: 'job_cancel', permission: 'dangerous', description: 'Cancel a local QVAC request or model job through the master.', parameters: { type: 'object', properties: { requestId: { type: 'string' }, modelId: { type: 'string' }, kind: { type: 'string' } } }, execute: async (input) => callQvac('cancel', input) }); + tools.push({ name: 'p2p_status', permission: 'read', description: 'Report optional P2P model/memory transport status; it is disabled by default.', parameters: { type: 'object', properties: {} }, execute: () => p2pStatus() }); + return tools; +} + +export { specs as PHASE9_SPECS }; diff --git a/test/phase9-tools.test.js b/test/phase9-tools.test.js new file mode 100644 index 0000000..5b232ab --- /dev/null +++ b/test/phase9-tools.test.js @@ -0,0 +1,15 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createPhase9Tools, PHASE9_SPECS } from '../skills/phase9-tools.js'; +import { SKILLS } from '../skills/catalog.js'; + +test('Phase 9 exposes the complete local QVAC capability surface', () => { + const tools = createPhase9Tools(); + assert.ok(PHASE9_SPECS.length >= 25); + assert.ok(tools.some((tool) => tool.name === 'embed')); + assert.ok(tools.some((tool) => tool.name === 'imagine')); + assert.ok(tools.some((tool) => tool.name === 'finetune_lora' && tool.permission === 'dangerous')); + assert.ok(tools.some((tool) => tool.name === 'job_cancel')); + assert.ok(SKILLS.some(([id]) => id === 'model-registry')); + for (const tool of tools) assert.ok(tool.parameters?.type === 'object', tool.name); +});