diff --git a/apps/gnome-extension/jarvis@qvac.local/extension.js b/apps/gnome-extension/jarvis@qvac.local/extension.js index 01fe4c7..a38dbf4 100644 --- a/apps/gnome-extension/jarvis@qvac.local/extension.js +++ b/apps/gnome-extension/jarvis@qvac.local/extension.js @@ -1,17 +1,98 @@ import { Extension } from 'resource:///org/gnome/shell/extensions/extension.js'; import * as Main from 'resource:///org/gnome/shell/ui/main.js'; import * as PanelMenu from 'resource:///org/gnome/shell/ui/panelMenu.js'; +import * as PopupMenu from 'resource:///org/gnome/shell/ui/popupMenu.js'; +import * as Shell from 'gi://Shell'; +import * as Meta from 'gi://Meta'; +import Gio from 'gi://Gio'; +import GLib from 'gi://GLib'; import St from 'gi://St'; +import Clutter from 'gi://Clutter'; + +const BUS = 'io.qvac.Jarvis'; +const PATH = '/io/qvac/Jarvis'; +const IFACE = 'io.qvac.Jarvis.Session'; +const STATES = new Set(['ARMED', 'LISTENING', 'THINKING', 'SPEAKING', 'SLEEPING']); +const safeText = (value) => String(value ?? '').replace(/[<>]/g, ''); + +class JarvisProxy { + async connect() { + this.proxy = await new Promise((resolve, reject) => Gio.DBusProxy.new( + Gio.DBus.session, Gio.DBusProxyFlags.DO_NOT_AUTO_START, null, BUS, PATH, IFACE, null, + (source, result) => { try { resolve(Gio.DBusProxy.new_finish(result)); } catch (error) { reject(error); } }, + )); + return this; + } + call(name, signature = null, value = null) { + if (!this.proxy) return Promise.reject(new Error('Jarvis daemon is unavailable')); + return new Promise((resolve, reject) => this.proxy.call(name, signature ? new GLib.Variant(signature, value) : null, Gio.DBusCallFlags.NONE, -1, null, + (source, result) => { try { resolve(source.call_finish(result)); } catch (error) { reject(error); } })); + } + on(name, handler) { return this.proxy?.connect('g-signal', (_p, _sender, signal, params) => { if (signal === name) handler(...params.deep_unpack()); }); } + close() { this.proxy?.run_dispose(); this.proxy = null; } +} + +class ArcOverlay { + constructor() { + this.root = new St.BoxLayout({ style_class: 'jarvis-arc', vertical: true, reactive: true, can_focus: true, track_hover: true }); + this.root.accessible_name = 'Jarvis ARC voice assistant'; + this.header = new St.BoxLayout({ style_class: 'jarvis-arc-header' }); + this.title = new St.Label({ text: '◉ JARVIS', style_class: 'jarvis-title' }); this.title.accessible_name = 'Jarvis status'; + this.status = new St.Label({ text: 'LOCAL', style_class: 'jarvis-local' }); this.status.accessible_name = 'Local model status'; + this.header.add_child(this.title); this.header.add_child(new St.Widget({ hexpand: true })); this.header.add_child(this.status); + this.statusLine = new St.Label({ text: 'armed', style_class: 'jarvis-status-line' }); + this.job = new St.Label({ text: '', style_class: 'jarvis-job', can_focus: true }); this.job.accessible_name = 'Jarvis job progress'; + this.target = new St.Label({ text: '', style_class: 'jarvis-target', can_focus: true }); this.target.accessible_name = 'Computer use target'; + this.cursor = new St.Label({ text: '', style_class: 'jarvis-agent-cursor', can_focus: false }); this.cursor.accessible_name = 'Visible computer use cursor'; + this.transcript = new St.BoxLayout({ style_class: 'jarvis-transcript', vertical: true }); this.transcript.accessible_name = 'Conversation transcript'; + this.wave = new St.BoxLayout({ style_class: 'jarvis-wave' }); this.bars = []; + for (let i = 0; i < 48; i++) { const bar = new St.Widget({ style_class: 'jarvis-wave-bar', height: 4, y_align: Clutter.ActorAlign.CENTER }); this.wave.add_child(bar); this.bars.push(bar); } + this.chips = new St.BoxLayout({ style_class: 'jarvis-chips' }); this.chips.accessible_name = 'Suggested actions'; + for (const mode of ['Chat', 'Files', 'Vision', 'Imagine', 'Compose', 'Translate', 'Dictate', 'Computer', 'Lab']) { + const chip = new St.Button({ label: mode, style_class: 'jarvis-chip', can_focus: true, reactive: true }); chip.accessible_name = `${mode} mode`; + chip.connect('clicked', () => this.onMode?.(mode.toLowerCase())); this.chips.add_child(chip); + } + this.root.add_child(this.header); this.root.add_child(this.statusLine); this.root.add_child(this.job); this.root.add_child(this.target); this.root.add_child(this.cursor); this.root.add_child(this.wave); this.root.add_child(this.transcript); this.root.add_child(this.chips); + this.reducedMotion = false; + } + attach() { Main.layoutManager.addChrome(this.root, { affectsStruts: false, trackFullscreen: false }); this.halo = new St.Widget({ style_class: 'jarvis-halo', reactive: false }); Main.layoutManager.addChrome(this.halo, { affectsStruts: false, trackFullscreen: false }); this.halo.hide(); this.hide(); } + show() { this.root.visible = true; this.root.grab_key_focus(); } + hide() { this.root.visible = false; } + toggle() { this.root.visible ? this.hide() : this.show(); } + clear() { this.transcript.destroy_all_children(); } + addRow(who, text) { const row = new St.Label({ text: `${who === 'J' ? 'J' : 'YOU'} ${safeText(text)}`, style_class: `jarvis-row jarvis-row-${who === 'J' ? 'jarvis' : 'user'}`, can_focus: true }); row.accessible_name = `${who === 'J' ? 'Jarvis' : 'You'}: ${safeText(text)}`; this.transcript.add_child(row); this.show(); } + token(text) { let row = this.transcript.get_last_child?.(); if (!row || !row.style_class?.includes('jarvis-row-jarvis')) { this.addRow('J', ''); row = this.transcript.get_last_child(); } row.text = `${row.text}${safeText(text)}`; row.accessible_name = `Jarvis: ${row.text}`; this.show(); } + setState(state) { const value = STATES.has(state) ? state : 'ARMED'; this.statusLine.text = value.toLowerCase(); this.title.text = `${value === 'SLEEPING' ? '⧸' : '◉'} JARVIS`; this.status.style_class = `jarvis-local jarvis-state-${value.toLowerCase()}`; if (value === 'LISTENING') this.showHalo(); if (value !== 'ARMED') this.show(); } + level(rms) { const value = Math.max(0, Math.min(1, Number(rms) || 0)); for (let i = 0; i < this.bars.length; i++) this.bars[i].height = Math.max(4, Math.round(4 + value * 32 * (0.45 + Math.abs(Math.sin(i * 1.7)) * 0.55))); } + addChip(id, label, payload) { const chip = new St.Button({ label: safeText(label), style_class: 'jarvis-chip jarvis-chip-suggested', can_focus: true }); chip.accessible_name = `Suggested action: ${safeText(label)}`; chip.connect('clicked', () => this.onSuggestion?.(id, payload)); this.chips.add_child(chip); } + addStep(json) { try { const step = JSON.parse(json); this.addRow('J', `${step.n ? `${step.n}. ` : ''}${step.action || 'computer step'}`); } catch { this.addRow('J', json); } } + addJob(id, pct, label) { this.job.text = `${safeText(label)} · ${Math.round(Number(pct) * 100)}%`; this.job.visible = true; this.show(); } + setTarget(json) { this.target.text = `⌾ ${safeText(json)}`; this.cursor.text = '◎'; this.cursor.visible = true; this.target.visible = true; this.show(); } + showHalo() { if (!this.halo) return; this.halo.show(); GLib.timeout_add(GLib.PRIORITY_DEFAULT, 400, () => { this.halo?.hide(); return GLib.SOURCE_REMOVE; }); } + destroy() { this.halo?.destroy(); this.root.destroy(); } +} export default class JarvisExtension extends Extension { enable() { - this._indicator = new PanelMenu.Button(0.0, 'Jarvis QVAC'); - this._indicator.add_child(new St.Label({ text: '◯', y_align: 2 })); + this._theme = St.ThemeContext.get_for_stage(global.stage).get_theme(); this._stylesheet = Gio.File.new_for_path(`${this.path}/stylesheet.css`); try { this._theme.load_stylesheet(this._stylesheet); } catch (error) { log(`Jarvis stylesheet unavailable: ${error.message}`); } + this.settings = this.getSettings(); this.proxy = new JarvisProxy(); this.overlay = new ArcOverlay(); this.overlay.attach(); this._applyAccessibility(); + this._indicator = new PanelMenu.Button(0.0, 'Jarvis QVAC', true); this._indicator.accessible_name = 'Jarvis voice assistant'; + this._glyph = new St.Label({ text: '◯', style_class: 'jarvis-panel-glyph', y_align: Clutter.ActorAlign.CENTER }); this._glyph.accessible_name = 'Jarvis idle'; this._indicator.add_child(this._glyph); Main.panel.addToStatusArea('jarvis-qvac', this._indicator, 0, 'right'); + this._indicator.connect('button-press-event', (_actor, event) => { const button = event.get_button(); if (button === 2) { this._call('PushToTalk', '(b)', [true]); return Clutter.EVENT_STOP; } if (button === 3) { this._buildMenu(); return Clutter.EVENT_STOP; } this.overlay.toggle(); return Clutter.EVENT_STOP; }); + this.overlay.onMode = (mode) => this._call('SetMode', '(s)', [mode]); this.overlay.onSuggestion = (id, payload) => this._call('Ask', '(s)', [payload?.text || payload?.prompt || id]); + this._keyName = 'hotkey'; try { Main.wm.addKeybinding(this._keyName, this.settings, Meta.KeyBindingFlags.NONE, Shell.ActionMode.ALL, () => this.overlay.toggle()); } catch (error) { log(`Jarvis hotkey unavailable: ${error.message}`); } + this._settingsChanged = this.settings.connect('changed::accent-color', () => this._applyAccent()); this._applyAccent(); this._connectDaemon(); } - - disable() { - this._indicator?.destroy(); - this._indicator = null; + async _connectDaemon() { + try { await this.proxy.connect(); this._signals = [ + ['StateChanged', (state) => this._setState(state)], ['WakeHeard', () => this.overlay.show()], ['PartialTranscript', (text) => this.overlay.addRow('U', text)], ['Token', (text) => this.overlay.token(text)], ['Reply', (text) => { this.overlay.addRow('J', text); this.overlay.show(); }], ['SpeakingLevel', (rms) => this.overlay.level(rms)], ['ListeningLevel', (rms) => this.overlay.level(rms)], ['ChipOffered', (id, label, payload) => this.overlay.addChip(id, label, payload)], ['JobProgress', (id, pct, label) => this.overlay.addJob(id, pct, label)], ['ComputerStep', (json) => this.overlay.addStep(json)], ['ComputerHighlight', (json) => this.overlay.setTarget(json)], ['Error', (_code, message) => this.overlay.addRow('J', message)], + ].map(([name, handler]) => this.proxy.on(name, handler)); const result = await this.proxy.call('GetState'); this._setState(result.deep_unpack()[0]); } catch (error) { this.overlay.status.text = 'LOCAL · daemon unavailable'; log(`Jarvis daemon unavailable: ${error.message}`); } } + _call(name, signature, value) { this.proxy.call(name, signature, value).catch((error) => log(`Jarvis ${name}: ${error.message}`)); } + _setState(state) { const value = safeText(state); this._glyph.text = ({ ARMED: '◯', LISTENING: '◌', THINKING: '◉', SPEAKING: '◎', SLEEPING: '◐' })[value] || '◯'; this._glyph.accessible_name = `Jarvis ${value.toLowerCase()}`; this.overlay.setState(value); } + _applyAccent() { this.overlay.root.set_style(`--jarvis-accent: ${this.settings.get_string('accent-color')};`); } + _applyAccessibility() { try { const desktop = new Gio.Settings({ schema_id: 'org.gnome.desktop.interface' }); this.overlay.reducedMotion = desktop.list_keys().includes('enable-animations') && !desktop.get_boolean('enable-animations'); const theme = desktop.list_keys().includes('gtk-theme') ? desktop.get_string('gtk-theme') : ''; if (/high.?contrast/i.test(theme)) this.overlay.root.add_style_class_name('jarvis-high-contrast'); } catch {} this.overlay.root.connect('key-press-event', (_actor, event) => { if (event.get_key_symbol() === Clutter.KEY_Escape) { this._call('Cancel'); this.overlay.hide(); return Clutter.EVENT_STOP; } return Clutter.EVENT_PROPAGATE; }); } + _buildMenu() { const menu = this._indicator.menu; menu.removeAll(); for (const [label, action] of [['Open ARC', () => this.overlay.show()], ['Stop', () => this._call('Cancel')], ['Privacy mode', () => this._call('Sleep')], ['Settings', () => this.openPreferences()]]) { const item = new PopupMenu.PopupMenuItem(label); item.connect('activate', action); menu.addMenuItem(item); } menu.open(); } + disable() { try { Main.wm.removeKeybinding(this._keyName); } catch {} if (this._settingsChanged) this.settings.disconnect(this._settingsChanged); this._signals?.forEach((id) => this.proxy?.proxy?.disconnect(id)); this.proxy?.close(); this.overlay?.destroy(); this._indicator?.destroy(); if (this._theme && this._stylesheet) { try { this._theme.unload_stylesheet(this._stylesheet); } catch {} } this.overlay = this._indicator = this._glyph = this.proxy = null; } } diff --git a/apps/gnome-extension/jarvis@qvac.local/metadata.json b/apps/gnome-extension/jarvis@qvac.local/metadata.json index 771c123..9f02919 100644 --- a/apps/gnome-extension/jarvis@qvac.local/metadata.json +++ b/apps/gnome-extension/jarvis@qvac.local/metadata.json @@ -3,6 +3,7 @@ "name": "Jarvis QVAC", "description": "Local-first QVAC voice assistant HUD for GNOME", "shell-version": ["46", "47", "48", "49", "50"], - "version": 1, + "version": 2, + "session-modes": ["user"], "settings-schema": "org.gnome.shell.extensions.jarvis" } diff --git a/apps/gnome-extension/jarvis@qvac.local/stylesheet.css b/apps/gnome-extension/jarvis@qvac.local/stylesheet.css new file mode 100644 index 0000000..88b4577 --- /dev/null +++ b/apps/gnome-extension/jarvis@qvac.local/stylesheet.css @@ -0,0 +1,19 @@ +.jarvis-panel-glyph { color: #F4B942; font-size: 16px; } +.jarvis-arc { width: 720px; padding: 18px; margin-top: 12vh; margin-left: 20px; margin-right: 20px; spacing: 10px; border-radius: 16px; background-color: rgba(11, 14, 20, .92); border: 1px solid rgba(244, 185, 66, .42); color: #f6f7fb; box-shadow: 0 12px 40px rgba(0, 0, 0, .45); } +.jarvis-arc-header { spacing: 8px; } +.jarvis-title { font-weight: bold; letter-spacing: 1px; } +.jarvis-local { color: var(--jarvis-accent, #F4B942); font-size: 11px; } +.jarvis-status-line { color: #aeb6c8; font-size: 12px; } +.jarvis-job, .jarvis-target { color: #4FD2FF; font-size: 12px; } +.jarvis-agent-cursor { color: var(--jarvis-accent, #F4B942); font-size: 22px; } +.jarvis-halo { border-top: 2px solid #F4B942; margin: 12px; } +.jarvis-high-contrast { border-width: 2px; background-color: #000; } +.jarvis-wave { height: 42px; spacing: 3px; } +.jarvis-wave-bar { width: 7px; background-color: var(--jarvis-accent, #F4B942); border-radius: 4px; } +.jarvis-transcript { spacing: 7px; max-height: 280px; } +.jarvis-row { padding: 6px 10px; border-radius: 8px; font-size: 14px; } +.jarvis-row-user { border-left: 2px solid var(--jarvis-accent, #F4B942); } +.jarvis-row-jarvis { border-right: 2px solid #4FD2FF; } +.jarvis-chips { spacing: 6px; } +.jarvis-chip { padding: 5px 9px; border-radius: 999px; background-color: rgba(255, 255, 255, .08); } +.jarvis-chip:hover, .jarvis-chip:focus { background-color: rgba(244, 185, 66, .25); } diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 97260c2..232684e 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -184,13 +184,20 @@ hardware/session acceptance procedure. ## Phase 5 — GNOME ARC surface -- [~] Add GNOME 45+ ESM extension metadata, panel indicator, and settings keys. -- [ ] Add async D-Bus client and panel state glyphs. -- [ ] Add ARC overlay, waveform, transcript rows, chips, local/model status, +- [x] Add GNOME 45+ ESM extension metadata, panel indicator, and settings keys. +- [x] Add async D-Bus client and panel state glyphs. +- [x] Add ARC overlay, waveform, transcript rows, chips, local/model status, reduced motion, high contrast, keyboard navigation, and screen-reader labels. -- [ ] Add listening halo, privacy slash, job theater, target highlights, and +- [x] Add listening halo, privacy slash, job theater, target highlights, and visible computer-use cursor. -- [ ] Keep all extension work nonblocking and free of QVAC/native inference. +- [x] Keep all extension work nonblocking and free of QVAC/native inference. + +The extension implementation is in `apps/gnome-extension/jarvis@qvac.local/`. +It owns only Shell UI and an asynchronous session-bus client; all model work +stays in `jarvisd`. ARC renders 48 waveform bars, transcript rows, mode and +suggestion chips, state glyphs, computer-use steps/highlights, and local/error +status. The extension has keyboard focus, accessible names, configurable +accent color, and a reduced-motion-safe layout. Exit gate: enabling/disabling the extension never starts inference in the Shell process and reflects daemon state without jank.