diff --git a/apps/gnome-extension/jarvis@qvac.local/extension.js b/apps/gnome-extension/jarvis@qvac.local/extension.js index 55ecf08..0a74097 100644 --- a/apps/gnome-extension/jarvis@qvac.local/extension.js +++ b/apps/gnome-extension/jarvis@qvac.local/extension.js @@ -2,75 +2,137 @@ import { Extension } from 'resource:///org/gnome/shell/extensions/extension.js'; import * as Main from 'resource:///org/gnome/shell/ui/main.js'; import * as PanelMenu from 'resource:///org/gnome/shell/ui/panelMenu.js'; import * as PopupMenu from 'resource:///org/gnome/shell/ui/popupMenu.js'; -import * as Shell from 'gi://Shell'; -import * as Meta from 'gi://Meta'; +import Shell from 'gi://Shell'; +import Meta from 'gi://Meta'; import Gio from 'gi://Gio'; import GLib from 'gi://GLib'; import St from 'gi://St'; import Clutter from 'gi://Clutter'; +import Pango from 'gi://Pango'; import { installShellService } from './shell-dbus.js'; const BUS = 'io.qvac.Jarvis'; const PATH = '/io/qvac/Jarvis'; const IFACE = 'io.qvac.Jarvis.Session'; +const OVERLAY_WIDTH = 720; const STATES = new Set(['ARMED', 'LISTENING', 'THINKING', 'SPEAKING', 'SLEEPING']); -const safeText = (value) => String(value ?? '').replace(/[<>]/g, ''); +const coerceText = (value) => { + if (value == null) return ''; + if (typeof value === 'string') return value === '[object Object]' ? '' : value; + if (typeof value === 'number' || typeof value === 'boolean') return String(value); + if (typeof value === 'object') { + if (typeof value.text === 'string') return value.text; + if (typeof value.message === 'string') return value.message; + if (typeof value.deep_unpack === 'function') return coerceText(value.deep_unpack()); + } + const fallback = String(value); + return fallback === '[object Object]' ? '' : fallback; +}; +const safeText = (value) => coerceText(value).replace(/[<>]/g, ''); class JarvisProxy { async connect() { - this.proxy = await new Promise((resolve, reject) => Gio.DBusProxy.new( + const proxy = await new Promise((resolve, reject) => Gio.DBusProxy.new( Gio.DBus.session, Gio.DBusProxyFlags.DO_NOT_AUTO_START, null, BUS, PATH, IFACE, null, (source, result) => { try { resolve(Gio.DBusProxy.new_finish(result)); } catch (error) { reject(error); } }, )); + if (this._closed) { proxy.run_dispose(); return this; } + this.proxy = proxy; + this._ownerSignal = proxy.connect('notify::g-name-owner', () => this.onOwnerChanged?.(this.owned())); return this; } + owned() { try { return Boolean(this.proxy?.g_name_owner || this.proxy?.get_name_owner?.()); } catch { return false; } } call(name, signature = null, value = null) { if (!this.proxy) return Promise.reject(new Error('Jarvis daemon is unavailable')); return new Promise((resolve, reject) => this.proxy.call(name, signature ? new GLib.Variant(signature, value) : null, Gio.DBusCallFlags.NONE, -1, null, (source, result) => { try { resolve(source.call_finish(result)); } catch (error) { reject(error); } })); } on(name, handler) { return this.proxy?.connect('g-signal', (_p, _sender, signal, params) => { if (signal === name) handler(...params.deep_unpack()); }); } - close() { this.proxy?.run_dispose(); this.proxy = null; } + close() { + this._closed = true; this.onOwnerChanged = null; + if (this._ownerSignal && this.proxy) { try { this.proxy.disconnect(this._ownerSignal); } catch {} } + this.proxy?.run_dispose(); this.proxy = null; + } } class ArcOverlay { constructor() { - this.root = new St.BoxLayout({ style_class: 'jarvis-arc', vertical: true, reactive: true, can_focus: true, track_hover: true }); + this.root = new St.BoxLayout({ style_class: 'jarvis-arc', vertical: true, reactive: true, can_focus: true, track_hover: true, x_expand: true }); this.root.accessible_name = 'Jarvis ARC voice assistant'; - this.header = new St.BoxLayout({ style_class: 'jarvis-arc-header' }); - this.title = new St.Label({ text: '◉ JARVIS', style_class: 'jarvis-title' }); this.title.accessible_name = 'Jarvis status'; + this.header = new St.BoxLayout({ style_class: 'jarvis-arc-header', x_expand: true }); + this.title = new St.Label({ text: '◉ JARVIS', style_class: 'jarvis-title', x_align: Clutter.ActorAlign.START }); this.title.accessible_name = 'Jarvis status'; + if (this.title.clutter_text) this.title.clutter_text.ellipsize = Pango.EllipsizeMode.NONE; this.status = new St.Label({ text: 'LOCAL', style_class: 'jarvis-local' }); this.status.accessible_name = 'Local model status'; - this.header.add_child(this.title); this.header.add_child(new St.Widget({ hexpand: true })); this.header.add_child(this.status); - this.statusLine = new St.Label({ text: 'armed', style_class: 'jarvis-status-line' }); - this.job = new St.Label({ text: '', style_class: 'jarvis-job', can_focus: true }); this.job.accessible_name = 'Jarvis job progress'; - this.target = new St.Label({ text: '', style_class: 'jarvis-target', can_focus: true }); this.target.accessible_name = 'Computer use target'; - this.cursor = new St.Label({ text: '', style_class: 'jarvis-agent-cursor', can_focus: false }); this.cursor.accessible_name = 'Visible computer use cursor'; - this.transcript = new St.BoxLayout({ style_class: 'jarvis-transcript', vertical: true }); this.transcript.accessible_name = 'Conversation transcript'; - this.wave = new St.BoxLayout({ style_class: 'jarvis-wave' }); this.bars = []; + if (this.status.clutter_text) this.status.clutter_text.ellipsize = Pango.EllipsizeMode.END; + this.header.add_child(this.title); this.header.add_child(new St.Widget({ x_expand: true })); this.header.add_child(this.status); + this.statusLine = new St.Label({ text: 'Super+Shift+J · Hold Talk to speak', style_class: 'jarvis-status-line' }); + this.job = new St.Label({ text: '', style_class: 'jarvis-job', can_focus: true, visible: false }); this.job.accessible_name = 'Jarvis job progress'; + this.target = new St.Label({ text: '', style_class: 'jarvis-target', can_focus: true, visible: false }); this.target.accessible_name = 'Computer use target'; + this.cursor = new St.Label({ text: '', style_class: 'jarvis-agent-cursor', can_focus: false, visible: false }); this.cursor.accessible_name = 'Visible computer use cursor'; + this.scroll = new St.ScrollView({ style_class: 'jarvis-transcript-scroll', overlay_scrollbars: true, x_expand: true, y_expand: true }); + try { this.scroll.hscrollbar_policy = St.PolicyType.NEVER; this.scroll.vscrollbar_policy = St.PolicyType.AUTOMATIC; } catch {} + this.transcript = new St.BoxLayout({ style_class: 'jarvis-transcript', vertical: true, x_expand: true }); this.transcript.accessible_name = 'Conversation transcript'; + if (typeof this.scroll.set_child === 'function') this.scroll.set_child(this.transcript); else this.scroll.add_child(this.transcript); + this.wave = new St.BoxLayout({ style_class: 'jarvis-wave', visible: false }); this.bars = []; for (let i = 0; i < 48; i++) { const bar = new St.Widget({ style_class: 'jarvis-wave-bar', height: 4, y_align: Clutter.ActorAlign.CENTER }); this.wave.add_child(bar); this.bars.push(bar); } this.chips = new St.BoxLayout({ style_class: 'jarvis-chips' }); this.chips.accessible_name = 'Suggested actions'; for (const mode of ['Chat', 'Files', 'Vision', 'Imagine', 'Compose', 'Translate', 'Dictate', 'Computer', 'Lab']) { const chip = new St.Button({ label: mode, style_class: 'jarvis-chip', can_focus: true, reactive: true }); chip.accessible_name = `${mode} mode`; chip.connect('clicked', () => this.onMode?.(mode.toLowerCase())); this.chips.add_child(chip); } - this.root.add_child(this.header); this.root.add_child(this.statusLine); this.root.add_child(this.job); this.root.add_child(this.target); this.root.add_child(this.cursor); this.root.add_child(this.wave); this.root.add_child(this.transcript); this.root.add_child(this.chips); + this.root.add_child(this.header); this.root.add_child(this.statusLine); this.root.add_child(this.job); this.root.add_child(this.target); this.root.add_child(this.cursor); this.root.add_child(this.wave); this.root.add_child(this.scroll); this.root.add_child(this.chips); + this.controls = new St.BoxLayout({ style_class: 'jarvis-chips' }); + this.talk = new St.Button({ label: 'Talk', style_class: 'jarvis-chip jarvis-talk', reactive: true, can_focus: true }); + this.talk.accessible_name = 'Hold to talk'; + this.talk.connect('button-press-event', () => { this.onTalk?.(true); return Clutter.EVENT_STOP; }); + this.talk.connect('button-release-event', () => { this.onTalk?.(false); return Clutter.EVENT_STOP; }); + this.controls.add_child(this.talk); + for (const [label, action] of [['Stop', () => this.onStop?.()], ['Close', () => this.hide()]]) { + const button = new St.Button({ label, style_class: 'jarvis-chip', reactive: true, can_focus: true }); + button.connect('clicked', action); this.controls.add_child(button); + } + this.entry = new St.Entry({ hint_text: 'Ask Jarvis…', can_focus: true, x_expand: true }); + this.entry.clutter_text.connect('activate', () => { const text = this.entry.get_text().trim(); if (text) { this.onAsk?.(text); this.entry.set_text(''); } }); + this.root.add_child(this.entry); this.root.add_child(this.controls); this.reducedMotion = false; } attach() { Main.layoutManager.addChrome(this.root, { affectsStruts: false, trackFullscreen: false }); this.halo = new St.Widget({ style_class: 'jarvis-halo', reactive: false }); Main.layoutManager.addChrome(this.halo, { affectsStruts: false, trackFullscreen: false }); this.halo.hide(); this.hide(); } - show() { this.root.visible = true; this.root.grab_key_focus(); } + show() { const monitor = Main.layoutManager.primaryMonitor; if (monitor) this.root.set_position(monitor.x + Math.max(0, Math.round((monitor.width - OVERLAY_WIDTH) / 2)), monitor.y + 64); this.root.visible = true; this.root.grab_key_focus(); } hide() { this.root.visible = false; } toggle() { this.root.visible ? this.hide() : this.show(); } clear() { this.transcript.destroy_all_children(); } - addRow(who, text) { const row = new St.Label({ text: `${who === 'J' ? 'J' : 'YOU'} ${safeText(text)}`, style_class: `jarvis-row jarvis-row-${who === 'J' ? 'jarvis' : 'user'}`, can_focus: true }); row.accessible_name = `${who === 'J' ? 'Jarvis' : 'You'}: ${safeText(text)}`; this.transcript.add_child(row); this.show(); } - token(text) { let row = this.transcript.get_last_child?.(); if (!row || !row.style_class?.includes('jarvis-row-jarvis')) { this.addRow('J', ''); row = this.transcript.get_last_child(); } row.text = `${row.text}${safeText(text)}`; row.accessible_name = `Jarvis: ${row.text}`; this.show(); } - setState(state) { const value = STATES.has(state) ? state : 'ARMED'; this.statusLine.text = value.toLowerCase(); this.title.text = `${value === 'SLEEPING' ? '⧸' : '◉'} JARVIS`; this.status.style_class = `jarvis-local jarvis-state-${value.toLowerCase()}`; if (value === 'LISTENING') this.showHalo(); if (value !== 'ARMED') this.show(); } - level(rms) { const value = Math.max(0, Math.min(1, Number(rms) || 0)); for (let i = 0; i < this.bars.length; i++) this.bars[i].height = Math.max(4, Math.round(4 + value * 32 * (0.45 + Math.abs(Math.sin(i * 1.7)) * 0.55))); } + addRow(who, text) { const body = safeText(text); const row = new St.Label({ text: `${who === 'J' ? 'J' : 'YOU'} ${body}`, style_class: `jarvis-row jarvis-row-${who === 'J' ? 'jarvis' : 'user'}`, can_focus: true }); row.accessible_name = `${who === 'J' ? 'Jarvis' : 'You'}: ${body}`; this.transcript.add_child(row); this.show(); return row; } + token(text) { const chunk = safeText(text); if (!chunk) return; let row = this.transcript.get_last_child?.(); if (!row || !String(row.style_class || '').includes('jarvis-row-jarvis')) { row = this.addRow('J', ''); } row.text = `${row.text}${chunk}`; row.accessible_name = `Jarvis: ${row.text}`; this.show(); } + finalizeReply(text) { + const spoken = safeText(text); + let row = this.transcript.get_last_child?.(); + if (row && String(row.style_class || '').includes('jarvis-row-jarvis')) { + const body = String(row.text || '').replace(/^J\s+/, ''); + if (spoken && !body.trim()) { row.text = `J ${spoken}`; row.accessible_name = `Jarvis: ${spoken}`; } + this.show(); return; + } + if (spoken) this.addRow('J', spoken); + } + setConnectionStatus(kind) { + const labels = { local: 'LOCAL', offline: 'LOCAL · daemon unavailable', 'voice-unavailable': 'LOCAL · voice unavailable' }; + this.status.text = labels[kind] || labels.local; + this.status.accessible_name = this.status.text; + } + setState(state) { + const value = STATES.has(state) ? state : 'ARMED'; + this.statusLine.text = `${value.toLowerCase()} · Super+Shift+J · Hold Talk to speak`; + this.title.text = `${value === 'SLEEPING' ? '⧸' : '◉'} JARVIS`; + this.status.style_class = `jarvis-local jarvis-state-${value.toLowerCase()}`; + this.wave.visible = value === 'LISTENING' || value === 'SPEAKING'; + if (value === 'LISTENING') this.showHalo(); if (value !== 'ARMED') this.show(); + } + level(rms) { this.wave.visible = true; const value = Math.max(0, Math.min(1, Number(rms) || 0)); for (let i = 0; i < this.bars.length; i++) this.bars[i].height = Math.max(4, Math.round(4 + value * 32 * (0.45 + Math.abs(Math.sin(i * 1.7)) * 0.55))); } addChip(id, label, payload) { const chip = new St.Button({ label: safeText(label), style_class: 'jarvis-chip jarvis-chip-suggested', can_focus: true }); chip.accessible_name = `Suggested action: ${safeText(label)}`; chip.connect('clicked', () => this.onSuggestion?.(id, payload)); this.chips.add_child(chip); } addStep(json) { try { const step = JSON.parse(json); this.addRow('J', `${step.n ? `${step.n}. ` : ''}${step.action || 'computer step'}`); } catch { this.addRow('J', json); } } addJob(id, pct, label) { this.job.text = `${safeText(label)} · ${Math.round(Number(pct) * 100)}%`; this.job.visible = true; this.show(); } setTarget(json) { this.target.text = `⌾ ${safeText(json)}`; this.cursor.text = '◎'; this.cursor.visible = true; this.target.visible = true; this.show(); } - showHalo() { if (!this.halo) return; this.halo.show(); GLib.timeout_add(GLib.PRIORITY_DEFAULT, 400, () => { this.halo?.hide(); return GLib.SOURCE_REMOVE; }); } - destroy() { this.halo?.destroy(); this.root.destroy(); } + showHalo() { if (!this.halo) return; this.halo.show(); if (this._haloTimeout) GLib.Source.remove(this._haloTimeout); this._haloTimeout = GLib.timeout_add(GLib.PRIORITY_DEFAULT, 400, () => { this._haloTimeout = 0; this.halo?.hide(); return GLib.SOURCE_REMOVE; }); } + destroy() { if (this._haloTimeout) GLib.Source.remove(this._haloTimeout); this._haloTimeout = 0; this.halo?.destroy(); this.halo = null; this.root.destroy(); } } export default class JarvisExtension extends Extension { @@ -78,24 +140,60 @@ export default class JarvisExtension extends Extension { this._theme = St.ThemeContext.get_for_stage(global.stage).get_theme(); this._stylesheet = Gio.File.new_for_path(`${this.path}/stylesheet.css`); try { this._theme.load_stylesheet(this._stylesheet); } catch (error) { log(`Jarvis stylesheet unavailable: ${error.message}`); } this.settings = this.getSettings(); this.proxy = new JarvisProxy(); this.overlay = new ArcOverlay(); this.overlay.attach(); this._applyAccessibility(); this._removeShellService = installShellService(); - this._indicator = new PanelMenu.Button(0.0, 'Jarvis QVAC', true); this._indicator.accessible_name = 'Jarvis voice assistant'; - this._glyph = new St.Label({ text: '◯', style_class: 'jarvis-panel-glyph', y_align: Clutter.ActorAlign.CENTER }); this._glyph.accessible_name = 'Jarvis idle'; this._indicator.add_child(this._glyph); + this._indicator = new PanelMenu.Button(0.0, 'Jarvis QVAC', false); this._indicator.accessible_name = 'Jarvis voice assistant'; + this._glyph = new St.Label({ text: '◯ Jarvis', style_class: 'jarvis-panel-glyph', y_align: Clutter.ActorAlign.CENTER }); this._glyph.accessible_name = 'Jarvis idle'; this._indicator.add_child(this._glyph); Main.panel.addToStatusArea('jarvis-qvac', this._indicator, 0, 'right'); - this._indicator.connect('button-press-event', (_actor, event) => { const button = event.get_button(); if (button === 2) { this._call('PushToTalk', '(b)', [true]); return Clutter.EVENT_STOP; } if (button === 3) { this._buildMenu(); return Clutter.EVENT_STOP; } this.overlay.toggle(); return Clutter.EVENT_STOP; }); + this._indicator.connect('button-press-event', (_actor, event) => { const button = event.get_button(); if (button === 2) { this._call('Arm'); return Clutter.EVENT_STOP; } if (button === 3) { this._buildMenu(); return Clutter.EVENT_STOP; } this.overlay.toggle(); return Clutter.EVENT_STOP; }); + this.overlay.onTalk = (pressed) => { if (pressed) { this._call('PushToTalk', '(b)', [true]); this._call('Arm'); } else { this._call('PushToTalk', '(b)', [false]); } }; + this.overlay.onStop = () => this._call('Cancel'); this.overlay.onAsk = (text) => { this.overlay.addRow('U', text); this._call('Ask', '(s)', [text]); }; this.overlay.onMode = (mode) => this._call('SetMode', '(s)', [mode]); this.overlay.onSuggestion = (id, payload) => this._call('Ask', '(s)', [payload?.text || payload?.prompt || id]); - this._keyName = 'hotkey'; try { Main.wm.addKeybinding(this._keyName, this.settings, Meta.KeyBindingFlags.NONE, Shell.ActionMode.ALL, () => this.overlay.toggle()); } catch (error) { log(`Jarvis hotkey unavailable: ${error.message}`); } - this._settingsChanged = this.settings.connect('changed::accent-color', () => this._applyAccent()); this._applyAccent(); this._connectDaemon(); + this._keyName = 'hotkey'; try { Main.wm.addKeybinding(this._keyName, this.settings, Meta.KeyBindingFlags.NONE, Shell.ActionMode.NORMAL | Shell.ActionMode.OVERVIEW, () => this.overlay.toggle()); } catch (error) { log(`Jarvis hotkey unavailable: ${error.message}`); } + this._settingsChanged = this.settings.connect('changed::accent-color', () => this._applyAccent()); this._applyAccent(); this._connectDaemon(); this.overlay.show(); try { this._lockChanged = Main.screenShield.connect('locked-changed', () => { if (Main.screenShield.locked) this._call('ComputerRevoke'); }); } catch {} } async _connectDaemon() { - try { await this.proxy.connect(); this._signals = [ - ['StateChanged', (state) => this._setState(state)], ['WakeHeard', () => this.overlay.show()], ['PartialTranscript', (text) => this.overlay.addRow('U', text)], ['Token', (text) => this.overlay.token(text)], ['Reply', (text) => { this.overlay.addRow('J', text); this.overlay.show(); }], ['SpeakingLevel', (rms) => this.overlay.level(rms)], ['ListeningLevel', (rms) => this.overlay.level(rms)], ['ChipOffered', (id, label, payload) => this.overlay.addChip(id, label, payload)], ['JobProgress', (id, pct, label) => this.overlay.addJob(id, pct, label)], ['ComputerStep', (json) => this.overlay.addStep(json)], ['ComputerHighlight', (json) => this.overlay.setTarget(json)], ['Error', (_code, message) => this.overlay.addRow('J', message)], - ].map(([name, handler]) => this.proxy.on(name, handler)); const result = await this.proxy.call('GetState'); this._setState(result.deep_unpack()[0]); } catch (error) { this.overlay.status.text = 'LOCAL · daemon unavailable'; log(`Jarvis daemon unavailable: ${error.message}`); } + const proxy = this.proxy; + try { + await proxy.connect(); if (this.proxy !== proxy) return; + this._signals = [ + ['StateChanged', (state) => { this._setState(state); this._refreshVoiceStatus(); }], + ['WakeHeard', () => this.overlay.show()], + ['PartialTranscript', (text) => this.overlay.addRow('U', text)], + ['Token', (text) => this.overlay.token(text)], + ['Reply', (text) => { this.overlay.finalizeReply(text); this.overlay.show(); }], + ['SpeakingLevel', (rms) => this.overlay.level(rms)], + ['ListeningLevel', (rms) => this.overlay.level(rms)], + ['ChipOffered', (id, label, payload) => this.overlay.addChip(id, label, payload)], + ['JobProgress', (id, pct, label) => this.overlay.addJob(id, pct, label)], + ['ComputerStep', (json) => this.overlay.addStep(json)], + ['ComputerHighlight', (json) => this.overlay.setTarget(json)], + ['Error', (code, message) => { this.overlay.addRow('J', message); if (coerceText(code) === 'VOICE_UNAVAILABLE') this.overlay.setConnectionStatus('voice-unavailable'); }], + ].map(([name, handler]) => proxy.on(name, handler)); + proxy.onOwnerChanged = () => { if (this.proxy !== proxy) return; this._syncDaemon(); }; + await this._syncDaemon(); + } catch (error) { if (this.proxy !== proxy) return; this.overlay?.setConnectionStatus('offline'); log(`Jarvis daemon unavailable: ${error.message}`); } } - _call(name, signature, value) { this.proxy.call(name, signature, value).catch((error) => log(`Jarvis ${name}: ${error.message}`)); } - _setState(state) { const value = safeText(state); this._glyph.text = ({ ARMED: '◯', LISTENING: '◌', THINKING: '◉', SPEAKING: '◎', SLEEPING: '◐' })[value] || '◯'; this._glyph.accessible_name = `Jarvis ${value.toLowerCase()}`; this.overlay.setState(value); } + async _syncDaemon() { + if (!this.proxy) return; + if (!this.proxy.owned()) { this.overlay?.setConnectionStatus('offline'); return; } + try { + const result = await this.proxy.call('GetState'); if (!this.overlay) return; + this._setState(result.deep_unpack()[0]); + await this._refreshVoiceStatus(); + } catch (error) { this.overlay?.setConnectionStatus('offline'); log(`Jarvis daemon unavailable: ${error.message}`); } + } + async _refreshVoiceStatus() { + if (!this.proxy?.owned?.() || !this.overlay) return; + try { + const runtime = await this.proxy.call('GetRuntimeStatus'); + const status = JSON.parse(runtime.deep_unpack()[0] || '{}'); + this.overlay.setConnectionStatus(status.voice ? 'local' : 'voice-unavailable'); + } catch { this.overlay.setConnectionStatus('local'); } + } + _call(name, signature, value) { this.proxy.call(name, signature, value).catch((error) => { log(`Jarvis ${name}: ${error.message}`); this.overlay?.addRow('J', `${name} failed: ${error.message}`); }); } + _setState(state) { const value = safeText(state); this._glyph.text = ({ ARMED: '◯', LISTENING: '◌', THINKING: '◉', SPEAKING: '◎', SLEEPING: '◐' })[value] || '◯'; this._glyph.text += ' Jarvis'; this._glyph.accessible_name = `Jarvis ${value.toLowerCase()}`; this.overlay.setState(value); } _applyAccent() { this.overlay.root.set_style(`--jarvis-accent: ${this.settings.get_string('accent-color')};`); } _applyAccessibility() { try { const desktop = new Gio.Settings({ schema_id: 'org.gnome.desktop.interface' }); this.overlay.reducedMotion = desktop.list_keys().includes('enable-animations') && !desktop.get_boolean('enable-animations'); const theme = desktop.list_keys().includes('gtk-theme') ? desktop.get_string('gtk-theme') : ''; if (/high.?contrast/i.test(theme)) this.overlay.root.add_style_class_name('jarvis-high-contrast'); } catch {} this.overlay.root.connect('key-press-event', (_actor, event) => { if (event.get_key_symbol() === Clutter.KEY_Escape) { this._call('Cancel'); this.overlay.hide(); return Clutter.EVENT_STOP; } return Clutter.EVENT_PROPAGATE; }); } - _buildMenu() { const menu = this._indicator.menu; menu.removeAll(); for (const [label, action] of [['Open ARC', () => this.overlay.show()], ['Stop', () => this._call('Cancel')], ['Privacy mode', () => this._call('Sleep')], ['Settings', () => this.openPreferences()]]) { const item = new PopupMenu.PopupMenuItem(label); item.connect('activate', action); menu.addMenuItem(item); } menu.open(); } + _buildMenu() { const menu = this._indicator.menu; menu.removeAll(); for (const [label, action] of [['Open ARC', () => this.overlay.show()], ['Talk', () => this._call('Arm')], ['Stop', () => this._call('Cancel')], ['Privacy mode', () => this._call('Sleep')], ['Settings', () => this.openPreferences()]]) { const item = new PopupMenu.PopupMenuItem(label); item.connect('activate', action); menu.addMenuItem(item); } menu.open(); } disable() { this._removeShellService?.(); try { Main.screenShield.disconnect(this._lockChanged); } catch {} try { Main.wm.removeKeybinding(this._keyName); } catch {} if (this._settingsChanged) this.settings.disconnect(this._settingsChanged); this._signals?.forEach((id) => this.proxy?.proxy?.disconnect(id)); this.proxy?.close(); this.overlay?.destroy(); this._indicator?.destroy(); if (this._theme && this._stylesheet) { try { this._theme.unload_stylesheet(this._stylesheet); } catch {} } this.overlay = this._indicator = this._glyph = this.proxy = null; } } diff --git a/apps/gnome-extension/jarvis@qvac.local/prefs.js b/apps/gnome-extension/jarvis@qvac.local/prefs.js index 8f2a5b8..2bd0085 100644 --- a/apps/gnome-extension/jarvis@qvac.local/prefs.js +++ b/apps/gnome-extension/jarvis@qvac.local/prefs.js @@ -1,7 +1,15 @@ import { ExtensionPreferences } from 'resource:///org/gnome/Shell/Extensions/js/extensions/prefs.js'; +import Adw from 'gi://Adw'; export default class JarvisPreferences extends ExtensionPreferences { fillPreferencesWindow(window) { window.set_title('Jarvis QVAC'); + const page = new Adw.PreferencesPage(); + const group = new Adw.PreferencesGroup({ title: 'Using Jarvis' }); + const settings = this.getSettings(); + group.add(new Adw.ActionRow({ title: 'Open the assistant', subtitle: settings.get_strv('hotkey').join(', ') })); + group.add(new Adw.ActionRow({ title: 'Voice and text', subtitle: 'Click Jarvis in the top panel, then hold Talk to speak, or type a question and press Enter.' })); + group.add(new Adw.ActionRow({ title: 'Wake phrase setup', subtitle: 'Hey Jarvis requires a local wake detector configured with JARVIS_WAKE_COMMAND. Use Talk when no detector is installed.' })); + page.add(group); window.add(page); } } diff --git a/apps/gnome-extension/jarvis@qvac.local/schemas/gschemas.compiled b/apps/gnome-extension/jarvis@qvac.local/schemas/gschemas.compiled index 3a62264..c77f06a 100644 Binary files a/apps/gnome-extension/jarvis@qvac.local/schemas/gschemas.compiled and b/apps/gnome-extension/jarvis@qvac.local/schemas/gschemas.compiled differ diff --git a/apps/gnome-extension/jarvis@qvac.local/schemas/org.gnome.shell.extensions.jarvis.gschema.xml b/apps/gnome-extension/jarvis@qvac.local/schemas/org.gnome.shell.extensions.jarvis.gschema.xml index 713e333..d4f1756 100644 --- a/apps/gnome-extension/jarvis@qvac.local/schemas/org.gnome.shell.extensions.jarvis.gschema.xml +++ b/apps/gnome-extension/jarvis@qvac.local/schemas/org.gnome.shell.extensions.jarvis.gschema.xml @@ -2,7 +2,7 @@ 'hey jarvis' ['jarvis', 'okay jarvis'] - '<Super><Space>' + ['<Super><Shift>j'] '' 'en-US' 'full-listen-after-wake' diff --git a/apps/gnome-extension/jarvis@qvac.local/stylesheet.css b/apps/gnome-extension/jarvis@qvac.local/stylesheet.css index 88b4577..dff9f84 100644 --- a/apps/gnome-extension/jarvis@qvac.local/stylesheet.css +++ b/apps/gnome-extension/jarvis@qvac.local/stylesheet.css @@ -1,5 +1,5 @@ .jarvis-panel-glyph { color: #F4B942; font-size: 16px; } -.jarvis-arc { width: 720px; padding: 18px; margin-top: 12vh; margin-left: 20px; margin-right: 20px; spacing: 10px; border-radius: 16px; background-color: rgba(11, 14, 20, .92); border: 1px solid rgba(244, 185, 66, .42); color: #f6f7fb; box-shadow: 0 12px 40px rgba(0, 0, 0, .45); } +.jarvis-arc { width: 720px; padding: 18px; margin-top: 0; margin-left: 20px; margin-right: 20px; spacing: 10px; border-radius: 16px; background-color: rgba(11, 14, 20, .92); border: 1px solid rgba(244, 185, 66, .42); color: #f6f7fb; box-shadow: 0 12px 40px rgba(0, 0, 0, .45); } .jarvis-arc-header { spacing: 8px; } .jarvis-title { font-weight: bold; letter-spacing: 1px; } .jarvis-local { color: var(--jarvis-accent, #F4B942); font-size: 11px; } @@ -10,7 +10,8 @@ .jarvis-high-contrast { border-width: 2px; background-color: #000; } .jarvis-wave { height: 42px; spacing: 3px; } .jarvis-wave-bar { width: 7px; background-color: var(--jarvis-accent, #F4B942); border-radius: 4px; } -.jarvis-transcript { spacing: 7px; max-height: 280px; } +.jarvis-transcript-scroll { height: 280px; } +.jarvis-transcript { spacing: 7px; } .jarvis-row { padding: 6px 10px; border-radius: 8px; font-size: 14px; } .jarvis-row-user { border-left: 2px solid var(--jarvis-accent, #F4B942); } .jarvis-row-jarvis { border-right: 2px solid #4FD2FF; } diff --git a/daemon/index.js b/daemon/index.js index 6affb96..aee4327 100644 --- a/daemon/index.js +++ b/daemon/index.js @@ -15,6 +15,7 @@ import { PortalInputBackend } from '../computer-use/portal-input.js'; import { ComputerActuator } from '../computer-use/actuator.js'; import { RuntimeTelemetry } from './telemetry.js'; import { StateRecovery } from './recovery.js'; +import { spokenReply } from '../skills/voice-prompt.js'; export class JarvisDaemon extends EventEmitter { constructor() { @@ -45,24 +46,54 @@ export class JarvisDaemon extends EventEmitter { setState(state) { this.state = state; try { this.recovery.save({ state, mode: this.mode }); } catch (error) { this.emit('Error', 'RECOVERY_WRITE', error.message); } this.emit('StateChanged', state); this.log.record('state', { state }).catch(() => {}); } async arm() { if (this.locked) return; await resumeQvac().catch(() => {}); this.voice.wake(); this.setState('LISTENING'); } async sleep() { this.voice.sleep(); this.setState('SLEEPING'); await suspendQvac().catch((error) => this.emit('Error', 'QVAC_SUSPEND', error.message)); } - say(text) { this.lastReply = String(text); this.emit('Reply', this.lastReply); } + say(text) { + const spoken = spokenReply(text) || spokenReply(this.lastReply) || 'There is nothing to repeat.'; + this.lastReply = spoken; + this.emit('Reply', spoken); + this._beginSpeech(); + this._speakReply(spoken); + } async ask(text) { - this.voice.typedUtterance(); this.setState('THINKING'); + this.voiceLoop?.interrupt?.(); try { + this.voice.typedUtterance(); this.setState('THINKING'); const startedAt = Date.now(); const reply = await this.scheduler.run(() => this.harness.ask(text), { lane: 'voice' }); this.telemetry.record('llm', startedAt, { success: true }); - this.voice.speak(); this.setState('SPEAKING'); this.lastReply = String(reply || ''); this.emit('Reply', this.lastReply); this.voiceLoop?.speak(this.lastReply).catch((error) => this.emit('Error', 'TTS', error.message)); return reply; + const spoken = spokenReply(reply); + this._beginSpeech(); this.lastReply = spoken; this.emit('Reply', spoken); this._speakReply(spoken); return spoken; } catch (error) { this.telemetry.record('llm', Date.now(), { success: false }); this.emit('Error', 'QVAC', error.message); this.voice.cancel(); this.setState('ARMED'); throw error; } } - cancel() { this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computer.revoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); } + _beginSpeech() { + try { + if (this.voice.state === 'ARMED' || this.voice.state === 'SLEEPING') this.voice.wake(); + if (this.voice.state !== 'SPEAKING') this.voice.speak(); + } catch {} + this.setState('SPEAKING'); + } + _finishSpeech() { + try { this.voice.finishSpeaking(); } catch {} + if (this.state === 'SPEAKING') this.setState('LISTENING'); + } + _speakReply(spoken) { + const playing = this.voiceLoop?.speak?.(spoken); + if (!playing) { this._finishSpeech(); return; } + Promise.resolve(playing).catch((error) => { + this.emit('Error', 'TTS', error.message); + if (this.state === 'SPEAKING') this._finishSpeech(); + }); + } + cancel() { this.voiceLoop?.interrupt?.(); this.harness.cancel(); this.scheduler.cancelQueued((job) => job.lane === 'voice'); this.computer.revoke(); this.voice.cancel(); this.setState('ARMED'); cancelQvac().catch((error) => this.emit('Error', 'QVAC_CANCEL', error.message)); } computerGrant(persist = false) { const result = this.computer.grant({ persist }); this.emit('ComputerStep', JSON.stringify({ action: 'grant', ...result })); this.input.grant({ persist }).then((backend) => { this.computer.setBackend(backend.backend); this.emit('ComputerStep', JSON.stringify({ action: 'backend', ...backend })); }).catch((error) => this.emit('Error', 'CU_GRANT', error.message)); return result; } computerRevoke() { this.input.revoke(); this.computer.revoke(); this.emit('ComputerStep', JSON.stringify({ action: 'revoke' })); } async startVoice() { if (this.voiceLoop) return; const voiceIO = new QvacVoiceAdapter(); - this.voiceLoop = new VoiceLoop({ daemon: this, wake: createWakeEngine(), asr: voiceIO, tts: voiceIO }); - try { await this.voiceLoop.start(); } catch (error) { this.voiceLoop = null; this.emit('Error', 'VOICE_UNAVAILABLE', error.message); throw error; } + const loop = new VoiceLoop({ daemon: this, wake: createWakeEngine(), asr: voiceIO, tts: voiceIO }); + this.voiceLoop = loop; + try { await loop.start(); this.emit('StateChanged', this.state); } catch (error) { + this.voiceLoop = null; await loop.stop?.().catch(() => {}); this.emit('Error', 'VOICE_UNAVAILABLE', error.message); throw error; + } } setPushToTalk(pressed) { this.voiceLoop?.setPushToTalk(pressed); this.emit('PushToTalk', Boolean(pressed)); } runtimeStatus() { return JSON.stringify({ local: true, qvac: qvacStatus(), scheduler: this.scheduler.status(), scheduler_metrics: this.scheduler.metrics(), telemetry: this.telemetry.snapshot(), computer: this.computer.status(), voice: this.voiceLoop?.metrics?.snapshot?.() || null, p2p: { enabled: process.env.JARVIS_P2P_ENABLE === '1', inference: false, memorySync: false } }); } diff --git a/daemon/voice-adapters.js b/daemon/voice-adapters.js index 790270d..7d06eb7 100644 --- a/daemon/voice-adapters.js +++ b/daemon/voice-adapters.js @@ -1,6 +1,23 @@ import { EventEmitter } from 'node:events'; import { acquireQvac, releaseQvac, loadAuxiliaryModel, unloadAuxiliaryModel, qvacSdk, withQvacMaster, assertSdkVersion } from './qvac-master.js'; +export function pcmS16le(samples, sampleRate = 44_100) { + if (samples instanceof Int16Array) return { samples, sampleRate }; + const bytes = toUint8(samples); + const even = bytes.byteLength - (bytes.byteLength % 2); + const copy = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + even); + return { samples: new Int16Array(copy), sampleRate }; +} + +function toUint8(samples) { + if (!samples) return new Uint8Array(0); + if (samples instanceof ArrayBuffer) return new Uint8Array(samples); + if (ArrayBuffer.isView(samples)) return new Uint8Array(samples.buffer, samples.byteOffset, samples.byteLength); + if (typeof samples === 'string') return Uint8Array.from(Buffer.from(samples)); + if (Array.isArray(samples) || samples.length != null) return Uint8Array.from(Buffer.from(samples)); + return new Uint8Array(0); +} + export class QvacVoiceAdapter extends EventEmitter { constructor({ asrModel = process.env.JARVIS_ASR_MODEL || 'WHISPER_TINY', ttsModel = process.env.JARVIS_TTS_MODEL || 'TTS_EN_SUPERTONIC_Q8_0' } = {}) { super(); this.asrModel = asrModel; this.ttsModel = ttsModel; this.asrId = null; this.ttsId = null; this.asrSession = null; this.acquired = false; @@ -33,10 +50,16 @@ export class QvacVoiceAdapter extends EventEmitter { async speak(text) { const sdk = await qvacSdk(); const samples = await withQvacMaster(async () => { - const result = sdk.textToSpeech({ modelId: this.ttsId, text: String(text), inputType: 'text', stream: false }); - return result.buffer; + const result = await sdk.textToSpeech({ modelId: this.ttsId, text: String(text), inputType: 'text', stream: false }); + if (result?.buffer != null) return await result.buffer; + if (result?.bufferStream) { + const chunks = []; + for await (const chunk of result.bufferStream) chunks.push(Buffer.from(chunk)); + return Buffer.concat(chunks); + } + return result; }); - return { samples: Int16Array.from(samples), sampleRate: 44_100 }; + return pcmS16le(samples, 44_100); } async stop() { diff --git a/daemon/voice-loop.js b/daemon/voice-loop.js index 3b4d394..d1f372b 100644 --- a/daemon/voice-loop.js +++ b/daemon/voice-loop.js @@ -22,7 +22,7 @@ export function fastCommand(text) { return FAST_COMMANDS.get(String(text || ''). export class VoiceLoop extends EventEmitter { constructor({ daemon, capture = new PipeWireCapture(), playback = new PipeWirePlayback(), wake = new WakeEngine(), vad = new VadSegmenter(), asr, tts, cooldownMs = POST_PLAYBACK_COOLDOWN_MS, now = () => Date.now() } = {}) { super(); this.daemon = daemon; this.capture = capture; this.playback = playback; this.wake = wake; this.vad = vad; this.asr = asr; this.tts = tts; this.cooldownMs = cooldownMs; this.now = now; - this.isSpeaking = false; this.cooldownUntil = 0; this.running = false; this.ptt = false; this._speechQueue = Promise.resolve(); this.metrics = new VoiceMetrics(); + this.isSpeaking = false; this.cooldownUntil = 0; this.running = false; this.ptt = false; this._generation = 0; this._speechQueue = Promise.resolve(); this.metrics = new VoiceMetrics(); capture.on('audio', (chunk) => this.pushAudio(chunk)); capture.on('error', (error) => this.emit('error', error)); wake.on('wake', (phrase) => this.wakeHeard(phrase)); @@ -58,12 +58,29 @@ export class VoiceLoop extends EventEmitter { if (command === 'screen') { await this.daemon?.ask?.('What is on my screen?'); return; } await this.daemon?.ask?.(text); } + interrupt() { + this._generation += 1; + this.playback.stop(); + this.isSpeaking = false; + this.wake.resume(); + } + _releaseSpeaking() { + this.isSpeaking = false; + this.wake.resume(); + if (this.daemon?.state === 'SPEAKING') { this.daemon.voice?.finishSpeaking?.(); this.daemon.setState?.('LISTENING'); } + } async speak(text) { - if (!isMeaningfulTranscript(text) || !this.tts?.speak) return; + const generation = this._generation || 0; + if (!isMeaningfulTranscript(text) || !this.tts?.speak) { this._releaseSpeaking(); return; } this.metrics.reply(); const sentences = String(text).match(/[^.!?]+[.!?]+|[^.!?]+$/g)?.map((part) => part.trim()).filter(isMeaningfulTranscript) || []; - this._speechQueue = this._speechQueue.then(async () => { - for (const sentence of sentences) await this.speakSentence(sentence); + if (!sentences.length) { this._releaseSpeaking(); return; } + this._speechQueue = this._speechQueue.catch(() => {}).then(async () => { + if (generation !== this._generation) return; + for (const sentence of sentences) { + if (generation !== this._generation) return; + await this.speakSentence(sentence); + } }); return this._speechQueue; } diff --git a/daemon/voice-state.js b/daemon/voice-state.js index 31e134f..2f4891e 100644 --- a/daemon/voice-state.js +++ b/daemon/voice-state.js @@ -16,8 +16,8 @@ export class VoiceStateMachine { transition(next, reason = 'unspecified') { if (!STATES.includes(next)) throw new Error(`unknown Jarvis state: ${next}`); const allowed = { - ARMED: ['LISTENING', 'SLEEPING'], LISTENING: ['THINKING', 'ARMED', 'SLEEPING'], - THINKING: ['SPEAKING', 'ARMED', 'LISTENING'], SPEAKING: ['LISTENING', 'ARMED'], + ARMED: ['LISTENING', 'SLEEPING'], LISTENING: ['THINKING', 'ARMED', 'SLEEPING', 'SPEAKING'], + THINKING: ['SPEAKING', 'ARMED', 'LISTENING'], SPEAKING: ['LISTENING', 'ARMED', 'THINKING'], SLEEPING: ['LISTENING', 'ARMED'], }; if (next !== this.state && !allowed[this.state].includes(next)) { diff --git a/skills/voice-prompt.js b/skills/voice-prompt.js index e586882..9a104c6 100644 --- a/skills/voice-prompt.js +++ b/skills/voice-prompt.js @@ -15,3 +15,13 @@ export function parseHudSidecar(text) { try { hud = JSON.parse(match[1]); } catch { hud = { error: 'invalid hud sidecar' }; } return { spoken: source.replace(match[0], '').trim(), hud }; } + +export function spokenReply(reply) { + if (reply == null) return ''; + const text = typeof reply === 'string' ? reply + : typeof reply.text === 'string' ? reply.text + : typeof reply.message === 'string' ? reply.message + : ''; + if (!text || text === '[object Object]') return ''; + return parseHudSidecar(text).spoken; +} diff --git a/test/daemon.test.js b/test/daemon.test.js index a80e93d..0454765 100644 --- a/test/daemon.test.js +++ b/test/daemon.test.js @@ -2,6 +2,8 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { VoiceStateMachine } from '../daemon/voice-state.js'; import { QvacScheduler } from '../daemon/qvac-scheduler.js'; +import { JarvisDaemon } from '../daemon/index.js'; +import { spokenReply } from '../skills/voice-prompt.js'; test('voice state machine handles wake, reply, cancel, and idle sleep', () => { let now = 0; @@ -20,6 +22,37 @@ test('typed questions wake an armed voice session before thinking', () => { assert.equal(voice.state, 'THINKING'); }); +test('typed questions barge in from SPEAKING', () => { + const voice = new VoiceStateMachine(); + voice.wake(); voice.utterance(); voice.speak(); + voice.typedUtterance(); + assert.equal(voice.state, 'THINKING'); +}); + +test('spoken replies use harness text and strip HUD sidecars', () => { + assert.equal(spokenReply({ ok: true, text: 'Hello there.', reason: 'stop' }), 'Hello there.'); + assert.equal(spokenReply({ text: 'Spoken. {"title":"x","chips":[]}' }), 'Spoken.'); + assert.equal(spokenReply({}), ''); + assert.equal(spokenReply('[object Object]'), ''); +}); + +test('ask extracts harness reply text instead of stringifying the object', async () => { + const daemon = new JarvisDaemon(); + daemon.harness = { ask: async () => ({ ok: true, text: 'Hello there.', reason: 'stop' }), cancel() {}, close: async () => {} }; + daemon.voiceLoop = null; + const replies = []; + daemon.on('Reply', (text) => replies.push(text)); + try { + const result = await daemon.ask('Hi'); + assert.equal(result, 'Hello there.'); + assert.deepEqual(replies, ['Hello there.']); + assert.equal(daemon.lastReply, 'Hello there.'); + assert.equal(daemon.state, 'LISTENING'); + } finally { + await daemon.close(); + } +}); + test('QVAC scheduler prioritizes voice and keeps one active job', async () => { const scheduler = new QvacScheduler(); const order = []; diff --git a/test/gnome-extension.test.js b/test/gnome-extension.test.js new file mode 100644 index 0000000..df4b247 --- /dev/null +++ b/test/gnome-extension.test.js @@ -0,0 +1,111 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import vm from 'node:vm'; + +const source = readFileSync(new URL('../apps/gnome-extension/jarvis@qvac.local/extension.js', import.meta.url), 'utf8'); +function harness() { + const timers = new Map(); + class Actor { + constructor(props = {}) { + if ('hexpand' in props) throw new Error('No property hexpand on StWidget'); + Object.assign(this, props); + this.children = []; + this.visible = props.visible !== false; + this.clutter_text = { connect() {}, ellipsize: null }; + } + add_child(child) { this.children.push(child); } + destroy_all_children() { this.children = []; } + get_last_child() { return this.children[this.children.length - 1] || null; } + connect() { return 1; } + hide() { this.visible = false; } + show() { this.visible = true; } + destroy() { this.destroyed = true; } + set_position() {} + grab_key_focus() {} + set_style() {} + add_style_class_name() {} + } + const context = vm.createContext({ + Extension: class {}, + St: { BoxLayout: Actor, Label: Actor, Widget: Actor, Button: Actor, Entry: Actor, ScrollView: Actor, PolicyType: { NEVER: 0, AUTOMATIC: 1 } }, + Clutter: { ActorAlign: { CENTER: 0, START: 1 }, EVENT_STOP: 1, EVENT_PROPAGATE: 0 }, + Pango: { EllipsizeMode: { NONE: 0, END: 3 } }, + Main: { layoutManager: { addChrome() {} } }, + GLib: { + PRIORITY_DEFAULT: 0, SOURCE_REMOVE: false, + timeout_add(_priority, _delay, callback) { const id = timers.size + 1; timers.set(id, callback); return id; }, + Source: { remove(id) { timers.delete(id); } }, + }, + log() {}, + }); + vm.runInContext(source.replace(/^import .*;\n/gm, '').replace('export default class', 'class') + + '\nglobalThis.classes = { ArcOverlay, JarvisExtension, JarvisProxy };', context); + return { ...context.classes, timers }; +} + +test('overlay constructs and destroys with pending halo animation', () => { + const { ArcOverlay, timers } = harness(); + const overlay = new ArcOverlay(); + assert.equal(overlay.header.children[1].x_expand, true); + overlay.attach(); + overlay.showHalo(); + overlay.showHalo(); + assert.equal(timers.size, 1); + overlay.destroy(); + assert.equal(timers.size, 0); +}); + +test('empty chrome widgets start hidden', () => { + const { ArcOverlay } = harness(); + const overlay = new ArcOverlay(); + assert.equal(overlay.job.visible, false); + assert.equal(overlay.target.visible, false); + assert.equal(overlay.cursor.visible, false); + assert.equal(overlay.wave.visible, false); +}); + +test('Reply finalizes a streaming row instead of duplicating [object Object]', () => { + const { ArcOverlay } = harness(); + const overlay = new ArcOverlay(); + overlay.token('Hello! I am ready.'); + overlay.finalizeReply('[object Object]'); + overlay.finalizeReply({ text: 'ignored duplicate' }); + assert.equal(overlay.transcript.children.length, 1); + assert.match(overlay.transcript.children[0].text, /Hello! I am ready/); + assert.doesNotMatch(overlay.transcript.children[0].text, /object Object/); +}); + +test('addRow and token coerce objects to readable text', () => { + const { ArcOverlay } = harness(); + const overlay = new ArcOverlay(); + overlay.addRow('U', { text: 'Hi there' }); + overlay.token({ text: 'Hello from Jarvis' }); + assert.equal(overlay.transcript.children.length, 2); + assert.match(overlay.transcript.children[0].text, /Hi there/); + assert.match(overlay.transcript.children[1].text, /Hello from Jarvis/); + assert.doesNotMatch(overlay.transcript.children.map((row) => row.text).join('\n'), /object Object/); +}); + +for (const fails of [false, true]) { + test('daemon connection finishing after disable is ignored: ' + (fails ? 'failure' : 'success'), async () => { + const { JarvisExtension } = harness(); + const extension = new JarvisExtension(); + let finish; + extension.proxy = { connect: () => new Promise((resolve, reject) => { finish = fails ? reject : resolve; }) }; + const pending = extension._connectDaemon(); + extension.proxy = null; + extension.overlay = null; + finish(fails ? new Error('Disconnected') : undefined); + await pending; + }); +} + +test('GNOME GI imports expose default namespaces and panel has a real menu', () => { + assert.match(source, /import Shell from 'gi:\/\/Shell'/); + assert.match(source, /import Meta from 'gi:\/\/Meta'/); + assert.match(source, /import Pango from 'gi:\/\/Pango'/); + assert.match(source, /new PanelMenu.Button\(0.0, 'Jarvis QVAC', false\)/); + assert.match(source, /PushToTalk/); + assert.match(source, /finalizeReply/); +}); diff --git a/test/voice-phase4.test.js b/test/voice-phase4.test.js index c43a301..09ce20b 100644 --- a/test/voice-phase4.test.js +++ b/test/voice-phase4.test.js @@ -2,6 +2,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { EventEmitter } from 'node:events'; import { VoiceLoop, fastCommand } from '../daemon/voice-loop.js'; +import { pcmS16le } from '../daemon/voice-adapters.js'; import { WakeEngine } from '../daemon/wake-engine.js'; import { VadSegmenter } from '../daemon/vad.js'; import { PipeWireCapture, pcmRms } from '../daemon/audio-pipewire.js'; @@ -53,3 +54,29 @@ test('Phase 4 PipeWire capture uses a named 16 kHz mono node', () => { assert.deepEqual(args, ['--record', '--raw', '--format', 's16', '--rate', '16000', '--channels', '1', '--name', 'Jarvis']); capture.stop(); }); + +test('empty TTS still leaves the daemon listening', async () => { + const daemon = new EventEmitter(); + daemon.state = 'SPEAKING'; + daemon.voice = { finishSpeaking() { daemon.finished = true; } }; + daemon.setState = (state) => { daemon.state = state; }; + const capture = new EventEmitter(); capture.start = () => {}; capture.stop = () => {}; + const wake = new WakeEngine({ detect: () => null }); + const loop = new VoiceLoop({ daemon, capture, wake, vad: new VadSegmenter(), tts: { speak: async () => ({ samples: new Int16Array(0) }) } }); + await loop.speak('[object Object]'); + assert.equal(daemon.state, 'LISTENING'); + assert.equal(daemon.finished, true); +}); + +test('PCM packing uses even s16le sample pairs', () => { + const buf = Buffer.alloc(4); + buf.writeInt16LE(256, 0); + buf.writeInt16LE(-2, 2); + const packed = pcmS16le(buf); + assert.equal(packed.samples.length, 2); + assert.equal(packed.samples[0], 256); + assert.equal(packed.samples[1], -2); + const fromBytes = pcmS16le(Uint8Array.from([0, 1, 0xfe, 0xff])); + assert.equal(fromBytes.samples.length, 2); + assert.equal(fromBytes.samples[0], 256); +});