Updates
Rolling release / release (push) Successful in 8m31s

This commit is contained in:
2026-09-13 15:08:05 -04:00
parent b56171da9b
commit c5ccaa490b
29 changed files with 933 additions and 146 deletions
+52 -1
View File
@@ -44,16 +44,67 @@ test('computer use semantic actuation previews, budgets, and audits actions', as
const audit = new ComputerAudit({ dir }); const session = new ComputerUseSession({ stepsMax: 1, audit }); const sent = [];
session.grant();
const actuator = new ComputerActuator({ session, input: { send: (event) => sent.push(event) }, find: async ({ ref }) => [{ ref, name: 'Save', role: 'push button', rect: [1, 2, 3, 4] }], atspiAction: async () => ({ semantic: true }), audit, sleep: async () => {} });
const result = await actuator.click({ ref: 'r1' }); assert.equal(result.ok, true); assert.equal(session.status().steps_used, 1);
const result = await actuator.click({ ref: 'r1' }); assert.equal(result.ok, true); assert.equal(result.name, 'Save'); assert.equal(session.status().steps_used, 1);
assert.equal(result.target, undefined);
assert.deepEqual(sent, []); const files = await (await import('node:fs/promises')).readdir(dir); assert.equal(files.length, 1); assert.match(await readFile(path.join(dir, files[0]), 'utf8'), /target_hash/);
});
test('semantic click falls back to the control center when AT-SPI has no click action', async () => {
const session = new ComputerUseSession(); session.grant(); const sent = [];
const actuator = new ComputerActuator({
session,
input: { send: (event) => sent.push(event) },
find: async ({ ref }) => [{ ref, name: 'Discord', role: 'frame', rect: [10, 20, 100, 40] }],
atspiAction: async () => { throw new Error('AT-SPI action unavailable: click'); },
sleep: async () => {},
});
const result = await actuator.click({ ref: 'r1' });
assert.equal(result.ok, true);
assert.equal(sent[0].type, 'pointer');
assert.equal(sent[0].x, 60);
assert.equal(sent[0].y, 40);
});
test('typing focuses the target then injects keys and does not stringify missing text', async () => {
const session = new ComputerUseSession(); session.grant(); const sent = [];
const actuator = new ComputerActuator({
session,
input: { send: (event) => sent.push(event) },
find: async () => [{ ref: 'r2', name: 'Message', role: 'entry', rect: [0, 0, 20, 10] }],
sleep: async () => {},
});
const result = await actuator.type({ ref: 'r2', text: 'hello' });
assert.equal(result.ok, true);
assert.equal(result.result.typed, 5);
assert.equal(sent[0].action, 'click');
assert.equal(sent[1].action, 'type');
assert.equal(sent[1].text, 'hello');
const empty = await actuator.type({ ref: 'r2' });
assert.equal(empty.result.typed, 0);
assert.equal(sent[3].text, '');
});
test('computer use refuses password targets and unconfirmed dangerous keys', async () => {
const session = new ComputerUseSession(); session.grant(); const actuator = new ComputerActuator({ session, input: { send() {} }, find: async () => [{ name: 'Password', role: 'password text' }], sleep: async () => {} });
await assert.rejects(() => actuator.type({ ref: 'password', text: 'secret' }), /password/);
await assert.rejects(() => actuator.key({ combo: 'alt+f4' }), /confirmation/);
});
test('Send buttons do not require extra confirmation after a desktop grant', async () => {
const session = new ComputerUseSession(); session.grant(); const sent = [];
const actuator = new ComputerActuator({
session,
input: { send: (event) => sent.push(event) },
find: async () => [{ ref: 'r3', name: 'Send', role: 'push button', rect: [0, 0, 40, 20] }],
atspiAction: async () => { throw new Error('AT-SPI action unavailable: click'); },
sleep: async () => {},
});
const result = await actuator.click({ ref: 'r3' });
assert.equal(result.ok, true);
assert.equal(sent[0].action, 'click');
assert.equal(sent[0].x, 20);
});
test('libei sender binds bitmask capabilities from libei.h', () => {
const pyDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../computer-use/py');
const compiled = spawnSync('python3', ['-m', 'py_compile', 'libei_sender.py', 'portal_remote_desktop.py', 'portal_screenshot.py', 'pw_framebuffer.py'], { cwd: pyDir, encoding: 'utf8' });
+52
View File
@@ -48,6 +48,8 @@ test('spoken replies use harness text and strip HUD sidecars', () => {
);
assert.equal(spokenReply({}), '');
assert.equal(spokenReply('[object Object]'), '');
assert.equal(spokenReply({ text: '{"ok":true,"action":"type","name":"Discord"}' }), '');
assert.equal(spokenReply({ text: 'Done. {"ok":true,"action":"click","ref":"r1"}' }), 'Done.');
assert.equal(
spokenReply({ text: '## Status\n- CPU is **fine**.\nUse `htop` if you want more.' }),
'Status\n- CPU is fine.\nUse htop if you want more.',
@@ -61,6 +63,21 @@ test('voice settings default TTS on and honor an explicit disable', () => {
assert.equal(voiceSettings({ tts_enabled: false }).ttsEnabled, false);
});
test('thought chunks forward SDK text onto Thinking', async () => {
const daemon = new JarvisDaemon();
const thoughts = [];
daemon.on('Thinking', (text) => thoughts.push(text));
try {
daemon.harness.emit('agent_thought_chunk', { type: 'agent_thought_chunk', text: 'Considering the lookup.' });
daemon.harness.emit('agent_thought_chunk', 'plain thought');
daemon.harness.emit('agent_thought_chunk', { delta: ' via delta' });
daemon.harness.emit('agent_thought_chunk', { text: '' });
assert.deepEqual(thoughts, ['Considering the lookup.', 'plain thought', ' via delta']);
} finally {
await daemon.close();
}
});
test('ask extracts harness reply text instead of stringifying the object', async () => {
const daemon = new JarvisDaemon();
daemon.harness = { ask: async () => ({ ok: true, text: 'Hello there.', reason: 'stop' }), cancel() {}, close: async () => {} };
@@ -216,3 +233,38 @@ test('cancel suppresses a late reply and revokes portal input', async () => {
assert.deepEqual(replies, []); assert.equal(daemon.state, 'ARMED'); assert.equal(revoked, true);
} finally { await daemon.close(); }
});
test('mute ignores arm and does not return to listening after speech', async () => {
const daemon = new JarvisDaemon();
const muted = [];
daemon.voiceLoop = {
setMuted(value) { muted.push(Boolean(value)); },
setPushToTalk(value) { this.ptt = value; },
interrupt() {},
status: { capture: false, wake: false, muted: false },
metrics: { snapshot() { return {}; } },
};
try {
daemon.setState('LISTENING');
daemon.setMuted(true);
assert.equal(daemon.muted, true);
assert.equal(daemon.listenEnabled, false);
assert.equal(daemon.state, 'ARMED');
assert.deepEqual(muted, [true]);
await daemon.arm();
assert.equal(daemon.state, 'ARMED');
daemon.setListening(true);
assert.equal(daemon.state, 'ARMED');
daemon.setState('SPEAKING');
daemon._finishSpeech();
assert.equal(daemon.state, 'ARMED');
const status = JSON.parse(daemon.runtimeStatus());
assert.equal(status.muted, true);
assert.equal(status.voice.muted, true);
daemon.setMuted(false);
daemon.setListening(false);
daemon.setState('SPEAKING');
daemon._finishSpeech();
assert.equal(daemon.state, 'ARMED');
} finally { await daemon.close(); }
});
+50 -14
View File
@@ -27,7 +27,8 @@ function harness() {
this.children = [];
this.visible = props.visible !== false;
this.text = props.text || props.hint_text || '';
this.clutter_text = { connect() {}, ellipsize: null };
this.clutter_text = { connect() {}, ellipsize: null, width: 0, text: this.text };
this.allocation = { x1: 0, x2: 320, get_width() { return this.x2 - this.x1; } };
this.vadjustment = {
value: 0, upper: 240, page_size: 80, handlers: {},
connect(name, handler) { this.handlers[name] = handler; },
@@ -35,6 +36,7 @@ function harness() {
};
}
get_vadjustment() { return this.vadjustment; }
get_allocation_box() { return this.allocation; }
add_child(child) { this.children.push(child); child.get_parent = () => this; }
remove_child(child) { this.children = this.children.filter((item) => item !== child); }
destroy_all_children() { this.children = []; }
@@ -83,7 +85,7 @@ function harness() {
const context = vm.createContext({
Extension: class {},
global: { stage: { get_key_focus() { return null; } } },
St: { BoxLayout, Label, Icon: Actor, Widget: Actor, Button: Actor, Entry: Actor, ScrollView, PolicyType: { NEVER: 0, AUTOMATIC: 1 } },
St: { BoxLayout, Label, Icon: Actor, Widget: Actor, Button: Actor, Entry: Actor, ScrollView, PolicyType: { NEVER: 0, AUTOMATIC: 1, ALWAYS: 2 } },
Clutter: { ActorAlign: { CENTER: 0, START: 1 }, EVENT_STOP: 1, EVENT_PROPAGATE: 0, KEY_space: 32, KEY_Return: 65293, KEY_Escape: 65307 },
Pango: { WrapMode: { WORD_CHAR: 2 }, EllipsizeMode: { NONE: 0, END: 3 } },
PopupMenu: {
@@ -141,6 +143,7 @@ test('empty chrome widgets start hidden', () => {
const cu = new ComputerUseChrome();
assert.equal(popup.confirm.visible, false);
assert.equal(popup.thinkingScroll.visible, false);
assert.equal(popup.thinkingPane.visible, false);
assert.equal(popup.chipScroll.visible, false);
assert.equal(cu.root.visible, false);
assert.equal(cu.target.visible, false);
@@ -262,7 +265,11 @@ test('GNOME GI imports expose default namespaces and panel has a real menu', ()
assert.match(extensionSource, /PopupMenuItem\('Settings'\)/);
assert.match(extensionSource, /PopupMenuItem\('Grant desktop'\)/);
assert.doesNotMatch(uiSource, /button-press-event', \(\) => Clutter\.EVENT_STOP/);
assert.match(uiSource, /notify::pressed/);
assert.match(uiSource, /onListen/);
assert.match(uiSource, /jarvis-mute/);
assert.match(extensionSource, /SetMuted/);
assert.match(extensionSource, /SetListening/);
assert.doesNotMatch(extensionSource, /this\._call\('Arm'\)/);
assert.match(extensionSource, /ComputerGrant/);
assert.match(extensionSource, /OpenExtensionPrefs/);
assert.match(uiSource, /finalizeReply/);
@@ -301,11 +308,16 @@ test('lazy ASR still shows the microphone as available when capture is up', () =
assert.match(popup.statusLine.text, /speech on/);
assert.match(popup.statusLine.text, /wake on/);
assert.doesNotMatch(popup.status.style_class, /jarvis-state-/);
assert.equal(popup.talk.label, 'Hold to talk');
assert.equal(popup.talk.label, 'Listening');
popup.setVoiceStatus({ tts: false, input: false, wake: false });
assert.match(popup.status.text, /Mic off/);
assert.match(popup.statusLine.text, /mic off/);
assert.equal(popup.talk.label, 'Mic unavailable');
assert.equal(popup.talk.label, 'Listening');
assert.equal(popup.talk.reactive, false);
popup.setVoiceStatus({ tts: false, input: false, wake: false, muted: true });
assert.equal(popup.status.text, 'Muted');
assert.match(popup.statusLine.text, /muted/);
assert.equal(popup.talk.reactive, false);
assert.match(extensionSource, /Boolean\(voice\.capture\)/);
assert.doesNotMatch(extensionSource, /voice\.asr && voice\.capture/);
});
@@ -450,7 +462,7 @@ test('Settings chip is in the header and opens preferences', () => {
assert.deepEqual(calls, ['settings', 'session']);
});
test('HUD chips and hold-to-talk receive clicks instead of swallowing them', () => {
test('HUD chips, listening, and mute receive clicks instead of swallowing them', () => {
const { ConversationView } = harness();
const popup = new ConversationView({ compact: true });
const calls = [];
@@ -458,63 +470,87 @@ test('HUD chips and hold-to-talk receive clicks instead of swallowing them', ()
popup.onReset = () => calls.push('reset');
popup.onExpand = () => calls.push('open');
popup.onSettings = () => calls.push('settings');
popup.onTalk = (pressed) => calls.push(pressed ? 'talk-down' : 'talk-up');
popup.onListen = () => calls.push('listen');
popup.onMute = (muted) => calls.push(muted ? 'mute' : 'unmute');
popup.stop.handlers.clicked();
popup.reset.handlers.clicked();
popup.expand.handlers.clicked();
popup.settings.handlers.clicked();
popup.talk.pressed = true;
popup.talk.handlers['notify::pressed']();
popup.talk.pressed = false;
popup.talk.handlers['notify::pressed']();
assert.deepEqual(calls, ['stop', 'reset', 'open', 'settings', 'talk-down', 'talk-up']);
popup.talk.handlers.clicked();
popup.mute.handlers.clicked();
assert.deepEqual(calls, ['stop', 'reset', 'open', 'settings', 'listen', 'mute']);
assert.equal(popup.stop.handlers['button-press-event'], undefined);
assert.equal(popup.settings.handlers['button-press-event'], undefined);
assert.equal(popup.talk.handlers['notify::pressed'], undefined);
assert.equal(popup.mute.label, 'Mute');
popup.setMuted(true);
assert.equal(popup.mute.label, 'Muted');
assert.match(popup.statusLine.text, /muted/);
popup.setState('LISTENING');
assert.equal(popup.talk.reactive, false);
assert.equal(popup.talk.visible, true);
popup.showTab('thinking');
assert.equal(popup.mute.visible, true);
assert.equal(popup.talk.visible, true);
});
test('closing the popup ends hold-to-talk', () => {
test('closing the popup does not stop listening', () => {
const { ConversationView } = harness();
const popup = new ConversationView({ compact: true });
const talks = [];
popup.onTalk = (pressed) => talks.push(pressed);
popup.onListen = () => talks.push('listen');
popup.endTalk();
assert.deepEqual(talks, [false]);
assert.deepEqual(talks, []);
});
test('Chat and Thinking tabs exist and thinking auto-follows', () => {
const { ConversationView } = harness();
const popup = new ConversationView({ compact: true });
const session = new ConversationView({ compact: false });
assert.equal(popup.thinkingPane.clip_to_allocation, true);
assert.equal(popup.thinkingPane.height, 160);
assert.equal(popup.thinkingPane.children[0], popup.thinkingScroll);
assert.equal(popup.thinkingScroll.overlay_scrollbars, false);
assert.equal(popup.thinkingScroll.vscrollbar_policy, 2);
assert.equal(popup.thinkingScroll.clip_to_allocation, true);
assert.equal(popup.thinkingScroll.enable_mouse_scrolling, true);
for (const view of [popup, session]) {
assert.equal(view.chatTab.label, 'Chat');
assert.equal(view.thinkTab.label, 'Thinking');
assert.equal(view.thinkingBox.children[0], view.thinking);
assert.equal(view.thinkingScroll.children[0], view.thinkingBox);
assert.equal(view.thinkingPane.visible, false);
assert.equal(view.thinkingScroll.visible, false);
assert.equal(view.scroll.visible, true);
view.setState('THINKING');
view.updateThinking('Considering the lookup. ');
assert.equal(view._tab, 'thinking');
assert.equal(view.thinkingPane.visible, true);
assert.equal(view.thinkingScroll.visible, true);
assert.equal(view.scroll.visible, false);
assert.match(view.thinking.text, /Considering the lookup/);
assert.equal(view.thinking.clutter_text.ellipsize, 0);
assert.ok(view.thinking.clutter_text.width >= 80);
assert.equal(view.thinkingScroll.vadjustment.value, 160);
view.token('Latest reply token');
view.scroll.vadjustment.value = 0;
view.setState('SPEAKING');
assert.equal(view._tab, 'chat');
assert.equal(view.scroll.visible, true);
assert.equal(view.thinkingPane.visible, false);
assert.equal(view.thinkingScroll.visible, false);
assert.equal(view.scroll.vadjustment.value, 160);
view.showTab('chat', { user: true });
view.updateThinking('still thinking');
assert.equal(view._tab, 'thinking');
assert.equal(view.thinkingPane.visible, true);
assert.equal(view.thinkingScroll.visible, true);
assert.match(view.thinking.text, /still thinking/);
view.thinkingScroll.vadjustment.value = 0;
view.showTab('thinking', { user: true });
assert.equal(view._tab, 'thinking');
assert.equal(view.thinkingPane.visible, true);
assert.equal(view.thinkingScroll.visible, true);
assert.equal(view.thinkingScroll.vadjustment.value, 160);
}
+31 -2
View File
@@ -9,6 +9,7 @@ import { ComputerUseSession } from '../computer-use/session.js';
import { ComputerActuator } from '../computer-use/actuator.js';
import { PortalInputBackend } from '../computer-use/portal-input.js';
import { createComputerObserveTools } from '../skills/computer-observe.js';
import { createComputerActTools } from '../skills/computer-act.js';
import { createPhase2Tools, filesystemRoots } from '../skills/phase2-tools.js';
import { VoiceStateMachine } from '../daemon/voice-state.js';
import { assertLocalEndpoint } from '../daemon/network-policy.js';
@@ -83,11 +84,16 @@ test('stale semantic refs fail before input and revocation during preview preven
await assert.rejects(actuator.click({ x: 1, y: 1 }), /inactive/);
});
test('desktop observation does not wait for a second Allow after Grant desktop', () => {
test('desktop observation and actuation do not wait for a second Allow after Grant desktop', () => {
const id = 'cu-observe-permission';
try {
custom.register(id, createComputerObserveTools({ computer: { status: () => ({ active: true }) }, observer: {} }));
custom.register(id, [
...createComputerObserveTools({ computer: { status: () => ({ active: true }) }, observer: {} }),
...createComputerActTools({ actuator: {} }),
]);
assert.equal(custom.needsPermission(id, 'cu_observe', 'ask'), false);
assert.equal(custom.needsPermission(id, 'cu_click', 'ask'), false);
assert.equal(custom.needsPermission(id, 'cu_type', 'ask'), false);
} finally { custom.clear(id); }
});
@@ -97,6 +103,29 @@ test('expired grants prevent desktop observation', async () => {
await assert.rejects(observe.execute(), /grant/);
});
test('observe tools return compact tree nodes without accessibility state dumps', async () => {
const computer = { status: () => ({ active: true }) };
const observer = {
observe: async () => ({
focused: { name: 'Discord' },
windows: [{ id: 1 }],
tree: [{ ref: 'r1', role: 'frame', name: 'Discord', rect: [0, 0, 10, 10], state: ['1', '8', '24'] }],
frame_source: 'pipewire',
screenshot_path: '/tmp/x.webp',
unavailable: [],
}),
find: async () => [{ ref: 'r1', role: 'entry', name: 'Message', rect: [1, 2, 3, 4], score: 90, state: ['focused'] }],
};
const tools = Object.fromEntries(createComputerObserveTools({ computer, observer }).map((tool) => [tool.name, tool]));
const seen = await tools.cu_observe.execute({});
assert.equal(seen.tree[0].ref, 'r1');
assert.equal(seen.tree[0].state, undefined);
assert.match(seen.hint, /Do not paste/);
const hits = await tools.cu_find.execute({ query: 'message' });
assert.equal(hits[0].score, 90);
assert.equal(hits[0].state, undefined);
});
function helper() {
const child = new EventEmitter();
child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); child.stdin = new EventEmitter(); child.stdin.writable = true; child.stdin.write = () => {};
+2 -1
View File
@@ -227,7 +227,8 @@ test('voice prompt tells the model not to chain extra terminal commands', () =>
assert.match(VOICE_SYSTEM_PROMPT, /Tool names, tool arguments/);
assert.match(VOICE_SYSTEM_PROMPT, /File tools may read any path they accept/);
assert.match(VOICE_SYSTEM_PROMPT, /Allow now/);
assert.match(VOICE_SYSTEM_PROMPT, /call cu_observe to read the live PipeWire frame buffer/);
assert.match(VOICE_SYSTEM_PROMPT, /call cu_observe, then cu_find or a tree ref, then cu_click and cu_type/);
assert.match(VOICE_SYSTEM_PROMPT, /Never paste tool JSON/);
assert.doesNotMatch(VOICE_SYSTEM_PROMPT, /Never claim cloud access/);
});
+43
View File
@@ -122,6 +122,49 @@ test('Hold Talk only mode ignores wake and automatic listening', () => {
loop.ptt = true; loop.pushAudio(Buffer.alloc(10)); assert.equal(vadFrames, 1);
});
test('mute stops capture and ignores wake, VAD, and push-to-talk', async () => {
const capture = new EventEmitter();
let started = 0;
let stopped = 0;
capture.start = () => { started += 1; };
capture.stop = () => { stopped += 1; };
let wakeFrames = 0;
let vadFrames = 0;
let heard = 0;
const wake = new EventEmitter();
wake.push = () => { wakeFrames += 1; };
wake.pause = () => {};
wake.resume = () => {};
wake.start = () => {};
wake.close = () => {};
const vad = new EventEmitter();
vad.push = () => { vadFrames += 1; };
vad.reset = () => {};
vad.end = () => {};
const daemon = new EventEmitter();
daemon.state = 'LISTENING';
daemon.arm = () => { heard += 1; };
const loop = new VoiceLoop({ daemon, capture, wake, vad, asr: { start: async () => {} } });
await loop.start();
assert.ok(started >= 1);
loop.setMuted(true);
assert.equal(loop.muted, true);
assert.equal(loop.status.muted, true);
assert.equal(loop.status.capture, false);
assert.equal(stopped, 1);
loop.pushAudio(Buffer.alloc(10));
assert.equal(wakeFrames, 0);
assert.equal(vadFrames, 0);
loop.wakeHeard('hey jarvis');
assert.equal(heard, 0);
await loop.setPushToTalk(true);
assert.equal(loop.ptt, false);
loop.interrupt();
loop.setMuted(false);
assert.equal(loop.status.capture, true);
await loop.stop();
});
test('VAD discards short noises and bounds the whole recording including pauses', () => {
const vad = new VadSegmenter({ params: { minSpeechDurationMs: 300, minSilenceDurationMs: 500, maxSpeechDurationMs: 600 } });
const speech = Buffer.alloc(3200); for (let i = 0; i < speech.length; i += 2) speech.writeInt16LE(20000, i);
+34
View File
@@ -89,3 +89,37 @@ Need a current answer.
assert.equal(recovered.calls[0].name, 'web_search');
assert.equal(recovered.calls[0].arguments.query, 'Ubuntu 26.04 release');
});
test('SDK thinking events use text and recover think tags from content', () => {
const events = require('../vendor/agent-harness/lib/events.js');
const sdkThink = events.normalizeCompletionEvent({ type: 'thinkingDelta', seq: 1, text: 'Considering the lookup.' });
assert.equal(sdkThink.type, 'thinkingDelta');
assert.equal(sdkThink.delta, 'Considering the lookup.');
const first = events.expandThinkEvents({ type: 'contentDelta', delta: 'Hello <think>reason' }, { inThink: false, carry: '' });
assert.deepEqual(first.events, [
{ type: 'contentDelta', delta: 'Hello ' },
{ type: 'thinkingDelta', delta: 'reason' },
]);
assert.equal(first.state.inThink, true);
const next = events.expandThinkEvents({ type: 'contentDelta', delta: 'ing</think>Answer' }, first.state);
assert.deepEqual(next.events, [
{ type: 'thinkingDelta', delta: 'ing' },
{ type: 'contentDelta', delta: 'Answer' },
]);
assert.equal(next.state.inThink, false);
});
test('click and type aliases recover as computer-use tools', () => {
const tools = [{ name: 'cu_click' }, { name: 'cu_type' }, { name: 'cu_observe' }];
const calls = toolParse.extractCalls(
'<tool_call><function=click><parameter=ref>\nr4\n</parameter></function></tool_call>',
tools,
);
assert.equal(calls[0].name, 'cu_click');
assert.equal(calls[0].arguments.ref, 'r4');
const typed = toolParse.extractCalls(
'<tool_call><function=type><parameter=text>\nhi\n</parameter></function></tool_call>',
tools,
);
assert.equal(typed[0].name, 'cu_type');
});