73 lines
6.3 KiB
JavaScript
73 lines
6.3 KiB
JavaScript
/** Opt-in live acceptance test. Operates only in a disposable GTK window. */
|
|
import { spawn } from 'node:child_process';
|
|
import { mkdtemp, readFile, writeFile, rm } from 'node:fs/promises';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { PortalInputBackend } from '../computer-use/portal-input.js';
|
|
import { DesktopObserver } from '../computer-use/observer.js';
|
|
import { ComputerUseSession } from '../computer-use/session.js';
|
|
import { ComputerActuator } from '../computer-use/actuator.js';
|
|
import { createComputerActTools } from '../skills/computer-act.js';
|
|
import { createComputerObserveTools } from '../skills/computer-observe.js';
|
|
|
|
const dir=await mkdtemp('/tmp/jarvis-cu-acceptance-');
|
|
const eventPath=path.join(dir,'events.json');
|
|
const title='Jarvis verification '+process.pid;
|
|
const fixture=spawn('python3',[fileURLToPath(new URL('../test/fixtures/cu-probe.py',import.meta.url))],{stdio:['ignore','ignore','pipe'],env:{...process.env,JARVIS_CU_PROBE_OUTPUT:eventPath,JARVIS_CU_PROBE_TITLE:title}});
|
|
let fixtureError='';fixture.stderr.on('data',d=>{fixtureError+=d;});fixture.on('error',e=>{fixtureError=e.message;});
|
|
const input=new PortalInputBackend();
|
|
const session=new ComputerUseSession({stepsMax:30});
|
|
const observer=new DesktopObserver({tmpDir:dir,framebuffer:{capture:p=>input.captureFrame(p),streams:()=>input.streams}});
|
|
const actuator=new ComputerActuator({session,input,find:async({ref})=>observer.lastTree.filter(n=>n.ref===ref),atspiAction:(target,action)=>observer.atspi.action(target,action)});
|
|
const tools=[...createComputerActTools({actuator}),...createComputerObserveTools({computer:session,observer})];
|
|
const results=[];
|
|
let previous, win;
|
|
const check=(condition,message)=>{if(!condition)throw Error(message);};
|
|
const events=async()=>JSON.parse(await readFile(eventPath,'utf8'));
|
|
async function waitFor(fn,label){const end=Date.now()+4000;while(Date.now()<end){try{if(await fn())return;}catch{}await delay(80);}throw Error(label);}
|
|
async function call(name,args){const focused=await observer.shell.focused();check(focused?.pid===win.pid,'Test window lost focus; stopped before input');return tools.find(t=>t.name===name).execute(args);}
|
|
async function step(name,fn){await fn();results.push(name);console.log('PASS '+name);}
|
|
try{
|
|
previous=await observer.shell.focused();
|
|
await waitFor(async()=>{win=(await observer.shell.windows()).windows.find(w=>w.title===title);return win;},'Test window unavailable: '+fixtureError);
|
|
console.log('Approve Ubuntu desktop sharing/control for the monitor containing '+title+'.');
|
|
const backend=await input.grant();session.grant();session.setBackend(backend.backend);
|
|
await observer.shell.focus(win.id);await delay(350);
|
|
let bundle;
|
|
await step('ScreenCast capture and accessibility observation',async()=>{
|
|
bundle=await call('cu_observe',{});
|
|
check(bundle.screenshot_path && bundle.tree.length,'Missing frame or accessibility controls: '+JSON.stringify(bundle.unavailable));
|
|
const bytes=await readFile(bundle.screenshot_path);check(bytes.length>1000,'Empty screenshot');
|
|
});
|
|
const target=name=>{const n=observer.lastTree.find(n=>n.name===name);check(n,'Missing '+name);return n;};
|
|
const center=n=>({x:n.rect[0]+n.rect[2]/2,y:n.rect[1]+n.rect[3]/2});
|
|
await step('AT-SPI semantic activation',async()=>{await call('cu_act',{ref:target('Verify click').ref,action:'click'});await waitFor(async()=>(await events()).clicks===1,'Semantic click not received');});
|
|
await step('Desktop-coordinate click',async()=>{await call('cu_click',center(target('Verify click')));await waitFor(async()=>(await events()).clicks===2,'Coordinate click not received');});
|
|
const sample='Jarvis verified! @#$% café ✓';
|
|
await step('Targeted Unicode typing',async()=>{await call('cu_type',{ref:target('Verification text').ref,text:sample});await waitFor(async()=>(await events()).text===sample,'Typed text differs: '+(await events()).text);});
|
|
await step('Keyboard shortcuts and navigation',async()=>{
|
|
await call('cu_key',{combo:'ctrl+a'});await call('cu_type',{text:'Desktop input verified.'});await call('cu_key',{combo:'home'});await call('cu_key',{combo:'end'});
|
|
await waitFor(async()=>{const e=await events();return e.text==='Desktop input verified.'&&e.keys.includes('Home')&&e.keys.includes('End');},'Shortcut/navigation failed');
|
|
});
|
|
await step('Save through app control',async()=>{await call('cu_click',{ref:target('Save verification text').ref});await waitFor(async()=>(await readFile(eventPath+'.txt','utf8'))==='Desktop input verified.','Saved text mismatch');});
|
|
const area=center(target('Verification pointer area'));
|
|
await step('Hover, double-click and right-click',async()=>{
|
|
await call('cu_hover',area);await call('cu_double_click',area);await call('cu_right_click',area);
|
|
await waitFor(async()=>{const e=await events();return e.pointer.some(p=>p.type==='5')&&e.pointer.some(p=>p.button===3);},'Pointer events missing');
|
|
});
|
|
await step('Horizontal and vertical scrolling',async()=>{await call('cu_scroll',{...area,dx:1,dy:2});await waitFor(async()=>(await events()).scroll.length>0,'Scroll not received');});
|
|
await step('Drag with motion and release',async()=>{
|
|
await call('cu_drag',{from:{x:area.x-80,y:area.y},to:{x:area.x+80,y:area.y+50}});
|
|
await waitFor(async()=>{const e=await events();const releases=e.pointer.filter(p=>p.type==='7'&&p.button===1);return releases.length>0&&Math.abs(releases.at(-1).x-(target('Verification pointer area').rect[2]/2+80))<8;},'Drag release missing or wrong coordinates');
|
|
});
|
|
await step('Revoke blocks subsequent input',async()=>{input.revoke();session.revoke();let refused=false;try{await actuator.key({combo:'enter'});}catch{refused=true;}check(refused&&!input.available,'Revoked session still accepts input');});
|
|
const report={passed:true,backend:backend.backend,eis_error:backend.eis_error,checks:results,at:new Date().toISOString()};
|
|
await writeFile(path.join(dir,'result.json'),JSON.stringify(report,null,2));console.log(JSON.stringify(report));
|
|
}catch(error){console.error('FAIL '+error.message);console.error('Passed before failure: '+results.join(', '));process.exitCode=1;}
|
|
finally{
|
|
input.revoke();session.revoke();fixture.kill('SIGTERM');
|
|
if(previous?.id)await observer.shell.focus(previous.id).catch(()=>{});
|
|
await rm(dir,{recursive:true,force:true});
|
|
}
|