38 lines
3.0 KiB
JavaScript
38 lines
3.0 KiB
JavaScript
const PERMISSION = 'read';
|
|
const inactive = (computer) => !computer?.status?.().active ? { unavailable: 'computer-use grant required', action: 'cu.grant' } : null;
|
|
|
|
export function summarizeNode(node) {
|
|
if (!node || typeof node !== 'object') return node;
|
|
const out = { ref: node.ref, role: node.role, name: node.name };
|
|
if (Array.isArray(node.rect)) out.rect = node.rect;
|
|
if (node.score != null) out.score = node.score;
|
|
return out;
|
|
}
|
|
|
|
export function summarizeTree(nodes, limit = 80) {
|
|
const list = Array.isArray(nodes) ? nodes : [];
|
|
return list.slice(0, limit).map(summarizeNode);
|
|
}
|
|
|
|
export function createComputerObserveTools({ computer, observer } = {}) {
|
|
const guard = () => { const result = inactive(computer); if (result) throw new Error(result.unavailable); };
|
|
return [
|
|
{ name: 'cu_observe', permission: PERMISSION, description: 'Look at the granted desktop: focused window, AT-SPI controls, and the live ScreenCast frame. Prefer this before clicking or typing. This is not the Screenshot portal.', parameters: { type: 'object', properties: { include_tree: { type: 'boolean' }, include_ocr: { type: 'boolean' }, include_vision: { type: 'boolean' } } }, execute: async ({ include_tree = true, include_ocr = false, include_vision = false } = {}) => {
|
|
guard();
|
|
const bundle = await observer.observe({ includeTree: include_tree, includeOcr: include_ocr, includeVision: include_vision });
|
|
return {
|
|
focused: bundle.focused || null,
|
|
windows: (bundle.windows || []).slice(0, 12),
|
|
tree: summarizeTree(bundle.tree),
|
|
frame_source: bundle.frame_source || null,
|
|
screenshot_path: bundle.screenshot_path || null,
|
|
unavailable: bundle.unavailable || [],
|
|
hint: 'Use cu_find or a tree ref with cu_click / cu_type. Do not paste this JSON to the user.',
|
|
};
|
|
} },
|
|
{ name: 'cu_zoom', permission: PERMISSION, description: 'Request a higher resolution local crop by rectangle or current AT-SPI ref.', parameters: { type: 'object', properties: { rect: { type: 'array', items: { type: 'number' } }, ref: { type: 'string' } } }, execute: async (args) => { guard(); return observer.zoom(args); } },
|
|
{ name: 'cu_tree', permission: PERMISSION, description: 'Return visible AT-SPI roles, names, bounds, and per-step refs.', parameters: { type: 'object', properties: { app: { type: 'string' }, focused_only: { type: 'boolean' }, max_nodes: { type: 'number' } } }, execute: async ({ focused_only = true, max_nodes = 80 } = {}) => { guard(); return summarizeTree(await observer.tree({ focusedOnly: focused_only, maxNodes: Math.min(400, Number(max_nodes) || 80) }), Math.min(80, Number(max_nodes) || 80)); } },
|
|
{ name: 'cu_find', permission: PERMISSION, description: 'Find a visible desktop element by accessible name/role.', parameters: { type: 'object', properties: { query: { type: 'string' }, role: { type: 'string' } }, required: ['query'] }, execute: async (args) => { guard(); return summarizeTree(await observer.find(args), 12); } },
|
|
];
|
|
}
|