88 lines
4.7 KiB
JavaScript
88 lines
4.7 KiB
JavaScript
import { mkdir } from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import { AtspiProvider } from './atspi.js';
|
|
import { FrameNormalizer } from './frame.js';
|
|
import { PortalScreenshot } from './portal-screenshot.js';
|
|
import { ShellProvider } from './shell-provider.js';
|
|
|
|
const normalize = (value) => String(value || '').toLowerCase().trim();
|
|
const score = (query, node) => { const q = normalize(query); const text = `${normalize(node.name)} ${normalize(node.role)}`; if (!q || !text) return 0; if (text === q) return 100; if (text.includes(q)) return 75; const words = q.split(/\s+/).filter((w) => text.includes(w)); return words.length ? 40 + words.length * 10 : 0; };
|
|
|
|
function timed(promise, ms, label) {
|
|
const timeoutMs = Number(ms) > 0 ? Number(ms) : 0;
|
|
if (!timeoutMs) return promise;
|
|
let timer;
|
|
return Promise.race([
|
|
promise,
|
|
new Promise((_, reject) => {
|
|
timer = setTimeout(() => reject(new Error(`${label} timed out`)), timeoutMs);
|
|
}),
|
|
]).finally(() => { if (timer) clearTimeout(timer); });
|
|
}
|
|
|
|
export class DesktopObserver {
|
|
constructor({ screenshot = new PortalScreenshot(), normalizer = new FrameNormalizer(), atspi = new AtspiProvider(), shell = new ShellProvider(), ocr, vision, tmpDir = '/tmp/jarvis-cu', timeouts = {} } = {}) {
|
|
this.screenshot = screenshot; this.normalizer = normalizer; this.atspi = atspi; this.shell = shell; this.ocr = ocr; this.vision = vision; this.tmpDir = tmpDir;
|
|
this.lastTree = [];
|
|
this.timeouts = { screenshot: timeouts.screenshot ?? 8000, tree: timeouts.tree ?? 5000, shell: timeouts.shell ?? 2000, ocr: timeouts.ocr ?? 8000, vision: timeouts.vision ?? 8000 };
|
|
}
|
|
|
|
async tree({ focusedOnly = true, maxNodes = 400 } = {}) {
|
|
const nodes = await timed(this.atspi.tree({ focusedOnly, maxNodes }), this.timeouts.tree, 'AT-SPI');
|
|
this.lastTree = nodes.map((node, index) => ({ ...node, ref: `r${index + 1}` }));
|
|
return this.lastTree;
|
|
}
|
|
|
|
async _captureFrame() {
|
|
const rawPath = await this.screenshot.capture(path.join(this.tmpDir, `observe-${Date.now()}.png`));
|
|
return this.normalizer.normalize(rawPath);
|
|
}
|
|
|
|
async _shellSnapshot() {
|
|
if (this.shell.connect) await this.shell.connect();
|
|
const [windows, focused] = await Promise.all([this.shell.windows(), this.shell.focused()]);
|
|
return { windows, focused };
|
|
}
|
|
|
|
async observe({ includeTree = true, includeOcr = false, includeVision = false } = {}) {
|
|
await mkdir(this.tmpDir, { recursive: true });
|
|
const unavailable = [];
|
|
const note = (label) => (error) => { unavailable.push(`${label}: ${error.message}`); return null; };
|
|
const [frame, shell, tree] = await Promise.all([
|
|
timed(this._captureFrame(), this.timeouts.screenshot, 'screenshot').catch(note('screenshot')),
|
|
timed(this._shellSnapshot(), this.timeouts.shell, 'Shell helper').catch(note('Shell helper')),
|
|
includeTree ? this.tree().catch((error) => { unavailable.push(`AT-SPI: ${error.message}`); return []; }) : Promise.resolve([]),
|
|
]);
|
|
const windows = shell?.windows || { available: false, windows: [] };
|
|
const result = {
|
|
monitor: null,
|
|
focused: shell?.focused || null,
|
|
windows: windows.windows || [],
|
|
tree,
|
|
screenshot_path: frame?.path || null,
|
|
ocr_blocks: [],
|
|
vision_hint: null,
|
|
unavailable,
|
|
};
|
|
if (shell && !windows.available && windows.reason) result.unavailable.push(windows.reason);
|
|
if (result.screenshot_path && includeOcr && this.ocr) result.ocr_blocks = await timed(this.ocr(result.screenshot_path), this.timeouts.ocr, 'OCR').catch((error) => { result.unavailable.push(`OCR: ${error.message}`); return []; });
|
|
if (result.screenshot_path && includeVision && this.vision) result.vision_hint = await timed(this.vision(result.screenshot_path), this.timeouts.vision, 'vision').catch((error) => { result.unavailable.push(`vision: ${error.message}`); return null; });
|
|
return result;
|
|
}
|
|
|
|
async zoom({ rect, ref } = {}) {
|
|
await mkdir(this.tmpDir, { recursive: true });
|
|
const node = ref ? this.lastTree.find((item) => item.ref === ref) : null;
|
|
const target = rect || node?.rect;
|
|
if (!target) throw new Error('rect or current tree ref is required');
|
|
const raw = await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`));
|
|
const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), target);
|
|
return { rect: target, screenshot_path: frame.path };
|
|
}
|
|
|
|
async find({ query, role } = {}) {
|
|
const nodes = this.lastTree.length ? this.lastTree : await this.tree();
|
|
return nodes.map((node) => ({ ...node, score: score(query, node) })).filter((node) => node.score && (!role || normalize(node.role) === normalize(role))).sort((a, b) => b.score - a.score).slice(0, 20);
|
|
}
|
|
}
|