import { mkdir } from 'node:fs/promises'; import path from 'node:path'; import { AtspiProvider } from './atspi.js'; import { FrameNormalizer } from './frame.js'; import { PortalScreenshot } from './portal-screenshot.js'; import { ShellProvider } from './shell-provider.js'; const normalize = (value) => String(value || '').toLowerCase().trim(); const score = (query, node) => { const q = normalize(query); const text = `${normalize(node.name)} ${normalize(node.role)}`; if (!q || !text) return 0; if (text === q) return 100; if (text.includes(q)) return 75; const words = q.split(/\s+/).filter((w) => text.includes(w)); return words.length ? 40 + words.length * 10 : 0; }; export class DesktopObserver { constructor({ screenshot = new PortalScreenshot(), normalizer = new FrameNormalizer(), atspi = new AtspiProvider(), shell = new ShellProvider(), ocr, vision, tmpDir = '/tmp/jarvis-cu' } = {}) { this.screenshot = screenshot; this.normalizer = normalizer; this.atspi = atspi; this.shell = shell; this.ocr = ocr; this.vision = vision; this.tmpDir = tmpDir; this.lastTree = []; } async tree({ focusedOnly = true, maxNodes = 400 } = {}) { const nodes = await this.atspi.tree({ focusedOnly, maxNodes }); this.lastTree = nodes.map((node, index) => ({ ...node, ref: `r${index + 1}` })); return this.lastTree; } async observe({ includeTree = true, includeOcr = true, includeVision = false } = {}) { await mkdir(this.tmpDir, { recursive: true }); const unavailable = []; let rawPath; let frame = { path: null }; try { rawPath = await this.screenshot.capture(path.join(this.tmpDir, `observe-${Date.now()}.png`)); frame = await this.normalizer.normalize(rawPath); } catch (error) { unavailable.push(`screenshot: ${error.message}`); } await this.shell.connect?.().catch((error) => unavailable.push(`Shell helper: ${error.message}`)); const [windows, focused, tree] = await Promise.all([this.shell.windows(), this.shell.focused(), includeTree ? this.tree().catch((error) => { unavailable.push(`AT-SPI: ${error.message}`); return []; }) : Promise.resolve([])]); const result = { monitor: null, focused, windows: windows.windows || [], tree, screenshot_path: frame.path, ocr_blocks: [], vision_hint: null, unavailable }; if (!windows.available) result.unavailable.push(windows.reason); if (includeOcr && this.ocr) result.ocr_blocks = await this.ocr(frame.path).catch((error) => { result.unavailable.push(`OCR: ${error.message}`); return []; }); if (includeVision && this.vision) result.vision_hint = await this.vision(frame.path).catch((error) => { result.unavailable.push(`vision: ${error.message}`); return null; }); return result; } async zoom({ rect, ref } = {}) { const node = ref ? this.lastTree.find((item) => item.ref === ref) : null; const target = rect || node?.rect; if (!target) throw new Error('rect or current tree ref is required'); const raw = await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`)); const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), target); return { rect: target, screenshot_path: frame.path }; } async find({ query, role } = {}) { const nodes = this.lastTree.length ? this.lastTree : await this.tree(); return nodes.map((node) => ({ ...node, score: score(query, node) })).filter((node) => node.score && (!role || normalize(node.role) === normalize(role))).sort((a, b) => b.score - a.score).slice(0, 20); } }