import { mkdir } from 'node:fs/promises'; import path from 'node:path'; import { AtspiProvider } from './atspi.js'; import { FrameNormalizer } from './frame.js'; import { PortalScreenshot } from './portal-screenshot.js'; import { ShellProvider } from './shell-provider.js'; const normalize = (value) => String(value || '').toLowerCase().trim(); const score = (query, node) => { const q = normalize(query); const text = `${normalize(node.name)} ${normalize(node.role)}`; if (!q || !text) return 0; if (text === q) return 100; if (text.includes(q)) return 75; const words = q.split(/\s+/).filter((w) => text.includes(w)); return words.length ? 40 + words.length * 10 : 0; }; function timed(promise, ms, label) { const timeoutMs = Number(ms) > 0 ? Number(ms) : 0; if (!timeoutMs) return promise; let timer; return Promise.race([ promise, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`${label} timed out`)), timeoutMs); }), ]).finally(() => { if (timer) clearTimeout(timer); }); } export class DesktopObserver { constructor({ screenshot = new PortalScreenshot(), normalizer = new FrameNormalizer(), atspi = new AtspiProvider(), shell = new ShellProvider(), framebuffer, ocr, vision, tmpDir = '/tmp/jarvis-cu', timeouts = {} } = {}) { this.screenshot = screenshot; this.normalizer = normalizer; this.atspi = atspi; this.shell = shell; this.framebuffer = framebuffer; this.ocr = ocr; this.vision = vision; this.tmpDir = tmpDir; this.lastTree = []; this._refSequence = 0; this.timeouts = { screenshot: timeouts.screenshot ?? 8000, tree: timeouts.tree ?? 5000, shell: timeouts.shell ?? 2000, ocr: timeouts.ocr ?? 8000, vision: timeouts.vision ?? 8000 }; } async tree({ focusedOnly = true, maxNodes = 400 } = {}) { this.lastTree = []; const nodes = await timed(this.atspi.tree({ focusedOnly, maxNodes }), this.timeouts.tree, 'AT-SPI'); const windows = (await timed(this.shell.windows(), this.timeouts.shell, 'Shell helper').catch(() => ({ windows: [] }))).windows || []; this.lastTree = nodes.map((node) => { const win = windows.find(w => w.pid === node.pid && (w.focused || windows.filter(x => x.pid === node.pid).length === 1)); const rect = node.rect?.slice(); if (win?.rect && node.window_rect && rect && rect[0] > -100000 && rect[1] > -100000) { const bounds = win.buffer_rect || [win.rect[0] - Math.max(0, node.window_rect[2] - win.rect[2]) / 2, win.rect[1] - Math.max(0, node.window_rect[3] - win.rect[3]) / 2]; rect[0] += bounds[0] - node.window_rect[0]; rect[1] += bounds[1] - node.window_rect[1]; } return { ...node, rect, ref: `r${++this._refSequence}` }; }); return this.lastTree; } async _captureFrame() { const dest = path.join(this.tmpDir, `observe-${Date.now()}.png`); if (this.framebuffer?.capture) { const rawPath = await this.framebuffer.capture(dest); const frame = await this.normalizer.normalize(rawPath); return { ...frame, source: 'pipewire' }; } const rawPath = await this.screenshot.capture(dest); const frame = await this.normalizer.normalize(rawPath); return { ...frame, source: 'screenshot' }; } async _shellSnapshot() { if (this.shell.connect) await this.shell.connect(); const [windows, focused] = await Promise.all([this.shell.windows(), this.shell.focused()]); return { windows, focused }; } async observe({ includeTree = true, includeOcr = false, includeVision = false } = {}) { await mkdir(this.tmpDir, { recursive: true }); const unavailable = []; const note = (label) => (error) => { unavailable.push(`${label}: ${error.message}`); return null; }; const [frame, shell, tree] = await Promise.all([ timed(this._captureFrame(), this.timeouts.screenshot, 'frame').catch(note('frame')), timed(this._shellSnapshot(), this.timeouts.shell, 'Shell helper').catch(note('Shell helper')), includeTree ? this.tree().catch((error) => { unavailable.push(`AT-SPI: ${error.message}`); return []; }) : Promise.resolve([]), ]); const windows = shell?.windows || { available: false, windows: [] }; const result = { monitor: null, focused: shell?.focused || null, windows: windows.windows || [], tree, screenshot_path: frame?.path || null, frame_source: frame?.source || null, frame: frame ? { width: frame.width, height: frame.height, source_width: frame.source_width, source_height: frame.source_height, scale: frame.scale } : null, streams: this.framebuffer?.streams?.() || [], coordinate_space: 'desktop logical pixels; tree rects use desktop coordinates. Scale screenshot coordinates to the stream size and add its position before clicking.', ocr_blocks: [], vision_hint: null, unavailable, }; if (shell && !windows.available && windows.reason) result.unavailable.push(windows.reason); if (result.screenshot_path && includeOcr && this.ocr) result.ocr_blocks = await timed(this.ocr(result.screenshot_path), this.timeouts.ocr, 'OCR').catch((error) => { result.unavailable.push(`OCR: ${error.message}`); return []; }); if (result.screenshot_path && includeVision && this.vision) result.vision_hint = await timed(this.vision(result.screenshot_path), this.timeouts.vision, 'vision').catch((error) => { result.unavailable.push(`vision: ${error.message}`); return null; }); return result; } async zoom({ rect, ref } = {}) { await mkdir(this.tmpDir, { recursive: true }); const node = ref ? this.lastTree.find((item) => item.ref === ref) : null; const target = rect || node?.rect; if (!target) throw new Error('rect or current tree ref is required'); const raw = this.framebuffer?.capture ? await this.framebuffer.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`)) : await this.screenshot.capture(path.join(this.tmpDir, `zoom-${Date.now()}.png`)); const stream = this.framebuffer?.streams?.()?.[0]; let crop = target; if (stream?.position && stream?.size) { const dimensions = await this.normalizer.normalize(raw); const sx = dimensions.source_width / stream.size[0], sy = dimensions.source_height / stream.size[1]; crop = [(target[0] - stream.position[0]) * sx, (target[1] - stream.position[1]) * sy, target[2] * sx, target[3] * sy]; if (crop[0] < 0 || crop[1] < 0 || crop[0] + crop[2] > dimensions.source_width || crop[1] + crop[3] > dimensions.source_height) throw new Error('zoom target lies outside the shared monitor'); } const frame = await this.normalizer.normalize(raw, path.join(this.tmpDir, `zoom-${Date.now()}.webp`), crop); return { rect: target, screenshot_path: frame.path }; } async find({ query, role } = {}) { const nodes = this.lastTree.length ? this.lastTree : await this.tree(); return nodes.map((node) => ({ ...node, score: score(query, node) })).filter((node) => node.score && (!role || normalize(node.role) === normalize(role))).sort((a, b) => b.score - a.score).slice(0, 20); } }