46 lines
3.3 KiB
JavaScript
46 lines
3.3 KiB
JavaScript
import { formatBrowserReading, PAGE_VISION_QUESTION } from '../browser-use/reading.js';
|
|
|
|
const BROWSER_ACTIONS = ['navigate', 'snapshot', 'click', 'type', 'press', 'scroll', 'wait'];
|
|
|
|
export function createBrowserTools({ browser } = {}) {
|
|
return [{
|
|
name: 'browser',
|
|
permission: 'read',
|
|
description: 'The only web tool. Drive the headed Jarvis Chromium window. Not computer-use and not web_search or web_fetch; those are removed. To search, navigate to https://duckduckgo.com/?q=QUERY, then click the best result ref and read that page before answering. The window opens on https://duckduckgo.com/. A result includes page_text for the full page after it was scrolled, plus headings, tables, visible labels, and image slices from top to bottom. Read all of page_text and speak a complete report from those facts, including a search-results page. Do not stop after thinking. The window stays open. action: navigate (url), snapshot, click (ref), type (ref + text, optional submit), press (key), scroll (dy), wait (ms). Snapshot or navigate before click or type; refs change. Cookie banners: snapshot, then click Accept by ref. If challenge is true, ask the user to finish the visible window, then snapshot again. Private and localhost urls are blocked. Do not speak refs or JSON.',
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
action: {
|
|
type: 'string',
|
|
enum: BROWSER_ACTIONS,
|
|
description: 'navigate opens url. snapshot lists numbered refs. click/type need ref from that snapshot. press needs key. scroll uses dy. wait uses ms.',
|
|
},
|
|
url: { type: 'string', description: 'Public http or https url for navigate.' },
|
|
ref: { type: 'string', description: 'Numbered control from the last snapshot, for click or type.' },
|
|
text: { type: 'string', description: 'Text to type into ref, or a click-by-label fallback, or wait-for text.' },
|
|
key: { type: 'string', description: 'Key or chord for press, such as Enter, Tab, Escape, Control+l.' },
|
|
selector: { type: 'string', description: 'Optional CSS selector when a snapshot ref is missing.' },
|
|
dy: { type: 'number', description: 'Vertical scroll pixels. Positive scrolls down.' },
|
|
dx: { type: 'number', description: 'Horizontal scroll pixels.' },
|
|
ms: { type: 'number', description: 'Milliseconds to wait, cap 20000.' },
|
|
submit: { type: 'boolean', description: 'After type, press Enter.' },
|
|
},
|
|
required: ['action'],
|
|
},
|
|
execute: async (input = {}) => {
|
|
if (!browser || typeof browser.call !== 'function') throw new Error('Jarvis browser helper unavailable');
|
|
const action = String(input.action || '').trim();
|
|
if (!action) throw new Error('action required');
|
|
const { action: _ignored, ...payload } = input;
|
|
const result = await browser.call(action, payload);
|
|
if (!result || typeof result !== 'object' || Array.isArray(result)) return result;
|
|
const reading = formatBrowserReading(result);
|
|
if (!reading) return result;
|
|
const images = Array.isArray(result.images) ? result.images.filter((image) => image && image.path) : [];
|
|
return { reading, ...(images.length ? { images, visionQuestion: PAGE_VISION_QUESTION } : {}) };
|
|
},
|
|
}];
|
|
}
|
|
|
|
export { BROWSER_ACTIONS, PAGE_VISION_QUESTION };
|