Computer Use Updates

2026-09-13 19:30:17 -04:00
parent c5ccaa490b
commit 599bfe440d
34 changed files with 1033 additions and 573 deletions
@@ -0,0 +1,68 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { createRequire } from 'node:module';
+const require = createRequire(import.meta.url);
+const web = require('../vendor/agent-harness/agent/web-search.js');
+const reader = require('../vendor/agent-harness/agent/web-reader.js');
+const response = (url, text, status = 200, headers = {}) => ({ url: String(url), status, text: async () => text, headers: { get: key => headers[key] || null } });
+
+test('reader extracts structured content, relative links, entities, pagination and find', () => {
+  const html = `<html><head><title>A &amp; B</title><meta name='description' content='A guide'></head><body><nav>Ignore navigation</nav><main><h1>Guide &#x1f680;</h1><p>${'Useful text. '.repeat(40)}</p><a href='../next?utm_source=x&amp;q=one'>Next</a><script>malicious()</script></main></body></html>`;
+  const page = reader.extractPage(html, 'https://example.com/docs/start', { max_chars: 200, find: 'Useful' });
+  assert.equal(page.title, 'A & B');
+  assert.equal(page.headings[0].text, 'Guide 🚀');
+  assert.equal(page.links[0].url, 'https://example.com/next?q=one');
+  assert.equal(page.metadata.description, 'A guide');
+  assert.equal(page.next_offset, 200);
+  assert.equal(page.matches.length, 20);
+  assert.doesNotMatch(page.text, /navigation|malicious/);
+  assert.equal(reader.extractPage(html, 'https://example.com', { offset: 200 }).text, reader.extractPage(html, 'https://example.com').text.slice(200));
+});
+
+test('DDG parses reordered, single-quoted attributes and snippets', () => {
+  const hits = web.parseDdgHtmlHits(`<a href='/l/?uddg=https%3A%2F%2Fexample.com%2Fa%253Fb' class='extra result__a'>Title &#8212; test</a><div class='result__snippet'>The snippet</div>`);
+  assert.equal(hits[0].url, 'https://example.com/a%3Fb');
+  assert.equal(hits[0].title, 'Title — test');
+  assert.equal(hits[0].snippet, 'The snippet');
+  assert.equal(web.parseDdgLiteHits(`<a href='https://example.com' class='result-link'>Lite</a>`)[0].title, 'Lite');
+});
+
+test('auto merges scraped engines and removes tracking duplicates without calling APIs', async () => {
+  const orig = globalThis.fetch; const calls = [];
+  globalThis.fetch = async url => {
+    calls.push(String(url));
+    if (String(url).includes('duckduckgo')) return response(url, '<a class="result__a" href="https://example.com/a?utm_source=ddg">Example</a>');
+    if (String(url).includes('bing')) return response(url, '<rss><item><title>Example</title><link>https://example.com/a</link></item><item><title>Other</title><link>https://other.example/b</link></item></rss>');
+    return response(url, '');
+  };
+  try {
+    const hits = await web.runWebSearch('example', { limit: 2, timeoutMs: 500 });
+    assert.equal(hits.length, 2);
+    assert.deepEqual(hits[0].sources, ['duckduckgo', 'bing_rss']);
+    assert.equal(hits[0].url, 'https://example.com/a');
+    assert.ok(calls.every(url => !/jina|api\.|\/api\//.test(url)));
+    assert.match((await web.runWebSearch('test', { engine: 'jina' })).error, /unknown engine/);
+  } finally { globalThis.fetch = orig; }
+});
+
+test('direct fetch checks redirects, rejects binary pages, and reports challenges', async () => {
+  const orig = globalThis.fetch; let calls = 0;
+  try {
+    globalThis.fetch = async url => { calls++; return response(url, '', 302, { location: 'http://127.0.0.1/secret' }); };
+    assert.match((await web.fetchPage('https://example.com')).error, /blocked/);
+    assert.equal(calls, 1);
+    globalThis.fetch = async url => response(url, 'binary', 200, { 'content-type': 'application/pdf' });
+    assert.match((await web.fetchPage('https://example.com')).error, /unsupported content type/);
+    globalThis.fetch = async url => response(url, '<p>Verify you are human</p>');
+    assert.match((await web.fetchPage('https://example.com')).warning, /challenge/);
+    globalThis.fetch = async url => response(url, 'x'.repeat(2 * 1024 * 1024 + 1));
+    assert.match((await web.fetchPage('https://example.com')).error, /limit/);
+  } finally { globalThis.fetch = orig; }
+});
+
+test('stalled streaming bodies are cancelled at the deadline', async () => {
+  let cancelled = false;
+  const res = new Response(new ReadableStream({ cancel() { cancelled = true; } }));
+  await assert.rejects(web.readBodyWithTimeout(res, 30), /timed out/);
+  assert.equal(cancelled, true);
+});