78 lines
4.5 KiB
JavaScript
78 lines
4.5 KiB
JavaScript
/** Small dependency-free HTML reader. Never executes page scripts. */
|
||
const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', ndash: '–', mdash: '—', hellip: '…', lsquo: '‘', rsquo: '’', ldquo: '“', rdquo: '”', copy: '©' };
|
||
function decodeEntities(value) {
|
||
return String(value || '').replace(/&(#x[\da-f]+|#\d+|[a-z]+);/gi, (all, key) => {
|
||
if (key[0] !== '#') return ENTITIES[key.toLowerCase()] || all;
|
||
const n = key[1].toLowerCase() === 'x' ? parseInt(key.slice(2), 16) : Number(key.slice(1));
|
||
return n > 0 && n <= 0x10ffff && !(n >= 0xd800 && n <= 0xdfff) ? String.fromCodePoint(n) : '�';
|
||
});
|
||
}
|
||
function attributes(tag) {
|
||
const out = {};
|
||
const re = /([^\s=<>/]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/g;
|
||
let m;
|
||
while ((m = re.exec(tag))) out[m[1].toLowerCase()] = decodeEntities(m[2] ?? m[3] ?? m[4]);
|
||
return out;
|
||
}
|
||
function canonicalUrl(raw, base) {
|
||
try {
|
||
const u = new URL(raw, base);
|
||
if (!/^https?:$/.test(u.protocol) || u.username || u.password) return '';
|
||
u.hash = '';
|
||
for (const key of [...u.searchParams.keys()]) if (/^utm_|^(fbclid|gclid|msclkid)$/i.test(key)) u.searchParams.delete(key);
|
||
return u.href;
|
||
} catch (_) { return ''; }
|
||
}
|
||
function cleanHtml(html) {
|
||
return String(html || '').replace(/<!--[\s\S]*?(?:-->|$)/g, '')
|
||
.replace(/<(script|style|noscript|svg|template|nav|footer|header|aside)\b[^>]*>[\s\S]*?<\/\1\s*>/gi, '');
|
||
}
|
||
function readableText(html) {
|
||
return decodeEntities(cleanHtml(html)
|
||
.replace(/<li\b[^>]*>/gi, '\n• ')
|
||
.replace(/<br\b[^>]*>|<\/(?:p|div|section|article|main|h[1-6]|li|tr|pre|blockquote)>/gi, '\n')
|
||
.replace(/<\/(?:td|th)>/gi, ' | ')
|
||
.replace(/<[^>]*>/g, ''))
|
||
.replace(/[\t \f\v]+/g, ' ').replace(/ *\n */g, '\n').replace(/\n{3,}/g, '\n\n').trim();
|
||
}
|
||
function bounded(value, fallback, min, max) {
|
||
return Number.isFinite(Number(value)) ? Math.min(max, Math.max(min, Math.floor(Number(value)))) : fallback;
|
||
}
|
||
function extractPage(html, url, opts = {}) {
|
||
const raw = String(html || '');
|
||
const title = readableText((raw.match(/<title\b[^>]*>([\s\S]*?)<\/title>/i) || [])[1] || '');
|
||
const cleaned = cleanHtml(raw);
|
||
const main = (cleaned.match(/<(?:article|main)\b[^>]*>([\s\S]*?)<\/(?:article|main)>/i) || [])[1];
|
||
const body = (cleaned.match(/<body\b[^>]*>([\s\S]*?)<\/body>/i) || [])[1];
|
||
const content = main && readableText(main).length >= 80 ? main : body || cleaned;
|
||
const full = readableText(content);
|
||
const links = [], headings = [], seen = new Set();
|
||
let m;
|
||
const re = /<a\b([^>]*)>([\s\S]*?)<\/a>/gi;
|
||
while ((m = re.exec(cleaned)) && links.length < 150) {
|
||
const attr = attributes(m[1]);
|
||
if (!attr.href) continue;
|
||
const href = canonicalUrl(attr.href, url), text = readableText(m[2]).slice(0, 240);
|
||
if (!href || seen.has(href)) continue;
|
||
seen.add(href); links.push({ id: links.length + 1, url: href, text: text || attr.title || href });
|
||
}
|
||
const hr = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi;
|
||
while ((m = hr.exec(content)) && headings.length < 80) headings.push({ level: Number(m[1]), text: readableText(m[2]).slice(0, 300) });
|
||
const metadata = {};
|
||
for (const tag of raw.match(/<meta\b[^>]*>/gi) || []) {
|
||
const a = attributes(tag), key = (a.name || a.property || '').toLowerCase();
|
||
if (['description', 'author', 'article:published_time', 'og:title', 'og:description'].includes(key)) metadata[key] = (a.content || '').slice(0, 1000);
|
||
}
|
||
const offset = bounded(opts.offset, 0, 0, full.length), max = bounded(opts.max_chars, 12000, 200, 30000);
|
||
const out = { title, text: full.slice(offset, offset + max), links, headings, metadata, offset, total_chars: full.length, next_offset: offset + max < full.length ? offset + max : null };
|
||
if (/captcha|verify (?:that )?you are human|verifying you are|enable javascript and cookies|unusual traffic/i.test(full.slice(0, 4000))) out.warning = 'Page may be a bot challenge; content is not verified.';
|
||
if (!full && /<script\b/i.test(raw)) out.warning = 'Page requires JavaScript rendering; direct scraping cannot execute scripts.';
|
||
if (opts.find) {
|
||
const needle = String(opts.find).slice(0, 500).toLowerCase(), lower = full.toLowerCase();
|
||
out.matches = [];
|
||
for (let pos = lower.indexOf(needle); pos >= 0 && out.matches.length < 20; pos = lower.indexOf(needle, pos + needle.length)) out.matches.push({ offset: pos, text: full.slice(Math.max(0, pos - 180), pos + needle.length + 180) });
|
||
}
|
||
return out;
|
||
}
|
||
module.exports = { decodeEntities, attributes, canonicalUrl, readableText, extractPage };
|