|
| 1 | +import type { |
| 2 | + WebContentExtractionRequest, |
| 3 | + WebContentExtractionResult, |
| 4 | + WebContentExtractorPlugin, |
| 5 | +} from "openclaw/plugin-sdk/web-content-extractor"; |
| 6 | +import { |
| 7 | + htmlToMarkdown, |
| 8 | + normalizeWhitespace, |
| 9 | + sanitizeHtml, |
| 10 | + stripInvisibleUnicode, |
| 11 | +} from "openclaw/plugin-sdk/web-content-extractor"; |
| 12 | + |
| 13 | +const READABILITY_MAX_HTML_CHARS = 1_000_000; |
| 14 | +const READABILITY_MAX_ESTIMATED_NESTING_DEPTH = 3_000; |
| 15 | + |
| 16 | +type ParsedHtml = { |
| 17 | + document: Document; |
| 18 | +}; |
| 19 | + |
| 20 | +type ParseHtml = (html: string) => ParsedHtml; |
| 21 | + |
| 22 | +type ReadabilityResult = { |
| 23 | + content?: string; |
| 24 | + textContent?: string | null; |
| 25 | + title?: string | null; |
| 26 | +}; |
| 27 | + |
| 28 | +type ReadabilityInstance = { |
| 29 | + parse(): ReadabilityResult | null; |
| 30 | +}; |
| 31 | + |
| 32 | +type ReadabilityConstructor = new ( |
| 33 | + document: Document, |
| 34 | + options: { charThreshold: number }, |
| 35 | +) => ReadabilityInstance; |
| 36 | + |
| 37 | +type ReadabilityModule = { |
| 38 | + Readability: ReadabilityConstructor; |
| 39 | +}; |
| 40 | + |
| 41 | +type LinkedomModule = { |
| 42 | + parseHTML: ParseHtml; |
| 43 | +}; |
| 44 | + |
| 45 | +const READABILITY_MODULE = "@mozilla/readability"; |
| 46 | +const LINKEDOM_MODULE = "linkedom"; |
| 47 | + |
| 48 | +let readabilityDepsPromise: |
| 49 | + | Promise<{ |
| 50 | + Readability: ReadabilityConstructor; |
| 51 | + parseHTML: ParseHtml; |
| 52 | + }> |
| 53 | + | undefined; |
| 54 | + |
| 55 | +async function loadReadabilityDeps(): Promise<{ |
| 56 | + Readability: ReadabilityConstructor; |
| 57 | + parseHTML: ParseHtml; |
| 58 | +}> { |
| 59 | + if (!readabilityDepsPromise) { |
| 60 | + readabilityDepsPromise = Promise.all([ |
| 61 | + import(READABILITY_MODULE) as Promise<ReadabilityModule>, |
| 62 | + import(LINKEDOM_MODULE) as Promise<LinkedomModule>, |
| 63 | + ]).then(([readability, linkedom]) => ({ |
| 64 | + Readability: readability.Readability, |
| 65 | + parseHTML: linkedom.parseHTML, |
| 66 | + })); |
| 67 | + } |
| 68 | + try { |
| 69 | + return await readabilityDepsPromise; |
| 70 | + } catch (error) { |
| 71 | + readabilityDepsPromise = undefined; |
| 72 | + throw error; |
| 73 | + } |
| 74 | +} |
| 75 | + |
| 76 | +function normalizeLowercaseStringOrEmpty(value: string): string { |
| 77 | + return value.trim().toLowerCase(); |
| 78 | +} |
| 79 | + |
| 80 | +function exceedsEstimatedHtmlNestingDepth(html: string, maxDepth: number): boolean { |
| 81 | + const voidTags = new Set([ |
| 82 | + "area", |
| 83 | + "base", |
| 84 | + "br", |
| 85 | + "col", |
| 86 | + "embed", |
| 87 | + "hr", |
| 88 | + "img", |
| 89 | + "input", |
| 90 | + "link", |
| 91 | + "meta", |
| 92 | + "param", |
| 93 | + "source", |
| 94 | + "track", |
| 95 | + "wbr", |
| 96 | + ]); |
| 97 | + |
| 98 | + let depth = 0; |
| 99 | + const len = html.length; |
| 100 | + for (let i = 0; i < len; i++) { |
| 101 | + if (html.charCodeAt(i) !== 60) { |
| 102 | + continue; |
| 103 | + } |
| 104 | + const next = html.charCodeAt(i + 1); |
| 105 | + if (next === 33 || next === 63) { |
| 106 | + continue; |
| 107 | + } |
| 108 | + |
| 109 | + let j = i + 1; |
| 110 | + let closing = false; |
| 111 | + if (html.charCodeAt(j) === 47) { |
| 112 | + closing = true; |
| 113 | + j += 1; |
| 114 | + } |
| 115 | + |
| 116 | + while (j < len && html.charCodeAt(j) <= 32) { |
| 117 | + j += 1; |
| 118 | + } |
| 119 | + |
| 120 | + const nameStart = j; |
| 121 | + while (j < len) { |
| 122 | + const c = html.charCodeAt(j); |
| 123 | + const isNameChar = |
| 124 | + (c >= 65 && c <= 90) || |
| 125 | + (c >= 97 && c <= 122) || |
| 126 | + (c >= 48 && c <= 57) || |
| 127 | + c === 58 || |
| 128 | + c === 45; |
| 129 | + if (!isNameChar) { |
| 130 | + break; |
| 131 | + } |
| 132 | + j += 1; |
| 133 | + } |
| 134 | + |
| 135 | + const tagName = normalizeLowercaseStringOrEmpty(html.slice(nameStart, j)); |
| 136 | + if (!tagName) { |
| 137 | + continue; |
| 138 | + } |
| 139 | + |
| 140 | + if (closing) { |
| 141 | + depth = Math.max(0, depth - 1); |
| 142 | + continue; |
| 143 | + } |
| 144 | + if (voidTags.has(tagName)) { |
| 145 | + continue; |
| 146 | + } |
| 147 | + |
| 148 | + let selfClosing = false; |
| 149 | + for (let k = j; k < len && k < j + 200; k++) { |
| 150 | + const c = html.charCodeAt(k); |
| 151 | + if (c === 62) { |
| 152 | + selfClosing = html.charCodeAt(k - 1) === 47; |
| 153 | + break; |
| 154 | + } |
| 155 | + } |
| 156 | + if (selfClosing) { |
| 157 | + continue; |
| 158 | + } |
| 159 | + |
| 160 | + depth += 1; |
| 161 | + if (depth > maxDepth) { |
| 162 | + return true; |
| 163 | + } |
| 164 | + } |
| 165 | + return false; |
| 166 | +} |
| 167 | + |
| 168 | +async function extractWithReadability( |
| 169 | + request: WebContentExtractionRequest, |
| 170 | +): Promise<WebContentExtractionResult | null> { |
| 171 | + const cleanHtml = await sanitizeHtml(request.html); |
| 172 | + if ( |
| 173 | + cleanHtml.length > READABILITY_MAX_HTML_CHARS || |
| 174 | + exceedsEstimatedHtmlNestingDepth(cleanHtml, READABILITY_MAX_ESTIMATED_NESTING_DEPTH) |
| 175 | + ) { |
| 176 | + return null; |
| 177 | + } |
| 178 | + try { |
| 179 | + const { Readability, parseHTML } = await loadReadabilityDeps(); |
| 180 | + const { document } = parseHTML(cleanHtml); |
| 181 | + try { |
| 182 | + (document as { baseURI?: string }).baseURI = request.url; |
| 183 | + } catch { |
| 184 | + // Best-effort base URI for relative links. |
| 185 | + } |
| 186 | + const reader = new Readability(document, { charThreshold: 0 }); |
| 187 | + const parsed = reader.parse(); |
| 188 | + if (!parsed?.content) { |
| 189 | + return null; |
| 190 | + } |
| 191 | + const title = parsed.title || undefined; |
| 192 | + if (request.extractMode === "text") { |
| 193 | + const text = stripInvisibleUnicode(normalizeWhitespace(parsed.textContent ?? "")); |
| 194 | + return text ? { text, title } : null; |
| 195 | + } |
| 196 | + const rendered = htmlToMarkdown(parsed.content); |
| 197 | + const text = stripInvisibleUnicode(rendered.text); |
| 198 | + return text ? { text, title: title ?? rendered.title } : null; |
| 199 | + } catch { |
| 200 | + return null; |
| 201 | + } |
| 202 | +} |
| 203 | + |
| 204 | +export function createReadabilityWebContentExtractor(): WebContentExtractorPlugin { |
| 205 | + return { |
| 206 | + id: "readability", |
| 207 | + label: "Readability", |
| 208 | + autoDetectOrder: 10, |
| 209 | + extract: extractWithReadability, |
| 210 | + }; |
| 211 | +} |
0 commit comments