Skip to content

Commit 2f64590

Browse files
vincentkocvincentkoc
authored
improve(web-fetch): speed up provider fallback loading (#98559)
* perf(web-fetch): add web fetch benchmark * perf(web-fetch): cache provider fallback resolution * perf(firecrawl): avoid duplicate scrape payload wrapping * perf(web-readability): reuse void tag lookup * fix(web-fetch): keep provider cache live * fix(web-fetch): invalidate provider cache on registry swap * perf(web-fetch): reduce html extraction overhead * fix(web-fetch): bound provider discovery cache * perf(web-fetch): avoid full plugin load on fallback
1 parent 1d839d3 commit 2f64590

17 files changed

Lines changed: 1170 additions & 93 deletions

‎extensions/firecrawl/src/firecrawl-client.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,7 @@ describe("parseFirecrawlScrapePayload", () => {
421421
expect(result.extractor).toBe("firecrawl");
422422
expect(result.extractMode).toBe("markdown");
423423
expect(result.text).toContain("# Hello");
424+
expect(result.wrappedLength).toBe((result.text as string).length);
424425
expect(result.truncated).toBe(false);
425426
});
426427

‎extensions/firecrawl/src/firecrawl-client.ts‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -475,6 +475,10 @@ export function parseFirecrawlScrapePayload(params: {
475475
}
476476
const rawText = params.extractMode === "text" ? markdownToText(markdown) : markdown;
477477
const truncated = truncateText(rawText, params.maxChars);
478+
const wrappedText = wrapExternalContent(truncated.text, {
479+
source: "web_fetch",
480+
includeWarning: false,
481+
});
478482
return {
479483
url: params.url,
480484
finalUrl:
@@ -498,14 +502,8 @@ export function parseFirecrawlScrapePayload(params: {
498502
},
499503
truncated: truncated.truncated,
500504
rawLength: rawText.length,
501-
wrappedLength: wrapExternalContent(truncated.text, {
502-
source: "web_fetch",
503-
includeWarning: false,
504-
}).length,
505-
text: wrapExternalContent(truncated.text, {
506-
source: "web_fetch",
507-
includeWarning: false,
508-
}),
505+
wrappedLength: wrappedText.length,
506+
text: wrappedText,
509507
warning:
510508
typeof params.payload.warning === "string" && params.payload.warning
511509
? wrapExternalContent(params.payload.warning, {

‎extensions/firecrawl/src/firecrawl-fetch-provider.ts‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,20 @@
11
// Firecrawl provider module implements model/runtime integration.
22
import { readPositiveIntegerParam } from "openclaw/plugin-sdk/param-readers";
3-
import type { WebFetchProviderPlugin } from "openclaw/plugin-sdk/provider-web-fetch";
4-
import { enablePluginInConfig } from "openclaw/plugin-sdk/provider-web-fetch";
5-
import { runFirecrawlScrape } from "./firecrawl-client.js";
3+
import {
4+
enablePluginInConfig,
5+
type WebFetchProviderPlugin,
6+
} from "openclaw/plugin-sdk/provider-web-fetch-contract";
67
import { FIRECRAWL_WEB_FETCH_PROVIDER_SHARED } from "./firecrawl-fetch-provider-shared.js";
78

9+
type FirecrawlClientModule = typeof import("./firecrawl-client.js");
10+
11+
let firecrawlClientModulePromise: Promise<FirecrawlClientModule> | undefined;
12+
13+
function loadFirecrawlClientModule(): Promise<FirecrawlClientModule> {
14+
firecrawlClientModulePromise ??= import("./firecrawl-client.js");
15+
return firecrawlClientModulePromise;
16+
}
17+
818
export function createFirecrawlWebFetchProvider(): WebFetchProviderPlugin {
919
return {
1020
...FIRECRAWL_WEB_FETCH_PROVIDER_SHARED,
@@ -21,6 +31,7 @@ export function createFirecrawlWebFetchProvider(): WebFetchProviderPlugin {
2131
? args.proxy
2232
: undefined;
2333
const storeInCache = typeof args.storeInCache === "boolean" ? args.storeInCache : undefined;
34+
const { runFirecrawlScrape } = await loadFirecrawlClientModule();
2435
return await runFirecrawlScrape({
2536
cfg: config,
2637
url,

‎extensions/web-readability/web-content-extractor.test.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,4 +61,15 @@ describe("web readability extractor", () => {
6161
expect(extracted.text).toContain("Main content starts here");
6262
expect(extracted.title).toBe("Example Article");
6363
});
64+
65+
it("does not count void tags toward the nesting limit", async () => {
66+
const extractor = createReadabilityWebContentExtractor();
67+
const html = SAMPLE_HTML.replace("<article>", `<article>${"<BR>".repeat(3100)}`);
68+
const result = await extractor.extract({
69+
html,
70+
url: "https://example.com/article",
71+
extractMode: "markdown",
72+
});
73+
expect(requireReadabilityResult(result).text).toContain("Main content starts here");
74+
});
6475
});

‎extensions/web-readability/web-content-extractor.ts‎

Lines changed: 22 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,22 @@ import {
1313

1414
const READABILITY_MAX_HTML_CHARS = 1_000_000;
1515
const READABILITY_MAX_ESTIMATED_NESTING_DEPTH = 3_000;
16+
const HTML_VOID_TAGS = new Set([
17+
"area",
18+
"base",
19+
"br",
20+
"col",
21+
"embed",
22+
"hr",
23+
"img",
24+
"input",
25+
"link",
26+
"meta",
27+
"param",
28+
"source",
29+
"track",
30+
"wbr",
31+
]);
1632

1733
type ParsedHtml = {
1834
document: Document;
@@ -74,28 +90,7 @@ async function loadReadabilityDeps(): Promise<{
7490
}
7591
}
7692

77-
function normalizeLowercaseStringOrEmpty(value: string): string {
78-
return value.trim().toLowerCase();
79-
}
80-
8193
function exceedsEstimatedHtmlNestingDepth(html: string, maxDepth: number): boolean {
82-
const voidTags = new Set([
83-
"area",
84-
"base",
85-
"br",
86-
"col",
87-
"embed",
88-
"hr",
89-
"img",
90-
"input",
91-
"link",
92-
"meta",
93-
"param",
94-
"source",
95-
"track",
96-
"wbr",
97-
]);
98-
9994
let depth = 0;
10095
const len = html.length;
10196
for (let i = 0; i < len; i++) {
@@ -133,16 +128,18 @@ function exceedsEstimatedHtmlNestingDepth(html: string, maxDepth: number): boole
133128
j += 1;
134129
}
135130

136-
const tagName = normalizeLowercaseStringOrEmpty(html.slice(nameStart, j));
137-
if (!tagName) {
131+
if (j === nameStart) {
138132
continue;
139133
}
140134

141135
if (closing) {
142-
depth = Math.max(0, depth - 1);
136+
if (depth > 0) {
137+
depth -= 1;
138+
}
143139
continue;
144140
}
145-
if (voidTags.has(tagName)) {
141+
const tagName = html.slice(nameStart, j).toLowerCase();
142+
if (HTML_VOID_TAGS.has(tagName)) {
146143
continue;
147144
}
148145

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1692,6 +1692,7 @@
16921692
"openclaw": "node scripts/run-node.mjs",
16931693
"openclaw:rpc": "node scripts/run-node.mjs agent --mode rpc --json",
16941694
"perf:issue-78851": "node --import tsx scripts/perf/issue-78851-model-resolution.ts",
1695+
"perf:web-fetch": "node --import tsx scripts/bench-web-fetch.ts",
16951696
"perf:kova:summary": "node scripts/kova-ci-summary.mjs",
16961697
"perf:source:summary": "node scripts/openclaw-performance-source-summary.mjs",
16971698
"plugin-sdk:api:check": "node --max-old-space-size=8192 --import tsx scripts/generate-plugin-sdk-api-baseline.ts --check",

0 commit comments

Comments
 (0)