Skip to content

Commit 0a14444

Browse files
joshavantjoshavant
committed
Bound successful provider response reads
1 parent 0a042f6 commit 0a14444

24 files changed

Lines changed: 384 additions & 88 deletions

‎docs/.generated/plugin-sdk-api-baseline.sha256‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
6620d5a6100d60f98cf13b8a13e3c46e9631400d1a1d7c0c6a22c490da810813 plugin-sdk-api-baseline.json
2-
961377a56fd0fb3307fb4be95dcb480610f14c717e1b82e4bf262dd5faaddcbc plugin-sdk-api-baseline.jsonl
1+
35b314075ff47453c5d57788861ca0c0e65d6a988b549ab2a2e1757b7590d140 plugin-sdk-api-baseline.json
2+
0dc8abcefccfe7d19280bde5fb2c0c69cf73b782d47e3759e2984baf904fe07c plugin-sdk-api-baseline.jsonl

‎extensions/duckduckgo/src/ddg-client.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Duckduckgo plugin module implements ddg client behavior.
22
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
3+
import { readProviderTextResponse } from "openclaw/plugin-sdk/provider-http";
34
import {
45
DEFAULT_CACHE_TTL_MINUTES,
56
DEFAULT_SEARCH_COUNT,
@@ -113,6 +114,10 @@ function isBotChallenge(html: string): boolean {
113114
return /g-recaptcha|are you a human|id="challenge-form"|name="challenge"/i.test(html);
114115
}
115116

117+
async function readDuckDuckGoHtmlResponse(response: Response): Promise<string> {
118+
return await readProviderTextResponse(response, "DuckDuckGo search");
119+
}
120+
116121
function parseDuckDuckGoHtml(html: string): DuckDuckGoResult[] {
117122
const results: DuckDuckGoResult[] = [];
118123
const resultRegex = /<a\b(?=[^>]*\bclass="[^"]*\bresult__a\b[^"]*")([^>]*)>([\s\S]*?)<\/a>/gi;
@@ -202,7 +207,7 @@ export async function runDuckDuckGoSearch(params: {
202207
);
203208
}
204209

205-
const html = await response.text();
210+
const html = await readDuckDuckGoHtmlResponse(response);
206211
if (isBotChallenge(html)) {
207212
throw new Error("DuckDuckGo returned a bot-detection challenge.");
208213
}
@@ -238,5 +243,6 @@ export const testing = {
238243
decodeHtmlEntities,
239244
isBotChallenge,
240245
parseDuckDuckGoHtml,
246+
readDuckDuckGoHtmlResponse,
241247
};
242248
export { testing as __testing };

‎extensions/duckduckgo/src/ddg-search-provider.test.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Duckduckgo tests cover ddg search provider plugin behavior.
22
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
3+
import { createStreamingResponse } from "../../test-support/streaming-error-response.js";
34
import { createDuckDuckGoWebSearchProvider as createDuckDuckGoWebSearchContractProvider } from "../web-search-contract-api.js";
45
import { DEFAULT_DDG_SAFE_SEARCH, resolveDdgRegion, resolveDdgSafeSearch } from "./config.js";
56

@@ -104,6 +105,24 @@ describe("duckduckgo web search provider", () => {
104105
expect(runDuckDuckGoSearch).not.toHaveBeenCalled();
105106
});
106107

108+
it("bounds successful DuckDuckGo HTML bodies without using response.text()", async () => {
109+
const streamed = createStreamingResponse({
110+
chunkCount: 32,
111+
chunkSize: 1024 * 1024,
112+
text: "x",
113+
headers: { "Content-Type": "text/html" },
114+
});
115+
const textSpy = vi.spyOn(streamed.response, "text").mockRejectedValue(new Error("unbounded"));
116+
117+
await expect(ddgClientTesting.readDuckDuckGoHtmlResponse(streamed.response)).rejects.toThrow(
118+
"DuckDuckGo search: text response exceeds 16777216 bytes",
119+
);
120+
121+
expect(streamed.getReadCount()).toBeLessThan(32);
122+
expect(streamed.wasCanceled()).toBe(true);
123+
expect(textSpy).not.toHaveBeenCalled();
124+
});
125+
107126
it("reads region from plugin config and normalizes empty values away", () => {
108127
expect(
109128
resolveDdgRegion({

‎extensions/firecrawl/src/firecrawl-client.ts‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Firecrawl plugin module implements firecrawl client behavior.
22
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
3+
import { readProviderJsonResponse } from "openclaw/plugin-sdk/provider-http";
34
import {
45
DEFAULT_CACHE_TTL_MINUTES,
56
markdownToText,
@@ -41,6 +42,7 @@ const SCRAPE_CACHE = new Map<
4142
>();
4243
const DEFAULT_SEARCH_COUNT = 5;
4344
const DEFAULT_SCRAPE_MAX_CHARS = 50_000;
45+
const FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES = 64 * 1024 * 1024;
4446
const ALLOWED_FIRECRAWL_HOSTS = new Set(["api.firecrawl.dev"]);
4547
const FIRECRAWL_SELF_HOSTED_PRIVATE_ERROR =
4648
"Firecrawl custom baseUrl must target a private or internal self-hosted endpoint.";
@@ -65,12 +67,9 @@ type FirecrawlSearchItem = {
6567
async function readFirecrawlJsonResponse(
6668
response: Response,
6769
label: string,
70+
opts?: { maxBytes?: number },
6871
): Promise<Record<string, unknown>> {
69-
try {
70-
return (await response.json()) as Record<string, unknown>;
71-
} catch (cause) {
72-
throw new Error(`${label}: malformed JSON response`, { cause });
73-
}
72+
return await readProviderJsonResponse<Record<string, unknown>>(response, label, opts);
7473
}
7574

7675
export type FirecrawlSearchParams = {
@@ -220,11 +219,9 @@ async function postFirecrawlJson<T>(
220219
const readJsonPayload = async (): Promise<Record<string, unknown> | null> => {
221220
const candidate = response as Response & { clone?: () => Response };
222221
const jsonResponse = typeof candidate.clone === "function" ? candidate.clone() : response;
223-
if (typeof jsonResponse.json !== "function") {
224-
return null;
225-
}
226222
try {
227-
const payload = await jsonResponse.json();
223+
const body = await readResponseText(jsonResponse, { maxBytes: 64_000 });
224+
const payload = JSON.parse(body.text) as unknown;
228225
return payload && typeof payload === "object" && !Array.isArray(payload)
229226
? (payload as Record<string, unknown>)
230227
: null;
@@ -579,7 +576,10 @@ export async function runFirecrawlScrape(
579576
},
580577
},
581578
async (response) => {
582-
const payloadLocal = await readFirecrawlJsonResponse(response, "Firecrawl fetch failed");
579+
const payloadLocal = await readFirecrawlJsonResponse(response, "Firecrawl fetch failed", {
580+
// Scrape can legitimately return page bodies before maxChars truncates parsed output.
581+
maxBytes: FIRECRAWL_SCRAPE_RESPONSE_MAX_BYTES,
582+
});
583583
if (payloadLocal.success === false) {
584584
const detail =
585585
typeof payloadLocal.error === "string"
@@ -613,6 +613,7 @@ export const testing = {
613613
assertFirecrawlScrapeTargetAllowed,
614614
parseFirecrawlScrapePayload,
615615
postFirecrawlJson,
616+
readFirecrawlJsonResponse,
616617
resolveEndpoint,
617618
validateFirecrawlBaseUrl,
618619
resolveSearchItems,

‎extensions/firecrawl/src/firecrawl-tools.test.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
33
import { mockPinnedHostnameResolution } from "openclaw/plugin-sdk/test-env";
44
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
5+
import { createStreamingResponse } from "../../test-support/streaming-error-response.js";
56
import {
67
DEFAULT_FIRECRAWL_BASE_URL,
78
DEFAULT_FIRECRAWL_MAX_AGE_MS,
@@ -966,6 +967,27 @@ describe("firecrawl tools", () => {
966967
).rejects.toThrow("Firecrawl Search API error: malformed JSON response");
967968
});
968969

970+
it("bounds successful Firecrawl JSON bodies before parsing", async () => {
971+
const streamed = createStreamingResponse({
972+
chunkCount: 32,
973+
chunkSize: 1024 * 1024,
974+
text: "x",
975+
headers: { "content-type": "application/json" },
976+
});
977+
const jsonSpy = vi.spyOn(streamed.response, "json").mockRejectedValue(new Error("unbounded"));
978+
979+
await expect(
980+
firecrawlClientTesting.readFirecrawlJsonResponse(
981+
streamed.response,
982+
"Firecrawl Search API error",
983+
),
984+
).rejects.toThrow("Firecrawl Search API error: JSON response exceeds 16777216 bytes");
985+
986+
expect(streamed.getReadCount()).toBeLessThan(32);
987+
expect(streamed.wasCanceled()).toBe(true);
988+
expect(jsonSpy).not.toHaveBeenCalled();
989+
});
990+
969991
it("reports malformed Firecrawl scrape JSON with a stable provider error", async () => {
970992
global.fetch = vi.fn(
971993
async () =>

‎extensions/ollama/src/embedding-provider.test.ts‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Ollama tests cover embedding provider plugin behavior.
22
import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth";
33
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
4+
import { createStreamingResponse } from "../../test-support/streaming-error-response.js";
45

56
const { fetchConfiguredLocalOriginWithSsrFGuardMock } = vi.hoisted(() => ({
67
fetchConfiguredLocalOriginWithSsrFGuardMock: vi.fn(
@@ -412,10 +413,40 @@ describe("ollama embedding provider", () => {
412413
});
413414

414415
await expect(provider.embedQuery("hello")).rejects.toThrow(
415-
"Ollama embed response returned malformed JSON",
416+
"Ollama embed response: malformed JSON response",
416417
);
417418
});
418419

420+
it("bounds successful embed JSON bodies before parsing", async () => {
421+
const streamed = createStreamingResponse({
422+
chunkCount: 32,
423+
chunkSize: 1024 * 1024,
424+
text: "x",
425+
headers: { "content-type": "application/json" },
426+
});
427+
const jsonSpy = vi.spyOn(streamed.response, "json").mockRejectedValue(new Error("unbounded"));
428+
vi.stubGlobal(
429+
"fetch",
430+
vi.fn(async () => streamed.response),
431+
);
432+
433+
const { provider } = await createOllamaEmbeddingProvider({
434+
config: {} as OpenClawConfig,
435+
provider: "ollama",
436+
model: "nomic-embed-text",
437+
fallback: "none",
438+
remote: { baseUrl: "http://127.0.0.1:11434" },
439+
});
440+
441+
await expect(provider.embedQuery("hello")).rejects.toThrow(
442+
"Ollama embed response: JSON response exceeds 16777216 bytes",
443+
);
444+
445+
expect(streamed.getReadCount()).toBeLessThan(32);
446+
expect(streamed.wasCanceled()).toBe(true);
447+
expect(jsonSpy).not.toHaveBeenCalled();
448+
});
449+
419450
it("rejects non-number embedding values instead of zeroing them", async () => {
420451
vi.stubGlobal(
421452
"fetch",

‎extensions/ollama/src/embedding-provider.ts‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,10 @@ import {
66
normalizeOptionalSecretInput,
77
} from "openclaw/plugin-sdk/provider-auth";
88
import { resolveEnvApiKey } from "openclaw/plugin-sdk/provider-auth-runtime";
9-
import { readResponseTextLimited } from "openclaw/plugin-sdk/provider-http";
9+
import {
10+
readProviderJsonResponse,
11+
readResponseTextLimited,
12+
} from "openclaw/plugin-sdk/provider-http";
1013
import { normalizeProviderId } from "openclaw/plugin-sdk/provider-model-shared";
1114
import {
1215
hasConfiguredSecretInput,
@@ -117,14 +120,9 @@ async function withRemoteHttpResponse<T>(params: {
117120
}
118121

119122
async function readOllamaEmbeddingJsonResponse(
120-
response: Pick<Response, "json">,
123+
response: Response,
121124
): Promise<{ embeddings?: unknown }> {
122-
let payload: unknown;
123-
try {
124-
payload = await response.json();
125-
} catch (cause) {
126-
throw new Error("Ollama embed response returned malformed JSON", { cause });
127-
}
125+
const payload = await readProviderJsonResponse<unknown>(response, "Ollama embed response");
128126
if (typeof payload !== "object" || payload === null || Array.isArray(payload)) {
129127
throw new Error("Ollama embed response returned a non-object JSON payload");
130128
}

‎extensions/ollama/src/web-search-provider.test.ts‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Ollama tests cover web search provider plugin behavior.
22
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
33
import { beforeEach, describe, expect, it, vi } from "vitest";
4+
import { createStreamingResponse } from "../../test-support/streaming-error-response.js";
45
import { createOllamaWebSearchProvider as createContractOllamaWebSearchProvider } from "../web-search-contract-api.js";
56
import {
67
testing,
@@ -403,7 +404,32 @@ describe("ollama web search provider", () => {
403404
config: createOllamaConfig(),
404405
query: "openclaw",
405406
}),
406-
).rejects.toThrow("Ollama web search returned malformed JSON");
407+
).rejects.toThrow("Ollama web search: malformed JSON response");
408+
});
409+
410+
it("bounds successful Ollama web search JSON bodies before parsing", async () => {
411+
const streamed = createStreamingResponse({
412+
chunkCount: 32,
413+
chunkSize: 1024 * 1024,
414+
text: "x",
415+
headers: { "content-type": "application/json" },
416+
});
417+
const jsonSpy = vi.spyOn(streamed.response, "json").mockRejectedValue(new Error("unbounded"));
418+
fetchWithSsrFGuardMock.mockResolvedValueOnce({
419+
response: streamed.response,
420+
release: vi.fn(async () => {}),
421+
});
422+
423+
await expect(
424+
runOllamaWebSearch({
425+
config: createOllamaConfig(),
426+
query: "openclaw",
427+
}),
428+
).rejects.toThrow("Ollama web search: JSON response exceeds 16777216 bytes");
429+
430+
expect(streamed.getReadCount()).toBeLessThan(32);
431+
expect(streamed.wasCanceled()).toBe(true);
432+
expect(jsonSpy).not.toHaveBeenCalled();
407433
});
408434

409435
it("warns when Ollama is not reachable during setup without cancelling", async () => {

‎extensions/ollama/src/web-search-provider.ts‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
normalizeOptionalSecretInput,
66
} from "openclaw/plugin-sdk/provider-auth";
77
import { resolveEnvApiKey } from "openclaw/plugin-sdk/provider-auth-runtime";
8+
import { readProviderJsonResponse } from "openclaw/plugin-sdk/provider-http";
89
import {
910
enablePluginInConfig,
1011
readPositiveIntegerParam,
@@ -67,11 +68,7 @@ type OllamaWebSearchAttempt = {
6768
};
6869

6970
async function readOllamaWebSearchResponse(response: Response): Promise<OllamaWebSearchResponse> {
70-
try {
71-
return (await response.json()) as OllamaWebSearchResponse;
72-
} catch (cause) {
73-
throw new Error("Ollama web search returned malformed JSON", { cause });
74-
}
71+
return await readProviderJsonResponse<OllamaWebSearchResponse>(response, "Ollama web search");
7572
}
7673

7774
function isOllamaCloudBaseUrl(baseUrl: string): boolean {

‎extensions/parallel/src/parallel-mcp-search.runtime.test.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { beforeEach, describe, expect, it, vi } from "vitest";
2+
import { createStreamingResponse } from "../../test-support/streaming-error-response.js";
23

34
type EndpointCall = {
45
url: string;
@@ -311,4 +312,27 @@ describe("runParallelMcpSearch", () => {
311312
expect(tracked.wasCanceled()).toBe(true);
312313
expect(textSpy).not.toHaveBeenCalled();
313314
});
315+
316+
it("bounds successful MCP bodies without using response.text()", async () => {
317+
const streamed = createStreamingResponse({
318+
chunkCount: 32,
319+
chunkSize: 1024 * 1024,
320+
text: "x",
321+
headers: { "Content-Type": "application/json" },
322+
});
323+
const textSpy = vi.spyOn(streamed.response, "text").mockRejectedValue(new Error("unbounded"));
324+
endpointMockState.responses.push(streamed.response);
325+
326+
const error = await runParallelMcpSearch({ searchQueries: ["x"], maxResults: 5 }).catch(
327+
(cause: unknown) => cause,
328+
);
329+
330+
expect(error).toBeInstanceOf(Error);
331+
expect((error as Error).message).toContain(
332+
"Parallel MCP: text response exceeds 16777216 bytes",
333+
);
334+
expect(streamed.getReadCount()).toBeLessThan(32);
335+
expect(streamed.wasCanceled()).toBe(true);
336+
expect(textSpy).not.toHaveBeenCalled();
337+
});
314338
});

0 commit comments

Comments
 (0)