Skip to content

Commit 0097363

Browse files
steipetesteipete
authored
fix(secrets): keep gateway alive when web provider refs fail (#109687)
* fix(secrets): isolate unavailable web providers * fix(secrets): keep web result type internal * fix(secrets): hide unavailable web selections
1 parent 9ac2f77 commit 0097363

16 files changed

Lines changed: 312 additions & 102 deletions

‎src/agents/tools/web-fetch.provider-fallback.test.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import { rm } from "node:fs/promises";
44
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
55
import type { OpenClawConfig } from "../../config/config.js";
6+
import { setActiveDegradedSecretOwners } from "../../secrets/runtime-degraded-state.js";
67
import { wrapExternalContent } from "../../security/external-content.js";
78
import { withFetchPreconnect } from "../../test-utils/fetch-mock.js";
89
import { createWebFetchTool } from "./web-fetch.js";
@@ -32,13 +33,43 @@ describe("web_fetch provider fallback normalization", () => {
3233
resolveWebFetchDefinitionMock.mockReset();
3334
runtimeState.activeSecretsRuntimeSnapshot = null;
3435
runtimeState.activeRuntimeWebToolsMetadata = null;
36+
setActiveDegradedSecretOwners([]);
3537
});
3638

3739
afterEach(() => {
3840
global.fetch = priorFetch;
3941
vi.restoreAllMocks();
4042
runtimeState.activeSecretsRuntimeSnapshot = null;
4143
runtimeState.activeRuntimeWebToolsMetadata = null;
44+
setActiveDegradedSecretOwners([]);
45+
});
46+
47+
it("returns typed unavailability for only the isolated fetch provider", async () => {
48+
setActiveDegradedSecretOwners([
49+
{
50+
ownerKind: "capability",
51+
ownerId: "web-fetch:firecrawl",
52+
state: "unavailable",
53+
paths: ["plugins.entries.firecrawl.config.webFetch.apiKey"],
54+
refKeys: ["env:default:MISSING_FIRECRAWL_KEY"],
55+
reason: "missing test ref",
56+
},
57+
]);
58+
const tool = createWebFetchTool({
59+
config: {
60+
tools: { web: { fetch: { provider: "firecrawl" } } },
61+
} as OpenClawConfig,
62+
});
63+
64+
await expect(
65+
tool?.execute?.("call-provider-fallback", { url: "https://example.com" }),
66+
).rejects.toMatchObject({
67+
name: "SecretSurfaceUnavailableError",
68+
code: "SECRET_SURFACE_UNAVAILABLE",
69+
ownerKind: "capability",
70+
ownerId: "web-fetch:firecrawl",
71+
});
72+
expect(resolveWebFetchDefinitionMock).not.toHaveBeenCalled();
4273
});
4374

4475
it("re-wraps and truncates provider fallback payloads before caching or returning", async () => {

‎src/agents/tools/web-fetch.ts‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ import { resolveWebProviderConfig } from "../../../packages/web-content-core/src
1515
import type { OpenClawConfig } from "../../config/types.openclaw.js";
1616
import { SsrFBlockedError, type LookupFn, type SsrFPolicy } from "../../infra/net/ssrf.js";
1717
import { logDebug } from "../../logger.js";
18+
import { assertSecretOwnerAvailable } from "../../secrets/runtime-degraded-state.js";
19+
import { runtimeWebSecretOwnerId } from "../../secrets/runtime-web-secret-owner.js";
1820
import type { RuntimeWebFetchMetadata } from "../../secrets/runtime-web-tools.types.js";
1921
import { wrapExternalContent, wrapWebContent } from "../../security/external-content.js";
2022
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
@@ -799,17 +801,22 @@ export function createWebFetchTool(options?: {
799801
description: "Fetch URL; extract readable markdown/text. Lightweight; no browser automation.",
800802
parameters: WebFetchSchema,
801803
execute: async (_toolCallId, args, signal, onUpdate) => {
802-
const { config, preferRuntimeProviders, runtimeWebFetch } = resolveWebFetchToolRuntimeContext(
803-
{
804+
const { config, preferRuntimeProviders, providerSelectionId, runtimeWebFetch } =
805+
resolveWebFetchToolRuntimeContext({
804806
config: options?.config,
805807
lateBindRuntimeConfig: options?.lateBindRuntimeConfig,
806808
runtimeWebFetch: options?.runtimeWebFetch,
807-
},
808-
);
809+
});
809810
const executionFetch = resolveFetchConfig(config);
810811
if (!resolveFetchEnabled({ fetch: executionFetch, sandboxed: options?.sandboxed })) {
811812
throw new Error("web_fetch is disabled.");
812813
}
814+
if (providerSelectionId) {
815+
assertSecretOwnerAvailable(
816+
"capability",
817+
runtimeWebSecretOwnerId("fetch", providerSelectionId),
818+
);
819+
}
813820
const providerCacheKey =
814821
normalizeOptionalLowercaseString(runtimeWebFetch?.selectedProvider) ??
815822
normalizeOptionalLowercaseString(runtimeWebFetch?.providerConfigured) ??

‎src/agents/tools/web-search.late-bind.test.ts‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// web_search late-binding tests cover runtime config and provider metadata
22
// selection at execution time.
3-
import { beforeEach, describe, expect, it, vi } from "vitest";
3+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
4+
import { setActiveDegradedSecretOwners } from "../../secrets/runtime-degraded-state.js";
45
import { createWebSearchTool } from "./web-search.js";
56

67
const mocks = vi.hoisted(() => ({
@@ -54,6 +55,10 @@ describe("web_search late-bound runtime fallback", () => {
5455
mocks.getActiveSecretsRuntimeConfigSnapshot.mockReturnValue(null);
5556
});
5657

58+
afterEach(() => {
59+
setActiveDegradedSecretOwners([]);
60+
});
61+
5762
it("falls back to options.runtimeWebSearch when active runtime web tools metadata is absent", async () => {
5863
const tool = createWebSearchTool({
5964
config: {},
@@ -171,4 +176,30 @@ describe("web_search late-bound runtime fallback", () => {
171176
);
172177
expect(mocks.runWebSearch).not.toHaveBeenCalled();
173178
});
179+
180+
it("returns typed unavailability for only the isolated search provider", async () => {
181+
setActiveDegradedSecretOwners([
182+
{
183+
ownerKind: "capability",
184+
ownerId: "web-search:brave",
185+
state: "unavailable",
186+
paths: ["tools.web.search.brave.apiKey"],
187+
refKeys: ["env:default:MISSING_BRAVE_KEY"],
188+
reason: "missing test ref",
189+
},
190+
]);
191+
const tool = createWebSearchTool({
192+
config: { tools: { web: { search: { provider: "brave" } } } },
193+
});
194+
195+
await expect(
196+
tool?.execute("call-search", { query: "openclaw" }, undefined),
197+
).rejects.toMatchObject({
198+
name: "SecretSurfaceUnavailableError",
199+
code: "SECRET_SURFACE_UNAVAILABLE",
200+
ownerKind: "capability",
201+
ownerId: "web-search:brave",
202+
});
203+
expect(mocks.runWebSearch).not.toHaveBeenCalled();
204+
});
174205
});

‎src/agents/tools/web-search.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
* Runs the configured runtime provider and returns normalized cached search results.
55
*/
66
import type { OpenClawConfig } from "../../config/types.openclaw.js";
7+
import { assertSecretOwnerAvailable } from "../../secrets/runtime-degraded-state.js";
8+
import { runtimeWebSecretOwnerId } from "../../secrets/runtime-web-secret-owner.js";
79
import type { RuntimeWebSearchMetadata } from "../../secrets/runtime-web-tools.types.js";
810
import { runWebSearch } from "../../web-search/runtime.js";
911
import type { AnyAgentTool } from "./common.js";
@@ -94,7 +96,7 @@ export function createWebSearchTool(options?: {
9496
execute: async (_toolCallId, args, signal) => {
9597
// Late binding lets long-lived agents pick up runtime web-search credentials/config without
9698
// rebuilding the tool object.
97-
const { config, preferRuntimeProviders, runtimeWebSearch } =
99+
const { config, preferRuntimeProviders, providerSelectionId, runtimeWebSearch } =
98100
resolveWebSearchToolRuntimeContext({
99101
config: options?.config,
100102
lateBindRuntimeConfig: options?.lateBindRuntimeConfig,
@@ -103,6 +105,12 @@ export function createWebSearchTool(options?: {
103105
if (isWebSearchDisabled(config)) {
104106
throw new Error("web_search is disabled.");
105107
}
108+
if (providerSelectionId) {
109+
assertSecretOwnerAvailable(
110+
"capability",
111+
runtimeWebSecretOwnerId("search", providerSelectionId),
112+
);
113+
}
106114
const result = await runWebSearch({
107115
config,
108116
agentDir: options?.agentDir,

‎src/agents/tools/web-tool-runtime-context.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ type WebProviderRuntimeMetadata = RuntimeWebFetchMetadata | RuntimeWebSearchMeta
1919
type ResolvedWebToolRuntimeContext<TMetadata extends WebProviderRuntimeMetadata> = {
2020
config?: OpenClawConfig;
2121
preferRuntimeProviders: boolean;
22+
providerSelectionId: string;
2223
runtimeMetadata?: TMetadata;
2324
};
2425

@@ -79,6 +80,7 @@ function resolveWebToolRuntimeContext<TMetadata extends WebProviderRuntimeMetada
7980
kind: params.kind,
8081
providerSelectionId,
8182
}),
83+
providerSelectionId,
8284
runtimeMetadata,
8385
};
8486
}
@@ -98,6 +100,7 @@ export function resolveWebSearchToolRuntimeContext(params: {
98100
return {
99101
config: resolved.config,
100102
preferRuntimeProviders: resolved.preferRuntimeProviders,
103+
providerSelectionId: resolved.providerSelectionId,
101104
runtimeWebSearch: resolved.runtimeMetadata,
102105
};
103106
}
@@ -117,6 +120,7 @@ export function resolveWebFetchToolRuntimeContext(params: {
117120
return {
118121
config: resolved.config,
119122
preferRuntimeProviders: resolved.preferRuntimeProviders,
123+
providerSelectionId: resolved.providerSelectionId,
120124
runtimeWebFetch: resolved.runtimeMetadata,
121125
};
122126
}

‎src/cli/command-secret-gateway.test.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,14 @@ vi.mock("../gateway/call.js", () => ({
2626
}));
2727

2828
vi.mock("../secrets/runtime-web-tools.js", () => ({
29-
resolveRuntimeWebTools: vi.fn(async () => ({})),
29+
resolveRuntimeWebTools: vi.fn(async () => ({
30+
metadata: {
31+
search: { providerSource: "none", diagnostics: [] },
32+
fetch: { providerSource: "none", diagnostics: [] },
33+
diagnostics: [],
34+
},
35+
degradedOwners: [],
36+
})),
3037
}));
3138

3239
vi.mock("../utils/message-channel.js", () => ({

‎src/gateway/server-startup-config.secrets.test.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -806,9 +806,9 @@ describe("gateway startup config secret preflight", () => {
806806

807807
it("does not emit degraded or recovered events for warning-only secret reloads", async () => {
808808
const warning: SecretResolverWarning = {
809-
code: "WEB_SEARCH_KEY_UNRESOLVED_FALLBACK_USED",
810-
path: "plugins.entries.google.config.webSearch.apiKey",
811-
message: "web search provider fell back to environment credentials",
809+
code: "WEB_SEARCH_AUTODETECT_SELECTED",
810+
path: "tools.web.search.provider",
811+
message: "web search provider was auto-detected",
812812
};
813813
const prepareRuntimeSecretsSnapshot = vi.fn(async ({ config }) => ({
814814
...preparedSnapshot(config),
@@ -844,7 +844,7 @@ describe("gateway startup config secret preflight", () => {
844844
expect(result.config).toBe(config);
845845
expect(result.warnings).toEqual([warning]);
846846
expect(logSecrets.warn).toHaveBeenCalledWith(
847-
"[WEB_SEARCH_KEY_UNRESOLVED_FALLBACK_USED] web search provider fell back to environment credentials",
847+
"[WEB_SEARCH_AUTODETECT_SELECTED] web search provider was auto-detected",
848848
);
849849
expect(emitStateEvent).not.toHaveBeenCalled();
850850
const preflightInput = callArg<{ config?: unknown }>(prepareRuntimeSecretsSnapshot);

‎src/gateway/server-startup-inactive-web-secret.test.ts‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
/** Gateway startup coverage for active and inactive web-provider SecretRefs. */
22
import { afterEach, describe, expect, it, vi } from "vitest";
33
import type { OpenClawConfig } from "../config/config.js";
4+
import { setActiveDegradedSecretOwners } from "../secrets/runtime-degraded-state.js";
45
import { getActiveSecretsRuntimeSnapshot } from "../secrets/runtime.js";
56
import { withEnvAsync } from "../test-utils/env.js";
67
import { getFreePort, installGatewayTestHooks, startGatewayServer } from "./test-helpers.js";
@@ -130,6 +131,7 @@ describe("gateway startup web-provider SecretRefs", () => {
130131
afterEach(async () => {
131132
await server?.close();
132133
server = undefined;
134+
setActiveDegradedSecretOwners([]);
133135
});
134136

135137
it("starts and warns when an unresolved web secret is provably inactive", async () => {
@@ -147,13 +149,27 @@ describe("gateway startup web-provider SecretRefs", () => {
147149
});
148150
});
149151

150-
it("fails closed when the unresolved web secret is active", async () => {
151-
await withEnvAsync({ [ACTIVE_SECRET_ENV]: undefined }, async () => {
152-
await writeConfig(buildConfig({ enabled: true, envVar: ACTIVE_SECRET_ENV }));
153-
154-
await expect(
155-
startGatewayServer(await getFreePort(), { auth: { mode: "none" } }),
156-
).rejects.toThrow(/Startup failed: required secrets are unavailable/);
157-
});
152+
it("starts with only the explicit active web provider unavailable", async () => {
153+
await withEnvAsync(
154+
{
155+
[ACTIVE_SECRET_ENV]: undefined,
156+
GEMINI_API_KEY: "test-gemini-api-key",
157+
},
158+
async () => {
159+
await writeConfig(buildConfig({ enabled: true, envVar: ACTIVE_SECRET_ENV }));
160+
161+
server = await startGatewayServer(await getFreePort(), { auth: { mode: "none" } });
162+
163+
const snapshot = getActiveSecretsRuntimeSnapshot();
164+
expect(snapshot?.degradedOwners).toContainEqual(
165+
expect.objectContaining({
166+
ownerKind: "capability",
167+
ownerId: "web-search:gemini",
168+
state: "unavailable",
169+
paths: [SECRET_PATH],
170+
}),
171+
);
172+
},
173+
);
158174
});
159175
});

‎src/secrets/runtime-owner-assignments.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,11 @@ function createDegradedOwner(assignments: SecretAssignment[], reason: string): D
6868
};
6969
}
7070

71-
function warnDegradedOwner(context: ResolverContext, owner: DegradedSecretOwner): void {
71+
/** Emits the canonical warning for one isolated runtime secret owner. */
72+
export function warnDegradedSecretOwner(
73+
context: ResolverContext,
74+
owner: DegradedSecretOwner,
75+
): void {
7276
pushWarning(context, {
7377
code: "SECRETS_OWNER_UNAVAILABLE",
7478
path: owner.paths[0]!,
@@ -159,7 +163,7 @@ export async function resolveAndApplySecretAssignments(params: {
159163
// would silently route this owner through env/profile fallback after its declared ref failed.
160164
const degradedOwner = createDegradedOwner(assignments, failureReason);
161165
degradedOwners.push(degradedOwner);
162-
warnDegradedOwner(params.context, degradedOwner);
166+
warnDegradedSecretOwner(params.context, degradedOwner);
163167
continue;
164168
}
165169
if (

‎src/secrets/runtime-web-secret-owner.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
/** Stable degraded-owner id for one configured web provider surface. */
2+
export function runtimeWebSecretOwnerId(kind: "search" | "fetch", providerId: string): string {
3+
return `web-${kind}:${providerId}`;
4+
}

0 commit comments

Comments
 (0)