Skip to content

Commit 5537bc9

Browse files
truffle-devsteipete
truffle-dev
and
steipete
authored
fix(media): allow Bedrock SDK auth for image and PDF tools (#72092)
* fix(media): allow aws-sdk auth mode for image and audio/video paths Media understanding tools failed for amazon-bedrock deployments using auth.mode "aws-sdk" (BYOK via role/SSO/profile creds). Each path called requireApiKey, which throws on the empty-key sentinel before the AWS SDK credential chain can resolve creds at call time. The image path (image.ts) resolves auth via getApiKeyForModel; the audio/video paths route through resolveProviderExecutionAuth. Both now mirror the chat path's allowMissingApiKeyModes allowance: when the resolved key is empty and the mode is aws-sdk, execute keyless and let the SDK resolve credentials. The image path skips setRuntimeApiKey so no empty-string secret is persisted; audio/video return the kind:"none" execution auth so the runner bypasses key rotation. Closes #72031 * fix(media): cover Bedrock PDF SDK auth * fix(pdf): register plugin completion streams * fix(media): recognize Bedrock SDK tool auth * chore: move release note to PR --------- Co-authored-by: Peter Steinberger <[email protected]>
1 parent e779abf commit 5537bc9

8 files changed

Lines changed: 265 additions & 6 deletions

File tree

‎src/agents/tools/image-tool.custom-provider-auth.regression.test.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ import { hasProviderAuthForTool } from "./model-config.helpers.js";
1515
const USER_PROVIDER = "hatchery-qwen3.6-plus";
1616
const USER_MODEL = "qwen3.6-plus";
1717
const USER_PRIMARY = `${USER_PROVIDER}/${USER_MODEL}`;
18+
const BEDROCK_PROVIDER = "amazon-bedrock";
19+
const BEDROCK_VISION_MODEL = "vision-1";
1820
const CONFIG_API_KEY = "sk-user-configured-key"; // pragma: allowlist secret
1921
const USER_PROVIDER_AUTH_ENV_KEYS = [
2022
"HATCHERY_QWEN3_6_PLUS_API_KEY",
@@ -74,6 +76,23 @@ function createUserReportedConfig(params?: { includeApiKey?: boolean }): OpenCla
7476
};
7577
}
7678

79+
function createBedrockSdkConfig(): OpenClawConfig {
80+
return {
81+
agents: { defaults: { model: { primary: `${BEDROCK_PROVIDER}/text-1` } } },
82+
models: {
83+
mode: "replace",
84+
providers: {
85+
[BEDROCK_PROVIDER]: {
86+
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
87+
auth: "aws-sdk",
88+
api: "bedrock-converse-stream",
89+
models: [makeVisionModel(BEDROCK_VISION_MODEL)],
90+
},
91+
},
92+
},
93+
};
94+
}
95+
7796
async function withEmptyAgentDir<T>(run: (agentDir: string) => Promise<T>): Promise<T> {
7897
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-image-auth-regression-"));
7998
try {
@@ -148,6 +167,28 @@ describe("image custom provider auth regression", () => {
148167
});
149168
});
150169

170+
it("registers config-only AWS SDK Bedrock image models", async () => {
171+
await withEmptyAgentDir(async (agentDir) => {
172+
vi.stubEnv("AWS_PROFILE", "");
173+
vi.stubEnv("AWS_ACCESS_KEY_ID", "");
174+
vi.stubEnv("AWS_SECRET_ACCESS_KEY", "");
175+
vi.stubEnv("AWS_BEARER_TOKEN_BEDROCK", "");
176+
const cfg = createBedrockSdkConfig();
177+
178+
expect(hasProviderAuthForTool({ provider: BEDROCK_PROVIDER, cfg })).toBe(true);
179+
expect(resolveImageModelConfigForTool({ cfg, agentDir })).toEqual({
180+
primary: `${BEDROCK_PROVIDER}/${BEDROCK_VISION_MODEL}`,
181+
});
182+
expect(
183+
resolveImageToolFactoryAvailable({
184+
config: cfg,
185+
agentDir,
186+
modelHasVision: true,
187+
}),
188+
).toBe(true);
189+
});
190+
});
191+
151192
it("executes deferred image tool discovery with config-backed auth and runtime key resolution", async () => {
152193
// This covers the production deferred-discovery path: registration can
153194
// avoid auth work, but execution still resolves the config-backed key.

‎src/agents/tools/media-tool-shared.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -673,6 +673,15 @@ export async function resolveModelRuntimeApiKey(params: {
673673
cfg: params.cfg,
674674
agentDir: params.agentDir,
675675
});
676+
// Bedrock's runtime client owns AWS credential-chain resolution. Keep the
677+
// empty sentinel out of auth storage and pass it through to the stream.
678+
if (
679+
!apiKeyInfo.apiKey?.trim() &&
680+
apiKeyInfo.mode === "aws-sdk" &&
681+
params.model.api === "bedrock-converse-stream"
682+
) {
683+
return "";
684+
}
676685
const apiKey = requireApiKey(apiKeyInfo, params.model.provider);
677686
params.authStorage.setRuntimeApiKey(params.model.provider, apiKey);
678687
return apiKey;

‎src/agents/tools/model-config.helpers.test.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,23 @@ describe("hasProviderAuthForTool", () => {
145145
expect(hasProviderAuthForTool({ provider: "hatchery", cfg })).toBe(true);
146146
});
147147

148+
it("accepts AWS SDK auth without a static credential", () => {
149+
const cfg = {
150+
models: {
151+
providers: {
152+
"amazon-bedrock": {
153+
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
154+
auth: "aws-sdk",
155+
api: "bedrock-converse-stream",
156+
models: [],
157+
},
158+
},
159+
},
160+
} as OpenClawConfig;
161+
162+
expect(hasProviderAuthForTool({ provider: "amazon-bedrock", cfg })).toBe(true);
163+
});
164+
148165
it("keeps auth-store profiles as valid tool auth", () => {
149166
// Tool-specific model selection should honor the same stored profile shape
150167
// used by agent sessions, not only process env/config keys.

‎src/agents/tools/model-config.helpers.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ import type { AuthProfileCredential, AuthProfileStore } from "../auth-profiles/t
2626
import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "../defaults.js";
2727
import {
2828
hasRuntimeAvailableProviderAuth,
29-
hasUsableCustomProviderApiKey,
3029
resolveProviderEntryApiKeyProfileReference,
3130
resolveEnvApiKey,
3231
} from "../model-auth.js";
@@ -133,6 +132,16 @@ export function hasProviderAuthForTool(params: {
133132
agentDir?: string;
134133
authStore?: AuthProfileStore;
135134
}): boolean {
135+
if (
136+
hasRuntimeAvailableProviderAuth({
137+
provider: params.provider,
138+
cfg: params.cfg,
139+
workspaceDir: params.workspaceDir,
140+
allowPluginSyntheticAuth: false,
141+
})
142+
) {
143+
return true;
144+
}
136145
if (
137146
hasAuthForProvider({
138147
provider: params.provider,
@@ -144,7 +153,7 @@ export function hasProviderAuthForTool(params: {
144153
) {
145154
return true;
146155
}
147-
return hasUsableCustomProviderApiKey(params.cfg, params.provider);
156+
return false;
148157
}
149158

150159
function formatProviderModelRef(provider: string, model: string): string {

‎src/agents/tools/pdf-tool.test.ts‎

Lines changed: 107 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import * as pdfModelConfigModule from "./pdf-tool.model-config.js";
1717
import { resetPdfToolAuthEnv, withTempPdfAgentDir } from "./pdf-tool.test-support.js";
1818

1919
const completeMock = vi.hoisted(() => vi.fn());
20+
const registerProviderStreamForModelMock = vi.hoisted(() => vi.fn());
2021

2122
vi.mock("../../llm/stream.js", async () => {
2223
const actual = await vi.importActual<typeof import("../../llm/stream.js")>("../../llm/stream.js");
@@ -26,6 +27,10 @@ vi.mock("../../llm/stream.js", async () => {
2627
};
2728
});
2829

30+
vi.mock("../provider-stream.js", () => ({
31+
registerProviderStreamForModel: registerProviderStreamForModelMock,
32+
}));
33+
2934
type PdfToolModule = typeof import("./pdf-tool.js");
3035
let createPdfTool: PdfToolModule["createPdfTool"];
3136
let PdfToolSchema: PdfToolModule["PdfToolSchema"];
@@ -126,9 +131,8 @@ async function stubPdfToolInfra(
126131
loadSpy.mockResolvedValue(FAKE_PDF_MEDIA as never);
127132
}
128133

129-
vi.spyOn(modelDiscovery, "discoverAuthStorage").mockReturnValue({
130-
setRuntimeApiKey: vi.fn(),
131-
} as never);
134+
const setRuntimeApiKey = vi.fn();
135+
vi.spyOn(modelDiscovery, "discoverAuthStorage").mockReturnValue({ setRuntimeApiKey } as never);
132136
const find =
133137
params?.modelFound === false
134138
? () => null
@@ -155,7 +159,7 @@ async function stubPdfToolInfra(
155159
vi.spyOn(modelAuth, "getApiKeyForModel").mockResolvedValue({ apiKey: "test-key" } as never);
156160
vi.spyOn(modelAuth, "requireApiKey").mockReturnValue("test-key");
157161

158-
return { loadSpy };
162+
return { loadSpy, setRuntimeApiKey };
159163
}
160164

161165
async function withManagedInboundPdf(
@@ -184,6 +188,7 @@ describe("createPdfTool", () => {
184188
beforeEach(() => {
185189
resetPdfToolAuthEnv();
186190
completeMock.mockReset();
191+
registerProviderStreamForModelMock.mockReset();
187192
});
188193

189194
afterEach(() => {
@@ -209,6 +214,51 @@ describe("createPdfTool", () => {
209214
});
210215
});
211216

217+
it("auto-selects Bedrock PDF models with AWS SDK auth", async () => {
218+
await withTempPdfAgentDir(async (agentDir) => {
219+
vi.stubEnv("AWS_PROFILE", "");
220+
vi.stubEnv("AWS_ACCESS_KEY_ID", "");
221+
vi.stubEnv("AWS_SECRET_ACCESS_KEY", "");
222+
vi.stubEnv("AWS_BEARER_TOKEN_BEDROCK", "");
223+
const cfg: OpenClawConfig = {
224+
agents: { defaults: { model: { primary: "amazon-bedrock/text-1" } } },
225+
models: {
226+
mode: "replace",
227+
providers: {
228+
"amazon-bedrock": {
229+
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
230+
auth: "aws-sdk",
231+
api: "bedrock-converse-stream",
232+
models: [
233+
{
234+
id: "text-1",
235+
name: "Bedrock Text",
236+
input: ["text"],
237+
contextWindow: 16_000,
238+
maxTokens: 4_096,
239+
reasoning: false,
240+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
241+
},
242+
{
243+
id: "vision-1",
244+
name: "Bedrock Vision",
245+
input: ["text", "image"],
246+
contextWindow: 16_000,
247+
maxTokens: 4_096,
248+
reasoning: false,
249+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
250+
},
251+
],
252+
},
253+
},
254+
},
255+
};
256+
257+
const tool = (await loadCreatePdfTool())({ config: cfg, agentDir });
258+
expect(typeof tool?.execute).toBe("function");
259+
});
260+
});
261+
212262
it("defers automatic model config resolution during registration (#76644)", async () => {
213263
const resolveSpy = vi.spyOn(pdfModelConfigModule, "resolvePdfModelConfigForTool");
214264
const cfg = withDefaultModel("openai/gpt-5.4");
@@ -652,6 +702,59 @@ describe("createPdfTool", () => {
652702
});
653703
});
654704

705+
it("uses the AWS SDK credential chain for Bedrock PDF models", async () => {
706+
await withTempPdfAgentDir(async (agentDir) => {
707+
const { setRuntimeApiKey } = await stubPdfToolInfra(agentDir, {
708+
provider: "amazon-bedrock",
709+
api: "bedrock-converse-stream",
710+
input: ["text", "image"],
711+
});
712+
vi.mocked(modelAuth.getApiKeyForModel).mockResolvedValue({
713+
apiKey: "",
714+
source: "aws-sdk default chain",
715+
mode: "aws-sdk",
716+
});
717+
vi.mocked(modelAuth.requireApiKey).mockImplementation(() => {
718+
throw new Error("Bedrock aws-sdk auth must not require a literal API key");
719+
});
720+
vi.spyOn(pdfExtractModule, "extractPdfContent").mockResolvedValue({
721+
text: "Extracted content",
722+
images: [],
723+
});
724+
completeMock.mockResolvedValue({
725+
role: "assistant",
726+
stopReason: "stop",
727+
content: [{ type: "text", text: "Bedrock summary" }],
728+
} as never);
729+
730+
const bedrockModel = "amazon-bedrock/us.anthropic.claude-sonnet-4-6";
731+
const tool = requirePdfTool(
732+
(await loadCreatePdfTool())({ config: withPdfModel(bedrockModel), agentDir }),
733+
);
734+
const result = await tool.execute("t1", {
735+
prompt: "summarize",
736+
pdf: "/tmp/doc.pdf",
737+
});
738+
739+
expect(result.content).toEqual([{ type: "text", text: "Bedrock summary" }]);
740+
expect(modelAuth.requireApiKey).not.toHaveBeenCalled();
741+
expect(setRuntimeApiKey).not.toHaveBeenCalled();
742+
expect(registerProviderStreamForModelMock).toHaveBeenCalledWith({
743+
model: expect.objectContaining({
744+
provider: "amazon-bedrock",
745+
api: "bedrock-converse-stream",
746+
}),
747+
cfg: expect.objectContaining({
748+
agents: expect.objectContaining({
749+
defaults: expect.objectContaining({ pdfModel: { primary: bedrockModel } }),
750+
}),
751+
}),
752+
agentDir,
753+
});
754+
expect(firstMockCall(completeMock, "complete")[2]).toMatchObject({ apiKey: "" });
755+
});
756+
});
757+
655758
it("passes password to PDF extraction fallback", async () => {
656759
await withTempPdfAgentDir(async (agentDir) => {
657760
await stubPdfToolInfra(agentDir, { provider: "openai", input: ["text"] });

‎src/agents/tools/pdf-tool.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import { loadWebMediaRaw } from "../../media/web-media.js";
2020
import { resolveUserPath } from "../../utils.js";
2121
import type { AuthProfileStore } from "../auth-profiles/types.js";
2222
import { getModelProviderRequestTransport } from "../provider-request-config.js";
23+
import { registerProviderStreamForModel } from "../provider-stream.js";
2324
import { optionalFiniteNumberSchema } from "../schema/typebox.js";
2425
import { readFiniteNumberParam, ToolInputError } from "./common.js";
2526
import { coerceImageModelConfig, type ImageModelConfig } from "./image-tool.helpers.js";
@@ -233,6 +234,15 @@ async function runPdfPrompt(params: {
233234
}
234235
}
235236

237+
// PDF-only model selections may not have loaded their provider plugin yet.
238+
// Register before complete() so plugin-owned APIs resolve on first use.
239+
registerProviderStreamForModel({
240+
model,
241+
cfg: effectiveCfg,
242+
agentDir: params.agentDir,
243+
...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}),
244+
});
245+
236246
const extractions = await getExtractions();
237247
const hasImages = extractions.some((e) => e.images.length > 0);
238248
if (hasImages && !model.input?.includes("image")) {

‎src/media-understanding/image.test.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,67 @@ describe("describeImageWithModel", () => {
301301
expect(fetchMock).not.toHaveBeenCalled();
302302
});
303303

304+
it("describes images keyless when amazon-bedrock resolves aws-sdk auth", async () => {
305+
getApiKeyForModelMock.mockResolvedValueOnce({
306+
apiKey: "",
307+
source: "profile:amazon-bedrock:default",
308+
mode: "aws-sdk",
309+
});
310+
// Faithful to runtime: requireApiKey throws on an empty resolved key. The
311+
// aws-sdk carve-out must return before reaching it.
312+
requireApiKeyMock.mockImplementation((auth: { apiKey?: string; mode?: string }) => {
313+
const key = auth.apiKey?.trim();
314+
if (!key) {
315+
throw new Error(
316+
`No API key resolved for provider "amazon-bedrock" (auth mode: ${auth.mode}).`,
317+
);
318+
}
319+
return key;
320+
});
321+
discoverModelsMock.mockReturnValue({
322+
find: vi.fn(() => ({
323+
provider: "amazon-bedrock",
324+
id: "us.anthropic.claude-sonnet-4-6-v1",
325+
input: ["text", "image"],
326+
api: "bedrock-converse-stream",
327+
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
328+
})),
329+
});
330+
completeMock.mockResolvedValue({
331+
role: "assistant",
332+
api: "bedrock-converse-stream",
333+
provider: "amazon-bedrock",
334+
model: "us.anthropic.claude-sonnet-4-6-v1",
335+
stopReason: "stop",
336+
timestamp: Date.now(),
337+
content: [{ type: "text", text: "an orange tabby cat" }],
338+
});
339+
340+
const result = await describeImageWithModel({
341+
cfg: {},
342+
agentDir: "/tmp/openclaw-agent",
343+
provider: "amazon-bedrock",
344+
model: "us.anthropic.claude-sonnet-4-6-v1",
345+
buffer: Buffer.from("png-bytes"),
346+
fileName: "image.png",
347+
mime: "image/png",
348+
prompt: "Describe the image.",
349+
timeoutMs: 1000,
350+
});
351+
352+
expect(result).toEqual({
353+
text: "an orange tabby cat",
354+
model: "us.anthropic.claude-sonnet-4-6-v1",
355+
});
356+
// The carve-out returns before requireApiKey and skips persisting an
357+
// empty-string secret; the empty key flows through to the model runtime.
358+
expect(requireApiKeyMock).not.toHaveBeenCalled();
359+
expect(setRuntimeApiKeyMock).not.toHaveBeenCalled();
360+
const completeCall = requireFirstMockCall(completeMock, "complete");
361+
expect(requireRecord(completeCall[2], "stream options").apiKey).toBe("");
362+
expect(fetchMock).not.toHaveBeenCalled();
363+
});
364+
304365
it("passes workspaceDir through MiniMax VLM fallback auth", async () => {
305366
const authStorage = {
306367
setRuntimeApiKey: setRuntimeApiKeyMock,

0 commit comments

Comments
 (0)