Skip to content

Commit f4d073e

Browse files
truffle-devtruffle-dev
committed
fix(media): allow aws-sdk auth mode for image and audio/video paths
Media understanding tools failed for amazon-bedrock deployments using auth.mode "aws-sdk" (BYOK via role/SSO/profile creds). Each path called requireApiKey, which throws on the empty-key sentinel before the AWS SDK credential chain can resolve creds at call time. The image path (image.ts) resolves auth via getApiKeyForModel; the audio/video paths route through resolveProviderExecutionAuth. Both now mirror the chat path's allowMissingApiKeyModes allowance: when the resolved key is empty and the mode is aws-sdk, execute keyless and let the SDK resolve credentials. The image path skips setRuntimeApiKey so no empty-string secret is persisted; audio/video return the kind:"none" execution auth so the runner bypasses key rotation. Closes #72031
1 parent 12c34fc commit f4d073e

4 files changed

Lines changed: 251 additions & 4 deletions

File tree

‎src/media-understanding/image.test.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,67 @@ describe("describeImageWithModel", () => {
300300
expect(fetchMock).not.toHaveBeenCalled();
301301
});
302302

303+
it("describes images keyless when amazon-bedrock resolves aws-sdk auth", async () => {
304+
getApiKeyForModelMock.mockResolvedValueOnce({
305+
apiKey: "",
306+
source: "profile:amazon-bedrock:default",
307+
mode: "aws-sdk",
308+
});
309+
// Faithful to runtime: requireApiKey throws on an empty resolved key. The
310+
// aws-sdk carve-out must return before reaching it.
311+
requireApiKeyMock.mockImplementation((auth: { apiKey?: string; mode?: string }) => {
312+
const key = auth.apiKey?.trim();
313+
if (!key) {
314+
throw new Error(
315+
`No API key resolved for provider "amazon-bedrock" (auth mode: ${auth.mode}).`,
316+
);
317+
}
318+
return key;
319+
});
320+
discoverModelsMock.mockReturnValue({
321+
find: vi.fn(() => ({
322+
provider: "amazon-bedrock",
323+
id: "us.anthropic.claude-sonnet-4-6-v1",
324+
input: ["text", "image"],
325+
api: "bedrock-converse-stream",
326+
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
327+
})),
328+
});
329+
completeMock.mockResolvedValue({
330+
role: "assistant",
331+
api: "bedrock-converse-stream",
332+
provider: "amazon-bedrock",
333+
model: "us.anthropic.claude-sonnet-4-6-v1",
334+
stopReason: "stop",
335+
timestamp: Date.now(),
336+
content: [{ type: "text", text: "an orange tabby cat" }],
337+
});
338+
339+
const result = await describeImageWithModel({
340+
cfg: {},
341+
agentDir: "/tmp/openclaw-agent",
342+
provider: "amazon-bedrock",
343+
model: "us.anthropic.claude-sonnet-4-6-v1",
344+
buffer: Buffer.from("png-bytes"),
345+
fileName: "image.png",
346+
mime: "image/png",
347+
prompt: "Describe the image.",
348+
timeoutMs: 1000,
349+
});
350+
351+
expect(result).toEqual({
352+
text: "an orange tabby cat",
353+
model: "us.anthropic.claude-sonnet-4-6-v1",
354+
});
355+
// The carve-out returns before requireApiKey and skips persisting an
356+
// empty-string secret; the empty key flows through to the model runtime.
357+
expect(requireApiKeyMock).not.toHaveBeenCalled();
358+
expect(setRuntimeApiKeyMock).not.toHaveBeenCalled();
359+
const completeCall = requireFirstMockCall(completeMock, "complete");
360+
expect(requireRecord(completeCall[2], "stream options").apiKey).toBe("");
361+
expect(fetchMock).not.toHaveBeenCalled();
362+
});
363+
303364
it("passes workspaceDir through MiniMax VLM fallback auth", async () => {
304365
const authStorage = {
305366
setRuntimeApiKey: setRuntimeApiKeyMock,

‎src/media-understanding/image.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,14 @@ async function prepareResolvedImageRuntime(
237237
preferredProfile: params.preferredProfile,
238238
store: params.authStore,
239239
});
240+
// Providers configured for aws-sdk auth (e.g. amazon-bedrock) resolve
241+
// credentials through the AWS SDK chain at call time rather than a static
242+
// key, so an empty resolved key is expected, not an error. Mirror the chat
243+
// path's allowMissingApiKeyModes allowance: pass the empty key through and
244+
// skip setRuntimeApiKey so we never persist an empty-string secret.
245+
if (!apiKeyInfo.apiKey?.trim() && apiKeyInfo.mode === "aws-sdk") {
246+
return { apiKey: "", model };
247+
}
240248
let apiKey = requireApiKey(apiKeyInfo, model.provider);
241249
// Image tool bypasses prepareRuntimeAuth — exchange OAuth token for
242250
// a short-lived Copilot API token so the integrator scope (vscode-chat)
@@ -423,8 +431,12 @@ async function resolveMinimaxVlmFallbackRuntime(params: {
423431
agentDir: params.agentDir,
424432
...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}),
425433
});
434+
// aws-sdk providers resolve credentials via the SDK chain at call time, so an
435+
// empty resolved key is expected; pass it through rather than throwing.
436+
const apiKey =
437+
!auth.apiKey?.trim() && auth.mode === "aws-sdk" ? "" : requireApiKey(auth, authProvider);
426438
return {
427-
apiKey: requireApiKey(auth, authProvider),
439+
apiKey,
428440
modelBaseUrl: resolveConfiguredProviderBaseUrl(params.cfg, params.provider),
429441
};
430442
}

‎src/media-understanding/runner.aws-sdk.test.ts‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
// aws-sdk auth-mode runner tests cover the audio and video describe paths for
2+
// providers (e.g. amazon-bedrock) whose credentials resolve through the AWS SDK
3+
// chain at call time rather than a static API key. The resolved auth carries no
4+
// key, so the runner must execute keyless instead of throwing missing-api-key.
5+
import fs from "node:fs/promises";
6+
import os from "node:os";
7+
import path from "node:path";
8+
import { describe, expect, it, vi } from "vitest";
9+
import { CUSTOM_LOCAL_AUTH_MARKER } from "../agents/model-auth-markers.js";
10+
import type { OpenClawConfig } from "../config/types.js";
11+
import { withEnvAsync } from "../test-utils/env.js";
12+
import { buildProviderRegistry, runCapability } from "./runner.js";
13+
import { withAudioFixture, withVideoFixture } from "./runner.test-utils.js";
14+
import type {
15+
AudioTranscriptionRequest,
16+
MediaUnderstandingProvider,
17+
VideoDescriptionRequest,
18+
} from "./types.js";
19+
20+
vi.mock("../plugins/capability-provider-runtime.js", async () => {
21+
const { createEmptyCapabilityProviderMockModule } = await import("./runner.test-mocks.js");
22+
return createEmptyCapabilityProviderMockModule();
23+
});
24+
25+
vi.mock("../plugins/providers.js", async (importOriginal) => ({
26+
...(await importOriginal()),
27+
resolveOwningPluginIdsForProvider: () => [],
28+
}));
29+
30+
// Clear AWS credentials so the resolved auth stays keyless and the test does not
31+
// depend on the host's SDK credential chain.
32+
const AUTH_ENV = {
33+
AWS_BEARER_TOKEN_BEDROCK: undefined,
34+
AWS_ACCESS_KEY_ID: undefined,
35+
AWS_SECRET_ACCESS_KEY: undefined,
36+
AWS_PROFILE: undefined,
37+
OPENCLAW_AGENT_DIR: undefined,
38+
} satisfies Record<string, string | undefined>;
39+
40+
const BEDROCK_PROVIDER = "amazon-bedrock";
41+
const BEDROCK_MODEL = "us.anthropic.claude-sonnet-4-6-v1";
42+
43+
function awsSdkProviderConfig(): Record<string, unknown> {
44+
return {
45+
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
46+
api: "bedrock-converse-stream",
47+
models: [],
48+
auth: "aws-sdk",
49+
};
50+
}
51+
52+
function createAudioProvider(
53+
id: string,
54+
transcribeAudio: (req: AudioTranscriptionRequest) => Promise<{ text: string; model?: string }>,
55+
): MediaUnderstandingProvider {
56+
return { id, capabilities: ["audio"], transcribeAudio };
57+
}
58+
59+
function createVideoProvider(
60+
id: string,
61+
describeVideo: (req: VideoDescriptionRequest) => Promise<{ text: string; model?: string }>,
62+
): MediaUnderstandingProvider {
63+
return { id, capabilities: ["video"], describeVideo };
64+
}
65+
66+
async function withIsolatedAgentDir<T>(run: (agentDir: string) => Promise<T>): Promise<T> {
67+
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-aws-sdk-media-"));
68+
try {
69+
return await run(agentDir);
70+
} finally {
71+
await fs.rm(agentDir, { recursive: true, force: true });
72+
}
73+
}
74+
75+
function createAudioCfg(): OpenClawConfig {
76+
return {
77+
models: { providers: { [BEDROCK_PROVIDER]: awsSdkProviderConfig() } },
78+
tools: {
79+
media: {
80+
audio: {
81+
enabled: true,
82+
models: [{ type: "provider", provider: BEDROCK_PROVIDER, model: BEDROCK_MODEL }],
83+
},
84+
},
85+
},
86+
} as unknown as OpenClawConfig;
87+
}
88+
89+
function createVideoCfg(): OpenClawConfig {
90+
return {
91+
models: { providers: { [BEDROCK_PROVIDER]: awsSdkProviderConfig() } },
92+
tools: {
93+
media: {
94+
video: {
95+
enabled: true,
96+
models: [{ type: "provider", provider: BEDROCK_PROVIDER, model: BEDROCK_MODEL }],
97+
},
98+
},
99+
},
100+
} as unknown as OpenClawConfig;
101+
}
102+
103+
describe("runCapability aws-sdk auth mode", () => {
104+
it("transcribes audio keyless when amazon-bedrock resolves aws-sdk auth", async () => {
105+
await withIsolatedAgentDir(async (agentDir) => {
106+
await withEnvAsync(AUTH_ENV, async () => {
107+
await withAudioFixture("openclaw-aws-sdk-audio", async ({ ctx, media, cache }) => {
108+
const transcribeAudio = vi.fn(async (req: AudioTranscriptionRequest) => ({
109+
text: `ok:${req.apiKey}`,
110+
model: req.model,
111+
}));
112+
113+
const result = await runCapability({
114+
capability: "audio",
115+
cfg: createAudioCfg(),
116+
ctx,
117+
attachments: cache,
118+
media,
119+
agentDir,
120+
providerRegistry: buildProviderRegistry({
121+
[BEDROCK_PROVIDER]: createAudioProvider(BEDROCK_PROVIDER, transcribeAudio),
122+
}),
123+
});
124+
125+
expect(result.decision.outcome).toBe("success");
126+
expect(result.outputs[0]?.text).toBe(`ok:${CUSTOM_LOCAL_AUTH_MARKER}`);
127+
expect(transcribeAudio).toHaveBeenCalledTimes(1);
128+
expect(transcribeAudio.mock.calls[0]?.[0].apiKey).toBe(CUSTOM_LOCAL_AUTH_MARKER);
129+
});
130+
});
131+
});
132+
});
133+
134+
it("describes video keyless when amazon-bedrock resolves aws-sdk auth", async () => {
135+
await withIsolatedAgentDir(async (agentDir) => {
136+
await withEnvAsync(AUTH_ENV, async () => {
137+
await withVideoFixture("openclaw-aws-sdk-video", async ({ ctx, media, cache }) => {
138+
const describeVideo = vi.fn(async (req: VideoDescriptionRequest) => ({
139+
text: `ok:${req.apiKey}`,
140+
model: req.model,
141+
}));
142+
143+
const result = await runCapability({
144+
capability: "video",
145+
cfg: createVideoCfg(),
146+
ctx,
147+
attachments: cache,
148+
media,
149+
agentDir,
150+
providerRegistry: buildProviderRegistry({
151+
[BEDROCK_PROVIDER]: createVideoProvider(BEDROCK_PROVIDER, describeVideo),
152+
}),
153+
});
154+
155+
expect(result.decision.outcome).toBe("success");
156+
expect(result.outputs[0]?.text).toBe(`ok:${CUSTOM_LOCAL_AUTH_MARKER}`);
157+
expect(describeVideo).toHaveBeenCalledTimes(1);
158+
expect(describeVideo.mock.calls[0]?.[0].apiKey).toBe(CUSTOM_LOCAL_AUTH_MARKER);
159+
});
160+
});
161+
});
162+
});
163+
});

‎src/media-understanding/runner.entries.ts‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -529,8 +529,9 @@ async function resolveProviderExecutionAuth(params: {
529529
};
530530
};
531531
const { isProviderAuthError, requireApiKey, resolveApiKeyForProvider } = await loadModelAuth();
532+
let resolvedAuth: Awaited<ReturnType<ResolveApiKeyForProvider>> | undefined;
532533
try {
533-
const auth = await resolveApiKeyForProvider({
534+
resolvedAuth = await resolveApiKeyForProvider({
534535
provider: params.providerId,
535536
cfg: params.cfg,
536537
profileId: params.entry.profile,
@@ -539,14 +540,14 @@ async function resolveProviderExecutionAuth(params: {
539540
workspaceDir: params.workspaceDir,
540541
modelApi,
541542
});
542-
const apiKey = requireApiKey(auth, params.providerId);
543+
const apiKey = requireApiKey(resolvedAuth, params.providerId);
543544
return {
544545
kind: "api-key",
545546
apiKeys: collectProviderApiKeysForExecution({
546547
provider: params.providerId,
547548
primaryApiKey: apiKey,
548549
}),
549-
source: auth.source,
550+
source: resolvedAuth.source,
550551
providerConfig,
551552
};
552553
} catch (err) {
@@ -560,6 +561,16 @@ async function resolveProviderExecutionAuth(params: {
560561
if (mediaAuth) {
561562
return mediaAuth;
562563
}
564+
// Providers configured for aws-sdk auth (e.g. Bedrock) authenticate through
565+
// the AWS credential chain rather than a literal API key, so a missing key
566+
// is expected, not an error. Execute keyless and let the SDK resolve creds.
567+
if (resolvedAuth?.mode === "aws-sdk") {
568+
return {
569+
kind: "none",
570+
source: resolvedAuth.source,
571+
providerConfig,
572+
};
573+
}
563574
throw err;
564575
}
565576
}

0 commit comments

Comments
 (0)