Skip to content

Commit 46a5a5e

Browse files
bek91bek91
authored
fix(media): route OAuth image defaults through Codex (#92824)
Route implicit OpenAI image understanding through the Codex app-server for eligible OpenAI OAuth profiles. Preserve scoped and persisted credential ownership plus the rotating-token refresh lifecycle for isolated clients. Fixes #87168 Thanks @bek91.
1 parent af09117 commit 46a5a5e

19 files changed

Lines changed: 2160 additions & 93 deletions

‎extensions/codex/media-understanding-provider.test.ts‎

Lines changed: 84 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,14 @@ import { buildCodexMediaUnderstandingProvider } from "./media-understanding-prov
55
import type { CodexAppServerClient } from "./src/app-server/client.js";
66
import type { CodexServerNotification, JsonValue } from "./src/app-server/protocol.js";
77

8+
const sharedClientMocks = vi.hoisted(() => ({
9+
createIsolatedCodexAppServerClient: vi.fn(),
10+
}));
11+
12+
vi.mock("./src/app-server/shared-client.js", () => ({
13+
createIsolatedCodexAppServerClient: sharedClientMocks.createIsolatedCodexAppServerClient,
14+
}));
15+
816
function codexModel(inputModalities: string[] = ["text", "image"]) {
917
return {
1018
id: "gpt-5.4",
@@ -169,6 +177,7 @@ function createFakeClient(options?: {
169177
requestHandlers.add(handler);
170178
return () => requestHandlers.delete(handler);
171179
},
180+
close: vi.fn(),
172181
} as unknown as CodexAppServerClient;
173182

174183
return { client, requests, approvalResponses };
@@ -178,13 +187,24 @@ describe("codex media understanding provider", () => {
178187
afterEach(() => {
179188
vi.useRealTimers();
180189
vi.restoreAllMocks();
190+
sharedClientMocks.createIsolatedCodexAppServerClient.mockReset();
181191
});
182192

183193
it("runs image understanding through a bounded Codex app-server turn", async () => {
184194
const { client, requests } = createFakeClient();
195+
const clientFactory = vi.fn(
196+
async (_startOptions, _authProfileId, _agentDir, _config) => client,
197+
);
185198
const provider = buildCodexMediaUnderstandingProvider({
186-
clientFactory: async () => client,
199+
clientFactory,
187200
});
201+
const cfg = {
202+
auth: {
203+
order: {
204+
openai: ["openai:work"],
205+
},
206+
},
207+
};
188208

189209
const result = await provider.describeImage?.({
190210
buffer: Buffer.from("image-bytes"),
@@ -194,7 +214,7 @@ describe("codex media understanding provider", () => {
194214
model: "gpt-5.4",
195215
prompt: "Describe briefly.",
196216
timeoutMs: 30_000,
197-
cfg: {},
217+
cfg,
198218
agentDir: "/tmp/openclaw-agent",
199219
});
200220

@@ -204,6 +224,12 @@ describe("codex media understanding provider", () => {
204224
"thread/start",
205225
"turn/start",
206226
]);
227+
expect(clientFactory).toHaveBeenCalledWith(
228+
expect.any(Object),
229+
undefined,
230+
"/tmp/openclaw-agent",
231+
cfg,
232+
);
207233
expect(requests[1]?.params).toEqual({
208234
model: "gpt-5.4",
209235
modelProvider: "openai",
@@ -236,6 +262,62 @@ describe("codex media understanding provider", () => {
236262
});
237263
});
238264

265+
it("treats a blank agent directory as absent when starting the app-server", async () => {
266+
const { client, requests } = createFakeClient();
267+
const clientFactory = vi.fn(async () => client);
268+
const provider = buildCodexMediaUnderstandingProvider({ clientFactory });
269+
const cfg = {};
270+
271+
await provider.describeImage?.({
272+
buffer: Buffer.from("image-bytes"),
273+
fileName: "image.png",
274+
mime: "image/png",
275+
provider: "codex",
276+
model: "gpt-5.4",
277+
timeoutMs: 30_000,
278+
cfg,
279+
agentDir: " ",
280+
});
281+
282+
expect(clientFactory).toHaveBeenCalledWith(expect.any(Object), undefined, undefined, cfg);
283+
expect(requests[1]?.params).toEqual(expect.objectContaining({ cwd: process.cwd() }));
284+
expect(requests[2]?.params).toEqual(expect.objectContaining({ cwd: process.cwd() }));
285+
});
286+
287+
it("passes the scoped auth store into isolated app-server startup", async () => {
288+
const { client } = createFakeClient();
289+
sharedClientMocks.createIsolatedCodexAppServerClient.mockResolvedValue(client);
290+
const provider = buildCodexMediaUnderstandingProvider();
291+
const authStore = {
292+
version: 1,
293+
profiles: {
294+
"openai:scoped": {
295+
type: "oauth" as const,
296+
provider: "openai",
297+
access: "scoped-access",
298+
refresh: "scoped-refresh",
299+
expires: Date.now() + 60_000,
300+
},
301+
},
302+
};
303+
304+
await provider.describeImage?.({
305+
buffer: Buffer.from("image-bytes"),
306+
fileName: "image.png",
307+
mime: "image/png",
308+
provider: "codex",
309+
model: "gpt-5.4",
310+
timeoutMs: 30_000,
311+
cfg: {},
312+
authStore,
313+
agentDir: "/tmp/openclaw-agent",
314+
});
315+
316+
expect(sharedClientMocks.createIsolatedCodexAppServerClient).toHaveBeenCalledWith(
317+
expect.objectContaining({ authProfileStore: authStore }),
318+
);
319+
});
320+
239321
it("clamps oversized image understanding turn timeouts", async () => {
240322
const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
241323
try {

‎extensions/codex/media-understanding-provider.ts‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,8 @@ async function describeCodexImages(
102102
profile: req.profile,
103103
timeoutMs: req.timeoutMs,
104104
agentDir: req.agentDir,
105+
authStore: req.authStore,
106+
cfg: req.cfg,
105107
options,
106108
taskLabel: "image understanding",
107109
developerInstructions:
@@ -123,6 +125,8 @@ type BoundedCodexVisionTurnParams = {
123125
profile?: string;
124126
timeoutMs: number;
125127
agentDir?: string;
128+
authStore?: ImagesDescriptionRequest["authStore"];
129+
cfg: ImagesDescriptionRequest["cfg"];
126130
options: CodexMediaUnderstandingProviderOptions;
127131
taskLabel: string;
128132
developerInstructions: string;
@@ -135,17 +139,22 @@ async function runBoundedCodexVisionTurn(params: BoundedCodexVisionTurnParams):
135139
pluginConfig: params.options.pluginConfig,
136140
});
137141
const timeoutMs = resolveTimerTimeoutMs(params.timeoutMs, 100, 100);
142+
const agentDir = params.agentDir?.trim() || undefined;
143+
const cwd = agentDir ?? process.cwd();
138144
const ownsClient = !params.options.clientFactory;
139145
// Tests inject a client factory; production creates an isolated app-server
140146
// client so media tasks cannot reuse the interactive attempt session.
141147
const client = params.options.clientFactory
142-
? await params.options.clientFactory(appServer.start, params.profile)
148+
? await params.options.clientFactory(appServer.start, params.profile, agentDir, params.cfg)
143149
: await import("./src/app-server/shared-client.js").then(
144150
({ createIsolatedCodexAppServerClient }) =>
145151
createIsolatedCodexAppServerClient({
146152
startOptions: appServer.start,
147153
timeoutMs,
148154
authProfileId: params.profile,
155+
agentDir,
156+
authProfileStore: params.authStore,
157+
config: params.cfg,
149158
}),
150159
);
151160
const abortController = new AbortController();
@@ -166,7 +175,7 @@ async function runBoundedCodexVisionTurn(params: BoundedCodexVisionTurnParams):
166175
{
167176
model: params.model,
168177
modelProvider: "openai",
169-
cwd: params.agentDir || process.cwd(),
178+
cwd,
170179
approvalPolicy: "on-request",
171180
sandbox: "read-only",
172181
serviceName: "OpenClaw",
@@ -193,7 +202,7 @@ async function runBoundedCodexVisionTurn(params: BoundedCodexVisionTurnParams):
193202
{
194203
threadId: thread.thread.id,
195204
input: params.input,
196-
cwd: params.agentDir || process.cwd(),
205+
cwd,
197206
approvalPolicy: "on-request",
198207
model: params.model,
199208
effort: "low",
@@ -242,6 +251,8 @@ async function extractCodexStructured(
242251
profile: req.profile,
243252
timeoutMs: req.timeoutMs,
244253
agentDir: req.agentDir,
254+
authStore: req.authStore,
255+
cfg: req.cfg,
245256
options,
246257
taskLabel: "structured extraction",
247258
developerInstructions:

0 commit comments

Comments
 (0)