Skip to content

Commit 028ed85

Browse files
shivamhwpPR Batch Testerjuliusmarmingeclaude
authored
fix(server): Codex generated images show in chat, including on Android with a Windows host (#19066)
Co-authored-by: PR Batch Tester <[email protected]> Co-authored-by: Julius Marminge <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 33d27e9 commit 028ed85

9 files changed

Lines changed: 583 additions & 1 deletion

File tree

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts‎

Lines changed: 211 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ const encodeStringJson = Schema.encodeEffect(Schema.fromJsonString(Schema.String
7777
const isProviderAdapterEventStreamError = Schema.is(
7878
ProviderAdapter.ProviderAdapterEventStreamError,
7979
);
80+
const encodeUnknownJson = Schema.encodeUnknownSync(Schema.fromJsonString(Schema.Unknown));
8081

8182
describe("CodexAdapterV2 context usage", () => {
8283
it("uses the current context rather than cumulative processed tokens", () => {
@@ -1086,6 +1087,7 @@ describe("CodexAdapterV2 post-settle continuation", () => {
10861087
Effect.void,
10871088
onRequest: (method: string, params: unknown) => Effect.Effect<void> = () => Effect.void,
10881089
readChildMetadata?: Parameters<typeof withCodexReplayChildMetadata>[2],
1090+
beforeEmitInbound?: CodexReplay.CodexAppServerReplayDriver["beforeEmitInbound"],
10891091
) =>
10901092
Effect.gen(function* () {
10911093
const fileSystem = yield* FileSystem.FileSystem;
@@ -1096,9 +1098,13 @@ describe("CodexAdapterV2 post-settle continuation", () => {
10961098
const changes = yield* makeChangeSignal;
10971099
const continuationRequests: Array<ProviderContinuationRequests.ProviderContinuationRequest> =
10981100
[];
1101+
const driver = yield* CodexReplay.makeReplayDriver(
1102+
transcript,
1103+
beforeEmitInbound === undefined ? {} : { beforeEmitInbound },
1104+
);
10991105
const clientFactory: CodexAdapterV2.CodexAppServerClientFactoryShape = {
11001106
open: (openInput) =>
1101-
Layer.build(CodexReplay.layerReplay(transcript)).pipe(
1107+
Layer.build(CodexReplay.layerReplayWithDriver(driver)).pipe(
11021108
Effect.mapError(
11031109
(cause) =>
11041110
new ProviderAdapter.ProviderAdapterOpenSessionError({
@@ -3237,6 +3243,210 @@ describe("CodexAdapterV2 post-settle continuation", () => {
32373243
});
32383244
};
32393245

3246+
it.effect.each([
3247+
{
3248+
name: "a text reply gains each saved image once",
3249+
reply: "Done.",
3250+
images: [
3251+
{ savedPath: "/tmp/a.png" },
3252+
{ savedPath: "/tmp/a.png" },
3253+
{ savedPath: "/tmp/failed.png", status: "failed" },
3254+
{
3255+
savedPath: "/tmp/limited.png",
3256+
failure: { type: "usageLimitExceeded", limitId: "image-generation" },
3257+
},
3258+
{ savedPath: null },
3259+
],
3260+
expected: "Done.\n\n![](</tmp/a.png>)",
3261+
},
3262+
{
3263+
name: "an image-only result becomes the reply",
3264+
reply: null,
3265+
images: [{ savedPath: "/tmp/a.png" }],
3266+
expected: "![](</tmp/a.png>)",
3267+
},
3268+
])("projects generated images: $name", ({ reply, images, expected }) =>
3269+
Effect.gen(function* () {
3270+
const scenario = "codex-generated-images";
3271+
const result = "aW1hZ2UtYnl0ZXMtbXVzdC1uZXZlci1yZWFjaC1jbGllbnRz";
3272+
const base = finalAnswerTranscript(
3273+
scenario,
3274+
reply === null ? [] : [{ id: "answer", text: reply }],
3275+
);
3276+
const harness = yield* makeCodexReplayHarness(
3277+
makeCodexReplayTranscript({
3278+
scenario,
3279+
entries: [
3280+
...base.entries.slice(0, -1),
3281+
...images.map((image, index) => ({
3282+
type: "emit_inbound" as const,
3283+
frame: {
3284+
method: "item/completed",
3285+
params: {
3286+
threadId: `native-${scenario}-thread`,
3287+
turnId: `native-${scenario}-turn`,
3288+
item: {
3289+
type: "imageGeneration",
3290+
id: `image-${index}`,
3291+
status: "completed",
3292+
result,
3293+
...image,
3294+
},
3295+
},
3296+
},
3297+
})),
3298+
...base.entries.slice(-1),
3299+
],
3300+
}),
3301+
);
3302+
yield* harness.runtime.startTurn(
3303+
makeCodexTestTurnInput({
3304+
threadId: harness.threadId,
3305+
providerThread: harness.providerThread,
3306+
now: yield* DateTime.now,
3307+
attemptId: RunAttemptId.make("attempt-generated-images"),
3308+
text: "Reply with the requested recovery marker.",
3309+
}),
3310+
);
3311+
yield* harness.firstTerminal;
3312+
3313+
const messages = assistantMessages(harness.events);
3314+
assert.strictEqual(messages.at(-1)?.message.text, expected);
3315+
assert.strictEqual(messages.at(-1)?.message.streaming, false);
3316+
assert.strictEqual(new Set(messages.map(({ message }) => message.id)).size, 1);
3317+
// Image bytes stay with Codex; clients load the saved file.
3318+
assert.notInclude(encodeUnknownJson(harness.events), result);
3319+
}).pipe(
3320+
Effect.scoped,
3321+
Effect.provide(
3322+
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
3323+
),
3324+
),
3325+
);
3326+
3327+
it.effect.each(["complete", "stop"] as const)(
3328+
"retains generating images after settlement until %s",
3329+
(outcome) =>
3330+
Effect.gen(function* () {
3331+
const scenario = "codex-retained-image";
3332+
const base = finalAnswerTranscript(scenario, [{ id: "answer", text: "Done." }]);
3333+
const completions = {
3334+
first: yield* Deferred.make<void>(),
3335+
second: yield* Deferred.make<void>(),
3336+
};
3337+
const projected = yield* Deferred.make<void>();
3338+
const drained = yield* Deferred.make<void>();
3339+
const firstText = "Done.\n\n![](</tmp/first.png>)";
3340+
const imageEvent = (image: "first" | "second", phase: "started" | "completed") => ({
3341+
type: "emit_inbound" as const,
3342+
label: `${image}-${phase}`,
3343+
frame: {
3344+
method: `item/${phase}`,
3345+
params: {
3346+
threadId: `native-${scenario}-thread`,
3347+
turnId: `native-${scenario}-turn`,
3348+
item: {
3349+
type: "imageGeneration",
3350+
id: image,
3351+
status: phase === "started" ? "inProgress" : "completed",
3352+
result: "",
3353+
...(phase === "completed" ? { savedPath: `/tmp/${image}.png` } : {}),
3354+
},
3355+
},
3356+
},
3357+
});
3358+
const harness = yield* makeCodexReplayHarness(
3359+
makeCodexReplayTranscript({
3360+
scenario,
3361+
entries: [
3362+
...base.entries.slice(0, -2),
3363+
imageEvent("first", "started"),
3364+
imageEvent("second", "started"),
3365+
...base.entries.slice(-2),
3366+
imageEvent("first", "completed"),
3367+
imageEvent("second", "completed"),
3368+
],
3369+
}),
3370+
(event) =>
3371+
event.type === "message.updated" && event.message.text === firstText
3372+
? Deferred.succeed(projected, undefined)
3373+
: event.type === "background_work.updated" && !event.pending
3374+
? Deferred.succeed(drained, undefined)
3375+
: Effect.void,
3376+
undefined,
3377+
undefined,
3378+
(entry) =>
3379+
entry.label === "first-completed"
3380+
? Deferred.await(completions.first)
3381+
: entry.label === "second-completed"
3382+
? Deferred.await(completions.second)
3383+
: Effect.void,
3384+
);
3385+
yield* harness.runtime.startTurn(
3386+
makeCodexTestTurnInput({
3387+
threadId: harness.threadId,
3388+
providerThread: harness.providerThread,
3389+
now: yield* DateTime.now,
3390+
attemptId: RunAttemptId.make("attempt-retained-image"),
3391+
text: "Reply with the requested recovery marker.",
3392+
}),
3393+
);
3394+
yield* harness.firstTerminal;
3395+
const pending = Effect.all([
3396+
harness.runtime.hasPendingBackgroundWorkForThread!(harness.providerThread),
3397+
harness.hasPendingBackgroundWork,
3398+
]);
3399+
assert.deepEqual(yield* pending, [true, true]);
3400+
assert.strictEqual(assistantMessages(harness.events).at(-1)?.message.text, "Done.");
3401+
const cleared = () =>
3402+
harness.events.some((e) => e.type === "background_work.updated" && !e.pending);
3403+
3404+
if (outcome === "complete") {
3405+
yield* Deferred.succeed(completions.first, undefined);
3406+
yield* Deferred.await(projected);
3407+
// The second image is still generating, so the first must not release the turn.
3408+
assert.deepEqual(yield* pending, [true, true]);
3409+
assert.isFalse(cleared());
3410+
yield* Deferred.succeed(completions.second, undefined);
3411+
} else {
3412+
yield* harness.runtime.interruptTurn({
3413+
providerThread: harness.providerThread,
3414+
providerTurnId: harness.terminalEvents()[0]!.providerTurnId,
3415+
requestRuntimeRestart: true,
3416+
});
3417+
}
3418+
yield* Deferred.await(drained);
3419+
assert.deepEqual(yield* pending, [false, false]);
3420+
assert.lengthOf(harness.terminalEvents(), 1);
3421+
// Run ingestion follows these updates, so they must bracket the terminal and the images.
3422+
const indexOf = (predicate: (event: ProviderAdapter.ProviderAdapterV2Event) => boolean) =>
3423+
harness.events.findIndex(predicate);
3424+
const started = indexOf((e) => e.type === "background_work.updated" && e.pending);
3425+
const terminal = indexOf((e) => e.type === "turn.terminal");
3426+
const finished = indexOf((e) => e.type === "background_work.updated" && !e.pending);
3427+
assert.isAtLeast(started, 0);
3428+
assert.isAbove(terminal, started);
3429+
assert.isAbove(finished, terminal);
3430+
if (outcome === "complete") {
3431+
const messages = assistantMessages(harness.events);
3432+
const projectedBoth = indexOf(
3433+
(e) =>
3434+
e.type === "message.updated" &&
3435+
e.message.text === `${firstText}\n\n![](</tmp/second.png>)`,
3436+
);
3437+
assert.isAbove(projectedBoth, terminal);
3438+
assert.isAbove(finished, projectedBoth);
3439+
assert.strictEqual(messages.at(-1)?.message.id, messages[0]?.message.id);
3440+
assert.strictEqual(messages.at(-1)?.message.streaming, false);
3441+
}
3442+
}).pipe(
3443+
Effect.scoped,
3444+
Effect.provide(
3445+
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
3446+
),
3447+
),
3448+
);
3449+
32403450
it.effect("suppresses a trailing empty final answer after a non-empty final answer", () =>
32413451
Effect.scoped(
32423452
Effect.gen(function* () {

0 commit comments

Comments
 (0)