Skip to content

Commit 5be6bc9

Browse files
authored
fix(codex): show image generation tool calls
Codex image generation now appears as a timeline tool row with prompt and transparency inputs, completion results, failures, and cancellation. Retain inputs on cancellation and give image-only replies a separate message ID so they cannot replace the tool row. Verified with a real image generation and collapsed/expanded UI captures. check:changed passes, including 152 tests; CI and agentic reviews pass on 86d2045. Model: GPT 6.1 Sol via the Codex harness.
1 parent d256170 commit 5be6bc9

2 files changed

Lines changed: 146 additions & 4 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4116,6 +4116,8 @@ describe("CodexAdapterV2 post-settle continuation", () => {
41164116
id: `image-${index}`,
41174117
status: "completed",
41184118
result,
4119+
revisedPrompt: "A tree shaped like T3",
4120+
transparentBackground: true,
41194121
...image,
41204122
},
41214123
},
@@ -4136,6 +4138,42 @@ describe("CodexAdapterV2 post-settle continuation", () => {
41364138
);
41374139
yield* harness.firstTerminal;
41384140

4141+
const imageItems = harness.events
4142+
.flatMap((event) => (event.type === "turn_item.updated" ? [event.turnItem] : []))
4143+
.filter((item) => item.type === "dynamic_tool" && item.toolName === "image_gen.imagegen");
4144+
const latestItems = new Map(
4145+
harness.events.flatMap((event) =>
4146+
event.type === "turn_item.updated" ? [[event.turnItem.id, event.turnItem] as const] : [],
4147+
),
4148+
);
4149+
assert.strictEqual(
4150+
[...latestItems.values()].filter((item) => item.type === "dynamic_tool").length,
4151+
images.length,
4152+
);
4153+
assert.strictEqual(imageItems.length, images.length);
4154+
assert.deepEqual(
4155+
imageItems.map((item) => item.status),
4156+
images.map((image) =>
4157+
("status" in image && image.status === "failed") || "failure" in image
4158+
? "failed"
4159+
: "completed",
4160+
),
4161+
);
4162+
for (const [index, item] of imageItems.entries()) {
4163+
assert.strictEqual(item.title, "Generate image");
4164+
if (item.type !== "dynamic_tool") continue;
4165+
assert.deepEqual(item.input, {
4166+
prompt: "A tree shaped like T3",
4167+
transparent_background: true,
4168+
});
4169+
const image = images[index]!;
4170+
assert.deepEqual(
4171+
item.output,
4172+
Object.fromEntries(
4173+
Object.entries({ status: "completed", ...image }).filter(([, value]) => value != null),
4174+
),
4175+
);
4176+
}
41394177
const messages = assistantMessages(harness.events);
41404178
assert.strictEqual(messages.at(-1)?.message.text, expected);
41414179
assert.strictEqual(messages.at(-1)?.message.streaming, false);
@@ -4179,6 +4217,8 @@ describe("CodexAdapterV2 post-settle continuation", () => {
41794217
item: {
41804218
type: "imageGeneration",
41814219
id: image,
4220+
revisedPrompt: "A friendly green tree",
4221+
transparentBackground: true,
41824222
status: phase === "started" ? "inProgress" : "completed",
41834223
result: "",
41844224
...(phase === "completed" ? { savedPath: `/tmp/${image}.png` } : {}),
@@ -4249,6 +4289,31 @@ describe("CodexAdapterV2 post-settle continuation", () => {
42494289
yield* Deferred.await(drained);
42504290
assert.deepEqual(yield* pending, [false, false]);
42514291
assert.lengthOf(harness.terminalEvents(), 1);
4292+
const imageItems = harness.events.flatMap((event) =>
4293+
event.type === "turn_item.updated" &&
4294+
event.turnItem.type === "dynamic_tool" &&
4295+
event.turnItem.toolName === "image_gen.imagegen"
4296+
? [event.turnItem]
4297+
: [],
4298+
);
4299+
assert.deepEqual(
4300+
imageItems.slice(0, 2).map((item) => item.status),
4301+
["running", "running"],
4302+
);
4303+
for (const id of new Set(imageItems.map((item) => item.id))) {
4304+
const updates = imageItems.filter((item) => item.id === id);
4305+
assert.lengthOf(updates, 2);
4306+
assert.strictEqual(
4307+
updates[1]?.status,
4308+
outcome === "complete" ? "completed" : "cancelled",
4309+
);
4310+
assert.strictEqual(updates[1]?.ordinal, updates[0]?.ordinal);
4311+
assert.deepEqual(updates[1]?.input, updates[0]?.input);
4312+
assert.deepEqual(updates[1]?.input, {
4313+
prompt: "A friendly green tree",
4314+
transparent_background: true,
4315+
});
4316+
}
42524317
// Run ingestion follows these updates, so they must bracket the terminal and the images.
42534318
const indexOf = (predicate: (event: ProviderAdapter.ProviderAdapterV2Event) => boolean) =>
42544319
harness.events.findIndex(predicate);

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts‎

Lines changed: 81 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1981,7 +1981,13 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
19811981
const imageTurns = new Map<
19821982
string,
19831983
{
1984-
readonly running: Set<string>;
1984+
readonly running: Map<
1985+
string,
1986+
{
1987+
readonly revisedPrompt: string | null;
1988+
readonly transparentBackground: boolean | null;
1989+
}
1990+
>;
19851991
readonly saved: Map<string, string>;
19861992
pending: boolean;
19871993
answer?: { readonly id: string; readonly text: string };
@@ -1990,7 +1996,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
19901996
const imageTurn = (nativeTurnId: string) => {
19911997
let state = imageTurns.get(nativeTurnId);
19921998
if (state === undefined) {
1993-
state = { running: new Set(), saved: new Map(), pending: false };
1999+
state = { running: new Map(), saved: new Map(), pending: false };
19942000
imageTurns.set(nativeTurnId, state);
19952001
}
19962002
return state;
@@ -2075,6 +2081,19 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
20752081
const clearImageWork = Effect.fn("CodexAdapterV2.clearImageWork")(function* (
20762082
context: ActiveCodexTurnContext,
20772083
) {
2084+
for (const [id, input] of imageTurns.get(context.nativeTurnId)?.running ?? []) {
2085+
yield* emitImageGenerationItem(
2086+
context,
2087+
{
2088+
type: "imageGeneration",
2089+
id,
2090+
...input,
2091+
status: "cancelled",
2092+
result: "",
2093+
},
2094+
true,
2095+
);
2096+
}
20782097
yield* emitImageWorkState(context, false);
20792098
imageTurns.delete(context.nativeTurnId);
20802099
});
@@ -3926,6 +3945,56 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
39263945
},
39273946
);
39283947

3948+
const emitImageGenerationItem = Effect.fn("CodexAdapterV2.emitImageGenerationItem")(
3949+
function* (
3950+
context: ActiveCodexTurnContext,
3951+
item: Extract<
3952+
CodexSchema.V2ItemCompletedNotification__ThreadItem,
3953+
{ type: "imageGeneration" }
3954+
>,
3955+
completed: boolean,
3956+
) {
3957+
const cancelled = completed && item.status === "cancelled";
3958+
const failed = item.status === "failed" || item.failure != null;
3959+
const artifacts = yield* buildDynamicToolArtifacts(context, {
3960+
type: "dynamicToolCall",
3961+
id: item.id,
3962+
namespace: "image_gen",
3963+
tool: "imagegen",
3964+
status: completed ? (failed ? "failed" : "completed") : "inProgress",
3965+
arguments: {
3966+
...(item.revisedPrompt == null ? {} : { prompt: item.revisedPrompt }),
3967+
...(item.transparentBackground == null
3968+
? {}
3969+
: { transparent_background: item.transparentBackground }),
3970+
},
3971+
});
3972+
yield* emitProviderEvent({
3973+
type: "node.updated",
3974+
driver: CODEX_PROVIDER,
3975+
node: { ...artifacts.node, ...(cancelled ? { status: "cancelled" } : {}) },
3976+
});
3977+
yield* emitProviderEvent({
3978+
type: "turn_item.updated",
3979+
driver: CODEX_PROVIDER,
3980+
turnItem: {
3981+
...artifacts.turnItem,
3982+
title: "Generate image",
3983+
...(cancelled ? { status: "cancelled" } : {}),
3984+
...(completed
3985+
? {
3986+
output: {
3987+
status: item.status,
3988+
...(item.savedPath == null ? {} : { savedPath: item.savedPath }),
3989+
...(item.failure == null ? {} : { failure: item.failure }),
3990+
},
3991+
}
3992+
: {}),
3993+
},
3994+
});
3995+
},
3996+
);
3997+
39293998
/**
39303999
* The item stays running until then, which keeps the run's event
39314000
* ingestion open past turn/completed; it runs off the notification
@@ -4980,7 +5049,11 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
49805049

49815050
if (payload.item.type === "imageGeneration") {
49825051
if (!(yield* Ref.get(interruptingNativeTurns)).has(payload.turnId)) {
4983-
imageTurn(payload.turnId).running.add(payload.item.id);
5052+
imageTurn(payload.turnId).running.set(payload.item.id, {
5053+
revisedPrompt: payload.item.revisedPrompt ?? null,
5054+
transparentBackground: payload.item.transparentBackground ?? null,
5055+
});
5056+
yield* emitImageGenerationItem(context, payload.item, false);
49845057
yield* emitImageWorkState(context, true);
49855058
}
49865059
return;
@@ -5266,6 +5339,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
52665339

52675340
if (payload.item.type === "imageGeneration") {
52685341
if ((yield* Ref.get(interruptingNativeTurns)).has(payload.turnId)) return;
5342+
yield* emitImageGenerationItem(context, payload.item, true);
52695343
const { id, savedPath, status, failure } = payload.item;
52705344
const state = imageTurn(payload.turnId);
52715345
if (savedPath?.trim() && status === "completed" && failure == null) {
@@ -6182,7 +6256,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
61826256
state.saved.clear();
61836257
return;
61846258
}
6185-
const message = { id: finalAnswer?.id ?? firstImageItemId, text };
6259+
const message = {
6260+
id: finalAnswer?.id ?? `generated-image-message:${firstImageItemId}`,
6261+
text,
6262+
};
61866263
state.answer = message;
61876264
const artifacts = yield* buildAgentMessageArtifacts(context, message, true);
61886265
yield* emitProviderEvent({

0 commit comments

Comments
 (0)