Skip to content

Commit 5702afb

Browse files
fix(server): retain image ingestion through ordered background completion
1 parent 5041b94 commit 5702afb

6 files changed

Lines changed: 124 additions & 26 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts‎

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4223,22 +4223,26 @@ describe("CodexAdapterV2 post-settle continuation", () => {
42234223
const scenario = "codex-retained-image";
42244224
const base = finalAnswerTranscript(scenario, [{ id: "answer", text: "Done." }]);
42254225
const completion = yield* Deferred.make<void>();
4226+
const secondCompletion = yield* Deferred.make<void>();
42264227
const projected = yield* Deferred.make<void>();
4228+
const drained = yield* Deferred.make<void>();
42274229
const expected = "Done.\n\n![](</tmp/late image.png>)";
4228-
const imageEvent = (phase: "started" | "completed") => ({
4230+
const imageEvent = (phase: "started" | "completed", second = false) => ({
42294231
type: "emit_inbound" as const,
4230-
label: `image-${phase}`,
4232+
label: `image-${phase}${second ? "-second" : ""}`,
42314233
frame: {
42324234
method: `item/${phase}`,
42334235
params: {
42344236
threadId: `native-${scenario}-thread`,
42354237
turnId: `native-${scenario}-turn`,
42364238
item: {
42374239
type: "imageGeneration",
4238-
id: "late-image",
4240+
id: second ? "second-image" : "late-image",
42394241
status: phase === "started" ? "inProgress" : "completed",
42404242
result: "",
4241-
...(phase === "completed" ? { savedPath: "/tmp/late image.png" } : {}),
4243+
...(phase === "completed"
4244+
? { savedPath: second ? "/tmp/second.png" : "/tmp/late image.png" }
4245+
: {}),
42424246
},
42434247
},
42444248
},
@@ -4249,17 +4253,26 @@ describe("CodexAdapterV2 post-settle continuation", () => {
42494253
entries: [
42504254
...base.entries.slice(0, -2),
42514255
imageEvent("started"),
4256+
imageEvent("started", true),
42524257
...base.entries.slice(-2),
42534258
imageEvent("completed"),
4259+
imageEvent("completed", true),
42544260
],
42554261
}),
42564262
(event) =>
42574263
event.type === "message.updated" && event.message.text === expected
42584264
? Deferred.succeed(projected, undefined)
4259-
: Effect.void,
4265+
: event.type === "background_work.updated" && !event.pending
4266+
? Deferred.succeed(drained, undefined)
4267+
: Effect.void,
42604268
undefined,
42614269
undefined,
4262-
(entry) => (entry.label === "image-completed" ? Deferred.await(completion) : Effect.void),
4270+
(entry) =>
4271+
entry.label === "image-completed"
4272+
? Deferred.await(completion)
4273+
: entry.label === "image-completed-second"
4274+
? Deferred.await(secondCompletion)
4275+
: Effect.void,
42634276
);
42644277
yield* harness.runtime.startTurn(
42654278
makeCodexTestTurnInput({
@@ -4280,6 +4293,21 @@ describe("CodexAdapterV2 post-settle continuation", () => {
42804293
if (outcome === "complete") {
42814294
yield* Deferred.succeed(completion, undefined);
42824295
yield* Deferred.await(projected);
4296+
assert.deepEqual(
4297+
[yield* threadPending, yield* harness.hasPendingBackgroundWork],
4298+
[true, true],
4299+
);
4300+
assert.isFalse(
4301+
harness.events.some(
4302+
(event) => event.type === "background_work.updated" && !event.pending,
4303+
),
4304+
);
4305+
yield* Deferred.succeed(secondCompletion, undefined);
4306+
yield* Deferred.await(drained);
4307+
assert.strictEqual(
4308+
assistantMessages(harness.events).at(-1)?.message.text,
4309+
`${expected}\n\n![](</tmp/second.png>)`,
4310+
);
42834311
const messages = assistantMessages(harness.events);
42844312
assert.strictEqual(messages.at(-1)?.message.id, messages[0]?.message.id);
42854313
assert.strictEqual(messages.at(-1)?.message.streaming, false);
@@ -4290,10 +4318,30 @@ describe("CodexAdapterV2 post-settle continuation", () => {
42904318
requestRuntimeRestart: true,
42914319
});
42924320
}
4321+
yield* Deferred.await(drained);
42934322
assert.deepEqual(
42944323
[yield* threadPending, yield* harness.hasPendingBackgroundWork],
42954324
[false, false],
42964325
);
4326+
const started = harness.events.findIndex(
4327+
(event) => event.type === "background_work.updated" && event.pending,
4328+
);
4329+
const terminal = harness.events.findIndex((event) => event.type === "turn.terminal");
4330+
const finished = harness.events.findIndex(
4331+
(event) => event.type === "background_work.updated" && !event.pending,
4332+
);
4333+
assert.isAtLeast(started, 0);
4334+
assert.isAbove(terminal, started);
4335+
assert.isAbove(finished, terminal);
4336+
if (outcome === "complete") {
4337+
assert.isAbove(
4338+
finished,
4339+
harness.events.findLastIndex(
4340+
(event) =>
4341+
event.type === "turn_item.updated" && event.turnItem.type === "assistant_message",
4342+
),
4343+
);
4344+
}
42974345
assert.lengthOf(harness.terminalEvents(), 1);
42984346
assert.deepEqual(pendingBefore, [true, true]);
42994347
}).pipe(

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1902,13 +1902,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
19021902
{
19031903
readonly running: Set<string>;
19041904
readonly saved: Map<string, string>;
1905+
pending: boolean;
19051906
answer?: { readonly id: string; readonly text: string };
19061907
}
19071908
>();
19081909
const imageTurn = (nativeTurnId: string) => {
19091910
let state = imageTurns.get(nativeTurnId);
19101911
if (state === undefined) {
1911-
state = { running: new Set(), saved: new Map() };
1912+
state = { running: new Set(), saved: new Map(), pending: false };
19121913
imageTurns.set(nativeTurnId, state);
19131914
}
19141915
return state;
@@ -1930,6 +1931,26 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
19301931
const emitProviderEvent = (event: ProviderAdapter.ProviderAdapterV2Event) =>
19311932
Queue.offer(events, event).pipe(Effect.asVoid);
19321933

1934+
const emitImageWorkState = Effect.fnUntraced(function* (
1935+
context: ActiveCodexTurnContext,
1936+
pending: boolean,
1937+
) {
1938+
const state = imageTurns.get(context.nativeTurnId);
1939+
if (state === undefined || state.pending === pending) return;
1940+
state.pending = pending;
1941+
yield* emitProviderEvent({
1942+
type: "background_work.updated",
1943+
driver: CODEX_PROVIDER,
1944+
providerTurnId: context.providerTurnId,
1945+
pending,
1946+
});
1947+
});
1948+
1949+
const clearImageWork = Effect.fnUntraced(function* (context: ActiveCodexTurnContext) {
1950+
yield* emitImageWorkState(context, false);
1951+
imageTurns.delete(context.nativeTurnId);
1952+
});
1953+
19331954
const emitGoalUpdate = Effect.fnUntraced(function* (nativeThreadId: string) {
19341955
const providerThread = rootProviderThreads.get(nativeThreadId);
19351956
if (providerThread === undefined || !goalsByNativeThread.has(nativeThreadId)) return;
@@ -4693,6 +4714,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
46934714
if (payload.item.type === "imageGeneration") {
46944715
if (!(yield* Ref.get(interruptingNativeTurns)).has(payload.turnId)) {
46954716
imageTurn(payload.turnId).running.add(payload.item.id);
4717+
yield* emitImageWorkState(context, true);
46964718
}
46974719
return;
46984720
}
@@ -4978,6 +5000,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
49785000
yield* projectGeneratedImages(context, payload.turnId);
49795001
}
49805002
state.running.delete(id);
5003+
yield* emitImageWorkState(context, state.running.size > 0 || state.saved.size > 0);
49815004
if (settled) {
49825005
yield* releaseSettledTurnIfIdle(payload.turnId);
49835006
}
@@ -5875,6 +5898,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
58755898
if (part.turnId === input.nativeTurnId) reasoningParts.delete(key);
58765899
}
58775900
yield* projectGeneratedImages(input.context, input.nativeTurnId);
5901+
if (input.status !== "completed") {
5902+
yield* clearImageWork(input.context);
5903+
} else {
5904+
const images = imageTurns.get(input.nativeTurnId);
5905+
if (images !== undefined) {
5906+
yield* emitImageWorkState(input.context, images.running.size > 0);
5907+
}
5908+
}
58785909
const nativeThreadId = input.context.providerThread.nativeThreadRef?.nativeId;
58795910
const activation =
58805911
nativeThreadId == null ? undefined : goalActivations.get(nativeThreadId);
@@ -7145,7 +7176,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
71457176
updated.delete(context.nativeTurnId);
71467177
return updated;
71477178
});
7148-
imageTurns.delete(context.nativeTurnId);
7179+
yield* clearImageWork(context);
71497180
yield* releaseSettledTurnIfIdle(context.nativeTurnId);
71507181
}
71517182
}),

‎apps/server/src/orchestration-v2/ProviderEventIngestor.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -674,6 +674,8 @@ export const layer: Layer.Layer<
674674
}),
675675
];
676676
}
677+
case "background_work.updated":
678+
return [];
677679
case "command_output.delta":
678680
// Live output is routed to the command output hub before ingestion.
679681
return [];

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1781,11 +1781,24 @@ it.effect(
17811781
turnItem: { ...itemEvent.turnItem, text: bothImagesText },
17821782
};
17831783
const terminal = rootTerminalEvent(ids, "completed");
1784+
const backgroundStarted = {
1785+
type: "background_work.updated",
1786+
driver,
1787+
providerTurnId: ids.rootProviderTurnId,
1788+
pending: true,
1789+
} satisfies ProviderAdapter.ProviderAdapterV2Event;
1790+
const backgroundCompleted = { ...backgroundStarted, pending: false };
17841791

17851792
const observed = yield* runBackgroundItemScenario(
17861793
key,
17871794
() =>
1788-
Stream.fromIterable([nodeEvent, messageEvent, itemEvent, terminal]).pipe(
1795+
Stream.fromIterable([
1796+
backgroundStarted,
1797+
nodeEvent,
1798+
messageEvent,
1799+
itemEvent,
1800+
terminal,
1801+
]).pipe(
17891802
// Both completions are queued when pending clears during root finalization.
17901803
// Separate chunks expose stop checks before the first node and between completions.
17911804
Stream.concat(Stream.succeed(nodeEvent)),
@@ -1794,6 +1807,7 @@ it.effect(
17941807
Stream.concat(Stream.succeed(nodeEvent)),
17951808
Stream.concat(Stream.succeed(secondLateMessage)),
17961809
Stream.concat(Stream.succeed(secondLateItem)),
1810+
Stream.concat(Stream.succeed(backgroundCompleted)),
17971811
),
17981812
{
17991813
keepEventStreamOpen: true,
@@ -1807,6 +1821,7 @@ it.effect(
18071821
);
18081822

18091823
assert.deepEqual(yield* Ref.get(ingested), [
1824+
backgroundStarted,
18101825
nodeEvent,
18111826
messageEvent,
18121827
itemEvent,
@@ -1817,6 +1832,7 @@ it.effect(
18171832
nodeEvent,
18181833
secondLateMessage,
18191834
secondLateItem,
1835+
backgroundCompleted,
18201836
]);
18211837
assert.deepEqual(observed, [
18221838
"turn_item:completed",

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 10 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -481,6 +481,8 @@ export function routeProviderEvent(
481481
state.ownedProviderTurnIds.has(event.runtimeRequest.providerTurnId)),
482482
state,
483483
];
484+
case "background_work.updated":
485+
return [state.ownedProviderTurnIds.has(event.providerTurnId), state];
484486
case "command_output.delta":
485487
// The session manager hands live output to the command output hub; runs never see it.
486488
return [false, state];
@@ -973,7 +975,7 @@ export const layer: Layer.Layer<
973975
const rootTerminalSeen = yield* Ref.make(false);
974976
const rootRunFinalized = yield* Ref.make(false);
975977
const providerThreadOwnerLost = yield* Ref.make(false);
976-
const pendingAssistantMessageNodes = new Set<NodeId>();
978+
const pendingBackgroundProviderTurns = new Set<ProviderTurnId>();
977979
const activeChildProviderTurns = yield* Ref.make<ReadonlySet<ProviderTurnId>>(new Set());
978980
const activeChildSubagents = yield* Ref.make<ReadonlySet<NodeId>>(new Set());
979981
const activeBackgroundTurnItems = yield* Ref.make<
@@ -1178,7 +1180,7 @@ export const layer: Layer.Layer<
11781180
) {
11791181
return true;
11801182
}
1181-
if (pendingAssistantMessageNodes.size > 0) {
1183+
if (pendingBackgroundProviderTurns.size > 0) {
11821184
return false;
11831185
}
11841186
const childProviderTurns = yield* Ref.get(activeChildProviderTurns);
@@ -1306,21 +1308,12 @@ export const layer: Layer.Layer<
13061308
yield* Ref.set(rootTerminalSeen, true);
13071309
yield* finalizeRootRun(event);
13081310
}
1309-
// Assistant artifacts arrive as node, message, then turn item. The adapter can
1310-
// clear its pending-work probe before these queued writes are ingested.
1311-
if (
1312-
event.type === "node.updated" &&
1313-
event.node.kind === "assistant_message" &&
1314-
event.node.status === "completed"
1315-
) {
1316-
pendingAssistantMessageNodes.add(event.node.id);
1317-
} else if (
1318-
event.type === "turn_item.updated" &&
1319-
event.turnItem.type === "assistant_message" &&
1320-
!event.turnItem.streaming &&
1321-
event.turnItem.nodeId !== null
1322-
) {
1323-
pendingAssistantMessageNodes.delete(event.turnItem.nodeId);
1311+
if (event.type === "background_work.updated") {
1312+
if (event.pending) {
1313+
pendingBackgroundProviderTurns.add(event.providerTurnId);
1314+
} else {
1315+
pendingBackgroundProviderTurns.delete(event.providerTurnId);
1316+
}
13241317
}
13251318
yield* trackChildLifecycle(event, deliveredEvent !== null);
13261319
}),

‎packages/provider-core/src/server/ProviderAdapter.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,14 @@ export const ProviderAdapterV2Event = Schema.Union([
119119
driver: ProviderDriverKind,
120120
turnItem: OrchestrationV2TurnItem,
121121
}),
122+
// Start before the root terminal; clear only after the final covered projection.
123+
// Consumer lifetime cannot depend on a producer probe that runs ahead of its queue.
124+
Schema.Struct({
125+
type: Schema.Literal("background_work.updated"),
126+
driver: ProviderDriverKind,
127+
providerTurnId: ProviderTurnId,
128+
pending: Schema.Boolean,
129+
}),
122130
/**
123131
* Raw output from a running command. Never persisted: the session manager
124132
* hands it to the command output hub, which only clients showing the row see.

0 commit comments

Comments
 (0)