Skip to content

Commit 45bcde1

Browse files
fix(server): clear Codex commands when their native thread closes (pingdotgg#19936)
Co-authored-by: Julius Marminge <[email protected]>
1 parent 6c3c31d commit 45bcde1

2 files changed

Lines changed: 160 additions & 0 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4973,6 +4973,134 @@ describe("CodexAdapterV2 post-settle continuation", () => {
49734973
),
49744974
);
49754975

4976+
it.effect.each([false, true])(
4977+
"keeps commands interrupted after native thread closure (persistent tool: %s)",
4978+
(retainContext) =>
4979+
Effect.gen(function* () {
4980+
const notification = (method: string, params: unknown) => ({
4981+
type: "emit_inbound" as const,
4982+
label: method,
4983+
frame: { method, params },
4984+
});
4985+
const monitor = (status: "inProgress" | "completed") =>
4986+
notification(status === "inProgress" ? "item/started" : "item/completed", {
4987+
threadId: BG_NATIVE_THREAD,
4988+
turnId: BG_NATIVE_TURN,
4989+
item: {
4990+
type: "dynamicToolCall",
4991+
id: "persistent-monitor",
4992+
namespace: "grok",
4993+
tool: "monitor",
4994+
status,
4995+
arguments: { persistent: true, command: "tail -f" },
4996+
},
4997+
});
4998+
// Replies fence notification handling on the same protocol reader.
4999+
const unsubscribe = (id: number, notifications: ReturnType<typeof notification>[] = []) => [
5000+
{
5001+
type: "expect_outbound" as const,
5002+
label: `unsubscribe-${id}`,
5003+
frame: { id, method: "thread/unsubscribe", params: { threadId: BG_NATIVE_THREAD } },
5004+
},
5005+
...notifications,
5006+
{
5007+
type: "emit_inbound" as const,
5008+
label: `unsubscribe-result-${id}`,
5009+
frame: { id, result: { status: "notSubscribed" } },
5010+
},
5011+
];
5012+
const closed = notification("thread/closed", { threadId: BG_NATIVE_THREAD });
5013+
const transcript = makeCodexReplayTranscript({
5014+
scenario: `codex-background-thread-closed-${retainContext}`,
5015+
entries: [
5016+
...backgroundExecTranscript.entries.slice(0, -2),
5017+
...(retainContext ? [monitor("inProgress")] : []),
5018+
backgroundExecTranscript.entries.at(-2)!,
5019+
...unsubscribe(4),
5020+
...unsubscribe(5, [notification("thread/closed", { threadId: "unrelated-thread" })]),
5021+
...unsubscribe(6, [closed]),
5022+
...unsubscribe(7, [
5023+
closed,
5024+
notification("item/completed", {
5025+
threadId: BG_NATIVE_THREAD,
5026+
turnId: BG_NATIVE_TURN,
5027+
item: backgroundCommandItem("completed"),
5028+
}),
5029+
...(retainContext ? [monitor("completed")] : []),
5030+
]),
5031+
],
5032+
});
5033+
const interrupted = yield* Deferred.make<void>();
5034+
const monitorCompleted = yield* Deferred.make<void>();
5035+
const harness = yield* makeCodexReplayHarness(transcript, (event) => {
5036+
if (event.type !== "turn_item.updated") return Effect.void;
5037+
if (
5038+
event.turnItem.nativeItemRef?.nativeId === BG_COMMAND_ITEM &&
5039+
event.turnItem.status === "interrupted"
5040+
) {
5041+
return Deferred.succeed(interrupted, undefined);
5042+
}
5043+
return event.turnItem.nativeItemRef?.nativeId === "persistent-monitor" &&
5044+
event.turnItem.status === "completed"
5045+
? Deferred.succeed(monitorCompleted, undefined)
5046+
: Effect.void;
5047+
});
5048+
yield* harness.runtime.startTurn(
5049+
makeCodexTestTurnInput({
5050+
threadId: harness.threadId,
5051+
providerThread: harness.providerThread,
5052+
now: yield* DateTime.now,
5053+
attemptId: RunAttemptId.make("attempt-codex-thread-closed"),
5054+
text: BG_PROMPT,
5055+
}),
5056+
);
5057+
yield* harness.firstTerminal;
5058+
const unload = harness.runtime.unloadThread!({ providerThread: harness.providerThread });
5059+
yield* unload;
5060+
assert.isTrue(yield* harness.hasPendingBackgroundWork);
5061+
yield* unload;
5062+
assert.isTrue(yield* harness.hasPendingBackgroundWork);
5063+
yield* unload;
5064+
assert.equal(yield* harness.hasPendingBackgroundWork, retainContext);
5065+
yield* Deferred.await(interrupted);
5066+
yield* unload;
5067+
if (retainContext) yield* Deferred.await(monitorCompleted);
5068+
assert.deepEqual(
5069+
{
5070+
items: harness.events.flatMap((event) =>
5071+
event.type === "turn_item.updated" &&
5072+
event.turnItem.nativeItemRef?.nativeId === BG_COMMAND_ITEM
5073+
? [event.turnItem.status]
5074+
: [],
5075+
),
5076+
nodes: harness.events.flatMap((event) =>
5077+
event.type === "node.updated" &&
5078+
event.node.nativeItemRef?.nativeId === BG_COMMAND_ITEM
5079+
? [event.node.status]
5080+
: [],
5081+
),
5082+
roots: harness.terminalEvents().map((event) => event.status),
5083+
continuations: harness.continuationRequests.length,
5084+
},
5085+
{
5086+
items: ["running", "interrupted"],
5087+
nodes: ["running", "interrupted"],
5088+
roots: ["completed"],
5089+
continuations: 0,
5090+
},
5091+
);
5092+
assert.isFalse(yield* harness.hasPendingBackgroundWork);
5093+
assert.isFalse(
5094+
yield* harness.runtime.hasPendingBackgroundWorkForThread!(harness.providerThread),
5095+
);
5096+
}).pipe(
5097+
Effect.scoped,
5098+
Effect.provide(
5099+
Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
5100+
),
5101+
),
5102+
);
5103+
49765104
const backgroundStopCases = [true, false, "still_running", "thread_unloaded"] as const;
49775105
const makeBackgroundStopTranscript = (terminated: (typeof backgroundStopCases)[number]) => {
49785106
const stillRunning = terminated === "still_running";

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1137,6 +1137,7 @@ function codexUsageLimitError(
11371137
}
11381138

11391139
interface ActiveCodexTurnContext {
1140+
closedCommandItemIds?: ReadonlySet<string>;
11401141
latestProviderFailure?: {
11411142
readonly nativeMessage: string;
11421143
readonly failure: OrchestrationV2ProviderFailure;
@@ -4674,6 +4675,35 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
46744675
});
46754676
});
46764677

4678+
// Unloading an idle thread kills its terminals without completing
4679+
// their command items, while the shared provider session stays alive.
4680+
yield* client.handleServerNotification("thread/closed", (payload) =>
4681+
Effect.gen(function* () {
4682+
const completedAt = yield* DateTime.now;
4683+
for (const context of (yield* Ref.get(settledTurns)).values()) {
4684+
if (context.providerThread.nativeThreadRef?.nativeId !== payload.threadId) continue;
4685+
const commands = (yield* Ref.get(runningCommandItemsByTurn)).get(
4686+
context.nativeTurnId,
4687+
);
4688+
if (commands !== undefined) {
4689+
context.closedCommandItemIds = new Set(commands.keys());
4690+
}
4691+
yield* terminalizeRunningCommandItems(
4692+
context,
4693+
context.nativeTurnId,
4694+
"interrupted",
4695+
completedAt,
4696+
);
4697+
yield* Ref.update(runningCommandItemsByTurn, (current) => {
4698+
const updated = new Map(current);
4699+
updated.delete(context.nativeTurnId);
4700+
return updated;
4701+
});
4702+
yield* releaseSettledTurnIfIdle(context.nativeTurnId);
4703+
}
4704+
}).pipe(Effect.orDie, turnTerminalizationPermit.withPermits(1)),
4705+
);
4706+
46774707
yield* client.handleServerNotification("item/started", (payload) =>
46784708
Effect.gen(function* () {
46794709
const context = yield* awaitActiveTurn(payload.turnId);
@@ -4858,6 +4888,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
48584888
}
48594889

48604890
if (payload.item.type === "commandExecution") {
4891+
// Other tools can retain the context after thread closure.
4892+
if (context.closedCommandItemIds?.has(payload.item.id)) return;
48614893
const turnDrained = yield* clearRunningCommandItem(payload.turnId, payload.item.id);
48624894
const artifacts = yield* buildCommandExecutionArtifacts(context, payload.item);
48634895
yield* emitProviderEvent({

0 commit comments

Comments
 (0)