Skip to content

Commit 4a3dab0

Browse files
fix(server): a send refused for background work no longer holds the queue
A queued send refused because background work was still running was classed as an unknown provider failure, so the next queued message held the whole queue and nothing released it. The refusal is now a validation failure: only that send fails and the queue keeps going. Mobile settings command ids are now random per attempt, so a remount no longer replays ids the server already refused. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent c36d0a1 commit 4a3dab0

3 files changed

Lines changed: 152 additions & 20 deletions

File tree

‎apps/mobile/src/state/use-thread-outbox-drain.ts‎

Lines changed: 6 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -184,13 +184,10 @@ function findCreationProject(
184184
}
185185

186186
// A refused settings change stays refused under its command id, so each retry
187-
// needs its own: the server can refuse one while background work runs.
188-
function settingsCommandId(
189-
message: QueuedThreadMessage,
190-
setting: string,
191-
attempt: number,
192-
): CommandId {
193-
return CommandId.make(`${message.commandId}:${setting}:${attempt}`);
187+
// needs its own, even after a remount: the server can refuse one while
188+
// background work runs.
189+
function settingsCommandId(message: QueuedThreadMessage, setting: string): CommandId {
190+
return CommandId.make(`${message.commandId}:${setting}:${randomHex(8)}`);
194191
}
195192

196193
export async function prepareQueuedMessageAttachments(
@@ -764,11 +761,7 @@ export function useThreadOutboxDrain(): void {
764761
const runtimeResult = await setThreadRuntimeMode({
765762
environmentId: queuedMessage.environmentId,
766763
input: {
767-
commandId: settingsCommandId(
768-
queuedMessage,
769-
"runtime-mode",
770-
retryAttemptRef.current.get(queuedMessage.messageId) ?? 0,
771-
),
764+
commandId: settingsCommandId(queuedMessage, "runtime-mode"),
772765
threadId: queuedMessage.threadId,
773766
runtimeMode: settings.runtimeMode,
774767
createdAt: queuedMessage.createdAt,
@@ -785,11 +778,7 @@ export function useThreadOutboxDrain(): void {
785778
const interactionResult = await setThreadInteractionMode({
786779
environmentId: queuedMessage.environmentId,
787780
input: {
788-
commandId: settingsCommandId(
789-
queuedMessage,
790-
"interaction-mode",
791-
retryAttemptRef.current.get(queuedMessage.messageId) ?? 0,
792-
),
781+
commandId: settingsCommandId(queuedMessage, "interaction-mode"),
793782
threadId: queuedMessage.threadId,
794783
interactionMode: settings.interactionMode,
795784
createdAt: queuedMessage.createdAt,

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -917,6 +917,9 @@ function lastDeliveredRunForProviderThread(
917917
);
918918
}
919919

920+
const BACKGROUND_WORK_REFUSAL =
921+
"Background work is still running in this thread, and this change would end it. Try again once it finishes, or stop the thread first.";
922+
920923
const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(function* () {
921924
const checkpointService = yield* CheckpointServiceV2;
922925
const commandPolicy = yield* CommandPolicyV2;
@@ -1335,7 +1338,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13351338
// Orchestrator refusals carry their user-facing reason as text.
13361339
...(typeof failureCause === "string" ? { message: failureCause } : {}),
13371340
code: handoffUnsupported ? "context_handoff_unsupported" : "queued_start_failed",
1338-
class: handoffUnsupported ? "validation_error" : "unknown",
1341+
// Background work refuses only this start; the queue behind it may run.
1342+
class:
1343+
handoffUnsupported || failureCause === BACKGROUND_WORK_REFUSAL
1344+
? "validation_error"
1345+
: "unknown",
13391346
});
13401347
yield* writeSystemEvents([
13411348
...(attempt !== undefined && rootNode !== undefined
@@ -2707,8 +2714,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
27072714
new OrchestratorDispatchError({
27082715
commandId: command.commandId,
27092716
commandType: command.type,
2710-
cause:
2711-
"Background work is still running in this thread, and this change would end it. Try again once it finishes, or stop the thread first.",
2717+
cause: BACKGROUND_WORK_REFUSAL,
27122718
});
27132719

27142720
const dispatchThreadMutation = Effect.fn("orchestrationV2.dispatch.threadMutation")(function* (

‎apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1002,3 +1002,140 @@ it.live.each(
10021002
}),
10031003
),
10041004
);
1005+
1006+
it.live("a refused queued send fails alone, and the queue behind it keeps running", () =>
1007+
Effect.scoped(
1008+
Effect.gen(function* () {
1009+
const name = "background-work-queue";
1010+
const cwd = yield* checkpointWorkspace(name);
1011+
const threadId = ThreadId.make(`thread:${name}`);
1012+
const targetId = ProviderInstanceId.make("codex-background-target");
1013+
const state = yield* Ref.make<RestartAdapterState>({
1014+
activeTurn: null,
1015+
opened: [],
1016+
started: [],
1017+
closedSessionCount: 0,
1018+
failedReplacementOpen: true,
1019+
});
1020+
const sessions: Array<ProviderAdapter.ProviderAdapterV2SessionRuntime> = [];
1021+
const adapters = [providerInstanceId, targetId].map((instanceId) => {
1022+
const base = makeRestartAdapter(state, exclusiveCapabilities, instanceId);
1023+
return {
1024+
...base,
1025+
openSession: (input) =>
1026+
base.openSession(input).pipe(
1027+
Effect.map((session) => {
1028+
const runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime = {
1029+
...session,
1030+
backgroundTasksEndWithProcess: true,
1031+
hasPendingBackgroundWork: Effect.succeed(true),
1032+
hasPendingBackgroundWorkForThread: () => Effect.succeed(false),
1033+
};
1034+
sessions.push(runtime);
1035+
return runtime;
1036+
}),
1037+
),
1038+
} satisfies ProviderAdapter.ProviderAdapterV2["Service"];
1039+
});
1040+
const layerRegistry = Layer.succeed(ProviderAdapterRegistry.ProviderAdapterRegistryV2, {
1041+
get: (instanceId) =>
1042+
Effect.succeed(adapters.find((adapter) => adapter.instanceId === instanceId)!),
1043+
list: () => Effect.succeed([providerInstanceId, targetId]),
1044+
getMetadata: () =>
1045+
Effect.succeed({
1046+
driver,
1047+
continuationKey: "codex:home:/shared",
1048+
enabled: true,
1049+
capabilities: exclusiveCapabilities,
1050+
}),
1051+
});
1052+
yield* Effect.gen(function* () {
1053+
const orchestrator = yield* Orchestrator.OrchestratorV2;
1054+
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1055+
yield* orchestrator.dispatch({
1056+
type: "thread.create",
1057+
commandId: CommandId.make(`${name}:create`),
1058+
threadId,
1059+
projectId: ProjectId.make(`project:${name}`),
1060+
title: name,
1061+
modelSelection: initialSelection,
1062+
runtimeMode: "full-access",
1063+
interactionMode: "default",
1064+
branch: null,
1065+
worktreePath: cwd,
1066+
createdBy: "user",
1067+
creationSource: "web",
1068+
});
1069+
const send = (
1070+
id: string,
1071+
dispatchMode: { readonly type: "start_immediately" | "queue_after_active" },
1072+
modelSelection?: ModelSelection,
1073+
) =>
1074+
orchestrator.dispatch({
1075+
type: "message.dispatch",
1076+
commandId: CommandId.make(`${name}:${id}`),
1077+
threadId,
1078+
messageId: MessageId.make(`${name}:${id}`),
1079+
text: id,
1080+
attachments: [],
1081+
...(modelSelection === undefined ? {} : { modelSelection }),
1082+
dispatchMode,
1083+
createdBy: "user",
1084+
creationSource: "web",
1085+
});
1086+
// A held queue never starts another turn, so holding ends the wait too.
1087+
const runningTurnOrHeldQueue = () =>
1088+
orchestrator.streamDomainEvents.pipe(
1089+
Stream.filter(
1090+
(event) =>
1091+
(event.type === "provider-turn.updated" && event.payload.status === "running") ||
1092+
(event.type === "run.updated" && event.payload.queueHeld === true),
1093+
),
1094+
Stream.take(1),
1095+
Stream.runDrain,
1096+
Effect.forkScoped,
1097+
);
1098+
const firstRunning = yield* runningTurnOrHeldQueue();
1099+
yield* send("first", { type: "start_immediately" });
1100+
yield* worker.drain();
1101+
yield* Fiber.join(firstRunning);
1102+
yield* worker.drain();
1103+
const before = yield* orchestrator.getThreadProjection(threadId);
1104+
const providerThread = before.providerThreads.find(
1105+
(thread) => thread.id === before.thread.activeProviderThreadId,
1106+
)!;
1107+
1108+
const targetSelection = { instanceId: targetId, model: initialSelection.model };
1109+
yield* send("other-1", { type: "queue_after_active" }, targetSelection);
1110+
yield* send("other-2", { type: "queue_after_active" }, targetSelection);
1111+
yield* send("same", { type: "queue_after_active" });
1112+
yield* worker.drain();
1113+
1114+
// Both sends to the other account are refused; the send that keeps
1115+
// the session starts on it.
1116+
const sameRunning = yield* runningTurnOrHeldQueue();
1117+
yield* sessions[0]!.interruptTurn({ providerThread } as never);
1118+
yield* Fiber.join(sameRunning);
1119+
yield* worker.drain();
1120+
1121+
const after = yield* orchestrator.getThreadProjection(threadId);
1122+
const status = (id: string) =>
1123+
after.runs.find((run) => run.userMessageId === `${name}:${id}`)?.status;
1124+
assert.deepEqual(["other-1", "other-2", "same"].map(status), [
1125+
"failed",
1126+
"failed",
1127+
"running",
1128+
]);
1129+
assert.isFalse(after.runs.some((run) => run.queueHeld === true));
1130+
assert.deepEqual(
1131+
after.providerSessions
1132+
.filter((session) => session.id === providerThread.providerSessionId)
1133+
.map((session) => session.status),
1134+
["ready"],
1135+
);
1136+
assert.equal(sessions.length, 1);
1137+
assert.equal((yield* Ref.get(state)).closedSessionCount, 0);
1138+
}).pipe(Effect.provide(ProviderReplayHarness.layerWithRegistry({ name }, layerRegistry)));
1139+
}),
1140+
),
1141+
);

0 commit comments

Comments
 (0)