Skip to content

Commit b9ad3da

Browse files
t3dotggclaude
andauthored
fix(server): cap automatic thread continuations and turn them on by default (#18461)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 12b34e8 commit b9ad3da

10 files changed

Lines changed: 270 additions & 38 deletions

File tree

‎apps/server/src/orchestration-v2/FoundationPersistence.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2779,6 +2779,8 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
27792779
{ id: sessionId, driver: "codex", providerInstanceId, status: "running" },
27802780
],
27812781
providerTurns: [{ providerThreadId, runAttemptId: attemptId, status: "running" }],
2782+
runtimeRequests: [],
2783+
turnItems: [],
27822784
} as unknown as OrchestrationV2ThreadProjection;
27832785
const recovery = yield* ProviderRuntimeRecovery.make.pipe(
27842786
Effect.provide(

‎apps/server/src/orchestration-v2/ProjectionRecovery.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -502,7 +502,7 @@ it.effect("includes shared sessions and provider-owned background rosters in rec
502502
preparedState.providerSessions.map((session) => session.id),
503503
[preparedSessionId],
504504
);
505-
assert.equal(restartContinuationRun(preparedState)?.id, preparedRunId);
505+
assert.equal(restartContinuationRun(preparedState, yield* DateTime.now)?.id, preparedRunId);
506506
assert.deepEqual(yield* projections.getUnreadableThreadIds(), []);
507507
const sql = yield* SqlClient.SqlClient;
508508
yield* sql`

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -561,7 +561,7 @@ export const make = Effect.gen(function* () {
561561
}
562562
const continuationRun =
563563
continueAfterRestart && trigger === "startup"
564-
? restartContinuationRun(projection)
564+
? restartContinuationRun(projection, now)
565565
: undefined;
566566
const effects: Array<EffectOutbox.PendingOrchestrationEffectV2> = continuationRun
567567
? [
@@ -697,7 +697,7 @@ export const make = Effect.gen(function* () {
697697
.continueThreadsAfterServerUpdate
698698
)
699699
return;
700-
const run = restartContinuationRun(projection);
700+
const run = restartContinuationRun(projection, yield* DateTime.now);
701701
if (!run) return;
702702
const commandId = CommandId.make(`command:restart-prepare:${run.id}`);
703703
yield* eventSink.writeWithEffects({

‎apps/server/src/orchestration-v2/RestartContinuation.test.ts‎

Lines changed: 151 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { assert, it } from "@effect/vitest";
22
import {
33
MessageId,
4+
NodeId,
45
ProjectId,
56
ProviderDriverKind,
67
ProviderInstanceId,
@@ -9,6 +10,7 @@ import {
910
ProviderTurnId,
1011
RunAttemptId,
1112
RunId,
13+
RuntimeRequestId,
1214
ThreadId,
1315
type OrchestrationV2Command,
1416
type OrchestrationV2ThreadProjection,
@@ -35,6 +37,7 @@ const sessionId = ProviderSessionId.make("session:restart");
3537
const attemptId = RunAttemptId.make("attempt:restart");
3638
// "No project" threads belong to the environment's Scratch project.
3739
const scratchProjectId = ProjectId.make("project:scratch");
40+
const now = DateTime.makeUnsafe("2026-10-03T12:00:00.000Z");
3841

3942
function makeProjection() {
4043
return {
@@ -90,7 +93,7 @@ function makeProjection() {
9093

9194
it("requires matching saved native state for an unfinished root run", () => {
9295
const projection = makeProjection();
93-
assert.equal(restartContinuationRun(projection)?.id, runId);
96+
assert.equal(restartContinuationRun(projection, now)?.id, runId);
9497
for (const invalid of [
9598
{ ...projection, thread: { ...projection.thread, archivedAt: {} } },
9699
{ ...projection, thread: { ...projection.thread, deletedAt: {} } },
@@ -123,7 +126,7 @@ it("requires matching saved native state for an unfinished root run", () => {
123126
"interrupted",
124127
].map((status) => ({ ...projection, runs: [{ ...projection.runs[0]!, status }] })),
125128
])
126-
assert.isUndefined(restartContinuationRun(invalid as OrchestrationV2ThreadProjection));
129+
assert.isUndefined(restartContinuationRun(invalid as OrchestrationV2ThreadProjection, now));
127130
});
128131

129132
it("continues a live turn whose session the adapter never marked running", () => {
@@ -135,9 +138,9 @@ it("continues a live turn whose session the adapter never marked running", () =>
135138
providerSessions: [{ ...projection.providerSessions[0]!, status }],
136139
}) as OrchestrationV2ThreadProjection;
137140
for (const status of ["starting", "ready", "running", "waiting"])
138-
assert.equal(restartContinuationRun(withSessionStatus(status))?.id, runId, status);
141+
assert.equal(restartContinuationRun(withSessionStatus(status), now)?.id, runId, status);
139142
for (const status of ["stopped", "error"])
140-
assert.isUndefined(restartContinuationRun(withSessionStatus(status)), status);
143+
assert.isUndefined(restartContinuationRun(withSessionStatus(status), now), status);
141144
});
142145

143146
it("recovers an admitted continuation after another crash before provider start", () => {
@@ -155,7 +158,7 @@ it("recovers an admitted continuation after another crash before provider start"
155158
providerSessions: [{ ...projection.providerSessions[0]!, status: "stopped" as const }],
156159
providerTurns: [],
157160
};
158-
assert.equal(restartContinuationRun(starting)?.id, runId);
161+
assert.equal(restartContinuationRun(starting, now)?.id, runId);
159162
});
160163

161164
it("does not continue settled root runs with restart-cancelled background work", () => {
@@ -176,11 +179,14 @@ it("does not continue settled root runs with restart-cancelled background work",
176179
for (const projectId of [scratchProjectId, projection.thread.projectId])
177180
for (const status of ["completed", "waiting"] as const)
178181
assert.isUndefined(
179-
restartContinuationRun({
180-
...settled,
181-
thread: { ...settled.thread, projectId },
182-
runs: [{ ...settled.runs[0]!, status }],
183-
}),
182+
restartContinuationRun(
183+
{
184+
...settled,
185+
thread: { ...settled.thread, projectId },
186+
runs: [{ ...settled.runs[0]!, status }],
187+
},
188+
now,
189+
),
184190
);
185191
});
186192

@@ -663,10 +669,13 @@ it.effect("continues a cut run past queued follow-ups, which stay held", () =>
663669
Effect.gen(function* () {
664670
const live = makeProjection();
665671
assert.equal(
666-
restartContinuationRun({
667-
...live,
668-
runs: [...live.runs, queuedFollowUp],
669-
} as unknown as OrchestrationV2ThreadProjection)?.id,
672+
restartContinuationRun(
673+
{
674+
...live,
675+
runs: [...live.runs, queuedFollowUp],
676+
} as unknown as OrchestrationV2ThreadProjection,
677+
now,
678+
)?.id,
670679
runId,
671680
);
672681
const cut = cutMidTurn();
@@ -692,10 +701,13 @@ it.effect("continues a resumed queued run that ran after an earlier continuation
692701
};
693702
const live = makeProjection();
694703
assert.equal(
695-
restartContinuationRun({
696-
...live,
697-
runs: [...live.runs, finishedContinuation],
698-
} as unknown as OrchestrationV2ThreadProjection)?.id,
704+
restartContinuationRun(
705+
{
706+
...live,
707+
runs: [...live.runs, finishedContinuation],
708+
} as unknown as OrchestrationV2ThreadProjection,
709+
now,
710+
)?.id,
699711
runId,
700712
);
701713
const cut = cutMidTurn({ completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00.000Z") });
@@ -835,3 +847,124 @@ it.effect("prepares later threads' continuations when one thread fails", () =>
835847
);
836848
}),
837849
);
850+
851+
it.effect("stops continuing after three automatic restart continuations in a row", () =>
852+
Effect.gen(function* () {
853+
const first = RunId.make("run:first-continuation");
854+
const second = RunId.make("run:second-continuation");
855+
const cut = cutMidTurn({ restartContinuationOfRunId: second });
856+
const withTwoLinks = {
857+
...cut,
858+
runs: [
859+
...cut.runs,
860+
{ ...cut.runs[0]!, id: second, restartContinuationOfRunId: first },
861+
{ ...cut.runs[0]!, id: first, restartContinuationOfRunId: undefined },
862+
],
863+
} as unknown as OrchestrationV2ThreadProjection;
864+
assert.lengthOf(yield* continuationTexts(withTwoLinks), 1);
865+
866+
const withThreeLinks = {
867+
...withTwoLinks,
868+
runs: withTwoLinks.runs.map((run) =>
869+
run.id === first ? { ...run, restartContinuationOfRunId: RunId.make("run:root") } : run,
870+
),
871+
} as unknown as OrchestrationV2ThreadProjection;
872+
assert.lengthOf(yield* continuationTexts(withThreeLinks), 0);
873+
}),
874+
);
875+
876+
it.effect.each([
877+
["user_input", "pending", "live", false],
878+
["user_input", "expired", "not_resumable", false],
879+
["user_input", "cancelled", "not_resumable", false],
880+
["command", "pending", "live", false],
881+
["command", "expired", "not_resumable", false],
882+
["user_input", "resolved", "live", true],
883+
["command", "cancelled", "live", true],
884+
["user_input", "pending", "message", true],
885+
["dynamic_tool_call", "pending", "live", true],
886+
["auth_refresh", "pending", "live", true],
887+
] as const)(
888+
"a %s request that is %s with %s capability allows continuation: %s",
889+
([kind, status, capability, shouldContinue]) =>
890+
Effect.gen(function* () {
891+
const base = makeProjection();
892+
const runtimeRequests = [
893+
{
894+
id: RuntimeRequestId.make("request:restart"),
895+
nodeId: NodeId.make("node:restart-question"),
896+
providerTurnId: base.providerTurns[0]!.id,
897+
kind,
898+
status,
899+
responseCapability:
900+
capability === "live"
901+
? { type: capability, providerSessionId: sessionId }
902+
: capability === "not_resumable"
903+
? { type: capability, reason: "The server restarted." }
904+
: { type: capability },
905+
},
906+
];
907+
const live = { ...base, runtimeRequests } as unknown as OrchestrationV2ThreadProjection;
908+
assert.equal(restartContinuationRun(live, now) !== undefined, shouldContinue);
909+
const cut = {
910+
...cutMidTurn(),
911+
runtimeRequests,
912+
} as unknown as OrchestrationV2ThreadProjection;
913+
assert.lengthOf(yield* continuationTexts(cut), shouldContinue ? 1 : 0);
914+
}),
915+
);
916+
917+
it.effect("does not wake a snoozed or settled thread, but continues once a snooze elapsed", () =>
918+
Effect.gen(function* () {
919+
const live = makeProjection();
920+
for (const held of [
921+
{ snoozedUntil: DateTime.add(now, { days: 1 }) },
922+
{ settledOverride: "settled" as const },
923+
]) {
924+
const thread = { ...live.thread, ...held };
925+
assert.isUndefined(restartContinuationRun({ ...live, thread }, now));
926+
assert.deepEqual(yield* continuationTexts({ ...cutMidTurn(), thread }), []);
927+
}
928+
const thread = { ...live.thread, snoozedUntil: DateTime.subtract(now, { minutes: 1 }) };
929+
assert.equal(restartContinuationRun({ ...live, thread }, now)?.id, runId);
930+
}),
931+
);
932+
933+
it.effect("does not continue a run waiting on a secret the restart closed", () =>
934+
Effect.gen(function* () {
935+
const completedAt = DateTime.makeUnsafe("2026-10-03T10:05:00.000Z");
936+
const secret = (secretStatus: string, at: DateTime.Utc | null) =>
937+
({ type: "secret_request", runId, secretStatus, completedAt: at }) as never;
938+
assert.isUndefined(
939+
restartContinuationRun({ ...makeProjection(), turnItems: [secret("pending", null)] }, now),
940+
);
941+
const cut = cutMidTurn({ completedAt });
942+
assert.deepEqual(
943+
yield* continuationTexts({ ...cut, turnItems: [secret("cancelled", completedAt)] }),
944+
[],
945+
);
946+
// A secret the user dismissed earlier in the run does not hold it.
947+
assert.lengthOf(
948+
yield* continuationTexts({
949+
...cut,
950+
turnItems: [secret("cancelled", DateTime.subtract(completedAt, { seconds: 1 }))],
951+
}),
952+
1,
953+
);
954+
}),
955+
);
956+
957+
it.effect.each(["completed", "failed", "interrupted"] as const)(
958+
"does not continue when the provider turn %s before the app run settled",
959+
(status) =>
960+
Effect.gen(function* () {
961+
const cut = cutMidTurn();
962+
assert.deepEqual(
963+
yield* continuationTexts({
964+
...cut,
965+
providerTurns: [{ ...cut.providerTurns[0]!, status }],
966+
}),
967+
[],
968+
);
969+
}),
970+
);

0 commit comments

Comments
 (0)