Skip to content

Commit 797f5bf

Browse files
fix(server): a failed turn start no longer leaves the thread stuck starting
The last budgeted attempt of a provider-turn.start or provider-turn.restart effect now fails the run for any failure, and the queue is held for the user to resume. These effects are no longer dead-lettered when the attempt budget runs out: if the run cannot be failed either, the effect keeps retrying on the capped backoff. A failed read in start's post-load recheck is now a start failure instead of "the run moved on". ProviderTurnStartService gains failStartingRun, which reuses the guarded settleRunBeforeStart write. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 9bd1d80 commit 797f5bf

5 files changed

Lines changed: 439 additions & 30 deletions

File tree

‎apps/server/src/orchestration-v2/EffectWorker.test.ts‎

Lines changed: 163 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,9 @@ function layerExecutorFor(input: {
8282
readonly threads?: Partial<ThreadManagementService.ThreadManagementService["Service"]>;
8383
readonly continueAfterRestart?: boolean;
8484
readonly interrupt?: ProviderTurnControlService.ProviderTurnControlServiceV2Shape["interrupt"];
85+
readonly failRunFailure?: boolean;
86+
readonly interruptFailure?: boolean;
87+
readonly startInterrupted?: boolean;
8588
}) {
8689
const record = (event: string) => Ref.update(input.events, (events) => [...events, event]);
8790
const layerDependencies = Layer.mergeAll(
@@ -95,6 +98,19 @@ function layerExecutorFor(input: {
9598
request.replacementProviderSessionId === undefined
9699
? "interrupt"
97100
: `interrupt:${request.replacementProviderSessionId}`,
101+
).pipe(
102+
Effect.andThen(
103+
input.interruptFailure === true
104+
? Effect.fail(
105+
new ProviderTurnControlService.ProviderTurnControlError({
106+
threadId,
107+
operation: "interrupt",
108+
providerTurnId,
109+
cause: "simulated interrupt failure",
110+
}),
111+
)
112+
: Effect.void,
113+
),
98114
),
99115
}),
100116
),
@@ -116,6 +132,7 @@ function layerExecutorFor(input: {
116132
start: () =>
117133
Effect.gen(function* () {
118134
yield* record("start");
135+
if (input.startInterrupted === true) return yield* Effect.interrupt;
119136
if (
120137
input.failFirstStart !== undefined &&
121138
(yield* Ref.getAndSet(input.failFirstStart, false))
@@ -126,6 +143,19 @@ function layerExecutorFor(input: {
126143
});
127144
}
128145
}),
146+
failStartingRun: () =>
147+
record("fail-run").pipe(
148+
Effect.andThen(
149+
input.failRunFailure === true
150+
? Effect.fail(
151+
new ProviderTurnStartService.ProviderTurnStartError({
152+
runId,
153+
cause: "simulated run failure write",
154+
}),
155+
)
156+
: Effect.void,
157+
),
158+
),
129159
}),
130160
),
131161
Layer.succeed(
@@ -775,7 +805,7 @@ it.effect("safely retries after replacement cleanup succeeds and start fails", (
775805

776806
const first = yield* Effect.gen(function* () {
777807
const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
778-
return yield* Effect.exit(executor.execute(effect));
808+
return yield* Effect.exit(executor.execute(effect, { willRetry: true }));
779809
}).pipe(Effect.provide(layer));
780810
assert.isTrue(Exit.isFailure(first));
781811

@@ -837,3 +867,135 @@ it.effect("settles a delegated child once its restart continuation fails for goo
837867
}).pipe(Effect.provide(layer));
838868
}),
839869
);
870+
871+
function startEffect(now: DateTime.Utc): EffectOutbox.OrchestrationEffectV2 {
872+
return {
873+
...restartEffect(now, { type: "detach" }),
874+
id: "effect:start",
875+
request: { type: "provider-turn.start", runId },
876+
};
877+
}
878+
879+
it.effect("fails the run when the last start attempt fails", () =>
880+
Effect.gen(function* () {
881+
const now = yield* DateTime.now;
882+
const events = yield* Ref.make<ReadonlyArray<string>>([]);
883+
const failFirstStart = yield* Ref.make(true);
884+
885+
yield* Effect.gen(function* () {
886+
const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
887+
yield* executor.execute(startEffect(now), { willRetry: false });
888+
}).pipe(Effect.provide(layerExecutorFor({ events, failFirstStart })));
889+
890+
assert.deepEqual(yield* Ref.get(events), ["start", "fail-run"]);
891+
}),
892+
);
893+
894+
it.effect("fails the run when a restart's last attempt cannot interrupt the old turn", () =>
895+
Effect.gen(function* () {
896+
const now = yield* DateTime.now;
897+
const events = yield* Ref.make<ReadonlyArray<string>>([]);
898+
const effect = restartEffect(now, { type: "detach" });
899+
900+
yield* Effect.gen(function* () {
901+
const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
902+
yield* executor.execute(effect, { willRetry: false });
903+
}).pipe(Effect.provide(layerExecutorFor({ events, interruptFailure: true })));
904+
905+
assert.deepEqual(yield* Ref.get(events), ["interrupt", "fail-run"]);
906+
}),
907+
);
908+
909+
it.effect("does not fail the run when its last start attempt is interrupted", () =>
910+
Effect.gen(function* () {
911+
const now = yield* DateTime.now;
912+
const events = yield* Ref.make<ReadonlyArray<string>>([]);
913+
914+
const exit = yield* Effect.gen(function* () {
915+
const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
916+
return yield* Effect.exit(executor.execute(startEffect(now), { willRetry: false }));
917+
}).pipe(Effect.provide(layerExecutorFor({ events, startInterrupted: true })));
918+
919+
assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause));
920+
assert.deepEqual(yield* Ref.get(events), ["start"]);
921+
}),
922+
);
923+
924+
it.effect("returns the start failure when the run cannot be failed either", () =>
925+
Effect.gen(function* () {
926+
const now = yield* DateTime.now;
927+
const events = yield* Ref.make<ReadonlyArray<string>>([]);
928+
const failFirstStart = yield* Ref.make(true);
929+
const effect = restartEffect(now, { type: "detach" });
930+
931+
const error = yield* Effect.gen(function* () {
932+
const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
933+
return yield* executor.execute(effect, { willRetry: false }).pipe(Effect.flip);
934+
}).pipe(Effect.provide(layerExecutorFor({ events, failFirstStart, failRunFailure: true })));
935+
936+
assert.deepEqual(yield* Ref.get(events), ["interrupt", "detach", "start", "fail-run"]);
937+
assert.include(Cause.pretty(Cause.fail(error)), "simulated first start failure");
938+
}),
939+
);
940+
941+
it.effect.each(["provider-turn.start", "provider-turn.restart"] as const)(
942+
"keeps retrying a %s effect past its attempt budget",
943+
(effectType) =>
944+
Effect.gen(function* () {
945+
const now = yield* DateTime.now;
946+
const workerId = "worker-run-start-past-budget";
947+
const claimedEffect: EffectOutbox.OrchestrationEffectV2 = {
948+
...restartEffect(now, { type: "detach" }),
949+
...(effectType === "provider-turn.start"
950+
? { request: { type: "provider-turn.start" as const, runId } }
951+
: {}),
952+
attemptCount: 5,
953+
leaseOwner: workerId,
954+
};
955+
const retryDelays = yield* Ref.make<ReadonlyArray<number>>([]);
956+
const terminalizations = yield* Ref.make(0);
957+
const willRetry = yield* Ref.make<ReadonlyArray<boolean | undefined>>([]);
958+
const outboxLayer = Layer.mock(EffectOutbox.EffectOutboxV2)({
959+
claimNext: () => Effect.succeed(Option.some(claimedEffect)),
960+
get: () => Effect.succeed(Option.some(claimedEffect)),
961+
awaitCancellation: () => Effect.never,
962+
clearCancellation: () => Effect.void,
963+
retry: ({ delayMs }) =>
964+
Ref.update(retryDelays, (delays) => [...delays, delayMs]).pipe(Effect.as(true)),
965+
fail: () => Ref.update(terminalizations, (count) => count + 1).pipe(Effect.as(true)),
966+
});
967+
const executorLayer = Layer.succeed(
968+
EffectWorker.OrchestrationEffectExecutorV2,
969+
EffectWorker.OrchestrationEffectExecutorV2.of({
970+
execute: (_effect, options) =>
971+
Ref.update(willRetry, (seen) => [...seen, options?.willRetry]).pipe(
972+
Effect.andThen(
973+
Effect.fail(
974+
new EffectWorker.OrchestrationEffectExecutionError({
975+
effectId: claimedEffect.id,
976+
effectType,
977+
cause: "simulated storage outage",
978+
}),
979+
),
980+
),
981+
),
982+
}),
983+
);
984+
const workerLayer = EffectWorker.layerWithOptions({ workerId, maxAttempts: 5 }).pipe(
985+
Layer.provide(Layer.merge(outboxLayer, executorLayer)),
986+
);
987+
988+
assert.isTrue(
989+
yield* EffectWorker.OrchestrationEffectWorkerV2.pipe(
990+
Effect.flatMap((worker) => worker.runOnce),
991+
Effect.provide(workerLayer),
992+
),
993+
);
994+
995+
// Each attempt past the budget is a last attempt: it starts the run or
996+
// fails it, and the effect itself is never abandoned.
997+
assert.deepEqual(yield* Ref.get(willRetry), [false]);
998+
assert.deepEqual(yield* Ref.get(retryDelays), [1_600]);
999+
assert.equal(yield* Ref.get(terminalizations), 0);
1000+
}),
1001+
);

‎apps/server/src/orchestration-v2/EffectWorker.ts‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { CommandId } from "@t3tools/contracts";
1+
import { CommandId, type RunId } from "@t3tools/contracts";
22
import * as Cause from "effect/Cause";
33
import * as Context from "effect/Context";
44
import * as DateTime from "effect/DateTime";
@@ -62,6 +62,15 @@ export function isNonRetryableProviderTurnControlFailure(
6262
);
6363
}
6464

65+
/**
66+
* Effects that move a run out of `starting`. Giving up on one would leave its
67+
* run `starting` until the user stops it or the server restarts, so the worker
68+
* keeps retrying it past the attempt budget until the run starts, fails, or is
69+
* interrupted.
70+
*/
71+
const ownsStartingRun = (effectType: EffectOutbox.OrchestrationEffectV2["request"]["type"]) =>
72+
effectType === "provider-turn.start" || effectType === "provider-turn.restart";
73+
6574
export interface OrchestrationEffectExecutorV2Shape {
6675
/**
6776
* Runs one claimed effect. `willRetry` is true when the worker will retry a
@@ -104,6 +113,36 @@ export const layerExecutor: Layer.Layer<
104113
yield* ThreadTitleRegenerationService.ThreadTitleRegenerationService;
105114
const threads = yield* ThreadManagementService.ThreadManagementService;
106115
const settings = yield* ServerSettings.ServerSettingsService;
116+
// The last attempt of a start effect fails the run instead of leaving it
117+
// `starting`. If the run cannot be failed either, the start failure goes
118+
// back to the worker, which keeps retrying.
119+
const failRunOnLastAttempt =
120+
(effect: EffectOutbox.OrchestrationEffectV2, runId: RunId, willRetry: boolean) =>
121+
<E, R>(start: Effect.Effect<void, E, R>) =>
122+
willRetry
123+
? start
124+
: start.pipe(
125+
Effect.catchCauseIf(
126+
(cause) => !Cause.hasInterruptsOnly(cause),
127+
(cause) =>
128+
Effect.logWarning(
129+
"Last run start attempt failed",
130+
{ effectId: effect.id, effectType: effect.request.type, runId },
131+
cause,
132+
).pipe(
133+
Effect.andThen(
134+
providerTurnStart.failStartingRun({ threadId: effect.threadId, runId }),
135+
),
136+
Effect.catchCause((failRunCause) =>
137+
Effect.logWarning(
138+
"Could not fail run after its last start attempt",
139+
{ effectId: effect.id, runId },
140+
failRunCause,
141+
).pipe(Effect.andThen(Effect.failCause(cause))),
142+
),
143+
),
144+
),
145+
);
107146
return OrchestrationEffectExecutorV2.of({
108147
execute: (effect, options) => {
109148
const willRetry = options?.willRetry ?? false;
@@ -153,6 +192,7 @@ export const layerExecutor: Layer.Layer<
153192
return providerTurnStart
154193
.start({ threadId: effect.threadId, runId: effect.request.runId, willRetry })
155194
.pipe(
195+
failRunOnLastAttempt(effect, effect.request.runId, willRetry),
156196
Effect.mapError(
157197
(cause) =>
158198
new OrchestrationEffectExecutionError({
@@ -335,6 +375,7 @@ export const layerExecutor: Layer.Layer<
335375
willRetry,
336376
}),
337377
),
378+
failRunOnLastAttempt(effect, effect.request.runId, willRetry),
338379
Effect.mapError(
339380
(cause) =>
340381
new OrchestrationEffectExecutionError({
@@ -717,7 +758,7 @@ export const layerWithOptions = (
717758
? yield* outbox
718759
.succeed({ effectId: effect.id, workerId })
719760
.pipe(Effect.onError((cause) => terminalizeClaim(effect, cause)))
720-
: effect.attemptCount >= maxAttempts
761+
: effect.attemptCount >= maxAttempts && !ownsStartingRun(effect.request.type)
721762
? yield* outbox
722763
.fail({ effectId: effect.id, workerId, error })
723764
.pipe(Effect.onError((cause) => terminalizeClaim(effect, cause)))

‎apps/server/src/orchestration-v2/FoundationPersistence.test.ts‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2570,6 +2570,81 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
25702570
}).pipe(Effect.provide(TestClock.layer())),
25712571
);
25722572

2573+
it.effect("keeps a run-start effect claimable past its attempt budget until it settles", () =>
2574+
Effect.gen(function* () {
2575+
const outbox = yield* EffectOutbox.EffectOutboxV2;
2576+
const effectId = "effect:foundation-run-start-past-budget";
2577+
const executions = yield* Ref.make(0);
2578+
const executorLayer = Layer.succeed(
2579+
EffectWorker.OrchestrationEffectExecutorV2,
2580+
EffectWorker.OrchestrationEffectExecutorV2.of({
2581+
execute: () =>
2582+
Ref.updateAndGet(executions, (count) => count + 1).pipe(
2583+
Effect.flatMap((attempt) =>
2584+
attempt < 3
2585+
? Effect.fail(
2586+
new EffectWorker.OrchestrationEffectExecutionError({
2587+
effectId,
2588+
effectType: "provider-turn.start",
2589+
cause: "simulated storage outage",
2590+
}),
2591+
)
2592+
: Effect.void,
2593+
),
2594+
),
2595+
}),
2596+
);
2597+
const workerLayer = EffectWorker.layerWithOptions({
2598+
workerId: "run-start-past-budget-worker",
2599+
maxAttempts: 1,
2600+
}).pipe(
2601+
Layer.provide(
2602+
Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), executorLayer),
2603+
),
2604+
);
2605+
const storedEffect = outbox
2606+
.get(effectId)
2607+
.pipe(Effect.map(Option.map(({ status, attemptCount }) => ({ status, attemptCount }))));
2608+
2609+
yield* Effect.gen(function* () {
2610+
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2611+
yield* outbox.enqueue([
2612+
{
2613+
id: effectId,
2614+
commandId: CommandId.make("command:foundation-run-start-past-budget"),
2615+
threadId: ThreadId.make("thread:foundation-run-start-past-budget"),
2616+
request: {
2617+
type: "provider-turn.start",
2618+
runId: RunId.make("run:foundation-run-start-past-budget"),
2619+
},
2620+
},
2621+
]);
2622+
2623+
// The only budgeted attempt fails: the row is rescheduled, not failed.
2624+
assert.isTrue(yield* worker.runOnce);
2625+
assert.deepEqual(
2626+
yield* storedEffect,
2627+
Option.some({ status: "pending" as const, attemptCount: 1 }),
2628+
);
2629+
assert.isFalse(yield* worker.runOnce);
2630+
2631+
yield* TestClock.adjust("100 millis");
2632+
assert.isTrue(yield* worker.runOnce);
2633+
assert.deepEqual(
2634+
yield* storedEffect,
2635+
Option.some({ status: "pending" as const, attemptCount: 2 }),
2636+
);
2637+
2638+
yield* TestClock.adjust("200 millis");
2639+
assert.isTrue(yield* worker.runOnce);
2640+
assert.deepEqual(
2641+
yield* storedEffect,
2642+
Option.some({ status: "succeeded" as const, attemptCount: 3 }),
2643+
);
2644+
}).pipe(Effect.provide(workerLayer));
2645+
}).pipe(Effect.provide(TestClock.layer())),
2646+
);
2647+
25732648
it.effect("runs distinct threads concurrently while serializing effects within a thread", () =>
25742649
Effect.gen(function* () {
25752650
const outbox = yield* EffectOutbox.EffectOutboxV2;

0 commit comments

Comments
 (0)