Skip to content

Commit 674b0e4

Browse files
committed
fix(orchestrator): Fail the run when a provider-turn.start effect dead-letters
Wire failFromDeadLetter onto the turn-start service and compensate before the outbox fails a start or restart that exhausted retries, matching the v2.1 port.
1 parent 57fca87 commit 674b0e4

4 files changed

Lines changed: 67 additions & 5 deletions

File tree

‎apps/server/src/orchestration-v2/EffectWorker.test.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ function makeExecutorLayer(input: {
130130
});
131131
}
132132
}),
133+
failFromDeadLetter: () => Effect.void,
133134
}),
134135
),
135136
Layer.succeed(
@@ -233,6 +234,7 @@ it.effect("requeues a claim when a pre-execution worker check fails", () =>
233234
const executorLayer = Layer.succeed(
234235
OrchestrationEffectExecutorV2,
235236
OrchestrationEffectExecutorV2.of({
237+
compensateDeadLetter: () => Effect.void,
236238
execute: () => Ref.update(executionCount, (count) => count + 1),
237239
}),
238240
);
@@ -306,6 +308,7 @@ it.effect("arms cancellation before the durable pre-execution check", () =>
306308
const executorLayer = Layer.succeed(
307309
OrchestrationEffectExecutorV2,
308310
OrchestrationEffectExecutorV2.of({
311+
compensateDeadLetter: () => Effect.void,
309312
execute: () =>
310313
Effect.yieldNow.pipe(Effect.andThen(Ref.update(executionCount, (count) => count + 1))),
311314
}),
@@ -374,6 +377,7 @@ it.effect("terminalizes a process-bound claim when success settlement fails", ()
374377
const executorLayer = Layer.succeed(
375378
OrchestrationEffectExecutorV2,
376379
OrchestrationEffectExecutorV2.of({
380+
compensateDeadLetter: () => Effect.void,
377381
execute: () => Ref.update(executionCount, (count) => count + 1),
378382
}),
379383
);
@@ -437,7 +441,10 @@ it.effect("requeues a replay-safe claim when success settlement fails", () =>
437441
});
438442
const executorLayer = Layer.succeed(
439443
OrchestrationEffectExecutorV2,
440-
OrchestrationEffectExecutorV2.of({ execute: () => Effect.void }),
444+
OrchestrationEffectExecutorV2.of({
445+
compensateDeadLetter: () => Effect.void,
446+
execute: () => Effect.void,
447+
}),
441448
);
442449
const workerLayer = effectWorkerLayerWithOptions({ workerId }).pipe(
443450
Layer.provide(Layer.merge(outboxLayer, executorLayer)),
@@ -504,6 +511,7 @@ it.effect("keeps a process-bound executor failure retryable when retry settlemen
504511
const executorLayer = Layer.succeed(
505512
OrchestrationEffectExecutorV2,
506513
OrchestrationEffectExecutorV2.of({
514+
compensateDeadLetter: () => Effect.void,
507515
execute: () =>
508516
Effect.fail(
509517
new OrchestrationEffectExecutionError({
@@ -576,6 +584,7 @@ it.effect("keeps a max-attempt replay-safe failure terminal when fail settlement
576584
const executorLayer = Layer.succeed(
577585
OrchestrationEffectExecutorV2,
578586
OrchestrationEffectExecutorV2.of({
587+
compensateDeadLetter: () => Effect.void,
579588
execute: () =>
580589
Effect.fail(
581590
new OrchestrationEffectExecutionError({

‎apps/server/src/orchestration-v2/EffectWorker.ts‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ export interface OrchestrationEffectExecutorV2Shape {
6666
readonly execute: (
6767
effect: OrchestrationEffectV2,
6868
) => Effect.Effect<void, OrchestrationEffectExecutionError>;
69+
readonly compensateDeadLetter: (
70+
effect: OrchestrationEffectV2,
71+
error: string,
72+
) => Effect.Effect<void>;
6973
}
7074

7175
export class OrchestrationEffectExecutorV2 extends Context.Service<
@@ -312,6 +316,29 @@ export const executorLayer: Layer.Layer<
312316
);
313317
}
314318
},
319+
compensateDeadLetter: (effect, error) => {
320+
switch (effect.request.type) {
321+
case "provider-turn.start":
322+
case "provider-turn.restart":
323+
return providerTurnStart
324+
.failFromDeadLetter({
325+
threadId: effect.threadId,
326+
runId: effect.request.runId,
327+
error,
328+
})
329+
.pipe(
330+
Effect.catchCause((cause) =>
331+
Effect.logError("Failed to fail the run for a dead-lettered turn start", {
332+
effectId: effect.id,
333+
effectType: effect.request.type,
334+
error: Cause.pretty(cause),
335+
}),
336+
),
337+
);
338+
default:
339+
return Effect.void;
340+
}
341+
},
315342
});
316343
}),
317344
);
@@ -532,9 +559,12 @@ export const layerWithOptions = (
532559
.succeed({ effectId: effect.id, workerId })
533560
.pipe(Effect.onError((cause) => terminalizeClaim(effect, cause)))
534561
: effect.attemptCount >= maxAttempts
535-
? yield* outbox
536-
.fail({ effectId: effect.id, workerId, error })
537-
.pipe(Effect.onError((cause) => terminalizeClaim(effect, cause)))
562+
? yield* Effect.uninterruptible(
563+
executor.compensateDeadLetter(effect, error).pipe(
564+
Effect.andThen(outbox.fail({ effectId: effect.id, workerId, error })),
565+
Effect.onError((cause) => terminalizeClaim(effect, cause)),
566+
),
567+
)
538568
: yield* outbox
539569
.retry({
540570
effectId: effect.id,

‎apps/server/src/orchestration-v2/FoundationPersistence.test.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -703,6 +703,7 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => {
703703
const executorLayer = Layer.succeed(
704704
OrchestrationEffectExecutorV2,
705705
OrchestrationEffectExecutorV2.of({
706+
compensateDeadLetter: () => Effect.void,
706707
execute: () => Ref.update(executionCount, (count) => count + 1),
707708
}),
708709
);
@@ -1173,6 +1174,7 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => {
11731174
const executorLayer = Layer.succeed(
11741175
OrchestrationEffectExecutorV2,
11751176
OrchestrationEffectExecutorV2.of({
1177+
compensateDeadLetter: () => Effect.void,
11761178
execute: () =>
11771179
Deferred.succeed(started, undefined).pipe(
11781180
Effect.andThen(Effect.never),
@@ -1318,7 +1320,10 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => {
13181320
});
13191321
const executorLayer = Layer.succeed(
13201322
OrchestrationEffectExecutorV2,
1321-
OrchestrationEffectExecutorV2.of({ execute: () => Effect.void }),
1323+
OrchestrationEffectExecutorV2.of({
1324+
compensateDeadLetter: () => Effect.void,
1325+
execute: () => Effect.void,
1326+
}),
13221327
);
13231328
const workerLayer = effectWorkerLayerWithOptions({
13241329
workerId: "settlement-race-worker",
@@ -1373,6 +1378,7 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => {
13731378
const executorLayer = Layer.succeed(
13741379
OrchestrationEffectExecutorV2,
13751380
OrchestrationEffectExecutorV2.of({
1381+
compensateDeadLetter: () => Effect.void,
13761382
execute: () => Ref.update(executionCount, (count) => count + 1),
13771383
}),
13781384
);
@@ -1697,6 +1703,7 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => {
16971703
const executorLayer = Layer.succeed(
16981704
OrchestrationEffectExecutorV2,
16991705
OrchestrationEffectExecutorV2.of({
1706+
compensateDeadLetter: () => Effect.void,
17001707
execute: () =>
17011708
Effect.gen(function* () {
17021709
const attempt = yield* Ref.updateAndGet(executions, (count) => count + 1);
@@ -1772,6 +1779,7 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => {
17721779
const executorLayer = Layer.succeed(
17731780
OrchestrationEffectExecutorV2,
17741781
OrchestrationEffectExecutorV2.of({
1782+
compensateDeadLetter: () => Effect.void,
17751783
execute: (effect) => {
17761784
const gate = gates.get(effect.id);
17771785
if (gate === undefined) return Effect.die(`Missing gate for ${effect.id}`);
@@ -1868,6 +1876,7 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => {
18681876
const executorLayer = Layer.succeed(
18691877
OrchestrationEffectExecutorV2,
18701878
OrchestrationEffectExecutorV2.of({
1879+
compensateDeadLetter: () => Effect.void,
18711880
execute: (effect) =>
18721881
Ref.update(executions, (current) => [...current, effect.id]).pipe(
18731882
Effect.andThen(
@@ -2197,6 +2206,7 @@ it.live("keeps claiming new work after repeated idle periods", () =>
21972206
const executorLayer = Layer.succeed(
21982207
OrchestrationEffectExecutorV2,
21992208
OrchestrationEffectExecutorV2.of({
2209+
compensateDeadLetter: () => Effect.void,
22002210
execute: (effect) => {
22012211
const completion = completed.get(effect.id);
22022212
return completion === undefined

‎apps/server/src/orchestration-v2/ProviderTurnStartService.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,11 @@ export interface ProviderTurnStartServiceV2Shape {
4949
readonly threadId: ThreadId;
5050
readonly runId: RunId;
5151
}) => Effect.Effect<void, ProviderTurnStartError>;
52+
readonly failFromDeadLetter: (input: {
53+
readonly threadId: ThreadId;
54+
readonly runId: RunId;
55+
readonly error: string;
56+
}) => Effect.Effect<void, ProviderTurnStartError>;
5257
}
5358

5459
export class ProviderTurnStartServiceV2 extends Context.Service<
@@ -989,6 +994,14 @@ export const layer: Layer.Layer<
989994
: new ProviderTurnStartError({ runId: input.runId, cause }),
990995
),
991996
),
997+
failFromDeadLetter: (input) =>
998+
failFromDeadLetter(input).pipe(
999+
Effect.mapError((cause) =>
1000+
isProviderTurnStartError(cause)
1001+
? cause
1002+
: new ProviderTurnStartError({ runId: input.runId, cause }),
1003+
),
1004+
),
9921005
});
9931006
}),
9941007
);

0 commit comments

Comments
 (0)