Skip to content

Commit 370dc97

Browse files
Alb11747codexclaude
committed
fix(server): persist run failures after storage recovers
Retry guarded failure finalization when ingestion and its terminal write fail. Bind recovery to the service lifetime so a rejected startup cannot abandon it, preserve run/attempt ownership on every retry, and throttle persistent failure warnings. Co-Authored-By: GPT-6.1-Sol at high effort <[email protected]> Co-Authored-By: GPT-6.1-Sol at high effort, implementation and investigation subagents <[email protected]> Co-Authored-By: GPT-6.1-Sol at high effort, startup recovery review subagent <[email protected]> Co-Authored-By: GPT-6.1-Sol at xhigh effort, independent review subagents <[email protected]> Co-Authored-By: Claude Opus 5.5, independent investigation and sanity review <[email protected]>
1 parent 86534db commit 370dc97

2 files changed

Lines changed: 321 additions & 57 deletions

File tree

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 246 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,11 @@ import * as DateTime from "effect/DateTime";
3333
import * as Deferred from "effect/Deferred";
3434
import * as Effect from "effect/Effect";
3535
import * as Layer from "effect/Layer";
36+
import * as Logger from "effect/Logger";
3637
import * as Option from "effect/Option";
3738
import * as Ref from "effect/Ref";
3839
import * as Stream from "effect/Stream";
40+
import * as TestClock from "effect/testing/TestClock";
3941

4042
import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
4143
import * as ServerSettings from "../serverSettings.ts";
@@ -3831,6 +3833,228 @@ it.effect("keeps completed runs completed when pull request refresh fails", () =
38313833
}),
38323834
);
38333835

3836+
it.effect.each(["typed", "defect"] as const)(
3837+
"settles failed runs after transient %s terminal write failures",
3838+
(failureKind) =>
3839+
Effect.gen(function* () {
3840+
const failedAt = yield* DateTime.now;
3841+
const failedWrite = yield* Deferred.make<void>();
3842+
const writeAttempts = yield* Ref.make(0);
3843+
const { observed, attempts, written } = yield* captureRootRunTermination({
3844+
key: `failure-finalization-retry:${failureKind}`,
3845+
shouldFinalizeRun: () => Effect.succeed(true),
3846+
ingestNormalized: () => Effect.die("ingestion failed"),
3847+
beforeTerminalWrite: () =>
3848+
Effect.gen(function* () {
3849+
const count = yield* Ref.updateAndGet(writeAttempts, (count) => count + 1);
3850+
if (count > 2) return true;
3851+
yield* Deferred.succeed(failedWrite, undefined);
3852+
const error = new EventSink.EventSinkWriteError({ eventCount: 5 });
3853+
return yield* failureKind === "typed" ? Effect.fail(error) : Effect.die(error);
3854+
}),
3855+
afterStart: (closed) =>
3856+
Effect.gen(function* () {
3857+
yield* Deferred.await(failedWrite);
3858+
yield* TestClock.adjust("1 second");
3859+
assert.isFalse(yield* Deferred.isDone(closed));
3860+
assert.equal(yield* Ref.get(writeAttempts), 2);
3861+
yield* TestClock.adjust("2 seconds");
3862+
}),
3863+
});
3864+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3865+
assert.deepEqual(attempts, ["failed"]);
3866+
assert.deepEqual(
3867+
written.map((item) => item.completedAt),
3868+
[failedAt],
3869+
);
3870+
assert.equal(yield* Ref.get(writeAttempts), 3);
3871+
}),
3872+
);
3873+
3874+
it.effect("throttles failure-finalization warnings without slowing recovery", () => {
3875+
const warnings: Array<unknown> = [];
3876+
const logger = Logger.make(({ message }) => {
3877+
const text = Array.isArray(message) ? message[0] : message;
3878+
if (text === "orchestration V2 run failure finalization retry") warnings.push(text);
3879+
});
3880+
return Effect.gen(function* () {
3881+
const failedWrite = yield* Deferred.make<void>();
3882+
const storageAvailable = yield* Ref.make(false);
3883+
const writeAttempts = yield* Ref.make(0);
3884+
const { observed } = yield* captureRootRunTermination({
3885+
key: "failure-finalization-warning-throttle",
3886+
shouldFinalizeRun: () => Effect.succeed(true),
3887+
ingestNormalized: () => Effect.die("ingestion failed"),
3888+
beforeTerminalWrite: () =>
3889+
Effect.gen(function* () {
3890+
yield* Ref.update(writeAttempts, (count) => count + 1);
3891+
if (yield* Ref.get(storageAvailable)) return true;
3892+
yield* Deferred.succeed(failedWrite, undefined);
3893+
return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
3894+
}),
3895+
afterStart: () =>
3896+
Effect.gen(function* () {
3897+
yield* Deferred.await(failedWrite);
3898+
yield* TestClock.adjust("1 minute");
3899+
assert.isAbove(yield* Ref.get(writeAttempts), 2);
3900+
assert.lengthOf(warnings, 1);
3901+
yield* TestClock.adjust("10 minutes");
3902+
assert.lengthOf(warnings, 2);
3903+
yield* Ref.set(storageAvailable, true);
3904+
yield* TestClock.adjust("30 seconds");
3905+
}),
3906+
});
3907+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3908+
assert.lengthOf(warnings, 2);
3909+
}).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false })));
3910+
});
3911+
3912+
it.effect("does not overwrite a superseding attempt when failure finalization retries", () =>
3913+
Effect.gen(function* () {
3914+
const failedWrite = yield* Deferred.make<void>();
3915+
const writeAttempts = yield* Ref.make(0);
3916+
const { observed, attempts, written } = yield* captureRootRunTermination({
3917+
key: "failure-finalization-superseded",
3918+
shouldFinalizeRun: () => Effect.succeed(true),
3919+
ingestNormalized: () => Effect.die("ingestion failed"),
3920+
beforeTerminalWrite: () =>
3921+
Effect.gen(function* () {
3922+
const count = yield* Ref.updateAndGet(writeAttempts, (count) => count + 1);
3923+
if (count > 1) return false;
3924+
yield* Deferred.succeed(failedWrite, undefined);
3925+
return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
3926+
}),
3927+
afterStart: (closed) =>
3928+
Effect.gen(function* () {
3929+
yield* Deferred.await(failedWrite);
3930+
yield* TestClock.adjust("1 second");
3931+
assert.equal(yield* Ref.get(writeAttempts), 2);
3932+
yield* Deferred.await(closed);
3933+
yield* TestClock.adjust("30 seconds");
3934+
}),
3935+
});
3936+
assert.deepEqual(observed, []);
3937+
assert.deepEqual(attempts, []);
3938+
assert.deepEqual(written, []);
3939+
assert.equal(yield* Ref.get(writeAttempts), 2);
3940+
}),
3941+
);
3942+
3943+
it.effect("does not overwrite a startup terminal when its consumer interrupts recovery", () =>
3944+
Effect.gen(function* () {
3945+
const failedWrite = yield* Deferred.make<void>();
3946+
const writeAttempts = yield* Ref.make(0);
3947+
const { observed, attempts } = yield* captureRootRunTermination({
3948+
key: "failure-finalization-start-rejected",
3949+
shouldFinalizeRun: () => Effect.succeed(true),
3950+
ingestNormalized: () => Effect.die("ingestion failed"),
3951+
beforeTerminalWrite: () =>
3952+
Effect.gen(function* () {
3953+
const count = yield* Ref.updateAndGet(writeAttempts, (count) => count + 1);
3954+
if (count > 1) return count === 2;
3955+
yield* Deferred.succeed(failedWrite, undefined);
3956+
return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
3957+
}),
3958+
startTurn: (input) =>
3959+
Deferred.await(failedWrite).pipe(
3960+
Effect.andThen(
3961+
Effect.fail(
3962+
new ProviderAdapter.ProviderAdapterTurnStartError({
3963+
driver,
3964+
threadId: input.threadId,
3965+
providerThreadId: input.providerThread.id,
3966+
runId: input.runId,
3967+
cause: "provider rejected the turn",
3968+
}),
3969+
),
3970+
),
3971+
),
3972+
afterStart: (closed) =>
3973+
Effect.gen(function* () {
3974+
yield* Deferred.await(closed);
3975+
// The service is still open here; only the consumer was interrupted.
3976+
yield* TestClock.adjust("30 seconds");
3977+
assert.equal(yield* Ref.get(writeAttempts), 3);
3978+
}),
3979+
});
3980+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3981+
assert.deepEqual(attempts, ["failed"]);
3982+
}),
3983+
);
3984+
3985+
it.effect("settles ingestion failure after startup finalization also fails", () =>
3986+
Effect.gen(function* () {
3987+
const failedWrite = yield* Deferred.make<void>();
3988+
const writeAttempts = yield* Ref.make(0);
3989+
const { observed, attempts } = yield* captureRootRunTermination({
3990+
key: "failure-finalization-start-write-failed",
3991+
shouldFinalizeRun: () => Effect.succeed(true),
3992+
ingestNormalized: () => Effect.die("ingestion failed"),
3993+
beforeTerminalWrite: () =>
3994+
Effect.gen(function* () {
3995+
const count = yield* Ref.updateAndGet(writeAttempts, (count) => count + 1);
3996+
if (count > 2) return true;
3997+
yield* Deferred.succeed(failedWrite, undefined);
3998+
return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
3999+
}),
4000+
startTurn: (input) =>
4001+
Deferred.await(failedWrite).pipe(
4002+
Effect.andThen(
4003+
Effect.fail(
4004+
new ProviderAdapter.ProviderAdapterTurnStartError({
4005+
driver,
4006+
threadId: input.threadId,
4007+
providerThreadId: input.providerThread.id,
4008+
runId: input.runId,
4009+
cause: "provider rejected the turn",
4010+
}),
4011+
),
4012+
),
4013+
),
4014+
onStartFailure: (error) =>
4015+
Effect.sync(() => assert.instanceOf(error, RunExecutionService.RunExecutionStartError)),
4016+
afterStart: (closed) =>
4017+
Effect.gen(function* () {
4018+
yield* Deferred.await(closed);
4019+
assert.equal(yield* Ref.get(writeAttempts), 2);
4020+
// Startup has failed and its consumer is gone, but the service is open.
4021+
yield* TestClock.adjust("1 second");
4022+
assert.equal(yield* Ref.get(writeAttempts), 3);
4023+
}),
4024+
});
4025+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
4026+
assert.deepEqual(attempts, ["failed"]);
4027+
}),
4028+
);
4029+
4030+
it.effect("interrupts failure-finalization retries when the service scope closes", () =>
4031+
Effect.gen(function* () {
4032+
const failedWrite = yield* Deferred.make<void>();
4033+
const writeAttempts = yield* Ref.make(0);
4034+
const exit = yield* captureRootRunTermination({
4035+
key: "failure-finalization-shutdown",
4036+
shouldFinalizeRun: () => Effect.succeed(true),
4037+
ingestNormalized: () => Effect.die("ingestion failed"),
4038+
beforeTerminalWrite: () =>
4039+
Ref.update(writeAttempts, (count) => count + 1).pipe(
4040+
Effect.andThen(Deferred.succeed(failedWrite, undefined)),
4041+
Effect.andThen(Effect.fail(new EventSink.EventSinkWriteError({ eventCount: 5 }))),
4042+
),
4043+
afterStart: (closed) =>
4044+
Effect.gen(function* () {
4045+
yield* Deferred.await(failedWrite);
4046+
yield* TestClock.adjust("1 second");
4047+
assert.isFalse(yield* Deferred.isDone(closed));
4048+
assert.equal(yield* Ref.get(writeAttempts), 2);
4049+
return yield* Effect.interrupt;
4050+
}),
4051+
}).pipe(Effect.exit);
4052+
assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause));
4053+
yield* TestClock.adjust("30 seconds");
4054+
assert.equal(yield* Ref.get(writeAttempts), 2);
4055+
}),
4056+
);
4057+
38344058
function captureRootRunTermination(input: {
38354059
readonly key: string;
38364060
readonly shouldFinalizeRun: () => Effect.Effect<boolean, ProjectionStore.ProjectionStoreV2Error>;
@@ -3844,9 +4068,15 @@ function captureRootRunTermination(input: {
38444068
ProviderAdapter.ProviderAdapterV2Error
38454069
>;
38464070
readonly startTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"];
4071+
readonly onStartFailure?: (
4072+
error: RunExecutionService.RunExecutionServiceV2Error,
4073+
) => Effect.Effect<void>;
38474074
readonly refreshAfterTurn?: Effect.Effect<void>;
38484075
readonly shouldStartProviderTurn?: RunExecutionService.RunExecutionServiceV2StartRootRunInput["shouldStartProviderTurn"];
38494076
readonly appContext?: ReturnType<McpAppModelContext.McpAppModelContext["Service"]["forThread"]>;
4077+
readonly ingestNormalized?: ProviderEventIngestor.ProviderEventIngestorV2["Service"]["ingestNormalized"];
4078+
readonly beforeTerminalWrite?: () => Effect.Effect<boolean, EventSink.EventSinkV2Error>;
4079+
readonly afterStart?: (closed: Deferred.Deferred<void>) => Effect.Effect<void>;
38504080
}) {
38514081
return Effect.gen(function* () {
38524082
const ids = backgroundScenarioIds(input.key);
@@ -3861,6 +4091,7 @@ function captureRootRunTermination(input: {
38614091
const writtenItems = yield* Ref.make<ReadonlyArray<OrchestrationV2TurnItem>>([]);
38624092
const finalEvents = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
38634093
const observed = yield* Ref.make<ReadonlyArray<string>>([]);
4094+
const attempts = yield* Ref.make<ReadonlyArray<OrchestrationV2RunAttempt["status"]>>([]);
38644095
const submittedEffects = yield* Ref.make<ReadonlyArray<PendingOrchestrationEffectV2>>([]);
38654096
const committedEffects = yield* Ref.make<ReadonlyArray<PendingOrchestrationEffectV2>>([]);
38664097
const ingestionDone = yield* Deferred.make<void>();
@@ -3876,6 +4107,9 @@ function captureRootRunTermination(input: {
38764107
if (event.type === "run.updated") {
38774108
yield* Ref.update(observed, (current) => [...current, `run:${event.payload.status}`]);
38784109
}
4110+
if (event.type === "run-attempt.updated") {
4111+
yield* Ref.update(attempts, (current) => [...current, event.payload.status]);
4112+
}
38794113
}
38804114
});
38814115
const layerTest = RunExecutionService.layer.pipe(
@@ -3906,7 +4140,10 @@ function captureRootRunTermination(input: {
39064140
Effect.gen(function* () {
39074141
const effects = payload.effects ?? [];
39084142
yield* Ref.update(submittedEffects, (current) => [...current, ...effects]);
3909-
if (input.rejectTerminalWrite === true) {
4143+
if (
4144+
input.rejectTerminalWrite === true ||
4145+
(input.beforeTerminalWrite !== undefined && !(yield* input.beforeTerminalWrite()))
4146+
) {
39104147
return { committed: false, storedEvents: [] };
39114148
}
39124149
yield* Ref.update(committedEffects, (current) => [...current, ...effects]);
@@ -3916,7 +4153,7 @@ function captureRootRunTermination(input: {
39164153
}),
39174154
IdAllocator.layer,
39184155
Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
3919-
ingestNormalized: () => Effect.succeed([]),
4156+
ingestNormalized: input.ingestNormalized ?? (() => Effect.succeed([])),
39204157
}),
39214158
ServerSettings.layerTest(),
39224159
Layer.succeed(RunFinalizationService.RunFinalizationObserver, {
@@ -3932,7 +4169,7 @@ function captureRootRunTermination(input: {
39324169

39334170
yield* Effect.gen(function* () {
39344171
const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
3935-
yield* runExecution.startRootRun({
4172+
const start = runExecution.startRootRun({
39364173
commandId: CommandId.make(`command:${input.key}`),
39374174
appThread: { id: ids.threadId } as OrchestrationV2AppThread,
39384175
providerSessionId: ProviderSessionId.make(`session:${input.key}`),
@@ -4021,13 +4258,18 @@ function captureRootRunTermination(input: {
40214258
},
40224259
},
40234260
});
4261+
yield* start.pipe(
4262+
Effect.catch((error) => input.onStartFailure?.(error) ?? Effect.fail(error)),
4263+
);
4264+
yield* input.afterStart?.(ingestionDone) ?? Effect.void;
4265+
yield* Deferred.await(ingestionDone);
40244266
}).pipe(Effect.provide(layerTest));
40254267

4026-
yield* Deferred.await(ingestionDone);
40274268
return {
40284269
written: yield* Ref.get(writtenItems),
40294270
finalEvents: yield* Ref.get(finalEvents),
40304271
observed: yield* Ref.get(observed),
4272+
attempts: yield* Ref.get(attempts),
40314273
submittedEffects: yield* Ref.get(submittedEffects),
40324274
committedEffects: yield* Ref.get(committedEffects),
40334275
};

0 commit comments

Comments
 (0)