Skip to content

Commit 1ce67d2

Browse files
Alb11747codexclaude
committed
fix(server): persist run failures after storage recovers
Retry guarded failure finalization when ingestion and its terminal write fail. Bind recovery to the service lifetime so a rejected startup cannot abandon it, preserve run/attempt ownership on every retry, and throttle persistent failure warnings. Upstream-PR: pingdotgg#19646 Co-Authored-By: GPT-6.1-Sol at high effort <[email protected]> Co-Authored-By: GPT-6.1-Sol at high effort, implementation and investigation subagents <[email protected]> Co-Authored-By: GPT-6.1-Sol at high effort, startup recovery review subagent <[email protected]> Co-Authored-By: GPT-6.1-Sol at xhigh effort, independent review subagents <[email protected]> Co-Authored-By: Claude Opus 5.5, independent investigation and sanity review <[email protected]>
1 parent 3c866c9 commit 1ce67d2

2 files changed

Lines changed: 321 additions & 60 deletions

File tree

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 246 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,11 @@ import * as DateTime from "effect/DateTime";
3333
import * as Deferred from "effect/Deferred";
3434
import * as Effect from "effect/Effect";
3535
import * as Layer from "effect/Layer";
36+
import * as Logger from "effect/Logger";
3637
import * as Option from "effect/Option";
3738
import * as Ref from "effect/Ref";
3839
import * as Stream from "effect/Stream";
40+
import * as TestClock from "effect/testing/TestClock";
3941

4042
import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
4143
import * as ServerSettings from "../serverSettings.ts";
@@ -3849,6 +3851,228 @@ it.effect("keeps completed runs completed when pull request refresh fails", () =
38493851
}),
38503852
);
38513853

3854+
it.effect.each(["typed", "defect"] as const)(
3855+
"settles failed runs after transient %s terminal write failures",
3856+
(failureKind) =>
3857+
Effect.gen(function* () {
3858+
const failedAt = yield* DateTime.now;
3859+
const failedWrite = yield* Deferred.make<void>();
3860+
const writeAttempts = yield* Ref.make(0);
3861+
const { observed, attempts, written } = yield* captureRootRunTermination({
3862+
key: `failure-finalization-retry:${failureKind}`,
3863+
shouldFinalizeRun: () => Effect.succeed(true),
3864+
ingestNormalized: () => Effect.die("ingestion failed"),
3865+
beforeTerminalWrite: () =>
3866+
Effect.gen(function* () {
3867+
const count = yield* Ref.updateAndGet(writeAttempts, (count) => count + 1);
3868+
if (count > 2) return true;
3869+
yield* Deferred.succeed(failedWrite, undefined);
3870+
const error = new EventSink.EventSinkWriteError({ eventCount: 5 });
3871+
return yield* failureKind === "typed" ? Effect.fail(error) : Effect.die(error);
3872+
}),
3873+
afterStart: (closed) =>
3874+
Effect.gen(function* () {
3875+
yield* Deferred.await(failedWrite);
3876+
yield* TestClock.adjust("1 second");
3877+
assert.isFalse(yield* Deferred.isDone(closed));
3878+
assert.equal(yield* Ref.get(writeAttempts), 2);
3879+
yield* TestClock.adjust("2 seconds");
3880+
}),
3881+
});
3882+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3883+
assert.deepEqual(attempts, ["failed"]);
3884+
assert.deepEqual(
3885+
written.map((item) => item.completedAt),
3886+
[failedAt],
3887+
);
3888+
assert.equal(yield* Ref.get(writeAttempts), 3);
3889+
}),
3890+
);
3891+
3892+
it.effect("throttles failure-finalization warnings without slowing recovery", () => {
3893+
const warnings: Array<unknown> = [];
3894+
const logger = Logger.make(({ message }) => {
3895+
const text = Array.isArray(message) ? message[0] : message;
3896+
if (text === "orchestration V2 run failure finalization retry") warnings.push(text);
3897+
});
3898+
return Effect.gen(function* () {
3899+
const failedWrite = yield* Deferred.make<void>();
3900+
const storageAvailable = yield* Ref.make(false);
3901+
const writeAttempts = yield* Ref.make(0);
3902+
const { observed } = yield* captureRootRunTermination({
3903+
key: "failure-finalization-warning-throttle",
3904+
shouldFinalizeRun: () => Effect.succeed(true),
3905+
ingestNormalized: () => Effect.die("ingestion failed"),
3906+
beforeTerminalWrite: () =>
3907+
Effect.gen(function* () {
3908+
yield* Ref.update(writeAttempts, (count) => count + 1);
3909+
if (yield* Ref.get(storageAvailable)) return true;
3910+
yield* Deferred.succeed(failedWrite, undefined);
3911+
return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
3912+
}),
3913+
afterStart: () =>
3914+
Effect.gen(function* () {
3915+
yield* Deferred.await(failedWrite);
3916+
yield* TestClock.adjust("1 minute");
3917+
assert.isAbove(yield* Ref.get(writeAttempts), 2);
3918+
assert.lengthOf(warnings, 1);
3919+
yield* TestClock.adjust("10 minutes");
3920+
assert.lengthOf(warnings, 2);
3921+
yield* Ref.set(storageAvailable, true);
3922+
yield* TestClock.adjust("30 seconds");
3923+
}),
3924+
});
3925+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3926+
assert.lengthOf(warnings, 2);
3927+
}).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false })));
3928+
});
3929+
3930+
it.effect("does not overwrite a superseding attempt when failure finalization retries", () =>
3931+
Effect.gen(function* () {
3932+
const failedWrite = yield* Deferred.make<void>();
3933+
const writeAttempts = yield* Ref.make(0);
3934+
const { observed, attempts, written } = yield* captureRootRunTermination({
3935+
key: "failure-finalization-superseded",
3936+
shouldFinalizeRun: () => Effect.succeed(true),
3937+
ingestNormalized: () => Effect.die("ingestion failed"),
3938+
beforeTerminalWrite: () =>
3939+
Effect.gen(function* () {
3940+
const count = yield* Ref.updateAndGet(writeAttempts, (count) => count + 1);
3941+
if (count > 1) return false;
3942+
yield* Deferred.succeed(failedWrite, undefined);
3943+
return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
3944+
}),
3945+
afterStart: (closed) =>
3946+
Effect.gen(function* () {
3947+
yield* Deferred.await(failedWrite);
3948+
yield* TestClock.adjust("1 second");
3949+
assert.equal(yield* Ref.get(writeAttempts), 2);
3950+
yield* Deferred.await(closed);
3951+
yield* TestClock.adjust("30 seconds");
3952+
}),
3953+
});
3954+
assert.deepEqual(observed, []);
3955+
assert.deepEqual(attempts, []);
3956+
assert.deepEqual(written, []);
3957+
assert.equal(yield* Ref.get(writeAttempts), 2);
3958+
}),
3959+
);
3960+
3961+
it.effect("does not overwrite a startup terminal when its consumer interrupts recovery", () =>
3962+
Effect.gen(function* () {
3963+
const failedWrite = yield* Deferred.make<void>();
3964+
const writeAttempts = yield* Ref.make(0);
3965+
const { observed, attempts } = yield* captureRootRunTermination({
3966+
key: "failure-finalization-start-rejected",
3967+
shouldFinalizeRun: () => Effect.succeed(true),
3968+
ingestNormalized: () => Effect.die("ingestion failed"),
3969+
beforeTerminalWrite: () =>
3970+
Effect.gen(function* () {
3971+
const count = yield* Ref.updateAndGet(writeAttempts, (count) => count + 1);
3972+
if (count > 1) return count === 2;
3973+
yield* Deferred.succeed(failedWrite, undefined);
3974+
return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
3975+
}),
3976+
startTurn: (input) =>
3977+
Deferred.await(failedWrite).pipe(
3978+
Effect.andThen(
3979+
Effect.fail(
3980+
new ProviderAdapter.ProviderAdapterTurnStartError({
3981+
driver,
3982+
threadId: input.threadId,
3983+
providerThreadId: input.providerThread.id,
3984+
runId: input.runId,
3985+
cause: "provider rejected the turn",
3986+
}),
3987+
),
3988+
),
3989+
),
3990+
afterStart: (closed) =>
3991+
Effect.gen(function* () {
3992+
yield* Deferred.await(closed);
3993+
// The service is still open here; only the consumer was interrupted.
3994+
yield* TestClock.adjust("30 seconds");
3995+
assert.equal(yield* Ref.get(writeAttempts), 3);
3996+
}),
3997+
});
3998+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3999+
assert.deepEqual(attempts, ["failed"]);
4000+
}),
4001+
);
4002+
4003+
it.effect("settles ingestion failure after startup finalization also fails", () =>
4004+
Effect.gen(function* () {
4005+
const failedWrite = yield* Deferred.make<void>();
4006+
const writeAttempts = yield* Ref.make(0);
4007+
const { observed, attempts } = yield* captureRootRunTermination({
4008+
key: "failure-finalization-start-write-failed",
4009+
shouldFinalizeRun: () => Effect.succeed(true),
4010+
ingestNormalized: () => Effect.die("ingestion failed"),
4011+
beforeTerminalWrite: () =>
4012+
Effect.gen(function* () {
4013+
const count = yield* Ref.updateAndGet(writeAttempts, (count) => count + 1);
4014+
if (count > 2) return true;
4015+
yield* Deferred.succeed(failedWrite, undefined);
4016+
return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
4017+
}),
4018+
startTurn: (input) =>
4019+
Deferred.await(failedWrite).pipe(
4020+
Effect.andThen(
4021+
Effect.fail(
4022+
new ProviderAdapter.ProviderAdapterTurnStartError({
4023+
driver,
4024+
threadId: input.threadId,
4025+
providerThreadId: input.providerThread.id,
4026+
runId: input.runId,
4027+
cause: "provider rejected the turn",
4028+
}),
4029+
),
4030+
),
4031+
),
4032+
onStartFailure: (error) =>
4033+
Effect.sync(() => assert.instanceOf(error, RunExecutionService.RunExecutionStartError)),
4034+
afterStart: (closed) =>
4035+
Effect.gen(function* () {
4036+
yield* Deferred.await(closed);
4037+
assert.equal(yield* Ref.get(writeAttempts), 2);
4038+
// Startup has failed and its consumer is gone, but the service is open.
4039+
yield* TestClock.adjust("1 second");
4040+
assert.equal(yield* Ref.get(writeAttempts), 3);
4041+
}),
4042+
});
4043+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
4044+
assert.deepEqual(attempts, ["failed"]);
4045+
}),
4046+
);
4047+
4048+
it.effect("interrupts failure-finalization retries when the service scope closes", () =>
4049+
Effect.gen(function* () {
4050+
const failedWrite = yield* Deferred.make<void>();
4051+
const writeAttempts = yield* Ref.make(0);
4052+
const exit = yield* captureRootRunTermination({
4053+
key: "failure-finalization-shutdown",
4054+
shouldFinalizeRun: () => Effect.succeed(true),
4055+
ingestNormalized: () => Effect.die("ingestion failed"),
4056+
beforeTerminalWrite: () =>
4057+
Ref.update(writeAttempts, (count) => count + 1).pipe(
4058+
Effect.andThen(Deferred.succeed(failedWrite, undefined)),
4059+
Effect.andThen(Effect.fail(new EventSink.EventSinkWriteError({ eventCount: 5 }))),
4060+
),
4061+
afterStart: (closed) =>
4062+
Effect.gen(function* () {
4063+
yield* Deferred.await(failedWrite);
4064+
yield* TestClock.adjust("1 second");
4065+
assert.isFalse(yield* Deferred.isDone(closed));
4066+
assert.equal(yield* Ref.get(writeAttempts), 2);
4067+
return yield* Effect.interrupt;
4068+
}),
4069+
}).pipe(Effect.exit);
4070+
assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause));
4071+
yield* TestClock.adjust("30 seconds");
4072+
assert.equal(yield* Ref.get(writeAttempts), 2);
4073+
}),
4074+
);
4075+
38524076
function captureRootRunTermination(input: {
38534077
readonly key: string;
38544078
readonly shouldFinalizeRun: () => Effect.Effect<boolean, ProjectionStore.ProjectionStoreV2Error>;
@@ -3862,9 +4086,15 @@ function captureRootRunTermination(input: {
38624086
ProviderAdapter.ProviderAdapterV2Error
38634087
>;
38644088
readonly startTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"];
4089+
readonly onStartFailure?: (
4090+
error: RunExecutionService.RunExecutionServiceV2Error,
4091+
) => Effect.Effect<void>;
38654092
readonly refreshAfterTurn?: Effect.Effect<void>;
38664093
readonly shouldStartProviderTurn?: RunExecutionService.RunExecutionServiceV2StartRootRunInput["shouldStartProviderTurn"];
38674094
readonly appContext?: ReturnType<McpAppModelContext.McpAppModelContext["Service"]["forThread"]>;
4095+
readonly ingestNormalized?: ProviderEventIngestor.ProviderEventIngestorV2["Service"]["ingestNormalized"];
4096+
readonly beforeTerminalWrite?: () => Effect.Effect<boolean, EventSink.EventSinkV2Error>;
4097+
readonly afterStart?: (closed: Deferred.Deferred<void>) => Effect.Effect<void>;
38684098
}) {
38694099
return Effect.gen(function* () {
38704100
const ids = backgroundScenarioIds(input.key);
@@ -3879,6 +4109,7 @@ function captureRootRunTermination(input: {
38794109
const writtenItems = yield* Ref.make<ReadonlyArray<OrchestrationV2TurnItem>>([]);
38804110
const finalEvents = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
38814111
const observed = yield* Ref.make<ReadonlyArray<string>>([]);
4112+
const attempts = yield* Ref.make<ReadonlyArray<OrchestrationV2RunAttempt["status"]>>([]);
38824113
const submittedEffects = yield* Ref.make<ReadonlyArray<PendingOrchestrationEffectV2>>([]);
38834114
const committedEffects = yield* Ref.make<ReadonlyArray<PendingOrchestrationEffectV2>>([]);
38844115
const ingestionDone = yield* Deferred.make<void>();
@@ -3894,6 +4125,9 @@ function captureRootRunTermination(input: {
38944125
if (event.type === "run.updated") {
38954126
yield* Ref.update(observed, (current) => [...current, `run:${event.payload.status}`]);
38964127
}
4128+
if (event.type === "run-attempt.updated") {
4129+
yield* Ref.update(attempts, (current) => [...current, event.payload.status]);
4130+
}
38974131
}
38984132
});
38994133
const layerTest = RunExecutionService.layer.pipe(
@@ -3925,7 +4159,10 @@ function captureRootRunTermination(input: {
39254159
Effect.gen(function* () {
39264160
const effects = payload.effects ?? [];
39274161
yield* Ref.update(submittedEffects, (current) => [...current, ...effects]);
3928-
if (input.rejectTerminalWrite === true) {
4162+
if (
4163+
input.rejectTerminalWrite === true ||
4164+
(input.beforeTerminalWrite !== undefined && !(yield* input.beforeTerminalWrite()))
4165+
) {
39294166
return { committed: false, storedEvents: [] };
39304167
}
39314168
yield* Ref.update(committedEffects, (current) => [...current, ...effects]);
@@ -3935,7 +4172,7 @@ function captureRootRunTermination(input: {
39354172
}),
39364173
IdAllocator.layer,
39374174
Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
3938-
ingestNormalized: () => Effect.succeed([]),
4175+
ingestNormalized: input.ingestNormalized ?? (() => Effect.succeed([])),
39394176
}),
39404177
ServerSettings.layerTest(),
39414178
Layer.succeed(RunFinalizationService.RunFinalizationObserver, {
@@ -3951,7 +4188,7 @@ function captureRootRunTermination(input: {
39514188

39524189
yield* Effect.gen(function* () {
39534190
const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
3954-
yield* runExecution.startRootRun({
4191+
const start = runExecution.startRootRun({
39554192
commandId: CommandId.make(`command:${input.key}`),
39564193
appThread: { id: ids.threadId } as OrchestrationV2AppThread,
39574194
providerSessionId: ProviderSessionId.make(`session:${input.key}`),
@@ -4040,13 +4277,18 @@ function captureRootRunTermination(input: {
40404277
},
40414278
},
40424279
});
4280+
yield* start.pipe(
4281+
Effect.catch((error) => input.onStartFailure?.(error) ?? Effect.fail(error)),
4282+
);
4283+
yield* input.afterStart?.(ingestionDone) ?? Effect.void;
4284+
yield* Deferred.await(ingestionDone);
40434285
}).pipe(Effect.provide(layerTest));
40444286

4045-
yield* Deferred.await(ingestionDone);
40464287
return {
40474288
written: yield* Ref.get(writtenItems),
40484289
finalEvents: yield* Ref.get(finalEvents),
40494290
observed: yield* Ref.get(observed),
4291+
attempts: yield* Ref.get(attempts),
40504292
submittedEffects: yield* Ref.get(submittedEffects),
40514293
committedEffects: yield* Ref.get(committedEffects),
40524294
};

0 commit comments

Comments
 (0)