Skip to content

Commit a12404f

Browse files
Alb11747codex
andcommitted
fix(server): persist run failures after storage recovers
Retry guarded failure finalization when ingestion and its terminal write fail. Bound recovery to the service lifetime and preserve run/attempt ownership on every retry. Co-Authored-By: GPT-6.1-Sol at high effort <[email protected]> Co-Authored-By: GPT-6.1-Sol at high effort, implementation and investigation subagents <[email protected]> Co-Authored-By: GPT-6.1-Sol at xhigh effort, independent review subagent <[email protected]>
1 parent 348f217 commit a12404f

2 files changed

Lines changed: 224 additions & 57 deletions

File tree

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 155 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import * as Layer from "effect/Layer";
3636
import * as Option from "effect/Option";
3737
import * as Ref from "effect/Ref";
3838
import * as Stream from "effect/Stream";
39+
import * as TestClock from "effect/testing/TestClock";
3940

4041
import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
4142
import * as ServerSettings from "../serverSettings.ts";
@@ -3814,6 +3815,145 @@ it.effect("keeps completed runs completed when pull request refresh fails", () =
38143815
}),
38153816
);
38163817

3818+
it.effect.each(["typed", "defect"] as const)(
3819+
"settles failed runs after transient %s terminal write failures",
3820+
(failureKind) =>
3821+
Effect.gen(function* () {
3822+
const failedAt = yield* DateTime.now;
3823+
const failedWrite = yield* Deferred.make<void>();
3824+
const writeAttempts = yield* Ref.make(0);
3825+
const { observed, attempts, written } = yield* captureRootRunTermination({
3826+
key: `failure-finalization-retry:${failureKind}`,
3827+
shouldFinalizeRun: () => Effect.succeed(true),
3828+
ingestNormalized: () => Effect.die("ingestion failed"),
3829+
beforeTerminalWrite: () =>
3830+
Effect.gen(function* () {
3831+
const count = yield* Ref.updateAndGet(writeAttempts, (count) => count + 1);
3832+
if (count > 2) return true;
3833+
yield* Deferred.succeed(failedWrite, undefined);
3834+
const error = new EventSink.EventSinkWriteError({ eventCount: 5 });
3835+
return yield* failureKind === "typed" ? Effect.fail(error) : Effect.die(error);
3836+
}),
3837+
afterStart: (closed) =>
3838+
Effect.gen(function* () {
3839+
yield* Deferred.await(failedWrite);
3840+
yield* TestClock.adjust("1 second");
3841+
assert.isFalse(yield* Deferred.isDone(closed));
3842+
assert.equal(yield* Ref.get(writeAttempts), 2);
3843+
yield* TestClock.adjust("2 seconds");
3844+
}),
3845+
});
3846+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3847+
assert.deepEqual(attempts, ["failed"]);
3848+
assert.deepEqual(
3849+
written.map((item) => item.completedAt),
3850+
[failedAt],
3851+
);
3852+
assert.equal(yield* Ref.get(writeAttempts), 3);
3853+
}),
3854+
);
3855+
3856+
it.effect("does not overwrite a superseding attempt when failure finalization retries", () =>
3857+
Effect.gen(function* () {
3858+
const failedWrite = yield* Deferred.make<void>();
3859+
const writeAttempts = yield* Ref.make(0);
3860+
const { observed, attempts, written } = yield* captureRootRunTermination({
3861+
key: "failure-finalization-superseded",
3862+
shouldFinalizeRun: () => Effect.succeed(true),
3863+
ingestNormalized: () => Effect.die("ingestion failed"),
3864+
beforeTerminalWrite: () =>
3865+
Effect.gen(function* () {
3866+
const count = yield* Ref.updateAndGet(writeAttempts, (count) => count + 1);
3867+
if (count > 1) return false;
3868+
yield* Deferred.succeed(failedWrite, undefined);
3869+
return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
3870+
}),
3871+
afterStart: (closed) =>
3872+
Effect.gen(function* () {
3873+
yield* Deferred.await(failedWrite);
3874+
yield* TestClock.adjust("1 second");
3875+
assert.equal(yield* Ref.get(writeAttempts), 2);
3876+
yield* Deferred.await(closed);
3877+
yield* TestClock.adjust("30 seconds");
3878+
}),
3879+
});
3880+
assert.deepEqual(observed, []);
3881+
assert.deepEqual(attempts, []);
3882+
assert.deepEqual(written, []);
3883+
assert.equal(yield* Ref.get(writeAttempts), 2);
3884+
}),
3885+
);
3886+
3887+
it.effect("stops ingestion recovery when a rejected provider start interrupts its consumer", () =>
3888+
Effect.gen(function* () {
3889+
const failedWrite = yield* Deferred.make<void>();
3890+
const writeAttempts = yield* Ref.make(0);
3891+
const { observed, attempts } = yield* captureRootRunTermination({
3892+
key: "failure-finalization-start-rejected",
3893+
shouldFinalizeRun: () => Effect.succeed(true),
3894+
ingestNormalized: () => Effect.die("ingestion failed"),
3895+
beforeTerminalWrite: () =>
3896+
Effect.gen(function* () {
3897+
const count = yield* Ref.updateAndGet(writeAttempts, (count) => count + 1);
3898+
if (count > 1) return true;
3899+
yield* Deferred.succeed(failedWrite, undefined);
3900+
return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
3901+
}),
3902+
startTurn: (input) =>
3903+
Deferred.await(failedWrite).pipe(
3904+
Effect.andThen(
3905+
Effect.fail(
3906+
new ProviderAdapter.ProviderAdapterTurnStartError({
3907+
driver,
3908+
threadId: input.threadId,
3909+
providerThreadId: input.providerThread.id,
3910+
runId: input.runId,
3911+
cause: "provider rejected the turn",
3912+
}),
3913+
),
3914+
),
3915+
),
3916+
afterStart: (closed) =>
3917+
Effect.gen(function* () {
3918+
yield* Deferred.await(closed);
3919+
// The service is still open here; only the consumer was interrupted.
3920+
yield* TestClock.adjust("30 seconds");
3921+
assert.equal(yield* Ref.get(writeAttempts), 2);
3922+
}),
3923+
});
3924+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3925+
assert.deepEqual(attempts, ["failed"]);
3926+
}),
3927+
);
3928+
3929+
it.effect("interrupts failure-finalization retries when the service scope closes", () =>
3930+
Effect.gen(function* () {
3931+
const failedWrite = yield* Deferred.make<void>();
3932+
const writeAttempts = yield* Ref.make(0);
3933+
const exit = yield* captureRootRunTermination({
3934+
key: "failure-finalization-shutdown",
3935+
shouldFinalizeRun: () => Effect.succeed(true),
3936+
ingestNormalized: () => Effect.die("ingestion failed"),
3937+
beforeTerminalWrite: () =>
3938+
Ref.update(writeAttempts, (count) => count + 1).pipe(
3939+
Effect.andThen(Deferred.succeed(failedWrite, undefined)),
3940+
Effect.andThen(Effect.fail(new EventSink.EventSinkWriteError({ eventCount: 5 }))),
3941+
),
3942+
afterStart: (closed) =>
3943+
Effect.gen(function* () {
3944+
yield* Deferred.await(failedWrite);
3945+
yield* TestClock.adjust("1 second");
3946+
assert.isFalse(yield* Deferred.isDone(closed));
3947+
assert.equal(yield* Ref.get(writeAttempts), 2);
3948+
return yield* Effect.interrupt;
3949+
}),
3950+
}).pipe(Effect.exit);
3951+
assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause));
3952+
yield* TestClock.adjust("30 seconds");
3953+
assert.equal(yield* Ref.get(writeAttempts), 2);
3954+
}),
3955+
);
3956+
38173957
function captureRootRunTermination(input: {
38183958
readonly key: string;
38193959
readonly shouldFinalizeRun: () => Effect.Effect<boolean, ProjectionStore.ProjectionStoreV2Error>;
@@ -3828,6 +3968,9 @@ function captureRootRunTermination(input: {
38283968
>;
38293969
readonly startTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"];
38303970
readonly refreshAfterTurn?: Effect.Effect<void>;
3971+
readonly ingestNormalized?: ProviderEventIngestor.ProviderEventIngestorV2["Service"]["ingestNormalized"];
3972+
readonly beforeTerminalWrite?: () => Effect.Effect<boolean, EventSink.EventSinkV2Error>;
3973+
readonly afterStart?: (closed: Deferred.Deferred<void>) => Effect.Effect<void>;
38313974
}) {
38323975
return Effect.gen(function* () {
38333976
const ids = backgroundScenarioIds(input.key);
@@ -3841,6 +3984,7 @@ function captureRootRunTermination(input: {
38413984
});
38423985
const writtenItems = yield* Ref.make<ReadonlyArray<OrchestrationV2TurnItem>>([]);
38433986
const observed = yield* Ref.make<ReadonlyArray<string>>([]);
3987+
const attempts = yield* Ref.make<ReadonlyArray<OrchestrationV2RunAttempt["status"]>>([]);
38443988
const submittedEffects = yield* Ref.make<ReadonlyArray<PendingOrchestrationEffectV2>>([]);
38453989
const committedEffects = yield* Ref.make<ReadonlyArray<PendingOrchestrationEffectV2>>([]);
38463990
const ingestionDone = yield* Deferred.make<void>();
@@ -3855,6 +3999,9 @@ function captureRootRunTermination(input: {
38553999
if (event.type === "run.updated") {
38564000
yield* Ref.update(observed, (current) => [...current, `run:${event.payload.status}`]);
38574001
}
4002+
if (event.type === "run-attempt.updated") {
4003+
yield* Ref.update(attempts, (current) => [...current, event.payload.status]);
4004+
}
38584005
}
38594006
});
38604007
const layerTest = RunExecutionService.layer.pipe(
@@ -3883,7 +4030,10 @@ function captureRootRunTermination(input: {
38834030
Effect.gen(function* () {
38844031
const effects = payload.effects ?? [];
38854032
yield* Ref.update(submittedEffects, (current) => [...current, ...effects]);
3886-
if (input.rejectTerminalWrite === true) {
4033+
if (
4034+
input.rejectTerminalWrite === true ||
4035+
(input.beforeTerminalWrite !== undefined && !(yield* input.beforeTerminalWrite()))
4036+
) {
38874037
return { committed: false, storedEvents: [] };
38884038
}
38894039
yield* Ref.update(committedEffects, (current) => [...current, ...effects]);
@@ -3893,7 +4043,7 @@ function captureRootRunTermination(input: {
38934043
}),
38944044
IdAllocator.layer,
38954045
Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
3896-
ingestNormalized: () => Effect.succeed([]),
4046+
ingestNormalized: input.ingestNormalized ?? (() => Effect.succeed([])),
38974047
}),
38984048
ServerSettings.layerTest(),
38994049
Layer.succeed(RunFinalizationService.RunFinalizationObserver, {
@@ -3995,12 +4145,14 @@ function captureRootRunTermination(input: {
39954145
},
39964146
},
39974147
});
4148+
yield* input.afterStart?.(ingestionDone) ?? Effect.void;
4149+
yield* Deferred.await(ingestionDone);
39984150
}).pipe(Effect.provide(layerTest));
39994151

4000-
yield* Deferred.await(ingestionDone);
40014152
return {
40024153
written: yield* Ref.get(writtenItems),
40034154
observed: yield* Ref.get(observed),
4155+
attempts: yield* Ref.get(attempts),
40044156
submittedEffects: yield* Ref.get(submittedEffects),
40054157
committedEffects: yield* Ref.get(committedEffects),
40064158
};

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 69 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,13 @@ import {
2727
import * as Context from "effect/Context";
2828
import * as Cause from "effect/Cause";
2929
import * as DateTime from "effect/DateTime";
30+
import * as Duration from "effect/Duration";
3031
import * as Effect from "effect/Effect";
3132
import * as Exit from "effect/Exit";
3233
import * as Fiber from "effect/Fiber";
3334
import * as Layer from "effect/Layer";
3435
import * as Ref from "effect/Ref";
36+
import * as Schedule from "effect/Schedule";
3537
import * as Schema from "effect/Schema";
3638
import * as Stream from "effect/Stream";
3739

@@ -572,6 +574,7 @@ export const layer: Layer.Layer<
572574
> = Layer.effect(
573575
RunExecutionServiceV2,
574576
Effect.gen(function* () {
577+
const scope = yield* Effect.scope;
575578
const checkpointService = yield* CheckpointService.CheckpointServiceV2;
576579
const eventSink = yield* EventSink.EventSinkV2;
577580
const idAllocator = yield* IdAllocator.IdAllocatorV2;
@@ -590,6 +593,7 @@ export const layer: Layer.Layer<
590593
readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
591594
readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection;
592595
readonly terminal: ProviderTerminalEvent;
596+
readonly completedAt?: DateTime.Utc;
593597
readonly failureItemPersisted: boolean;
594598
readonly refreshAfterTurn: Effect.Effect<void>;
595599
readonly writeIfRunCurrent?: {
@@ -598,7 +602,7 @@ export const layer: Layer.Layer<
598602
};
599603
}) =>
600604
Effect.gen(function* () {
601-
const completedAt = yield* DateTime.now;
605+
const completedAt = input.completedAt ?? (yield* DateTime.now);
602606
const finalizedAttempt: OrchestrationV2RunAttempt | null = {
603607
...input.attempt,
604608
status: input.terminal.status,
@@ -1322,61 +1326,72 @@ export const layer: Layer.Layer<
13221326
}),
13231327
),
13241328
Effect.catchCause((cause) =>
1325-
Ref.get(rootRunFinalized).pipe(
1326-
Effect.flatMap((finalized) =>
1327-
Effect.logWarning("orchestration V2 provider event ingestion failed", {
1328-
runId: input.run.id,
1329-
cause,
1330-
}).pipe(
1331-
Effect.andThen(
1332-
finalized
1333-
? Effect.void
1334-
: Ref.get(latestProviderThread).pipe(
1335-
Effect.flatMap((providerThread) =>
1336-
Ref.get(latestTurnItemOrdinal).pipe(
1337-
Effect.flatMap((latestItemOrdinal) =>
1338-
Ref.get(openRunOwnedSubagents).pipe(
1339-
Effect.flatMap((openSubagents) =>
1340-
writeFinalRunEvents({
1341-
run: input.run,
1342-
rootNode: input.rootNode,
1343-
checkpointScope: input.checkpointScope,
1344-
providerThread,
1345-
attempt: input.attempt,
1346-
// The failure may be the ownership
1347-
// read itself, so check in the write.
1348-
writeIfRunCurrent: {
1349-
activeAttemptId: input.attempt.id,
1350-
expectedStatus: "running",
1351-
},
1352-
openRunOwnedSubagents: openSubagents,
1353-
terminal: makeFailedTerminalEvent(
1354-
makeProviderFailure({
1355-
cause: Cause.squash(cause),
1356-
class: "unknown",
1357-
}),
1358-
latestItemOrdinal + 1,
1359-
),
1360-
failureItemPersisted: false,
1361-
refreshAfterTurn,
1362-
}),
1363-
),
1364-
),
1365-
),
1366-
),
1367-
),
1368-
),
1329+
Effect.gen(function* () {
1330+
if (Cause.hasInterruptsOnly(cause)) return yield* Effect.failCause(cause);
1331+
yield* Effect.logWarning("orchestration V2 provider event ingestion failed", {
1332+
runId: input.run.id,
1333+
cause,
1334+
});
1335+
if (yield* Ref.get(rootRunFinalized)) return;
1336+
const providerThread = yield* Ref.get(latestProviderThread);
1337+
const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal);
1338+
const openSubagents = yield* Ref.get(openRunOwnedSubagents);
1339+
const completedAt = yield* DateTime.now;
1340+
// Once ingestion fails there will be no later consumer to
1341+
// settle this run. Keep its failure until storage recovers or
1342+
// the atomic ownership gate rejects this attempt.
1343+
yield* writeFinalRunEvents({
1344+
run: input.run,
1345+
rootNode: input.rootNode,
1346+
checkpointScope: input.checkpointScope,
1347+
providerThread,
1348+
attempt: input.attempt,
1349+
// The failure may be the ownership read itself.
1350+
writeIfRunCurrent: {
1351+
activeAttemptId: input.attempt.id,
1352+
expectedStatus: "running",
1353+
},
1354+
openRunOwnedSubagents: openSubagents,
1355+
completedAt,
1356+
terminal: makeFailedTerminalEvent(
1357+
makeProviderFailure({ cause: Cause.squash(cause), class: "unknown" }),
1358+
latestItemOrdinal + 1,
1359+
),
1360+
failureItemPersisted: false,
1361+
refreshAfterTurn,
1362+
}).pipe(
1363+
Effect.exit,
1364+
Effect.flatMap((exit) =>
1365+
Effect.gen(function* () {
1366+
if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) {
1367+
return yield* exit;
1368+
}
1369+
yield* Effect.logWarning("orchestration V2 run failure finalization retry", {
1370+
runId: input.run.id,
1371+
attemptId: input.attempt.id,
1372+
cause: exit.cause,
1373+
});
1374+
return yield* Effect.fail(exit.cause);
1375+
}),
1376+
),
1377+
// SQL commits can fail as defects, not only typed errors.
1378+
Effect.retry({
1379+
schedule: Schedule.exponential("1 second").pipe(
1380+
Schedule.modifyDelay(({ duration }) =>
1381+
Effect.succeed(Duration.min(duration, Duration.seconds(30))),
1382+
),
13691383
),
1370-
Effect.mapError(
1371-
(writeCause) =>
1372-
new RunExecutionIngestError({
1373-
runId: input.run.id,
1374-
cause: { ingest: cause, write: writeCause },
1375-
}),
1384+
}),
1385+
// Recovery cannot outlive service shutdown or a consumer
1386+
// interruption while the provider turn is being started.
1387+
(recovery) =>
1388+
Effect.acquireUseRelease(
1389+
Effect.forkIn(recovery, scope),
1390+
Fiber.join,
1391+
Fiber.interrupt,
13761392
),
1377-
),
1378-
),
1379-
),
1393+
);
1394+
}),
13801395
),
13811396
Effect.ensuring(eventSubscription.close),
13821397
Effect.forkDetach,

0 commit comments

Comments
 (0)