Skip to content

Commit 801dbf6

Browse files
Alb11747codexclaudejuliusmarminge
authored
fix(server): persist run failures after storage recovers (pingdotgg#19646)
Co-authored-by: GPT-6.1-Sol at high effort <[email protected]> Co-authored-by: Claude Opus 5.5, independent investigation and sanity review <[email protected]> Co-authored-by: Julius Marminge <[email protected]>
1 parent aac2865 commit 801dbf6

2 files changed

Lines changed: 314 additions & 57 deletions

File tree

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 245 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import * as Deferred from "effect/Deferred";
3636
import * as Effect from "effect/Effect";
3737
import * as Fiber from "effect/Fiber";
3838
import * as Layer from "effect/Layer";
39+
import * as Logger from "effect/Logger";
3940
import * as Option from "effect/Option";
4041
import * as Ref from "effect/Ref";
4142
import * as Stream from "effect/Stream";
@@ -4104,6 +4105,230 @@ it.effect("keeps completed runs completed when pull request refresh fails", () =
41044105
}),
41054106
);
41064107

4108+
it.effect(
4109+
"recovers ingestion failure through typed errors and defects with bounded backoff",
4110+
() => {
4111+
const warnings: Array<unknown> = [];
4112+
const logger = Logger.make(({ message }) => {
4113+
const text = Array.isArray(message) ? message[0] : message;
4114+
if (text === "orchestration V2 run failure finalization retry") warnings.push(text);
4115+
});
4116+
return Effect.gen(function* () {
4117+
const failIngestion = yield* Deferred.make<void>();
4118+
const failedWrite = yield* Deferred.make<void>();
4119+
const writeTimes: Array<number> = [];
4120+
let storageAvailable = false;
4121+
let failedAt = yield* DateTime.now;
4122+
const { finalEvents, observed, written } = yield* captureRootRunTermination({
4123+
key: "recovery-backoff",
4124+
shouldFinalizeRun: () => Effect.succeed(true),
4125+
ingestNormalized: () =>
4126+
Deferred.await(failIngestion).pipe(Effect.andThen(Effect.die("ingestion failed"))),
4127+
beforeTerminalWrite: () =>
4128+
Effect.gen(function* () {
4129+
writeTimes.push(DateTime.toEpochMillis(yield* DateTime.now));
4130+
if (storageAvailable) return true;
4131+
yield* Deferred.succeed(failedWrite, undefined);
4132+
const error = new EventSink.EventSinkWriteError({ eventCount: 5 });
4133+
return yield* writeTimes.length % 2 === 1 ? Effect.fail(error) : Effect.die(error);
4134+
}),
4135+
afterStart: () =>
4136+
Effect.gen(function* () {
4137+
// Separate startup, ingestion failure, and eventual storage recovery.
4138+
yield* TestClock.adjust("5 seconds");
4139+
failedAt = yield* DateTime.now;
4140+
yield* Deferred.succeed(failIngestion, undefined);
4141+
yield* Deferred.await(failedWrite);
4142+
yield* TestClock.adjust("999 millis");
4143+
assert.lengthOf(writeTimes, 1);
4144+
yield* TestClock.adjust("1 millis");
4145+
assert.lengthOf(writeTimes, 2);
4146+
assert.lengthOf(warnings, 1);
4147+
yield* TestClock.adjust("598999 millis");
4148+
const offsets = writeTimes.map((time) => time - DateTime.toEpochMillis(failedAt));
4149+
assert.deepEqual(offsets, [
4150+
0,
4151+
1000,
4152+
3000,
4153+
7000,
4154+
15000,
4155+
31000,
4156+
...Array.from({ length: 18 }, (_, index) => 61000 + index * 30000),
4157+
]);
4158+
assert.lengthOf(warnings, 1);
4159+
yield* TestClock.adjust("1001 millis");
4160+
assert.lengthOf(warnings, 2);
4161+
storageAvailable = true;
4162+
yield* TestClock.adjust("30 seconds");
4163+
}),
4164+
});
4165+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
4166+
assert.lengthOf(warnings, 2);
4167+
assert.sameMembers(
4168+
finalEvents.map((event) => event.type),
4169+
[
4170+
"run-attempt.updated",
4171+
"turn-item.updated",
4172+
"run.updated",
4173+
"node.updated",
4174+
"provider-thread.updated",
4175+
],
4176+
);
4177+
for (const event of finalEvents) {
4178+
assert.deepEqual(event.occurredAt, failedAt);
4179+
if (
4180+
event.type === "run.updated" ||
4181+
event.type === "run-attempt.updated" ||
4182+
event.type === "node.updated"
4183+
) {
4184+
assert.equal(event.payload.status, "failed");
4185+
assert.deepEqual(event.payload.completedAt, failedAt);
4186+
}
4187+
if (event.type === "provider-thread.updated")
4188+
assert.deepEqual(event.payload.updatedAt, failedAt);
4189+
}
4190+
assert.lengthOf(written, 1);
4191+
assert.equal(written[0]?.type, "error");
4192+
assert.deepEqual(written[0]?.completedAt, failedAt);
4193+
if (written[0]?.type === "error")
4194+
assert.equal(written[0].failure.message, "Provider turn failed.");
4195+
}).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false })));
4196+
},
4197+
);
4198+
4199+
it.effect("stops recovery when the atomic gate rejects a superseded attempt", () =>
4200+
Effect.gen(function* () {
4201+
const key = "recovery-successor";
4202+
const ids = backgroundScenarioIds(key);
4203+
const failedWrite = yield* Deferred.make<void>();
4204+
let activeAttemptId = ids.attemptId;
4205+
let writes = 0;
4206+
const { finalEvents, observed, written } = yield* captureRootRunTermination({
4207+
key,
4208+
shouldFinalizeRun: () => Effect.succeed(true),
4209+
ingestNormalized: () => Effect.die("ingestion failed"),
4210+
beforeTerminalWrite: (request) =>
4211+
Effect.gen(function* () {
4212+
writes++;
4213+
assert.equal(request.runId, ids.runId);
4214+
assert.equal(request.activeAttemptId, ids.attemptId);
4215+
assert.equal(request.expectedStatus, "running");
4216+
if (writes > 1) return request.activeAttemptId === activeAttemptId;
4217+
yield* Deferred.succeed(failedWrite, undefined);
4218+
return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
4219+
}),
4220+
afterStart: () =>
4221+
Effect.gen(function* () {
4222+
yield* Deferred.await(failedWrite);
4223+
activeAttemptId = RunAttemptId.make("attempt:successor");
4224+
yield* TestClock.adjust("1 second");
4225+
assert.equal(writes, 2);
4226+
yield* TestClock.adjust("10 minutes");
4227+
assert.equal(writes, 2);
4228+
}),
4229+
});
4230+
assert.deepEqual(finalEvents, []);
4231+
assert.deepEqual(written, []);
4232+
assert.deepEqual(observed, []);
4233+
}),
4234+
);
4235+
4236+
it.effect.each([true, false])(
4237+
"keeps recovery alive after startup cancels its consumer (startup terminal persisted: %s)",
4238+
(startupPersisted) =>
4239+
Effect.gen(function* () {
4240+
const failedWrite = yield* Deferred.make<void>();
4241+
let writes = 0;
4242+
let status: OrchestrationV2Run["status"] = "running";
4243+
const { observed, finalEvents, written } = yield* captureRootRunTermination({
4244+
key: `recovery-startup:${startupPersisted}`,
4245+
shouldFinalizeRun: () => Effect.succeed(true),
4246+
ingestNormalized: () => Effect.die("ingestion failed"),
4247+
beforeTerminalWrite: (request) =>
4248+
Effect.gen(function* () {
4249+
writes++;
4250+
if (writes === 1 || (writes === 2 && !startupPersisted)) {
4251+
yield* Deferred.succeed(failedWrite, undefined);
4252+
return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
4253+
}
4254+
if (request.expectedStatus !== status) return false;
4255+
status = "failed";
4256+
return true;
4257+
}),
4258+
startTurn: (input) =>
4259+
Deferred.await(failedWrite).pipe(
4260+
Effect.andThen(
4261+
Effect.fail(
4262+
new ProviderAdapter.ProviderAdapterTurnStartError({
4263+
driver,
4264+
threadId: input.threadId,
4265+
providerThreadId: input.providerThread.id,
4266+
runId: input.runId,
4267+
cause: "provider rejected startup",
4268+
}),
4269+
),
4270+
),
4271+
),
4272+
onStartFailure: (error) =>
4273+
Effect.sync(() => {
4274+
assert.isFalse(startupPersisted);
4275+
assert.instanceOf(error, RunExecutionService.RunExecutionStartError);
4276+
}),
4277+
afterStart: (consumerClosed) =>
4278+
Effect.gen(function* () {
4279+
yield* Deferred.await(consumerClosed);
4280+
assert.equal(writes, 2);
4281+
yield* TestClock.adjust("1 second");
4282+
assert.equal(writes, 3);
4283+
yield* TestClock.adjust("10 minutes");
4284+
assert.equal(writes, 3);
4285+
}),
4286+
});
4287+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
4288+
assert.lengthOf(
4289+
finalEvents.filter(
4290+
(event) => event.type === "run-attempt.updated" && event.payload.status === "failed",
4291+
),
4292+
1,
4293+
);
4294+
assert.lengthOf(written, 1);
4295+
assert.equal(written[0]?.type, "error");
4296+
if (written[0]?.type === "error")
4297+
assert.include(
4298+
written[0].failure.message,
4299+
startupPersisted ? "provider could not start this turn" : "Provider turn failed.",
4300+
);
4301+
}),
4302+
);
4303+
4304+
it.effect("stops pending recovery at service shutdown", () =>
4305+
Effect.gen(function* () {
4306+
const failedWrite = yield* Deferred.make<void>();
4307+
let writes = 0;
4308+
const exit = yield* captureRootRunTermination({
4309+
key: "recovery-shutdown",
4310+
shouldFinalizeRun: () => Effect.succeed(true),
4311+
ingestNormalized: () => Effect.die("ingestion failed"),
4312+
beforeTerminalWrite: () =>
4313+
Effect.gen(function* () {
4314+
writes++;
4315+
yield* Deferred.succeed(failedWrite, undefined);
4316+
return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
4317+
}),
4318+
afterStart: () =>
4319+
Effect.gen(function* () {
4320+
yield* Deferred.await(failedWrite);
4321+
yield* TestClock.adjust("1 second");
4322+
assert.equal(writes, 2);
4323+
return yield* Effect.interrupt;
4324+
}),
4325+
}).pipe(Effect.exit);
4326+
assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause));
4327+
yield* TestClock.adjust("10 minutes");
4328+
assert.equal(writes, 2);
4329+
}),
4330+
);
4331+
41074332
function captureRootRunTermination(input: {
41084333
readonly key: string;
41094334
readonly shouldFinalizeRun: () => Effect.Effect<boolean, ProjectionStore.ProjectionStoreV2Error>;
@@ -4117,10 +4342,18 @@ function captureRootRunTermination(input: {
41174342
ProviderAdapter.ProviderAdapterV2Error
41184343
>;
41194344
readonly startTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"];
4345+
readonly onStartFailure?: (
4346+
error: RunExecutionService.RunExecutionServiceV2Error,
4347+
) => Effect.Effect<void>;
41204348
readonly refreshAfterTurn?: Effect.Effect<void>;
41214349
readonly onSubscriptionClose?: Effect.Effect<void>;
41224350
readonly shouldStartProviderTurn?: RunExecutionService.RunExecutionServiceV2StartRootRunInput["shouldStartProviderTurn"];
41234351
readonly appContext?: ReturnType<McpAppModelContext.McpAppModelContext["Service"]["forThread"]>;
4352+
readonly ingestNormalized?: ProviderEventIngestor.ProviderEventIngestorV2["Service"]["ingestNormalized"];
4353+
readonly beforeTerminalWrite?: (
4354+
request: Parameters<EventSink.EventSinkV2["Service"]["writeIfRunCurrent"]>[0],
4355+
) => Effect.Effect<boolean, EventSink.EventSinkV2Error>;
4356+
readonly afterStart?: (closed: Deferred.Deferred<void>) => Effect.Effect<void>;
41244357
}) {
41254358
return Effect.gen(function* () {
41264359
const ids = backgroundScenarioIds(input.key);
@@ -4181,7 +4414,11 @@ function captureRootRunTermination(input: {
41814414
Effect.gen(function* () {
41824415
const effects = payload.effects ?? [];
41834416
yield* Ref.update(submittedEffects, (current) => [...current, ...effects]);
4184-
if (input.rejectTerminalWrite === true) {
4417+
if (
4418+
input.rejectTerminalWrite === true ||
4419+
(input.beforeTerminalWrite !== undefined &&
4420+
!(yield* input.beforeTerminalWrite(payload)))
4421+
) {
41854422
return { committed: false, storedEvents: [] };
41864423
}
41874424
yield* Ref.update(committedEffects, (current) => [...current, ...effects]);
@@ -4191,7 +4428,7 @@ function captureRootRunTermination(input: {
41914428
}),
41924429
IdAllocator.layer,
41934430
Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
4194-
ingestNormalized: () => Effect.succeed([]),
4431+
ingestNormalized: input.ingestNormalized ?? (() => Effect.succeed([])),
41954432
}),
41964433
ServerSettings.layerTest(),
41974434
Layer.succeed(RunFinalizationService.RunFinalizationObserver, {
@@ -4207,7 +4444,7 @@ function captureRootRunTermination(input: {
42074444

42084445
yield* Effect.gen(function* () {
42094446
const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
4210-
yield* runExecution.startRootRun({
4447+
const start = runExecution.startRootRun({
42114448
commandId: CommandId.make(`command:${input.key}`),
42124449
appThread: { id: ids.threadId } as OrchestrationV2AppThread,
42134450
providerSessionId: ProviderSessionId.make(`session:${input.key}`),
@@ -4298,9 +4535,13 @@ function captureRootRunTermination(input: {
42984535
},
42994536
},
43004537
});
4538+
yield* start.pipe(
4539+
Effect.catch((error) => input.onStartFailure?.(error) ?? Effect.fail(error)),
4540+
);
4541+
yield* input.afterStart?.(ingestionDone) ?? Effect.void;
4542+
yield* Deferred.await(ingestionDone);
43014543
}).pipe(Effect.provide(layerTest));
43024544

4303-
yield* Deferred.await(ingestionDone);
43044545
return {
43054546
written: yield* Ref.get(writtenItems),
43064547
finalEvents: yield* Ref.get(finalEvents),

0 commit comments

Comments
 (0)