Skip to content

Commit 152d44d

Browse files
fix(server): settle exhausted turn starts without failing newer attempts
1 parent 78a3fc1 commit 152d44d

5 files changed

Lines changed: 655 additions & 748 deletions

File tree

‎apps/server/src/orchestration-v2/EffectWorker.test.ts‎

Lines changed: 1 addition & 231 deletions
Original file line numberDiff line numberDiff line change
@@ -83,10 +83,6 @@ function layerExecutorFor(input: {
8383
readonly threads?: Partial<ThreadManagementService.ThreadManagementService["Service"]>;
8484
readonly continueAfterRestart?: boolean;
8585
readonly interrupt?: ProviderTurnControlService.ProviderTurnControlServiceV2Shape["interrupt"];
86-
readonly failRunFailure?: boolean;
87-
readonly failRunStateMissing?: boolean;
88-
readonly interruptFailure?: boolean;
89-
readonly startInterrupted?: boolean;
9086
}) {
9187
const record = (event: string) => Ref.update(input.events, (events) => [...events, event]);
9288
const layerDependencies = Layer.mergeAll(
@@ -100,19 +96,6 @@ function layerExecutorFor(input: {
10096
request.replacementProviderSessionId === undefined
10197
? "interrupt"
10298
: `interrupt:${request.replacementProviderSessionId}`,
103-
).pipe(
104-
Effect.andThen(
105-
input.interruptFailure === true
106-
? Effect.fail(
107-
new ProviderTurnControlService.ProviderTurnControlError({
108-
threadId,
109-
operation: "interrupt",
110-
providerTurnId,
111-
cause: "simulated interrupt failure",
112-
}),
113-
)
114-
: Effect.void,
115-
),
11699
),
117100
}),
118101
),
@@ -132,10 +115,10 @@ function layerExecutorFor(input: {
132115
Layer.succeed(
133116
ProviderTurnStartService.ProviderTurnStartServiceV2,
134117
ProviderTurnStartService.ProviderTurnStartServiceV2.of({
118+
failStartingRun: () => Effect.void,
135119
start: () =>
136120
Effect.gen(function* () {
137121
yield* record("start");
138-
if (input.startInterrupted === true) return yield* Effect.interrupt;
139122
if (
140123
input.failFirstStart !== undefined &&
141124
(yield* Ref.getAndSet(input.failFirstStart, false))
@@ -146,21 +129,6 @@ function layerExecutorFor(input: {
146129
});
147130
}
148131
}),
149-
failStartingRun: () =>
150-
Effect.gen(function* () {
151-
yield* record("fail-run");
152-
if (input.failRunStateMissing === true) {
153-
return yield* new ProviderTurnStartService.ProviderTurnStartRunStateMissingError({
154-
runId,
155-
});
156-
}
157-
if (input.failRunFailure === true) {
158-
return yield* new ProviderTurnStartService.ProviderTurnStartError({
159-
runId,
160-
cause: "simulated run failure write",
161-
});
162-
}
163-
}),
164132
}),
165133
),
166134
Layer.succeed(
@@ -937,201 +905,3 @@ it.effect.each(["confirmed", "unconfirmed", "failed"] as const)(
937905
);
938906
}),
939907
);
940-
941-
function startEffect(now: DateTime.Utc): EffectOutbox.OrchestrationEffectV2 {
942-
return {
943-
...restartEffect(now, { type: "detach" }),
944-
id: "effect:start",
945-
request: { type: "provider-turn.start", runId },
946-
};
947-
}
948-
949-
it.effect("fails the run when the last start attempt fails", () =>
950-
Effect.gen(function* () {
951-
const now = yield* DateTime.now;
952-
const events = yield* Ref.make<ReadonlyArray<string>>([]);
953-
const failFirstStart = yield* Ref.make(true);
954-
955-
yield* Effect.gen(function* () {
956-
const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
957-
yield* executor.execute(startEffect(now), { willRetry: false });
958-
}).pipe(Effect.provide(layerExecutorFor({ events, failFirstStart })));
959-
960-
assert.deepEqual(yield* Ref.get(events), ["start", "fail-run"]);
961-
}),
962-
);
963-
964-
it.effect("fails the run when a restart's last attempt cannot interrupt the old turn", () =>
965-
Effect.gen(function* () {
966-
const now = yield* DateTime.now;
967-
const events = yield* Ref.make<ReadonlyArray<string>>([]);
968-
const effect = restartEffect(now, { type: "detach" });
969-
970-
yield* Effect.gen(function* () {
971-
const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
972-
yield* executor.execute(effect, { willRetry: false });
973-
}).pipe(Effect.provide(layerExecutorFor({ events, interruptFailure: true })));
974-
975-
assert.deepEqual(yield* Ref.get(events), ["interrupt", "fail-run"]);
976-
}),
977-
);
978-
979-
it.effect("does not fail the run when its last start attempt is interrupted", () =>
980-
Effect.gen(function* () {
981-
const now = yield* DateTime.now;
982-
const events = yield* Ref.make<ReadonlyArray<string>>([]);
983-
984-
const exit = yield* Effect.gen(function* () {
985-
const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
986-
return yield* Effect.exit(executor.execute(startEffect(now), { willRetry: false }));
987-
}).pipe(Effect.provide(layerExecutorFor({ events, startInterrupted: true })));
988-
989-
assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause));
990-
assert.deepEqual(yield* Ref.get(events), ["start"]);
991-
}),
992-
);
993-
994-
it.effect("returns the start failure when the run cannot be failed either", () =>
995-
Effect.gen(function* () {
996-
const now = yield* DateTime.now;
997-
const events = yield* Ref.make<ReadonlyArray<string>>([]);
998-
const failFirstStart = yield* Ref.make(true);
999-
const effect = restartEffect(now, { type: "detach" });
1000-
1001-
const error = yield* Effect.gen(function* () {
1002-
const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
1003-
return yield* executor.execute(effect, { willRetry: false }).pipe(Effect.flip);
1004-
}).pipe(Effect.provide(layerExecutorFor({ events, failFirstStart, failRunFailure: true })));
1005-
1006-
assert.deepEqual(yield* Ref.get(events), ["interrupt", "detach", "start", "fail-run"]);
1007-
assert.include(Cause.pretty(Cause.fail(error)), "simulated first start failure");
1008-
}),
1009-
);
1010-
1011-
it.effect("returns an unsettleable run instead of the start failure", () =>
1012-
Effect.gen(function* () {
1013-
const now = yield* DateTime.now;
1014-
const events = yield* Ref.make<ReadonlyArray<string>>([]);
1015-
const failFirstStart = yield* Ref.make(true);
1016-
1017-
const error = yield* Effect.gen(function* () {
1018-
const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
1019-
return yield* executor.execute(startEffect(now), { willRetry: false }).pipe(Effect.flip);
1020-
}).pipe(
1021-
Effect.provide(layerExecutorFor({ events, failFirstStart, failRunStateMissing: true })),
1022-
);
1023-
1024-
assert.deepEqual(yield* Ref.get(events), ["start", "fail-run"]);
1025-
assert.isTrue(ProviderTurnStartService.isProviderTurnStartRunStateMissingError(error.cause));
1026-
}),
1027-
);
1028-
1029-
it.effect.each(["provider-turn.start", "provider-turn.restart"] as const)(
1030-
"keeps retrying a %s effect past its attempt budget",
1031-
(effectType) =>
1032-
Effect.gen(function* () {
1033-
const now = yield* DateTime.now;
1034-
const workerId = "worker-run-start-past-budget";
1035-
const claimedEffect: EffectOutbox.OrchestrationEffectV2 = {
1036-
...restartEffect(now, { type: "detach" }),
1037-
...(effectType === "provider-turn.start"
1038-
? { request: { type: "provider-turn.start" as const, runId } }
1039-
: {}),
1040-
attemptCount: 5,
1041-
leaseOwner: workerId,
1042-
};
1043-
const retryDelays = yield* Ref.make<ReadonlyArray<number>>([]);
1044-
const terminalizations = yield* Ref.make(0);
1045-
const willRetry = yield* Ref.make<ReadonlyArray<boolean | undefined>>([]);
1046-
const outboxLayer = Layer.mock(EffectOutbox.EffectOutboxV2)({
1047-
claimNext: () => Effect.succeed(Option.some(claimedEffect)),
1048-
get: () => Effect.succeed(Option.some(claimedEffect)),
1049-
awaitCancellation: () => Effect.never,
1050-
clearCancellation: () => Effect.void,
1051-
retry: ({ delayMs }) =>
1052-
Ref.update(retryDelays, (delays) => [...delays, delayMs]).pipe(Effect.as(true)),
1053-
fail: () => Ref.update(terminalizations, (count) => count + 1).pipe(Effect.as(true)),
1054-
});
1055-
const executorLayer = Layer.succeed(
1056-
EffectWorker.OrchestrationEffectExecutorV2,
1057-
EffectWorker.OrchestrationEffectExecutorV2.of({
1058-
execute: (_effect, options) =>
1059-
Ref.update(willRetry, (seen) => [...seen, options?.willRetry]).pipe(
1060-
Effect.andThen(
1061-
Effect.fail(
1062-
new EffectWorker.OrchestrationEffectExecutionError({
1063-
effectId: claimedEffect.id,
1064-
effectType,
1065-
cause: "simulated storage outage",
1066-
}),
1067-
),
1068-
),
1069-
),
1070-
}),
1071-
);
1072-
const workerLayer = EffectWorker.layerWithOptions({ workerId, maxAttempts: 5 }).pipe(
1073-
Layer.provide(Layer.merge(outboxLayer, executorLayer)),
1074-
);
1075-
1076-
assert.isTrue(
1077-
yield* EffectWorker.OrchestrationEffectWorkerV2.pipe(
1078-
Effect.flatMap((worker) => worker.runOnce),
1079-
Effect.provide(workerLayer),
1080-
),
1081-
);
1082-
1083-
// Each attempt past the budget is a last attempt: it starts the run or
1084-
// fails it, and the effect itself is never abandoned.
1085-
assert.deepEqual(yield* Ref.get(willRetry), [false]);
1086-
assert.deepEqual(yield* Ref.get(retryDelays), [1_600]);
1087-
assert.equal(yield* Ref.get(terminalizations), 0);
1088-
}),
1089-
);
1090-
1091-
it.effect("fails a run-start effect for good when its run has no state left to settle", () =>
1092-
Effect.gen(function* () {
1093-
const now = yield* DateTime.now;
1094-
const workerId = "worker-run-start-unsettleable";
1095-
const claimedEffect: EffectOutbox.OrchestrationEffectV2 = {
1096-
...startEffect(now),
1097-
attemptCount: 5,
1098-
leaseOwner: workerId,
1099-
};
1100-
const retries = yield* Ref.make(0);
1101-
const terminalizations = yield* Ref.make(0);
1102-
const outboxLayer = Layer.mock(EffectOutbox.EffectOutboxV2)({
1103-
claimNext: () => Effect.succeed(Option.some(claimedEffect)),
1104-
get: () => Effect.succeed(Option.some(claimedEffect)),
1105-
awaitCancellation: () => Effect.never,
1106-
clearCancellation: () => Effect.void,
1107-
retry: () => Ref.update(retries, (count) => count + 1).pipe(Effect.as(true)),
1108-
fail: () => Ref.update(terminalizations, (count) => count + 1).pipe(Effect.as(true)),
1109-
});
1110-
const executorLayer = Layer.succeed(
1111-
EffectWorker.OrchestrationEffectExecutorV2,
1112-
EffectWorker.OrchestrationEffectExecutorV2.of({
1113-
execute: () =>
1114-
Effect.fail(
1115-
new EffectWorker.OrchestrationEffectExecutionError({
1116-
effectId: claimedEffect.id,
1117-
effectType: claimedEffect.request.type,
1118-
cause: new ProviderTurnStartService.ProviderTurnStartRunStateMissingError({ runId }),
1119-
}),
1120-
),
1121-
}),
1122-
);
1123-
const workerLayer = EffectWorker.layerWithOptions({ workerId, maxAttempts: 5 }).pipe(
1124-
Layer.provide(Layer.merge(outboxLayer, executorLayer)),
1125-
);
1126-
1127-
assert.isTrue(
1128-
yield* EffectWorker.OrchestrationEffectWorkerV2.pipe(
1129-
Effect.flatMap((worker) => worker.runOnce),
1130-
Effect.provide(workerLayer),
1131-
),
1132-
);
1133-
1134-
assert.equal(yield* Ref.get(retries), 0);
1135-
assert.equal(yield* Ref.get(terminalizations), 1);
1136-
}),
1137-
);

0 commit comments

Comments
 (0)