Skip to content

Commit f33f3bf

Browse files
committed
fix(server): persist provider interruption after ingestion failure
1 parent d66cf27 commit f33f3bf

3 files changed

Lines changed: 67 additions & 22 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,7 @@ function makeLocalCommandHarness(input: {
274274
readonly failReadsAfterRunning?: boolean;
275275
readonly previousTurn?: {
276276
readonly runStatus: "failed" | "completed";
277-
readonly turnStatus: "running" | "completed";
277+
readonly turnStatus: "running" | "completed" | "interrupted";
278278
readonly differentThread?: boolean;
279279
readonly liveSession?: boolean;
280280
readonly interruptFailure?: boolean;
@@ -507,7 +507,7 @@ function makeLocalCommandHarness(input: {
507507
nativeTurnRef: null,
508508
status: input.previousTurn.turnStatus,
509509
startedAt: now,
510-
completedAt: input.previousTurn.turnStatus === "completed" ? now : null,
510+
completedAt: input.previousTurn.turnStatus === "running" ? null : now,
511511
},
512512
],
513513
};
@@ -801,6 +801,7 @@ effectIt.effect.each([
801801
{ runStatus: "completed", turnStatus: "completed" },
802802
{ runStatus: "failed", turnStatus: "running", differentThread: true },
803803
{ runStatus: "failed", turnStatus: "running", liveSession: false },
804+
{ runStatus: "failed", turnStatus: "interrupted", interruptFailure: true },
804805
] as const)("continues without interrupting when recovery does not apply: %j", (previousTurn) =>
805806
Effect.gen(function* () {
806807
const harness = makeLocalCommandHarness({ text: "Continue", previousTurn });

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2487,7 +2487,7 @@ it.effect("refreshes pull requests after a provider stream exits with an error",
24872487
}),
24882488
);
24892489

2490-
it.effect.each(["failure", "defect", "interrupt-failure"] as const)(
2490+
it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as const)(
24912491
"stops the live provider before settling an ingestion %s",
24922492
(failureMode) =>
24932493
Effect.gen(function* () {
@@ -2498,9 +2498,10 @@ it.effect.each(["failure", "defect", "interrupt-failure"] as const)(
24982498
ReadonlyArray<ProviderAdapter.ProviderAdapterV2InterruptInput>
24992499
>([]);
25002500
const ids = backgroundScenarioIds(`ingestion:${failureMode}`);
2501-
const { observed } = yield* captureRootRunTermination({
2501+
const { observed, finalEvents } = yield* captureRootRunTermination({
25022502
key: `ingestion:${failureMode}`,
25032503
initialProviderTurnId: null,
2504+
rejectTerminalWrite: failureMode === "ownership-lost",
25042505
shouldFinalizeRun: () => Effect.succeed(true),
25052506
startTurn: () =>
25062507
Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(stopped))),
@@ -2559,7 +2560,20 @@ it.effect.each(["failure", "defect", "interrupt-failure"] as const)(
25592560
return true;
25602561
}),
25612562
});
2562-
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
2563+
if (failureMode === "ownership-lost") {
2564+
assert.deepEqual(finalEvents, []);
2565+
assert.deepEqual(observed, []);
2566+
} else {
2567+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
2568+
const repaired = finalEvents.find((event) => event.type === "provider-turn.updated");
2569+
if (failureMode === "interrupt-failure") {
2570+
assert.isUndefined(repaired);
2571+
} else {
2572+
assert.equal(repaired?.payload.id, ids.rootProviderTurnId);
2573+
assert.equal(repaired?.payload.status, "interrupted");
2574+
assert.isNotNull(repaired?.payload.completedAt);
2575+
}
2576+
}
25632577
}),
25642578
);
25652579

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 47 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -622,6 +622,7 @@ export const layer: Layer.Layer<
622622
readonly shouldFinalizeRun?: () => Effect.Effect<boolean, ProjectionStoreV2Error>;
623623
readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
624624
readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection;
625+
readonly interruptedProviderTurn?: OrchestrationV2ProviderTurn;
625626
readonly terminal: ProviderTerminalEvent;
626627
readonly completedAt?: DateTime.Utc;
627628
readonly failureItemPersisted: boolean;
@@ -743,6 +744,24 @@ export const layer: Layer.Layer<
743744
: [],
744745
events: [
745746
...cascadedSubagentEvents,
747+
...(input.interruptedProviderTurn === undefined
748+
? []
749+
: [
750+
{
751+
id: yield* allocateEventId(),
752+
type: "provider-turn.updated" as const,
753+
threadId: input.run.threadId,
754+
runId: input.run.id,
755+
nodeId: input.rootNode.id,
756+
providerInstanceId: input.run.providerInstanceId,
757+
occurredAt: completedAt,
758+
payload: {
759+
...input.interruptedProviderTurn,
760+
status: "interrupted" as const,
761+
completedAt,
762+
},
763+
},
764+
]),
746765
...(finalizedAttempt === null
747766
? []
748767
: [
@@ -962,6 +981,7 @@ export const layer: Layer.Layer<
962981
const terminalEvent = yield* Ref.make<ProviderTerminalEvent | null>(null);
963982
const latestTurnItemOrdinal = yield* Ref.make(input.providerTurnOrdinal * 100);
964983
const latestProviderThread = yield* Ref.make(input.providerThread);
984+
const latestRootProviderTurn = yield* Ref.make<OrchestrationV2ProviderTurn | null>(null);
965985
const routeIdentity: ProviderEventRouteIdentity = {
966986
threadId: input.run.threadId,
967987
runId: input.run.id,
@@ -1318,6 +1338,13 @@ export const layer: Layer.Layer<
13181338
Stream.tap(({ event, accepted }) =>
13191339
Effect.gen(function* () {
13201340
if (!accepted) return;
1341+
if (
1342+
event.type === "provider_turn.updated" &&
1343+
event.providerTurn.runAttemptId === input.attempt.id &&
1344+
event.providerTurn.providerThreadId === input.providerThread.id
1345+
) {
1346+
yield* Ref.set(latestRootProviderTurn, event.providerTurn);
1347+
}
13211348
let storedEventCount = 0;
13221349
const deliveredEvent = filterAssistantEvent(
13231350
event,
@@ -1437,24 +1464,26 @@ export const layer: Layer.Layer<
14371464
if (yield* Ref.get(rootRunFinalized)) return;
14381465
const providerThread = yield* Ref.get(latestProviderThread);
14391466
const routing = yield* Ref.get(eventRouting);
1440-
// Failed ingestion can leave the provider running after its run settles.
1441-
if (routing.rootProviderTurnId !== null && !routing.rootTurnEnded) {
1442-
yield* input.session
1443-
.interruptTurn({
1444-
providerThread,
1445-
providerTurnId: routing.rootProviderTurnId,
1446-
})
1447-
.pipe(
1448-
Effect.catchCauseIf(
1449-
(interruptCause) => !Cause.hasInterruptsOnly(interruptCause),
1450-
(interruptCause) =>
1451-
Effect.logWarning(
1452-
"orchestration V2 failed to interrupt turn after ingestion failure",
1453-
{ runId: input.run.id, cause: interruptCause },
1467+
// The failed consumer cannot persist the adapter's terminal event.
1468+
const interruptedProviderTurn =
1469+
routing.rootProviderTurnId !== null && !routing.rootTurnEnded
1470+
? yield* input.session
1471+
.interruptTurn({
1472+
providerThread,
1473+
providerTurnId: routing.rootProviderTurnId,
1474+
})
1475+
.pipe(
1476+
Effect.as(yield* Ref.get(latestRootProviderTurn)),
1477+
Effect.catchCauseIf(
1478+
(interruptCause) => !Cause.hasInterruptsOnly(interruptCause),
1479+
(interruptCause) =>
1480+
Effect.logWarning(
1481+
"orchestration V2 failed to interrupt turn after ingestion failure",
1482+
{ runId: input.run.id, cause: interruptCause },
1483+
).pipe(Effect.as(null)),
14541484
),
1455-
),
1456-
);
1457-
}
1485+
)
1486+
: null;
14581487
const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal);
14591488
const openSubagents = yield* Ref.get(openRunOwnedSubagents);
14601489
let nextRetryWarningAt = completedAt;
@@ -1469,6 +1498,7 @@ export const layer: Layer.Layer<
14691498
expectedStatus: "running",
14701499
},
14711500
openRunOwnedSubagents: openSubagents,
1501+
...(interruptedProviderTurn === null ? {} : { interruptedProviderTurn }),
14721502
completedAt,
14731503
terminal: makeFailedTerminalEvent(
14741504
makeProviderFailure({ cause: Cause.squash(cause), class: "unknown" }),

0 commit comments

Comments
 (0)