Skip to content

Commit e77a48e

Browse files
Alb11747codexjuliusmarmingeclaude
authored
fix(server): fail runs whose provider stream ends without a terminal (pingdotgg#19131)
Co-authored-by: GPT-6.1-Sol at high effort <[email protected]> Co-authored-by: Julius Marminge <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 76cb251 commit e77a48e

4 files changed

Lines changed: 161 additions & 76 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1746,7 +1746,7 @@ it.effect("ProviderSessionManagerV2 releases sessions in parallel on server shut
17461746
}),
17471747
);
17481748

1749-
it.effect("ProviderSessionManagerV2 closes event subscriptions normally on server shutdown", () =>
1749+
it.effect("ProviderSessionManagerV2 interrupts event subscriptions on server shutdown", () =>
17501750
Effect.gen(function* () {
17511751
const state = yield* Ref.make(emptyState);
17521752
const effect = Effect.gen(function* () {
@@ -1781,7 +1781,10 @@ it.effect("ProviderSessionManagerV2 closes event subscriptions normally on serve
17811781

17821782
yield* manager.shutdown;
17831783

1784-
assert.isEmpty(yield* bufferedSubscription.events.pipe(Stream.runCollect));
1784+
// Buffered events are dropped and the consumer sees an interruption, not
1785+
// a clean end, so it leaves run settlement to restart recovery.
1786+
const exit = yield* bufferedSubscription.events.pipe(Stream.runCollect, Effect.exit);
1787+
assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause));
17851788
});
17861789

17871790
yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000 })));

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -593,12 +593,15 @@ export const layerWithOptions = (
593593
);
594594
});
595595

596-
const closeSubscribers = (entry: LiveSessionEntry) =>
596+
// Server shutdown interrupts subscribers rather than ending them: a
597+
// cleanly ended stream without a terminal fails its run, while an
598+
// interrupted one leaves run settlement to restart recovery.
599+
const interruptSubscribers = (entry: LiveSessionEntry) =>
597600
Effect.gen(function* () {
598601
const subscribers = yield* Ref.getAndSet(entry.eventSubscribers, new Map());
599602
yield* Effect.forEach(
600603
subscribers.values(),
601-
(queue) => Queue.clear(queue).pipe(Effect.andThen(Queue.end(queue))),
604+
(queue) => Queue.clear(queue).pipe(Effect.andThen(Queue.interrupt(queue))),
602605
{ discard: true },
603606
);
604607
});
@@ -1078,7 +1081,7 @@ export const layerWithOptions = (
10781081
if (input.gracefulSubscribers === true) {
10791082
yield* endSubscribers(entry);
10801083
} else if (input.reason === "server_shutdown") {
1081-
yield* closeSubscribers(entry);
1084+
yield* interruptSubscribers(entry);
10821085
} else {
10831086
yield* failSubscribers(
10841087
entry,
@@ -2096,15 +2099,17 @@ export const layerWithOptions = (
20962099
}).pipe(logReleaseFailure(entry.runtime.providerSessionId));
20972100
return;
20982101
}
2099-
const cause = Exit.isFailure(exit)
2100-
? exit.cause
2101-
: Cause.fail(
2102-
new ProviderAdapter.ProviderAdapterEventStreamError({
2103-
driver: entry.runtime.driver,
2104-
providerSessionId: entry.runtime.providerSessionId,
2105-
cause: "Provider event stream ended unexpectedly.",
2106-
}),
2107-
);
2102+
// Subscribers read an interrupt as server shutdown, so keep it for that.
2103+
const cause =
2104+
Exit.isFailure(exit) && !Cause.hasInterruptsOnly(exit.cause)
2105+
? exit.cause
2106+
: Cause.fail(
2107+
new ProviderAdapter.ProviderAdapterEventStreamError({
2108+
driver: entry.runtime.driver,
2109+
providerSessionId: entry.runtime.providerSessionId,
2110+
cause: "Provider event stream ended unexpectedly.",
2111+
}),
2112+
);
21082113
yield* publishToSubscribers(entry.eventSubscribers, {
21092114
type: "failure",
21102115
cause,

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 66 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3742,9 +3742,11 @@ it.effect("does not refresh pull requests for auxiliary or stale provider termin
37423742
{ ...ids, rootProviderTurnId: ProviderTurnId.make("turn:previous") },
37433743
"interrupted",
37443744
),
3745+
rootTerminalEvent(ids, "cancelled"),
37453746
]),
37463747
});
3747-
assert.deepEqual(observed, []);
3748+
// Only the root terminal settles the run and refreshes pull requests.
3749+
assert.deepEqual(observed, ["run:cancelled", "pull-requests-refreshed"]);
37483750
}),
37493751
);
37503752

@@ -3772,6 +3774,58 @@ it.effect("refreshes pull requests after a provider stream exits with an error",
37723774
}),
37733775
);
37743776

3777+
it.effect("fails the run when its event stream ends without a root terminal", () =>
3778+
Effect.gen(function* () {
3779+
const { observed, finalEvents } = yield* captureRootRunTermination({
3780+
key: "stream-end-without-terminal",
3781+
shouldFinalizeRun: () => Effect.succeed(true),
3782+
events: () => Stream.empty,
3783+
});
3784+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3785+
assert.equal(
3786+
finalEvents.find((event) => event.type === "provider-thread.updated")?.payload.status,
3787+
"idle",
3788+
);
3789+
}),
3790+
);
3791+
3792+
it.effect("leaves the run to restart recovery when shutdown interrupts its event stream", () =>
3793+
Effect.gen(function* () {
3794+
const { written, observed } = yield* captureRootRunTermination({
3795+
key: "stream-end:shutdown",
3796+
shouldFinalizeRun: () => Effect.succeed(true),
3797+
events: () => Stream.failCause(Cause.interrupt()),
3798+
});
3799+
assert.deepEqual(written, []);
3800+
assert.deepEqual(observed, []);
3801+
}),
3802+
);
3803+
3804+
it.effect("does not start a turn whose run settled during the app context read", () =>
3805+
Effect.gen(function* () {
3806+
const contextReadStarted = yield* Deferred.make<void>();
3807+
const runFailed = yield* Deferred.make<void>();
3808+
const current = yield* Ref.make(true);
3809+
const providerCalls = yield* Ref.make(0);
3810+
const { observed } = yield* captureRootRunTermination({
3811+
key: "stream-end:context-read",
3812+
shouldStartProviderTurn: () => Ref.get(current),
3813+
shouldFinalizeRun: () => Ref.get(current),
3814+
events: () => Stream.unwrap(Deferred.await(contextReadStarted).pipe(Effect.as(Stream.empty))),
3815+
appContext: Deferred.succeed(contextReadStarted, undefined).pipe(
3816+
Effect.andThen(Deferred.await(runFailed)),
3817+
Effect.as([]),
3818+
),
3819+
refreshAfterTurn: Ref.set(current, false).pipe(
3820+
Effect.andThen(Deferred.succeed(runFailed, undefined)),
3821+
),
3822+
startTurn: () => Ref.update(providerCalls, (count) => count + 1),
3823+
});
3824+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3825+
assert.equal(yield* Ref.get(providerCalls), 0);
3826+
}),
3827+
);
3828+
37753829
it.effect("refreshes pull requests only once when startup failure closes its event stream", () =>
37763830
Effect.gen(function* () {
37773831
const ingestionStarted = yield* Deferred.make<void>();
@@ -3828,6 +3882,8 @@ function captureRootRunTermination(input: {
38283882
>;
38293883
readonly startTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"];
38303884
readonly refreshAfterTurn?: Effect.Effect<void>;
3885+
readonly shouldStartProviderTurn?: RunExecutionService.RunExecutionServiceV2StartRootRunInput["shouldStartProviderTurn"];
3886+
readonly appContext?: ReturnType<McpAppModelContext.McpAppModelContext["Service"]["forThread"]>;
38313887
}) {
38323888
return Effect.gen(function* () {
38333889
const ids = backgroundScenarioIds(input.key);
@@ -3840,6 +3896,7 @@ function captureRootRunTermination(input: {
38403896
status: "running",
38413897
});
38423898
const writtenItems = yield* Ref.make<ReadonlyArray<OrchestrationV2TurnItem>>([]);
3899+
const finalEvents = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
38433900
const observed = yield* Ref.make<ReadonlyArray<string>>([]);
38443901
const submittedEffects = yield* Ref.make<ReadonlyArray<PendingOrchestrationEffectV2>>([]);
38453902
const committedEffects = yield* Ref.make<ReadonlyArray<PendingOrchestrationEffectV2>>([]);
@@ -3848,6 +3905,7 @@ function captureRootRunTermination(input: {
38483905
Ref.update(writtenItems, (current) => [...current, payload]);
38493906
const captureFinalEvents = (events: ReadonlyArray<OrchestrationV2DomainEvent>) =>
38503907
Effect.gen(function* () {
3908+
yield* Ref.update(finalEvents, (current) => [...current, ...events]);
38513909
for (const event of events) {
38523910
if (event.type === "turn-item.updated") {
38533911
yield* captureTurnItem(event.payload);
@@ -3860,7 +3918,9 @@ function captureRootRunTermination(input: {
38603918
const layerTest = RunExecutionService.layer.pipe(
38613919
Layer.provide(
38623920
Layer.mergeAll(
3863-
McpAppModelContext.layerEmpty,
3921+
Layer.mock(McpAppModelContext.McpAppModelContext)({
3922+
forThread: () => input.appContext ?? Effect.succeed([]),
3923+
}),
38643924
Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
38653925
Layer.mock(EventSink.EventSinkV2)({
38663926
write: (payload) =>
@@ -3970,6 +4030,9 @@ function captureRootRunTermination(input: {
39704030
attemptId: ids.attemptId,
39714031
providerTurnOrdinal: 1,
39724032
shouldFinalizeRun: input.shouldFinalizeRun,
4033+
...(input.shouldStartProviderTurn === undefined
4034+
? {}
4035+
: { shouldStartProviderTurn: input.shouldStartProviderTurn }),
39734036
...(input.hasUnpairedRunInterruptRequest === undefined
39744037
? {}
39754038
: {
@@ -4000,6 +4063,7 @@ function captureRootRunTermination(input: {
40004063
yield* Deferred.await(ingestionDone);
40014064
return {
40024065
written: yield* Ref.get(writtenItems),
4066+
finalEvents: yield* Ref.get(finalEvents),
40034067
observed: yield* Ref.get(observed),
40044068
submittedEffects: yield* Ref.get(submittedEffects),
40054069
committedEffects: yield* Ref.get(committedEffects),

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 73 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1316,83 +1316,85 @@ export const layer: Layer.Layer<
13161316
Effect.gen(function* () {
13171317
const terminal = yield* Ref.get(terminalEvent);
13181318
if (terminal === null) {
1319-
return;
1319+
// A session release interrupts or fails the stream; a clean
1320+
// end without a terminal means the run would never settle.
1321+
return yield* new RunExecutionIngestError({
1322+
runId: input.run.id,
1323+
cause: new ProviderAdapter.ProviderAdapterEventStreamError({
1324+
driver: input.providerThread.driver,
1325+
providerSessionId: input.providerSessionId,
1326+
cause: "Provider event stream ended without a root terminal.",
1327+
}),
1328+
});
13201329
}
13211330
yield* finalizeRootRun(terminal);
13221331
}),
13231332
),
1324-
Effect.catchCause((cause) =>
1325-
Ref.get(rootRunFinalized).pipe(
1326-
Effect.flatMap((finalized) =>
1327-
Effect.logWarning("orchestration V2 provider event ingestion failed", {
1328-
runId: input.run.id,
1329-
cause,
1330-
}).pipe(
1331-
Effect.andThen(
1332-
finalized
1333-
? Effect.void
1334-
: Ref.get(latestProviderThread).pipe(
1335-
Effect.flatMap((providerThread) =>
1336-
Ref.get(latestTurnItemOrdinal).pipe(
1337-
Effect.flatMap((latestItemOrdinal) =>
1338-
Ref.get(openRunOwnedSubagents).pipe(
1339-
Effect.flatMap((openSubagents) =>
1340-
writeFinalRunEvents({
1341-
run: input.run,
1342-
rootNode: input.rootNode,
1343-
checkpointScope: input.checkpointScope,
1344-
providerThread,
1345-
attempt: input.attempt,
1346-
// The failure may be the ownership
1347-
// read itself, so check in the write.
1348-
writeIfRunCurrent: {
1349-
activeAttemptId: input.attempt.id,
1350-
expectedStatus: "running",
1351-
},
1352-
openRunOwnedSubagents: openSubagents,
1353-
terminal: makeFailedTerminalEvent(
1354-
makeProviderFailure({
1355-
cause: Cause.squash(cause),
1356-
class: "unknown",
1357-
}),
1358-
latestItemOrdinal + 1,
1359-
),
1360-
failureItemPersisted: false,
1361-
refreshAfterTurn,
1362-
}),
1333+
// Server shutdown interrupts the subscription and leaves the run to
1334+
// restart recovery; every other failure fails the run.
1335+
Effect.catchCauseIf(
1336+
(cause) => !Cause.hasInterruptsOnly(cause),
1337+
(cause) =>
1338+
Ref.get(rootRunFinalized).pipe(
1339+
Effect.flatMap((finalized) =>
1340+
Effect.logWarning("orchestration V2 provider event ingestion failed", {
1341+
runId: input.run.id,
1342+
cause,
1343+
}).pipe(
1344+
Effect.andThen(
1345+
finalized
1346+
? Effect.void
1347+
: Ref.get(latestProviderThread).pipe(
1348+
Effect.flatMap((providerThread) =>
1349+
Ref.get(latestTurnItemOrdinal).pipe(
1350+
Effect.flatMap((latestItemOrdinal) =>
1351+
Ref.get(openRunOwnedSubagents).pipe(
1352+
Effect.flatMap((openSubagents) =>
1353+
writeFinalRunEvents({
1354+
run: input.run,
1355+
rootNode: input.rootNode,
1356+
checkpointScope: input.checkpointScope,
1357+
providerThread,
1358+
attempt: input.attempt,
1359+
// The failure may be the ownership
1360+
// read itself, so check in the write.
1361+
writeIfRunCurrent: {
1362+
activeAttemptId: input.attempt.id,
1363+
expectedStatus: "running",
1364+
},
1365+
openRunOwnedSubagents: openSubagents,
1366+
terminal: makeFailedTerminalEvent(
1367+
makeProviderFailure({
1368+
cause: Cause.squash(cause),
1369+
class: "unknown",
1370+
}),
1371+
latestItemOrdinal + 1,
1372+
),
1373+
failureItemPersisted: false,
1374+
refreshAfterTurn,
1375+
}),
1376+
),
13631377
),
13641378
),
13651379
),
13661380
),
13671381
),
1368-
),
1369-
),
1370-
Effect.mapError(
1371-
(writeCause) =>
1372-
new RunExecutionIngestError({
1373-
runId: input.run.id,
1374-
cause: { ingest: cause, write: writeCause },
1375-
}),
1382+
),
1383+
Effect.mapError(
1384+
(writeCause) =>
1385+
new RunExecutionIngestError({
1386+
runId: input.run.id,
1387+
cause: { ingest: cause, write: writeCause },
1388+
}),
1389+
),
13761390
),
13771391
),
13781392
),
1379-
),
13801393
),
13811394
Effect.ensuring(eventSubscription.close),
13821395
Effect.forkDetach,
13831396
);
13841397

1385-
// A failed read fails the start below, so the run is recorded as
1386-
// failed instead of staying active with no provider turn.
1387-
const shouldStart =
1388-
input.shouldStartProviderTurn === undefined
1389-
? Exit.succeed(true)
1390-
: yield* Effect.exit(input.shouldStartProviderTurn());
1391-
if (Exit.isSuccess(shouldStart) && !shouldStart.value) {
1392-
yield* Fiber.interrupt(providerEventFiber);
1393-
return;
1394-
}
1395-
13961398
// A provider turn is a sign that the session is still alive. Keep
13971399
// its already-issued MCP credential valid even when the agent goes
13981400
// a long time between browser-tool calls.
@@ -1436,6 +1438,17 @@ export const layer: Layer.Layer<
14361438
runtimePolicy: input.runtimePolicy,
14371439
...(appContext.length === 0 ? {} : { appContext }),
14381440
};
1441+
// Check after the context read: ingestion can settle the run while
1442+
// it awaits. A failed ownership read fails the start below.
1443+
const shouldStart =
1444+
input.shouldStartProviderTurn === undefined
1445+
? Exit.succeed(true)
1446+
: yield* Effect.exit(input.shouldStartProviderTurn());
1447+
if (Exit.isSuccess(shouldStart) && !shouldStart.value) {
1448+
yield* Fiber.interrupt(providerEventFiber);
1449+
return;
1450+
}
1451+
14391452
const compact =
14401453
input.message.attachments.length === 0 &&
14411454
input.message.text.trim().toLowerCase() === "/compact";

0 commit comments

Comments
 (0)