Skip to content

Commit 41e5eb4

Browse files
AnobleSCMclaude
andcommitted
fix(server): bound the orphaned provider turn stop
The stop runs as a finalizer, so it cannot be interrupted. A provider whose interrupt never answers would hold the ingestion fiber forever, and a turn-start failure that interrupts that fiber would hang with it. The stop now gives up after 15 seconds, longer than the Claude adapter's own 10-second close wait, and logs a warning. The interrupt-failure test now checks that the interrupt was attempted and that its failure was logged, not only that the failed run end was written. A new test blocks the interrupt and advances the test clock. Refs #15542 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 19bb632 commit 41e5eb4

2 files changed

Lines changed: 49 additions & 10 deletions

File tree

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,16 @@ import {
2929
} from "@t3tools/contracts";
3030
import * as Cause from "effect/Cause";
3131
import * as Exit from "effect/Exit";
32+
import * as Fiber from "effect/Fiber";
3233
import * as DateTime from "effect/DateTime";
3334
import * as Deferred from "effect/Deferred";
3435
import * as Effect from "effect/Effect";
3536
import * as Layer from "effect/Layer";
37+
import * as Logger from "effect/Logger";
3638
import * as Option from "effect/Option";
3739
import * as Ref from "effect/Ref";
3840
import * as Stream from "effect/Stream";
41+
import * as TestClock from "effect/testing/TestClock";
3942

4043
import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
4144
import * as ServerSettings from "../serverSettings.ts";
@@ -3787,25 +3790,58 @@ it.effect("stops the provider turn when event ingestion fails before the root te
37873790
}),
37883791
);
37893792

3790-
it.effect("still records the failed run when stopping the orphaned provider turn fails", () =>
3791-
Effect.gen(function* () {
3793+
it.effect("still records the failed run when stopping the orphaned provider turn fails", () => {
3794+
const logged: Array<unknown> = [];
3795+
const logger = Logger.make(({ message }) => {
3796+
logged.push(Array.isArray(message) ? message[0] : message);
3797+
});
3798+
return Effect.gen(function* () {
3799+
const attempts = yield* Ref.make(0);
37923800
const { observed } = yield* captureRootRunTermination({
37933801
key: "ingest-failure:interrupt-error",
37943802
shouldFinalizeRun: () => Effect.succeed(true),
37953803
events: (ids) =>
37963804
Stream.make(runningRootProviderTurnEvent(ids)).pipe(Stream.concat(Stream.never)),
37973805
ingestNormalized: failedEventWrite,
37983806
interruptTurn: (request) =>
3799-
Effect.fail(
3800-
new ProviderAdapter.ProviderAdapterInterruptError({
3801-
driver,
3802-
providerThreadId: request.providerThread.id,
3803-
providerTurnId: request.providerTurnId,
3804-
cause: "no live query",
3805-
}),
3807+
Ref.update(attempts, (count) => count + 1).pipe(
3808+
Effect.andThen(
3809+
Effect.fail(
3810+
new ProviderAdapter.ProviderAdapterInterruptError({
3811+
driver,
3812+
providerThreadId: request.providerThread.id,
3813+
providerTurnId: request.providerTurnId,
3814+
cause: "no live query",
3815+
}),
3816+
),
3817+
),
38063818
),
38073819
});
38083820
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3821+
assert.equal(yield* Ref.get(attempts), 1);
3822+
assert.include(
3823+
logged,
3824+
"orchestration V2 could not stop the provider turn after ingestion failed",
3825+
);
3826+
}).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false })));
3827+
});
3828+
3829+
it.effect("gives up on stopping an orphaned provider turn that never answers", () =>
3830+
Effect.gen(function* () {
3831+
const interruptStarted = yield* Deferred.make<void>();
3832+
const capture = yield* captureRootRunTermination({
3833+
key: "ingest-failure:interrupt-hangs",
3834+
shouldFinalizeRun: () => Effect.succeed(true),
3835+
events: (ids) =>
3836+
Stream.make(runningRootProviderTurnEvent(ids)).pipe(Stream.concat(Stream.never)),
3837+
ingestNormalized: failedEventWrite,
3838+
interruptTurn: () =>
3839+
Deferred.succeed(interruptStarted, undefined).pipe(Effect.andThen(Effect.never)),
3840+
}).pipe(Effect.forkChild);
3841+
yield* Deferred.await(interruptStarted);
3842+
yield* TestClock.adjust("15 seconds");
3843+
const { observed } = yield* Fiber.join(capture);
3844+
assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
38093845
}),
38103846
);
38113847

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1015,7 +1015,9 @@ export const layer: Layer.Layer<
10151015
});
10161016
// Failing the run on an ingest error does not end the provider's
10171017
// turn. Stop it, or it keeps running tools nobody sees, and the
1018-
// thread's next turn is refused while it is still active.
1018+
// thread's next turn is refused while it is still active. Bounded,
1019+
// because it runs as a finalizer: a provider that never answers
1020+
// would otherwise hold the ingestion fiber forever.
10191021
const stopOrphanedProviderTurn = Effect.gen(function* () {
10201022
const providerTurnId = (yield* Ref.get(eventRouting)).rootProviderTurnId;
10211023
if (providerTurnId === null) {
@@ -1026,6 +1028,7 @@ export const layer: Layer.Layer<
10261028
providerTurnId,
10271029
});
10281030
}).pipe(
1031+
Effect.timeout("15 seconds"),
10291032
Effect.catchCause((cause) =>
10301033
Effect.logWarning(
10311034
"orchestration V2 could not stop the provider turn after ingestion failed",

0 commit comments

Comments
 (0)