Skip to content

Commit 9954e9b

Browse files
committed
fix(server): drain provider events before releasing startup ownership
Retain session subscription termination atomically so buffered terminal and background events drain and late subscribers receive the same closure. Stop cancelling the ingestion source independently of its commit coordinator. Mark ACP turn configuration and context-handoff delivery failures at their definite pre-prompt boundary, preserving ambiguous native-start ownership. Include attachment refusal and process-bound recovery corrections, with late-subscription and provider-switch coverage. Verified pinned check:changed (499 tests) and targeted CI adapter/replay/batching suites (213 tests). Implemented with openai-codex/gpt-6.1-sol in Pi.
1 parent 6e3d432 commit 9954e9b

10 files changed

Lines changed: 150 additions & 91 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -773,6 +773,7 @@ it.effect(
773773
assert.deepEqual(committedInput?.cancelUnsettledEffects?.effectTypes, [
774774
"provider-turn.start",
775775
"provider-turn.interrupt",
776+
"provider-turn.interrupt-start",
776777
"provider-turn.steer",
777778
"provider-turn.restart",
778779
"runtime-request.respond",

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1946,8 +1946,10 @@ it.effect("ProviderSessionManagerV2 interrupts event subscriptions on server shu
19461946

19471947
yield* manager.shutdown;
19481948

1949-
const exit = yield* bufferedSubscription.events.pipe(Stream.runCollect, Effect.exit);
1950-
assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause));
1949+
for (const subscription of [bufferedSubscription, yield* runtime.subscribeEvents!]) {
1950+
const exit = yield* subscription.events.pipe(Stream.runCollect, Effect.exit);
1951+
assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause));
1952+
}
19511953
});
19521954

19531955
yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000 })));
@@ -1977,7 +1979,10 @@ it.effect("ProviderSessionManagerV2 drains subscribers when the provider stops",
19771979
runtimePolicy,
19781980
});
19791981
const subscription = yield* runtime.subscribeEvents!;
1980-
const collected = yield* subscription.events.pipe(Stream.runCollect, Effect.forkScoped);
1982+
const closed = yield* Deferred.make<void>();
1983+
const unsubscribe = yield* runtime.onSessionClosed!(
1984+
Deferred.succeed(closed, undefined).pipe(Effect.asVoid),
1985+
);
19811986
const adapterQueue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId));
19821987
assert.isDefined(adapterQueue);
19831988
const providerThreadId = idAllocator.derive.providerThread({
@@ -2009,11 +2014,15 @@ it.effect("ProviderSessionManagerV2 drains subscribers when the provider stops",
20092014
});
20102015
yield* Queue.end(adapterQueue!);
20112016

2012-
const events = Array.from(yield* Fiber.join(collected));
2017+
yield* Deferred.await(closed);
2018+
const events = Array.from(yield* subscription.events.pipe(Stream.runCollect));
20132019
assert.deepEqual(
20142020
events.map((event) => event.type),
20152021
["turn.terminal", "provider_session.updated"],
20162022
);
2023+
const late = yield* runtime.subscribeEvents!;
2024+
assert.deepEqual(Array.from(yield* late.events.pipe(Stream.runCollect)), []);
2025+
yield* unsubscribe;
20172026
assert.isTrue(Option.isNone(yield* manager.get(providerSessionId)));
20182027
assert.equal((yield* Ref.get(state)).closeCount, 1);
20192028
});
@@ -3055,6 +3064,9 @@ it.effect("ProviderSessionManagerV2 releases sessions when provider event stream
30553064
const projection = yield* projectionStore.getThreadProjection(threadId);
30563065

30573066
assert.isTrue(Option.isNone(liveSession));
3067+
const late = yield* runtime.subscribeEvents!;
3068+
const lateExit = yield* late.events.pipe(Stream.runCollect, Effect.exit);
3069+
assert.isTrue(Exit.isFailure(lateExit) && !Cause.hasInterruptsOnly(lateExit.cause));
30583070
assert.equal(runtimeState.closeCount, 1);
30593071
assert.equal(projection.providerSessions.at(-1)?.status, "error");
30603072
assert.equal(

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 62 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -223,9 +223,7 @@ interface LiveSessionEntry {
223223
readonly supportsMultipleProviderThreads: boolean;
224224
readonly runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime;
225225
readonly exposedRuntime: ProviderAdapter.ProviderAdapterV2SessionRuntime;
226-
readonly eventSubscribers: Ref.Ref<
227-
ReadonlyMap<number, Queue.Queue<ProviderSessionEventSignal, Cause.Done>>
228-
>;
226+
readonly eventSubscribers: Ref.Ref<ProviderSessionSubscribers>;
229227
readonly closeObservers: Ref.Ref<ReadonlySet<Effect.Effect<void>> | null>;
230228
readonly requestEventPermit: Semaphore.Semaphore;
231229
readonly scope: Scope.Closeable;
@@ -249,6 +247,16 @@ type ProviderSessionEventSignal =
249247
readonly cause: Cause.Cause<ProviderAdapter.ProviderAdapterV2Error>;
250248
};
251249

250+
type ProviderSessionEventTermination =
251+
| { readonly type: "end" }
252+
| { readonly type: "interrupt" }
253+
| Extract<ProviderSessionEventSignal, { readonly type: "failure" }>;
254+
255+
interface ProviderSessionSubscribers {
256+
readonly queues: ReadonlyMap<number, Queue.Queue<ProviderSessionEventSignal, Cause.Done>>;
257+
readonly terminal: ProviderSessionEventTermination | null;
258+
}
259+
252260
export interface ProviderSessionManagerV2LayerOptions {
253261
readonly idleTimeoutMs?: number;
254262
readonly maxIdlePinMs?: number;
@@ -519,64 +527,52 @@ export const layerWithOptions = (
519527
);
520528

521529
const publishToSubscribers = (
522-
subscribers: Ref.Ref<
523-
ReadonlyMap<number, Queue.Queue<ProviderSessionEventSignal, Cause.Done>>
524-
>,
530+
subscribers: LiveSessionEntry["eventSubscribers"],
525531
signal: ProviderSessionEventSignal,
526532
) =>
527533
Ref.get(subscribers).pipe(
528534
Effect.flatMap((current) =>
529-
Effect.forEach(current.values(), (queue) => Queue.offer(queue, signal), {
535+
Effect.forEach(current.queues.values(), (queue) => Queue.offer(queue, signal), {
530536
discard: true,
531537
}),
532538
),
533539
);
534540

535-
const failSubscribers = Effect.fn("ProviderSessionManager.failSubscribers")(function* (
536-
entry: LiveSessionEntry,
537-
detail: string,
538-
) {
539-
const error = new ProviderAdapter.ProviderAdapterEventStreamError({
540-
driver: entry.runtime.driver,
541-
providerSessionId: entry.runtime.providerSessionId,
542-
cause: detail,
543-
});
544-
const subscribers = yield* Ref.getAndSet(entry.eventSubscribers, new Map());
545-
yield* Effect.forEach(
546-
subscribers.values(),
547-
(queue) =>
548-
Queue.offer(queue, {
549-
type: "failure",
550-
cause: Cause.fail(error),
551-
}),
552-
{ discard: true },
553-
);
554-
});
541+
const terminateQueue = (
542+
queue: Queue.Queue<ProviderSessionEventSignal, Cause.Done>,
543+
terminal: ProviderSessionEventTermination,
544+
) =>
545+
terminal.type === "interrupt"
546+
? Queue.clear(queue).pipe(Effect.andThen(Queue.interrupt(queue)), Effect.asVoid)
547+
: (terminal.type === "failure" ? Queue.offer(queue, terminal) : Effect.void).pipe(
548+
Effect.andThen(Queue.end(queue)),
549+
Effect.asVoid,
550+
);
555551

556-
// Server shutdown interrupts subscribers rather than ending them: a
557-
// cleanly ended stream without a terminal fails its run, while an
558-
// interrupted one leaves run settlement to restart recovery.
559-
const interruptSubscribers = Effect.fn("ProviderSessionManager.interruptSubscribers")(
560-
function* (entry: LiveSessionEntry) {
561-
const subscribers = yield* Ref.getAndSet(entry.eventSubscribers, new Map());
562-
yield* Effect.forEach(
563-
subscribers.values(),
564-
(queue) => Queue.clear(queue).pipe(Effect.andThen(Queue.interrupt(queue))),
565-
{ discard: true },
552+
const terminateSubscribers = Effect.fn("ProviderSessionManager.terminateSubscribers")(
553+
function* (entry: LiveSessionEntry, terminal: ProviderSessionEventTermination) {
554+
const queues = yield* Ref.modify(entry.eventSubscribers, (current) =>
555+
current.terminal === null
556+
? [current.queues, { queues: new Map(), terminal }]
557+
: [current.queues, current],
566558
);
559+
yield* Effect.forEach(queues.values(), (queue) => terminateQueue(queue, terminal), {
560+
discard: true,
561+
});
567562
},
568563
);
569564

570-
// Server shutdown intentionally clears them; a provider-announced Stop
571-
// must let consumers drain them before the stream completes.
572-
const endSubscribers = Effect.fn("ProviderSessionManager.endSubscribers")(function* (
573-
entry: LiveSessionEntry,
574-
) {
575-
const subscribers = yield* Ref.getAndSet(entry.eventSubscribers, new Map());
576-
yield* Effect.forEach(subscribers.values(), (queue) => Queue.end(queue), {
577-
discard: true,
565+
const failSubscribers = (entry: LiveSessionEntry, detail: string) =>
566+
terminateSubscribers(entry, {
567+
type: "failure",
568+
cause: Cause.fail(
569+
new ProviderAdapter.ProviderAdapterEventStreamError({
570+
driver: entry.runtime.driver,
571+
providerSessionId: entry.runtime.providerSessionId,
572+
cause: detail,
573+
}),
574+
),
578575
});
579-
});
580576

581577
const cancelIdleFiber = (fiber: Fiber.Fiber<void, never> | null) =>
582578
fiber === null ? Effect.void : Fiber.interrupt(fiber).pipe(Effect.ignore);
@@ -1046,9 +1042,10 @@ export const layerWithOptions = (
10461042
yield* cancelIdleFiber(entry.idleFiber);
10471043
}
10481044
if (input.gracefulSubscribers === true) {
1049-
yield* endSubscribers(entry);
1045+
yield* terminateSubscribers(entry, { type: "end" });
10501046
} else if (input.reason === "server_shutdown") {
1051-
yield* interruptSubscribers(entry);
1047+
// Shutdown discards buffered work; interruption leaves settlement to recovery.
1048+
yield* terminateSubscribers(entry, { type: "interrupt" });
10521049
} else {
10531050
yield* failSubscribers(
10541051
entry,
@@ -1677,24 +1674,24 @@ export const layerWithOptions = (
16771674

16781675
const makeEventSubscription = Effect.fn("ProviderSessionManager.makeEventSubscription")(
16791676
function* (
1680-
subscribers: Ref.Ref<
1681-
ReadonlyMap<number, Queue.Queue<ProviderSessionEventSignal, Cause.Done>>
1682-
>,
1677+
subscribers: LiveSessionEntry["eventSubscribers"],
16831678
): Effect.fn.Return<ProviderAdapter.ProviderAdapterV2EventSubscription> {
16841679
const queue = yield* Queue.unbounded<ProviderSessionEventSignal, Cause.Done>();
16851680
const subscriberId = yield* Ref.getAndUpdate(nextSubscriberId, (value) => value + 1);
1686-
yield* Ref.update(subscribers, (current) => {
1687-
const updated = new Map(current);
1688-
updated.set(subscriberId, queue);
1689-
return updated;
1681+
const terminal = yield* Ref.modify(subscribers, (current) => {
1682+
if (current.terminal !== null) return [current.terminal, current] as const;
1683+
const queues = new Map(current.queues);
1684+
queues.set(subscriberId, queue);
1685+
return [null, { ...current, queues }] as const;
16901686
});
1687+
if (terminal !== null) yield* terminateQueue(queue, terminal);
16911688
const close = Ref.modify(subscribers, (current) => {
1692-
if (!current.has(subscriberId)) {
1689+
if (!current.queues.has(subscriberId)) {
16931690
return [false, current] as const;
16941691
}
1695-
const updated = new Map(current);
1696-
updated.delete(subscriberId);
1697-
return [true, updated] as const;
1692+
const queues = new Map(current.queues);
1693+
queues.delete(subscriberId);
1694+
return [true, { ...current, queues }] as const;
16981695
}).pipe(
16991696
Effect.flatMap((removed) =>
17001697
removed
@@ -1716,9 +1713,7 @@ export const layerWithOptions = (
17161713

17171714
const decorateRuntime = (
17181715
runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime,
1719-
eventSubscribers: Ref.Ref<
1720-
ReadonlyMap<number, Queue.Queue<ProviderSessionEventSignal, Cause.Done>>
1721-
>,
1716+
eventSubscribers: LiveSessionEntry["eventSubscribers"],
17221717
closeObservers: LiveSessionEntry["closeObservers"],
17231718
): ProviderAdapter.ProviderAdapterV2SessionRuntime => {
17241719
const providerSessionId = runtime.providerSessionId;
@@ -2116,11 +2111,7 @@ export const layerWithOptions = (
21162111
cause: "Provider event stream ended unexpectedly.",
21172112
}),
21182113
);
2119-
yield* publishToSubscribers(entry.eventSubscribers, {
2120-
type: "failure",
2121-
cause,
2122-
});
2123-
yield* Ref.set(entry.eventSubscribers, new Map());
2114+
yield* terminateSubscribers(entry, { type: "failure", cause });
21242115
// lastError is the thread banner, so it gets the same bounded
21252116
// message as a failed turn; the raw cause survives only in the log.
21262117
yield* Effect.logWarning("orchestration-v2.provider-session-runtime-failed", {
@@ -2279,9 +2270,10 @@ export const layerWithOptions = (
22792270
attributes: { provider: adapter.driver, operation: "open" },
22802271
}),
22812272
);
2282-
const eventSubscribers = yield* Ref.make<
2283-
ReadonlyMap<number, Queue.Queue<ProviderSessionEventSignal, Cause.Done>>
2284-
>(new Map());
2273+
const eventSubscribers = yield* Ref.make<ProviderSessionSubscribers>({
2274+
queues: new Map(),
2275+
terminal: null,
2276+
});
22852277
const closeObservers = yield* Ref.make<ReadonlySet<Effect.Effect<void>> | null>(
22862278
new Set(),
22872279
);

‎apps/server/src/orchestration-v2/ProviderStartStop.integration.test.ts‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ it.effect.each([
5353
"identity-timeout-force",
5454
"unsupported-compact",
5555
"prompt-refusal",
56+
"attachment-refusal",
5657
"manager-release",
5758
"manager-release-interrupt",
5859
"provider-stopped",
@@ -117,16 +118,23 @@ it.effect.each([
117118
turn,
118119
) =>
119120
Effect.gen(function* () {
120-
if (scenario === "prompt-refusal") {
121+
if (scenario === "prompt-refusal" || scenario === "attachment-refusal") {
121122
return yield* new ProviderAdapter.ProviderAdapterTurnStartError({
122123
driver,
123124
threadId: turn.threadId,
124125
providerThreadId: turn.providerThread.id,
125126
runId: turn.runId,
126-
cause: {
127-
_tag: "ClaudeAppendSystemPromptFileError",
128-
path: "/task-owned/missing-file",
129-
},
127+
cause:
128+
scenario === "attachment-refusal"
129+
? new ProviderAdapter.ProviderAdapterProtocolError({
130+
driver,
131+
reason: "attachment_unreadable",
132+
detail: "Synthetic unreadable attachment before prompt delivery",
133+
})
134+
: {
135+
_tag: "ClaudeAppendSystemPromptFileError",
136+
path: "/task-owned/missing-file",
137+
},
130138
});
131139
}
132140
yield* Deferred.succeed(entered, undefined);
@@ -387,6 +395,7 @@ it.effect.each([
387395
if (
388396
scenario === "unsupported-compact" ||
389397
scenario === "prompt-refusal" ||
398+
scenario === "attachment-refusal" ||
390399
lateSubscription
391400
) {
392401
yield* Fiber.join(startup);

‎apps/server/src/orchestration-v2/ProviderTurnStartService.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1582,7 +1582,15 @@ export const layer: Layer.Layer<
15821582
}),
15831583
}),
15841584
persist: persistHandoff,
1585-
});
1585+
}).pipe(
1586+
Effect.mapError(
1587+
(cause) =>
1588+
new ProviderAdapter.ProviderAdapterTurnPreparationError({
1589+
driver: session.driver,
1590+
cause,
1591+
}),
1592+
),
1593+
);
15861594
if (!(yield* isCurrentAttemptInStatus("running"))) return;
15871595
const start = compact ? session.compactThread! : session.startTurn;
15881596
// Like history, the side chat context waits for a turn that sends a prompt.

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1763,23 +1763,16 @@ export const layer: Layer.Layer<
17631763
}),
17641764
),
17651765
);
1766-
// Only an out-of-stream closure observer needs to wake an idle subscription.
1767-
const providerEvents =
1768-
input.session.onSessionClosed === undefined
1769-
? rawProviderEvents
1770-
: rawProviderEvents.pipe(Stream.interruptWhen(Deferred.await(sessionClosed)));
17711766
const providerEventFiber = yield* eventSink
17721767
.consumeProviderEvents(
1773-
providerEvents,
1768+
rawProviderEvents,
17741769
(event) =>
17751770
Effect.gen(function* () {
17761771
yield* ingestEvent(yield* routeEvent(event));
17771772
}),
17781773
Effect.gen(function* () {
17791774
if (yield* agentTurnsSettled) yield* endPreviewActivity;
1780-
return (
1781-
(yield* Deferred.isDone(sessionClosed)) || (yield* shouldStopProviderEventIngestion)
1782-
);
1775+
return yield* shouldStopProviderEventIngestion;
17831776
}),
17841777
Ref.get(rootTerminalSeen).pipe(Effect.map((seen) => !seen)),
17851778
)

‎apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,21 @@ function makeTestAdapter(input: {
249249
input.failStartOnce !== undefined &&
250250
(yield* Ref.getAndSet(input.failStartOnce, false))
251251
)
252-
return yield* unimplemented(input.driver, "turn start failed after injection");
252+
return yield* unimplemented(input.driver, "turn start failed after injection").pipe(
253+
Effect.mapError(
254+
(cause) =>
255+
new ProviderAdapter.ProviderAdapterTurnStartError({
256+
driver: input.driver,
257+
threadId: turnInput.threadId,
258+
providerThreadId: turnInput.providerThread.id,
259+
runId: turnInput.runId,
260+
cause: new ProviderAdapter.ProviderAdapterTurnPreparationError({
261+
driver: input.driver,
262+
cause,
263+
}),
264+
}),
265+
),
266+
);
253267
if (
254268
input.refuseStarts !== undefined &&
255269
(yield* Ref.getAndUpdate(input.refuseStarts, (left) => Math.max(0, left - 1))) > 0

0 commit comments

Comments
 (0)