Skip to content

Commit 196060d

Browse files
t3dotggclaude
andcommitted
fix(server): settled and idle-released sessions no longer leave threads Waiting
When a provider session is released (Settle or archive detaching it, or idle release), its process is gone, but the background-task roster it kept on its provider threads stayed. The thread kept showing Waiting with a Stop button until the user pressed Stop. Release now records the task ids on the released session's provider threads as soon as the session leaves the live map, including the thread whose detach released it, and removes those ids once the process has closed. Tasks a same-id replacement session reports meanwhile stay. Server shutdown keeps rosters for startup recovery. Fixes #15239 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent bc95bbd commit 196060d

2 files changed

Lines changed: 270 additions & 3 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2769,6 +2769,169 @@ it.effect("ProviderSessionManagerV2 persists release when session scope close ha
27692769
}),
27702770
);
27712771

2772+
// Seeds a thread with a background command on a provider thread of the
2773+
// session it then opens, and one on a provider thread of another session.
2774+
const openSessionWithBackgroundCommand = (fixtureName: string) =>
2775+
Effect.gen(function* () {
2776+
const eventSink = yield* EventSink.EventSinkV2;
2777+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
2778+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
2779+
const now = yield* DateTime.now;
2780+
const threadId = yield* idAllocator.allocate.thread({
2781+
fixtureName,
2782+
projectId: yield* idAllocator.allocate.project({ fixtureName }),
2783+
});
2784+
const providerSessionId = yield* idAllocator.allocate.providerSession({
2785+
providerInstanceId: modelSelection.instanceId,
2786+
threadId,
2787+
});
2788+
const otherProviderSessionId = yield* idAllocator.allocate.providerSession({
2789+
providerInstanceId: modelSelection.instanceId,
2790+
threadId,
2791+
});
2792+
const pendingBackgroundTasks = [
2793+
{ taskId: "dev-server", kind: "command", description: "python3 -m http.server" },
2794+
] as const;
2795+
const providerThreads = [
2796+
{
2797+
...makeProviderThread({ idAllocator, threadId, providerSessionId, now }),
2798+
pendingBackgroundTasks,
2799+
},
2800+
{
2801+
...makeProviderThread({
2802+
idAllocator,
2803+
threadId,
2804+
providerSessionId: otherProviderSessionId,
2805+
now,
2806+
nativeThreadId: "native-thread-other-session",
2807+
}),
2808+
pendingBackgroundTasks,
2809+
},
2810+
];
2811+
const writeProviderThread = (payload: OrchestrationV2ProviderThread) =>
2812+
Effect.gen(function* () {
2813+
yield* eventSink.write({
2814+
events: [
2815+
{
2816+
id: yield* idAllocator.allocate.event({ threadId }),
2817+
type: "provider-thread.updated",
2818+
threadId,
2819+
driver: CODEX_DRIVER,
2820+
occurredAt: now,
2821+
payload,
2822+
},
2823+
],
2824+
});
2825+
});
2826+
yield* eventSink.write({
2827+
events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
2828+
});
2829+
yield* Effect.forEach(providerThreads, writeProviderThread);
2830+
yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
2831+
// The task ids each seeded provider thread lists now.
2832+
const rosters = Effect.gen(function* () {
2833+
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2834+
const projection = yield* projectionStore.getThreadProjection(threadId);
2835+
return providerThreads.map((seeded) =>
2836+
(
2837+
projection.providerThreads.find((thread) => thread.id === seeded.id)
2838+
?.pendingBackgroundTasks ?? []
2839+
).map((task) => task.taskId),
2840+
);
2841+
});
2842+
return { threadId, providerSessionId, providerThreads, writeProviderThread, rosters };
2843+
});
2844+
2845+
it.effect(
2846+
"ProviderSessionManagerV2 clears the background roster of a session a detach releases",
2847+
() =>
2848+
Effect.gen(function* () {
2849+
const state = yield* Ref.make(emptyState);
2850+
const effect = Effect.gen(function* () {
2851+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
2852+
const { threadId, providerSessionId, rosters } = yield* openSessionWithBackgroundCommand(
2853+
"provider-session-manager-detach-roster",
2854+
);
2855+
assert.deepStrictEqual(yield* rosters, [["dev-server"], ["dev-server"]]);
2856+
2857+
// Settle and archive detach the thread, which releases an exclusive
2858+
// (Claude-like) session.
2859+
yield* manager.detach({ providerSessionId, threadId, detail: "Thread settled." });
2860+
2861+
assert.isTrue(Option.isNone(yield* manager.get(providerSessionId)));
2862+
assert.deepStrictEqual(yield* rosters, [[], ["dev-server"]]);
2863+
});
2864+
2865+
yield* effect.pipe(
2866+
Effect.provide(
2867+
layerTest({ state, idleTimeoutMs: 60_000, capabilities: ExclusiveCapabilities }),
2868+
),
2869+
);
2870+
}),
2871+
);
2872+
2873+
it.effect("ProviderSessionManagerV2 clears the background roster of an idle-released session", () =>
2874+
Effect.gen(function* () {
2875+
const state = yield* Ref.make(emptyState);
2876+
const effect = Effect.gen(function* () {
2877+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
2878+
const { providerSessionId, rosters } = yield* openSessionWithBackgroundCommand(
2879+
"provider-session-manager-idle-roster",
2880+
);
2881+
assert.deepStrictEqual(yield* rosters, [["dev-server"], ["dev-server"]]);
2882+
2883+
yield* TestClock.adjust("1 second");
2884+
yield* Effect.yieldNow;
2885+
2886+
assert.isTrue(Option.isNone(yield* manager.get(providerSessionId)));
2887+
assert.deepStrictEqual(yield* rosters, [[], ["dev-server"]]);
2888+
});
2889+
2890+
yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 })));
2891+
}),
2892+
);
2893+
2894+
it.effect(
2895+
"ProviderSessionManagerV2 keeps background tasks a replacement reports during release",
2896+
() =>
2897+
Effect.gen(function* () {
2898+
const state = yield* Ref.make(emptyState);
2899+
const scopeCloseReached = yield* Deferred.make<void>();
2900+
const effect = Effect.gen(function* () {
2901+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
2902+
const { providerSessionId, providerThreads, writeProviderThread, rosters } =
2903+
yield* openSessionWithBackgroundCommand("provider-session-manager-replacement-roster");
2904+
const release = yield* manager.close(providerSessionId).pipe(Effect.forkChild);
2905+
2906+
// While the old process closes, a replacement session with the same id
2907+
// reports a task of its own next to the dead one.
2908+
yield* Deferred.await(scopeCloseReached);
2909+
yield* writeProviderThread({
2910+
...providerThreads[0]!,
2911+
pendingBackgroundTasks: [
2912+
...providerThreads[0]!.pendingBackgroundTasks,
2913+
{ taskId: "replacement-task", kind: "command" },
2914+
],
2915+
});
2916+
yield* TestClock.adjust("30 seconds");
2917+
yield* Fiber.join(release);
2918+
2919+
assert.deepStrictEqual(yield* rosters, [["replacement-task"], ["dev-server"]]);
2920+
});
2921+
2922+
yield* effect.pipe(
2923+
Effect.provide(
2924+
layerTest({
2925+
state,
2926+
idleTimeoutMs: 60_000,
2927+
hangSessionScopeClose: true,
2928+
scopeCloseReached,
2929+
}),
2930+
),
2931+
);
2932+
}),
2933+
);
2934+
27722935
it.effect("ProviderSessionManagerV2 defers idle release while background work is pending", () =>
27732936
Effect.gen(function* () {
27742937
const state = yield* Ref.make(emptyState);

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 107 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -774,14 +774,106 @@ export const layerWithOptions = (
774774
}
775775
});
776776

777+
// Reads the background tasks a released session's process left on its
778+
// provider threads. Read as soon as the session leaves the live map:
779+
// after that, a replacement session with the same id can open and report
780+
// tasks of its own on the same provider threads. Server shutdown keeps
781+
// rosters, since startup recovery clears them and records the cancelled
782+
// work.
783+
const readReleasedBackgroundTasks = (input: {
784+
readonly entry: LiveSessionEntry;
785+
readonly reason: ProviderSessionReleaseReason;
786+
readonly detachedThreadId?: ThreadId;
787+
}) =>
788+
Effect.gen(function* () {
789+
if (input.reason === "server_shutdown") return [];
790+
const providerSessionId = input.entry.runtime.providerSessionId;
791+
// A detach that released the session already removed its thread
792+
// from the attached set.
793+
const threadIds = new Set(input.entry.attachedThreadIds);
794+
if (input.detachedThreadId !== undefined) threadIds.add(input.detachedThreadId);
795+
const released = [];
796+
for (const threadId of threadIds) {
797+
const projection = yield* projectionStore.getThreadRecords(threadId, [
798+
"providerThreads",
799+
]);
800+
for (const providerThread of projection.providerThreads) {
801+
const tasks = providerThread.pendingBackgroundTasks ?? [];
802+
if (providerThread.providerSessionId !== providerSessionId || tasks.length === 0) {
803+
continue;
804+
}
805+
released.push({
806+
threadId,
807+
providerSessionId,
808+
providerThreadId: providerThread.id,
809+
taskIds: new Set(tasks.map((task) => task.taskId)),
810+
});
811+
}
812+
}
813+
return released;
814+
}).pipe(
815+
Effect.catchCause((cause) =>
816+
Effect.logWarning("orchestration-v2.provider-session-release-roster-read-failed", {
817+
providerSessionId: input.entry.runtime.providerSessionId,
818+
cause,
819+
}).pipe(Effect.as([])),
820+
),
821+
);
822+
type ReleasedBackgroundTasks = Effect.Success<
823+
ReturnType<typeof readReleasedBackgroundTasks>
824+
>[number];
825+
826+
// Removes the tasks `readReleasedBackgroundTasks` found from their
827+
// rosters. Their process is gone, so nothing will report them ending,
828+
// and a leftover roster keeps the thread Waiting (#15239). Tasks a
829+
// replacement session reported since are left in place.
830+
const writeReleasedBackgroundWorkEvents = (
831+
released: ReadonlyArray<ReleasedBackgroundTasks>,
832+
) =>
833+
Effect.forEach(
834+
released,
835+
({ threadId, providerSessionId, providerThreadId, taskIds }) =>
836+
Effect.gen(function* () {
837+
const projection = yield* projectionStore.getThreadRecords(threadId, [
838+
"providerThreads",
839+
]);
840+
const providerThread = projection.providerThreads.find(
841+
(candidate) => candidate.id === providerThreadId,
842+
);
843+
const tasks = providerThread?.pendingBackgroundTasks ?? [];
844+
const remaining = tasks.filter((task) => !taskIds.has(task.taskId));
845+
if (providerThread === undefined || remaining.length === tasks.length) return;
846+
const now = yield* DateTime.now;
847+
yield* eventSink.write({
848+
events: [
849+
{
850+
id: yield* idAllocator.allocate.event({ threadId, providerSessionId }),
851+
type: "provider-thread.updated",
852+
threadId,
853+
driver: providerThread.driver,
854+
providerInstanceId: providerThread.providerInstanceId,
855+
occurredAt: now,
856+
payload: {
857+
...providerThread,
858+
pendingBackgroundTasks: remaining,
859+
updatedAt: now,
860+
},
861+
},
862+
],
863+
});
864+
}),
865+
{ discard: true },
866+
);
867+
777868
// Records a released session as stopped and resolves the live runtime
778-
// requests it left. Each write runs even if the other fails. Once a
779-
// replacement session opens with the same id, it owns the session status,
780-
// so only the requests are settled.
869+
// requests and background-task rosters it left. Each write runs even if
870+
// another fails. Once a replacement session opens with the same id, it
871+
// owns the session status, so only the requests are settled.
781872
const writeReleaseRecords = (input: {
782873
readonly entry: LiveSessionEntry;
783874
readonly reason: ProviderSessionReleaseReason;
784875
readonly detail?: string;
876+
readonly releasedBackgroundTasks: ReadonlyArray<ReleasedBackgroundTasks>;
785877
readonly releasedAt: DateTime.Utc;
786878
readonly replaced: boolean;
787879
}) =>
@@ -795,6 +887,7 @@ export const layerWithOptions = (
795887
input.entry.requestEventPermit.withPermits(1),
796888
),
797889
),
890+
Effect.exit(writeReleasedBackgroundWorkEvents(input.releasedBackgroundTasks)),
798891
],
799892
{ concurrency: 1 },
800893
).pipe(Effect.flatMap(Exit.asVoidAll));
@@ -937,6 +1030,8 @@ export const layerWithOptions = (
9371030
readonly providerSessionId: ProviderSessionId;
9381031
readonly reason: ProviderSessionReleaseReason;
9391032
readonly detail?: string;
1033+
/** The thread whose detach released the session; it is no longer attached. */
1034+
readonly detachedThreadId?: ThreadId;
9401035
readonly cancelIdleFiber?: boolean;
9411036
readonly onlyIfIdleGeneration?: number;
9421037
readonly gracefulSubscribers?: boolean;
@@ -948,6 +1043,13 @@ export const layerWithOptions = (
9481043
onNone: () => Effect.void,
9491044
onSome: (entry) =>
9501045
Effect.gen(function* () {
1046+
const releasedBackgroundTasks = yield* readReleasedBackgroundTasks({
1047+
entry,
1048+
reason: input.reason,
1049+
...(input.detachedThreadId === undefined
1050+
? {}
1051+
: { detachedThreadId: input.detachedThreadId }),
1052+
});
9511053
if (input.cancelIdleFiber !== false) {
9521054
yield* cancelIdleFiber(entry.idleFiber);
9531055
}
@@ -969,6 +1071,7 @@ export const layerWithOptions = (
9691071
entry,
9701072
reason: input.reason,
9711073
...(input.detail === undefined ? {} : { detail: input.detail }),
1074+
releasedBackgroundTasks,
9721075
releasedAt,
9731076
};
9741077
const recorded = yield* Effect.exit(
@@ -2406,6 +2509,7 @@ export const layerWithOptions = (
24062509
providerSessionId: input.providerSessionId,
24072510
reason: "manual_shutdown",
24082511
...(input.detail === undefined ? {} : { detail: input.detail }),
2512+
detachedThreadId: input.threadId,
24092513
});
24102514
return;
24112515
}

0 commit comments

Comments
 (0)