Skip to content

Commit 9085d08

Browse files
SunkenInTimeclaude
andcommitted
fix(server): a release leaves a replacement's status alone even after the replacement closes
Checking whether a replacement is live missed one that opened and closed while the failed session was still closing; the old failure then landed after the replacement's own status. The manager now tracks the latest runtime opened under each id until its release is recorded, and only that runtime writes its release status. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 1186e74 commit 9085d08

2 files changed

Lines changed: 104 additions & 20 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 70 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -336,7 +336,8 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
336336
readonly hasPendingBackgroundWorkForThread?: Effect.Effect<boolean>;
337337
/** False models a runtime whose background tasks can outlive its process. */
338338
readonly backgroundTasksEndWithProcess?: boolean;
339-
readonly hangSessionScopeClose?: boolean;
339+
/** "first" wedges only the first session opened. */
340+
readonly hangSessionScopeClose?: boolean | "first";
340341
readonly startTurn?: Effect.Effect<void>;
341342
readonly beforeUnload?: Effect.Effect<void>;
342343
/** Registers the process's closeCount finalizer before `beforeOpen` runs. */
@@ -401,7 +402,11 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
401402
});
402403
if (options.spawnBeforeOpen !== true) {
403404
yield* countClose;
404-
if (options.hangSessionScopeClose === true) yield* hangClose;
405+
if (
406+
options.hangSessionScopeClose === true ||
407+
(options.hangSessionScopeClose === "first" && (yield* Ref.get(state)).openCount === 1)
408+
)
409+
yield* hangClose;
405410
}
406411

407412
return {
@@ -489,7 +494,7 @@ function layerTest(input: {
489494
readonly hasPendingBackgroundWork?: Effect.Effect<boolean>;
490495
readonly hasPendingBackgroundWorkForThread?: Effect.Effect<boolean>;
491496
readonly backgroundTasksEndWithProcess?: boolean;
492-
readonly hangSessionScopeClose?: boolean;
497+
readonly hangSessionScopeClose?: boolean | "first";
493498
readonly startTurn?: Effect.Effect<void>;
494499
readonly beforeUnload?: Effect.Effect<void>;
495500
readonly spawnBeforeOpen?: boolean;
@@ -3912,6 +3917,68 @@ it.effect(
39123917
}),
39133918
);
39143919

3920+
it.effect("ProviderSessionManagerV2 leaves a closed replacement's status to it", () =>
3921+
Effect.gen(function* () {
3922+
const state = yield* Ref.make(emptyState);
3923+
const scopeCloseReached = yield* Deferred.make<void>();
3924+
yield* Effect.gen(function* () {
3925+
const eventSink = yield* EventSink.EventSinkV2;
3926+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
3927+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
3928+
const threadId = ThreadId.make("thread-provider-session-manager-closed-replacement");
3929+
const providerSessionId = yield* idAllocator.allocate.providerSession({
3930+
providerInstanceId: modelSelection.instanceId,
3931+
threadId,
3932+
});
3933+
yield* eventSink.write({
3934+
events: [
3935+
yield* makeThreadCreatedEvent({ idAllocator, threadId, now: yield* DateTime.now }),
3936+
],
3937+
});
3938+
yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
3939+
3940+
// The first runtime's events end unexpectedly, and its release stalls in scope close.
3941+
const events = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId));
3942+
assert.isDefined(events);
3943+
yield* Queue.end(events!);
3944+
yield* Deferred.await(scopeCloseReached);
3945+
3946+
// A replacement opens and closes meanwhile.
3947+
yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
3948+
yield* manager.close(providerSessionId);
3949+
const statuses = yield* eventSink
3950+
.stream({ threadId, afterSequence: yield* eventSink.latestSequence({ threadId }) })
3951+
.pipe(
3952+
Stream.filter(
3953+
({ event }) =>
3954+
event.type === "provider-session.updated" ||
3955+
event.type === "provider-session.attached",
3956+
),
3957+
Stream.takeUntil(({ event }) => event.type === "provider-session.attached"),
3958+
Stream.map(({ event }) => event.type),
3959+
Stream.runCollect,
3960+
Effect.forkScoped,
3961+
);
3962+
3963+
// The stalled close ends; the next open waits on the open lock until the
3964+
// first release has written its records.
3965+
yield* TestClock.adjust("30 seconds");
3966+
yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
3967+
3968+
assert.deepEqual(Array.from(yield* Fiber.join(statuses)), ["provider-session.attached"]);
3969+
}).pipe(
3970+
Effect.provide(
3971+
layerTest({
3972+
state,
3973+
idleTimeoutMs: 60_000,
3974+
hangSessionScopeClose: "first",
3975+
scopeCloseReached,
3976+
}),
3977+
),
3978+
);
3979+
}),
3980+
);
3981+
39153982
it.effect("ProviderSessionManagerV2 keeps each failed release's cleanup", () =>
39163983
Effect.gen(function* () {
39173984
const state = yield* Ref.make(emptyState);

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 34 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,10 @@ export const layerWithOptions = (
421421
}),
422422
);
423423
const sessions = yield* Ref.make(new Map<string, LiveSessionEntry>());
424+
// The latest session opened under each id, kept until its release records it.
425+
const latestOpened = yield* Ref.make(
426+
new Map<string, ProviderAdapter.ProviderAdapterV2SessionRuntime>(),
427+
);
424428
// One retry per released entry, so a later release with the same id
425429
// cannot drop cleanup for threads only the earlier session served.
426430
const releaseRecordRetries = yield* FiberSet.make();
@@ -917,6 +921,27 @@ export const layerWithOptions = (
917921
{ concurrency: 1 },
918922
).pipe(Effect.flatMap(Exit.asVoidAll));
919923

924+
// A replacement under the same id, like the next thread of a shared
925+
// runtime, owns the session's status from its open on, even after it
926+
// closes. Only the latest session opened under an id writes its release
927+
// status; the entry is dropped once that is written. Callers hold the
928+
// open lock, so no replacement opens mid-write.
929+
const writeOwnReleaseRecords = (
930+
input: Omit<Parameters<typeof writeReleaseRecords>[0], "replaced">,
931+
) =>
932+
Effect.gen(function* () {
933+
const key = sessionKey(input.entry.runtime.providerSessionId);
934+
const replaced = (yield* Ref.get(latestOpened)).get(key) !== input.entry.runtime;
935+
yield* writeReleaseRecords({ ...input, replaced });
936+
if (replaced) return;
937+
yield* Ref.update(latestOpened, (current) => {
938+
if (current.get(key) !== input.entry.runtime) return current;
939+
const updated = new Map(current);
940+
updated.delete(key);
941+
return updated;
942+
});
943+
});
944+
920945
// The session already left the live map, so a later release finds
921946
// nothing to do. Without a retry the UI would keep a ready session and
922947
// answerable approvals until a server restart. Each attempt holds the
@@ -927,13 +952,7 @@ export const layerWithOptions = (
927952
const providerSessionId = input.entry.runtime.providerSessionId;
928953
const attempt = Effect.gen(function* () {
929954
const exit = yield* Effect.exit(
930-
sessionOpen.withLock(
931-
providerSessionId,
932-
Effect.gen(function* () {
933-
const replaced = (yield* Ref.get(sessions)).has(sessionKey(providerSessionId));
934-
yield* writeReleaseRecords({ ...input, replaced });
935-
}),
936-
),
955+
sessionOpen.withLock(providerSessionId, writeOwnReleaseRecords(input)),
937956
);
938957
if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) return yield* exit;
939958
yield* Effect.logWarning("orchestration-v2.provider-session-release-records-failed", {
@@ -1102,18 +1121,15 @@ export const layerWithOptions = (
11021121
releasedAt,
11031122
};
11041123
// A thread of a shared runtime can reopen it under the same
1105-
// id while this closes. The open lock orders the check and
1106-
// write against that, so a replacement keeps its status.
1107-
const writeRecords = Effect.gen(function* () {
1108-
const replaced = (yield* Ref.get(sessions)).has(
1109-
sessionKey(input.providerSessionId),
1110-
);
1111-
yield* writeReleaseRecords({ ...records, replaced });
1112-
});
1124+
// id while this closes; the open lock orders that against
1125+
// this write.
11131126
const recorded = yield* Effect.exit(
11141127
input.openLockHeld === true
1115-
? writeRecords
1116-
: sessionOpen.withLock(input.providerSessionId, writeRecords),
1128+
? writeOwnReleaseRecords(records)
1129+
: sessionOpen.withLock(
1130+
input.providerSessionId,
1131+
writeOwnReleaseRecords(records),
1132+
),
11171133
);
11181134
if (Exit.isFailure(recorded)) {
11191135
yield* retryReleaseRecords(records);
@@ -2331,6 +2347,7 @@ export const layerWithOptions = (
23312347
updated.set(key, entry);
23322348
return updated;
23332349
});
2350+
yield* Ref.update(latestOpened, (current) => new Map(current).set(key, runtime));
23342351
// The entry now guards the credential via its recorded id, so
23352352
// the pre-open reservation can be dropped.
23362353
yield* dropReservation;

0 commit comments

Comments
 (0)