Skip to content

Commit 387a592

Browse files
SunkenInTimeclaude
andcommitted
fix(server): a release leaves a replacement's status alone even after the replacement closes
Checking whether a replacement is live missed one that opened and closed while the failed session was still closing; the old failure then landed after the replacement's own status. The manager now tracks the latest runtime opened under each id until its release is recorded, and only that runtime writes its release status. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent fa4e324 commit 387a592

2 files changed

Lines changed: 104 additions & 20 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderSessionManager.test.ts‎

Lines changed: 70 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,8 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
318318
}) => Effect.Effect<void>;
319319
readonly hasPendingBackgroundWork?: Effect.Effect<boolean>;
320320
readonly hasPendingBackgroundWorkForThread?: Effect.Effect<boolean>;
321-
readonly hangSessionScopeClose?: boolean;
321+
/** "first" wedges only the first session opened. */
322+
readonly hangSessionScopeClose?: boolean | "first";
322323
readonly startTurn?: Effect.Effect<void>;
323324
readonly beforeUnload?: Effect.Effect<void>;
324325
/** Registers the process's closeCount finalizer before `beforeOpen` runs. */
@@ -383,7 +384,11 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
383384
});
384385
if (options.spawnBeforeOpen !== true) {
385386
yield* countClose;
386-
if (options.hangSessionScopeClose === true) yield* hangClose;
387+
if (
388+
options.hangSessionScopeClose === true ||
389+
(options.hangSessionScopeClose === "first" && (yield* Ref.get(state)).openCount === 1)
390+
)
391+
yield* hangClose;
387392
}
388393

389394
return {
@@ -465,7 +470,7 @@ function layerTest(input: {
465470
};
466471
readonly hasPendingBackgroundWork?: Effect.Effect<boolean>;
467472
readonly hasPendingBackgroundWorkForThread?: Effect.Effect<boolean>;
468-
readonly hangSessionScopeClose?: boolean;
473+
readonly hangSessionScopeClose?: boolean | "first";
469474
readonly startTurn?: Effect.Effect<void>;
470475
readonly beforeUnload?: Effect.Effect<void>;
471476
readonly spawnBeforeOpen?: boolean;
@@ -3606,6 +3611,68 @@ it.effect(
36063611
}),
36073612
);
36083613

3614+
it.effect("ProviderSessionManagerV2 leaves a closed replacement's status to it", () =>
3615+
Effect.gen(function* () {
3616+
const state = yield* Ref.make(emptyState);
3617+
const scopeCloseReached = yield* Deferred.make<void>();
3618+
yield* Effect.gen(function* () {
3619+
const eventSink = yield* EventSink.EventSinkV2;
3620+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
3621+
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
3622+
const threadId = ThreadId.make("thread-provider-session-manager-closed-replacement");
3623+
const providerSessionId = yield* idAllocator.allocate.providerSession({
3624+
providerInstanceId: modelSelection.instanceId,
3625+
threadId,
3626+
});
3627+
yield* eventSink.write({
3628+
events: [
3629+
yield* makeThreadCreatedEvent({ idAllocator, threadId, now: yield* DateTime.now }),
3630+
],
3631+
});
3632+
yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
3633+
3634+
// The first runtime's events end unexpectedly, and its release stalls in scope close.
3635+
const events = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId));
3636+
assert.isDefined(events);
3637+
yield* Queue.end(events!);
3638+
yield* Deferred.await(scopeCloseReached);
3639+
3640+
// A replacement opens and closes meanwhile.
3641+
yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
3642+
yield* manager.close(providerSessionId);
3643+
const statuses = yield* eventSink
3644+
.stream({ threadId, afterSequence: yield* eventSink.latestSequence({ threadId }) })
3645+
.pipe(
3646+
Stream.filter(
3647+
({ event }) =>
3648+
event.type === "provider-session.updated" ||
3649+
event.type === "provider-session.attached",
3650+
),
3651+
Stream.takeUntil(({ event }) => event.type === "provider-session.attached"),
3652+
Stream.map(({ event }) => event.type),
3653+
Stream.runCollect,
3654+
Effect.forkScoped,
3655+
);
3656+
3657+
// The stalled close ends; the next open waits on the open lock until the
3658+
// first release has written its records.
3659+
yield* TestClock.adjust("30 seconds");
3660+
yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
3661+
3662+
assert.deepEqual(Array.from(yield* Fiber.join(statuses)), ["provider-session.attached"]);
3663+
}).pipe(
3664+
Effect.provide(
3665+
layerTest({
3666+
state,
3667+
idleTimeoutMs: 60_000,
3668+
hangSessionScopeClose: "first",
3669+
scopeCloseReached,
3670+
}),
3671+
),
3672+
);
3673+
}),
3674+
);
3675+
36093676
it.effect("ProviderSessionManagerV2 keeps each failed release's cleanup", () =>
36103677
Effect.gen(function* () {
36113678
const state = yield* Ref.make(emptyState);

‎apps/server/src/orchestration-v2/ProviderSessionManager.ts‎

Lines changed: 34 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -419,6 +419,10 @@ export const layerWithOptions = (
419419
}),
420420
);
421421
const sessions = yield* Ref.make(new Map<string, LiveSessionEntry>());
422+
// The latest session opened under each id, kept until its release records it.
423+
const latestOpened = yield* Ref.make(
424+
new Map<string, ProviderAdapter.ProviderAdapterV2SessionRuntime>(),
425+
);
422426
// One retry per released entry, so a later release with the same id
423427
// cannot drop cleanup for threads only the earlier session served.
424428
const releaseRecordRetries = yield* FiberSet.make();
@@ -799,6 +803,27 @@ export const layerWithOptions = (
799803
{ concurrency: 1 },
800804
).pipe(Effect.flatMap(Exit.asVoidAll));
801805

806+
// A replacement under the same id, like the next thread of a shared
807+
// runtime, owns the session's status from its open on, even after it
808+
// closes. Only the latest session opened under an id writes its release
809+
// status; the entry is dropped once that is written. Callers hold the
810+
// open lock, so no replacement opens mid-write.
811+
const writeOwnReleaseRecords = (
812+
input: Omit<Parameters<typeof writeReleaseRecords>[0], "replaced">,
813+
) =>
814+
Effect.gen(function* () {
815+
const key = sessionKey(input.entry.runtime.providerSessionId);
816+
const replaced = (yield* Ref.get(latestOpened)).get(key) !== input.entry.runtime;
817+
yield* writeReleaseRecords({ ...input, replaced });
818+
if (replaced) return;
819+
yield* Ref.update(latestOpened, (current) => {
820+
if (current.get(key) !== input.entry.runtime) return current;
821+
const updated = new Map(current);
822+
updated.delete(key);
823+
return updated;
824+
});
825+
});
826+
802827
// The session already left the live map, so a later release finds
803828
// nothing to do. Without a retry the UI would keep a ready session and
804829
// answerable approvals until a server restart. Each attempt holds the
@@ -809,13 +834,7 @@ export const layerWithOptions = (
809834
const providerSessionId = input.entry.runtime.providerSessionId;
810835
const attempt = Effect.gen(function* () {
811836
const exit = yield* Effect.exit(
812-
sessionOpen.withLock(
813-
providerSessionId,
814-
Effect.gen(function* () {
815-
const replaced = (yield* Ref.get(sessions)).has(sessionKey(providerSessionId));
816-
yield* writeReleaseRecords({ ...input, replaced });
817-
}),
818-
),
837+
sessionOpen.withLock(providerSessionId, writeOwnReleaseRecords(input)),
819838
);
820839
if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) return yield* exit;
821840
yield* Effect.logWarning("orchestration-v2.provider-session-release-records-failed", {
@@ -974,18 +993,15 @@ export const layerWithOptions = (
974993
releasedAt,
975994
};
976995
// A thread of a shared runtime can reopen it under the same
977-
// id while this closes. The open lock orders the check and
978-
// write against that, so a replacement keeps its status.
979-
const writeRecords = Effect.gen(function* () {
980-
const replaced = (yield* Ref.get(sessions)).has(
981-
sessionKey(input.providerSessionId),
982-
);
983-
yield* writeReleaseRecords({ ...records, replaced });
984-
});
996+
// id while this closes; the open lock orders that against
997+
// this write.
985998
const recorded = yield* Effect.exit(
986999
input.openLockHeld === true
987-
? writeRecords
988-
: sessionOpen.withLock(input.providerSessionId, writeRecords),
1000+
? writeOwnReleaseRecords(records)
1001+
: sessionOpen.withLock(
1002+
input.providerSessionId,
1003+
writeOwnReleaseRecords(records),
1004+
),
9891005
);
9901006
if (Exit.isFailure(recorded)) {
9911007
yield* retryReleaseRecords(records);
@@ -2201,6 +2217,7 @@ export const layerWithOptions = (
22012217
updated.set(key, entry);
22022218
return updated;
22032219
});
2220+
yield* Ref.update(latestOpened, (current) => new Map(current).set(key, runtime));
22042221
// The entry now guards the credential via its recorded id, so
22052222
// the pre-open reservation can be dropped.
22062223
yield* dropReservation;

0 commit comments

Comments
 (0)