Skip to content

Commit 02452f1

Browse files
committed
fix(server): serialize provider admission snapshots
Keep reservation release ordered with provider session snapshots so a startup handoff cannot disappear from both views during admission. Cover turn and handover interleavings, same-thread followups, interruption cleanup, and idempotent release.
1 parent b355c3d commit 02452f1

2 files changed

Lines changed: 181 additions & 46 deletions

File tree

‎apps/server/src/orchestration/UsageLimitReservations.test.ts‎

Lines changed: 124 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
import type { ProviderSession, ThreadId } from "@t3tools/contracts";
22
import { describe, expect, it } from "@effect/vitest";
3+
import * as Deferred from "effect/Deferred";
34
import * as Layer from "effect/Layer";
45
import { ProviderService } from "../provider/Services/ProviderService.ts";
56
import * as Effect from "effect/Effect";
7+
import * as Fiber from "effect/Fiber";
68

79
import { MAX_CONCURRENT_PROVIDER_TURNS } from "./ConcurrentTurnPolicy.ts";
810
import { make as makeService, layer, UsageLimitReservations } from "./UsageLimitReservations.ts";
@@ -17,7 +19,69 @@ function session(index: number): ProviderSession {
1719
} as ProviderSession;
1820
}
1921

22+
function assertReleasedReservationRemainsVisible(kind: "turn" | "handover") {
23+
return Effect.scoped(
24+
Effect.gen(function* () {
25+
let runningSessions = Array.from({ length: MAX_CONCURRENT_PROVIDER_TURNS - 1 }, (_, index) =>
26+
session(index),
27+
);
28+
let pauseSnapshot = false;
29+
const snapshotStarted = yield* Deferred.make<void>();
30+
const resumeSnapshot = yield* Deferred.make<void>();
31+
const reservations = yield* make(() =>
32+
Effect.gen(function* () {
33+
const snapshot = [...runningSessions];
34+
if (pauseSnapshot) {
35+
yield* Deferred.succeed(snapshotStarted, undefined);
36+
yield* Deferred.await(resumeSnapshot);
37+
}
38+
return snapshot;
39+
}),
40+
);
41+
const reserve = kind === "turn" ? reservations.reserveTurn : reservations.reserveHandover;
42+
43+
expect(
44+
yield* reserve({
45+
key: `${kind}:eighth`,
46+
threadId: "thread-eighth" as ThreadId,
47+
}),
48+
).toBeUndefined();
49+
50+
pauseSnapshot = true;
51+
const ninthReservation = yield* reserve({
52+
key: `${kind}:ninth`,
53+
threadId: "thread-ninth" as ThreadId,
54+
}).pipe(Effect.forkChild);
55+
yield* Deferred.await(snapshotStarted);
56+
runningSessions = [
57+
...runningSessions,
58+
{
59+
...session(MAX_CONCURRENT_PROVIDER_TURNS - 1),
60+
threadId: "thread-eighth" as ThreadId,
61+
},
62+
];
63+
const releaseEighth = yield* reservations.release(`${kind}:eighth`).pipe(Effect.forkChild);
64+
yield* Effect.yieldNow;
65+
expect(releaseEighth.pollUnsafe()).toBeUndefined();
66+
yield* Deferred.succeed(resumeSnapshot, undefined);
67+
68+
expect(yield* Fiber.join(ninthReservation)).toMatchObject({
69+
code: "concurrent-turn-limit",
70+
});
71+
yield* Fiber.join(releaseEighth);
72+
}),
73+
);
74+
}
75+
2076
describe("UsageLimitReservations", () => {
77+
it.effect("keeps a released turn reservation visible to an in-flight snapshot", () =>
78+
assertReleasedReservationRemainsVisible("turn"),
79+
);
80+
81+
it.effect("keeps a released handover reservation visible to an in-flight snapshot", () =>
82+
assertReleasedReservationRemainsVisible("handover"),
83+
);
84+
2185
it.effect("shares admission capacity between runtime consumers", () =>
2286
Effect.gen(function* () {
2387
const turnConsumer = yield* UsageLimitReservations;
@@ -86,7 +150,65 @@ describe("UsageLimitReservations", () => {
86150
}),
87151
);
88152

89-
it.effect("releases a handover slot after completion", () =>
153+
it.effect("preserves same-thread followups when provider capacity is full", () =>
154+
Effect.gen(function* () {
155+
const reservations = yield* make(() =>
156+
Effect.succeed(
157+
Array.from({ length: MAX_CONCURRENT_PROVIDER_TURNS }, (_, index) => session(index)),
158+
),
159+
);
160+
161+
expect(
162+
yield* reservations.reserveTurn({
163+
key: "turn:followup",
164+
threadId: "running-0" as ThreadId,
165+
}),
166+
).toBeUndefined();
167+
yield* reservations.release("turn:followup");
168+
expect(
169+
yield* reservations.reserveTurn({
170+
key: "turn:new-thread",
171+
threadId: "thread-new" as ThreadId,
172+
}),
173+
).toMatchObject({ code: "concurrent-turn-limit" });
174+
}),
175+
);
176+
177+
it.effect("releases admission synchronization when snapshot lookup is interrupted", () =>
178+
Effect.scoped(
179+
Effect.gen(function* () {
180+
let pauseSnapshot = true;
181+
const snapshotStarted = yield* Deferred.make<void>();
182+
const resumeSnapshot = yield* Deferred.make<void>();
183+
const reservations = yield* make(() =>
184+
pauseSnapshot
185+
? Deferred.succeed(snapshotStarted, undefined).pipe(
186+
Effect.andThen(Deferred.await(resumeSnapshot)),
187+
Effect.as<ReadonlyArray<ProviderSession>>([]),
188+
)
189+
: Effect.succeed([]),
190+
);
191+
const interruptedReservation = yield* reservations
192+
.reserveTurn({
193+
key: "turn:interrupted",
194+
threadId: "thread-interrupted" as ThreadId,
195+
})
196+
.pipe(Effect.forkChild);
197+
yield* Deferred.await(snapshotStarted);
198+
yield* Fiber.interrupt(interruptedReservation);
199+
200+
pauseSnapshot = false;
201+
expect(
202+
yield* reservations.reserveTurn({
203+
key: "turn:after-interrupt",
204+
threadId: "thread-after-interrupt" as ThreadId,
205+
}),
206+
).toBeUndefined();
207+
}),
208+
),
209+
);
210+
211+
it.effect("releases a handover slot idempotently after completion", () =>
90212
Effect.gen(function* () {
91213
const reservations = yield* make(() =>
92214
Effect.succeed(
@@ -101,6 +223,7 @@ describe("UsageLimitReservations", () => {
101223
}),
102224
).toBeUndefined();
103225
yield* reservations.release("handover:first");
226+
yield* reservations.release("handover:first");
104227
expect(
105228
yield* reservations.reserveHandover({
106229
key: "handover:second",

‎apps/server/src/orchestration/UsageLimitReservations.ts‎

Lines changed: 57 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import type { ThreadId } from "@t3tools/contracts";
22
import * as Context from "effect/Context";
33
import * as Layer from "effect/Layer";
44
import * as Effect from "effect/Effect";
5+
import * as Semaphore from "effect/Semaphore";
56

67
import type { ProviderServiceError } from "../provider/Errors.ts";
78
import { ProviderService } from "../provider/Services/ProviderService.ts";
@@ -33,65 +34,76 @@ export class UsageLimitReservations extends Context.Service<
3334
export const make = Effect.gen(function* () {
3435
const providerService = yield* ProviderService;
3536
const reservations = new Map<string, Reservation>();
37+
const reservationMutex = yield* Semaphore.make(1);
3638

3739
const release: UsageLimitReservations["Service"]["release"] = (key) =>
38-
Effect.sync(() => {
39-
reservations.delete(key);
40-
});
40+
reservationMutex.withPermit(
41+
Effect.sync(() => {
42+
reservations.delete(key);
43+
}),
44+
);
4145

4246
const reserveTurn: UsageLimitReservations["Service"]["reserveTurn"] = Effect.fn(
4347
"UsageLimitReservations.reserveTurn",
4448
)(function* (input: Parameters<UsageLimitReservations["Service"]["reserveTurn"]>[0]) {
45-
const sessions = yield* providerService.listSessions();
46-
return yield* Effect.sync(() => {
47-
const values = [...reservations.values()];
48-
const violation = evaluateTurnStartLimits({
49-
threadId: input.threadId,
50-
sessions,
51-
reservedTurnThreadIds: values
52-
.filter((reservation) => reservation.kind === "turn")
53-
.map((reservation) => reservation.threadId),
54-
reservedHandoverCount: values.filter((reservation) => reservation.kind === "handover")
55-
.length,
56-
});
57-
if (violation) return violation;
49+
return yield* reservationMutex.withPermit(
50+
Effect.gen(function* () {
51+
const sessions = yield* providerService.listSessions();
52+
return yield* Effect.sync(() => {
53+
const values = [...reservations.values()];
54+
const violation = evaluateTurnStartLimits({
55+
threadId: input.threadId,
56+
sessions,
57+
reservedTurnThreadIds: values
58+
.filter((reservation) => reservation.kind === "turn")
59+
.map((reservation) => reservation.threadId),
60+
reservedHandoverCount: values.filter((reservation) => reservation.kind === "handover")
61+
.length,
62+
});
63+
if (violation) return violation;
5864

59-
reservations.set(input.key, { kind: "turn", threadId: input.threadId });
60-
return undefined;
61-
});
65+
reservations.set(input.key, { kind: "turn", threadId: input.threadId });
66+
return undefined;
67+
});
68+
}),
69+
);
6270
});
6371

6472
const reserveHandover: UsageLimitReservations["Service"]["reserveHandover"] = Effect.fn(
6573
"UsageLimitReservations.reserveHandover",
6674
)(function* (input: Parameters<UsageLimitReservations["Service"]["reserveHandover"]>[0]) {
67-
const sessions = yield* providerService.listSessions();
68-
return yield* Effect.sync(() => {
69-
const values = [...reservations.values()];
70-
if (
71-
values.some(
72-
(reservation) =>
73-
reservation.kind === "handover" && reservation.threadId === input.threadId,
74-
)
75-
) {
76-
return {
77-
code: "handover-in-progress",
78-
detail: "A handover is already being generated for this thread.",
79-
} satisfies UsageLimitViolation;
80-
}
75+
return yield* reservationMutex.withPermit(
76+
Effect.gen(function* () {
77+
const sessions = yield* providerService.listSessions();
78+
return yield* Effect.sync(() => {
79+
const values = [...reservations.values()];
80+
if (
81+
values.some(
82+
(reservation) =>
83+
reservation.kind === "handover" && reservation.threadId === input.threadId,
84+
)
85+
) {
86+
return {
87+
code: "handover-in-progress",
88+
detail: "A handover is already being generated for this thread.",
89+
} satisfies UsageLimitViolation;
90+
}
8191

82-
const violation = evaluateHandoverStartLimits({
83-
sessions,
84-
reservedTurnThreadIds: values
85-
.filter((reservation) => reservation.kind === "turn")
86-
.map((reservation) => reservation.threadId),
87-
reservedHandoverCount: values.filter((reservation) => reservation.kind === "handover")
88-
.length,
89-
});
90-
if (violation) return violation;
92+
const violation = evaluateHandoverStartLimits({
93+
sessions,
94+
reservedTurnThreadIds: values
95+
.filter((reservation) => reservation.kind === "turn")
96+
.map((reservation) => reservation.threadId),
97+
reservedHandoverCount: values.filter((reservation) => reservation.kind === "handover")
98+
.length,
99+
});
100+
if (violation) return violation;
91101

92-
reservations.set(input.key, { kind: "handover", threadId: input.threadId });
93-
return undefined;
94-
});
102+
reservations.set(input.key, { kind: "handover", threadId: input.threadId });
103+
return undefined;
104+
});
105+
}),
106+
);
95107
});
96108

97109
return UsageLimitReservations.of({ reserveTurn, reserveHandover, release });

0 commit comments

Comments
 (0)