Skip to content

Commit e6b5f34

Browse files
committed
fix(server): fence deletion cleanup with thread creation
1 parent 14f266f commit e6b5f34

7 files changed

Lines changed: 46 additions & 39 deletions

File tree

‎apps/server/integration/OrchestrationEngineHarness.integration.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -373,7 +373,7 @@ export const makeOrchestrationIntegrationHarness = (
373373
Layer.provideMerge(
374374
Layer.succeed(ThreadDeletionReactor, {
375375
start: () => Effect.void,
376-
drain: Effect.void,
376+
drainThrough: () => Effect.void,
377377
}),
378378
),
379379
Layer.provideMerge(

‎apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ describe("OrchestrationReactor", () => {
6161
started.push("thread-deletion-reactor");
6262
return Effect.void;
6363
},
64-
drain: Effect.void,
64+
drainThrough: () => Effect.void,
6565
}),
6666
),
6767
Layer.provideMerge(

‎apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ describe("ThreadDeletionReactor drain", () => {
123123
// Sequence 1 is fully cleaned and the worker queue is idle. Sequence
124124
// 2 is committed and published but still in flight to the subscriber.
125125
yield* Ref.set(latestSequence, 2);
126-
const drained = yield* Effect.forkChild(reactor.drain);
126+
const drained = yield* Effect.forkChild(reactor.drainThrough(2));
127127
yield* Effect.yieldNow;
128128
yield* Effect.yieldNow;
129129
expect(stops).toEqual([1]);

‎apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,10 +81,10 @@ const make = Effect.gen(function* () {
8181

8282
const worker = yield* makeDrainableWorker(processThreadDeletedSafely);
8383

84-
// Highest event sequence the subscriber has handed to the worker. Events
85-
// are published after they commit, so a caller that reads latestSequence
86-
// and waits for this to catch up knows every deletion up to that point is
87-
// at least enqueued; the worker drain then covers the in-flight ones.
84+
// Highest event sequence the subscriber has handed to the worker. Waiting
85+
// through a successful thread.created sequence covers every deletion that
86+
// was ahead of that create in the engine queue; the worker drain then covers
87+
// the in-flight cleanup.
8888
const seenSequence = yield* SubscriptionRef.make(0);
8989
const noteSeen = (sequence: number) =>
9090
SubscriptionRef.update(seenSequence, (seen) => Math.max(seen, sequence));
@@ -105,8 +105,9 @@ const make = Effect.gen(function* () {
105105
);
106106
});
107107

108-
const drain: ThreadDeletionReactorShape["drain"] = Effect.gen(function* () {
109-
const target = yield* orchestrationEngine.latestSequence;
108+
const drainThrough: ThreadDeletionReactorShape["drainThrough"] = Effect.fn(
109+
"ThreadDeletionReactor.drainThrough",
110+
)(function* (target) {
110111
yield* SubscriptionRef.changes(seenSequence).pipe(
111112
Stream.filter((seen) => seen >= target),
112113
Stream.runHead,
@@ -116,7 +117,7 @@ const make = Effect.gen(function* () {
116117

117118
return {
118119
start,
119-
drain,
120+
drainThrough,
120121
} satisfies ThreadDeletionReactorShape;
121122
});
122123

‎apps/server/src/orchestration/Services/ThreadDeletionReactor.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@ export interface ThreadDeletionReactorShape {
2323
readonly start: () => Effect.Effect<void, never, Scope.Scope>;
2424

2525
/**
26-
* Resolves once every thread.deleted published up to now has been handed
27-
* to the worker and the worker's queue is empty and idle. Callers that
28-
* re-create a deleted thread id wait on this so the old incarnation's
29-
* cleanup cannot run against the new one.
26+
* Resolves once every thread.deleted at or before the supplied event
27+
* sequence has been handed to the worker and the worker is empty and idle.
28+
* A successful thread.create sequence is the fence callers use before the
29+
* new incarnation can own runtime resources.
3030
*/
31-
readonly drain: Effect.Effect<void>;
31+
readonly drainThrough: (sequence: number) => Effect.Effect<void>;
3232
}
3333

3434
/**

‎apps/server/src/server.test.ts‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -798,7 +798,7 @@ const buildAppUnderTest = (options?: {
798798
}),
799799
Layer.mock(ThreadDeletionReactor)({
800800
start: () => Effect.void,
801-
drain: Effect.void,
801+
drainThrough: () => Effect.void,
802802
...options?.layers?.threadDeletionReactor,
803803
}),
804804
),
@@ -8449,22 +8449,24 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
84498449
}).pipe(Effect.provide(NodeHttpServer.layerTest)),
84508450
);
84518451

8452-
it.effect("waits for deletion cleanup before re-creating a thread id", () =>
8452+
it.effect("drains deletion cleanup through the re-created thread event", () =>
84538453
Effect.gen(function* () {
84548454
// A draft retry reuses the thread id its failed bootstrap deleted. The
84558455
// deletion reactor stops sessions and closes terminals by that id, so
8456-
// both thread.create paths must let its queue drain first.
8456+
// both thread.create paths use the created event as a fence, then drain
8457+
// cleanup before handing the new incarnation to resource-owning work.
84578458
const trace: Array<string> = [];
84588459
const drainRequested = yield* Deferred.make<void>();
84598460
const cleanupDone = yield* Deferred.make<void>();
84608461
yield* buildAppUnderTest({
84618462
layers: {
84628463
threadDeletionReactor: {
8463-
drain: Effect.gen(function* () {
8464-
trace.push("drain");
8465-
yield* Deferred.succeed(drainRequested, undefined);
8466-
yield* Deferred.await(cleanupDone);
8467-
}),
8464+
drainThrough: (sequence) =>
8465+
Effect.gen(function* () {
8466+
trace.push(`drain:${sequence}`);
8467+
yield* Deferred.succeed(drainRequested, undefined);
8468+
yield* Deferred.await(cleanupDone);
8469+
}),
84688470
},
84698471
orchestrationEngine: {
84708472
dispatch: (command) =>
@@ -8500,15 +8502,16 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
85008502
}),
85018503
);
85028504
yield* Deferred.await(drainRequested);
8503-
assert.deepEqual(trace, ["drain"]);
8505+
assert.deepEqual(trace, ["thread.create", "drain:1"]);
85048506
yield* Deferred.succeed(cleanupDone, undefined);
85058507
yield* Fiber.join(directCreate);
85068508
}),
85078509
),
85088510
);
8509-
assert.deepEqual(trace, ["drain", "thread.create"]);
8511+
assert.deepEqual(trace, ["thread.create", "drain:1"]);
85108512

8511-
// Cleanup is already released; the bootstrap path must still drain first.
8513+
// Cleanup is already released; the bootstrap path must still drain
8514+
// between creating the thread and starting its turn.
85128515
trace.length = 0;
85138516
yield* Effect.scoped(
85148517
withWsRpcClient(wsUrl, (client) =>
@@ -8547,7 +8550,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
85478550
}),
85488551
),
85498552
);
8550-
assert.deepEqual(trace, ["drain", "thread.create", "thread.turn.start"]);
8553+
assert.deepEqual(trace, ["thread.create", "drain:1", "thread.turn.start"]);
85518554
}).pipe(Effect.provide(NodeHttpServer.layerTest)),
85528555
);
85538556

‎apps/server/src/ws.ts‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -462,11 +462,6 @@ const makeWsRpcLayer = (
462462
const projectionSnapshotQuery = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery;
463463
const orchestrationEngine = yield* OrchestrationEngine.OrchestrationEngineService;
464464
const threadDeletionReactor = yield* ThreadDeletionReactor;
465-
// A draft retry re-creates the thread id its failed bootstrap deleted.
466-
// Session stop and terminal close are keyed by that id and run in the
467-
// deletion reactor's queue, so let the old incarnation's cleanup finish
468-
// before the new one can own those resources.
469-
const awaitThreadDeletionCleanup = threadDeletionReactor.drain;
470465
const analytics = yield* AnalyticsService.AnalyticsService;
471466
// Every command dispatched on this connection carries the connecting
472467
// client's origin, including server-generated bootstrap sub-commands:
@@ -1033,8 +1028,7 @@ const makeWsRpcLayer = (
10331028

10341029
const bootstrapProgram = Effect.gen(function* () {
10351030
if (bootstrap?.createThread) {
1036-
yield* awaitThreadDeletionCleanup;
1037-
yield* dispatchFromClient({
1031+
const created = yield* dispatchFromClient({
10381032
type: "thread.create",
10391033
commandId: yield* serverCommandId("bootstrap-thread-create"),
10401034
threadId: command.threadId,
@@ -1047,6 +1041,11 @@ const makeWsRpcLayer = (
10471041
worktreePath: bootstrap.createThread.worktreePath,
10481042
createdAt: bootstrap.createThread.createdAt,
10491043
});
1044+
// The successful create is a fence in the engine command queue:
1045+
// every delete for the prior incarnation committed before it.
1046+
// Drain through that event before setup or turn start can own
1047+
// terminals and provider sessions under the reused thread id.
1048+
yield* threadDeletionReactor.drainThrough(created.sequence);
10501049
createdThread = true;
10511050
}
10521051

@@ -1133,11 +1132,15 @@ const makeWsRpcLayer = (
11331132
const dispatchEffect =
11341133
normalizedCommand.type === "thread.turn.start" && normalizedCommand.bootstrap
11351134
? dispatchBootstrapTurnStart(normalizedCommand)
1136-
: (normalizedCommand.type === "thread.create"
1137-
? awaitThreadDeletionCleanup
1138-
: Effect.void
1139-
).pipe(
1140-
Effect.andThen(dispatchFromClient(normalizedCommand)),
1135+
: dispatchFromClient(normalizedCommand).pipe(
1136+
Effect.tap(({ sequence }) =>
1137+
// Returning from thread.create is the handoff point at which
1138+
// clients may start resources for the new incarnation. Use
1139+
// its event sequence as the exact deletion-cleanup fence.
1140+
normalizedCommand.type === "thread.create"
1141+
? threadDeletionReactor.drainThrough(sequence)
1142+
: Effect.void,
1143+
),
11411144
Effect.mapError((cause) =>
11421145
toDispatchCommandError(cause, "Failed to dispatch orchestration command"),
11431146
),

0 commit comments

Comments
 (0)