Skip to content

Commit 1c06a88

Browse files
shivamhwpclaude
andcommitted
fix(server): deletion drain covers events still in flight to the subscriber
The drain that gates thread.create only waited for the reactor's queue to empty, but thread.deleted reaches that queue through an asynchronous subscriber. A retry arriving between publish and enqueue could re-create the id and then have the old cleanup stop the new session. The reactor now tracks the highest event sequence its subscriber has handed on, and drain first waits for that to reach the engine's latestSequence before draining the worker. Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent 26f2119 commit 1c06a88

3 files changed

Lines changed: 137 additions & 11 deletions

File tree

‎apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts‎

Lines changed: 103 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,35 @@
1-
import { ThreadId } from "@t3tools/contracts";
1+
import {
2+
CommandId,
3+
CorrelationId,
4+
EventId,
5+
type OrchestrationEvent,
6+
ThreadId,
7+
} from "@t3tools/contracts";
8+
import { it as effectIt } from "@effect/vitest";
29
import * as Cause from "effect/Cause";
10+
import * as Deferred from "effect/Deferred";
311
import * as Effect from "effect/Effect";
412
import * as Exit from "effect/Exit";
13+
import * as Fiber from "effect/Fiber";
14+
import * as Layer from "effect/Layer";
15+
import * as Ref from "effect/Ref";
16+
import * as Stream from "effect/Stream";
517
import { describe, expect, it } from "vite-plus/test";
618

7-
import { logCleanupCauseUnlessInterrupted } from "./ThreadDeletionReactor.ts";
19+
import {
20+
ProviderService,
21+
type ProviderServiceShape,
22+
} from "../../provider/Services/ProviderService.ts";
23+
import * as TerminalManager from "../../terminal/Manager.ts";
24+
import {
25+
OrchestrationEngineService,
26+
type OrchestrationEngineShape,
27+
} from "../Services/OrchestrationEngine.ts";
28+
import { ThreadDeletionReactor } from "../Services/ThreadDeletionReactor.ts";
29+
import {
30+
logCleanupCauseUnlessInterrupted,
31+
ThreadDeletionReactorLive,
32+
} from "./ThreadDeletionReactor.ts";
833

934
describe("logCleanupCauseUnlessInterrupted", () => {
1035
const threadId = ThreadId.make("thread-deletion-reactor-test");
@@ -36,3 +61,79 @@ describe("logCleanupCauseUnlessInterrupted", () => {
3661
}
3762
});
3863
});
64+
65+
describe("ThreadDeletionReactor drain", () => {
66+
const now = "2026-01-01T00:00:00.000Z";
67+
const threadId = ThreadId.make("thread-deletion-reactor-drain");
68+
const deletedEvent = (sequence: number): OrchestrationEvent => ({
69+
sequence,
70+
eventId: EventId.make(`evt-deleted-${sequence}`),
71+
aggregateKind: "thread",
72+
aggregateId: threadId,
73+
type: "thread.deleted",
74+
occurredAt: now,
75+
commandId: CommandId.make(`cmd-deleted-${sequence}`),
76+
causationEventId: null,
77+
correlationId: CorrelationId.make(`cmd-deleted-${sequence}`),
78+
metadata: {},
79+
payload: { threadId, deletedAt: now },
80+
});
81+
82+
effectIt.effect("waits for a published deletion the subscriber has not consumed yet", () =>
83+
Effect.gen(function* () {
84+
const stops: Array<number> = [];
85+
const firstCleanupDone = yield* Deferred.make<void>();
86+
// The engine has already committed and published sequence 2, but the
87+
// subscriber has not received it yet: the stream releases it on demand.
88+
const releaseSecondEvent = yield* Deferred.make<void>();
89+
const latestSequence = yield* Ref.make(0);
90+
const engine = {
91+
latestSequence: Ref.get(latestSequence),
92+
streamDomainEvents: Stream.concat(
93+
Stream.make(deletedEvent(1)),
94+
Stream.fromEffect(Deferred.await(releaseSecondEvent)).pipe(
95+
Stream.map(() => deletedEvent(2)),
96+
),
97+
),
98+
} as unknown as OrchestrationEngineShape;
99+
const providerService = {
100+
stopSession: () =>
101+
Effect.gen(function* () {
102+
stops.push(stops.length + 1);
103+
if (stops.length === 1) {
104+
yield* Deferred.succeed(firstCleanupDone, undefined);
105+
}
106+
}),
107+
} as unknown as ProviderServiceShape;
108+
const terminalManager = {
109+
close: () => Effect.void,
110+
} as unknown as TerminalManager.TerminalManager["Service"];
111+
const layer = ThreadDeletionReactorLive.pipe(
112+
Layer.provide(Layer.succeed(ProviderService, providerService)),
113+
Layer.provide(Layer.succeed(TerminalManager.TerminalManager, terminalManager)),
114+
Layer.provide(Layer.succeed(OrchestrationEngineService, engine)),
115+
);
116+
117+
yield* Effect.scoped(
118+
Effect.gen(function* () {
119+
const reactor = yield* ThreadDeletionReactor;
120+
yield* reactor.start();
121+
yield* Deferred.await(firstCleanupDone);
122+
123+
// Sequence 1 is fully cleaned and the worker queue is idle. Sequence
124+
// 2 is committed and published but still in flight to the subscriber.
125+
yield* Ref.set(latestSequence, 2);
126+
const drained = yield* Effect.forkChild(reactor.drain);
127+
yield* Effect.yieldNow;
128+
yield* Effect.yieldNow;
129+
expect(stops).toEqual([1]);
130+
expect(drained.pollUnsafe()).toBeUndefined();
131+
132+
yield* Deferred.succeed(releaseSecondEvent, undefined);
133+
yield* Fiber.join(drained);
134+
expect(stops).toEqual([1, 2]);
135+
}),
136+
).pipe(Effect.provide(layer));
137+
}),
138+
);
139+
});

‎apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts‎

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import * as Cause from "effect/Cause";
44
import * as Effect from "effect/Effect";
55
import * as Layer from "effect/Layer";
66
import * as Stream from "effect/Stream";
7+
import * as SubscriptionRef from "effect/SubscriptionRef";
78

89
import { ProviderService } from "../../provider/Services/ProviderService.ts";
910
import * as TerminalManager from "../../terminal/Manager.ts";
@@ -80,20 +81,42 @@ const make = Effect.gen(function* () {
8081

8182
const worker = yield* makeDrainableWorker(processThreadDeletedSafely);
8283

84+
// Highest event sequence the subscriber has handed to the worker. Events
85+
// are published after they commit, so a caller that reads latestSequence
86+
// and waits for this to catch up knows every deletion up to that point is
87+
// at least enqueued; the worker drain then covers the in-flight ones.
88+
const seenSequence = yield* SubscriptionRef.make(0);
89+
const noteSeen = (sequence: number) =>
90+
SubscriptionRef.update(seenSequence, (seen) => Math.max(seen, sequence));
91+
8392
const start: ThreadDeletionReactorShape["start"] = Effect.fn("start")(function* () {
8493
yield* forkParked(
85-
Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => {
86-
if (event.type !== "thread.deleted") {
87-
return Effect.void;
88-
}
89-
return worker.enqueue(event);
90-
}),
94+
Stream.runForEach(
95+
orchestrationEngine.streamDomainEvents.pipe(
96+
// Events that landed before the subscription are not replayed, so
97+
// start the watermark at the current head instead of zero.
98+
Stream.onStart(orchestrationEngine.latestSequence.pipe(Effect.flatMap(noteSeen))),
99+
),
100+
(event) =>
101+
(event.type === "thread.deleted" ? worker.enqueue(event) : Effect.void).pipe(
102+
Effect.andThen(noteSeen(event.sequence)),
103+
),
104+
),
105+
);
106+
});
107+
108+
const drain: ThreadDeletionReactorShape["drain"] = Effect.gen(function* () {
109+
const target = yield* orchestrationEngine.latestSequence;
110+
yield* SubscriptionRef.changes(seenSequence).pipe(
111+
Stream.filter((seen) => seen >= target),
112+
Stream.runHead,
91113
);
114+
yield* worker.drain;
92115
});
93116

94117
return {
95118
start,
96-
drain: worker.drain,
119+
drain,
97120
} satisfies ThreadDeletionReactorShape;
98121
});
99122

‎apps/server/src/orchestration/Services/ThreadDeletionReactor.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,10 @@ export interface ThreadDeletionReactorShape {
2323
readonly start: () => Effect.Effect<void, never, Scope.Scope>;
2424

2525
/**
26-
* Resolves when the internal processing queue is empty and idle.
27-
* Intended for test use to replace timing-sensitive sleeps.
26+
* Resolves once every thread.deleted published up to now has been handed
27+
* to the worker and the worker's queue is empty and idle. Callers that
28+
* re-create a deleted thread id wait on this so the old incarnation's
29+
* cleanup cannot run against the new one.
2830
*/
2931
readonly drain: Effect.Effect<void>;
3032
}

0 commit comments

Comments
 (0)