Skip to content

Commit 4eb57ab

Browse files
committed
fix(server): isolate provider command lanes by thread with atomic lane creation
Key provider command processing by threadId so one thread's slow commands no longer head-of-line block other threads, while preserving per-thread FIFO. Create keyed lanes under a semaphore so concurrent enqueues for a new key atomically reserve a single lane instead of racing check-then-create and splitting the key's FIFO across lanes. Idle-lane cleanup revalidates under the same lock and closes scopes outside it.
1 parent b906ce2 commit 4eb57ab

3 files changed

Lines changed: 176 additions & 3 deletions

File tree

‎apps/server/src/orchestration/Layers/ProviderCommandReactor.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import * as Option from "effect/Option";
2727
import * as Schedule from "effect/Schedule";
2828
import * as Schema from "effect/Schema";
2929
import * as Stream from "effect/Stream";
30-
import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker";
30+
import { makeDrainableWorker, makeKeyedDrainableWorker } from "@t3tools/shared/DrainableWorker";
3131

3232
import { resolveThreadWorkspaceCwd } from "../../checkpointing/Utils.ts";
3333
import { increment, orchestrationEventsProcessedTotal } from "../../observability/Metrics.ts";
@@ -1763,7 +1763,10 @@ const make = Effect.gen(function* () {
17631763
}),
17641764
);
17651765

1766-
const worker = yield* makeDrainableWorker(processDomainEventSafely);
1766+
const worker = yield* makeKeyedDrainableWorker(
1767+
(event: ProviderIntentEvent) => event.payload.threadId,
1768+
processDomainEventSafely,
1769+
);
17671770

17681771
const start: ProviderCommandReactorShape["start"] = Effect.fn("start")(function* () {
17691772
const interruptedTitleRegenerations = yield* findInterruptedThreadTitleRegenerations().pipe(

‎packages/shared/src/DrainableWorker.test.ts‎

Lines changed: 74 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { describe, expect } from "vite-plus/test";
33
import * as Deferred from "effect/Deferred";
44
import * as Effect from "effect/Effect";
55

6-
import { makeDrainableWorker } from "./DrainableWorker.ts";
6+
import { makeDrainableWorker, makeKeyedDrainableWorker } from "./DrainableWorker.ts";
77

88
describe("makeDrainableWorker", () => {
99
it.live("waits for work enqueued during active processing before draining", () =>
@@ -55,3 +55,76 @@ describe("makeDrainableWorker", () => {
5555
),
5656
);
5757
});
58+
59+
describe("makeKeyedDrainableWorker", () => {
60+
it.live("runs different keys independently while preserving each key's FIFO", () =>
61+
Effect.scoped(
62+
Effect.gen(function* () {
63+
const processed: string[] = [];
64+
const firstStarted = yield* Deferred.make<void>();
65+
const secondStarted = yield* Deferred.make<void>();
66+
const releaseFirst = yield* Deferred.make<void>();
67+
const worker = yield* makeKeyedDrainableWorker(
68+
(item: { readonly key: string; readonly value: string }) => item.key,
69+
(item) =>
70+
Effect.gen(function* () {
71+
if (item.value === "a1") {
72+
yield* Deferred.succeed(firstStarted, undefined);
73+
yield* Deferred.await(releaseFirst);
74+
}
75+
processed.push(item.value);
76+
if (item.value === "b1") {
77+
yield* Deferred.succeed(secondStarted, undefined);
78+
}
79+
}),
80+
);
81+
82+
yield* worker.enqueue({ key: "a", value: "a1" });
83+
yield* Deferred.await(firstStarted);
84+
yield* worker.enqueue({ key: "a", value: "a2" });
85+
yield* worker.enqueue({ key: "b", value: "b1" });
86+
yield* Deferred.await(secondStarted);
87+
expect(processed).toEqual(["b1"]);
88+
89+
yield* Deferred.succeed(releaseFirst, undefined);
90+
yield* worker.drain;
91+
expect(processed).toEqual(["b1", "a1", "a2"]);
92+
}),
93+
),
94+
);
95+
96+
it.live("creates a single lane under concurrent same-key enqueues", () =>
97+
Effect.scoped(
98+
Effect.gen(function* () {
99+
const processed: number[] = [];
100+
const processingFibers: number[] = [];
101+
// Hold every item until all enqueues have been offered, so the lane
102+
// can neither drain nor be cleaned up mid-race.
103+
const gate = yield* Deferred.make<void>();
104+
const worker = yield* makeKeyedDrainableWorker(
105+
(_item: number) => "shared",
106+
(item) =>
107+
Effect.gen(function* () {
108+
processingFibers.push(yield* Effect.fiberId);
109+
yield* Deferred.await(gate);
110+
processed.push(item);
111+
}),
112+
);
113+
114+
const count = 50;
115+
yield* Effect.all(
116+
Array.from({ length: count }, (_, index) => worker.enqueue(index)),
117+
{ concurrency: "unbounded", discard: true },
118+
);
119+
yield* Deferred.succeed(gate, undefined);
120+
yield* worker.drain;
121+
122+
expect(processed).toHaveLength(count);
123+
expect(processed.toSorted((a, b) => a - b)).toEqual(
124+
Array.from({ length: count }, (_, index) => index),
125+
);
126+
expect(new Set(processingFibers).size).toBe(1);
127+
}),
128+
),
129+
);
130+
});

‎packages/shared/src/DrainableWorker.ts‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
*/
1111
import * as Scope from "effect/Scope";
1212
import * as Effect from "effect/Effect";
13+
import * as Exit from "effect/Exit";
14+
import * as Semaphore from "effect/Semaphore";
1315
import * as TxQueue from "effect/TxQueue";
1416
import * as TxRef from "effect/TxRef";
1517

@@ -28,6 +30,101 @@ export interface DrainableWorker<A> {
2830
readonly drain: Effect.Effect<void>;
2931
}
3032

33+
/**
34+
* Create drainable FIFO lanes keyed by an item field.
35+
*
36+
* Lane lookup, creation, and idle cleanup all serialize on a semaphore, so
37+
* concurrent enqueues for a not-yet-seen key atomically reserve a single
38+
* lane. A plain check-then-create on the lane map would let two fibers both
39+
* observe a missing key and each create a lane, splitting the key's FIFO
40+
* across lanes.
41+
*
42+
* Each key gets its own FIFO lane while different keys process concurrently.
43+
* `drain` waits for every lane present when draining starts. Idle lanes are
44+
* removed once drained so key cardinality stays bounded.
45+
*
46+
* @param keyOf - Derives the lane key for an item.
47+
* @param process - The effect to run for each queued item.
48+
* @returns A `DrainableWorker` with `enqueue` and `drain`.
49+
*/
50+
export const makeKeyedDrainableWorker = <A, K, E, R>(
51+
keyOf: (item: A) => K,
52+
process: (item: A) => Effect.Effect<void, E, R>,
53+
): Effect.Effect<DrainableWorker<A>, never, Scope.Scope | R> =>
54+
Effect.gen(function* () {
55+
const context = yield* Effect.context<R>();
56+
const scope = yield* Scope.Scope;
57+
const laneLock = yield* Semaphore.make(1);
58+
type Entry = {
59+
readonly worker: DrainableWorker<A>;
60+
readonly scope: Scope.Closeable;
61+
version: number;
62+
};
63+
const entries = new Map<K, Entry>();
64+
const processInParentScope = (item: A) =>
65+
process(item).pipe(Effect.provide(context), Scope.provide(scope));
66+
67+
const removeWhenIdle = (key: K, entry: Entry): Effect.Effect<void> =>
68+
Effect.gen(function* () {
69+
while (true) {
70+
const observedVersion = yield* laneLock.withPermit(
71+
Effect.sync(() => (entries.get(key) === entry ? entry.version : undefined)),
72+
);
73+
if (observedVersion === undefined) {
74+
return;
75+
}
76+
yield* entry.worker.drain;
77+
const removed = yield* laneLock.withPermit(
78+
Effect.sync(() => {
79+
if (entries.get(key) !== entry || entry.version !== observedVersion) {
80+
return false;
81+
}
82+
entries.delete(key);
83+
return true;
84+
}),
85+
);
86+
if (removed) {
87+
yield* Scope.close(entry.scope, Exit.void);
88+
return;
89+
}
90+
}
91+
});
92+
93+
const enqueue = (item: A): Effect.Effect<void> =>
94+
laneLock.withPermit(
95+
Effect.gen(function* () {
96+
const key = keyOf(item);
97+
const existing = entries.get(key);
98+
if (existing !== undefined) {
99+
existing.version += 1;
100+
yield* existing.worker.enqueue(item);
101+
return;
102+
}
103+
const laneScope = yield* Scope.fork(scope, "sequential");
104+
const worker = yield* makeDrainableWorker(processInParentScope).pipe(
105+
Scope.provide(laneScope),
106+
);
107+
const entry: Entry = { worker, scope: laneScope, version: 1 };
108+
entries.set(key, entry);
109+
yield* worker.enqueue(item);
110+
yield* removeWhenIdle(key, entry).pipe(Effect.forkIn(scope));
111+
}),
112+
);
113+
114+
const drain: DrainableWorker<A>["drain"] = laneLock
115+
.withPermit(Effect.sync(() => Array.from(entries.values())))
116+
.pipe(
117+
Effect.flatMap((lanes) =>
118+
Effect.forEach(lanes, (entry) => entry.worker.drain, {
119+
concurrency: "unbounded",
120+
discard: true,
121+
}),
122+
),
123+
);
124+
125+
return { enqueue, drain } satisfies DrainableWorker<A>;
126+
});
127+
31128
/**
32129
* Create a drainable worker that processes items from an unbounded queue.
33130
*

0 commit comments

Comments
 (0)