Skip to content

Commit 938e2af

Browse files
fix(shared): DrainableWorker keeps running after a failed item
When the processor failed or died on one item, the error escaped the worker loop and its fiber exited. Later items were never processed, and anything waiting on `drain` hung forever because the outstanding count of the queued items was never decremented. AgentAwarenessRelay's publish worker and ThreadSettlementService's sweep worker pass `process` functions that can still die, so one bad item stopped them for the rest of the server's life. The worker now catches each item's cause, logs it with `Effect.logError`, and keeps going; the outstanding count is still decremented in `Effect.ensuring`. Closing the worker's scope still stops it: in Effect 4.0.1 an interrupt of the worker fiber skips `catchCause` handlers, so only the item's own failure, defect, or self-interruption is caught. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 5eb8773 commit 938e2af

2 files changed

Lines changed: 68 additions & 4 deletions

File tree

‎packages/shared/src/DrainableWorker.test.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,67 @@
11
import { it } from "@effect/vitest";
22
import { describe, expect } from "vite-plus/test";
3+
import * as Cause from "effect/Cause";
34
import * as Deferred from "effect/Deferred";
45
import * as Effect from "effect/Effect";
6+
import * as Exit from "effect/Exit";
7+
import * as Fiber from "effect/Fiber";
8+
import * as Logger from "effect/Logger";
9+
import * as Scope from "effect/Scope";
510

611
import { makeDrainableWorker } from "./DrainableWorker.ts";
712

813
describe("makeDrainableWorker", () => {
14+
it.effect("logs a failed or defective item and keeps processing later items", () => {
15+
const loggedErrors: Array<unknown> = [];
16+
const logger = Logger.make(({ logLevel, cause }) => {
17+
if (logLevel === "Error") loggedErrors.push(Cause.squash(cause));
18+
});
19+
20+
return Effect.scoped(
21+
Effect.gen(function* () {
22+
const processed: string[] = [];
23+
const workerFiber = yield* Deferred.make<Fiber.Fiber<unknown, unknown>>();
24+
const worker = yield* makeDrainableWorker((item: string) =>
25+
Effect.gen(function* () {
26+
yield* Deferred.succeed(workerFiber, yield* Effect.fiber);
27+
if (item === "fail") return yield* Effect.fail("typed failure");
28+
if (item === "die") return yield* Effect.die("defect");
29+
processed.push(item);
30+
}),
31+
);
32+
33+
yield* worker.enqueue("fail");
34+
yield* worker.enqueue("die");
35+
yield* worker.enqueue("ok");
36+
37+
// A stopped worker never drains; its exit settles the race instead of a hang.
38+
yield* Effect.raceFirst(worker.drain, Fiber.await(yield* Deferred.await(workerFiber)));
39+
40+
expect(processed).toEqual(["ok"]);
41+
expect(loggedErrors).toEqual(["typed failure", "defect"]);
42+
}),
43+
).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false })));
44+
});
45+
46+
it.effect("stops when its scope closes during an item", () =>
47+
Effect.gen(function* () {
48+
const scope = yield* Scope.make();
49+
const workerFiber = yield* Deferred.make<Fiber.Fiber<unknown, unknown>>();
50+
const worker = yield* makeDrainableWorker(() =>
51+
Effect.gen(function* () {
52+
yield* Deferred.succeed(workerFiber, yield* Effect.fiber);
53+
return yield* Effect.never;
54+
}),
55+
).pipe(Scope.provide(scope));
56+
57+
yield* worker.enqueue("item");
58+
const fiber = yield* Deferred.await(workerFiber);
59+
yield* Scope.close(scope, Exit.void);
60+
61+
expect(Exit.hasInterrupts(yield* Fiber.await(fiber))).toBe(true);
62+
}),
63+
);
64+
965
it.live("waits for work enqueued during active processing before draining", () =>
1066
Effect.scoped(
1167
Effect.gen(function* () {

‎packages/shared/src/DrainableWorker.ts‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@ export interface DrainableWorker<A> {
3434
* The worker is forked into the current scope and will be interrupted when
3535
* the scope closes. A finalizer shuts down the queue.
3636
*
37+
* An item that fails or dies is logged and skipped; the worker keeps
38+
* processing later items and `drain` still resolves.
39+
*
3740
* @param process - The effect to run for each queued item.
3841
* @returns A `DrainableWorker` with `queue` and `drain`.
3942
*/
@@ -45,10 +48,15 @@ export const makeDrainableWorker = <A, E, R>(
4548
const outstanding = yield* TxRef.make(0);
4649

4750
yield* TxQueue.take(queue).pipe(
48-
Effect.tap((a) =>
49-
Effect.ensuring(
50-
process(a),
51-
TxRef.update(outstanding, (n) => n - 1),
51+
Effect.flatMap((a) =>
52+
// `suspend` turns a `process` that throws while building its effect
53+
// into this item's defect instead of the loop's.
54+
Effect.suspend(() => process(a)).pipe(
55+
// Interrupting the worker fiber (its scope closing) skips this
56+
// handler, so only the item's own failure, defect, or interruption
57+
// lands here and the loop continues.
58+
Effect.catchCause((cause) => Effect.logError("DrainableWorker item failed", cause)),
59+
Effect.ensuring(TxRef.update(outstanding, (n) => n - 1)),
5260
),
5361
),
5462
Effect.forever,

0 commit comments

Comments
 (0)