Skip to content

Commit 2cbe737

Browse files
voida6claude
andcommitted
fix(server): scheduled run listener resumes forever and settles each outcome once
- Decide a cancel once and retry only the save, so a failed first write no longer makes the retry a silent no-op that leaves the row running. - Resume the run event stream forever with a capped backoff instead of giving up after five tries, which left later dispatches with no listener. - Start the stream before the projection catch-up so a terminal event between them is delivered rather than missed. Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
1 parent 0e9db72 commit 2cbe737

4 files changed

Lines changed: 139 additions & 69 deletions

File tree

‎apps/server/src/scheduledTasks/ScheduledTaskService.test.ts‎

Lines changed: 84 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -487,6 +487,12 @@ const makeTurnHarness = (
487487
readonly runs?: ReadonlyArray<OrchestrationV2Run>;
488488
/** How many times the event stream fails before it starts to deliver events. */
489489
readonly streamFailures?: number;
490+
/** Makes the stream a live tail: events from before it was opened are not delivered. */
491+
readonly liveTail?: boolean;
492+
/** Runs when the projection's runs are read, so an event can land mid catch-up. */
493+
readonly duringRunsRead?: (
494+
events: Queue.Queue<OrchestrationV2DomainEvent>,
495+
) => Effect.Effect<void>;
490496
} = {},
491497
) =>
492498
Effect.gen(function* () {
@@ -503,22 +509,27 @@ const makeTurnHarness = (
503509
Effect.andThen(options.duringDispatch?.(events, input.messageId) ?? Effect.void),
504510
Effect.as(undefined as never),
505511
),
506-
getThreadRecords: (_threadId, _fields, filter) =>
507-
Effect.succeed({
508-
messages: (filter?.messageIds ?? []).map((id) => ({
509-
id,
510-
runId: options.messageRunId ?? null,
511-
})),
512-
runs: (options.runs ?? []).filter(
513-
(run) => filter?.runIds === undefined || filter.runIds.includes(run.id),
514-
),
515-
} as never),
512+
getThreadRecords: (_threadId, fields: ReadonlyArray<string>, filter) =>
513+
Effect.gen(function* () {
514+
if (fields.includes("runs")) yield* options.duringRunsRead?.(events) ?? Effect.void;
515+
return {
516+
messages: (filter?.messageIds ?? []).map((id) => ({
517+
id,
518+
runId: options.messageRunId ?? null,
519+
})),
520+
runs: (options.runs ?? []).filter(
521+
(run) => filter?.runIds === undefined || filter.runIds.includes(run.id),
522+
),
523+
} as never;
524+
}),
516525
streamDomainEvents: Stream.unwrap(
517526
Ref.getAndUpdate(subscriptions, (n) => n + 1).pipe(
518-
Effect.map((attempt) =>
527+
Effect.flatMap((attempt) =>
519528
attempt < (options.streamFailures ?? 0)
520-
? Stream.fail(new Error("event stream dropped") as never)
521-
: Stream.fromQueue(events),
529+
? Effect.succeed(Stream.fail(new Error("event stream dropped") as never))
530+
: (options.liveTail ? Queue.clear(events) : Effect.void).pipe(
531+
Effect.as(Stream.fromQueue(events)),
532+
),
522533
),
523534
),
524535
),
@@ -754,17 +765,74 @@ it.effect("a dropped event stream is resumed, and runs that ended meanwhile are
754765
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
755766
);
756767

757-
it.effect("runs are released as failed once the event stream cannot be resumed", () =>
768+
it.effect("the event stream keeps being resumed after many failures", () =>
758769
Effect.gen(function* () {
759-
const harness = yield* makeTurnHarness({ streamFailures: 1_000 });
770+
const runId = RunId.make("run:late");
771+
const harness = yield* makeTurnHarness({
772+
streamFailures: 8,
773+
messageRunId: runId,
774+
runs: [runOf(runId, MessageId.make("unused"), "failed")],
775+
});
760776
yield* Effect.gen(function* () {
761777
const service = yield* ScheduledTaskService.ScheduledTaskService;
762778
const { task } = yield* upsertBoundTask(service);
763779
yield* service.runNow({ id: task.id });
780+
yield* Queue.take(harness.dispatched);
764781
yield* TestClock.adjust("10 minutes");
765782
yield* untilStatus(service, task.id, "failed");
783+
assert.equal(yield* Ref.get(harness.subscriptions), 9);
784+
}).pipe(Effect.provide(harness.layer));
785+
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
786+
);
787+
788+
it.effect("an event landing while the resumed listener catches up is not missed", () =>
789+
Effect.gen(function* () {
790+
const runId = RunId.make("run:handoff");
791+
const harness = yield* makeTurnHarness({
792+
streamFailures: 1,
793+
liveTail: true,
794+
messageRunId: runId,
795+
// The projection read is stale: the run ends right after it.
796+
runs: [runOf(runId, MessageId.make("unused"), "running")],
797+
duringRunsRead: (events) =>
798+
Queue.offer(events, runEvent(runOf(runId, MessageId.make("unused"), "completed"))).pipe(
799+
Effect.asVoid,
800+
),
801+
});
802+
yield* Effect.gen(function* () {
803+
const service = yield* ScheduledTaskService.ScheduledTaskService;
804+
const { task } = yield* upsertBoundTask(service);
805+
yield* service.runNow({ id: task.id });
806+
yield* Queue.take(harness.dispatched);
807+
yield* TestClock.adjust("30 seconds");
808+
yield* untilStatus(service, task.id, "succeeded");
809+
}).pipe(Effect.provide(harness.layer));
810+
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
811+
);
812+
813+
it.effect("a cancelled run is still recorded when its first save fails", () =>
814+
Effect.gen(function* () {
815+
const harness = yield* makeTurnHarness();
816+
yield* Effect.gen(function* () {
817+
const service = yield* ScheduledTaskService.ScheduledTaskService;
818+
const sql = yield* SqlClient.SqlClient;
819+
const { task } = yield* upsertBoundTask(service);
820+
yield* service.runNow({ id: task.id });
821+
const messageId = yield* Queue.take(harness.dispatched);
822+
// Fails the first write only; the retry must still record the cancel.
823+
yield* sql`CREATE TABLE fail_once (id INTEGER)`;
824+
yield* sql`INSERT INTO fail_once VALUES (1)`;
825+
yield* sql`CREATE TRIGGER fail_first_outcome BEFORE UPDATE ON scheduled_tasks
826+
WHEN OLD.last_run_status = 'running' AND NEW.last_run_status = 'failed'
827+
AND EXISTS (SELECT 1 FROM fail_once)
828+
BEGIN DELETE FROM fail_once; SELECT RAISE(FAIL, 'outcome unavailable'); END`;
829+
yield* Queue.offerAll(harness.events, [
830+
runEvent(runOf("run:cancelled", messageId, "queued")),
831+
runEvent(runOf("run:cancelled", messageId, "cancelled")),
832+
]);
833+
yield* untilStatus(service, task.id, "failed");
766834
const [settled] = (yield* service.list()).tasks;
767-
assert.equal(settled?.lastRunError, "Run outcomes could no longer be followed.");
835+
assert.equal(settled?.lastRunError, "The run was cancelled.");
768836
}).pipe(Effect.provide(harness.layer));
769837
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
770838
);

‎apps/server/src/scheduledTasks/ScheduledTaskService.ts‎

Lines changed: 55 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import * as Data from "effect/Data";
3131
import * as DateTime from "effect/DateTime";
3232
import * as Effect from "effect/Effect";
3333
import * as Exit from "effect/Exit";
34+
import * as Fiber from "effect/Fiber";
3435
import * as Duration from "effect/Duration";
3536
import * as Metric from "effect/Metric";
3637
import * as Layer from "effect/Layer";
@@ -718,25 +719,6 @@ export const layer = Layer.effect(
718719
if (rows.length > 0) yield* notifyChanged;
719720
});
720721

721-
// Promoting a queued message to a steer cancels its queued run first and
722-
// moves the message into the active run afterwards, so a cancel only ends
723-
// the run if the message still belongs to the cancelled run.
724-
const settleEnded = Effect.fn("ScheduledTaskService.settleEnded")(function* (run: SettledRun) {
725-
if (run.cancelledRunId !== null) {
726-
const moved =
727-
run.threadId === null
728-
? null
729-
: yield* threadManagement
730-
.getThreadRecords(run.threadId, ["messages"], { messageIds: [run.messageId] })
731-
.pipe(
732-
Effect.map(({ messages }) => messages[0]?.runId ?? null),
733-
Effect.orElseSucceed(() => null),
734-
);
735-
if (!trackedRuns.confirmCancel(run.messageId, moved)) return;
736-
}
737-
yield* recordRunEnd(run);
738-
});
739-
740722
// Best-effort escape hatch: if anything fails between markRunning and
741723
// markCompleted, write a full terminal record so runDueTasks neither skips
742724
// the task forever (it filters out 'running' rows) nor re-fires it
@@ -782,8 +764,26 @@ export const layer = Layer.effect(
782764

783765
// Each outcome stands alone: one that still cannot be saved after retries is
784766
// released as failed so its row does not stay `running` and block the task.
785-
const settleOutcome = (run: SettledRun) =>
786-
settleEnded(run).pipe(
767+
const settleOutcome = Effect.fn("ScheduledTaskService.settleOutcome")(function* (
768+
run: SettledRun,
769+
) {
770+
// Promoting a queued message to a steer cancels its queued run first and
771+
// moves the message into the active run afterwards, so a cancel only ends
772+
// the run if the message still belongs to the cancelled run. Decided once:
773+
// confirming forgets the run, so it must not be repeated by a retry.
774+
if (run.cancelledRunId !== null) {
775+
const moved =
776+
run.threadId === null
777+
? null
778+
: yield* threadManagement
779+
.getThreadRecords(run.threadId, ["messages"], { messageIds: [run.messageId] })
780+
.pipe(
781+
Effect.map(({ messages }) => messages[0]?.runId ?? null),
782+
Effect.orElseSucceed(() => null),
783+
);
784+
if (!trackedRuns.confirmCancel(run.messageId, moved)) return;
785+
}
786+
yield* recordRunEnd(run).pipe(
787787
Effect.retry({ times: 2 }),
788788
Effect.catchCause((cause) =>
789789
Effect.logWarning("Could not record scheduled task run outcome", { cause }).pipe(
@@ -801,6 +801,7 @@ export const layer = Layer.effect(
801801
),
802802
),
803803
);
804+
});
804805

805806
const runTask = Effect.fn("ScheduledTaskService.runTask")(function* (
806807
task: ScheduledTask,
@@ -1178,30 +1179,44 @@ export const layer = Layer.effect(
11781179
),
11791180
);
11801181

1182+
// The stream resumes after a cursor it fixes on its first pull. Starting
1183+
// that pull before the catch-up read means an event landing in between is
1184+
// delivered by the stream instead of missed, and a run seen by both is
1185+
// settled once because the tracker forgets it when it ends.
1186+
const runEvents = Stream.fromPull(
1187+
Effect.gen(function* () {
1188+
const pull = yield* Stream.toPull(threadManagement.streamDomainEvents);
1189+
const first = yield* Effect.forkScoped(pull, { startImmediately: true });
1190+
yield* catchUpRuns;
1191+
let joined = false;
1192+
return Effect.suspend(() => {
1193+
if (joined) return pull;
1194+
joined = true;
1195+
return Fiber.join(first);
1196+
});
1197+
}),
1198+
);
1199+
11811200
// Dispatching a run only gets it accepted; its turn ends later, so the
1182-
// outcome is read from the thread's run events. If the stream drops it is
1183-
// resumed with a bounded backoff; once that is spent the runs still waiting
1184-
// are released as failed, because no one will report their end.
1201+
// outcome is read from the thread's run events. A dropped stream is resumed
1202+
// forever with a capped backoff: runs dispatched meanwhile stay tracked and
1203+
// are caught up on the next subscribe.
11851204
yield* forkParked(
1186-
catchUpRuns.pipe(
1187-
Effect.andThen(
1188-
Stream.runForEach(threadManagement.streamDomainEvents, (event) => {
1189-
// Every domain event passes through here, so the common case stays allocation-free.
1190-
const ended = trackedRuns.observe(event);
1191-
return ended.length === 0
1192-
? Effect.void
1193-
: Effect.forEach(ended, settleOutcome, { discard: true });
1194-
}),
1195-
),
1205+
Stream.runForEach(runEvents, (event) => {
1206+
// Every domain event passes through here, so the common case stays allocation-free.
1207+
const ended = trackedRuns.observe(event);
1208+
return ended.length === 0
1209+
? Effect.void
1210+
: Effect.forEach(ended, settleOutcome, { discard: true });
1211+
}).pipe(
11961212
Effect.tapCause((cause) =>
11971213
Effect.logWarning("Scheduled task run event stream failed", { cause }),
11981214
),
1199-
Effect.retry({ schedule: Schedule.exponential("1 second"), times: 5 }),
1200-
Effect.catchCause(() =>
1201-
Effect.forEach(
1202-
trackedRuns.abandonAll("Run outcomes could no longer be followed."),
1203-
settleOutcome,
1204-
{ discard: true },
1215+
Effect.retry(
1216+
Schedule.exponential("1 second").pipe(
1217+
Schedule.modifyDelay(({ duration }) =>
1218+
Effect.succeed(Duration.min(duration, Duration.minutes(1))),
1219+
),
12051220
),
12061221
),
12071222
),

‎apps/server/src/scheduledTasks/TrackedRuns.test.ts‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -105,14 +105,6 @@ it("lists only dispatched runs as pending, and follows a run bound from the proj
105105
assert.deepEqual(runs.pending(), []);
106106
});
107107

108-
it("fails every dispatched run when events can no longer be followed", () => {
109-
const runs = makeTrackedRuns();
110-
runs.track(messageId, taskId, "start");
111-
runs.dispatched(messageId, threadId);
112-
assert.deepEqual(runs.abandonAll("gave up"), [settledFor("failed", "gave up")]);
113-
assert.deepEqual(runs.abandonAll("gave up"), []);
114-
});
115-
116108
it("fails a run whose thread is deleted before the turn ends", () => {
117109
const runs = makeTrackedRuns();
118110
runs.track(messageId, taskId, "start");

‎apps/server/src/scheduledTasks/TrackedRuns.ts‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -131,11 +131,6 @@ export function makeTrackedRuns() {
131131
const run = runs.get(messageId);
132132
if (run !== undefined) run.runId = runId;
133133
},
134-
/** Ends every dispatched run as failed, for when events can no longer be followed. */
135-
abandonAll: (error: string): ReadonlyArray<SettledRun> =>
136-
[...runs.values()].flatMap((run) =>
137-
run.dispatched ? settled(end(run, { status: "failed", error, cancelledRunId: null })) : [],
138-
),
139134
/**
140135
* Rules on a cancelled run, given the run its message belongs to now.
141136
* Returns false when the message moved into another run, which is then

0 commit comments

Comments
 (0)