Skip to content

Commit 018dee7

Browse files
voida6claude
andcommitted
fix(server): scheduled run catch-up no longer strands a run it cannot read
- Retry the projection read a few times, then release the run as failed and stop tracking it, so one unreadable thread neither leaves its row running nor makes the listener resubscribe forever. - Record a thread deleted while the stream was down as failed; deletion keeps the projection flagged rather than removing it. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 2cbe737 commit 018dee7

4 files changed

Lines changed: 100 additions & 20 deletions

File tree

‎apps/server/src/scheduledTasks/ScheduledTaskService.test.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -487,6 +487,10 @@ const makeTurnHarness = (
487487
readonly runs?: ReadonlyArray<OrchestrationV2Run>;
488488
/** How many times the event stream fails before it starts to deliver events. */
489489
readonly streamFailures?: number;
490+
/** Reads the thread as deleted, which the projection keeps as a flagged record. */
491+
readonly threadDeleted?: boolean;
492+
/** Makes every projection read fail, as for a thread whose records are gone. */
493+
readonly recordsUnreadable?: boolean;
490494
/** Makes the stream a live tail: events from before it was opened are not delivered. */
491495
readonly liveTail?: boolean;
492496
/** Runs when the projection's runs are read, so an event can land mid catch-up. */
@@ -511,8 +515,11 @@ const makeTurnHarness = (
511515
),
512516
getThreadRecords: (_threadId, fields: ReadonlyArray<string>, filter) =>
513517
Effect.gen(function* () {
518+
if (options.recordsUnreadable)
519+
return yield* Effect.fail(new Error("no records") as never);
514520
if (fields.includes("runs")) yield* options.duringRunsRead?.(events) ?? Effect.void;
515521
return {
522+
thread: { deletedAt: options.threadDeleted ? epoch : null },
516523
messages: (filter?.messageIds ?? []).map((id) => ({
517524
id,
518525
runId: options.messageRunId ?? null,
@@ -836,3 +843,39 @@ it.effect("a cancelled run is still recorded when its first save fails", () =>
836843
}).pipe(Effect.provide(harness.layer));
837844
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
838845
);
846+
847+
it.effect(
848+
"a run whose thread cannot be read during catch-up is released, not retried forever",
849+
() =>
850+
Effect.gen(function* () {
851+
const harness = yield* makeTurnHarness({ streamFailures: 1, recordsUnreadable: true });
852+
yield* Effect.gen(function* () {
853+
const service = yield* ScheduledTaskService.ScheduledTaskService;
854+
const { task } = yield* upsertBoundTask(service);
855+
yield* service.runNow({ id: task.id });
856+
yield* Queue.take(harness.dispatched);
857+
yield* TestClock.adjust("1 minute");
858+
yield* untilStatus(service, task.id, "failed");
859+
const [settled] = (yield* service.list()).tasks;
860+
assert.equal(settled?.lastRunError, "Could not read how the run ended.");
861+
// The stream was resubscribed once for the drop, not again for the unreadable thread.
862+
assert.equal(yield* Ref.get(harness.subscriptions), 2);
863+
}).pipe(Effect.provide(harness.layer));
864+
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
865+
);
866+
867+
it.effect("a thread deleted while the stream was down fails its run on catch-up", () =>
868+
Effect.gen(function* () {
869+
const harness = yield* makeTurnHarness({ streamFailures: 1, threadDeleted: true });
870+
yield* Effect.gen(function* () {
871+
const service = yield* ScheduledTaskService.ScheduledTaskService;
872+
const { task } = yield* upsertBoundTask(service);
873+
yield* service.runNow({ id: task.id });
874+
yield* Queue.take(harness.dispatched);
875+
yield* TestClock.adjust("30 seconds");
876+
yield* untilStatus(service, task.id, "failed");
877+
const [settled] = (yield* service.list()).tasks;
878+
assert.equal(settled?.lastRunError, "The thread was deleted.");
879+
}).pipe(Effect.provide(harness.layer));
880+
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
881+
);

‎apps/server/src/scheduledTasks/ScheduledTaskService.ts‎

Lines changed: 45 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -762,6 +762,15 @@ export const layer = Layer.effect(
762762
),
763763
);
764764

765+
const releaseRun = (run: Pick<SettledRun, "taskId" | "startedAtIso">, message: string) =>
766+
Effect.result(findTask(run.taskId)).pipe(
767+
Effect.flatMap((found) =>
768+
Result.isSuccess(found) && found.success !== null
769+
? releaseStuckRun(found.success, message, run.startedAtIso, undefined)
770+
: Effect.void,
771+
),
772+
);
773+
765774
// Each outcome stands alone: one that still cannot be saved after retries is
766775
// released as failed so its row does not stay `running` and block the task.
767776
const settleOutcome = Effect.fn("ScheduledTaskService.settleOutcome")(function* (
@@ -787,17 +796,7 @@ export const layer = Layer.effect(
787796
Effect.retry({ times: 2 }),
788797
Effect.catchCause((cause) =>
789798
Effect.logWarning("Could not record scheduled task run outcome", { cause }).pipe(
790-
Effect.andThen(Effect.result(findTask(run.taskId))),
791-
Effect.flatMap((found) =>
792-
Result.isSuccess(found) && found.success !== null
793-
? releaseStuckRun(
794-
found.success,
795-
"Could not record how the run ended.",
796-
run.startedAtIso,
797-
undefined,
798-
)
799-
: Effect.void,
800-
),
799+
Effect.andThen(releaseRun(run, "Could not record how the run ended.")),
801800
),
802801
),
803802
);
@@ -1147,11 +1146,28 @@ export const layer = Layer.effect(
11471146
const catchUpRuns = Effect.suspend(() =>
11481147
Effect.forEach(
11491148
trackedRuns.pending(),
1150-
({ messageId, threadId }) =>
1149+
(pending) =>
11511150
Effect.gen(function* () {
1152-
const { messages } = yield* threadManagement.getThreadRecords(threadId, ["messages"], {
1153-
messageIds: [messageId],
1154-
});
1151+
const { messageId, threadId } = pending;
1152+
const { thread, messages } = yield* threadManagement.getThreadRecords(
1153+
threadId,
1154+
["messages"],
1155+
{ messageIds: [messageId] },
1156+
);
1157+
const occurredAt = yield* DateTime.now;
1158+
// Deleting a thread keeps its projection, flagged, so a delete missed
1159+
// with the stream shows up here rather than as a read failure.
1160+
if (thread.deletedAt !== null) {
1161+
const ended = trackedRuns.observe({
1162+
id: EventId.make(`scheduled-task-catch-up:${threadId}:deleted`),
1163+
type: "thread.deleted",
1164+
threadId,
1165+
occurredAt,
1166+
payload: undefined as never,
1167+
});
1168+
yield* Effect.forEach(ended, settleOutcome, { discard: true });
1169+
return;
1170+
}
11551171
const runId = messages[0]?.runId ?? null;
11561172
if (runId === null) return;
11571173
const { runs } = yield* threadManagement.getThreadRecords(threadId, ["runs"], {
@@ -1160,7 +1176,6 @@ export const layer = Layer.effect(
11601176
const run = runs[0];
11611177
if (run === undefined) return;
11621178
trackedRuns.bindRun(messageId, runId);
1163-
const occurredAt = yield* DateTime.now;
11641179
const ended = trackedRuns.observe({
11651180
id: EventId.make(`scheduled-task-catch-up:${runId}`),
11661181
type: "run.updated",
@@ -1171,8 +1186,21 @@ export const layer = Layer.effect(
11711186
});
11721187
yield* Effect.forEach(ended, settleOutcome, { discard: true });
11731188
}).pipe(
1189+
// A thread that cannot be read (deleted, records gone) would otherwise
1190+
// be retried by every resubscribe and nothing else would settle its
1191+
// row, so after a few tries the run is released as failed.
1192+
Effect.retry({ times: 3, schedule: Schedule.exponential("500 millis") }),
11741193
Effect.catchCause((cause) =>
1175-
Effect.logWarning("Could not catch up a scheduled task run", { messageId, cause }),
1194+
Effect.logWarning("Could not catch up a scheduled task run", {
1195+
messageId: pending.messageId,
1196+
cause,
1197+
}).pipe(
1198+
Effect.andThen(
1199+
Effect.sync(() => trackedRuns.forgetTask(pending.taskId)).pipe(
1200+
Effect.andThen(releaseRun(pending, "Could not read how the run ended.")),
1201+
),
1202+
),
1203+
),
11761204
),
11771205
),
11781206
{ discard: true },

‎apps/server/src/scheduledTasks/TrackedRuns.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ it("lists only dispatched runs as pending, and follows a run bound from the proj
9797
runs.track(messageId, taskId, "start");
9898
assert.deepEqual(runs.pending(), []);
9999
runs.dispatched(messageId, threadId);
100-
assert.deepEqual(runs.pending(), [{ messageId, threadId }]);
100+
assert.deepEqual(runs.pending(), [{ taskId, startedAtIso: "start", messageId, threadId }]);
101101
runs.bindRun(messageId, RunId.make("run:caught-up"));
102102
assert.deepEqual(runs.observe(runEvent("run:caught-up", "completed", MessageId.make("other"))), [
103103
settledFor("succeeded", null),

‎apps/server/src/scheduledTasks/TrackedRuns.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,10 +120,19 @@ export function makeTrackedRuns() {
120120
return held === null ? null : end(run, held);
121121
},
122122
/** Dispatched runs still waiting for an end, for catching up after missed events. */
123-
pending: (): ReadonlyArray<{ readonly messageId: MessageId; readonly threadId: ThreadId }> =>
123+
pending: (): ReadonlyArray<
124+
Pick<SettledRun, "taskId" | "startedAtIso" | "messageId"> & { readonly threadId: ThreadId }
125+
> =>
124126
[...runs.values()].flatMap((run) =>
125127
run.dispatched && run.threadId !== null
126-
? [{ messageId: run.messageId, threadId: run.threadId }]
128+
? [
129+
{
130+
taskId: run.taskId,
131+
startedAtIso: run.startedAtIso,
132+
messageId: run.messageId,
133+
threadId: run.threadId,
134+
},
135+
]
127136
: [],
128137
),
129138
/** Follows a run found by reading the projection rather than by events. */

0 commit comments

Comments
 (0)