Skip to content

Commit 9eb5649

Browse files
saphidclaude
andcommitted
fix(server): resume the scheduled-task archive reactor after a stream failure
The reactor that pauses tasks bound to archived or deleted threads ended for good on any failure of its event stream, such as a transient read error or a full stream buffer, until the server restarted. It now tracks the last event it handled and resubscribes from there after a short delay. Replaying an archive is harmless because pausing is idempotent. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 8df991a commit 9eb5649

2 files changed

Lines changed: 99 additions & 16 deletions

File tree

‎apps/server/src/scheduledTasks/ScheduledTaskService.test.ts‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import * as Schema from "effect/Schema";
3737
import * as TestClock from "effect/testing/TestClock";
3838

3939
import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
40+
import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
4041
import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
4142
import * as SecretRequests from "../secrets/SecretRequests.ts";
4243
import * as SqlitePersistence from "../persistence/Sqlite.ts";
@@ -2876,6 +2877,78 @@ it.effect("the domain-event reactor pauses tasks on thread archive and delete",
28762877
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
28772878
);
28782879

2880+
it.effect("the domain-event reactor resumes after its event stream fails", () =>
2881+
Effect.gen(function* () {
2882+
yield* Effect.scoped(
2883+
Effect.gen(function* () {
2884+
const threadId = ThreadId.make("thread:reactor-resumes");
2885+
const firstFailed = yield* Deferred.make<void>();
2886+
const cursors: Array<number | undefined> = [];
2887+
// The first subscription fails like a transient read error. The resumed
2888+
// one delivers the archive committed after the reactor's cursor.
2889+
const threadManagement = Layer.mock(ThreadManagementService.ThreadManagementService)({
2890+
streamStoredEventsFrom: (input) =>
2891+
Stream.unwrap(
2892+
Effect.gen(function* () {
2893+
cursors.push(input?.afterSequence);
2894+
if (cursors.length === 1) {
2895+
yield* Deferred.succeed(firstFailed, undefined);
2896+
return Stream.fail(
2897+
new Orchestrator.OrchestratorDomainEventStreamError({
2898+
cause: "database is locked",
2899+
}),
2900+
);
2901+
}
2902+
return Stream.make({
2903+
sequence: (input?.afterSequence ?? 0) + 1,
2904+
commandId: null,
2905+
event: {
2906+
id: EventId.make("event:reactor-resumes"),
2907+
type: "thread.archived",
2908+
threadId,
2909+
providerInstanceId: ProviderInstanceId.make("codex"),
2910+
occurredAt: DateTime.makeUnsafe(boundThreadArchivedAt),
2911+
payload: { id: threadId },
2912+
} as unknown as OrchestrationV2DomainEvent,
2913+
} satisfies OrchestrationV2StoredEvent).pipe(Stream.concat(Stream.never));
2914+
}),
2915+
),
2916+
});
2917+
const context = yield* Layer.build(
2918+
ScheduledTaskService.layer.pipe(
2919+
Layer.provide(
2920+
Layer.mergeAll(
2921+
NodeCrypto.layer,
2922+
Scheduler.layer,
2923+
Layer.mock(ThreadLaunchService.ThreadLaunchService)({}),
2924+
Layer.mock(SecretRequests.SecretRequests)({}),
2925+
threadManagement,
2926+
),
2927+
),
2928+
),
2929+
);
2930+
const tasks = Context.get(context, ScheduledTaskService.ScheduledTaskService);
2931+
yield* setBoundThreadState(threadId, "active");
2932+
const task = yield* tasks.upsert({ ...boundTaskInput, enabled: true, threadId });
2933+
yield* setBoundThreadState(threadId, "archived");
2934+
yield* Deferred.await(firstFailed);
2935+
2936+
// Well before the task is due, so its fire-time check cannot pause it.
2937+
yield* TestClock.adjust("10 seconds");
2938+
2939+
const paused = yield* tasks.subscribeList().pipe(
2940+
Stream.map(({ tasks: all }) => all.find((candidate) => candidate.id === task.task.id)),
2941+
Stream.filter((candidate) => candidate !== undefined && candidate.enabled === false),
2942+
Stream.runHead,
2943+
);
2944+
assert.isTrue(Option.isSome(paused));
2945+
assert.equal(cursors.length, 2);
2946+
assert.equal(cursors[1], cursors[0]);
2947+
}),
2948+
);
2949+
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
2950+
);
2951+
28792952
it.effect("replays an archive committed after the startup sweep checked its thread", () =>
28802953
Effect.gen(function* () {
28812954
yield* Effect.scoped(

‎apps/server/src/scheduledTasks/ScheduledTaskService.ts‎

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import * as Predicate from "effect/Predicate";
3838
import * as PubSub from "effect/PubSub";
3939
import * as Ref from "effect/Ref";
4040
import * as Result from "effect/Result";
41+
import * as Schedule from "effect/Schedule";
4142
import * as Schema from "effect/Schema";
4243
import * as Semaphore from "effect/Semaphore";
4344
import * as Stream from "effect/Stream";
@@ -1539,26 +1540,35 @@ export const layer = Layer.effect(
15391540
// subscribes, and the forked reactor subscribes some time after this
15401541
// layer is built. Resume from the sequence read before the sweep so an
15411542
// archive committed in that gap is replayed rather than missed.
1542-
const reactorCursor = yield* latestEventSeq.pipe(
1543-
Effect.map(Option.some),
1544-
Effect.catchCause((cause) =>
1545-
Effect.logWarning("Could not read the event cursor for the archive reactor", {
1546-
cause,
1547-
}).pipe(Effect.as(Option.none<number>())),
1543+
const reactorCursor = yield* Ref.make(
1544+
yield* latestEventSeq.pipe(
1545+
Effect.map(Option.some),
1546+
Effect.catchCause((cause) =>
1547+
Effect.logWarning("Could not read the event cursor for the archive reactor", {
1548+
cause,
1549+
}).pipe(Effect.as(Option.none<number>())),
1550+
),
15481551
),
15491552
);
1550-
const lifecycleEvents = Option.match(reactorCursor, {
1551-
onNone: () => threadManagement.streamDomainEvents,
1552-
onSome: (afterSequence) =>
1553-
threadManagement
1554-
.streamStoredEventsFrom({ afterSequence })
1555-
.pipe(Stream.map((stored) => stored.event)),
1556-
});
1557-
yield* Stream.runForEach(lifecycleEvents, (event) =>
1558-
event.type === "thread.archived" || event.type === "thread.deleted"
1553+
// A failed read or a full stream buffer ends the subscription. The reactor
1554+
// resubscribes after the last event it handled, so pausing is delayed, not
1555+
// stopped until restart. Pausing an already paused task is a no-op.
1556+
const lifecycleEvents = Stream.unwrap(
1557+
Ref.get(reactorCursor).pipe(
1558+
Effect.flatMap(Option.match({ onNone: () => latestEventSeq, onSome: Effect.succeed })),
1559+
Effect.map((afterSequence) => threadManagement.streamStoredEventsFrom({ afterSequence })),
1560+
),
1561+
);
1562+
yield* Stream.runForEach(lifecycleEvents, ({ sequence, event }) =>
1563+
(event.type === "thread.archived" || event.type === "thread.deleted"
15591564
? pauseLogged(event.threadId)
1560-
: Effect.void,
1565+
: Effect.void
1566+
).pipe(Effect.andThen(Ref.set(reactorCursor, Option.some(sequence)))),
15611567
).pipe(
1568+
Effect.tapError((error) =>
1569+
Effect.logWarning("Schedule task archive reactor failed; resubscribing", { error }),
1570+
),
1571+
Effect.retry({ schedule: Schedule.spaced("5 seconds") }),
15621572
Effect.catchCause((cause) =>
15631573
Effect.logWarning("Schedule task archive reactor stopped", { cause }),
15641574
),

0 commit comments

Comments
 (0)