Skip to content

Commit 0e9db72

Browse files
voida6claude
andcommitted
fix(server): scheduled run outcomes survive early ends, promoted steers and a dropped event stream
- Persist the dispatch bookkeeping before an outcome can replace it, so a turn that ends right after dispatch still advances next_run_at. - A queued run cancelled only because its message was promoted into the active run no longer ends the task; the message's current run is checked first. - An outcome that cannot be saved is retried, then the run is released as failed instead of staying running. - The event listener resumes with a bounded backoff, catches up runs that ended meanwhile from the thread projection, and releases the rest as failed once retries are spent. Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
1 parent 404b655 commit 0e9db72

4 files changed

Lines changed: 442 additions & 90 deletions

File tree

‎apps/server/src/scheduledTasks/ScheduledTaskService.test.ts‎

Lines changed: 202 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -474,37 +474,77 @@ const providerErrorEvent = (runId: string, message: string): OrchestrationV2Doma
474474
});
475475

476476
// Dispatches to a bound thread succeed at once; the run's events are fed by hand.
477-
const makeTurnHarness = Effect.gen(function* () {
478-
const events = yield* Queue.unbounded<OrchestrationV2DomainEvent>();
479-
const dispatched = yield* Queue.unbounded<MessageId>();
480-
const layer = ScheduledTaskService.layer.pipe(
481-
Layer.provide(
482-
Layer.mergeAll(
483-
Layer.mock(ThreadLaunchService.ThreadLaunchService)({}),
484-
Layer.mock(ThreadManagementService.ThreadManagementService)({
485-
sendToThread: (input) =>
486-
Queue.offer(dispatched, input.messageId).pipe(Effect.as(undefined as never)),
487-
streamDomainEvents: Stream.fromQueue(events),
488-
}),
489-
Layer.mock(SecretRequests.SecretRequests)({}),
490-
Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
491-
namedProjectsRoot: "/projects",
492-
isScratchProject: () => Effect.succeed(false),
493-
}),
494-
NodeCrypto.layer,
495-
Scheduler.layer,
477+
const makeTurnHarness = (
478+
options: {
479+
/** Runs inside the dispatch, so the turn's events can land before it returns. */
480+
readonly duringDispatch?: (
481+
events: Queue.Queue<OrchestrationV2DomainEvent>,
482+
messageId: MessageId,
483+
) => Effect.Effect<void>;
484+
/** The run the thread's projection says the scheduled message now belongs to. */
485+
readonly messageRunId?: RunId;
486+
/** The runs the thread's projection holds, as read back after the event stream drops. */
487+
readonly runs?: ReadonlyArray<OrchestrationV2Run>;
488+
/** How many times the event stream fails before it starts to deliver events. */
489+
readonly streamFailures?: number;
490+
} = {},
491+
) =>
492+
Effect.gen(function* () {
493+
const events = yield* Queue.unbounded<OrchestrationV2DomainEvent>();
494+
const dispatched = yield* Queue.unbounded<MessageId>();
495+
const subscriptions = yield* Ref.make(0);
496+
const layer = ScheduledTaskService.layer.pipe(
497+
Layer.provide(
498+
Layer.mergeAll(
499+
Layer.mock(ThreadLaunchService.ThreadLaunchService)({}),
500+
Layer.mock(ThreadManagementService.ThreadManagementService)({
501+
sendToThread: (input) =>
502+
Queue.offer(dispatched, input.messageId).pipe(
503+
Effect.andThen(options.duringDispatch?.(events, input.messageId) ?? Effect.void),
504+
Effect.as(undefined as never),
505+
),
506+
getThreadRecords: (_threadId, _fields, filter) =>
507+
Effect.succeed({
508+
messages: (filter?.messageIds ?? []).map((id) => ({
509+
id,
510+
runId: options.messageRunId ?? null,
511+
})),
512+
runs: (options.runs ?? []).filter(
513+
(run) => filter?.runIds === undefined || filter.runIds.includes(run.id),
514+
),
515+
} as never),
516+
streamDomainEvents: Stream.unwrap(
517+
Ref.getAndUpdate(subscriptions, (n) => n + 1).pipe(
518+
Effect.map((attempt) =>
519+
attempt < (options.streamFailures ?? 0)
520+
? Stream.fail(new Error("event stream dropped") as never)
521+
: Stream.fromQueue(events),
522+
),
523+
),
524+
),
525+
}),
526+
Layer.mock(SecretRequests.SecretRequests)({}),
527+
Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
528+
namedProjectsRoot: "/projects",
529+
isScratchProject: () => Effect.succeed(false),
530+
}),
531+
NodeCrypto.layer,
532+
Scheduler.layer,
533+
),
496534
),
497-
),
498-
);
499-
return { events, dispatched, layer };
500-
});
535+
);
536+
return { events, dispatched, subscriptions, layer };
537+
});
501538

502-
const upsertBoundTask = (service: ScheduledTaskService.ScheduledTaskService["Service"]) =>
539+
const upsertBoundTask = (
540+
service: ScheduledTaskService.ScheduledTaskService["Service"],
541+
enabled = false,
542+
) =>
503543
service.upsert({
504544
id: ScheduledTaskId.make("task:turn-outcome"),
505545
title: "Check orchestration",
506546
prompt: "Check progress",
507-
enabled: false,
547+
enabled,
508548
schedule: { type: "interval", everyMs: 60_000 },
509549
projectId: ProjectId.make("project:test"),
510550
threadId: turnThreadId,
@@ -527,7 +567,7 @@ const untilStatus = (
527567

528568
it.effect("a failed turn replaces the dispatched run's status with the provider error", () =>
529569
Effect.gen(function* () {
530-
const harness = yield* makeTurnHarness;
570+
const harness = yield* makeTurnHarness();
531571
yield* Effect.gen(function* () {
532572
const service = yield* ScheduledTaskService.ScheduledTaskService;
533573
const { task } = yield* upsertBoundTask(service);
@@ -553,7 +593,7 @@ it.effect("a failed turn replaces the dispatched run's status with the provider
553593

554594
it.effect("a completed turn records the run as succeeded", () =>
555595
Effect.gen(function* () {
556-
const harness = yield* makeTurnHarness;
596+
const harness = yield* makeTurnHarness();
557597
yield* Effect.gen(function* () {
558598
const service = yield* ScheduledTaskService.ScheduledTaskService;
559599
const { task } = yield* upsertBoundTask(service);
@@ -572,7 +612,7 @@ it.effect("a completed turn records the run as succeeded", () =>
572612

573613
it.effect("an earlier run's turn ending cannot overwrite a newer run", () =>
574614
Effect.gen(function* () {
575-
const harness = yield* makeTurnHarness;
615+
const harness = yield* makeTurnHarness();
576616
yield* Effect.gen(function* () {
577617
const service = yield* ScheduledTaskService.ScheduledTaskService;
578618
const { task } = yield* upsertBoundTask(service);
@@ -594,3 +634,137 @@ it.effect("an earlier run's turn ending cannot overwrite a newer run", () =>
594634
}).pipe(Effect.provide(harness.layer));
595635
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
596636
);
637+
638+
const messageMovedEvent = (messageId: MessageId, runId: string): OrchestrationV2DomainEvent => ({
639+
id: EventId.make(`event:${messageId}:moved`),
640+
type: "message.updated",
641+
threadId: turnThreadId,
642+
runId: RunId.make(runId),
643+
occurredAt: epoch,
644+
payload: {
645+
id: messageId,
646+
threadId: turnThreadId,
647+
runId: RunId.make(runId),
648+
nodeId: null,
649+
role: "user",
650+
text: "Check progress",
651+
attachments: [],
652+
streaming: false,
653+
createdAt: epoch,
654+
updatedAt: epoch,
655+
createdBy: "user",
656+
creationSource: "web",
657+
},
658+
});
659+
660+
it.effect("a turn that ends while the dispatch is being recorded still advances the schedule", () =>
661+
Effect.gen(function* () {
662+
const harness = yield* makeTurnHarness({
663+
duringDispatch: (events, messageId) =>
664+
Queue.offerAll(events, [
665+
runEvent(runOf("run:fast", messageId, "queued")),
666+
runEvent(runOf("run:fast", messageId, "failed")),
667+
]),
668+
});
669+
yield* Effect.gen(function* () {
670+
const service = yield* ScheduledTaskService.ScheduledTaskService;
671+
const { task } = yield* upsertBoundTask(service, true);
672+
yield* TestClock.adjust("30 seconds");
673+
yield* service.runNow({ id: task.id });
674+
yield* untilStatus(service, task.id, "failed");
675+
const [settled] = (yield* service.list()).tasks;
676+
assert.equal(settled?.runCount, 1);
677+
// Dispatched 30 seconds after the task was saved, so the next run is 60 seconds on from then.
678+
assert.equal(
679+
settled?.nextRunAt,
680+
DateTime.formatIso(DateTime.makeUnsafe(Date.parse(task.nextRunAt!) + 30_000)),
681+
);
682+
}).pipe(Effect.provide(harness.layer));
683+
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
684+
);
685+
686+
it.effect("a queued message promoted to a steer keeps its run going in the active run", () =>
687+
Effect.gen(function* () {
688+
const harness = yield* makeTurnHarness({ messageRunId: RunId.make("run:active") });
689+
yield* Effect.gen(function* () {
690+
const service = yield* ScheduledTaskService.ScheduledTaskService;
691+
const { task } = yield* upsertBoundTask(service);
692+
yield* service.runNow({ id: task.id });
693+
const messageId = yield* Queue.take(harness.dispatched);
694+
yield* Queue.offerAll(harness.events, [
695+
runEvent(runOf("run:queued", messageId, "queued")),
696+
// Promotion cancels the queued run before it moves the message into the active one.
697+
runEvent(runOf("run:queued", messageId, "cancelled")),
698+
messageMovedEvent(messageId, "run:active"),
699+
runEvent(runOf("run:active", MessageId.make("message:active-turn"), "completed")),
700+
]);
701+
yield* service.subscribeList().pipe(
702+
Stream.filter(({ tasks }) => tasks.some((t) => t.lastRunStatus !== "running")),
703+
Stream.take(1),
704+
Stream.runDrain,
705+
);
706+
const [settled] = (yield* service.list()).tasks;
707+
assert.equal(settled?.lastRunStatus, "succeeded");
708+
assert.isNull(settled?.lastRunError);
709+
}).pipe(Effect.provide(harness.layer));
710+
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
711+
);
712+
713+
it.effect("a run whose outcome cannot be saved is released instead of left running", () =>
714+
Effect.gen(function* () {
715+
const harness = yield* makeTurnHarness();
716+
yield* Effect.gen(function* () {
717+
const service = yield* ScheduledTaskService.ScheduledTaskService;
718+
const sql = yield* SqlClient.SqlClient;
719+
const { task } = yield* upsertBoundTask(service);
720+
yield* service.runNow({ id: task.id });
721+
const messageId = yield* Queue.take(harness.dispatched);
722+
// Only the release path may write the terminal state.
723+
yield* sql`CREATE TRIGGER fail_outcome BEFORE UPDATE ON scheduled_tasks
724+
WHEN OLD.last_run_status = 'running' AND NEW.last_run_status = 'succeeded'
725+
BEGIN SELECT RAISE(FAIL, 'outcome unavailable'); END`;
726+
yield* Queue.offerAll(harness.events, [
727+
runEvent(runOf("run:unsaved", messageId, "completed")),
728+
]);
729+
yield* untilStatus(service, task.id, "failed");
730+
const [settled] = (yield* service.list()).tasks;
731+
assert.equal(settled?.lastRunError, "Could not record how the run ended.");
732+
}).pipe(Effect.provide(harness.layer));
733+
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
734+
);
735+
736+
it.effect("a dropped event stream is resumed, and runs that ended meanwhile are caught up", () =>
737+
Effect.gen(function* () {
738+
const runId = RunId.make("run:during-gap");
739+
const harness = yield* makeTurnHarness({
740+
streamFailures: 1,
741+
messageRunId: runId,
742+
runs: [runOf(runId, MessageId.make("unused"), "completed")],
743+
});
744+
yield* Effect.gen(function* () {
745+
const service = yield* ScheduledTaskService.ScheduledTaskService;
746+
const { task } = yield* upsertBoundTask(service);
747+
yield* service.runNow({ id: task.id });
748+
yield* Queue.take(harness.dispatched);
749+
// The run completed while no one was listening; the resumed listener finds it.
750+
yield* TestClock.adjust("30 seconds");
751+
yield* untilStatus(service, task.id, "succeeded");
752+
assert.equal(yield* Ref.get(harness.subscriptions), 2);
753+
}).pipe(Effect.provide(harness.layer));
754+
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
755+
);
756+
757+
it.effect("runs are released as failed once the event stream cannot be resumed", () =>
758+
Effect.gen(function* () {
759+
const harness = yield* makeTurnHarness({ streamFailures: 1_000 });
760+
yield* Effect.gen(function* () {
761+
const service = yield* ScheduledTaskService.ScheduledTaskService;
762+
const { task } = yield* upsertBoundTask(service);
763+
yield* service.runNow({ id: task.id });
764+
yield* TestClock.adjust("10 minutes");
765+
yield* untilStatus(service, task.id, "failed");
766+
const [settled] = (yield* service.list()).tasks;
767+
assert.equal(settled?.lastRunError, "Run outcomes could no longer be followed.");
768+
}).pipe(Effect.provide(harness.layer));
769+
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
770+
);

0 commit comments

Comments
 (0)