Skip to content

Commit 2911fbd

Browse files
fix(server): judge a scheduled run's preparation by its first exit
The watcher read the run's current status, so a retried preparation could hide its first failure, and a provider failure after setup could be blamed on setup. It now replays the new thread's run updates and decides from the run's first status other than "preparing". Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent ec0fb71 commit 2911fbd

2 files changed

Lines changed: 92 additions & 51 deletions

File tree

‎apps/server/src/scheduledTasks/ScheduledTaskService.test.ts‎

Lines changed: 73 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -398,12 +398,18 @@ it.effect("records a run whose worktree cannot be made as failed instead of laun
398398
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
399399
);
400400

401-
it.effect("marks a run failed when its workspace preparation fails after launch", () =>
401+
// Runs one worktree task whose launched thread reports the given run updates,
402+
// from this run unless marked as another run's.
403+
const runWithRunUpdates = (
404+
updates: Stream.Stream<{ readonly status: string; readonly otherRun?: true }>,
405+
afterRun: (
406+
service: ScheduledTaskService.ScheduledTaskService["Service"],
407+
) => Effect.Effect<void, ScheduledTaskError>,
408+
) =>
402409
Effect.gen(function* () {
403410
const threadId = ThreadId.make("thread:scheduled-preparation");
404411
const runId = RunId.make("run:scheduled-preparation");
405412
const messageId = yield* Deferred.make<string>();
406-
const reads = yield* Ref.make(0);
407413
const layerDependencies = Layer.mergeAll(
408414
NodeCrypto.layer,
409415
Scheduler.layer,
@@ -415,21 +421,28 @@ it.effect("marks a run failed when its workspace preparation fails after launch"
415421
),
416422
}),
417423
Layer.mock(ThreadManagementService.ThreadManagementService)({
418-
getThreadEventSequence: () => Effect.succeed(0),
419-
// Preparing when the scheduler first looks, failed once the run moves on.
420-
getThreadRecords: () =>
421-
Effect.all([Deferred.await(messageId), Ref.getAndUpdate(reads, (count) => count + 1)]).pipe(
422-
Effect.map(
423-
([userMessageId, count]) =>
424-
({
425-
runs: [{ id: runId, userMessageId, status: count === 0 ? "preparing" : "failed" }],
426-
}) as never,
424+
streamStoredEventsFrom: () =>
425+
Stream.unwrap(
426+
Deferred.await(messageId).pipe(
427+
Effect.map((userMessageId) =>
428+
updates.pipe(
429+
Stream.map(
430+
({ status, otherRun }) =>
431+
({
432+
event: {
433+
type: "run.updated",
434+
payload: {
435+
id: runId,
436+
userMessageId: otherRun ? "message:other-run" : userMessageId,
437+
status,
438+
},
439+
},
440+
}) as never,
441+
),
442+
),
443+
),
427444
),
428445
),
429-
streamStoredEventsFrom: () =>
430-
Stream.make({
431-
event: { type: "run.updated", payload: { id: runId, status: "failed" } },
432-
} as never),
433446
}),
434447
Layer.mock(SecretRequests.SecretRequests)({}),
435448
);
@@ -450,12 +463,50 @@ it.effect("marks a run failed when its workspace preparation fails after launch"
450463
);
451464
const { task: ran } = yield* service.runNow({ id: task.id });
452465
assert.equal(ran.lastRunStatus, "succeeded");
453-
const failed = yield* service.subscribeList().pipe(
454-
Stream.map((list) => list.tasks[0]),
455-
Stream.filter((listed) => listed?.lastRunStatus === "failed"),
456-
Stream.runHead,
457-
);
458-
assert.include(Option.getOrUndefined(failed)?.lastRunError, "Workspace preparation failed");
466+
yield* afterRun(service);
459467
}).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))));
460-
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
468+
}).pipe(Effect.provide(SqlitePersistence.layerMemory));
469+
470+
it.effect("marks a run failed when its workspace preparation fails after launch", () =>
471+
// Another run moving on does not count. Preparation fails, then the user
472+
// retries it: the run's first exit from preparation decides.
473+
runWithRunUpdates(
474+
Stream.make(
475+
{ status: "starting", otherRun: true } as const,
476+
{ status: "preparing" },
477+
{ status: "failed" },
478+
{ status: "preparing" },
479+
),
480+
(service) =>
481+
Effect.gen(function* () {
482+
const failed = yield* service.subscribeList().pipe(
483+
Stream.map((list) => list.tasks[0]),
484+
Stream.filter((listed) => listed?.lastRunStatus === "failed"),
485+
Stream.runHead,
486+
);
487+
assert.include(Option.getOrUndefined(failed)?.lastRunError, "Workspace preparation failed");
488+
}),
489+
),
490+
);
491+
492+
it.effect("leaves a run succeeded when the provider fails after preparation", () =>
493+
Effect.gen(function* () {
494+
const closed = yield* Deferred.make<void>();
495+
const readFailure = yield* Deferred.make<void>();
496+
// This run's later "failed" is produced only if read, so reading past
497+
// "starting" is visible.
498+
const updates = Stream.concat(
499+
Stream.make({ status: "preparing" }, { status: "starting" }),
500+
Stream.fromEffect(
501+
Deferred.succeed(readFailure, undefined).pipe(Effect.as({ status: "failed" })),
502+
),
503+
).pipe(Stream.ensuring(Deferred.succeed(closed, undefined)));
504+
yield* runWithRunUpdates(updates, (service) =>
505+
Effect.gen(function* () {
506+
yield* Deferred.await(closed);
507+
assert.isFalse(yield* Deferred.isDone(readFailure));
508+
assert.equal((yield* service.list()).tasks[0]?.lastRunStatus, "succeeded");
509+
}),
510+
);
511+
}),
461512
);

‎apps/server/src/scheduledTasks/ScheduledTaskService.ts‎

Lines changed: 19 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -675,45 +675,35 @@ export const layer = Layer.effect(
675675
);
676676

677677
// A launch is accepted before its workspace is prepared, so a run counts as
678-
// succeeded first. Wait for that run to leave preparation and record a
679-
// failed preparation against this run only: last_run_at is the run's
680-
// identity, so a later run's status is never overwritten.
678+
// succeeded first. The first time the launched run leaves preparation says
679+
// how preparation ended; a later retry or provider failure is not about the
680+
// workspace. A failure is recorded against this run only: last_run_at is
681+
// the run's identity, so a later run's status is never overwritten.
681682
const recordFailedPreparation = Effect.fn("ScheduledTaskService.recordFailedPreparation")(
682683
function* (input: {
683684
readonly id: ScheduledTaskId;
684685
readonly startedAtIso: string;
685686
readonly threadId: ThreadId;
686687
readonly messageId: MessageId;
687688
}) {
688-
const afterSequence = yield* threadManagement.getThreadEventSequence(input.threadId);
689-
const readRun = threadManagement
690-
.getThreadRecords(input.threadId, ["runs"])
689+
// The thread is new, so replaying its run updates from the start is cheap.
690+
const leftPreparation = yield* threadManagement
691+
.streamStoredEventsFrom({
692+
threadId: input.threadId,
693+
afterSequence: 0,
694+
eventType: "run.updated",
695+
})
691696
.pipe(
692-
Effect.map((records) =>
693-
records.runs.find((run) => run.userMessageId === input.messageId),
697+
Stream.map((stored) =>
698+
stored.event.type === "run.updated" &&
699+
stored.event.payload.userMessageId === input.messageId
700+
? stored.event.payload.status
701+
: "preparing",
694702
),
703+
Stream.filter((status) => status !== "preparing"),
704+
Stream.runHead,
695705
);
696-
let run = yield* readRun;
697-
if (run?.status === "preparing") {
698-
const runId = run.id;
699-
yield* threadManagement
700-
.streamStoredEventsFrom({
701-
threadId: input.threadId,
702-
afterSequence,
703-
eventType: "run.updated",
704-
})
705-
.pipe(
706-
Stream.filter(
707-
(stored) =>
708-
stored.event.type === "run.updated" &&
709-
stored.event.payload.id === runId &&
710-
stored.event.payload.status !== "preparing",
711-
),
712-
Stream.runHead,
713-
);
714-
run = yield* readRun;
715-
}
716-
if (run?.status !== "failed") return;
706+
if (Option.getOrUndefined(leftPreparation) !== "failed") return;
717707
yield* sql`
718708
UPDATE scheduled_tasks
719709
SET last_run_status = 'failed',

0 commit comments

Comments
 (0)