Skip to content

Commit 922020b

Browse files
fix(server): a scheduled run the provider fails afterwards shows as failed
A run was recorded as succeeded once its message was accepted. Follow the dispatched run and, when it ends failed, record the provider's error on the task. Fixes #30844 AI-assisted; verified with the tests and tools listed above.
1 parent 6ab3277 commit 922020b

2 files changed

Lines changed: 129 additions & 2 deletions

File tree

‎apps/server/src/scheduledTasks/ScheduledTaskService.test.ts‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import * as Effect from "effect/Effect";
1717
import * as Exit from "effect/Exit";
1818
import * as Layer from "effect/Layer";
1919
import * as Logger from "effect/Logger";
20+
import * as Option from "effect/Option";
2021
import * as SqlClient from "effect/sql/SqlClient";
2122

2223
import * as Fiber from "effect/Fiber";
@@ -418,6 +419,89 @@ it.effect("thread cleanup notifies subscribers and a finishing run cannot recrea
418419
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
419420
);
420421

422+
it.effect("a run the provider fails after accepting its message records the failure", () =>
423+
Effect.gen(function* () {
424+
const projectId = ProjectId.make("project:test");
425+
const ended = yield* Deferred.make<void>();
426+
const sent = yield* Ref.make("");
427+
const run = { id: "run:2", status: "failed", rootNodeId: "node:root" };
428+
const layer = ScheduledTaskService.layer.pipe(
429+
Layer.provide(
430+
Layer.mergeAll(
431+
ManagedPolicy.layerTest(),
432+
Layer.mock(ThreadLaunchService.ThreadLaunchService)({}),
433+
Layer.mock(ThreadManagementService.ThreadManagementService)({
434+
sendToThread: (input) => Ref.set(sent, input.messageId).pipe(Effect.as({} as never)),
435+
getThreadShell: () => Effect.succeed(threadShell(projectId)),
436+
getProjectThreadRecords: () =>
437+
Effect.map(
438+
Ref.get(sent),
439+
(userMessageId) =>
440+
({
441+
runs: [
442+
{ id: "run:1", status: "completed", userMessageId: "message:earlier" },
443+
{ ...run, userMessageId },
444+
],
445+
turnItems: [
446+
{
447+
type: "error",
448+
status: "failed",
449+
runId: "run:2",
450+
nodeId: "node:root",
451+
failure: { message: "Weekly usage limit reached" },
452+
},
453+
],
454+
}) as never,
455+
),
456+
waitForThread: (input) =>
457+
Deferred.await(ended).pipe(
458+
Effect.as({
459+
threadId: input.threadId,
460+
run: { ...run, id: input.runId, userMessageId: "" },
461+
timedOut: false,
462+
} as never),
463+
),
464+
}),
465+
Layer.mock(SecretRequests.SecretRequests)({}),
466+
Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
467+
namedProjectsRoot: "/projects",
468+
isScratchProject: () => Effect.succeed(false),
469+
}),
470+
NodeCrypto.layer,
471+
Scheduler.layer,
472+
),
473+
),
474+
);
475+
yield* Effect.gen(function* () {
476+
const service = yield* ScheduledTaskService.ScheduledTaskService;
477+
const { task } = yield* service.upsert({
478+
id: ScheduledTaskId.make("task:quota"),
479+
title: "Quota bookkeeping",
480+
prompt: "Update the notes",
481+
enabled: true,
482+
schedule: { type: "interval", everyMs: 60_000 },
483+
projectId,
484+
threadId: ThreadId.make("thread:quota"),
485+
workspaceStrategy: { type: "root" },
486+
modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" },
487+
runtimeMode: "full-access",
488+
interactionMode: "default",
489+
});
490+
yield* service.runNow({ id: task.id });
491+
const failed = yield* service.subscribeList().pipe(
492+
Stream.map(({ tasks }) => tasks.find((candidate) => candidate.id === task.id)),
493+
Stream.filter((current) => current?.lastRunStatus === "failed"),
494+
Stream.runHead,
495+
Effect.forkChild,
496+
);
497+
yield* Deferred.succeed(ended, undefined);
498+
const recorded = Option.getOrUndefined(yield* Fiber.join(failed));
499+
assert.equal(recorded?.lastRunError, "Weekly usage limit reached");
500+
assert.equal(recorded?.runCount, 1);
501+
}).pipe(Effect.provide(layer));
502+
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
503+
);
504+
421505
it.effect("a bound run follows its thread when promotion moves it mid-send", () =>
422506
Effect.gen(function* () {
423507
const scratch = ProjectId.make("project:scratch");

‎apps/server/src/scheduledTasks/ScheduledTaskService.ts‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ import * as SecretRequests from "../secrets/SecretRequests.ts";
5757
import * as ManagedPolicy from "../policy/ManagedPolicy.ts";
5858
import * as DueHint from "../scheduling/DueHint.ts";
5959
import * as Scheduler from "../scheduling/Scheduler.ts";
60+
import { latestRootProviderFailure } from "@t3tools/shared/orchestrationV2ThreadError";
6061
import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts";
6162
import {
6263
redactHeaders,
@@ -306,6 +307,7 @@ function iso(value: DateTime.DateTime): string {
306307
return DateTime.formatIso(DateTime.toUtc(value));
307308
}
308309

310+
const RUN_OUTCOME_WAIT_MS = 24 * 60 * 60 * 1_000;
309311
const localNow = DateTime.withCurrentZoneLocal(DateTime.nowInCurrentZone);
310312

311313
function nextRunAt(
@@ -459,6 +461,7 @@ export const layer = Layer.effect(
459461
const mac = yield* Mac.Mac;
460462
const threadLaunch = yield* ThreadLaunchService.ThreadLaunchService;
461463
const threadManagement = yield* ThreadManagementService.ThreadManagementService;
464+
const serviceScope = yield* Effect.scope;
462465
const managedFolders = yield* ManagedProjectFolders.ManagedProjectFolders;
463466
const secretRequests = yield* SecretRequests.SecretRequests;
464467
const scheduler = yield* Scheduler.Scheduler;
@@ -863,6 +866,37 @@ export const layer = Layer.effect(
863866
),
864867
);
865868

869+
// A dispatch is recorded once the message is accepted; the provider can still
870+
// reject the turn after that, so follow the run and record its failure.
871+
const recordRunFailure = Effect.fn("ScheduledTaskService.recordRunFailure")(function* (
872+
taskId: ScheduledTaskId,
873+
startedAtIso: string,
874+
threadId: ThreadId,
875+
messageId: MessageId,
876+
) {
877+
const shell = yield* threadManagement.getThreadShell(threadId);
878+
if (shell === null) return;
879+
const thread = { projectId: shell.projectId, threadId };
880+
const { runs } = yield* threadManagement.getProjectThreadRecords(thread, ["runs"]);
881+
const dispatched = runs.find((run) => run.userMessageId === messageId);
882+
if (dispatched === undefined) return;
883+
const { run } = yield* threadManagement.waitForThread({
884+
...thread,
885+
runId: dispatched.id,
886+
timeoutMs: RUN_OUTCOME_WAIT_MS,
887+
});
888+
if (run?.status !== "failed") return;
889+
const { turnItems } = yield* threadManagement.getProjectThreadRecords(thread, ["turnItems"]);
890+
const error = latestRootProviderFailure(run, turnItems)?.message ?? "The run failed.";
891+
const updated = yield* sql`
892+
UPDATE scheduled_tasks
893+
SET last_run_status = 'failed', last_run_error = ${error}
894+
WHERE task_id = ${taskId} AND last_run_at = ${startedAtIso} AND last_run_status = 'succeeded'
895+
RETURNING task_id
896+
`;
897+
if (updated.length > 0) yield* notifyChanged;
898+
});
899+
866900
const runTask = Effect.fn("ScheduledTaskService.runTask")(function* (
867901
task: ScheduledTask,
868902
trigger: "scheduled" | "manual" | "webhook",
@@ -1109,6 +1143,17 @@ export const layer = Layer.effect(
11091143
),
11101144
);
11111145
if (written) yield* notifyChanged;
1146+
if (written && threadId !== null) {
1147+
yield* recordRunFailure(active.id, startedAtIso, threadId, messageId).pipe(
1148+
Effect.catchCause((cause) =>
1149+
Effect.logWarning("Could not record scheduled run outcome", {
1150+
taskId: active.id,
1151+
cause,
1152+
}),
1153+
),
1154+
Effect.forkIn(serviceScope),
1155+
);
1156+
}
11121157
// The stored row can belong to a replacement, so report this dispatch separately.
11131158
return { task: recorded, threadId, dispatchFailed: !runSucceeded };
11141159
}).pipe(
@@ -1922,8 +1967,6 @@ export const layer = Layer.effect(
19221967
});
19231968
});
19241969

1925-
const serviceScope = yield* Effect.scope;
1926-
19271970
const releaseFailedDelivery = Effect.fn("ScheduledTaskService.releaseFailedDelivery")(
19281971
function* (deliveryId: string) {
19291972
yield* releaseReplay(deliveryId);

0 commit comments

Comments
 (0)