Skip to content

Commit ce9e016

Browse files
t3dotggclaude
andauthored
fix(server): interrupted scheduled task runs no longer stay busy (#28474)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 6328c84 commit ce9e016

3 files changed

Lines changed: 220 additions & 92 deletions

File tree

‎apps/server/src/scheduledTasks/ScheduledTaskService.test.ts‎

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import * as Scheduler from "../scheduling/Scheduler.ts";
22
import * as NodeUtil from "node:util";
33

44
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
5-
import { assert, it } from "@effect/vitest";
5+
import { assert, it, vi } from "@effect/vitest";
66
import {
77
ProjectId,
88
ProviderInstanceId,
@@ -12,6 +12,7 @@ import {
1212
} from "@t3tools/contracts";
1313
import * as DateTime from "effect/DateTime";
1414
import * as Effect from "effect/Effect";
15+
import * as Exit from "effect/Exit";
1516
import * as Layer from "effect/Layer";
1617
import * as Logger from "effect/Logger";
1718
import * as SqlClient from "effect/sql/SqlClient";
@@ -476,3 +477,70 @@ it.effect("a bound run follows its thread when promotion moves it mid-send", ()
476477
}).pipe(Effect.provide(layer));
477478
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
478479
);
480+
481+
it.effect("a run interrupted right after it reserves the task does not block the next run", () =>
482+
Effect.gen(function* () {
483+
const launches = yield* Ref.make(0);
484+
const layer = ScheduledTaskService.layer.pipe(
485+
Layer.provide(
486+
Layer.mergeAll(
487+
Layer.mock(ThreadLaunchService.ThreadLaunchService)({
488+
launch: () =>
489+
Ref.update(launches, (n) => n + 1).pipe(
490+
Effect.as({ threadId: ThreadId.make("thread:launched"), resumed: false } as never),
491+
),
492+
}),
493+
Layer.mock(ThreadManagementService.ThreadManagementService)({}),
494+
Layer.mock(SecretRequests.SecretRequests)({}),
495+
Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
496+
namedProjectsRoot: "/projects",
497+
isScratchProject: () => Effect.succeed(false),
498+
}),
499+
NodeCrypto.layer,
500+
Layer.succeed(Scheduler.Scheduler, { register: () => Effect.void }),
501+
),
502+
),
503+
);
504+
yield* Effect.gen(function* () {
505+
const service = yield* ScheduledTaskService.ScheduledTaskService;
506+
const { task } = yield* service.upsert({
507+
id: ScheduledTaskId.make("task:interrupted-reservation"),
508+
title: "Check in",
509+
prompt: "Check progress",
510+
enabled: true,
511+
schedule: { type: "interval", everyMs: 60_000 },
512+
projectId: ProjectId.make("project:test"),
513+
workspaceStrategy: { type: "root" },
514+
modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" },
515+
runtimeMode: "full-access",
516+
interactionMode: "default",
517+
});
518+
519+
// The reservation is the first Set.add of the task id; interrupt the run right there.
520+
const add = Set.prototype.add;
521+
let interrupted = false;
522+
const first = yield* Effect.gen(function* () {
523+
using _reserve = vi.spyOn(Set.prototype, "add").mockImplementation(function (
524+
this: Set<unknown>,
525+
value: unknown,
526+
) {
527+
const result = add.call(this, value);
528+
if (value === task.id && !interrupted) {
529+
interrupted = true;
530+
Fiber.getCurrent()?.interruptUnsafe();
531+
}
532+
return result;
533+
});
534+
return yield* service
535+
.runNow({ id: task.id })
536+
.pipe(Effect.forkChild, Effect.flatMap(Fiber.await));
537+
});
538+
assert.isTrue(interrupted);
539+
assert.isTrue(Exit.hasInterrupts(first));
540+
541+
const second = yield* service.runNow({ id: task.id });
542+
assert.equal(second.threadId, ThreadId.make("thread:launched"));
543+
assert.equal(yield* Ref.get(launches), 1);
544+
}).pipe(Effect.provide(layer));
545+
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
546+
);

‎apps/server/src/scheduledTasks/ScheduledTaskService.ts‎

Lines changed: 99 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -786,20 +786,7 @@ export const layer = Layer.effect(
786786
// Read before reserving: nothing between the reservation and its release may fail.
787787
const startedAt = yield* localNow;
788788
const startedAtIso = iso(startedAt);
789-
const reserved = yield* Ref.modify(activeRuns, (active) => {
790-
if (active.has(task.id)) return [false, active] as const;
791-
const next = new Set(active);
792-
next.add(task.id);
793-
return [true, next] as const;
794-
});
795-
if (!reserved) {
796-
if (trigger !== "scheduled") {
797-
return yield* taskError("Schedule task is already running.", { taskId: task.id });
798-
}
799-
return { task, threadId: null, dispatchFailed: false };
800-
}
801-
802-
return yield* Effect.gen(function* () {
789+
const run = Effect.gen(function* () {
803790
// Re-read stale poll snapshots so deleted, paused or postponed tasks cannot fire.
804791
const active = yield* findTask(task.id);
805792
if (active === null) {
@@ -1018,13 +1005,30 @@ export const layer = Layer.effect(
10181005
Effect.onError((cause) =>
10191006
releaseStuckRun(task, startedAtIso, errorMessage(cause), webhook?.deliveryId),
10201007
),
1021-
Effect.ensuring(
1022-
Ref.update(activeRuns, (active) => {
1023-
const next = new Set(active);
1024-
next.delete(task.id);
1025-
return next;
1026-
}),
1027-
),
1008+
);
1009+
1010+
// Reserve and release as one bracket so an interrupt cannot leave the task marked busy.
1011+
return yield* Effect.acquireUseRelease(
1012+
Ref.modify(activeRuns, (active) => {
1013+
if (active.has(task.id)) return [false, active] as const;
1014+
const next = new Set(active);
1015+
next.add(task.id);
1016+
return [true, next] as const;
1017+
}),
1018+
(reserved) =>
1019+
reserved
1020+
? run
1021+
: trigger !== "scheduled"
1022+
? Effect.fail(taskError("Schedule task is already running.", { taskId: task.id }))
1023+
: Effect.succeed({ task, threadId: null, dispatchFailed: false }),
1024+
(reserved) =>
1025+
reserved
1026+
? Ref.update(activeRuns, (active) => {
1027+
const next = new Set(active);
1028+
next.delete(task.id);
1029+
return next;
1030+
})
1031+
: Effect.void,
10281032
);
10291033
});
10301034

@@ -2243,83 +2247,87 @@ export const layer = Layer.effect(
22432247
}
22442248
yield* Effect.forEach(
22452249
tasks,
2246-
({ task_id: taskId }) =>
2247-
Effect.gen(function* () {
2248-
const reserved = yield* Ref.modify(pendingInFlight, (active) =>
2249-
active.has(taskId)
2250-
? ([false, active] as const)
2251-
: ([true, new Set(active).add(taskId)] as const),
2252-
);
2253-
if (!reserved) return;
2254-
const permit = yield* webhookPermit(taskId);
2255-
// One worker per task waits on its permit, then drains persisted rows in order.
2256-
// Other tasks remain independent and ticks never reload an active task's payloads.
2257-
const dispatch = Effect.gen(function* () {
2258-
while (true) {
2259-
const [entry] = yield* sql<{
2260-
delivery_id: string;
2261-
task_json: string;
2262-
request_json: string;
2263-
state: string;
2264-
}>`
2250+
({ task_id: taskId }) => {
2251+
// One worker per task waits on its permit, then drains persisted rows in order.
2252+
// Other tasks remain independent and ticks never reload an active task's payloads.
2253+
const dispatch = Effect.gen(function* () {
2254+
while (true) {
2255+
const [entry] = yield* sql<{
2256+
delivery_id: string;
2257+
task_json: string;
2258+
request_json: string;
2259+
state: string;
2260+
}>`
22652261
SELECT delivery_id, task_json, request_json, state FROM scheduled_task_webhook_pending
22662262
WHERE task_id = ${taskId} AND state IN ('dispatching', 'running', 'completed', 'failed') ORDER BY due_at, rowid LIMIT 1
22672263
`.pipe(NodeSqliteClient.readOnly);
2268-
if (entry === undefined) return;
2269-
const remove = sql`DELETE FROM scheduled_task_webhook_pending WHERE delivery_id = ${entry.delivery_id}`;
2270-
if (entry.state === "completed" || entry.state === "failed") {
2271-
if (entry.state === "failed") {
2272-
yield* markDeliveryFailed(entry.delivery_id, "The run failed to start.");
2273-
yield* releaseFailedDelivery(entry.delivery_id);
2274-
}
2275-
yield* remove;
2276-
continue;
2277-
}
2278-
const prepared = yield* Effect.gen(function* () {
2279-
const task = yield* decodePendingTask(entry.task_json);
2280-
const request = yield* decodePendingRequest(entry.request_json);
2281-
const frameId = Buffer.from(yield* crypto.randomBytes(6)).toString("hex");
2282-
return { task, rendered: renderWebhookPrompt(task.prompt, request, frameId) };
2283-
}).pipe(Effect.exit);
2284-
if (Exit.isFailure(prepared)) {
2285-
if (Cause.hasInterruptsOnly(prepared.cause))
2286-
return yield* Effect.failCause(prepared.cause);
2287-
yield* markDeliveryFailed(
2288-
entry.delivery_id,
2289-
"The pending webhook payload could not be prepared.",
2290-
);
2291-
yield* releaseFailedDelivery(entry.delivery_id);
2292-
yield* remove;
2293-
continue;
2294-
}
2295-
const { task, rendered } = prepared.value;
2296-
if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) {
2297-
yield* markDeliveryFailed(entry.delivery_id, PROMPT_TOO_LONG);
2264+
if (entry === undefined) return;
2265+
const remove = sql`DELETE FROM scheduled_task_webhook_pending WHERE delivery_id = ${entry.delivery_id}`;
2266+
if (entry.state === "completed" || entry.state === "failed") {
2267+
if (entry.state === "failed") {
2268+
yield* markDeliveryFailed(entry.delivery_id, "The run failed to start.");
22982269
yield* releaseFailedDelivery(entry.delivery_id);
2299-
} else {
2300-
yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL, rendered_prompt = ${rendered.prompt.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT)}, missing_fields_json = ${encodeMissingFieldsJson(rendered.missing)} WHERE delivery_id = ${entry.delivery_id}`;
2301-
if (!(yield* runWebhookDelivery(task, entry.delivery_id, rendered.prompt)))
2302-
return;
23032270
}
23042271
yield* remove;
2272+
continue;
23052273
}
2306-
});
2307-
yield* dispatch.pipe(
2308-
permit.withPermits(1),
2309-
Effect.catch((cause) =>
2310-
Effect.logWarning("Pending webhook dispatch failed", { cause }),
2311-
),
2312-
Effect.ensuring(
2313-
Ref.update(pendingInFlight, (active) => {
2314-
const next = new Set(active);
2315-
next.delete(taskId);
2316-
return next;
2317-
}),
2318-
),
2319-
Effect.withSpan("ScheduledTaskService.dispatchPendingWebhook", { root: true }),
2320-
Effect.forkIn(serviceScope),
2321-
);
2322-
}),
2274+
const prepared = yield* Effect.gen(function* () {
2275+
const task = yield* decodePendingTask(entry.task_json);
2276+
const request = yield* decodePendingRequest(entry.request_json);
2277+
const frameId = Buffer.from(yield* crypto.randomBytes(6)).toString("hex");
2278+
return { task, rendered: renderWebhookPrompt(task.prompt, request, frameId) };
2279+
}).pipe(Effect.exit);
2280+
if (Exit.isFailure(prepared)) {
2281+
if (Cause.hasInterruptsOnly(prepared.cause))
2282+
return yield* Effect.failCause(prepared.cause);
2283+
yield* markDeliveryFailed(
2284+
entry.delivery_id,
2285+
"The pending webhook payload could not be prepared.",
2286+
);
2287+
yield* releaseFailedDelivery(entry.delivery_id);
2288+
yield* remove;
2289+
continue;
2290+
}
2291+
const { task, rendered } = prepared.value;
2292+
if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) {
2293+
yield* markDeliveryFailed(entry.delivery_id, PROMPT_TOO_LONG);
2294+
yield* releaseFailedDelivery(entry.delivery_id);
2295+
} else {
2296+
yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL, rendered_prompt = ${rendered.prompt.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT)}, missing_fields_json = ${encodeMissingFieldsJson(rendered.missing)} WHERE delivery_id = ${entry.delivery_id}`;
2297+
if (!(yield* runWebhookDelivery(task, entry.delivery_id, rendered.prompt))) return;
2298+
}
2299+
yield* remove;
2300+
}
2301+
});
2302+
// The worker owns its reservation, so an interrupt anywhere still releases it.
2303+
return Effect.acquireUseRelease(
2304+
Ref.modify(pendingInFlight, (active) =>
2305+
active.has(taskId)
2306+
? ([false, active] as const)
2307+
: ([true, new Set(active).add(taskId)] as const),
2308+
),
2309+
(reserved) =>
2310+
reserved
2311+
? webhookPermit(taskId).pipe(
2312+
Effect.flatMap((permit) => dispatch.pipe(permit.withPermits(1))),
2313+
Effect.catch((cause) =>
2314+
Effect.logWarning("Pending webhook dispatch failed", { cause }),
2315+
),
2316+
Effect.withSpan("ScheduledTaskService.dispatchPendingWebhook", {
2317+
root: true,
2318+
}),
2319+
)
2320+
: Effect.void,
2321+
(reserved) =>
2322+
reserved
2323+
? Ref.update(pendingInFlight, (active) => {
2324+
const next = new Set(active);
2325+
next.delete(taskId);
2326+
return next;
2327+
})
2328+
: Effect.void,
2329+
).pipe(Effect.forkIn(serviceScope));
2330+
},
23232331
{ discard: true },
23242332
);
23252333
});

‎apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import * as DateTime from "effect/DateTime";
1212
import * as Crypto from "effect/Crypto";
1313
import * as Deferred from "effect/Deferred";
1414
import * as Effect from "effect/Effect";
15+
import * as Exit from "effect/Exit";
1516
import * as Fiber from "effect/Fiber";
1617
import * as Layer from "effect/Layer";
1718
import * as Queue from "effect/Queue";
@@ -1005,6 +1006,57 @@ it.effect("recovers a persisted coalescing window after restarting the service",
10051006
);
10061007
});
10071008

1009+
it.effect("a pending dispatch interrupted at its permit does not block later polls", () => {
1010+
const workers = new Map<string, Effect.Effect<void>>();
1011+
// One shared dispatcher lets the test run the retry's forked worker to completion.
1012+
const mixed = new EffectScheduler.MixedScheduler("sync");
1013+
const dispatcher = mixed.makeDispatcher();
1014+
return withService(
1015+
({ service, sql, launches }) =>
1016+
Effect.gen(function* () {
1017+
const { task } = yield* service.upsert(
1018+
yield* webhookTaskInput({
1019+
schedule: {
1020+
type: "webhook",
1021+
dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 },
1022+
},
1023+
}),
1024+
);
1025+
yield* service.triggerWebhook(requestFor(task));
1026+
yield* TestClock.adjust("10 seconds");
1027+
const poll = workers.get("webhook-coalescing")!;
1028+
1029+
const entered = yield* Deferred.make<Fiber.Fiber<unknown, unknown>>();
1030+
const interruptAtPermit = Tracer.make({
1031+
span: (options) => {
1032+
if (options.name === "ScheduledTaskService.webhookPermit") {
1033+
const fiber = Fiber.getCurrent()!;
1034+
Deferred.doneUnsafe(entered, Effect.succeed(fiber));
1035+
fiber.interruptUnsafe();
1036+
}
1037+
return new Tracer.NativeSpan(options);
1038+
},
1039+
});
1040+
const firstPoll = yield* poll.pipe(Effect.withTracer(interruptAtPermit), Effect.forkChild);
1041+
assert.isTrue(Exit.hasInterrupts(yield* Fiber.await(yield* Deferred.await(entered))));
1042+
yield* Fiber.await(firstPoll);
1043+
1044+
yield* poll;
1045+
dispatcher.flush();
1046+
1047+
assert.equal(yield* Queue.size(launches), 1);
1048+
assert.lengthOf(yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`, 0);
1049+
}),
1050+
{ workers },
1051+
).pipe(
1052+
Effect.provideService(EffectScheduler.Scheduler, {
1053+
executionMode: mixed.executionMode,
1054+
makeDispatcher: () => dispatcher,
1055+
shouldYield: (fiber) => mixed.shouldYield(fiber),
1056+
}),
1057+
);
1058+
});
1059+
10081060
it.effect("rolls back replay and pending state when recording a delivery fails", () => {
10091061
const workers = new Map<string, Effect.Effect<void>>();
10101062
return withService(

0 commit comments

Comments
 (0)