Skip to content

Commit 6376b4b

Browse files
t3dotggclaude
andcommitted
fix(server): interrupted scheduled task runs no longer stay busy
runTask reserved the task in activeRuns and dispatchPending reserved it in pendingInFlight before their cleanup was installed. An interrupt in that gap left the task marked busy until restart. Both reservations now use Effect.acquireUseRelease, and the webhook worker owns its own reservation through the permit wait and drain. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 8b74a69 commit 6376b4b

3 files changed

Lines changed: 220 additions & 92 deletions

File tree

‎apps/server/src/scheduledTasks/ScheduledTaskService.test.ts‎

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import * as Scheduler from "../scheduling/Scheduler.ts";
22
import * as NodeUtil from "node:util";
33

44
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
5-
import { assert, it } from "@effect/vitest";
5+
import { assert, it, vi } from "@effect/vitest";
66
import {
77
ProjectId,
88
ProviderInstanceId,
@@ -12,6 +12,7 @@ import {
1212
} from "@t3tools/contracts";
1313
import * as DateTime from "effect/DateTime";
1414
import * as Effect from "effect/Effect";
15+
import * as Exit from "effect/Exit";
1516
import * as Layer from "effect/Layer";
1617
import * as Logger from "effect/Logger";
1718
import * as SqlClient from "effect/sql/SqlClient";
@@ -476,3 +477,70 @@ it.effect("a bound run follows its thread when promotion moves it mid-send", ()
476477
}).pipe(Effect.provide(layer));
477478
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
478479
);
480+
481+
it.effect("a run interrupted right after it reserves the task does not block the next run", () =>
482+
Effect.gen(function* () {
483+
const launches = yield* Ref.make(0);
484+
const layer = ScheduledTaskService.layer.pipe(
485+
Layer.provide(
486+
Layer.mergeAll(
487+
Layer.mock(ThreadLaunchService.ThreadLaunchService)({
488+
launch: () =>
489+
Ref.update(launches, (n) => n + 1).pipe(
490+
Effect.as({ threadId: ThreadId.make("thread:launched"), resumed: false } as never),
491+
),
492+
}),
493+
Layer.mock(ThreadManagementService.ThreadManagementService)({}),
494+
Layer.mock(SecretRequests.SecretRequests)({}),
495+
Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
496+
namedProjectsRoot: "/projects",
497+
isScratchProject: () => Effect.succeed(false),
498+
}),
499+
NodeCrypto.layer,
500+
Layer.succeed(Scheduler.Scheduler, { register: () => Effect.void }),
501+
),
502+
),
503+
);
504+
yield* Effect.gen(function* () {
505+
const service = yield* ScheduledTaskService.ScheduledTaskService;
506+
const { task } = yield* service.upsert({
507+
id: ScheduledTaskId.make("task:interrupted-reservation"),
508+
title: "Check in",
509+
prompt: "Check progress",
510+
enabled: true,
511+
schedule: { type: "interval", everyMs: 60_000 },
512+
projectId: ProjectId.make("project:test"),
513+
workspaceStrategy: { type: "root" },
514+
modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" },
515+
runtimeMode: "full-access",
516+
interactionMode: "default",
517+
});
518+
519+
// The reservation is the first Set.add of the task id; interrupt the run right there.
520+
const add = Set.prototype.add;
521+
let interrupted = false;
522+
const first = yield* Effect.gen(function* () {
523+
using _reserve = vi.spyOn(Set.prototype, "add").mockImplementation(function (
524+
this: Set<unknown>,
525+
value: unknown,
526+
) {
527+
const result = add.call(this, value);
528+
if (value === task.id && !interrupted) {
529+
interrupted = true;
530+
Fiber.getCurrent()?.interruptUnsafe();
531+
}
532+
return result;
533+
});
534+
return yield* service
535+
.runNow({ id: task.id })
536+
.pipe(Effect.forkChild, Effect.flatMap(Fiber.await));
537+
});
538+
assert.isTrue(interrupted);
539+
assert.isTrue(Exit.hasInterrupts(first));
540+
541+
const second = yield* service.runNow({ id: task.id });
542+
assert.equal(second.threadId, ThreadId.make("thread:launched"));
543+
assert.equal(yield* Ref.get(launches), 1);
544+
}).pipe(Effect.provide(layer));
545+
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
546+
);

‎apps/server/src/scheduledTasks/ScheduledTaskService.ts‎

Lines changed: 99 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -734,20 +734,7 @@ export const layer = Layer.effect(
734734
readonly dispatched: Ref.Ref<boolean>;
735735
},
736736
) {
737-
const reserved = yield* Ref.modify(activeRuns, (active) => {
738-
if (active.has(task.id)) return [false, active] as const;
739-
const next = new Set(active);
740-
next.add(task.id);
741-
return [true, next] as const;
742-
});
743-
if (!reserved) {
744-
if (trigger !== "scheduled") {
745-
return yield* taskError("Schedule task is already running.", { taskId: task.id });
746-
}
747-
return { task, threadId: null };
748-
}
749-
750-
return yield* Effect.gen(function* () {
737+
const run = Effect.gen(function* () {
751738
const startedAt = yield* localNow;
752739
const startedAtIso = iso(startedAt);
753740

@@ -944,13 +931,30 @@ export const layer = Layer.effect(
944931
return { task: completed, threadId };
945932
}).pipe(
946933
Effect.onError((cause) => releaseStuckRun(task, errorMessage(cause), webhook?.deliveryId)),
947-
Effect.ensuring(
948-
Ref.update(activeRuns, (active) => {
949-
const next = new Set(active);
950-
next.delete(task.id);
951-
return next;
952-
}),
953-
),
934+
);
935+
936+
// Reserve and release as one bracket so an interrupt cannot leave the task marked busy.
937+
return yield* Effect.acquireUseRelease(
938+
Ref.modify(activeRuns, (active) => {
939+
if (active.has(task.id)) return [false, active] as const;
940+
const next = new Set(active);
941+
next.add(task.id);
942+
return [true, next] as const;
943+
}),
944+
(reserved) =>
945+
reserved
946+
? run
947+
: trigger !== "scheduled"
948+
? Effect.fail(taskError("Schedule task is already running.", { taskId: task.id }))
949+
: Effect.succeed({ task, threadId: null }),
950+
(reserved) =>
951+
reserved
952+
? Ref.update(activeRuns, (active) => {
953+
const next = new Set(active);
954+
next.delete(task.id);
955+
return next;
956+
})
957+
: Effect.void,
954958
);
955959
});
956960

@@ -2155,83 +2159,87 @@ export const layer = Layer.effect(
21552159
}
21562160
yield* Effect.forEach(
21572161
tasks,
2158-
({ task_id: taskId }) =>
2159-
Effect.gen(function* () {
2160-
const reserved = yield* Ref.modify(pendingInFlight, (active) =>
2161-
active.has(taskId)
2162-
? ([false, active] as const)
2163-
: ([true, new Set(active).add(taskId)] as const),
2164-
);
2165-
if (!reserved) return;
2166-
const permit = yield* webhookPermit(taskId);
2167-
// One worker per task waits on its permit, then drains persisted rows in order.
2168-
// Other tasks remain independent and ticks never reload an active task's payloads.
2169-
const dispatch = Effect.gen(function* () {
2170-
while (true) {
2171-
const [entry] = yield* sql<{
2172-
delivery_id: string;
2173-
task_json: string;
2174-
request_json: string;
2175-
state: string;
2176-
}>`
2162+
({ task_id: taskId }) => {
2163+
// One worker per task waits on its permit, then drains persisted rows in order.
2164+
// Other tasks remain independent and ticks never reload an active task's payloads.
2165+
const dispatch = Effect.gen(function* () {
2166+
while (true) {
2167+
const [entry] = yield* sql<{
2168+
delivery_id: string;
2169+
task_json: string;
2170+
request_json: string;
2171+
state: string;
2172+
}>`
21772173
SELECT delivery_id, task_json, request_json, state FROM scheduled_task_webhook_pending
21782174
WHERE task_id = ${taskId} AND state IN ('dispatching', 'running', 'completed', 'failed') ORDER BY due_at, rowid LIMIT 1
21792175
`.pipe(NodeSqliteClient.readOnly);
2180-
if (entry === undefined) return;
2181-
const remove = sql`DELETE FROM scheduled_task_webhook_pending WHERE delivery_id = ${entry.delivery_id}`;
2182-
if (entry.state === "completed" || entry.state === "failed") {
2183-
if (entry.state === "failed") {
2184-
yield* markDeliveryFailed(entry.delivery_id, "The run failed to start.");
2185-
yield* releaseFailedDelivery(entry.delivery_id);
2186-
}
2187-
yield* remove;
2188-
continue;
2189-
}
2190-
const prepared = yield* Effect.gen(function* () {
2191-
const task = yield* decodePendingTask(entry.task_json);
2192-
const request = yield* decodePendingRequest(entry.request_json);
2193-
const frameId = Buffer.from(yield* crypto.randomBytes(6)).toString("hex");
2194-
return { task, rendered: renderWebhookPrompt(task.prompt, request, frameId) };
2195-
}).pipe(Effect.exit);
2196-
if (Exit.isFailure(prepared)) {
2197-
if (Cause.hasInterruptsOnly(prepared.cause))
2198-
return yield* Effect.failCause(prepared.cause);
2199-
yield* markDeliveryFailed(
2200-
entry.delivery_id,
2201-
"The pending webhook payload could not be prepared.",
2202-
);
2203-
yield* releaseFailedDelivery(entry.delivery_id);
2204-
yield* remove;
2205-
continue;
2206-
}
2207-
const { task, rendered } = prepared.value;
2208-
if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) {
2209-
yield* markDeliveryFailed(entry.delivery_id, PROMPT_TOO_LONG);
2176+
if (entry === undefined) return;
2177+
const remove = sql`DELETE FROM scheduled_task_webhook_pending WHERE delivery_id = ${entry.delivery_id}`;
2178+
if (entry.state === "completed" || entry.state === "failed") {
2179+
if (entry.state === "failed") {
2180+
yield* markDeliveryFailed(entry.delivery_id, "The run failed to start.");
22102181
yield* releaseFailedDelivery(entry.delivery_id);
2211-
} else {
2212-
yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL, rendered_prompt = ${rendered.prompt.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT)}, missing_fields_json = ${encodeMissingFieldsJson(rendered.missing)} WHERE delivery_id = ${entry.delivery_id}`;
2213-
if (!(yield* runWebhookDelivery(task, entry.delivery_id, rendered.prompt)))
2214-
return;
22152182
}
22162183
yield* remove;
2184+
continue;
22172185
}
2218-
});
2219-
yield* dispatch.pipe(
2220-
permit.withPermits(1),
2221-
Effect.catch((cause) =>
2222-
Effect.logWarning("Pending webhook dispatch failed", { cause }),
2223-
),
2224-
Effect.ensuring(
2225-
Ref.update(pendingInFlight, (active) => {
2226-
const next = new Set(active);
2227-
next.delete(taskId);
2228-
return next;
2229-
}),
2230-
),
2231-
Effect.withSpan("ScheduledTaskService.dispatchPendingWebhook", { root: true }),
2232-
Effect.forkIn(serviceScope),
2233-
);
2234-
}),
2186+
const prepared = yield* Effect.gen(function* () {
2187+
const task = yield* decodePendingTask(entry.task_json);
2188+
const request = yield* decodePendingRequest(entry.request_json);
2189+
const frameId = Buffer.from(yield* crypto.randomBytes(6)).toString("hex");
2190+
return { task, rendered: renderWebhookPrompt(task.prompt, request, frameId) };
2191+
}).pipe(Effect.exit);
2192+
if (Exit.isFailure(prepared)) {
2193+
if (Cause.hasInterruptsOnly(prepared.cause))
2194+
return yield* Effect.failCause(prepared.cause);
2195+
yield* markDeliveryFailed(
2196+
entry.delivery_id,
2197+
"The pending webhook payload could not be prepared.",
2198+
);
2199+
yield* releaseFailedDelivery(entry.delivery_id);
2200+
yield* remove;
2201+
continue;
2202+
}
2203+
const { task, rendered } = prepared.value;
2204+
if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) {
2205+
yield* markDeliveryFailed(entry.delivery_id, PROMPT_TOO_LONG);
2206+
yield* releaseFailedDelivery(entry.delivery_id);
2207+
} else {
2208+
yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL, rendered_prompt = ${rendered.prompt.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT)}, missing_fields_json = ${encodeMissingFieldsJson(rendered.missing)} WHERE delivery_id = ${entry.delivery_id}`;
2209+
if (!(yield* runWebhookDelivery(task, entry.delivery_id, rendered.prompt))) return;
2210+
}
2211+
yield* remove;
2212+
}
2213+
});
2214+
// The worker owns its reservation, so an interrupt anywhere still releases it.
2215+
return Effect.acquireUseRelease(
2216+
Ref.modify(pendingInFlight, (active) =>
2217+
active.has(taskId)
2218+
? ([false, active] as const)
2219+
: ([true, new Set(active).add(taskId)] as const),
2220+
),
2221+
(reserved) =>
2222+
reserved
2223+
? webhookPermit(taskId).pipe(
2224+
Effect.flatMap((permit) => dispatch.pipe(permit.withPermits(1))),
2225+
Effect.catch((cause) =>
2226+
Effect.logWarning("Pending webhook dispatch failed", { cause }),
2227+
),
2228+
Effect.withSpan("ScheduledTaskService.dispatchPendingWebhook", {
2229+
root: true,
2230+
}),
2231+
)
2232+
: Effect.void,
2233+
(reserved) =>
2234+
reserved
2235+
? Ref.update(pendingInFlight, (active) => {
2236+
const next = new Set(active);
2237+
next.delete(taskId);
2238+
return next;
2239+
})
2240+
: Effect.void,
2241+
).pipe(Effect.forkIn(serviceScope));
2242+
},
22352243
{ discard: true },
22362244
);
22372245
});

‎apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import * as DateTime from "effect/DateTime";
1212
import * as Crypto from "effect/Crypto";
1313
import * as Deferred from "effect/Deferred";
1414
import * as Effect from "effect/Effect";
15+
import * as Exit from "effect/Exit";
1516
import * as Fiber from "effect/Fiber";
1617
import * as Layer from "effect/Layer";
1718
import * as Queue from "effect/Queue";
@@ -1005,6 +1006,57 @@ it.effect("recovers a persisted coalescing window after restarting the service",
10051006
);
10061007
});
10071008

1009+
it.effect("a pending dispatch interrupted at its permit does not block later polls", () => {
1010+
const workers = new Map<string, Effect.Effect<void>>();
1011+
// One shared dispatcher lets the test run the retry's forked worker to completion.
1012+
const mixed = new EffectScheduler.MixedScheduler("sync");
1013+
const dispatcher = mixed.makeDispatcher();
1014+
return withService(
1015+
({ service, sql, launches }) =>
1016+
Effect.gen(function* () {
1017+
const { task } = yield* service.upsert(
1018+
yield* webhookTaskInput({
1019+
schedule: {
1020+
type: "webhook",
1021+
dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 },
1022+
},
1023+
}),
1024+
);
1025+
yield* service.triggerWebhook(requestFor(task));
1026+
yield* TestClock.adjust("10 seconds");
1027+
const poll = workers.get("webhook-coalescing")!;
1028+
1029+
const entered = yield* Deferred.make<Fiber.Fiber<unknown, unknown>>();
1030+
const interruptAtPermit = Tracer.make({
1031+
span: (options) => {
1032+
if (options.name === "ScheduledTaskService.webhookPermit") {
1033+
const fiber = Fiber.getCurrent()!;
1034+
Deferred.doneUnsafe(entered, Effect.succeed(fiber));
1035+
fiber.interruptUnsafe();
1036+
}
1037+
return new Tracer.NativeSpan(options);
1038+
},
1039+
});
1040+
const firstPoll = yield* poll.pipe(Effect.withTracer(interruptAtPermit), Effect.forkChild);
1041+
assert.isTrue(Exit.hasInterrupts(yield* Fiber.await(yield* Deferred.await(entered))));
1042+
yield* Fiber.await(firstPoll);
1043+
1044+
yield* poll;
1045+
dispatcher.flush();
1046+
1047+
assert.equal(yield* Queue.size(launches), 1);
1048+
assert.lengthOf(yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`, 0);
1049+
}),
1050+
{ workers },
1051+
).pipe(
1052+
Effect.provideService(EffectScheduler.Scheduler, {
1053+
executionMode: mixed.executionMode,
1054+
makeDispatcher: () => dispatcher,
1055+
shouldYield: (fiber) => mixed.shouldYield(fiber),
1056+
}),
1057+
);
1058+
});
1059+
10081060
it.effect("rolls back replay and pending state when recording a delivery fails", () => {
10091061
const workers = new Map<string, Effect.Effect<void>>();
10101062
return withService(

0 commit comments

Comments
 (0)