Skip to content

Commit 812789c

Browse files
fix(server): webhook saves and dispatch hold up under concurrency
- A save that sends no secret keeps the live one, so a stale editor cannot bring back a replaced signing secret. - A run aborts when its task was deleted after the re-read. - Cap the deliveries one task may hold at once (20) instead of parking a fiber per request behind a stuck run. - Measure the logged body limit in bytes. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 457e5bf commit 812789c

2 files changed

Lines changed: 122 additions & 7 deletions

File tree

‎apps/server/src/scheduledTasks/ScheduledTaskService.ts‎

Lines changed: 48 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ export const WEBHOOK_ROUTE_PREFIX = "/api/hooks";
4949
const WEBHOOK_DELIVERY_RETENTION = 50;
5050
/** Body text kept in the delivery log. Larger bodies are cut and flagged. */
5151
const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024;
52+
/** Deliveries one task may hold at once, running or waiting their turn. */
53+
const WEBHOOK_MAX_QUEUED_PER_TASK = 20;
5254
/** Accepted deliveries per task per minute, enforced here as well as on the relay because the tunnel hostname is public too. */
5355
const WEBHOOK_RATE_LIMIT_PER_MINUTE = 60;
5456

@@ -346,6 +348,7 @@ export const layer = Layer.effect(
346348
const webhookPermits = yield* Ref.make<ReadonlyMap<ScheduledTaskId, Semaphore.Semaphore>>(
347349
new Map(),
348350
);
351+
const webhookQueued = yield* Ref.make<ReadonlyMap<ScheduledTaskId, number>>(new Map());
349352
const webhookRateWindows = yield* Ref.make<ReadonlyMap<ScheduledTaskId, RateWindow>>(new Map());
350353
const activeRuns = yield* Ref.make<ReadonlySet<ScheduledTaskId>>(new Set());
351354
// Sliding(1) coalesces the dirty-signal: every notification triggers a
@@ -462,7 +465,12 @@ export const layer = Layer.effect(
462465
const saveTask = (
463466
task: ScheduledTask,
464467
requireExisting: boolean,
465-
webhook: { readonly token: string | null; readonly secret: string | null },
468+
webhook: {
469+
readonly token: string | null;
470+
readonly secret: string | null;
471+
/** False when the save carried no new secret, so a concurrent change survives. */
472+
readonly secretChanged: boolean;
473+
},
466474
) =>
467475
sql<{ task_id: string }>`
468476
INSERT INTO scheduled_tasks (
@@ -535,7 +543,10 @@ export const layer = Layer.effect(
535543
WHEN excluded.webhook_token IS NULL THEN NULL
536544
ELSE COALESCE(scheduled_tasks.webhook_token, excluded.webhook_token)
537545
END,
538-
webhook_secret = excluded.webhook_secret
546+
webhook_secret = CASE
547+
WHEN ${webhook.secretChanged ? 1 : 0} = 1 THEN excluded.webhook_secret
548+
ELSE scheduled_tasks.webhook_secret
549+
END
539550
RETURNING task_id
540551
`.pipe(
541552
Effect.mapError((cause) =>
@@ -565,17 +576,22 @@ export const layer = Layer.effect(
565576
// a completing run cannot resurrect a deleted task or clobber concurrent
566577
// edits to the task definition.
567578
const markRunning = (id: ScheduledTaskId, startedAtIso: string) =>
568-
sql`
579+
sql<{ task_id: string }>`
569580
UPDATE scheduled_tasks
570581
SET updated_at = ${startedAtIso},
571582
last_run_at = ${startedAtIso},
572583
last_run_status = 'running',
573584
last_run_error = NULL
574585
WHERE task_id = ${id}
586+
RETURNING task_id
575587
`.pipe(
576588
Effect.mapError((cause) =>
577589
taskError("Could not mark schedule task as running.", { taskId: id, cause }),
578590
),
591+
// A task deleted after the re-read must not be dispatched from the stale snapshot.
592+
Effect.flatMap((rows) =>
593+
rows.length > 0 ? Effect.void : taskError("Schedule task not found.", { taskId: id }),
594+
),
579595
);
580596

581597
const markCompleted = (input: {
@@ -959,14 +975,16 @@ export const layer = Layer.effect(
959975
input.schedule.type === "webhook" ? input.schedule.signature : null;
960976
const secret =
961977
signature == null ? null : (signature.secret ?? existing?.secret ?? null);
978+
const secretChanged =
979+
signature == null || signature.secret !== undefined || existing === null;
962980
if (signature != null && secret === null) {
963981
return yield* taskError("A webhook signature check needs a signing secret.", {
964982
taskId: id,
965983
});
966984
}
967-
return { token, secret };
985+
return { token, secret, secretChanged };
968986
})
969-
: { token: null, secret: null };
987+
: { token: null, secret: null, secretChanged: true };
970988
const scheduleUnchanged =
971989
existingTask !== null &&
972990
existingTask.enabled === input.enabled &&
@@ -1152,7 +1170,10 @@ export const layer = Layer.effect(
11521170
readonly missing: ReadonlyArray<string>;
11531171
readonly renderedPrompt: string | null;
11541172
}) => {
1155-
const truncated = input.request.bodyText.length > WEBHOOK_DELIVERY_LOG_BODY_LIMIT;
1173+
const truncated = input.request.body.byteLength > WEBHOOK_DELIVERY_LOG_BODY_LIMIT;
1174+
const loggedBody = truncated
1175+
? new TextDecoder().decode(input.request.body.subarray(0, WEBHOOK_DELIVERY_LOG_BODY_LIMIT))
1176+
: input.request.bodyText;
11561177
return sql
11571178
.withTransaction(
11581179
Effect.gen(function* () {
@@ -1167,7 +1188,7 @@ export const layer = Layer.effect(
11671188
SELECT
11681189
${input.id}, ${input.taskId}, ${input.receivedAt}, ${input.request.method},
11691190
${input.request.query}, ${encodeHeadersJson(redactHeaders(input.request.headers))},
1170-
${truncated ? input.request.bodyText.slice(0, WEBHOOK_DELIVERY_LOG_BODY_LIMIT) : input.request.bodyText},
1191+
${loggedBody},
11711192
${input.request.body.byteLength}, ${truncated ? 1 : 0}, ${input.outcome},
11721193
${input.signatureVerified ? 1 : 0}, ${encodeMissingFieldsJson(input.missing)},
11731194
${input.renderedPrompt}, NULL
@@ -1325,6 +1346,25 @@ export const layer = Layer.effect(
13251346
missing: rendered.missing,
13261347
renderedPrompt: rendered.prompt,
13271348
});
1349+
// Bound the deliveries waiting behind a slow run, so steady traffic to
1350+
// a stuck task cannot pile up parked fibers.
1351+
const queued = yield* Ref.modify(webhookQueued, (counts) => {
1352+
const count = counts.get(task.id) ?? 0;
1353+
return count >= WEBHOOK_MAX_QUEUED_PER_TASK
1354+
? ([false, counts] as const)
1355+
: ([true, new Map(counts).set(task.id, count + 1)] as const);
1356+
});
1357+
if (!queued) {
1358+
yield* markDeliveryFailed(deliveryId, "Too many deliveries are waiting for this task.");
1359+
return { _tag: "rate_limited" as const };
1360+
}
1361+
const release = Ref.update(webhookQueued, (counts) => {
1362+
const next = new Map(counts);
1363+
const count = (next.get(task.id) ?? 1) - 1;
1364+
if (count <= 0) next.delete(task.id);
1365+
else next.set(task.id, count);
1366+
return next;
1367+
});
13281368
const permit = yield* webhookPermit(task.id);
13291369
yield* runTask(task, "webhook", { deliveryId, prompt: rendered.prompt }).pipe(
13301370
Effect.flatMap((completed) =>
@@ -1340,6 +1380,7 @@ export const layer = Layer.effect(
13401380
),
13411381
),
13421382
permit.withPermits(1),
1383+
Effect.ensuring(release),
13431384
Effect.forkIn(serviceScope),
13441385
);
13451386
return { _tag: "accepted" as const, deliveryId };

‎apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -354,6 +354,80 @@ it.effect("a delivery queued behind a run does not start once the task is paused
354354
}),
355355
);
356356

357+
it.effect("a save without a secret keeps a secret changed after it was read", () =>
358+
withService(({ service }) =>
359+
Effect.gen(function* () {
360+
const signature = { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" };
361+
const { task } = yield* service.upsert(
362+
yield* webhookTaskInput({
363+
schedule: { type: "webhook", signature: { ...signature, secret: "old" } },
364+
}),
365+
);
366+
yield* service.upsert(
367+
yield* webhookTaskInput({
368+
schedule: { type: "webhook", signature: { ...signature, secret: "new" } },
369+
}),
370+
);
371+
// A form opened before the change saves without sending a secret.
372+
yield* service.upsert(
373+
yield* webhookTaskInput({ title: "Edited", schedule: { type: "webhook", signature } }),
374+
);
375+
const sign = (secret: string) =>
376+
`sha256=${NodeCrypto.createHmac("sha256", secret).update(pullRequestBody).digest("hex")}`;
377+
const withSignature = (secret: string) =>
378+
requestFor(task, {
379+
headers: { "content-type": "application/json", "x-hub-signature-256": sign(secret) },
380+
});
381+
assert.equal(
382+
(yield* service.triggerWebhook(withSignature("old")))._tag,
383+
"rejected_signature",
384+
);
385+
assert.equal((yield* service.triggerWebhook(withSignature("new")))._tag, "accepted");
386+
}),
387+
),
388+
);
389+
390+
it.effect("caps deliveries waiting behind a stuck run", () =>
391+
Effect.gen(function* () {
392+
const gate = yield* Deferred.make<void>();
393+
yield* withService(
394+
({ service, launches }) =>
395+
Effect.gen(function* () {
396+
const { task } = yield* service.upsert(yield* webhookTaskInput());
397+
yield* service.triggerWebhook(requestFor(task));
398+
yield* Queue.take(launches);
399+
// The cap counts the running delivery too: 19 more wait, the next is refused.
400+
const waiting = yield* Effect.forEach(Array.from({ length: 20 }), () =>
401+
service.triggerWebhook(requestFor(task)),
402+
);
403+
assert.equal(waiting.filter((result) => result._tag === "accepted").length, 19);
404+
assert.equal(waiting.at(-1)?._tag, "rate_limited");
405+
yield* Deferred.succeed(gate, undefined);
406+
}),
407+
{ gate },
408+
);
409+
}),
410+
);
411+
412+
it.effect("logs a body's first 64 KiB by bytes, not characters", () =>
413+
withService(({ service }) =>
414+
Effect.gen(function* () {
415+
const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
416+
// 30 000 three-byte characters: under 64 Ki characters, over 64 KiB.
417+
const text = "界".repeat(30_000);
418+
const body = new TextEncoder().encode(text);
419+
yield* service.triggerWebhook(requestFor(task, { body, bodyText: text }));
420+
const [summary] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
421+
const { delivery } = yield* service.getWebhookDelivery({
422+
id: task.id,
423+
deliveryId: summary!.id,
424+
});
425+
assert.isTrue(delivery.bodyTruncated);
426+
assert.isAtMost(new TextEncoder().encode(delivery.body).byteLength, 64 * 1024 + 3);
427+
}),
428+
),
429+
);
430+
357431
it.effect("deleting a task removes its delivery log", () =>
358432
withService(({ service }) =>
359433
Effect.gen(function* () {

0 commit comments

Comments
 (0)