Skip to content

Commit fb412b6

Browse files
only21milclaude
andauthored
fix(server): cap delivery history for oversized coalesced webhooks (#24046)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 8f79233 commit fb412b6

2 files changed

Lines changed: 79 additions & 29 deletions

File tree

‎apps/server/src/scheduledTasks/ScheduledTaskService.ts‎

Lines changed: 45 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1470,6 +1470,35 @@ export const layer = Layer.effect(
14701470
WHERE delivery_id = ${deliveryId}`.pipe(Effect.asVoid, Effect.mapError(dedupeError));
14711471
};
14721472

1473+
// Each suppression bucket retains its own rows, never real run history.
1474+
const deliveryBucket = (outcome: string, error: string | undefined) =>
1475+
outcome === "filtered"
1476+
? "filtered"
1477+
: outcome === "duplicate" ||
1478+
outcome === "coalesced" ||
1479+
outcome === "rate_limited" ||
1480+
error === PROMPT_TOO_LONG
1481+
? "dedupe"
1482+
: "main";
1483+
const pruneDeliveries = (
1484+
taskId: string,
1485+
retentionBucket: ReturnType<typeof deliveryBucket>,
1486+
) => {
1487+
const rowBucket = sql.literal(`CASE WHEN outcome = 'filtered' THEN 'filtered'
1488+
WHEN outcome IN ('duplicate', 'coalesced', 'rate_limited') OR error = 'The filled-in prompt is too long.' THEN 'dedupe'
1489+
ELSE 'main' END`);
1490+
return sql`
1491+
DELETE FROM scheduled_task_webhook_deliveries
1492+
WHERE task_id = ${taskId} AND ${rowBucket} = ${retentionBucket}
1493+
AND delivery_id NOT IN (
1494+
SELECT delivery_id FROM scheduled_task_webhook_deliveries
1495+
WHERE task_id = ${taskId} AND ${rowBucket} = ${retentionBucket}
1496+
ORDER BY received_at DESC, rowid DESC
1497+
LIMIT ${retentionBucket === "main" ? WEBHOOK_DELIVERY_RETENTION : retentionBucket === "filtered" ? WEBHOOK_FILTERED_DELIVERY_RETENTION : WEBHOOK_DEDUPE_DELIVERY_RETENTION}
1498+
)
1499+
`;
1500+
};
1501+
14731502
const recordDelivery = (input: {
14741503
readonly id: string;
14751504
readonly taskId: ScheduledTaskId;
@@ -1511,30 +1540,7 @@ export const layer = Layer.effect(
15111540
rendered_prompt = excluded.rendered_prompt,
15121541
error = excluded.error
15131542
`;
1514-
// Each suppression bucket retains its own rows, never real run history.
1515-
const bucket = (outcome: string, error: string | undefined) =>
1516-
outcome === "filtered"
1517-
? "filtered"
1518-
: outcome === "duplicate" ||
1519-
outcome === "coalesced" ||
1520-
outcome === "rate_limited" ||
1521-
error === PROMPT_TOO_LONG
1522-
? "dedupe"
1523-
: "main";
1524-
const retentionBucket = bucket(input.outcome, input.error);
1525-
const rowBucket = sql.literal(`CASE WHEN outcome = 'filtered' THEN 'filtered'
1526-
WHEN outcome IN ('duplicate', 'coalesced', 'rate_limited') OR error = 'The filled-in prompt is too long.' THEN 'dedupe'
1527-
ELSE 'main' END`);
1528-
yield* sql`
1529-
DELETE FROM scheduled_task_webhook_deliveries
1530-
WHERE task_id = ${input.taskId} AND ${rowBucket} = ${retentionBucket}
1531-
AND delivery_id NOT IN (
1532-
SELECT delivery_id FROM scheduled_task_webhook_deliveries
1533-
WHERE task_id = ${input.taskId} AND ${rowBucket} = ${retentionBucket}
1534-
ORDER BY received_at DESC, rowid DESC
1535-
LIMIT ${retentionBucket === "main" ? WEBHOOK_DELIVERY_RETENTION : retentionBucket === "filtered" ? WEBHOOK_FILTERED_DELIVERY_RETENTION : WEBHOOK_DEDUPE_DELIVERY_RETENTION}
1536-
)
1537-
`;
1543+
yield* pruneDeliveries(input.taskId, deliveryBucket(input.outcome, input.error));
15381544
}),
15391545
)
15401546
.pipe(
@@ -1545,11 +1551,21 @@ export const layer = Layer.effect(
15451551
};
15461552

15471553
const markDeliveryFailed = (deliveryId: string, message: string) =>
1548-
sql`
1549-
UPDATE scheduled_task_webhook_deliveries
1550-
SET outcome = 'dispatch_failed', error = ${message}
1551-
WHERE delivery_id = ${deliveryId}
1552-
`.pipe(Effect.ignore);
1554+
sql
1555+
.withTransaction(
1556+
Effect.gen(function* () {
1557+
const [updated] = yield* sql<{ task_id: string }>`
1558+
UPDATE scheduled_task_webhook_deliveries
1559+
SET outcome = 'dispatch_failed', error = ${message}
1560+
WHERE delivery_id = ${deliveryId}
1561+
RETURNING task_id
1562+
`;
1563+
// An oversized prompt moves the row into the suppression bucket.
1564+
if (updated !== undefined)
1565+
yield* pruneDeliveries(updated.task_id, deliveryBucket("dispatch_failed", message));
1566+
}),
1567+
)
1568+
.pipe(Effect.ignore);
15531569

15541570
/** Sliding one-minute window for deliveries that can start work. */
15551571
const takeRateSlot = (

‎apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1320,6 +1320,40 @@ it.effect.each([119_000, 120_000])(
13201320
},
13211321
);
13221322

1323+
it.effect("oversized coalesced windows stay within the suppression bucket's limit", () => {
1324+
const workers = new Map<string, Effect.Effect<void>>();
1325+
return withService(
1326+
({ service, sql, launches }) =>
1327+
Effect.gen(function* () {
1328+
const { task } = yield* service.upsert(
1329+
yield* webhookTaskInput({
1330+
prompt: `{{body.absent}}${" ".repeat(10_000)}{{body.text}}`,
1331+
schedule: {
1332+
type: "webhook",
1333+
dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 },
1334+
},
1335+
}),
1336+
);
1337+
const bodyText = JSON.stringify({ group: "B", text: "x".repeat(120_000) });
1338+
for (let i = 0; i < 21; i++) {
1339+
const accepted = yield* service.triggerWebhook(
1340+
requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }),
1341+
);
1342+
assert.equal(accepted._tag === "accepted" && accepted.outcome, "accepted");
1343+
yield* TestClock.adjust("10 seconds");
1344+
yield* drainPendingDispatch(workers);
1345+
}
1346+
assert.equal(yield* Queue.size(launches), 0);
1347+
const counts = yield* sql<{
1348+
outcome: string;
1349+
count: number;
1350+
}>`SELECT outcome, COUNT(*) AS count FROM scheduled_task_webhook_deliveries GROUP BY outcome`;
1351+
assert.deepEqual(counts, [{ outcome: "dispatch_failed", count: 20 }]);
1352+
}),
1353+
{ workers },
1354+
);
1355+
});
1356+
13231357
it.effect.each([1, 11])(
13241358
"keeps relay claims for %i coalesced updates after their window closes",
13251359
(count) => {

0 commit comments

Comments
 (0)