Skip to content

Commit c03e5a6

Browse files
fix(scheduled-tasks): bound filtered logging and keep concurrent filter edits
- Filtered requests still take no run slot, but only 10 a minute per task are written to the delivery log; past that they are counted in metrics and answered 202. - Web and mobile forms send filters only when that draft changed them, so a title-only save can't overwrite filters another device changed. - The delivery list leaves out outcomes shipped clients can't decode unless the client sends acceptAnyOutcome, which the web log now does. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 33f064b commit c03e5a6

10 files changed

Lines changed: 166 additions & 26 deletions

File tree

‎apps/mobile/src/features/settings/scheduledTaskDraft.test.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ describe("scheduleDraftForTask", () => {
3737
type: "webhook",
3838
signature: null,
3939
maxDeliveryAgeMinutes: null,
40-
filters: [],
4140
});
4241
});
4342

@@ -63,7 +62,7 @@ describe("scheduleDraftForTask", () => {
6362
const saved = scheduleFromDraft(
6463
scheduleDraftForTask({ schedule: { type: "webhook", signature } }),
6564
);
66-
expect(saved).toEqual({ type: "webhook", signature, maxDeliveryAgeMinutes: null, filters: [] });
65+
expect(saved).toEqual({ type: "webhook", signature, maxDeliveryAgeMinutes: null });
6766
expect(saved?.type === "webhook" && saved.signature && "secret" in saved.signature).toBe(false);
6867
});
6968

@@ -73,7 +72,12 @@ describe("scheduleDraftForTask", () => {
7372
{ path: "body.label.name", op: "in", value: ["needs review", "agent:fix"] },
7473
] as const;
7574
const draft = scheduleDraftForTask({ schedule: { type: "webhook", signature: null, filters } });
76-
expect(scheduleFromDraft(draft)).toMatchObject({ filters });
75+
// Untouched rows are left out, so the server keeps whatever is stored now.
76+
expect(scheduleFromDraft(draft)).not.toHaveProperty("filters");
77+
expect(scheduleFromDraft({ ...draft, filters: [] })).toMatchObject({ filters: [] });
78+
expect(scheduleFromDraft({ ...draft, filters: draft.filters.slice(1) })).toMatchObject({
79+
filters: filters.slice(1),
80+
});
7781
expect(
7882
scheduleFromDraft({ ...draft, filters: [{ path: "", op: "exists", value: "" }] }),
7983
).toBeNull();

‎apps/mobile/src/features/settings/scheduledTaskDraft.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
parseMaxDeliveryAge,
1414
parseWebhookFilterDrafts,
1515
webhookFilterDrafts,
16+
webhookFilterDraftsChanged,
1617
type WebhookFilterDraft,
1718
} from "@t3tools/client-runtime/scheduled-task-webhook";
1819
import {
@@ -56,6 +57,8 @@ export type ScheduleDraft = {
5657
readonly signature: ScheduledTaskWebhookSignature | null;
5758
readonly maxDeliveryAgeMinutes: string;
5859
readonly filters: ReadonlyArray<WebhookFilterDraft>;
60+
/** The filter rows the form opened with. */
61+
readonly savedFilters: ReadonlyArray<WebhookFilterDraft>;
5962
};
6063

6164
export const DEFAULT_SCHEDULE: ScheduleDraft = {
@@ -66,6 +69,7 @@ export const DEFAULT_SCHEDULE: ScheduleDraft = {
6669
signature: null,
6770
maxDeliveryAgeMinutes: "",
6871
filters: [],
72+
savedFilters: [],
6973
};
7074

7175
export function scheduleDraftForTask(task: Pick<ScheduledTask, "schedule">): ScheduleDraft {
@@ -94,6 +98,7 @@ export function scheduleDraftForTask(task: Pick<ScheduledTask, "schedule">): Sch
9498
? ""
9599
: String(task.schedule.maxDeliveryAgeMinutes),
96100
filters: webhookFilterDrafts(task.schedule.filters),
101+
savedFilters: webhookFilterDrafts(task.schedule.filters),
97102
};
98103
}
99104
}
@@ -116,8 +121,10 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche
116121
prefix: draft.signature.prefix,
117122
},
118123
maxDeliveryAgeMinutes,
119-
// Always sent: the server keeps stored filters when the key is missing.
120-
filters: parsedFilters.filters,
124+
// Only edited filters are sent; the server keeps the stored ones otherwise.
125+
...(webhookFilterDraftsChanged(draft.savedFilters, draft.filters)
126+
? { filters: parsedFilters.filters }
127+
: {}),
121128
};
122129
}
123130
if (draft.mode === "interval") {

‎apps/server/src/scheduledTasks/ScheduledTaskService.ts‎

Lines changed: 46 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,20 @@ export const WEBHOOK_ROUTE_PREFIX = "/api/hooks";
6161
const WEBHOOK_DELIVERY_RETENTION = 50;
6262
/** Kept apart from the rest, so a stream of filtered requests can't evict real deliveries. */
6363
const WEBHOOK_FILTERED_DELIVERY_RETENTION = 20;
64+
/** The outcomes clients decoded before outcomes became open-ended. */
65+
const LEGACY_DELIVERY_OUTCOMES: ReadonlySet<string> = new Set([
66+
"accepted",
67+
"dispatch_failed",
68+
"rejected_signature",
69+
"disabled",
70+
"rate_limited",
71+
"expired",
72+
]);
73+
/**
74+
* Filtered requests take no run slot, so this bounds the log writes they
75+
* cause instead; past it a filtered request is only counted in metrics.
76+
*/
77+
const WEBHOOK_FILTERED_LOG_PER_MINUTE = 10;
6478
const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024;
6579
const WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT = 64 * 1024;
6680
const WEBHOOK_MAX_QUEUED_PER_TASK = 20;
@@ -391,6 +405,9 @@ export const layer = Layer.effect(
391405
// finish late release their own count, never a recreated task's.
392406
const webhookQueued = yield* Ref.make<ReadonlyMap<string, number>>(new Map());
393407
const webhookRateWindows = yield* Ref.make<ReadonlyMap<ScheduledTaskId, RateWindow>>(new Map());
408+
const webhookFilteredLogWindows = yield* Ref.make<
409+
ReadonlyMap<ScheduledTaskId, ReadonlyArray<number>>
410+
>(new Map());
394411
const activeRuns = yield* Ref.make<ReadonlySet<ScheduledTaskId>>(new Set());
395412
// Sliding(1) coalesces the dirty-signal: every notification triggers a
396413
// full list() re-emit anyway, so a slow subscriber only ever needs the
@@ -1125,6 +1142,11 @@ export const layer = Layer.effect(
11251142
next.delete(id);
11261143
return next;
11271144
}),
1145+
Ref.update(webhookFilteredLogWindows, (windows) => {
1146+
const next = new Map(windows);
1147+
next.delete(id);
1148+
return next;
1149+
}),
11281150
Ref.update(webhookPermits, (permits) => {
11291151
const next = new Map(permits);
11301152
next.delete(id);
@@ -1236,7 +1258,12 @@ export const layer = Layer.effect(
12361258
WHERE task_id = ${input.id}
12371259
ORDER BY received_at DESC, rowid DESC
12381260
`.pipe(
1239-
Effect.map((rows) => ({ deliveries: rows.map(decodeDeliverySummary) })),
1261+
Effect.map((rows) => ({
1262+
deliveries: (input.acceptAnyOutcome === true
1263+
? rows
1264+
: rows.filter((row) => LEGACY_DELIVERY_OUTCOMES.has(row.outcome))
1265+
).map(decodeDeliverySummary),
1266+
})),
12401267
Effect.mapError((cause) =>
12411268
taskError("Could not list webhook deliveries.", { taskId: input.id, cause }),
12421269
),
@@ -1372,6 +1399,18 @@ export const layer = Layer.effect(
13721399
},
13731400
);
13741401

1402+
/** Sliding one-minute window of filtered requests written to the delivery log. */
1403+
const takeFilteredLogSlot = (id: ScheduledTaskId, nowMs: number) =>
1404+
Ref.modify(
1405+
webhookFilteredLogWindows,
1406+
(windows): readonly [boolean, ReadonlyMap<ScheduledTaskId, ReadonlyArray<number>>] => {
1407+
const recent = (windows.get(id) ?? []).filter((at) => nowMs - at < 60_000);
1408+
return recent.length >= WEBHOOK_FILTERED_LOG_PER_MINUTE
1409+
? [false, windows]
1410+
: [true, new Map(windows).set(id, [...recent, nowMs])];
1411+
},
1412+
);
1413+
13751414
const webhookPermit = Effect.fn("ScheduledTaskService.webhookPermit")(function* (
13761415
id: ScheduledTaskId,
13771416
) {
@@ -1558,10 +1597,12 @@ export const layer = Layer.effect(
15581597
if (task.enabled && signatureVerified && !tooOld) {
15591598
const failedFilter = findFailedWebhookFilter(schedule.filters ?? [], request);
15601599
if (failedFilter !== null) {
1561-
yield* log("filtered", {
1562-
signatureVerified: signature !== null,
1563-
error: describeFailedWebhookFilter(failedFilter),
1564-
});
1600+
if (yield* takeFilteredLogSlot(task.id, DateTime.toEpochMillis(now))) {
1601+
yield* log("filtered", {
1602+
signatureVerified: signature !== null,
1603+
error: describeFailedWebhookFilter(failedFilter),
1604+
});
1605+
}
15651606
yield* observe("filtered");
15661607
return yield* releaseClaim({
15671608
_tag: "accepted" as const,

‎apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts‎

Lines changed: 47 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1019,7 +1019,10 @@ it.effect("checks filters after the signature and max age, and runs only matchin
10191019
assert.deepInclude(matched, { _tag: "accepted", outcome: "accepted" });
10201020
assert.include((yield* Queue.take(launches)).initialMessage?.text ?? "", "pull/45");
10211021

1022-
const deliveries = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
1022+
const deliveries = (yield* service.listWebhookDeliveries({
1023+
id: task.id,
1024+
acceptAnyOutcome: true,
1025+
})).deliveries;
10231026
const filtered = deliveries
10241027
.filter((entry) => entry.outcome === "filtered")
10251028
.map((entry) => [entry.signatureVerified, entry.error]);
@@ -1042,7 +1045,7 @@ const githubEvent = (event: string) => ({
10421045
headers: { "content-type": "application/json", "x-github-event": event },
10431046
});
10441047

1045-
it.effect("filtered deliveries take no rate slot and cannot evict real deliveries", () =>
1048+
it.effect("filtered deliveries take no run slot and log at most 10 rows a minute", () =>
10461049
withService(({ service, launches }) =>
10471050
Effect.gen(function* () {
10481051
const { task } = yield* service.upsert(yield* pullRequestFilterInput());
@@ -1059,11 +1062,25 @@ it.effect("filtered deliveries take no rate slot and cannot evict real deliverie
10591062
service.triggerWebhook(requestFor(task, githubEvent("push"))),
10601063
);
10611064

1062-
const outcomes = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.map(
1063-
(delivery) => delivery.outcome,
1064-
);
1065-
assert.equal(outcomes.filter((outcome) => outcome === "filtered").length, 20);
1065+
const outcomes = (yield* service.listWebhookDeliveries({
1066+
id: task.id,
1067+
acceptAnyOutcome: true,
1068+
})).deliveries.map((delivery) => delivery.outcome);
1069+
// 75 filtered requests in one window: only the first 10 are written.
1070+
assert.equal(outcomes.filter((outcome) => outcome === "filtered").length, 10);
10661071
assert.equal(outcomes.filter((outcome) => outcome === "accepted").length, 1);
1072+
1073+
// Later windows log again, and retention keeps the filtered rows at 20.
1074+
for (const _ of [1, 2]) {
1075+
yield* TestClock.adjust("61 seconds");
1076+
yield* Effect.forEach(Array.from({ length: 15 }), () =>
1077+
service.triggerWebhook(requestFor(task, githubEvent("push"))),
1078+
);
1079+
}
1080+
const later = (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true }))
1081+
.deliveries;
1082+
assert.equal(later.filter((delivery) => delivery.outcome === "filtered").length, 20);
1083+
assert.equal(later.filter((delivery) => delivery.outcome === "accepted").length, 1);
10671084
}),
10681085
),
10691086
);
@@ -1116,3 +1133,27 @@ it.effect("a save without filters keeps the stored ones and an empty list clears
11161133
}),
11171134
),
11181135
);
1136+
1137+
it.effect("leaves filtered rows out of the list for clients that only decode older outcomes", () =>
1138+
withService(({ service, launches }) =>
1139+
Effect.gen(function* () {
1140+
const { task } = yield* service.upsert(yield* pullRequestFilterInput());
1141+
yield* service.triggerWebhook(requestFor(task, githubEvent("push")));
1142+
yield* service.triggerWebhook(requestFor(task, githubEvent("pull_request")));
1143+
yield* Queue.take(launches);
1144+
const legacy = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
1145+
assert.deepEqual(
1146+
legacy.map((delivery) => delivery.outcome),
1147+
["accepted"],
1148+
);
1149+
const current = (yield* service.listWebhookDeliveries({
1150+
id: task.id,
1151+
acceptAnyOutcome: true,
1152+
})).deliveries;
1153+
assert.sameMembers(
1154+
current.map((delivery) => delivery.outcome),
1155+
["accepted", "filtered"],
1156+
);
1157+
}),
1158+
),
1159+
);

‎apps/web/src/components/settings/ScheduledTasksSettings.tsx‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ const EMPTY_DRAFT: DraftState = {
152152
signatureSecret: "",
153153
maxDeliveryAgeMinutes: "",
154154
filters: [],
155+
savedFilters: [],
155156
};
156157

157158
function WebhookFiltersField({
@@ -813,7 +814,10 @@ function WebhookDeliveriesDialog({
813814
readonly onClose: () => void;
814815
}) {
815816
const deliveriesQuery = useEnvironmentQuery(
816-
serverEnvironment.scheduledTaskWebhookDeliveries({ environmentId, input: { id: task.id } }),
817+
serverEnvironment.scheduledTaskWebhookDeliveries({
818+
environmentId,
819+
input: { id: task.id, acceptAnyOutcome: true },
820+
}),
817821
);
818822
const [selectedId, setSelectedId] = useState<ScheduledTaskWebhookDeliverySummary["id"] | null>(
819823
null,

‎apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -183,13 +183,11 @@ describe("webhook scheduled tasks", () => {
183183
type: "webhook",
184184
signature,
185185
maxDeliveryAgeMinutes: null,
186-
filters: [],
187186
});
188187
expect(scheduleFromDraft({ ...draft, signatureSecret: " new " })).toEqual({
189188
type: "webhook",
190189
signature: { ...signature, secret: "new" },
191190
maxDeliveryAgeMinutes: null,
192-
filters: [],
193191
});
194192
});
195193

@@ -201,7 +199,6 @@ describe("webhook scheduled tasks", () => {
201199
type: "webhook",
202200
signature: null,
203201
maxDeliveryAgeMinutes: null,
204-
filters: [],
205202
});
206203
});
207204

@@ -222,7 +219,7 @@ describe("webhook scheduled tasks", () => {
222219
}
223220
});
224221

225-
it("round-trips filters and refuses to build a schedule with an invalid one", () => {
222+
it("sends filters only when the form changed them", () => {
226223
const filters = [
227224
{ path: "headers.x-github-event", op: "equals", value: "pull_request" },
228225
{ path: "body.action", op: "in", value: ["opened", "reopened"] },
@@ -232,8 +229,12 @@ describe("webhook scheduled tasks", () => {
232229
schedule: { type: "webhook", signature, filters },
233230
});
234231
expect(draft.filters.map((filter) => filter.op)).toEqual(["equals", "in"]);
235-
expect(scheduleFromDraft(draft)).toMatchObject({ filters });
236-
// Sent even when empty, so removing every row clears the stored filters.
232+
// A title-only save leaves the key out, so the server keeps whatever is stored now.
233+
expect(scheduleFromDraft({ ...draft, title: "Renamed" })).not.toHaveProperty("filters");
234+
expect(scheduleFromDraft({ ...draft, filters: draft.filters.slice(0, 1) })).toMatchObject({
235+
filters: filters.slice(0, 1),
236+
});
237+
// Removing every row is a change, so it clears the stored filters.
237238
expect(scheduleFromDraft({ ...draft, filters: [] })).toMatchObject({ filters: [] });
238239
expect(
239240
scheduleFromDraft({ ...draft, filters: [{ path: "body.action", op: "in", value: " , " }] }),

‎apps/web/src/components/settings/scheduledTasksSettings.logic.ts‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
parseMaxDeliveryAge,
1414
parseWebhookFilterDrafts,
1515
webhookFilterDrafts,
16+
webhookFilterDraftsChanged,
1617
type WebhookFilterDraft,
1718
} from "@t3tools/client-runtime/scheduled-task-webhook";
1819

@@ -83,6 +84,8 @@ export interface DraftState {
8384
readonly signatureSecret: string;
8485
readonly maxDeliveryAgeMinutes: string;
8586
readonly filters: ReadonlyArray<WebhookFilterDraft>;
87+
/** The filter rows the form opened with. */
88+
readonly savedFilters: ReadonlyArray<WebhookFilterDraft>;
8689
}
8790

8891
export const WEBHOOK_SIGNATURE_DEFAULTS = {
@@ -109,8 +112,10 @@ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedul
109112
}
110113
: null,
111114
maxDeliveryAgeMinutes,
112-
// Always sent: the server keeps stored filters when the key is missing.
113-
filters: parsedFilters.filters,
115+
// Only edited filters are sent; the server keeps the stored ones otherwise.
116+
...(webhookFilterDraftsChanged(draft.savedFilters, draft.filters)
117+
? { filters: parsedFilters.filters }
118+
: {}),
114119
};
115120
}
116121
if (draft.scheduleMode === "interval") {
@@ -127,6 +132,7 @@ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedul
127132

128133
export function taskToDraft(task: ScheduledTask): DraftState {
129134
const schedule = task.schedule;
135+
const filters = webhookFilterDrafts(schedule.type === "webhook" ? schedule.filters : undefined);
130136
const weekdays =
131137
schedule.type === "fixed_time" && schedule.weekdays && schedule.weekdays.length > 0
132138
? new Set(schedule.weekdays)
@@ -171,7 +177,8 @@ export function taskToDraft(task: ScheduledTask): DraftState {
171177
schedule.type === "webhook" && schedule.maxDeliveryAgeMinutes != null
172178
? String(schedule.maxDeliveryAgeMinutes)
173179
: "",
174-
filters: webhookFilterDrafts(schedule.type === "webhook" ? schedule.filters : undefined),
180+
filters,
181+
savedFilters: filters,
175182
};
176183
}
177184

‎packages/client-runtime/src/scheduledTaskWebhook.test.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import {
44
parseMaxDeliveryAge,
55
parseWebhookFilterDrafts,
66
webhookFilterDrafts,
7+
webhookFilterDraftsChanged,
78
} from "./scheduledTaskWebhook.ts";
89

910
describe("parseMaxDeliveryAge", () => {
@@ -49,4 +50,12 @@ describe("webhook filter drafts", () => {
4950
}));
5051
expect(parseWebhookFilterDrafts(seventeen)).toEqual({ error: "Use at most 16 filters." });
5152
});
53+
54+
it("notices added, removed and edited rows but not an untouched list", () => {
55+
const saved = webhookFilterDrafts([{ path: "body.action", op: "equals", value: "opened" }]);
56+
expect(webhookFilterDraftsChanged(saved, [...saved])).toBe(false);
57+
expect(webhookFilterDraftsChanged(saved, [])).toBe(true);
58+
expect(webhookFilterDraftsChanged(saved, [{ ...saved[0]!, value: "closed" }])).toBe(true);
59+
expect(webhookFilterDraftsChanged([], [])).toBe(false);
60+
});
5261
});

0 commit comments

Comments
 (0)