Skip to content

Commit 21c3da4

Browse files
fix(relay): held webhooks retry every 10 s for 3 minutes after a wake
The environment wakes its inbox when cloudflared registers a tunnel connection, but Cloudflare can take a few minutes to route the hostname to it. The first push landed in that window, and the inbox then backed off 30 s, 1 min, 2 min, so an end-to-end test saw a held webhook arrive 3.5 minutes after the environment was back. Retry delays now come from one Schedule: every 10 s for 18 attempts, then 30 s doubling up to 10 min. A wake resets it to the start. Pushes that do not deliver log why (status or error), so a cold start can be told apart from a timeout. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 07afd5c commit 21c3da4

3 files changed

Lines changed: 84 additions & 24 deletions

File tree

‎infra/relay/src/hooks/HookInboxObject.ts‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,19 +44,32 @@ export class HookInboxObject extends Cloudflare.DurableObject<
4444
const deliver = (baseUrl: string, hook: HookInboxStore.HeldHook) =>
4545
sendUpstream(baseUrl, hook).pipe(
4646
Effect.result,
47-
Effect.map((result): HookInboxStore.DeliveryOutcome => {
47+
Effect.map((result) => {
4848
// Unreachable or timed out: a timeout may still have run it, and the
4949
// environment drops a delivery id it has already seen.
50-
if (Result.isFailure(result) || Option.isNone(result.success)) return "unreachable";
50+
if (Result.isFailure(result)) {
51+
return { outcome: "unreachable" as const, reason: result.failure._tag };
52+
}
53+
if (Option.isNone(result.success))
54+
return { outcome: "unreachable" as const, reason: "timeout" };
5155
const status = result.success.value.status;
52-
if (UNREACHABLE_STATUSES.has(status)) return "unreachable";
53-
return BUSY_STATUSES.has(status) ? "busy" : "delivered";
56+
const outcome: HookInboxStore.DeliveryOutcome = UNREACHABLE_STATUSES.has(status)
57+
? "unreachable"
58+
: BUSY_STATUSES.has(status)
59+
? "busy"
60+
: "delivered";
61+
return { outcome, reason: `status ${status}` };
5462
}),
55-
Effect.tap((outcome) =>
63+
Effect.tap(({ outcome, reason }) =>
5664
outcome === "delivered"
5765
? Effect.void
58-
: Effect.logInfo("Held webhook not delivered yet", { outcome, deliveryId: hook.id }),
66+
: Effect.logInfo("Held webhook not delivered yet", {
67+
outcome,
68+
reason,
69+
deliveryId: hook.id,
70+
}),
5971
),
72+
Effect.map(({ outcome }) => outcome),
6073
Effect.provide(FetchHttpClient.layer),
6174
);
6275

‎infra/relay/src/hooks/HookInboxStore.test.ts‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -68,10 +68,9 @@ describe("HookInboxStore", () => {
6868
const offline = deliverer(() => "unreachable");
6969
const now = yield* Clock.currentTimeMillis;
7070
// Stops at the first failure, so order is kept.
71-
expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 30_000);
71+
expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000);
7272
expect(offline.sent.map((entry) => entry.hook.id)).toEqual(["first"]);
73-
expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 60_000);
74-
expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 120_000);
73+
expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000);
7574

7675
const online = deliverer(() => "delivered");
7776
expect(yield* HookInboxStore.deliverDue(online.send)).toBeNull();
@@ -116,10 +115,15 @@ describe("HookInboxStore", () => {
116115
),
117116
);
118117

119-
it("caps the wait between attempts at 10 minutes", () => {
120-
expect(HookInboxStore.retryDelayMs(1)).toBe(30_000);
121-
expect(HookInboxStore.retryDelayMs(20)).toBe(10 * 60_000);
122-
});
118+
it.effect("retries every 10 s for 3 minutes, then backs off to 10 minutes", () =>
119+
Effect.gen(function* () {
120+
const delays = yield* Effect.forEach(
121+
[1, 18, 19, 20, 21, 23, 24, 40],
122+
HookInboxStore.retryDelayMs,
123+
);
124+
expect(delays).toEqual([10_000, 10_000, 30_000, 60_000, 120_000, 480_000, 600_000, 600_000]);
125+
}),
126+
);
123127

124128
it.effect("waking resets the backoff and reports whether anything waits", () =>
125129
withInbox(
@@ -130,10 +134,18 @@ describe("HookInboxStore", () => {
130134
yield* HookInboxStore.deliverDue(offline.send);
131135
yield* HookInboxStore.deliverDue(offline.send);
132136

133-
expect(yield* HookInboxStore.wake(BASE_URL)).toBe(true);
137+
// Past the fast phase: the environment was away a while.
138+
for (let failure = 0; failure < 20; failure++) {
139+
yield* HookInboxStore.deliverDue(offline.send);
140+
}
134141
const now = yield* Clock.currentTimeMillis;
135-
// Back to the first step, and sent to where the environment is now.
136-
expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 30_000);
142+
expect(yield* HookInboxStore.deliverDue(offline.send)).toBeGreaterThan(now + 60_000);
143+
144+
// A wake means its tunnel just connected, which Cloudflare may not
145+
// route to for a few minutes: back to retrying every 10 s, sent to
146+
// where the environment is now.
147+
expect(yield* HookInboxStore.wake(BASE_URL)).toBe(true);
148+
expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000);
137149
expect(offline.sent.at(-1)?.baseUrl).toBe(BASE_URL);
138150
}),
139151
),

‎infra/relay/src/hooks/HookInboxStore.ts‎

Lines changed: 43 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
import * as Clock from "effect/Clock";
22
import * as DateTime from "effect/DateTime";
3+
import * as Duration from "effect/Duration";
34
import * as Effect from "effect/Effect";
45
import * as Option from "effect/Option";
6+
import * as Pull from "effect/Pull";
7+
import * as Schedule from "effect/Schedule";
58
import * as Schema from "effect/Schema";
69
import * as SqlClient from "effect/unstable/sql/SqlClient";
710

@@ -26,8 +29,28 @@ export const HOOK_INBOX_MAX_PER_HOOK = 100;
2629
const DELIVERIES_PER_RUN = 20;
2730
/** Requests read per run, so requests behind a busy hook's backlog are still reached. */
2831
const ROWS_READ_PER_RUN = 200;
29-
const FIRST_RETRY_MS = 30_000;
30-
const MAX_RETRY_MS = 10 * 60_000;
32+
/** Wait before trying a hook whose environment answered busy again. */
33+
const BUSY_RETRY_MS = 30_000;
34+
const MAX_RETRY_DELAY = Duration.minutes(10);
35+
36+
/**
37+
* Delays between delivery attempts while the environment is unreachable,
38+
* indexed by consecutive failures since it was last reached or woke us.
39+
* A wake means its tunnel just connected, but Cloudflare can take a few
40+
* minutes to route the hostname to it, so the first 3 minutes retry every
41+
* 10 s. After that the environment is likely gone again: 30 s, 1 min, 2 min,
42+
* ... up to 10 min.
43+
*/
44+
export const retrySchedule = Schedule.spaced("10 seconds").pipe(
45+
Schedule.upTo({ times: 18 }),
46+
Schedule.concat(
47+
Schedule.exponential("30 seconds").pipe(
48+
Schedule.modifyDelay(({ duration }) =>
49+
Effect.succeed(Duration.min(Duration.fromInputUnsafe(duration), MAX_RETRY_DELAY)),
50+
),
51+
),
52+
),
53+
);
3154

3255
export interface HeldHook {
3356
readonly id: string;
@@ -52,9 +75,21 @@ export interface HeldHook {
5275
*/
5376
export type DeliveryOutcome = "delivered" | "busy" | "unreachable";
5477

55-
/** 30 s, 1 min, 2 min, ... up to 10 min between attempts while the environment stays away. */
56-
export const retryDelayMs = (failures: number) =>
57-
Math.min(FIRST_RETRY_MS * 2 ** Math.max(0, failures - 1), MAX_RETRY_MS);
78+
/** The `retrySchedule` delay after `failures` consecutive unreachable attempts. */
79+
export const retryDelayMs = Effect.fn("HookInboxStore.retryDelayMs")(function* (failures: number) {
80+
const step = yield* Schedule.toStep(retrySchedule);
81+
let delay = MAX_RETRY_DELAY;
82+
for (let attempt = 0; attempt < Math.max(1, failures); attempt++) {
83+
const next = yield* step(0, undefined).pipe(
84+
Effect.map(([, duration]) => Option.some(duration)),
85+
// The schedule never ends; stay at the cap if it ever does.
86+
Pull.catchDone(() => Effect.succeedNone),
87+
);
88+
if (Option.isNone(next)) break;
89+
delay = next.value;
90+
}
91+
return Duration.toMillis(delay);
92+
});
5893

5994
const HeadersJson = Schema.fromJsonString(Schema.Record(Schema.String, Schema.String));
6095
const encodeHeaders = Schema.encodeSync(HeadersJson);
@@ -161,7 +196,7 @@ export const hold = Effect.fn("HookInboxStore.hold")(function* (hook: HeldHook,
161196
if (inserted.length === 0) return null;
162197
yield* setTarget(baseUrl, { resetFailures: false });
163198
const target = yield* readTarget;
164-
return (yield* Clock.currentTimeMillis) + retryDelayMs(Math.max(1, target?.failures ?? 0));
199+
return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(target?.failures ?? 0));
165200
});
166201

167202
/**
@@ -231,7 +266,7 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* <R>(
231266
if (outcome === "unreachable") {
232267
failures += 1;
233268
yield* setFailures(failures);
234-
return (yield* Clock.currentTimeMillis) + retryDelayMs(failures);
269+
return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(failures));
235270
}
236271
if (failures !== 0) {
237272
failures = 0;
@@ -247,5 +282,5 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* <R>(
247282
if (!(yield* hasPending)) return null;
248283
// Everything left this run was busy: give those tasks a moment to drain.
249284
const now = yield* Clock.currentTimeMillis;
250-
return delivered === 0 && busyHooks.size > 0 ? now + FIRST_RETRY_MS : now;
285+
return delivered === 0 && busyHooks.size > 0 ? now + BUSY_RETRY_MS : now;
251286
});

0 commit comments

Comments
 (0)