Skip to content

Commit b399bb4

Browse files
fix(relay): held requests behind busy hooks' backlogs are reached in the same run
A delivery run read one fixed batch of the oldest held requests. When busy hooks filled that batch, other hooks' requests behind it waited for the busy retry. A run now keeps reading past busy hooks until it has tried every other held request. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 0e33197 commit b399bb4

2 files changed

Lines changed: 77 additions & 44 deletions

File tree

‎infra/relay/src/hooks/HookInboxStore.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,27 @@ describe("HookInboxStore", () => {
101101
),
102102
);
103103

104+
it.effect("reaches a hook queued behind busy hooks' full backlogs in the same run", () =>
105+
withInbox(
106+
Effect.gen(function* () {
107+
for (const name of ["stuck-a", "stuck-b"]) {
108+
const stuck = { rawHookId: name, hookKey: name };
109+
for (let index = 0; index < HookInboxStore.HOOK_INBOX_MAX_PER_HOOK; index++) {
110+
yield* HookInboxStore.hold(yield* hook(`${name}-${index}`, stuck), BASE_URL);
111+
}
112+
}
113+
yield* HookInboxStore.hold(yield* hook("other-1"), BASE_URL);
114+
const busy = deliverer((held) => (held.hookKey.startsWith("stuck") ? "busy" : "delivered"));
115+
yield* HookInboxStore.deliverDue(busy.send);
116+
expect(busy.sent.map((entry) => entry.hook.id)).toEqual([
117+
"stuck-a-0",
118+
"stuck-b-0",
119+
"other-1",
120+
]);
121+
}),
122+
),
123+
);
124+
104125
it.effect("drops a held request it cannot read instead of stalling on it", () =>
105126
withInbox(
106127
Effect.gen(function* () {

‎infra/relay/src/hooks/HookInboxStore.ts‎

Lines changed: 56 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ export const HOOK_INBOX_MAX_BYTES = 50 * 1_048_576;
2727
export const HOOK_INBOX_MAX_PER_HOOK = 100;
2828
/** Requests pushed per alarm run; the next run starts right away while more wait. */
2929
const DELIVERIES_PER_RUN = 20;
30-
/** Requests read per run, so requests behind a busy hook's backlog are still reached. */
31-
const ROWS_READ_PER_RUN = 200;
30+
/** Requests read at a time; a run keeps reading past busy hooks' backlogs. */
31+
const ROWS_READ_PER_PAGE = 50;
3232
/** Wait before trying a hook whose environment answered busy again. */
3333
const BUSY_RETRY_MS = 30_000;
3434
const MAX_RETRY_DELAY = Duration.minutes(10);
@@ -102,6 +102,7 @@ const encodeHeaders = Schema.encodeSync(HeadersJson);
102102
const decodeHeaders = Schema.decodeUnknownOption(HeadersJson);
103103

104104
interface HeldHookRow {
105+
readonly seq: number;
105106
readonly id: string;
106107
readonly received_at: string;
107108
readonly method: string;
@@ -279,10 +280,16 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* <R>(
279280
RETURNING id
280281
`;
281282
const target = yield* readTarget;
282-
const batch = yield* sql<HeldHookRow>`
283-
SELECT id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body
284-
FROM held_hooks ORDER BY seq LIMIT ${ROWS_READ_PER_RUN}
285-
`;
283+
/** Oldest held requests after `afterSeq`, skipping hooks already found busy. */
284+
const readPage = (afterSeq: number, skip: ReadonlySet<string>) =>
285+
sql<HeldHookRow>`
286+
SELECT seq, id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body
287+
FROM held_hooks
288+
WHERE seq > ${afterSeq}
289+
${skip.size === 0 ? sql`` : sql`AND hook_key NOT IN ${sql.in([...skip])}`}
290+
ORDER BY seq LIMIT ${ROWS_READ_PER_PAGE}
291+
`;
292+
let batch = yield* readPage(0, new Set());
286293
if (batch.length === 0 || target === null) {
287294
if (target === null) yield* sql`DELETE FROM held_hooks`;
288295
// Empty, so the next request held starts the schedule from the top.
@@ -314,46 +321,51 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* <R>(
314321
"relay.inbox.held_bytes": backlog.bytes,
315322
});
316323
});
317-
for (const row of batch) {
318-
if (sent === DELIVERIES_PER_RUN) break;
319-
// Later requests to a busy hook wait their turn, so its order is kept.
320-
if (busyHooks.has(row.hook_key)) continue;
321-
const headers = decodeHeaders(row.headers);
322-
if (Option.isNone(headers)) {
323-
// Unreadable: it can never be delivered, and must not block the rest.
324+
// Pages run out only once every non-busy request has been tried, so a
325+
// backlog behind busy hooks never hides another hook's requests.
326+
pages: while (batch.length > 0) {
327+
for (const row of batch) {
328+
if (sent === DELIVERIES_PER_RUN) break pages;
329+
// Later requests to a busy hook wait their turn, so its order is kept.
330+
if (busyHooks.has(row.hook_key)) continue;
331+
const headers = decodeHeaders(row.headers);
332+
if (Option.isNone(headers)) {
333+
// Unreadable: it can never be delivered, and must not block the rest.
334+
yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`;
335+
unreadable += 1;
336+
continue;
337+
}
338+
sent += 1;
339+
const outcome = yield* send(target.base_url, {
340+
id: row.id,
341+
receivedAt: row.received_at,
342+
method: row.method,
343+
rawHookId: row.raw_hook_id,
344+
rawToken: row.raw_token,
345+
hookKey: row.hook_key,
346+
query: row.query,
347+
headers: headers.value,
348+
body: row.body,
349+
});
350+
if (outcome === "unreachable") {
351+
failures = yield* countFailure;
352+
yield* annotateRun("unreachable");
353+
return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(failures));
354+
}
355+
if (failures !== 0) {
356+
failures = 0;
357+
yield* resetFailures;
358+
}
359+
if (outcome === "busy") {
360+
busyHooks.add(row.hook_key);
361+
continue;
362+
}
324363
yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`;
325-
unreadable += 1;
326-
continue;
327-
}
328-
sent += 1;
329-
const outcome = yield* send(target.base_url, {
330-
id: row.id,
331-
receivedAt: row.received_at,
332-
method: row.method,
333-
rawHookId: row.raw_hook_id,
334-
rawToken: row.raw_token,
335-
hookKey: row.hook_key,
336-
query: row.query,
337-
headers: headers.value,
338-
body: row.body,
339-
});
340-
if (outcome === "unreachable") {
341-
failures = yield* countFailure;
342-
yield* annotateRun("unreachable");
343-
return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(failures));
344-
}
345-
if (failures !== 0) {
346-
failures = 0;
347-
yield* resetFailures;
348-
}
349-
if (outcome === "busy") {
350-
busyHooks.add(row.hook_key);
351-
continue;
364+
delivered += 1;
365+
const receivedAtMs = DateTime.toEpochMillis(DateTime.makeUnsafe(row.received_at));
366+
longestWaitMs = Math.max(longestWaitMs, startedAt - receivedAtMs);
352367
}
353-
yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`;
354-
delivered += 1;
355-
const receivedAtMs = DateTime.toEpochMillis(DateTime.makeUnsafe(row.received_at));
356-
longestWaitMs = Math.max(longestWaitMs, startedAt - receivedAtMs);
368+
batch = yield* readPage(batch.at(-1)!.seq, busyHooks);
357369
}
358370
if (!(yield* hasPending)) {
359371
yield* resetFailures;

0 commit comments

Comments
 (0)