@@ -27,8 +27,8 @@ export const HOOK_INBOX_MAX_BYTES = 50 * 1_048_576;
2727export const HOOK_INBOX_MAX_PER_HOOK = 100 ;
2828/** Requests pushed per alarm run; the next run starts right away while more wait. */
2929const DELIVERIES_PER_RUN = 20 ;
30- /** Requests read per run, so requests behind a busy hook's backlog are still reached . */
31- const ROWS_READ_PER_RUN = 200 ;
30+ /** Requests read at a time; a run keeps reading past busy hooks' backlogs . */
31+ const ROWS_READ_PER_PAGE = 50 ;
3232/** Wait before trying a hook whose environment answered busy again. */
3333const BUSY_RETRY_MS = 30_000 ;
3434const MAX_RETRY_DELAY = Duration . minutes ( 10 ) ;
@@ -102,6 +102,7 @@ const encodeHeaders = Schema.encodeSync(HeadersJson);
102102const decodeHeaders = Schema . decodeUnknownOption ( HeadersJson ) ;
103103
104104interface HeldHookRow {
105+ readonly seq : number ;
105106 readonly id : string ;
106107 readonly received_at : string ;
107108 readonly method : string ;
@@ -279,10 +280,16 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* <R>(
279280 RETURNING id
280281 ` ;
281282 const target = yield * readTarget ;
282- const batch = yield * sql < HeldHookRow > `
283- SELECT id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body
284- FROM held_hooks ORDER BY seq LIMIT ${ ROWS_READ_PER_RUN }
285- ` ;
283+ /** Oldest held requests after `afterSeq`, skipping hooks already found busy. */
284+ const readPage = ( afterSeq : number , skip : ReadonlySet < string > ) =>
285+ sql < HeldHookRow > `
286+ SELECT seq, id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body
287+ FROM held_hooks
288+ WHERE seq > ${ afterSeq }
289+ ${ skip . size === 0 ? sql `` : sql `AND hook_key NOT IN ${ sql . in ( [ ...skip ] ) } ` }
290+ ORDER BY seq LIMIT ${ ROWS_READ_PER_PAGE }
291+ ` ;
292+ let batch = yield * readPage ( 0 , new Set ( ) ) ;
286293 if ( batch . length === 0 || target === null ) {
287294 if ( target === null ) yield * sql `DELETE FROM held_hooks` ;
288295 // Empty, so the next request held starts the schedule from the top.
@@ -314,46 +321,51 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* <R>(
314321 "relay.inbox.held_bytes" : backlog . bytes ,
315322 } ) ;
316323 } ) ;
317- for ( const row of batch ) {
318- if ( sent === DELIVERIES_PER_RUN ) break ;
319- // Later requests to a busy hook wait their turn, so its order is kept.
320- if ( busyHooks . has ( row . hook_key ) ) continue ;
321- const headers = decodeHeaders ( row . headers ) ;
322- if ( Option . isNone ( headers ) ) {
323- // Unreadable: it can never be delivered, and must not block the rest.
324+ // Pages run out only once every non-busy request has been tried, so a
325+ // backlog behind busy hooks never hides another hook's requests.
326+ pages: while ( batch . length > 0 ) {
327+ for ( const row of batch ) {
328+ if ( sent === DELIVERIES_PER_RUN ) break pages;
329+ // Later requests to a busy hook wait their turn, so its order is kept.
330+ if ( busyHooks . has ( row . hook_key ) ) continue ;
331+ const headers = decodeHeaders ( row . headers ) ;
332+ if ( Option . isNone ( headers ) ) {
333+ // Unreadable: it can never be delivered, and must not block the rest.
334+ yield * sql `DELETE FROM held_hooks WHERE id = ${ row . id } ` ;
335+ unreadable += 1 ;
336+ continue ;
337+ }
338+ sent += 1 ;
339+ const outcome = yield * send ( target . base_url , {
340+ id : row . id ,
341+ receivedAt : row . received_at ,
342+ method : row . method ,
343+ rawHookId : row . raw_hook_id ,
344+ rawToken : row . raw_token ,
345+ hookKey : row . hook_key ,
346+ query : row . query ,
347+ headers : headers . value ,
348+ body : row . body ,
349+ } ) ;
350+ if ( outcome === "unreachable" ) {
351+ failures = yield * countFailure ;
352+ yield * annotateRun ( "unreachable" ) ;
353+ return ( yield * Clock . currentTimeMillis ) + ( yield * retryDelayMs ( failures ) ) ;
354+ }
355+ if ( failures !== 0 ) {
356+ failures = 0 ;
357+ yield * resetFailures ;
358+ }
359+ if ( outcome === "busy" ) {
360+ busyHooks . add ( row . hook_key ) ;
361+ continue ;
362+ }
324363 yield * sql `DELETE FROM held_hooks WHERE id = ${ row . id } ` ;
325- unreadable += 1 ;
326- continue ;
327- }
328- sent += 1 ;
329- const outcome = yield * send ( target . base_url , {
330- id : row . id ,
331- receivedAt : row . received_at ,
332- method : row . method ,
333- rawHookId : row . raw_hook_id ,
334- rawToken : row . raw_token ,
335- hookKey : row . hook_key ,
336- query : row . query ,
337- headers : headers . value ,
338- body : row . body ,
339- } ) ;
340- if ( outcome === "unreachable" ) {
341- failures = yield * countFailure ;
342- yield * annotateRun ( "unreachable" ) ;
343- return ( yield * Clock . currentTimeMillis ) + ( yield * retryDelayMs ( failures ) ) ;
344- }
345- if ( failures !== 0 ) {
346- failures = 0 ;
347- yield * resetFailures ;
348- }
349- if ( outcome === "busy" ) {
350- busyHooks . add ( row . hook_key ) ;
351- continue ;
364+ delivered += 1 ;
365+ const receivedAtMs = DateTime . toEpochMillis ( DateTime . makeUnsafe ( row . received_at ) ) ;
366+ longestWaitMs = Math . max ( longestWaitMs , startedAt - receivedAtMs ) ;
352367 }
353- yield * sql `DELETE FROM held_hooks WHERE id = ${ row . id } ` ;
354- delivered += 1 ;
355- const receivedAtMs = DateTime . toEpochMillis ( DateTime . makeUnsafe ( row . received_at ) ) ;
356- longestWaitMs = Math . max ( longestWaitMs , startedAt - receivedAtMs ) ;
368+ batch = yield * readPage ( batch . at ( - 1 ) ! . seq , busyHooks ) ;
357369 }
358370 if ( ! ( yield * hasPending ) ) {
359371 yield * resetFailures ;
0 commit comments