@@ -16,16 +16,19 @@ import {
1616 visibleThreadPullRequests ,
1717} from "@t3tools/shared/threadPullRequests" ;
1818import * as Cause from "effect/Cause" ;
19+ import * as Clock from "effect/Clock" ;
1920import * as Context from "effect/Context" ;
2021import * as Crypto from "effect/Crypto" ;
2122import * as DateTime from "effect/DateTime" ;
2223import * as Effect from "effect/Effect" ;
2324import * as Layer from "effect/Layer" ;
25+ import * as Schema from "effect/Schema" ;
2426import * as Schedule from "effect/Schedule" ;
2527import type * as Scope from "effect/Scope" ;
2628import * as Semaphore from "effect/Semaphore" ;
2729import * as Stream from "effect/Stream" ;
2830
31+ import { PullRequestProviderError } from "../pullRequest/PullRequestProvider.ts" ;
2932import * as PullRequestService from "../pullRequest/PullRequestService.ts" ;
3033import { forkParked } from "../serverActivation.ts" ;
3134import * as Orchestrator from "./Orchestrator.ts" ;
@@ -36,6 +39,8 @@ const SLOW_SYNC_INTERVAL_MS = 15 * 60 * 1_000;
3639/** Shell commands that can merge or close a pull request without a merge notification. */
3740const PULL_REQUEST_CLOSE_COMMAND = / \b (?: g h \s + p r | g l a b \s + m r ) \s + (?: m e r g e | c l o s e ) \b / u;
3841
42+ const isPullRequestProviderError = Schema . is ( PullRequestProviderError ) ;
43+
3944type SnapshotFields = Omit < ThreadPullRequestSnapshot , "syncedAt" > ;
4045
4146interface LinkEntry {
@@ -107,6 +112,21 @@ function stacksEqual(
107112 ) ;
108113}
109114
115+ function skipReason ( cause : Cause . Cause < unknown > ) : string {
116+ const error = Cause . squash ( cause ) ;
117+ return error instanceof Error ? error . message : String ( error ) ;
118+ }
119+
120+ /** When a host read failed because the host is rate limited, the time that pause ends. */
121+ function rateLimitRetryAt ( cause : Cause . Cause < unknown > ) : number | undefined {
122+ let error : unknown = Cause . squash ( cause ) ;
123+ while ( error instanceof Error ) {
124+ if ( isPullRequestProviderError ( error ) && error . reason === "rate-limited" ) return error . retryAt ;
125+ error = error . cause ;
126+ }
127+ return undefined ;
128+ }
129+
110130function isUnsettled ( thread : ProjectionStore . ProjectionThreadPullRequests ) : boolean {
111131 return thread . settledOverride !== "settled" && thread . settledAt === null ;
112132}
@@ -122,7 +142,10 @@ export class PullRequestSyncReactor extends Context.Service<
122142 {
123143 readonly start : ( ) => Effect . Effect < void , never , Scope . Scope > ;
124144 readonly drain : Effect . Effect < void > ;
125- /** Force the next sweep to re-read this pull request, even when its snapshot is terminal. */
145+ /**
146+ * Force the next sweep to re-read this pull request, even when its snapshot is terminal.
147+ * While its host is rate limited, the read waits for the first sweep after the pause.
148+ */
126149 readonly requestSync : ( key : ThreadPullRequestKey ) => Effect . Effect < void > ;
127150 }
128151> ( ) ( "t3/orchestration-v2/PullRequestSyncReactor" ) { }
@@ -141,6 +164,10 @@ export const make = Effect.gen(function* () {
141164 // linking dozens of pull requests) is read together and shares the summary batches.
142165 let requestedSweepQueued = false ;
143166 const retryStacks = new Set < string > ( ) ;
167+ // Rate limit pauses by project and host, since each project reads with its own credential.
168+ // A paused host refuses every read without asking it, so the sweep leaves its pull requests
169+ // due until the pause ends rather than failing each of them every minute.
170+ const pausedUntil = new Map < string , number > ( ) ;
144171
145172 const isDue = ( key : string , entries : ReadonlyArray < LinkEntry > , nowMs : number ) : boolean => {
146173 if ( requested . has ( key ) || retryStacks . has ( key ) ) return true ;
@@ -279,10 +306,15 @@ export const make = Effect.gen(function* () {
279306 } ) ) ,
280307 Effect . catchCauseIf (
281308 ( cause ) => ! Cause . hasInterruptsOnly ( cause ) ,
282- ( ) =>
283- Effect . logWarning ( "pull request stack lookup failed" , {
284- key,
285- } ) . pipe ( Effect . as ( null ) ) ,
309+ ( cause ) =>
310+ rateLimitRetryAt ( cause ) !== undefined
311+ ? // The sweep records the pause and holds the host's other reads until it ends.
312+ Effect . sync ( ( ) => retryStacks . add ( key ) ) . pipe (
313+ Effect . andThen ( Effect . failCause ( cause ) ) ,
314+ )
315+ : Effect . logWarning ( "pull request stack lookup failed" , {
316+ key,
317+ } ) . pipe ( Effect . as ( null ) ) ,
286318 ) ,
287319 ) ;
288320 if ( needsStack ) {
@@ -312,18 +344,50 @@ export const make = Effect.gen(function* () {
312344 ) ;
313345 } ) ;
314346
347+ // Failed host reads by reason: how many, and the first key that failed that way.
348+ const skips = new Map < string , { count : number ; readonly key : string } > ( ) ;
349+ const readGroup = ( key : string , entries : ReadonlyArray < LinkEntry > , pauseKey : string ) =>
350+ syncGroup ( key , entries ) . pipe (
351+ Effect . catchCause ( ( cause ) => {
352+ if ( Cause . hasInterruptsOnly ( cause ) ) return Effect . failCause ( cause ) ;
353+ const retryAt = rateLimitRetryAt ( cause ) ;
354+ if ( retryAt !== undefined ) {
355+ pausedUntil . set ( pauseKey , Math . max ( retryAt , pausedUntil . get ( pauseKey ) ?? 0 ) ) ;
356+ }
357+ const reason = skipReason ( cause ) ;
358+ const skip = skips . get ( reason ) ;
359+ if ( skip === undefined ) skips . set ( reason , { count : 1 , key } ) ;
360+ else skip . count += 1 ;
361+ return Effect . void ;
362+ } ) ,
363+ ) ;
315364 yield * Effect . forEach (
316365 groups ,
317- ( [ key , entries ] ) =>
318- ( scope === "all" || requested . has ( key ) ) && isDue ( key , entries , nowMs )
319- ? syncGroup ( key , entries ) . pipe (
320- Effect . catchCause ( logSkipped ( "pull request sync skipped" , { key } ) ) ,
321- )
322- : Effect . void ,
366+ ( [ key , entries ] ) => {
367+ if ( ! ( ( scope === "all" || requested . has ( key ) ) && isDue ( key , entries , nowMs ) ) ) {
368+ return Effect . void ;
369+ }
370+ const first = entries [ 0 ] ! ;
371+ const pauseKey = `${ first . thread . projectId } \0${ normalizeThreadPullRequestKey ( first . link ) . host } ` ;
372+ // Checked against the clock as each read starts, so a pause found earlier in this sweep
373+ // holds the rest, and one that ends during the sweep lets the rest through.
374+ return Clock . currentTimeMillis . pipe (
375+ Effect . flatMap ( ( startedAtMs ) =>
376+ ( pausedUntil . get ( pauseKey ) ?? 0 ) > startedAtMs
377+ ? Effect . void
378+ : readGroup ( key , entries , pauseKey ) ,
379+ ) ,
380+ ) ;
381+ } ,
323382 // As wide as one batched summary read, so the sweep's reads on a host arrive together and
324383 // GitHub answers them in one request rather than one `gh pr view` apiece.
325384 { concurrency : 25 , discard : true } ,
326385 ) ;
386+ // A host failure such as a signed-out CLI fails every due pull request the same way, so a
387+ // sweep reports one line per reason rather than one per pull request.
388+ for ( const [ reason , { count, key } ] of skips ) {
389+ yield * Effect . logWarning ( "pull request sync skipped" , { count, key, reason } ) ;
390+ }
327391 } ) ;
328392
329393 const worker = yield * makeDrainableWorker ( ( scope : "all" | "requested" ) =>
0 commit comments