@@ -123,6 +123,7 @@ function supersedesReading(current: PullRequestReading | undefined, next: PullRe
123123const DEFAULT_REPOSITORY_LIST_LIMIT = 99 ;
124124const REPOSITORY_CONCURRENCY = 12 ;
125125const REPOSITORY_SEARCH_CHUNK = 100 ;
126+ const FILTERED_LIST_READS = 5 ;
126127
127128/**
128129 * Every read leaves the process — a CLI per repository, against hosts whose limits are low
@@ -341,7 +342,7 @@ interface ListCursor extends ProviderListCursor {
341342}
342343
343344const LIST_CURSOR_PATTERN =
344- / ^ ( \d { 4 } - \d { 2 } - \d { 2 } T \d { 2 } : \d { 2 } : \d { 2 } (?: \. \d { 1 , 9 } ) ? (?: Z | [ + - ] \d { 2 } : \d { 2 } ) ) \| ( \d { 1 , 9 } ) \| ( \d { 1 , 9 } (?: , \d { 1 , 9 } ) * ) ? $ / ;
345+ / ^ ( \d { 4 } - \d { 2 } - \d { 2 } T \d { 2 } : \d { 2 } : \d { 2 } (?: \. \d { 1 , 9 } ) ? (?: Z | [ + - ] \d { 2 } : \d { 2 } ) ) \| ( \d { 1 , 9 } ) \| ( \d { 1 , 9 } (?: , \d { 1 , 9 } ) * ) ? (?: \| ( [ \w . : ~ + - ] { 1 , 128 } ) ) ? $ / ;
345346
346347function parseListCursor ( raw : string ) : ListCursor | null {
347348 const match = LIST_CURSOR_PATTERN . exec ( raw ) ;
@@ -351,6 +352,7 @@ function parseListCursor(raw: string): ListCursor | null {
351352 updatedBefore : match [ 1 ] ! ,
352353 delivered : Number ( match [ 2 ] ) ,
353354 seenAt : seenAt === undefined ? [ ] : seenAt . split ( "," ) . map ( Number ) ,
355+ ...( match [ 4 ] === undefined ? { } : { resumeAfter : match [ 4 ] } ) ,
354356 } ;
355357}
356358
@@ -368,27 +370,30 @@ function nextListCursor(
368370 delivered : ReadonlyArray < ProviderChangeRequest > ,
369371 /** A provider may consume malformed offset-paged rows that never appear in `delivered`. */
370372 cursorAdvance = delivered . length ,
373+ resumeAfter ?: string ,
371374) : string | null {
372375 if ( fetched . length === 0 ) return null ;
373376 // Taken from what the host answered rather than from what survived de-duplication: a slice can
374377 // be entirely rows already sent — a hundred change requests touched in the same second is one
375378 // repository's boring afternoon — and reading "nothing new" as "nothing left" would end the
376379 // walk on the instant it was stuck on, with everything older unreachable for good.
377380 const oldest = fetched . reduce ( ( left , right ) => ( right . updatedAt < left . updatedAt ? right : left ) ) ;
378- return listCursorAt ( previous , oldest . updatedAt , fetched , cursorAdvance ) ;
381+ return listCursorAt ( previous , oldest . updatedAt , fetched , cursorAdvance , resumeAfter ) ;
379382}
380383
381384function listCursorAt (
382385 previous : ListCursor | undefined ,
383386 boundary : string ,
384387 fetched : ReadonlyArray < ProviderChangeRequest > ,
385388 deliveredCount : number ,
389+ resumeAfter ?: string ,
386390) : string {
387391 const seenAt = new Set ( [
388392 ...( previous ?. updatedBefore === boundary ? previous . seenAt : [ ] ) ,
389393 ...fetched . filter ( ( item ) => item . updatedAt === boundary ) . map ( ( item ) => item . number ) ,
390394 ] ) ;
391- return `${ boundary } |${ ( previous ?. delivered ?? 0 ) + deliveredCount } |${ [ ...seenAt ] . join ( "," ) } ` ;
395+ const cursor = `${ boundary } |${ ( previous ?. delivered ?? 0 ) + deliveredCount } |${ [ ...seenAt ] . join ( "," ) } ` ;
396+ return resumeAfter === undefined ? cursor : `${ cursor } |${ resumeAfter } ` ;
392397}
393398
394399function isProviderUnusable ( error : PullRequestProviderError ) : boolean {
@@ -1165,11 +1170,13 @@ export const make = Effect.gen(function* () {
11651170 const cursorOf = ( project : SupportedProject ) : ListCursor | undefined =>
11661171 continuation ?. get ( project . cursorKey ) ;
11671172
1168- const readRepository = ( project : SupportedProject ) : Effect . Effect < RepositoryBatch > => {
1173+ const readSlice = (
1174+ project : SupportedProject ,
1175+ cursor : ListCursor | undefined ,
1176+ ) : Effect . Effect < RepositoryBatch > => {
11691177 {
11701178 const viewer = viewers [ project . host ] ! ;
11711179 const key = project . cursorKey ;
1172- const cursor = cursorOf ( project ) ;
11731180 return project . api
11741181 . listChangeRequests ( {
11751182 cwd : project . project . workspaceRoot ,
@@ -1188,6 +1195,7 @@ export const make = Effect.gen(function* () {
11881195 updatedBefore : cursor . updatedBefore ,
11891196 delivered : cursor . delivered ,
11901197 seenAt : cursor . seenAt ,
1198+ resumeAfter : cursor . resumeAfter ,
11911199 } ,
11921200 } ) ,
11931201 } )
@@ -1214,7 +1222,13 @@ export const make = Effect.gen(function* () {
12141222 truncated : page . truncated ,
12151223 nextCursor :
12161224 page . continues && page . truncated
1217- ? nextListCursor ( cursor , page . items , items , page . cursorAdvance )
1225+ ? nextListCursor (
1226+ cursor ,
1227+ page . items ,
1228+ items ,
1229+ page . cursorAdvance ,
1230+ page . resumeAfter ,
1231+ )
12181232 : null ,
12191233 } ;
12201234 } ) ,
@@ -1235,6 +1249,32 @@ export const make = Effect.gen(function* () {
12351249 }
12361250 } ;
12371251
1252+ // Filters a host cannot apply thin out the slices it hands over, so a repository keeps
1253+ // reading until it has a page's worth or runs out. The reads are bounded: a filter that
1254+ // matches nothing would otherwise walk the repository's whole history in one request.
1255+ const readRepository = ( project : SupportedProject ) : Effect . Effect < RepositoryBatch > => {
1256+ const fill = ( batch : RepositoryBatch , reads : number ) : Effect . Effect < RepositoryBatch > => {
1257+ const cursor = batch . nextCursor === null ? null : parseListCursor ( batch . nextCursor ) ;
1258+ if (
1259+ input . filters === undefined ||
1260+ cursor === null ||
1261+ batch . entries . length >= limit ||
1262+ reads >= FILTERED_LIST_READS
1263+ ) {
1264+ return Effect . succeed ( batch ) ;
1265+ }
1266+ return readSlice ( project , cursor ) . pipe (
1267+ Effect . flatMap ( ( next ) =>
1268+ // A slice that failed keeps the cursor before it, so the reader can try again.
1269+ next . errors . length > 0
1270+ ? Effect . succeed ( batch )
1271+ : fill ( { ...next , entries : [ ...batch . entries , ...next . entries ] } , reads + 1 ) ,
1272+ ) ,
1273+ ) ;
1274+ } ;
1275+ return readSlice ( project , cursorOf ( project ) ) . pipe ( Effect . flatMap ( ( batch ) => fill ( batch , 1 ) ) ) ;
1276+ } ;
1277+
12381278 const readTogether = (
12391279 chunk : ReadonlyArray < SupportedProject > ,
12401280 ) : Effect . Effect < ReadonlyArray < RepositoryBatch > > => {
0 commit comments