@@ -6,8 +6,10 @@ import * as FileSystem from "effect/FileSystem";
66import * as Layer from "effect/Layer" ;
77import * as Option from "effect/Option" ;
88import * as Schema from "effect/Schema" ;
9+ import * as Semaphore from "effect/Semaphore" ;
910import {
1011 NonNegativeInt ,
12+ SourceControlProviderKind ,
1113 TrimmedNonEmptyString ,
1214 type SourceControlProviderAuth ,
1315 type SourceControlRepositoryCloneUrls ,
@@ -32,12 +34,14 @@ import * as SourceControlHost from "@t3tools/source-control-core/server/SourceCo
3234import { readSourceControlHostSettings } from "@t3tools/source-control-core/client/definition" ;
3335
3436import * as BitbucketClient from "../client/definition.ts" ;
35- import { retryAtFromHeader } from "@t3tools/source-control-core/server/SourceControlRateLimit" ;
37+ import * as SourceControlRateLimit from "@t3tools/source-control-core/server/SourceControlRateLimit" ;
3638
3739const DEFAULT_API_BASE_URL = "https://api.bitbucket.org/2.0" ;
3840const DEFAULT_MAX_RESPONSE_BYTES = 8 * 1024 * 1024 ;
3941const MAX_REDIRECTS = 3 ;
4042const BITBUCKET_API_REQUEST_TIMEOUT = "30 seconds" ;
43+ /** A sweep over many branches would otherwise reach Bitbucket as one burst of lookups. */
44+ const CONCURRENCY = 4 ;
4145
4246const BitbucketApiEnvConfig = Config . all ( {
4347 baseUrl : Config . String ( "T3CODE_BITBUCKET_API_BASE_URL" ) . pipe (
@@ -262,6 +266,7 @@ export const BitbucketApiError = Schema.Union([
262266 BitbucketRepositoryRemoteNotFoundError ,
263267 BitbucketPullRequestBodyReadError ,
264268 BitbucketCheckoutError ,
269+ SourceControlRateLimit . SourceControlRateLimitPausedError ,
265270] ) ;
266271export type BitbucketApiError = typeof BitbucketApiError . Type ;
267272const isBitbucketApiError = Schema . is ( BitbucketApiError ) ;
@@ -602,12 +607,20 @@ function originOf(value: string): string | null {
602607 }
603608}
604609
610+ function hostOf ( value : string ) : string | null {
611+ try {
612+ return new URL ( value ) . host ;
613+ } catch {
614+ return null ;
615+ }
616+ }
617+
605618const responseError = Effect . fn ( "BitbucketApi.responseError" ) ( function * (
606619 operation : BitbucketApiOperation ,
607620 response : HttpClientResponse . HttpClientResponse ,
608621) : Effect . fn . Return < never , BitbucketApiError > {
609622 const now = yield * Clock . currentTimeMillis ;
610- const retryAt = retryAtFromHeader ( response . headers [ "retry-after" ] , now ) ;
623+ const retryAt = SourceControlRateLimit . retryAtFromHeader ( response . headers [ "retry-after" ] , now ) ;
611624 const collected = yield * collectUint8StreamText ( {
612625 stream : response . stream ,
613626 maxBytes : DEFAULT_MAX_RESPONSE_BYTES ,
@@ -637,6 +650,12 @@ export const make = Effect.gen(function* () {
637650 const git = host . git ;
638651 const httpClient = yield * HttpClient . HttpClient ;
639652 const fileSystem = yield * FileSystem . FileSystem ;
653+ const limits = yield * SourceControlRateLimit . SourceControlRateLimit ;
654+ const slots = yield * Semaphore . make ( CONCURRENCY ) ;
655+ const rateLimitKey = {
656+ provider : SourceControlProviderKind . make ( "bitbucket" ) ,
657+ host : hostOf ( config . baseUrl ) ?? config . baseUrl ,
658+ } ;
640659
641660 const apiUrl = ( path : string ) => `${ config . baseUrl . replace ( / \/ + $ / u, "" ) } ${ path } ` ;
642661
@@ -673,6 +692,39 @@ export const make = Effect.gen(function* () {
673692 ) ,
674693 ) ;
675694
695+ /**
696+ * Every request waits for one of the shared slots, then honors the pause a 429 starts for the
697+ * whole host. Each queued lookup would otherwise spend its own 429 and keep the quota exhausted.
698+ * Background reads fail at once while paused; a write or an interactive read has a user waiting
699+ * on it, so it still goes out.
700+ */
701+ const throttled = < A , R > (
702+ method : string ,
703+ effect : Effect . Effect < A , BitbucketApiError , R > ,
704+ ) : Effect . Effect < A , BitbucketApiError , R > =>
705+ slots . withPermit (
706+ SourceControlRateLimit . Interactive . pipe (
707+ Effect . flatMap ( ( interactive ) =>
708+ limits . check (
709+ rateLimitKey ,
710+ interactive || method !== "GET" ? { allowPaused : true } : undefined ,
711+ ) ,
712+ ) ,
713+ Effect . flatMap ( ( lease ) =>
714+ effect . pipe (
715+ Effect . tap ( ( ) => limits . recordSuccess ( { ...rateLimitKey , lease } ) ) ,
716+ Effect . tapError ( ( error ) =>
717+ ( error . _tag === "BitbucketResponseError" ||
718+ error . _tag === "BitbucketResponseBodyReadError" ) &&
719+ error . status === 429
720+ ? limits . recordRateLimit ( { ...rateLimitKey , lease, retryAt : error . retryAt } )
721+ : Effect . void ,
722+ ) ,
723+ ) ,
724+ ) ,
725+ ) ,
726+ ) ;
727+
676728 const decodeResponse = < S extends Schema . Top > (
677729 operation : BitbucketApiOperation ,
678730 schema : S ,
@@ -698,20 +750,23 @@ export const make = Effect.gen(function* () {
698750 request : HttpClientRequest . HttpClientRequest ,
699751 schema : S ,
700752 ) : Effect . Effect < S [ "Type" ] , BitbucketApiError , S [ "DecodingServices" ] > =>
701- withAuth ( request . pipe ( HttpClientRequest . acceptJson ) ) . pipe (
702- Effect . flatMap ( httpClient . execute ) ,
703- Effect . mapError (
704- ( cause ) =>
705- new BitbucketRequestError ( {
706- operation,
707- cause,
708- } ) ,
753+ throttled (
754+ request . method ,
755+ withAuth ( request . pipe ( HttpClientRequest . acceptJson ) ) . pipe (
756+ Effect . flatMap ( httpClient . execute ) ,
757+ Effect . mapError (
758+ ( cause ) =>
759+ new BitbucketRequestError ( {
760+ operation,
761+ cause,
762+ } ) ,
763+ ) ,
764+ Effect . flatMap ( ( response ) => decodeResponse ( operation , schema , response ) ) ,
765+ Effect . timeout ( BITBUCKET_API_REQUEST_TIMEOUT ) ,
766+ Effect . catchTags ( {
767+ TimeoutError : ( cause ) => Effect . fail ( new BitbucketRequestError ( { operation, cause } ) ) ,
768+ } ) ,
709769 ) ,
710- Effect . flatMap ( ( response ) => decodeResponse ( operation , schema , response ) ) ,
711- Effect . timeout ( BITBUCKET_API_REQUEST_TIMEOUT ) ,
712- Effect . catchTags ( {
713- TimeoutError : ( cause ) => Effect . fail ( new BitbucketRequestError ( { operation, cause } ) ) ,
714- } ) ,
715770 ) ;
716771
717772 const resolveRepository = Effect . fn ( "BitbucketApi.resolveRepository" ) ( function * ( input : {
@@ -913,38 +968,41 @@ export const make = Effect.gen(function* () {
913968 } ;
914969
915970 const request : BitbucketApi [ "Service" ] [ "request" ] = ( input ) =>
916- send ( { ...input , redirects : 0 } ) . pipe (
917- Effect . flatMap ( ( response ) =>
918- HttpClientResponse . matchStatus ( {
919- // Read through the body stream rather than `text`, so an oversized diff is stopped as
920- // it arrives instead of being materialized whole and then cut.
921- "2xx" : ( success ) =>
922- collectUint8StreamText ( {
923- stream : success . stream ,
924- maxBytes : input . maxBytes ?? DEFAULT_MAX_RESPONSE_BYTES ,
925- } ) . pipe (
926- Effect . mapError (
927- ( cause ) =>
928- new BitbucketResponseBodyReadError ( {
929- operation : "request" ,
930- status : success . status ,
931- cause,
932- } ) ,
971+ throttled (
972+ input . method ,
973+ send ( { ...input , redirects : 0 } ) . pipe (
974+ Effect . flatMap ( ( response ) =>
975+ HttpClientResponse . matchStatus ( {
976+ // Read through the body stream rather than `text`, so an oversized diff is stopped as
977+ // it arrives instead of being materialized whole and then cut.
978+ "2xx" : ( success ) =>
979+ collectUint8StreamText ( {
980+ stream : success . stream ,
981+ maxBytes : input . maxBytes ?? DEFAULT_MAX_RESPONSE_BYTES ,
982+ } ) . pipe (
983+ Effect . mapError (
984+ ( cause ) =>
985+ new BitbucketResponseBodyReadError ( {
986+ operation : "request" ,
987+ status : success . status ,
988+ cause,
989+ } ) ,
990+ ) ,
991+ Effect . map ( ( collected ) => ( {
992+ body : collected . text ,
993+ truncated : collected . truncated ,
994+ } ) ) ,
933995 ) ,
934- Effect . map ( ( collected ) => ( {
935- body : collected . text ,
936- truncated : collected . truncated ,
937- } ) ) ,
938- ) ,
939- orElse : ( failed ) => responseError ( "request" , failed ) ,
940- } ) ( response ) ,
996+ orElse : ( failed ) => responseError ( "request" , failed ) ,
997+ } ) ( response ) ,
998+ ) ,
999+ // This also caps redirect hops: no request can finish headers then stall its body forever.
1000+ Effect . timeout ( BITBUCKET_API_REQUEST_TIMEOUT ) ,
1001+ Effect . catchTags ( {
1002+ TimeoutError : ( cause ) =>
1003+ Effect . fail ( new BitbucketRequestError ( { operation : "request" , cause } ) ) ,
1004+ } ) ,
9411005 ) ,
942- // This also caps redirect hops: no request can finish headers then stall its body forever.
943- Effect . timeout ( BITBUCKET_API_REQUEST_TIMEOUT ) ,
944- Effect . catchTags ( {
945- TimeoutError : ( cause ) =>
946- Effect . fail ( new BitbucketRequestError ( { operation : "request" , cause } ) ) ,
947- } ) ,
9481006 ) ;
9491007
9501008 return BitbucketApi . of ( {
0 commit comments