Skip to content

Commit f9eb783

Browse files
PR Batch Testerclaude
andcommitted
fix(bitbucket): stop the startup sweep from flooding Bitbucket with 429s
Bitbucket requests had no concurrency cap and no shared pause, so after the first 429 every queued branch lookup still went out and got its own 429. They now share four request slots and the host-wide pause GitHub already uses. Background reads fail fast while paused; writes and interactive reads still go through. Closes #23885 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent b3c0109 commit f9eb783

5 files changed

Lines changed: 209 additions & 47 deletions

File tree

‎apps/server/src/sourceControl/BitbucketApi.test.ts‎

Lines changed: 87 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,19 @@ import { assert, it, vi } from "@effect/vitest";
33
import * as NodeServices from "@effect/platform-node/NodeServices";
44
import * as ConfigProvider from "effect/ConfigProvider";
55
import * as DateTime from "effect/DateTime";
6+
import * as Deferred from "effect/Deferred";
67
import * as Effect from "effect/Effect";
8+
import * as Fiber from "effect/Fiber";
79
import * as FileSystem from "effect/FileSystem";
810
import * as Layer from "effect/Layer";
911
import * as Option from "effect/Option";
12+
import * as Queue from "effect/Queue";
1013
import * as Schema from "effect/Schema";
14+
import * as TestClock from "effect/testing/TestClock";
1115
import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http";
1216

1317
import * as BitbucketApi from "@t3tools/source-control-bitbucket/server/BitbucketApi";
18+
import * as SourceControlRateLimit from "@t3tools/source-control-core/server/SourceControlRateLimit";
1419
import * as ServerSettings from "../serverSettings.ts";
1520
import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
1621
import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts";
@@ -60,6 +65,8 @@ const repositoryJson = {
6065

6166
function makeLayer(input: {
6267
readonly response: (request: HttpClientRequest.HttpClientRequest) => Response;
68+
/** Holds the answer until it resolves, in place of answering at once. */
69+
readonly answer?: (response: Response) => Effect.Effect<Response>;
6370
readonly requestFailure?: (
6471
request: HttpClientRequest.HttpClientRequest,
6572
) => HttpClientError.HttpClientError;
@@ -72,7 +79,9 @@ function makeLayer(input: {
7279
? Effect.never
7380
: input.requestFailure
7481
? Effect.fail(input.requestFailure(request))
75-
: Effect.succeed(HttpClientResponse.fromWeb(request, input.response(request))),
82+
: (input.answer ?? Effect.succeed)(input.response(request)).pipe(
83+
Effect.map((response) => HttpClientResponse.fromWeb(request, response)),
84+
),
7685
);
7786
const gitMock = {
7887
readConfigValue: vi.fn<GitVcsDriver.GitVcsDriver["Service"]["readConfigValue"]>(() =>
@@ -144,6 +153,7 @@ function makeLayer(input: {
144153

145154
const layer = BitbucketApi.layer.pipe(
146155
Layer.provide(ServerSourceControlHost.layer),
156+
Layer.provide(SourceControlRateLimit.layer),
147157
Layer.provide(Layer.mock(VcsProcess.VcsProcess)({})),
148158
Layer.provide(
149159
Layer.succeed(
@@ -736,3 +746,79 @@ it.effect("cuts a response short rather than reading an unbounded diff into memo
736746
),
737747
),
738748
);
749+
750+
it.effect("pauses background reads after a 429 until Bitbucket's retry-after passes", () => {
751+
let answered = 0;
752+
const { execute, layer } = makeLayer({
753+
response: () => {
754+
answered += 1;
755+
return answered === 1
756+
? new Response("Rate limit exceeded", { status: 429, headers: { "retry-after": "60" } })
757+
: Response.json({ values: [] });
758+
},
759+
});
760+
761+
return Effect.gen(function* () {
762+
const bitbucket = yield* BitbucketApi.BitbucketApi;
763+
const lookup = bitbucket.listPullRequests({
764+
cwd: "/repo",
765+
headSelector: "feature/a",
766+
state: "all",
767+
});
768+
769+
const refused = yield* Effect.flip(lookup);
770+
assert.strictEqual(refused._tag, "BitbucketResponseError");
771+
const paused = yield* Effect.flip(lookup);
772+
assert.strictEqual(paused._tag, "SourceControlRateLimitPausedError");
773+
assert.strictEqual(execute.mock.calls.length, 1);
774+
775+
// A user waiting on an answer is not held back by the background pause.
776+
yield* lookup.pipe(Effect.provideService(SourceControlRateLimit.Interactive, true));
777+
yield* bitbucket.request({
778+
method: "POST",
779+
url: "/repositories/acme/web/pullrequests/1/approve",
780+
});
781+
assert.strictEqual(execute.mock.calls.length, 3);
782+
783+
yield* TestClock.adjust("61 seconds");
784+
yield* lookup;
785+
assert.strictEqual(execute.mock.calls.length, 4);
786+
}).pipe(Effect.provide(layer));
787+
});
788+
789+
it.effect("keeps a sweep over many branches to a few Bitbucket requests at a time", () =>
790+
Effect.gen(function* () {
791+
const started = yield* Queue.unbounded<void>();
792+
const release = yield* Deferred.make<void>();
793+
let inFlight = 0;
794+
let peak = 0;
795+
const { execute, layer } = makeLayer({
796+
response: () => Response.json({ values: [] }),
797+
answer: (response) =>
798+
Effect.gen(function* () {
799+
inFlight += 1;
800+
peak = Math.max(peak, inFlight);
801+
yield* Queue.offer(started, undefined);
802+
yield* Deferred.await(release);
803+
inFlight -= 1;
804+
return response;
805+
}),
806+
});
807+
808+
yield* Effect.gen(function* () {
809+
const bitbucket = yield* BitbucketApi.BitbucketApi;
810+
const sweep = yield* Effect.forEach(
811+
Array.from({ length: 12 }, (_, index) => `feature/${index}`),
812+
(branch) =>
813+
bitbucket.listPullRequests({ cwd: "/repo", headSelector: branch, state: "all" }),
814+
{ concurrency: "unbounded", discard: true },
815+
).pipe(Effect.forkChild);
816+
for (let index = 0; index < 4; index += 1) yield* Queue.take(started);
817+
yield* Deferred.succeed(release, undefined);
818+
yield* Fiber.join(sweep);
819+
}).pipe(Effect.provide(layer));
820+
821+
assert.strictEqual(peak, 4);
822+
assert.strictEqual(execute.mock.calls.length, 12);
823+
}),
824+
);

‎apps/server/src/sourceControl/builtInDrivers.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import * as GitLabCli from "@t3tools/source-control-gitlab/server/GitLabCli";
1616
import * as GitLabPullRequestCli from "@t3tools/source-control-gitlab/server/GitLabPullRequestCli";
1717
import * as GitLabDriver from "@t3tools/source-control-gitlab/server/driver";
1818
import type { SourceControlDriver } from "@t3tools/source-control-core/server/driver";
19+
import * as SourceControlRateLimit from "@t3tools/source-control-core/server/SourceControlRateLimit";
1920
import * as Layer from "effect/Layer";
2021

2122
import * as ServerSourceControlHost from "./ServerSourceControlHost.ts";
@@ -41,7 +42,9 @@ export const layer = Layer.mergeAll(
4142
// reader shares, so the server builds it once, here.
4243
GitHubPullRequestApi.layer.pipe(Layer.provideMerge(GitHubRest.layerWithDependencies)),
4344
AzureDevOpsPullRequestCli.layer.pipe(Layer.provideMerge(AzureDevOpsCli.layer)),
44-
BitbucketPullRequestApi.layer.pipe(Layer.provideMerge(BitbucketApi.layer)),
45+
BitbucketPullRequestApi.layer.pipe(
46+
Layer.provideMerge(BitbucketApi.layer.pipe(Layer.provide(SourceControlRateLimit.layer))),
47+
),
4548
ForgejoCli.layer,
4649
GitCafeApi.layer.pipe(Layer.provideMerge(GitCafeCredentials.layer)),
4750
GitLabPullRequestCli.layer.pipe(Layer.provideMerge(GitLabCli.layer)),

‎packages/source-control-bitbucket/src/server/BitbucketApi.ts‎

Lines changed: 103 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ import * as FileSystem from "effect/FileSystem";
66
import * as Layer from "effect/Layer";
77
import * as Option from "effect/Option";
88
import * as Schema from "effect/Schema";
9+
import * as Semaphore from "effect/Semaphore";
910
import {
1011
NonNegativeInt,
12+
SourceControlProviderKind,
1113
TrimmedNonEmptyString,
1214
type SourceControlProviderAuth,
1315
type SourceControlRepositoryCloneUrls,
@@ -32,12 +34,14 @@ import * as SourceControlHost from "@t3tools/source-control-core/server/SourceCo
3234
import { readSourceControlHostSettings } from "@t3tools/source-control-core/client/definition";
3335

3436
import * as BitbucketClient from "../client/definition.ts";
35-
import { retryAtFromHeader } from "@t3tools/source-control-core/server/SourceControlRateLimit";
37+
import * as SourceControlRateLimit from "@t3tools/source-control-core/server/SourceControlRateLimit";
3638

3739
const DEFAULT_API_BASE_URL = "https://api.bitbucket.org/2.0";
3840
const DEFAULT_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
3941
const MAX_REDIRECTS = 3;
4042
const BITBUCKET_API_REQUEST_TIMEOUT = "30 seconds";
43+
/** A sweep over many branches would otherwise reach Bitbucket as one burst of lookups. */
44+
const CONCURRENCY = 4;
4145

4246
const BitbucketApiEnvConfig = Config.all({
4347
baseUrl: Config.String("T3CODE_BITBUCKET_API_BASE_URL").pipe(
@@ -262,6 +266,7 @@ export const BitbucketApiError = Schema.Union([
262266
BitbucketRepositoryRemoteNotFoundError,
263267
BitbucketPullRequestBodyReadError,
264268
BitbucketCheckoutError,
269+
SourceControlRateLimit.SourceControlRateLimitPausedError,
265270
]);
266271
export type BitbucketApiError = typeof BitbucketApiError.Type;
267272
const isBitbucketApiError = Schema.is(BitbucketApiError);
@@ -602,12 +607,20 @@ function originOf(value: string): string | null {
602607
}
603608
}
604609

610+
function hostOf(value: string): string | null {
611+
try {
612+
return new URL(value).host;
613+
} catch {
614+
return null;
615+
}
616+
}
617+
605618
const responseError = Effect.fn("BitbucketApi.responseError")(function* (
606619
operation: BitbucketApiOperation,
607620
response: HttpClientResponse.HttpClientResponse,
608621
): Effect.fn.Return<never, BitbucketApiError> {
609622
const now = yield* Clock.currentTimeMillis;
610-
const retryAt = retryAtFromHeader(response.headers["retry-after"], now);
623+
const retryAt = SourceControlRateLimit.retryAtFromHeader(response.headers["retry-after"], now);
611624
const collected = yield* collectUint8StreamText({
612625
stream: response.stream,
613626
maxBytes: DEFAULT_MAX_RESPONSE_BYTES,
@@ -637,6 +650,12 @@ export const make = Effect.gen(function* () {
637650
const git = host.git;
638651
const httpClient = yield* HttpClient.HttpClient;
639652
const fileSystem = yield* FileSystem.FileSystem;
653+
const limits = yield* SourceControlRateLimit.SourceControlRateLimit;
654+
const slots = yield* Semaphore.make(CONCURRENCY);
655+
const rateLimitKey = {
656+
provider: SourceControlProviderKind.make("bitbucket"),
657+
host: hostOf(config.baseUrl) ?? config.baseUrl,
658+
};
640659

641660
const apiUrl = (path: string) => `${config.baseUrl.replace(/\/+$/u, "")}${path}`;
642661

@@ -673,6 +692,39 @@ export const make = Effect.gen(function* () {
673692
),
674693
);
675694

695+
/**
696+
* Every request waits for one of the shared slots, then honors the pause a 429 starts for the
697+
* whole host. Each queued lookup would otherwise spend its own 429 and keep the quota exhausted.
698+
* Background reads fail at once while paused; a write or an interactive read has a user waiting
699+
* on it, so it still goes out.
700+
*/
701+
const throttled = <A, R>(
702+
method: string,
703+
effect: Effect.Effect<A, BitbucketApiError, R>,
704+
): Effect.Effect<A, BitbucketApiError, R> =>
705+
slots.withPermit(
706+
SourceControlRateLimit.Interactive.pipe(
707+
Effect.flatMap((interactive) =>
708+
limits.check(
709+
rateLimitKey,
710+
interactive || method !== "GET" ? { allowPaused: true } : undefined,
711+
),
712+
),
713+
Effect.flatMap((lease) =>
714+
effect.pipe(
715+
Effect.tap(() => limits.recordSuccess({ ...rateLimitKey, lease })),
716+
Effect.tapError((error) =>
717+
(error._tag === "BitbucketResponseError" ||
718+
error._tag === "BitbucketResponseBodyReadError") &&
719+
error.status === 429
720+
? limits.recordRateLimit({ ...rateLimitKey, lease, retryAt: error.retryAt })
721+
: Effect.void,
722+
),
723+
),
724+
),
725+
),
726+
);
727+
676728
const decodeResponse = <S extends Schema.Top>(
677729
operation: BitbucketApiOperation,
678730
schema: S,
@@ -698,20 +750,23 @@ export const make = Effect.gen(function* () {
698750
request: HttpClientRequest.HttpClientRequest,
699751
schema: S,
700752
): Effect.Effect<S["Type"], BitbucketApiError, S["DecodingServices"]> =>
701-
withAuth(request.pipe(HttpClientRequest.acceptJson)).pipe(
702-
Effect.flatMap(httpClient.execute),
703-
Effect.mapError(
704-
(cause) =>
705-
new BitbucketRequestError({
706-
operation,
707-
cause,
708-
}),
753+
throttled(
754+
request.method,
755+
withAuth(request.pipe(HttpClientRequest.acceptJson)).pipe(
756+
Effect.flatMap(httpClient.execute),
757+
Effect.mapError(
758+
(cause) =>
759+
new BitbucketRequestError({
760+
operation,
761+
cause,
762+
}),
763+
),
764+
Effect.flatMap((response) => decodeResponse(operation, schema, response)),
765+
Effect.timeout(BITBUCKET_API_REQUEST_TIMEOUT),
766+
Effect.catchTags({
767+
TimeoutError: (cause) => Effect.fail(new BitbucketRequestError({ operation, cause })),
768+
}),
709769
),
710-
Effect.flatMap((response) => decodeResponse(operation, schema, response)),
711-
Effect.timeout(BITBUCKET_API_REQUEST_TIMEOUT),
712-
Effect.catchTags({
713-
TimeoutError: (cause) => Effect.fail(new BitbucketRequestError({ operation, cause })),
714-
}),
715770
);
716771

717772
const resolveRepository = Effect.fn("BitbucketApi.resolveRepository")(function* (input: {
@@ -913,38 +968,41 @@ export const make = Effect.gen(function* () {
913968
};
914969

915970
const request: BitbucketApi["Service"]["request"] = (input) =>
916-
send({ ...input, redirects: 0 }).pipe(
917-
Effect.flatMap((response) =>
918-
HttpClientResponse.matchStatus({
919-
// Read through the body stream rather than `text`, so an oversized diff is stopped as
920-
// it arrives instead of being materialized whole and then cut.
921-
"2xx": (success) =>
922-
collectUint8StreamText({
923-
stream: success.stream,
924-
maxBytes: input.maxBytes ?? DEFAULT_MAX_RESPONSE_BYTES,
925-
}).pipe(
926-
Effect.mapError(
927-
(cause) =>
928-
new BitbucketResponseBodyReadError({
929-
operation: "request",
930-
status: success.status,
931-
cause,
932-
}),
971+
throttled(
972+
input.method,
973+
send({ ...input, redirects: 0 }).pipe(
974+
Effect.flatMap((response) =>
975+
HttpClientResponse.matchStatus({
976+
// Read through the body stream rather than `text`, so an oversized diff is stopped as
977+
// it arrives instead of being materialized whole and then cut.
978+
"2xx": (success) =>
979+
collectUint8StreamText({
980+
stream: success.stream,
981+
maxBytes: input.maxBytes ?? DEFAULT_MAX_RESPONSE_BYTES,
982+
}).pipe(
983+
Effect.mapError(
984+
(cause) =>
985+
new BitbucketResponseBodyReadError({
986+
operation: "request",
987+
status: success.status,
988+
cause,
989+
}),
990+
),
991+
Effect.map((collected) => ({
992+
body: collected.text,
993+
truncated: collected.truncated,
994+
})),
933995
),
934-
Effect.map((collected) => ({
935-
body: collected.text,
936-
truncated: collected.truncated,
937-
})),
938-
),
939-
orElse: (failed) => responseError("request", failed),
940-
})(response),
996+
orElse: (failed) => responseError("request", failed),
997+
})(response),
998+
),
999+
// This also caps redirect hops: no request can finish headers then stall its body forever.
1000+
Effect.timeout(BITBUCKET_API_REQUEST_TIMEOUT),
1001+
Effect.catchTags({
1002+
TimeoutError: (cause) =>
1003+
Effect.fail(new BitbucketRequestError({ operation: "request", cause })),
1004+
}),
9411005
),
942-
// This also caps redirect hops: no request can finish headers then stall its body forever.
943-
Effect.timeout(BITBUCKET_API_REQUEST_TIMEOUT),
944-
Effect.catchTags({
945-
TimeoutError: (cause) =>
946-
Effect.fail(new BitbucketRequestError({ operation: "request", cause })),
947-
}),
9481006
);
9491007

9501008
return BitbucketApi.of({

‎packages/source-control-bitbucket/src/server/BitbucketPullRequestProvider.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1+
import { SourceControlProviderKind } from "@t3tools/contracts";
12
import { describe, expect, it } from "@effect/vitest";
23
import * as Effect from "effect/Effect";
34
import * as Layer from "effect/Layer";
45
import * as Result from "effect/Result";
6+
import * as SourceControlRateLimit from "@t3tools/source-control-core/server/SourceControlRateLimit";
57

68
import * as BitbucketApi from "./BitbucketApi.ts";
79
import * as BitbucketPullRequestApi from "./BitbucketPullRequestApi.ts";
@@ -188,6 +190,16 @@ describe("bitbucketProviderFailure", () => {
188190
expect(bitbucketProviderFailure(responseError(401)).reason).toBe("unauthenticated");
189191
expect(bitbucketProviderFailure(responseError(403)).reason).toBe("failed");
190192
});
193+
194+
it("reports a request held back by the host-wide pause as rate limited until it ends", () => {
195+
const paused = new SourceControlRateLimit.SourceControlRateLimitPausedError({
196+
provider: SourceControlProviderKind.make("bitbucket"),
197+
host: "api.bitbucket.org",
198+
retryAt: 120_000,
199+
});
200+
201+
expect(bitbucketProviderFailure(paused)).toEqual({ reason: "rate-limited", retryAt: 120_000 });
202+
});
191203
});
192204

193205
describe("bitbucketViewerPermissions", () => {

0 commit comments

Comments
 (0)