Skip to content

Commit f62e021

Browse files
feat(relay): tell hosts when to retry during a Cloudflare rate limit
Link, tunnel registration and recovery failures caused by a Cloudflare 429 or the shared pause after one now carry Retry-After with the remaining pause, so hosts that honor it (#22108) wait the pause out instead of retrying into it. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent af7a32f commit f62e021

3 files changed

Lines changed: 134 additions & 0 deletions

File tree

‎infra/relay/src/environments/ManagedEndpointProvider.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -438,6 +438,30 @@ function cloudflareRateLimitPauseMillis(cause: unknown): number {
438438
return Duration.toMillis(Duration.min(pause, CLOUDFLARE_MAX_RATE_LIMIT_PAUSE));
439439
}
440440

441+
/**
442+
* How long a caller should wait before retrying, when `cause` is a Cloudflare
443+
* rate limit or a call refused during the pause that follows one.
444+
*/
445+
export function cloudflareRetryAfterMillis(cause: unknown): number | undefined {
446+
for (let current = cause, depth = 0; depth < 8; depth++) {
447+
if (typeof current !== "object" || current === null) return undefined;
448+
if ("_tag" in current && current._tag === "CloudflareApiPaused") {
449+
const retryAfterMillis: unknown = Reflect.get(current, "retryAfterMillis");
450+
return typeof retryAfterMillis === "number" ? retryAfterMillis : undefined;
451+
}
452+
if (
453+
("_tag" in current &&
454+
(current._tag === "TooManyRequests" || current._tag === "CloudflareRateLimited")) ||
455+
("status" in current && current.status === 429)
456+
) {
457+
return cloudflareRateLimitPauseMillis(current);
458+
}
459+
if (!("cause" in current)) return undefined;
460+
current = current.cause;
461+
}
462+
return undefined;
463+
}
464+
441465
// The SDK's default policy sleeps through Retry-After inside the request, which
442466
// held every caller until its timeout and kept the account over its limit.
443467
// Rate limits surface at once instead; other transient failures still retry.

‎infra/relay/src/http/Api.test.ts‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,102 @@ describe("device listing compatibility", () => {
187187
});
188188
});
189189

190+
describe("Cloudflare rate-limit retry hint", () => {
191+
const provisioningFailed = (cause?: unknown) =>
192+
new ManagedEndpointProvider.ManagedEndpointProvisioningFailed({
193+
userId: "user-1",
194+
environmentId: "env-1",
195+
stage: "ensure-tunnel",
196+
...(cause === undefined
197+
? {}
198+
: {
199+
cause: new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
200+
operation: "list",
201+
cause,
202+
}),
203+
}),
204+
});
205+
206+
const linkResponse = (failure: ManagedEndpointProvider.ManagedEndpointProvisioningFailed) => {
207+
const layerHandlers = RelayHttpApi.layerClientApi.pipe(
208+
HttpRouter.provideRequest(
209+
Layer.mergeAll(
210+
Layer.mock(EnvironmentCredentials.EnvironmentCredentials, {}),
211+
Layer.mock(EnvironmentLinks.EnvironmentLinks, {}),
212+
Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}),
213+
Layer.mock(RelayDb.RelayTransactions, {}),
214+
Layer.mock(HookInbox.HookInbox, {}),
215+
),
216+
),
217+
Layer.provide(
218+
Layer.mergeAll(
219+
Layer.succeed(RelayConfiguration.RelayConfiguration, relaySettings),
220+
NodeCryptoLayer.layer,
221+
Layer.mock(RelayTokens.RelayTokens, { resolveDpopAccessTokenScopes: () => null }),
222+
Layer.mock(EnvironmentLinker.EnvironmentLinker, { link: () => Effect.fail(failure) }),
223+
Layer.mock(EnvironmentLinks.EnvironmentLinks, {}),
224+
Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}),
225+
Layer.mock(Devices.Devices, {}),
226+
),
227+
),
228+
Layer.provideMerge(
229+
Layer.succeed(RelayClientAuth, {
230+
clientBearer: (effect) =>
231+
Effect.provideService(effect, RelayClientPrincipal, {
232+
userId: "user-1",
233+
token: "test-token",
234+
}),
235+
}),
236+
),
237+
);
238+
return Effect.acquireRelease(
239+
Effect.sync(() =>
240+
HttpRouter.toWebHandler(
241+
HttpApiBuilder.layer(HttpApi.make("RelayApi").add(RelayApi.groups.client)).pipe(
242+
Layer.provide(layerHandlers),
243+
Layer.provide(HttpServer.layerServices),
244+
),
245+
{ disableLogger: true },
246+
),
247+
),
248+
(app) => Effect.promise(() => app.dispose()),
249+
).pipe(
250+
Effect.flatMap((app) =>
251+
Effect.promise(() =>
252+
app.handler(
253+
new Request("https://relay.example.test/v1/client/environment-links", {
254+
method: "POST",
255+
headers: { authorization: "Bearer test-token", "content-type": "application/json" },
256+
body: JSON.stringify({
257+
proof: "signed-proof",
258+
notificationsEnabled: true,
259+
liveActivitiesEnabled: true,
260+
managedTunnelsEnabled: true,
261+
}),
262+
}),
263+
),
264+
),
265+
),
266+
);
267+
};
268+
269+
it.effect("tells a link refused during a Cloudflare pause when to retry", () =>
270+
Effect.gen(function* () {
271+
const paused = yield* linkResponse(
272+
provisioningFailed(
273+
new ManagedEndpointProvider.CloudflareApiPaused({ retryAfterMillis: 24_100 }),
274+
),
275+
);
276+
expect(paused.status).toBe(503);
277+
expect(paused.headers.get("retry-after")).toBe("25");
278+
279+
const other = yield* linkResponse(provisioningFailed());
280+
expect(other.status).toBe(503);
281+
expect(other.headers.get("retry-after")).toBeNull();
282+
}).pipe(Effect.scoped),
283+
);
284+
});
285+
190286
describe("relay client authentication", () => {
191287
it.effect("preserves the existing Clerk session JWT path", () =>
192288
Effect.gen(function* () {

‎infra/relay/src/http/Api.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,17 @@ const appendRelayCredentialResponseHeaders = HttpEffect.appendPreResponseHandler
102102
),
103103
);
104104

105+
// T3 Code hosts never retry sooner than Retry-After, so during a Cloudflare
106+
// rate limit they wait out the pause instead of retrying into it.
107+
const appendCloudflareRetryAfter = (error: unknown) => {
108+
const retryAfterMillis = ManagedEndpointProvider.cloudflareRetryAfterMillis(error);
109+
if (retryAfterMillis === undefined) return Effect.void;
110+
const seconds = String(Math.max(1, Math.ceil(retryAfterMillis / 1000)));
111+
return HttpEffect.appendPreResponseHandler((_request, response) =>
112+
Effect.succeed(HttpServerResponse.setHeader(response, "retry-after", seconds)),
113+
);
114+
};
115+
105116
const appendRelayDpopChallengeHeader = HttpEffect.appendPreResponseHandler((_request, response) =>
106117
Effect.succeed(
107118
response.status === 401
@@ -874,6 +885,7 @@ export const layerClientApi = HttpApiBuilder.group(
874885
cloudMintPublicKey: config.cloudMintPublicKey,
875886
};
876887
},
888+
Effect.tapError(appendCloudflareRetryAfter),
877889
mapErrorTags({
878890
EnvironmentLinkProofExpired: (_error, traceId) =>
879891
new RelayEnvironmentLinkProofExpiredError({
@@ -1335,6 +1347,7 @@ export const layerServerApi = HttpApiBuilder.group(
13351347
origin: payload.origin,
13361348
});
13371349
},
1350+
Effect.tapError(appendCloudflareRetryAfter),
13381351
Effect.catchTags({
13391352
ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})),
13401353
ManagedEndpointProvisioningNotConfigured: () =>
@@ -1370,6 +1383,7 @@ export const layerServerApi = HttpApiBuilder.group(
13701383
origin: payload.origin,
13711384
});
13721385
},
1386+
Effect.tapError(appendCloudflareRetryAfter),
13731387
Effect.catchTags({
13741388
ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})),
13751389
ManagedEndpointProvisioningNotConfigured: () =>

0 commit comments

Comments
 (0)