Skip to content

Commit c4264fd

Browse files
fix(relay): tunnel recovery no longer drives the Cloudflare API into rate limits
Host recovery requests re-ran a full tunnel provision (list, ingress, DNS, token) even when the tunnel was ready for the same origin, and the Cloudflare SDK slept through every 429 until the relay's 8 s timeout. On 2026-10-19 that tipped the account into sustained rate limiting: links and recoveries both stalled into 503/504 and their retries kept the limit in place. - Recovery of a ready tunnel for an unchanged origin now only checks the tunnel exists and fetches its token, falling back to a full provision otherwise. Links still fully provision. - Cloudflare rate limits are no longer retried inside the request, and a 429 pauses this isolate's Cloudflare calls for its Retry-After so callers fail fast instead of extending the limit. Refs #16810 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 03c5bc8 commit c4264fd

5 files changed

Lines changed: 303 additions & 32 deletions

File tree

‎infra/relay/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
},
1414
"dependencies": {
1515
"@clerk/backend": "catalog:",
16+
"@distilled.cloud/cloudflare": "1.0.0-rc.13",
1617
"@effect/sql-pg": "catalog:",
1718
"@effect/sql-sqlite-do": "catalog:",
1819
"@noble/curves": "catalog:",

‎infra/relay/src/environments/ManagedEndpointProvider.test.ts‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import * as Cloudflare from "alchemy/Cloudflare";
66
import * as Effect from "effect/Effect";
77
import * as Layer from "effect/Layer";
88
import * as Redacted from "effect/Redacted";
9+
import * as TestClock from "effect/testing/TestClock";
910

1011
import * as RelayConfiguration from "../Config.ts";
1112
import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
@@ -754,6 +755,127 @@ describe("ManagedEndpointProvider", () => {
754755
}).pipe(Effect.provide(layer));
755756
});
756757

758+
it.effect("recovers a ready tunnel for the same origin with only reads", () => {
759+
const tunnelCalls: TunnelCall[] = [];
760+
const dnsCalls: DnsCall[] = [];
761+
const allocationCalls: AllocationCall[] = [];
762+
const layer = layerProvider(
763+
makePersistentTunnelClient(tunnelCalls),
764+
makeDnsClient(dnsCalls),
765+
makeAllocations(allocationCalls),
766+
);
767+
768+
return Effect.gen(function* () {
769+
const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
770+
const request = {
771+
userId: "user_ABC",
772+
environmentId: "env_ABC",
773+
origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
774+
} as const;
775+
const provisioned = yield* provider.provision(request);
776+
tunnelCalls.length = 0;
777+
dnsCalls.length = 0;
778+
allocationCalls.length = 0;
779+
780+
const recovered = yield* provider.provision({ ...request, reuseReadyTunnel: true });
781+
782+
expect(recovered).toEqual(provisioned);
783+
expect(tunnelCalls.map((call) => call.operation)).toEqual(["get", "getToken"]);
784+
expect(dnsCalls).toEqual([]);
785+
expect(allocationCalls.map((call) => call.operation)).toEqual(["reserve"]);
786+
}).pipe(Effect.provide(layer));
787+
});
788+
789+
it.effect("fully provisions a recovery when the ready tunnel cannot be reused", () => {
790+
const tunnelCalls: TunnelCall[] = [];
791+
const tunnelClient = makePersistentTunnelClient(tunnelCalls);
792+
const layer = layerProvider(tunnelClient);
793+
794+
return Effect.gen(function* () {
795+
const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
796+
const request = {
797+
userId: "user_ABC",
798+
environmentId: "env_ABC",
799+
origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
800+
reuseReadyTunnel: true,
801+
} as const;
802+
yield* provider.provision(request);
803+
804+
tunnelCalls.length = 0;
805+
yield* provider.provision({
806+
...request,
807+
origin: { localHttpHost: "127.0.0.1", localHttpPort: 3774 },
808+
});
809+
expect(tunnelCalls.map((call) => call.operation)).toEqual([
810+
"list",
811+
"putConfiguration",
812+
"getToken",
813+
]);
814+
815+
yield* tunnelClient.delete("tunnel-id");
816+
tunnelCalls.length = 0;
817+
yield* provider.provision({
818+
...request,
819+
origin: { localHttpHost: "127.0.0.1", localHttpPort: 3774 },
820+
});
821+
expect(tunnelCalls.map((call) => call.operation)).toEqual([
822+
"get",
823+
"list",
824+
"create",
825+
"putConfiguration",
826+
"getToken",
827+
]);
828+
}).pipe(Effect.provide(layer));
829+
});
830+
831+
it.effect("stops calling Cloudflare until a rate limit has passed", () => {
832+
let listCalls = 0;
833+
const tunnelClient = {
834+
list: () =>
835+
Effect.suspend(() => {
836+
listCalls += 1;
837+
return Effect.fail({ _tag: "TooManyRequests", message: "Rate limited" });
838+
}),
839+
} as unknown as Cloudflare.Tunnel.ReadWriteTunnelClient;
840+
const layer = ManagedEndpointProvider.layerCloudflareBindings(
841+
tunnelClient,
842+
{} as Cloudflare.DNS.ReadWriteDnsClient,
843+
{} as Alchemy.BaseRuntimeContext,
844+
).pipe(
845+
Layer.provideMerge(NodeServices.layer),
846+
Layer.provide(RelayConfiguration.layer(config)),
847+
Layer.provide(
848+
Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, makeAllocations()),
849+
),
850+
Layer.provide(Layer.succeed(ManagedTunnelLimits.ManagedTunnelLimits, makeTunnelLimits())),
851+
);
852+
853+
return Effect.gen(function* () {
854+
const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
855+
const provision = provider
856+
.provision({
857+
userId: "user_ABC",
858+
environmentId: "env_ABC",
859+
origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
860+
})
861+
.pipe(Effect.flip);
862+
863+
const rateLimited = yield* provision;
864+
const paused = yield* provision;
865+
expect(listCalls).toBe(1);
866+
expect(ManagedEndpointProvider.isCloudflareRateLimited(rateLimited)).toBe(true);
867+
expect(paused).toMatchObject({
868+
_tag: "ManagedEndpointProvisioningFailed",
869+
stage: "ensure-tunnel",
870+
cause: { cause: { _tag: "CloudflareApiPaused" } },
871+
});
872+
873+
yield* TestClock.adjust("30 seconds");
874+
yield* provision;
875+
expect(listCalls).toBe(2);
876+
}).pipe(Effect.provide(layer));
877+
});
878+
757879
it.effect("recreates a checkpointed DNS record when it was removed externally", () => {
758880
const dnsCalls: DnsCall[] = [];
759881
const allocationCalls: AllocationCall[] = [];

0 commit comments

Comments
 (0)