Skip to content

Commit 4feb0dc

Browse files
fix(relay): releasing a tunnel that still has a connector no longer 500s
Cloudflare refuses to delete a tunnel while a connector is attached ("This tunnel has active connections"). Shutdown release treated that as an upstream failure, so ~45% of releases (about 3k a day across 900 environments) answered upstream_unavailable and logged errors. The tunnel was not released, which is exactly what ok:false already means: the environment keeps its connector config and the reaper deletes the tunnel once it has been down past the grace period. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 25d5c7c commit 4feb0dc

2 files changed

Lines changed: 59 additions & 2 deletions

File tree

‎infra/relay/src/environments/ManagedEndpointProvider.test.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1534,6 +1534,36 @@ describe("ManagedEndpointProvider", () => {
15341534
}).pipe(Effect.provide(layer));
15351535
});
15361536

1537+
it.effect("reports a tunnel with an attached connector as not released", () => {
1538+
const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
1539+
...makeTunnelClient(),
1540+
delete: (tunnelId) =>
1541+
Effect.fail(
1542+
new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
1543+
operation: "delete",
1544+
tunnelId,
1545+
cause: {
1546+
_tag: "BadRequest",
1547+
code: 1022,
1548+
message:
1549+
"This tunnel has active connections. Please stop all cloudflared replicas, or wait a few minutes for connections to close, then try again.",
1550+
},
1551+
}),
1552+
),
1553+
});
1554+
const layer = providerLayer(tunnelClient, makeDnsClient(), makeAllocations());
1555+
1556+
return Effect.gen(function* () {
1557+
const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
1558+
const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
1559+
yield* provider.provision({
1560+
...key,
1561+
origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
1562+
});
1563+
expect(yield* provider.release(key)).toBe(false);
1564+
}).pipe(Effect.provide(layer));
1565+
});
1566+
15371567
it.effect("treats an absent allocation as already deprovisioned", () => {
15381568
const tunnelCalls: TunnelCall[] = [];
15391569
const dnsCalls: DnsCall[] = [];

‎infra/relay/src/environments/ManagedEndpointProvider.ts‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -395,6 +395,25 @@ export function isManagedEndpointNotFound(cause: unknown): boolean {
395395
return "cause" in cause && isManagedEndpointNotFound(cause.cause);
396396
}
397397

398+
/**
399+
* Cloudflare refuses to delete a tunnel while a connector is still attached,
400+
* either one that has not finished draining or another runtime still serving
401+
* the tunnel.
402+
*/
403+
export function isManagedEndpointTunnelInUse(cause: unknown): boolean {
404+
if (typeof cause !== "object" || cause === null) {
405+
return false;
406+
}
407+
if (
408+
"message" in cause &&
409+
typeof cause.message === "string" &&
410+
cause.message.includes("has active connections")
411+
) {
412+
return true;
413+
}
414+
return "cause" in cause && isManagedEndpointTunnelInUse(cause.cause);
415+
}
416+
398417
type ManagedEndpointClientError = ManagedEndpointTunnelClientError | ManagedEndpointDnsClientError;
399418

400419
const ignoreNotFound = <A>(
@@ -873,8 +892,16 @@ export const make = Effect.gen(function* () {
873892
if (finalGeneration === null) {
874893
return false;
875894
}
876-
yield* deleteTunnel;
877-
return true;
895+
// A connector still attached means the tunnel is not released. That
896+
// is the same answer as losing the claim: the caller keeps its config,
897+
// and the reaper deletes the tunnel once it has been down long enough.
898+
return yield* deleteTunnel.pipe(
899+
Effect.as(true),
900+
Effect.catchIf(
901+
(error) => isManagedEndpointTunnelInUse(error.cause),
902+
() => Effect.succeed(false),
903+
),
904+
);
878905
}),
879906
)
880907
.pipe(

0 commit comments

Comments
 (0)