Skip to content

Commit 740bda4

Browse files
fix(server): recover a deleted tunnel when Cloudflare says "Tunnel not found" (#16648)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent c4481b9 commit 740bda4

2 files changed

Lines changed: 127 additions & 5 deletions

File tree

‎apps/server/src/cloud/ManagedEndpointRuntime.test.ts‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,11 +185,27 @@ describe("CloudManagedEndpointRuntime", () => {
185185
'2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Invalid tunnel secret" connIndex=0',
186186
),
187187
).toBe(true);
188+
// Seen in production on 2026-10-06 after the relay deleted an idle tunnel.
189+
expect(
190+
ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
191+
'2026-10-06T12:00:00Z ERR Register tunnel error from server side error="Unauthorized: Tunnel not found" connIndex=0 event=0 ip=198.41.200.23',
192+
),
193+
).toBe(true);
194+
expect(
195+
ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
196+
'2026-10-06T12:00:00Z ERR Register tunnel error from server side error="Tunnel not found" connIndex=0',
197+
),
198+
).toBe(true);
188199
expect(
189200
ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
190201
'2026-06-17T02:00:00Z ERR Register tunnel error from server side error="connection timed out" connIndex=0',
191202
),
192203
).toBe(false);
204+
expect(
205+
ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
206+
'2026-06-17T02:00:00Z ERR Failed to serve tunnel connection error="Unauthorized: Tunnel not found" connIndex=0',
207+
),
208+
).toBe(false);
193209
});
194210

195211
it.effect("keeps recovery requests sent before the server starts consuming them", () =>
@@ -318,6 +334,81 @@ describe("CloudManagedEndpointRuntime", () => {
318334
}),
319335
);
320336

337+
it.effect("requests recovery while the connector never registers a connection", () =>
338+
Effect.gen(function* () {
339+
const output = yield* Queue.unbounded<Uint8Array>();
340+
const spawner = ChildProcessSpawner.make(() =>
341+
Effect.gen(function* () {
342+
const handle = makeHandle({
343+
pid: 800,
344+
onKill: () => {},
345+
output: Stream.fromQueue(output),
346+
});
347+
yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
348+
return handle;
349+
}),
350+
);
351+
const runtime = yield* buildCloudManagedEndpointRuntime(spawner);
352+
const requests = yield* Queue.unbounded<RelayManagedEndpointRuntimeConfig>();
353+
yield* runtime.recoveryRequests.pipe(
354+
Stream.runForEach((config) => Queue.offer(requests, config)),
355+
Effect.forkChild,
356+
);
357+
const config = {
358+
providerKind: "cloudflare_tunnel" as const,
359+
connectorToken: "token",
360+
tunnelId: "silently-deleted",
361+
};
362+
yield* runtime.applyConfig(config);
363+
364+
yield* TestClock.adjust(Duration.minutes(2));
365+
expect(yield* Queue.size(requests)).toBe(0);
366+
yield* TestClock.adjust(Duration.minutes(1));
367+
expect(yield* Queue.take(requests)).toEqual(config);
368+
// Still unconnected, so it asks again.
369+
yield* TestClock.adjust(Duration.minutes(3));
370+
expect(yield* Queue.take(requests)).toEqual(config);
371+
}).pipe(Effect.provide(TestClock.layer())),
372+
);
373+
374+
it.effect("does not request recovery once the connector has connected", () =>
375+
Effect.gen(function* () {
376+
const output = yield* Queue.unbounded<Uint8Array>();
377+
const spawner = ChildProcessSpawner.make(() =>
378+
Effect.gen(function* () {
379+
const handle = makeHandle({
380+
pid: 801,
381+
onKill: () => {},
382+
output: Stream.fromQueue(output),
383+
});
384+
yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
385+
return handle;
386+
}),
387+
);
388+
const runtime = yield* buildCloudManagedEndpointRuntime(spawner);
389+
const requests = yield* Queue.unbounded<RelayManagedEndpointRuntimeConfig>();
390+
yield* runtime.recoveryRequests.pipe(
391+
Stream.runForEach((config) => Queue.offer(requests, config)),
392+
Effect.forkChild,
393+
);
394+
yield* runtime.applyConfig({
395+
providerKind: "cloudflare_tunnel",
396+
connectorToken: "token",
397+
tunnelId: "tunnel-1",
398+
});
399+
yield* Queue.offer(
400+
output,
401+
new TextEncoder().encode(
402+
"2026-10-06T00:00:00Z INF Registered tunnel connection connIndex=0\n",
403+
),
404+
);
405+
yield* Stream.runHead(runtime.tunnelConnected);
406+
407+
yield* TestClock.adjust(Duration.minutes(10));
408+
expect(yield* Queue.size(requests)).toBe(0);
409+
}).pipe(Effect.provide(TestClock.layer())),
410+
);
411+
321412
it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () =>
322413
Effect.gen(function* () {
323414
const spawned: Array<ChildProcess.StandardCommand> = [];

‎apps/server/src/cloud/ManagedEndpointRuntime.ts‎

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ interface ActiveConnector {
5959
readonly configKey: string;
6060
readonly config: RelayManagedEndpointRuntimeConfig;
6161
readonly startedAtMillis: number;
62+
readonly connected: Ref.Ref<boolean>;
6263
}
6364

6465
// A connector that exits before running this long is treated as part of a
@@ -71,6 +72,11 @@ const RELAY_RESTART_BACKOFF_BASE_MS = 1_000;
7172
const RELAY_RESTART_BACKOFF_MAX_MS = 60_000;
7273
// Newly created tunnels can fail authorization briefly while Cloudflare propagates their token.
7374
const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4;
75+
// A connector that never registers a connection may hold a token for a tunnel
76+
// that no longer exists, rejected in wording the output check does not know.
77+
// Ask for recovery after this long, and again at this interval while it stays
78+
// unconnected; the relay hands back the same tunnel when it is still live.
79+
const CONNECTOR_REGISTRATION_TIMEOUT = Duration.minutes(3);
7480

7581
export function classifyRelayClientOutput(line: string): "connected" | "warning" | "debug" {
7682
if (/\bRegistered tunnel connection\b/iu.test(line)) {
@@ -84,14 +90,16 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning"
8490

8591
/**
8692
* Cloudflare's edge rejects a connector whose tunnel was deleted or whose
87-
* token no longer matches. Current edge output is
88-
* `error="Failed to get tunnel"` with no prefix; older edges prefixed the
89-
* same messages with `Unauthorized:`. Match both so recovery fires on either.
93+
* token no longer matches. The edge words this differently over time
94+
* (`Failed to get tunnel`, `Tunnel not found`, ...), sometimes prefixed with
95+
* `Unauthorized:`. Treat any `Unauthorized:` registration error as a rejection,
96+
* plus the unprefixed messages seen so far. Transient rejections while a new
97+
* tunnel's token propagates are absorbed by requiring several in a row.
9098
*/
9199
export function isRejectedRelayClientTunnelOutput(line: string): boolean {
92100
return (
93101
/\bRegister tunnel error from server side\b/iu.test(line) &&
94-
/error="(?:Unauthorized:\s*)?(?:Failed to get tunnel|Record for tunnel not found|Invalid tunnel secret)"/iu.test(
102+
/error="(?:Unauthorized:[^"]*|Failed to get tunnel|Tunnel not found|Record for tunnel not found|Invalid tunnel secret)"/iu.test(
95103
line,
96104
)
97105
);
@@ -220,6 +228,24 @@ export const make = Effect.gen(function* () {
220228
Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })),
221229
);
222230

231+
// Requests recovery while the connector has not registered a connection,
232+
// once per timeout, until it connects or is replaced.
233+
const watchConnectorRegistration = (connector: ActiveConnector) =>
234+
Effect.gen(function* () {
235+
yield* Effect.sleep(CONNECTOR_REGISTRATION_TIMEOUT);
236+
if (yield* Ref.get(connector.connected)) return true;
237+
yield* Effect.logWarning(
238+
"Relay client has not registered a tunnel connection; requesting recovery",
239+
{
240+
pid: Number(connector.child.pid),
241+
tunnelId: connector.config.tunnelId,
242+
tunnelName: connector.config.tunnelName,
243+
},
244+
);
245+
yield* Queue.offer(recoveryRequests, connector.config);
246+
return false;
247+
}).pipe(Effect.repeat({ until: (connected) => connected }));
248+
223249
const observeConnectorOutput = (connector: ActiveConnector) => {
224250
let rejectedRegistrations = 0;
225251

@@ -239,7 +265,10 @@ export const make = Effect.gen(function* () {
239265
switch (classifyRelayClientOutput(line)) {
240266
case "connected":
241267
rejectedRegistrations = 0;
242-
return Effect.logInfo("Relay client tunnel connection registered", attributes).pipe(
268+
return Ref.set(connector.connected, true).pipe(
269+
Effect.andThen(
270+
Effect.logInfo("Relay client tunnel connection registered", attributes),
271+
),
243272
Effect.andThen(Queue.offer(tunnelConnections, undefined)),
244273
Effect.asVoid,
245274
);
@@ -370,10 +399,12 @@ export const make = Effect.gen(function* () {
370399
configKey: nextConfigKey,
371400
config,
372401
startedAtMillis: yield* Clock.currentTimeMillis,
402+
connected: yield* Ref.make(false),
373403
} satisfies ActiveConnector;
374404
yield* Ref.set(activeRef, connector);
375405
yield* Effect.forkIn(observeConnectorOutput(connector), connectorScope);
376406
yield* Effect.forkIn(superviseConnector(connector), connectorScope);
407+
yield* Effect.forkIn(watchConnectorRegistration(connector), connectorScope);
377408
return {
378409
status: "running",
379410
providerKind: "cloudflare_tunnel",

0 commit comments

Comments
 (0)