Skip to content

Commit 8d7b5e9

Browse files
t3dotggclaudejuliusmarminge
authored
fix(connect): remove tunnels after hosts go offline (pingdotgg#9386)
Co-authored-by: Claude Fable 5.1 <[email protected]> Co-authored-by: Julius Marminge <[email protected]> Co-authored-by: Julius Marminge <[email protected]>
1 parent d4a3345 commit 8d7b5e9

36 files changed

Lines changed: 7563 additions & 388 deletions

‎.github/workflows/deploy-relay.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ jobs:
2828
RELAY_DOMAIN: ${{ vars.RELAY_DOMAIN }}
2929
RELAY_API_ZONE_NAME: ${{ vars.RELAY_API_ZONE_NAME }}
3030
RELAY_TUNNEL_ZONE_NAME: ${{ vars.RELAY_TUNNEL_ZONE_NAME }}
31+
RELAY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_TUNNEL_CLEANUP_MODE }}
3132
CLERK_PUBLISHABLE_KEY: ${{ vars.CLERK_PUBLISHABLE_KEY }}
3233
CLERK_JWT_AUDIENCE: ${{ vars.CLERK_JWT_AUDIENCE }}
3334
APNS_ENVIRONMENT: ${{ vars.APNS_ENVIRONMENT }}

‎apps/server/src/cloud/CliState.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
88
import { ServerConfig } from "../config.ts";
99
import * as CliState from "./CliState.ts";
1010
import {
11+
CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
1112
CLOUD_ENDPOINT_RUNTIME_CONFIG,
1213
CLOUD_LINKED_USER_ID,
1314
CLOUD_MINT_PUBLIC_KEY,
@@ -24,6 +25,7 @@ const persistedCloudLinkSecrets = [
2425
RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
2526
CLOUD_MINT_PUBLIC_KEY,
2627
CLOUD_ENDPOINT_RUNTIME_CONFIG,
28+
CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
2729
PUBLISH_AGENT_ACTIVITY_SECRET,
2830
] as const;
2931

‎apps/server/src/cloud/CliState.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import * as Option from "effect/Option";
33

44
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
55
import {
6+
CLOUD_ENDPOINT_CONFIRMED_ORIGIN,
67
CLOUD_ENDPOINT_RUNTIME_CONFIG,
78
CLOUD_LINKED_USER_ID,
89
CLOUD_MINT_PUBLIC_KEY,
@@ -67,6 +68,7 @@ export const clearPersistedCloudLink = Effect.gen(function* () {
6768
secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
6869
secrets.remove(CLOUD_MINT_PUBLIC_KEY),
6970
secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
71+
secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
7072
secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
7173
],
7274
{ concurrency: "unbounded" },

‎apps/server/src/cloud/ManagedEndpointRuntime.test.ts‎

Lines changed: 214 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@ import * as Fiber from "effect/Fiber";
77
import * as Layer from "effect/Layer";
88
import * as Option from "effect/Option";
99
import * as PlatformError from "effect/PlatformError";
10+
import * as Queue from "effect/Queue";
1011
import * as Sink from "effect/Sink";
1112
import * as Stream from "effect/Stream";
1213
import * as TestClock from "effect/testing/TestClock";
1314
import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
15+
import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
1416
import * as RelayClient from "@t3tools/shared/relayClient";
1517

1618
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
@@ -62,6 +64,7 @@ function makeHandle(input: {
6264
readonly onKill: () => void;
6365
readonly isRunning?: () => boolean;
6466
readonly exitCode?: Effect.Effect<ChildProcessSpawner.ExitCode>;
67+
readonly output?: Stream.Stream<Uint8Array>;
6568
}) {
6669
return ChildProcessSpawner.makeHandle({
6770
pid: ChildProcessSpawner.ProcessId(input.pid),
@@ -75,13 +78,70 @@ function makeHandle(input: {
7578
stdin: Sink.drain,
7679
stdout: Stream.empty,
7780
stderr: Stream.empty,
78-
all: Stream.empty,
81+
all: input.output ?? Stream.empty,
7982
getInputFd: () => Sink.drain,
8083
getOutputFd: () => Stream.empty,
8184
});
8285
}
8386

8487
describe("CloudManagedEndpointRuntime", () => {
88+
it("retries connector startup failures but stops for unsupported runtimes", () => {
89+
expect(
90+
ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({
91+
status: "failed",
92+
failure: "not-installed",
93+
reason: "The relay client is not installed.",
94+
}),
95+
).toBe(true);
96+
expect(
97+
ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({
98+
status: "failed",
99+
failure: "spawn-failed",
100+
reason: "spawn failed",
101+
}),
102+
).toBe(true);
103+
expect(
104+
ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({
105+
status: "failed",
106+
failure: "unsupported-platform",
107+
reason: "Relay client is unsupported on linux-arm.",
108+
}),
109+
).toBe(false);
110+
expect(
111+
ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ status: "unsupported" }),
112+
).toBe(false);
113+
});
114+
115+
it.effect("serializes updates to persisted cloud link state", () =>
116+
Effect.gen(function* () {
117+
const firstEntered = yield* Deferred.make<void>();
118+
const releaseFirst = yield* Deferred.make<void>();
119+
const secondEntered = yield* Deferred.make<void>();
120+
const runtime = yield* buildCloudManagedEndpointRuntime(
121+
ChildProcessSpawner.make(() => Effect.die("unused")),
122+
);
123+
124+
const first = yield* runtime
125+
.withLinkStateLock(
126+
Deferred.succeed(firstEntered, undefined).pipe(
127+
Effect.andThen(Deferred.await(releaseFirst)),
128+
),
129+
)
130+
.pipe(Effect.forkChild);
131+
yield* Deferred.await(firstEntered);
132+
133+
const second = yield* runtime
134+
.withLinkStateLock(Deferred.succeed(secondEntered, undefined))
135+
.pipe(Effect.forkChild);
136+
expect(yield* Deferred.isDone(secondEntered)).toBe(false);
137+
138+
yield* Deferred.succeed(releaseFirst, undefined);
139+
yield* Fiber.join(first);
140+
yield* Fiber.join(second);
141+
expect(yield* Deferred.isDone(secondEntered)).toBe(true);
142+
}),
143+
);
144+
85145
it("classifies Cloudflare connection and warning output", () => {
86146
expect(
87147
ManagedEndpointRuntime.classifyRelayClientOutput(
@@ -109,6 +169,125 @@ describe("CloudManagedEndpointRuntime", () => {
109169
).toBe("warning");
110170
});
111171

172+
it("recognizes tunnel authorization failures without matching ordinary transport errors", () => {
173+
expect(
174+
ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
175+
'2026-09-15T06:30:43Z ERR Register tunnel error from server side error="Failed to get tunnel" connIndex=0 event=0 ip=198.41.200.23',
176+
),
177+
).toBe(true);
178+
expect(
179+
ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
180+
'2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Record for tunnel not found" connIndex=0',
181+
),
182+
).toBe(true);
183+
expect(
184+
ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
185+
'2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Invalid tunnel secret" connIndex=0',
186+
),
187+
).toBe(true);
188+
expect(
189+
ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput(
190+
'2026-06-17T02:00:00Z ERR Register tunnel error from server side error="connection timed out" connIndex=0',
191+
),
192+
).toBe(false);
193+
});
194+
195+
it.effect("keeps recovery requests sent before the server starts consuming them", () =>
196+
Effect.gen(function* () {
197+
const runtime = yield* buildCloudManagedEndpointRuntime(
198+
ChildProcessSpawner.make(() => Effect.die("unused")),
199+
);
200+
const config = {
201+
providerKind: "cloudflare_tunnel" as const,
202+
connectorToken: "token",
203+
tunnelId: "tunnel-1",
204+
};
205+
206+
yield* runtime.requestRecovery(config);
207+
208+
expect(Option.getOrNull(yield* Stream.runHead(runtime.recoveryRequests))).toEqual(config);
209+
}),
210+
);
211+
212+
it.effect("recovers a rejected tunnel without waiting for the connector to exit", () =>
213+
Effect.gen(function* () {
214+
const output = yield* Queue.unbounded<Uint8Array>();
215+
const firstBatchObserved = yield* Deferred.make<void>();
216+
const secondBatchObserved = yield* Deferred.make<void>();
217+
const recoveryRequested = yield* Deferred.make<RelayManagedEndpointRuntimeConfig>();
218+
const recoveryRetried = yield* Deferred.make<RelayManagedEndpointRuntimeConfig>();
219+
let recoveryRequestCount = 0;
220+
const spawned: Array<number> = [];
221+
const encoder = new TextEncoder();
222+
const connectorOutput = Stream.fromQueue(output).pipe(
223+
Stream.tap((chunk) => {
224+
const line = new TextDecoder().decode(chunk);
225+
if (line === "first checkpoint\n") {
226+
return Deferred.succeed(firstBatchObserved, undefined).pipe(Effect.asVoid);
227+
}
228+
if (line === "second checkpoint\n") {
229+
return Deferred.succeed(secondBatchObserved, undefined).pipe(Effect.asVoid);
230+
}
231+
return Effect.void;
232+
}),
233+
);
234+
const spawner = ChildProcessSpawner.make(() =>
235+
Effect.gen(function* () {
236+
const pid = 600;
237+
spawned.push(pid);
238+
const handle = makeHandle({ pid, onKill: () => {}, output: connectorOutput });
239+
yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
240+
return handle;
241+
}),
242+
);
243+
const runtime = yield* buildCloudManagedEndpointRuntime(spawner);
244+
const config = {
245+
providerKind: "cloudflare_tunnel" as const,
246+
connectorToken: "token",
247+
tunnelId: "deleted-tunnel",
248+
};
249+
const rejectedLine =
250+
'2026-09-15T06:30:43Z ERR Register tunnel error from server side error="Failed to get tunnel" connIndex=0 event=0 ip=198.41.200.23\n';
251+
252+
yield* runtime.recoveryRequests.pipe(
253+
Stream.runForEach((requested) => {
254+
recoveryRequestCount += 1;
255+
return Deferred.succeed(
256+
recoveryRequestCount === 1 ? recoveryRequested : recoveryRetried,
257+
requested,
258+
).pipe(Effect.asVoid);
259+
}),
260+
Effect.forkChild,
261+
);
262+
yield* runtime.applyConfig(config);
263+
264+
yield* Queue.offer(output, encoder.encode(rejectedLine.repeat(3)));
265+
yield* Queue.offer(output, encoder.encode("first checkpoint\n"));
266+
yield* Deferred.await(firstBatchObserved);
267+
expect(yield* Deferred.isDone(recoveryRequested)).toBe(false);
268+
269+
yield* Queue.offer(
270+
output,
271+
encoder.encode(
272+
"2026-06-17T02:00:00Z INF Registered tunnel connection connIndex=0\n" +
273+
rejectedLine.repeat(3),
274+
),
275+
);
276+
yield* Queue.offer(output, encoder.encode("second checkpoint\n"));
277+
yield* Deferred.await(secondBatchObserved);
278+
expect(yield* Deferred.isDone(recoveryRequested)).toBe(false);
279+
280+
yield* Queue.offer(output, encoder.encode(rejectedLine));
281+
282+
expect(yield* Deferred.await(recoveryRequested)).toEqual(config);
283+
284+
yield* Queue.offer(output, encoder.encode(rejectedLine.repeat(4)));
285+
286+
expect(yield* Deferred.await(recoveryRetried)).toEqual(config);
287+
expect(spawned).toEqual([600]);
288+
}),
289+
);
290+
112291
it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () =>
113292
Effect.gen(function* () {
114293
const spawned: Array<ChildProcess.StandardCommand> = [];
@@ -156,8 +335,8 @@ describe("CloudManagedEndpointRuntime", () => {
156335

157336
expect(spawned.map((command) => command.command)).toEqual(["cloudflared", "cloudflared"]);
158337
expect(spawned.map((command) => command.args)).toEqual([
159-
["tunnel", "run"],
160-
["tunnel", "run"],
338+
["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"],
339+
["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"],
161340
]);
162341
expect(spawned.map((command) => command.options.env?.TUNNEL_TOKEN)).toEqual([
163342
"token-1",
@@ -378,6 +557,37 @@ describe("CloudManagedEndpointRuntime", () => {
378557
}).pipe(Effect.provide(TestClock.layer())),
379558
);
380559

560+
it.effect("a recovery that returns the same config keeps the crash backoff", () =>
561+
Effect.gen(function* () {
562+
const { spawner, spawned, exits, spawnSignals } = yield* makeCrashLoopSpawner(900, 4);
563+
const runtime = yield* buildCloudManagedEndpointRuntime(spawner);
564+
const config = {
565+
providerKind: "cloudflare_tunnel" as const,
566+
connectorToken: "same-token",
567+
tunnelId: "same-tunnel",
568+
};
569+
// The startup consumer re-applies whatever the relay hands back. When the
570+
// relay confirms the current tunnel, that must not look like a config change.
571+
yield* runtime.recoveryRequests.pipe(
572+
Stream.runForEach((requested) => runtime.applyConfig(requested).pipe(Effect.asVoid)),
573+
Effect.forkChild,
574+
);
575+
576+
yield* runtime.applyConfig(config);
577+
yield* Deferred.succeed(exits[0]!, ChildProcessSpawner.ExitCode(1));
578+
yield* Deferred.await(spawnSignals[1]!);
579+
expect(spawned).toEqual([900, 901]);
580+
581+
// Second rapid crash still waits out the base delay.
582+
yield* Deferred.succeed(exits[1]!, ChildProcessSpawner.ExitCode(1));
583+
yield* TestClock.adjust(Duration.millis(999));
584+
expect(spawned).toEqual([900, 901]);
585+
yield* TestClock.adjust(Duration.millis(1));
586+
yield* Deferred.await(spawnSignals[2]!);
587+
expect(spawned).toEqual([900, 901, 902]);
588+
}).pipe(Effect.provide(TestClock.layer())),
589+
);
590+
381591
it.effect("an explicit config change clears the backoff and preempts a delayed restart", () =>
382592
Effect.gen(function* () {
383593
const { spawner, spawned, exits, spawnSignals } = yield* makeCrashLoopSpawner(800, 3);
@@ -510,6 +720,7 @@ describe("CloudManagedEndpointRuntime", () => {
510720
expect(status).toEqual({
511721
status: "failed",
512722
providerKind: "cloudflare_tunnel",
723+
failure: "not-installed",
513724
reason: "The relay client is not installed.",
514725
});
515726
expect(spawn).not.toHaveBeenCalled();

0 commit comments

Comments
 (0)