Skip to content

Commit 101f8b2

Browse files
t3dotggclaude
andauthored
fix(server): relay client updates no longer drop the host off T3 Connect (pingdotgg#17366)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 43f8a8d commit 101f8b2

3 files changed

Lines changed: 123 additions & 23 deletions

File tree

‎apps/server/src/cloud/ManagedEndpointRuntime.test.ts‎

Lines changed: 82 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -851,7 +851,7 @@ describe("CloudManagedEndpointRuntime", () => {
851851
);
852852

853853
it.effect(
854-
"runs an older relay client while the pinned one installs, then restarts on it and prunes",
854+
"runs an older relay client while the pinned one installs, then hands off to it and prunes",
855855
() =>
856856
Effect.gen(function* () {
857857
const olderClient = {
@@ -870,15 +870,19 @@ describe("CloudManagedEndpointRuntime", () => {
870870
const releaseInstall = yield* Deferred.make<void>();
871871
const pruned = yield* Deferred.make<void>();
872872
const spawned = yield* Queue.unbounded<string>();
873-
const killed: Array<string> = [];
873+
const olderStopped = yield* Deferred.make<void>();
874874
const output = yield* Queue.unbounded<Uint8Array>();
875875
const spawner = ChildProcessSpawner.make((command) =>
876876
Effect.gen(function* () {
877877
const executable = ChildProcess.isStandardCommand(command) ? command.command : "";
878878
yield* Queue.offer(spawned, executable);
879879
const handle = makeHandle({
880880
pid: executable === pinnedClient.executablePath ? 2 : 1,
881-
onKill: () => killed.push(executable),
881+
onKill: () => {
882+
if (executable === olderClient.executablePath) {
883+
Deferred.doneUnsafe(olderStopped, Effect.void);
884+
}
885+
},
882886
...(executable === pinnedClient.executablePath
883887
? { output: Stream.fromQueue(output) }
884888
: {}),
@@ -916,14 +920,16 @@ describe("CloudManagedEndpointRuntime", () => {
916920

917921
yield* Deferred.succeed(releaseInstall, undefined);
918922
expect(yield* Queue.take(spawned)).toBe(pinnedClient.executablePath);
919-
expect(killed).toEqual([olderClient.executablePath]);
923+
// The older connector keeps serving until the pinned one registers.
924+
expect(Option.isNone(yield* Deferred.poll(olderStopped))).toBe(true);
920925

921926
yield* Queue.offer(
922927
output,
923928
new TextEncoder().encode(
924929
"2026-10-08T00:00:00Z INF Registered tunnel connection connIndex=0\n",
925930
),
926931
);
932+
yield* Deferred.await(olderStopped);
927933
yield* Deferred.await(pruned);
928934
}),
929935
);
@@ -1118,7 +1124,7 @@ describe("CloudManagedEndpointRuntime", () => {
11181124
}),
11191125
);
11201126

1121-
it.effect("requests recovery when the updated relay client fails to start", () =>
1127+
it.effect("keeps the older connector when the updated relay client fails to start", () =>
11221128
Effect.gen(function* () {
11231129
const olderClient = {
11241130
status: "available",
@@ -1133,18 +1139,26 @@ describe("CloudManagedEndpointRuntime", () => {
11331139
version: RelayClient.CLOUDFLARED_VERSION,
11341140
} as const;
11351141
let installed = false;
1142+
let olderKilled = false;
11361143
const releaseInstall = yield* Deferred.make<void>();
1144+
const pinnedAttempted = yield* Deferred.make<void>();
11371145
const spawner = ChildProcessSpawner.make((command) =>
11381146
Effect.gen(function* () {
11391147
const executable = ChildProcess.isStandardCommand(command) ? command.command : "";
11401148
if (executable === pinnedClient.executablePath) {
1149+
yield* Deferred.succeed(pinnedAttempted, undefined);
11411150
return yield* PlatformError.systemError({
11421151
_tag: "PermissionDenied",
11431152
module: "ChildProcess",
11441153
method: "spawn",
11451154
});
11461155
}
1147-
const handle = makeHandle({ pid: 6, onKill: () => {} });
1156+
const handle = makeHandle({
1157+
pid: 6,
1158+
onKill: () => {
1159+
olderKilled = true;
1160+
},
1161+
});
11481162
yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
11491163
return handle;
11501164
}),
@@ -1171,6 +1185,68 @@ describe("CloudManagedEndpointRuntime", () => {
11711185
const config = { providerKind: "cloudflare_tunnel", connectorToken: "token" } as const;
11721186
expect(yield* runtime.applyConfig(config)).toMatchObject({ status: "running", pid: 6 });
11731187

1188+
yield* Deferred.succeed(releaseInstall, undefined);
1189+
yield* Deferred.await(pinnedAttempted);
1190+
// The failed swap leaves the older connector active and running.
1191+
expect(yield* runtime.applyConfig(config)).toMatchObject({ status: "running", pid: 6 });
1192+
expect(olderKilled).toBe(false);
1193+
}),
1194+
);
1195+
1196+
it.effect("requests recovery when the older connector exited during a failed swap", () =>
1197+
Effect.gen(function* () {
1198+
const olderClient = {
1199+
status: "available",
1200+
executablePath: "/managed/2025.9.0/cloudflared",
1201+
source: "managed",
1202+
version: "2025.9.0",
1203+
} as const;
1204+
const pinnedClient = {
1205+
status: "available",
1206+
executablePath: "/managed/pinned/cloudflared",
1207+
source: "managed",
1208+
version: RelayClient.CLOUDFLARED_VERSION,
1209+
} as const;
1210+
let installed = false;
1211+
const releaseInstall = yield* Deferred.make<void>();
1212+
const spawner = ChildProcessSpawner.make((command) =>
1213+
Effect.gen(function* () {
1214+
const executable = ChildProcess.isStandardCommand(command) ? command.command : "";
1215+
if (executable === pinnedClient.executablePath) {
1216+
return yield* PlatformError.systemError({
1217+
_tag: "PermissionDenied",
1218+
module: "ChildProcess",
1219+
method: "spawn",
1220+
});
1221+
}
1222+
// The older connector has exited by the time the swap fails.
1223+
const handle = makeHandle({ pid: 12, onKill: () => {}, isRunning: () => !installed });
1224+
yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
1225+
return handle;
1226+
}),
1227+
);
1228+
const runtime = yield* buildCloudManagedEndpointRuntime(
1229+
spawner,
1230+
Layer.succeed(
1231+
RelayClient.RelayClient,
1232+
RelayClient.RelayClient.of({
1233+
resolve: Effect.sync(() => (installed ? pinnedClient : olderClient)),
1234+
install: Deferred.await(releaseInstall).pipe(
1235+
Effect.andThen(
1236+
Effect.sync(() => {
1237+
installed = true;
1238+
return pinnedClient;
1239+
}),
1240+
),
1241+
),
1242+
installWithProgress: () => Effect.die("unused"),
1243+
pruneManagedVersions: Effect.void,
1244+
}),
1245+
),
1246+
);
1247+
const config = { providerKind: "cloudflare_tunnel", connectorToken: "token" } as const;
1248+
expect(yield* runtime.applyConfig(config)).toMatchObject({ status: "running", pid: 12 });
1249+
11741250
yield* Deferred.succeed(releaseInstall, undefined);
11751251
expect(Option.getOrThrow(yield* Stream.runHead(runtime.recoveryRequests))).toEqual(config);
11761252
}),

‎apps/server/src/cloud/ManagedEndpointRuntime.ts‎

Lines changed: 36 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay
22
import * as RelayClient from "@t3tools/shared/relayClient";
33
import * as Clock from "effect/Clock";
44
import * as Context from "effect/Context";
5+
import * as Deferred from "effect/Deferred";
56
import * as Duration from "effect/Duration";
67
import * as Effect from "effect/Effect";
78
import * as Exit from "effect/Exit";
@@ -60,7 +61,8 @@ interface ActiveConnector {
6061
readonly configKey: string;
6162
readonly config: RelayManagedEndpointRuntimeConfig;
6263
readonly startedAtMillis: number;
63-
readonly connected: Ref.Ref<boolean>;
64+
/** Completes when the connector first registers a tunnel connection. */
65+
readonly registered: Deferred.Deferred<void>;
6466
}
6567

6668
// A connector that exits before running this long is treated as part of a
@@ -252,9 +254,10 @@ export const make = Effect.gen(function* () {
252254
Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })),
253255
);
254256

255-
// Installs the pinned relay client in the background, then restarts the
256-
// connector on it. The running connector, if any, keeps serving until the new
257-
// binary is installed and validated, so an update costs one brief reconnect.
257+
// Installs the pinned relay client in the background, then moves the connector
258+
// to it. The new connector starts next to the old one on the same tunnel, and
259+
// the old one stops only once the new one registers, so clients never see the
260+
// relay lose this host during the swap.
258261
const installPinnedRelayClient = Effect.gen(function* () {
259262
if (yield* Ref.getAndSet(installInFlightRef, true)) return;
260263
yield* Effect.logInfo("Installing the pinned relay client", {
@@ -286,17 +289,37 @@ export const make = Effect.gen(function* () {
286289
) {
287290
return;
288291
}
289-
yield* Effect.logInfo("Relay client installed; restarting the connector on it", {
292+
yield* Effect.logInfo("Relay client installed; moving the connector to it", {
290293
version: installed.version,
291294
previousVersion: active?.executable.version,
292295
});
293-
yield* stopActive;
296+
// Detach the old connector so the reconcile starts a new one beside it.
297+
const previous = yield* Ref.getAndSet(activeRef, null);
294298
const status = yield* reconcileConfig(desiredConfig);
295-
// The old connector is gone and no supervisor watches a failed spawn,
296-
// so ask for recovery like an exited connector would.
297-
if (status.status === "failed") {
299+
const next = yield* Ref.get(activeRef);
300+
if (!next) {
298301
yield* Effect.logWarning("Relay client did not start after the update", status);
299-
yield* Queue.offer(recoveryRequests, desiredConfig);
302+
// A still-running old connector keeps serving and the retry loop
303+
// tries the pin again. One that exited while detached was skipped
304+
// by its supervisor, so ask for recovery like an exited connector.
305+
const previousRunning = previous
306+
? yield* previous.child.isRunning.pipe(Effect.orElseSucceed(() => false))
307+
: false;
308+
if (previous && previousRunning) {
309+
yield* Ref.set(activeRef, previous);
310+
} else {
311+
yield* stopConnector(previous);
312+
yield* Queue.offer(recoveryRequests, desiredConfig);
313+
}
314+
return;
315+
}
316+
// Runs in the new connector's scope: the old one stops when the new
317+
// one registers, or as soon as the new one is stopped or replaced.
318+
if (previous) {
319+
yield* Deferred.await(next.registered).pipe(
320+
Effect.ensuring(stopConnector(previous)),
321+
Effect.forkIn(next.scope),
322+
);
300323
}
301324
}),
302325
),
@@ -357,7 +380,7 @@ export const make = Effect.gen(function* () {
357380
const watchConnectorRegistration = (connector: ActiveConnector) =>
358381
Effect.gen(function* () {
359382
yield* Effect.sleep(CONNECTOR_REGISTRATION_TIMEOUT);
360-
if (yield* Ref.get(connector.connected)) return true;
383+
if (yield* Deferred.isDone(connector.registered)) return true;
361384
yield* Effect.logWarning(
362385
"Relay client has not registered a tunnel connection; requesting recovery",
363386
{
@@ -389,7 +412,7 @@ export const make = Effect.gen(function* () {
389412
switch (classifyRelayClientOutput(line)) {
390413
case "connected":
391414
rejectedRegistrations = 0;
392-
return Ref.set(connector.connected, true).pipe(
415+
return Deferred.succeed(connector.registered, undefined).pipe(
393416
Effect.andThen(
394417
Effect.logInfo("Relay client tunnel connection registered", attributes),
395418
),
@@ -539,7 +562,7 @@ export const make = Effect.gen(function* () {
539562
configKey: nextConfigKey,
540563
config,
541564
startedAtMillis: yield* Clock.currentTimeMillis,
542-
connected: yield* Ref.make(false),
565+
registered: yield* Deferred.make<void>(),
543566
} satisfies ActiveConnector;
544567
yield* Ref.set(activeRef, connector);
545568
yield* Effect.forkIn(observeConnectorOutput(connector), connectorScope);

‎docs/internals/t3-connect.md‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -136,10 +136,11 @@ The host runs `cloudflared` pinned by `CLOUDFLARED_VERSION` in
136136
[`relayClient.ts`](../../packages/shared/src/relayClient.ts), and bumping it there
137137
(version, URLs, and checksums) is the whole release step. A linked host that
138138
starts a server with a new pin keeps its connector up on the newest older managed
139-
release, installs the pinned one in the background, then restarts only the
140-
connector child on it. If that restart fails to spawn, the host requests recovery
141-
as it would for an exited connector. A host with no relay client installs one
142-
first. A `cloudflared` on `PATH` or an explicit `T3CODE_CLOUDFLARED_PATH` is the
139+
release, installs the pinned one in the background, then starts a second
140+
connector on it for the same tunnel. The old connector stops only after the new
141+
one registers, so the swap never takes the host off the relay. If the new one
142+
fails to spawn, the old one keeps serving and the install retries later. A host
143+
with no relay client installs one first. A `cloudflared` on `PATH` or an explicit `T3CODE_CLOUDFLARED_PATH` is the
143144
user's choice and is never replaced, since linking asks before downloading.
144145

145146
Once the pinned release registers a connection, managed releases older than the

0 commit comments

Comments
 (0)