Skip to content

Commit b62a17c

Browse files
ishaankoishaanko
committed
fix(connect): reclaim a T3 Connect tunnel routed to another local port
The relay routes a managed tunnel to the origin its environment registered last. When another process with the same environment credentials registers its own port and exits, the server that owns the tunnel keeps a healthy connector whose every request fails with "connection refused" on the dead port. Nothing re-registered until the server restarted. The owner now reads the ingress cloudflared received every minute. When it names an origin other than the one this server listens on, the server registers its origin again. Repeated mismatches back off to 30 minutes so a relay that cannot fix it is not asked every minute.
1 parent 96ffd7a commit b62a17c

5 files changed

Lines changed: 180 additions & 8 deletions

File tree

‎apps/server/src/cloud/ConnectReachability.test.ts‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,13 @@ import { describe, expect, it } from "@effect/vitest";
77
import { classifyReachability, type ReachabilityFacts } from "./ConnectReachability.ts";
88

99
const checkedAt = "2026-10-19T00:00:00.000Z";
10-
const connected = { readyConnections: 4, hostname: "env.example.test", protocol: "quic" };
11-
const notConnected = { readyConnections: 0, hostname: null, protocol: null };
10+
const connected = {
11+
readyConnections: 4,
12+
hostname: "env.example.test",
13+
origin: null,
14+
protocol: "quic",
15+
};
16+
const notConnected = { readyConnections: 0, hostname: null, origin: null, protocol: null };
1217
const facts = (overrides: Partial<ReachabilityFacts>): ReachabilityFacts => ({
1318
managedTunnelConfigured: true,
1419
connector: connected,
@@ -40,7 +45,12 @@ describe("classifyReachability", () => {
4045
expect(
4146
classifyReachability(
4247
facts({
43-
connector: { readyConnections: null, hostname: "env.example.test", protocol: null },
48+
connector: {
49+
readyConnections: null,
50+
hostname: "env.example.test",
51+
origin: null,
52+
protocol: null,
53+
},
4454
endpoint: { kind: "status", status: 502 },
4555
edgeReachable: false,
4656
}),
@@ -51,7 +61,12 @@ describe("classifyReachability", () => {
5161
expect(
5262
classifyReachability(
5363
facts({
54-
connector: { readyConnections: null, hostname: "env.example.test", protocol: null },
64+
connector: {
65+
readyConnections: null,
66+
hostname: "env.example.test",
67+
origin: null,
68+
protocol: null,
69+
},
5570
endpoint: { kind: "dns_failed" },
5671
edgeReachable: false,
5772
}),

‎apps/server/src/cloud/ManagedEndpointRuntime.ts‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ export interface ConnectorFacts {
4848
readonly readyConnections: number | null;
4949
/** The public tunnel hostname from the remote config cloudflared receives once it connects. */
5050
readonly hostname: string | null;
51+
/** The local origin that remote config routes the public hostname to. */
52+
readonly origin: string | null;
5153
readonly protocol: string | null;
5254
}
5355

@@ -107,7 +109,14 @@ const ConnectorReadyBody = Schema.Struct({ readyConnections: Schema.Number });
107109
const ConnectorConfigBody = Schema.Struct({
108110
config: Schema.Struct({
109111
ingress: Schema.optional(
110-
Schema.NullOr(Schema.Array(Schema.Struct({ hostname: Schema.optional(Schema.String) }))),
112+
Schema.NullOr(
113+
Schema.Array(
114+
Schema.Struct({
115+
hostname: Schema.optional(Schema.String),
116+
service: Schema.optional(Schema.String),
117+
}),
118+
),
119+
),
111120
),
112121
}),
113122
});
@@ -766,7 +775,12 @@ export const make = Effect.gen(function* () {
766775
}
767776
const protocol = yield* Ref.get(active.protocol);
768777
if (!(yield* Deferred.isDone(active.metricsAddress))) {
769-
return { readyConnections: null, hostname: null, protocol } satisfies ConnectorFacts;
778+
return {
779+
readyConnections: null,
780+
hostname: null,
781+
origin: null,
782+
protocol,
783+
} satisfies ConnectorFacts;
770784
}
771785
const metricsUrl = `http://${yield* Deferred.await(active.metricsAddress)}`;
772786
const [ready, config] = yield* Effect.all(
@@ -776,9 +790,11 @@ export const make = Effect.gen(function* () {
776790
],
777791
{ concurrency: 2 },
778792
);
793+
const hostnameRule = config?.config.ingress?.find((rule) => rule.hostname);
779794
return {
780795
readyConnections: ready?.readyConnections ?? null,
781-
hostname: config?.config.ingress?.find((rule) => rule.hostname)?.hostname ?? null,
796+
hostname: hostnameRule?.hostname ?? null,
797+
origin: hostnameRule?.service ?? null,
782798
protocol,
783799
} satisfies ConnectorFacts;
784800
});

‎apps/server/src/cloud/managedTunnelStartup.test.ts‎

Lines changed: 79 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
import { describe, expect, it } from "@effect/vitest";
22
import * as Effect from "effect/Effect";
3+
import * as Fiber from "effect/Fiber";
4+
import * as TestClock from "effect/testing/TestClock";
35

4-
import { retryManagedTunnelRegistration } from "./managedTunnelStartup.ts";
6+
import {
7+
retryManagedTunnelRegistration,
8+
watchManagedTunnelOrigin,
9+
} from "./managedTunnelStartup.ts";
510

611
describe("retryManagedTunnelRegistration", () => {
712
it.effect("does not fall back or retry when registration is permanently rejected", () =>
@@ -26,3 +31,76 @@ describe("retryManagedTunnelRegistration", () => {
2631
}),
2732
);
2833
});
34+
35+
describe("watchManagedTunnelOrigin", () => {
36+
const localOrigin = "http://127.0.0.1:3773";
37+
38+
/** Runs the watcher for `minutes`, serving each check the next ingress in `ingress`. */
39+
const watch = (input: {
40+
readonly ingress: ReadonlyArray<string | null>;
41+
readonly minutes: number;
42+
readonly failRegistration?: boolean;
43+
}) =>
44+
Effect.gen(function* () {
45+
let checks = 0;
46+
const registeredAtMinute: Array<number> = [];
47+
let minute = 0;
48+
const fiber = yield* watchManagedTunnelOrigin({
49+
localOrigin,
50+
readIngressOrigin: Effect.sync(
51+
() => input.ingress[Math.min(checks++, input.ingress.length - 1)] ?? null,
52+
),
53+
register: Effect.suspend(() => {
54+
registeredAtMinute.push(minute);
55+
return input.failRegistration ? Effect.fail("relay unavailable") : Effect.void;
56+
}),
57+
}).pipe(Effect.forkChild);
58+
for (; minute < input.minutes;) {
59+
minute += 1;
60+
yield* TestClock.adjust("1 minute");
61+
}
62+
yield* Fiber.interrupt(fiber);
63+
return { checks, registeredAtMinute };
64+
});
65+
66+
it.effect("registers this server again when the ingress names another local port", () =>
67+
Effect.gen(function* () {
68+
const result = yield* watch({
69+
ingress: ["http://127.0.0.1:46169", localOrigin],
70+
minutes: 5,
71+
});
72+
expect(result.registeredAtMinute).toEqual([1]);
73+
}),
74+
);
75+
76+
it.effect("leaves a matching or unknown ingress alone", () =>
77+
Effect.gen(function* () {
78+
const result = yield* watch({
79+
ingress: [null, `${localOrigin}/`, localOrigin],
80+
minutes: 5,
81+
});
82+
expect(result.checks).toBe(5);
83+
expect(result.registeredAtMinute).toEqual([]);
84+
}),
85+
);
86+
87+
it.effect("keeps checking after a failed registration", () =>
88+
Effect.gen(function* () {
89+
const result = yield* watch({
90+
ingress: ["http://127.0.0.1:46169", "http://127.0.0.1:46169", localOrigin],
91+
minutes: 8,
92+
failRegistration: true,
93+
});
94+
expect(result.registeredAtMinute).toEqual([1, 3]);
95+
// Minutes 1, 3, and 7; the match at 7 restores the one-minute interval.
96+
expect(result.checks).toBe(4);
97+
}),
98+
);
99+
100+
it.effect("backs off while the relay keeps routing elsewhere", () =>
101+
Effect.gen(function* () {
102+
const result = yield* watch({ ingress: ["http://127.0.0.1:46169"], minutes: 20 });
103+
expect(result.registeredAtMinute).toEqual([1, 3, 7, 15]);
104+
}),
105+
);
106+
});

‎apps/server/src/cloud/managedTunnelStartup.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
2+
import * as Cause from "effect/Cause";
23
import * as Clock from "effect/Clock";
34
import * as Duration from "effect/Duration";
45
import * as Effect from "effect/Effect";
@@ -216,3 +217,57 @@ export const makeManagedTunnelRecovery = Effect.fn("ManagedTunnelRecovery.make")
216217
),
217218
);
218219
});
220+
221+
const MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL = Duration.minutes(1);
222+
223+
const sameOrigin = (left: string, right: string) => {
224+
try {
225+
return new URL(left).origin === new URL(right).origin;
226+
} catch {
227+
return false;
228+
}
229+
};
230+
231+
/**
232+
* The relay routes the tunnel to the origin this environment registered last.
233+
* Only the server that owns the T3 home runs the connector, so an ingress that
234+
* names any other origin is stale: a process with this environment's
235+
* credentials registered its own port, then went away, and every request now
236+
* fails with connection refused. Checks the connector's ingress every minute
237+
* and registers this server's origin again when it points elsewhere. While the
238+
* mismatch persists, each attempt doubles the wait, up to 30 minutes, so a
239+
* relay that cannot fix it is not asked every minute.
240+
*/
241+
export const watchManagedTunnelOrigin = Effect.fn("ManagedTunnelOrigin.watch")(function* <
242+
E,
243+
R,
244+
>(options: {
245+
readonly localOrigin: string;
246+
/** The service URL cloudflared routes the public hostname to, or `null` while unknown. */
247+
readonly readIngressOrigin: Effect.Effect<string | null>;
248+
readonly register: Effect.Effect<unknown, E, R>;
249+
}) {
250+
let wait = MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL;
251+
while (true) {
252+
yield* Effect.sleep(wait);
253+
const ingressOrigin = yield* options.readIngressOrigin;
254+
if (ingressOrigin === null || sameOrigin(ingressOrigin, options.localOrigin)) {
255+
wait = MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL;
256+
continue;
257+
}
258+
yield* Effect.logWarning(
259+
"T3 Connect routes this server's tunnel to another origin; registering this server again",
260+
{ ingressOrigin, localOrigin: options.localOrigin },
261+
);
262+
yield* options.register.pipe(
263+
Effect.catchCause((cause) =>
264+
Cause.hasInterrupts(cause)
265+
? Effect.interrupt
266+
: Effect.logWarning("Failed to register this server's T3 Connect origin again", {
267+
cause,
268+
}),
269+
),
270+
);
271+
wait = Duration.min(Duration.times(wait, 2), MANAGED_TUNNEL_RECOVERY_MAX_COOLDOWN);
272+
}
273+
});

‎apps/server/src/server.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ import {
151151
managedTunnelStartupAction,
152152
retryDesiredLinkReconcile,
153153
retryManagedTunnelRegistration,
154+
watchManagedTunnelOrigin,
154155
} from "./cloud/managedTunnelStartup.ts";
155156
import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts";
156157
import * as CloudCliState from "./cloud/CliState.ts";
@@ -893,6 +894,13 @@ const layerMakeServer = Layer.unwrap(
893894
Stream.runForEach(recoverManagedTunnel),
894895
Effect.forkScoped,
895896
);
897+
yield* watchManagedTunnelOrigin({
898+
localOrigin,
899+
readIngressOrigin: endpointRuntime.inspectConnector.pipe(
900+
Effect.map((connector) => connector?.origin ?? null),
901+
),
902+
register: cloudLink.registerManagedTunnelRecovery(localOrigin),
903+
}).pipe(Effect.forkScoped);
896904
const wantsCliLink = hasCloudPublicConfig
897905
? yield* CloudCliState.readCliDesiredCloudLink.pipe(
898906
Effect.catch((cause) =>

0 commit comments

Comments
 (0)