Skip to content

Commit c192c43

Browse files
shivamhwpPR Batch Testerclaudemaria-rcks
authored
fix(server): restart T3 Connect's tunnel client when Cloudflare drops it (pingdotgg#24946)
Co-authored-by: PR Batch Tester <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]> Co-authored-by: maria-rcks <[email protected]>
1 parent b9522c8 commit c192c43

6 files changed

Lines changed: 531 additions & 22 deletions

File tree

‎apps/server/src/cloud/CloudLink.lifecycle.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
206206
}),
207207
recoveryRequests: Stream.empty,
208208
requestRecovery: () => Effect.void,
209+
restartConnector: () => Effect.succeed(false),
209210
tunnelConnected: Stream.empty,
210211
inspectConnector: Effect.succeed(null),
211212
readinessChanged: Stream.empty,

‎apps/server/src/cloud/ConnectReachability.test.ts‎

Lines changed: 308 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,32 @@
11
// @effect-diagnostics nodeBuiltinImport:off - reads the user guide to check its anchors.
22
import * as NodeFS from "node:fs";
3+
import * as NodeNet from "node:net";
34

45
import { T3_CONNECT_TROUBLESHOOTING_SECTIONS } from "@t3tools/contracts";
56
import { describe, expect, it } from "@effect/vitest";
7+
import * as Clock from "effect/Clock";
8+
import * as Deferred from "effect/Deferred";
9+
import * as Duration from "effect/Duration";
10+
import * as Effect from "effect/Effect";
11+
import * as Fiber from "effect/Fiber";
12+
import * as Layer from "effect/Layer";
13+
import * as Queue from "effect/Queue";
14+
import * as Stream from "effect/Stream";
15+
import * as TestClock from "effect/testing/TestClock";
16+
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
17+
import { ChildProcessSpawner } from "effect/process";
18+
import { vi } from "vite-plus/test";
619

20+
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
21+
import * as ConnectReachability from "./ConnectReachability.ts";
722
import { classifyReachability, type ReachabilityFacts } from "./ConnectReachability.ts";
23+
import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
24+
25+
vi.mock("node:net", { spy: true });
826

927
const checkedAt = "2026-10-19T00:00:00.000Z";
10-
const connected = { readyConnections: 4, hostname: "env.example.test", protocol: "quic" };
11-
const notConnected = { readyConnections: 0, hostname: null, protocol: null };
28+
const connected = { pid: 1, readyConnections: 4, hostname: "env.example.test", protocol: "quic" };
29+
const notConnected = { pid: 1, readyConnections: 0, hostname: null, protocol: null };
1230
const facts = (overrides: Partial<ReachabilityFacts>): ReachabilityFacts => ({
1331
managedTunnelConfigured: true,
1432
connector: connected,
@@ -22,6 +40,8 @@ describe("classifyReachability", () => {
2240
it.each([
2341
[{ kind: "status", status: 502 }, "origin_unreachable"],
2442
[{ kind: "status", status: 530 }, "no_connector"],
43+
[{ kind: "status", status: 530, cloudflareError: "1033" }, "no_connector"],
44+
[{ kind: "status", status: 530, cloudflareError: "1016" }, "unexpected_response"],
2545
[{ kind: "network_error" }, "timeout"],
2646
] as const)("maps endpoint result %o to %s", (endpoint, cause) => {
2747
expect(classifyReachability(facts({ endpoint, edgeReachable: true }), checkedAt)).toMatchObject(
@@ -40,7 +60,12 @@ describe("classifyReachability", () => {
4060
expect(
4161
classifyReachability(
4262
facts({
43-
connector: { readyConnections: null, hostname: "env.example.test", protocol: null },
63+
connector: {
64+
pid: 1,
65+
readyConnections: null,
66+
hostname: "env.example.test",
67+
protocol: null,
68+
},
4469
endpoint: { kind: "status", status: 502 },
4570
edgeReachable: false,
4671
}),
@@ -51,7 +76,12 @@ describe("classifyReachability", () => {
5176
expect(
5277
classifyReachability(
5378
facts({
54-
connector: { readyConnections: null, hostname: "env.example.test", protocol: null },
79+
connector: {
80+
pid: 1,
81+
readyConnections: null,
82+
hostname: "env.example.test",
83+
protocol: null,
84+
},
5585
endpoint: { kind: "dns_failed" },
5686
edgeReachable: false,
5787
}),
@@ -74,6 +104,280 @@ describe("classifyReachability", () => {
74104
});
75105
});
76106

107+
interface ServedResponse {
108+
readonly status: number;
109+
readonly body: string | ReadableStream<Uint8Array> | { readonly cloudflareError: number };
110+
}
111+
112+
// What Cloudflare serves for its 530 page: HTML unless the client accepts JSON.
113+
const cloudflareErrorPage = (request: HttpClientRequest.HttpClientRequest, code: number) =>
114+
request.headers.accept === "application/json"
115+
? JSON.stringify({ status: 530, error_code: code, error_name: "tunnel_error" })
116+
: `<!doctype html><h1><span>Error</span> <span>${code}</span></h1>`;
117+
118+
const reachabilityLayer = (
119+
respond: Effect.Effect<ServedResponse>,
120+
restartConnector: (pid: number, stillDead: Effect.Effect<boolean>) => Effect.Effect<boolean>,
121+
) =>
122+
ConnectReachability.layer.pipe(
123+
Layer.provide(
124+
Layer.mergeAll(
125+
Layer.mock(ManagedEndpointRuntime.CloudManagedEndpointRuntime)({
126+
inspectConnector: Effect.succeed(connected),
127+
readinessChanged: Stream.never,
128+
restartConnector,
129+
}),
130+
Layer.mock(ServerSecretStore.ServerSecretStore)({
131+
get: () => Effect.succeedSome(new Uint8Array()),
132+
}),
133+
Layer.succeed(
134+
HttpClient.HttpClient,
135+
HttpClient.make((request) =>
136+
respond.pipe(
137+
Effect.map(({ status, body }) =>
138+
HttpClientResponse.fromWeb(
139+
request,
140+
new Response(
141+
typeof body === "object" && "cloudflareError" in body
142+
? cloudflareErrorPage(request, body.cloudflareError)
143+
: body,
144+
{ status },
145+
),
146+
),
147+
),
148+
),
149+
),
150+
),
151+
Layer.succeed(
152+
ChildProcessSpawner.ChildProcessSpawner,
153+
ChildProcessSpawner.make(() => Effect.die("no tailscale")),
154+
),
155+
),
156+
),
157+
);
158+
159+
const mockEdgeConnect = () =>
160+
vi.spyOn(NodeNet, "connect").mockImplementation(() => {
161+
const socket = new NodeNet.Socket();
162+
queueMicrotask(() => socket.emit("connect"));
163+
return socket;
164+
});
165+
166+
describe("ConnectReachability stale connector recovery", () => {
167+
it.effect("restarts cloudflared when Cloudflare has no connector but it reports ready", () =>
168+
Effect.gen(function* () {
169+
using _connect = mockEdgeConnect();
170+
const restarts = yield* Queue.unbounded<number>();
171+
// Cloudflare answers 530 while cloudflared still reports four ready connections.
172+
let status = 530;
173+
let body = { cloudflareError: 1033 };
174+
const layer = reachabilityLayer(
175+
Effect.sync(() => ({ status, body })),
176+
(pid, stillDead) =>
177+
stillDead.pipe(Effect.tap((dead) => (dead ? Queue.offer(restarts, pid) : Effect.void))),
178+
);
179+
// The scheduled checks run in the layer's scope, so the assertions run inside it.
180+
yield* Effect.gen(function* () {
181+
const reachability = yield* ConnectReachability.ConnectReachability;
182+
183+
// The first sighting schedules its own confirming check two minutes later.
184+
expect(yield* reachability.check).toMatchObject({ cause: "no_connector" });
185+
yield* TestClock.adjust(Duration.minutes(1));
186+
expect(yield* Queue.size(restarts)).toBe(0);
187+
yield* TestClock.adjust(Duration.minutes(1));
188+
expect(yield* Queue.take(restarts)).toBe(connected.pid);
189+
190+
// A restart that did not help doubles the wait.
191+
yield* reachability.check;
192+
yield* TestClock.adjust(Duration.minutes(2));
193+
expect(yield* Queue.size(restarts)).toBe(0);
194+
yield* TestClock.adjust(Duration.minutes(2));
195+
expect(yield* Queue.take(restarts)).toBe(connected.pid);
196+
197+
// Reaching the endpoint resets the wait, and a 530 for a broken DNS record is left alone.
198+
status = 200;
199+
expect(yield* reachability.check).toMatchObject({ status: "reachable" });
200+
status = 530;
201+
body = { cloudflareError: 1016 };
202+
expect(yield* reachability.check).toMatchObject({ cause: "unexpected_response" });
203+
yield* TestClock.adjust(Duration.minutes(2));
204+
expect(yield* Queue.size(restarts)).toBe(0);
205+
206+
body = { cloudflareError: 1033 };
207+
yield* reachability.check;
208+
yield* TestClock.adjust(Duration.minutes(2));
209+
expect(yield* Queue.take(restarts)).toBe(connected.pid);
210+
}).pipe(Effect.provide(layer));
211+
}),
212+
);
213+
214+
it.effect("starts a new streak after a healthy check that a newer check overtook", () =>
215+
Effect.gen(function* () {
216+
using _connect = mockEdgeConnect();
217+
const restartTimes = yield* Queue.unbounded<number>();
218+
const healthyRequested = yield* Deferred.make<void>();
219+
const healthyAnswer = yield* Deferred.make<void>();
220+
let status = 530;
221+
const layer = reachabilityLayer(
222+
Effect.gen(function* () {
223+
const served = status;
224+
if (served === 200) {
225+
yield* Deferred.succeed(healthyRequested, undefined);
226+
yield* Deferred.await(healthyAnswer);
227+
}
228+
return { status: served, body: { cloudflareError: 1033 } };
229+
}),
230+
(_pid, stillDead) =>
231+
stillDead.pipe(
232+
Effect.tap((dead) =>
233+
dead
234+
? Clock.currentTimeMillis.pipe(
235+
Effect.flatMap((now) => Queue.offer(restartTimes, now)),
236+
)
237+
: Effect.void,
238+
),
239+
),
240+
);
241+
yield* Effect.gen(function* () {
242+
const reachability = yield* ConnectReachability.ConnectReachability;
243+
const startedAt = yield* Clock.currentTimeMillis;
244+
yield* reachability.check;
245+
yield* TestClock.adjust(Duration.minutes(1));
246+
247+
// The healthy answer arrives after a newer check already reported the tunnel dead.
248+
status = 200;
249+
const healthy = yield* Effect.forkChild(reachability.check);
250+
yield* Deferred.await(healthyRequested);
251+
status = 530;
252+
yield* reachability.check;
253+
yield* Deferred.succeed(healthyAnswer, undefined);
254+
expect(yield* Fiber.join(healthy)).toMatchObject({ status: "reachable" });
255+
256+
// The grace period starts again at the next dead check, a minute after the first one.
257+
yield* reachability.check;
258+
yield* TestClock.adjust(Duration.minutes(2));
259+
expect((yield* Queue.take(restartTimes)) - startedAt).toBe(
260+
Duration.toMillis(Duration.minutes(3)),
261+
);
262+
}).pipe(Effect.provide(layer));
263+
}),
264+
);
265+
266+
it.effect("rejects a restart when an overtaken healthy check finishes while it waits", () =>
267+
Effect.gen(function* () {
268+
using _connect = mockEdgeConnect();
269+
const healthyRequested = yield* Deferred.make<void>();
270+
const healthyAnswer = yield* Deferred.make<void>();
271+
const restartRequested = yield* Deferred.make<void>();
272+
const releaseRestart = yield* Deferred.make<void>();
273+
const guardChecks = yield* Queue.unbounded<boolean>();
274+
let status = 530;
275+
const layer = reachabilityLayer(
276+
Effect.gen(function* () {
277+
const served = status;
278+
if (served === 200) {
279+
yield* Deferred.succeed(healthyRequested, undefined);
280+
yield* Deferred.await(healthyAnswer);
281+
}
282+
return { status: served, body: { cloudflareError: 1033 } };
283+
}),
284+
(_pid, stillDead) =>
285+
Effect.gen(function* () {
286+
yield* Deferred.succeed(restartRequested, undefined);
287+
yield* Deferred.await(releaseRestart);
288+
const accepted = yield* stillDead;
289+
yield* Queue.offer(guardChecks, accepted);
290+
return accepted;
291+
}),
292+
);
293+
yield* Effect.gen(function* () {
294+
const reachability = yield* ConnectReachability.ConnectReachability;
295+
yield* reachability.check;
296+
yield* TestClock.adjust(Duration.seconds(119));
297+
298+
status = 200;
299+
const healthy = yield* Effect.forkChild(reachability.check);
300+
yield* Deferred.await(healthyRequested);
301+
status = 530;
302+
yield* TestClock.adjust(Duration.seconds(1));
303+
yield* Deferred.await(restartRequested);
304+
305+
yield* Deferred.succeed(healthyAnswer, undefined);
306+
expect(yield* Fiber.join(healthy)).toMatchObject({ status: "reachable" });
307+
yield* Deferred.succeed(releaseRestart, undefined);
308+
expect(yield* Queue.take(guardChecks)).toBe(false);
309+
310+
yield* reachability.check;
311+
yield* TestClock.adjust(Duration.minutes(1));
312+
expect(yield* Queue.size(guardChecks)).toBe(0);
313+
yield* TestClock.adjust(Duration.minutes(1));
314+
expect(yield* Queue.take(guardChecks)).toBe(true);
315+
}).pipe(Effect.provide(layer));
316+
}),
317+
);
318+
319+
it.effect("drops the confirming check of a streak that already ended", () =>
320+
Effect.gen(function* () {
321+
using _connect = mockEdgeConnect();
322+
const restarts = yield* Queue.unbounded<number>();
323+
let status = 530;
324+
let requests = 0;
325+
const layer = reachabilityLayer(
326+
Effect.sync(() => {
327+
requests += 1;
328+
return { status, body: { cloudflareError: 1033 } };
329+
}),
330+
(pid, stillDead) =>
331+
stillDead.pipe(Effect.tap((dead) => (dead ? Queue.offer(restarts, pid) : Effect.void))),
332+
);
333+
yield* Effect.gen(function* () {
334+
const reachability = yield* ConnectReachability.ConnectReachability;
335+
yield* reachability.check;
336+
yield* TestClock.adjust(Duration.minutes(1));
337+
status = 200;
338+
yield* reachability.check;
339+
status = 530;
340+
yield* reachability.check;
341+
342+
const before = requests;
343+
yield* TestClock.adjust(Duration.minutes(1));
344+
expect(requests).toBe(before);
345+
yield* TestClock.adjust(Duration.minutes(1));
346+
expect(yield* Queue.take(restarts)).toBe(connected.pid);
347+
}).pipe(Effect.provide(layer));
348+
}),
349+
);
350+
351+
it.effect("keeps a 530 whose error body never finishes", () =>
352+
Effect.gen(function* () {
353+
using _connect = mockEdgeConnect();
354+
let markBodyRead = () => {};
355+
const bodyRead = new Promise<void>((resolve) => {
356+
markBodyRead = resolve;
357+
});
358+
const layer = reachabilityLayer(
359+
Effect.sync(() => ({
360+
status: 530,
361+
body: new ReadableStream({
362+
pull: () => {
363+
markBodyRead();
364+
return new Promise<void>(() => {});
365+
},
366+
}),
367+
})),
368+
() => Effect.succeed(false),
369+
);
370+
yield* Effect.gen(function* () {
371+
const reachability = yield* ConnectReachability.ConnectReachability;
372+
const check = yield* Effect.forkChild(reachability.check);
373+
yield* Effect.promise(() => bodyRead);
374+
yield* TestClock.adjust(Duration.seconds(2));
375+
expect(yield* Fiber.join(check)).toMatchObject({ cause: "no_connector" });
376+
}).pipe(Effect.provide(layer));
377+
}),
378+
);
379+
});
380+
77381
describe("T3 Connect troubleshooting links", () => {
78382
it("each section names a heading in the user guide", () => {
79383
const guide = NodeFS.readFileSync(

0 commit comments

Comments
 (0)