Skip to content

Commit 19da7b2

Browse files
PR Batch Testerclaude
andcommitted
fix(server): restart cloudflared when Cloudflare drops its connections
After a host network change, cloudflared can keep reporting ready connections on its local /ready endpoint while Cloudflare answers 530 (no live connector). The runtime's readiness watch only restarts on 503, so the connector stayed dead until someone killed it by hand. The reachability check now restarts the connector through the runtime once a no_connector result with ready local connections lasts two minutes for the same process. Each restart that does not help doubles the wait, up to an hour; a healthy result resets it. The runtime ignores the request once that process is no longer the active connector, and the existing supervisor starts the replacement with its usual backoff. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 5b35402 commit 19da7b2

5 files changed

Lines changed: 385 additions & 9 deletions

File tree

‎apps/server/src/cloud/CloudLink.lifecycle.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
197197
}),
198198
recoveryRequests: Stream.empty,
199199
requestRecovery: () => Effect.void,
200+
restartConnector: () => Effect.void,
200201
tunnelConnected: Stream.empty,
201202
inspectConnector: Effect.succeed(null),
202203
readinessChanged: Stream.empty,

‎apps/server/src/cloud/ConnectReachability.test.ts‎

Lines changed: 304 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,34 @@
22
import * as NodeFS from "node:fs";
33

44
import { T3_CONNECT_TROUBLESHOOTING_SECTIONS } from "@t3tools/contracts";
5+
import * as RelayClient from "@t3tools/shared/relayClient";
56
import { describe, expect, it } from "@effect/vitest";
7+
import * as Deferred from "effect/Deferred";
8+
import * as Duration from "effect/Duration";
9+
import * as Effect from "effect/Effect";
10+
import * as Exit from "effect/Exit";
11+
import * as Fiber from "effect/Fiber";
12+
import * as Layer from "effect/Layer";
13+
import * as Option from "effect/Option";
14+
import * as PlatformError from "effect/PlatformError";
15+
import * as Queue from "effect/Queue";
16+
import * as Ref from "effect/Ref";
17+
import * as Sink from "effect/Sink";
18+
import * as Stream from "effect/Stream";
19+
import * as TestClock from "effect/testing/TestClock";
20+
import { HttpClient, HttpClientResponse } from "effect/http";
21+
import * as HttpClientError from "effect/http/HttpClientError";
22+
import { ChildProcess, ChildProcessSpawner } from "effect/process";
623

24+
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
25+
import { CLOUD_ENDPOINT_RUNTIME_CONFIG } from "./config.ts";
26+
import * as ConnectReachability from "./ConnectReachability.ts";
727
import { classifyReachability, type ReachabilityFacts } from "./ConnectReachability.ts";
28+
import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
829

930
const checkedAt = "2026-10-19T00:00:00.000Z";
10-
const connected = { readyConnections: 4, hostname: "env.example.test", protocol: "quic" };
11-
const notConnected = { readyConnections: 0, hostname: null, protocol: null };
31+
const connected = { pid: 1, readyConnections: 4, hostname: "env.example.test", protocol: "quic" };
32+
const notConnected = { pid: 1, readyConnections: 0, hostname: null, protocol: null };
1233
const facts = (overrides: Partial<ReachabilityFacts>): ReachabilityFacts => ({
1334
managedTunnelConfigured: true,
1435
connector: connected,
@@ -40,7 +61,12 @@ describe("classifyReachability", () => {
4061
expect(
4162
classifyReachability(
4263
facts({
43-
connector: { readyConnections: null, hostname: "env.example.test", protocol: null },
64+
connector: {
65+
pid: 1,
66+
readyConnections: null,
67+
hostname: "env.example.test",
68+
protocol: null,
69+
},
4470
endpoint: { kind: "status", status: 502 },
4571
edgeReachable: false,
4672
}),
@@ -51,7 +77,12 @@ describe("classifyReachability", () => {
5177
expect(
5278
classifyReachability(
5379
facts({
54-
connector: { readyConnections: null, hostname: "env.example.test", protocol: null },
80+
connector: {
81+
pid: 1,
82+
readyConnections: null,
83+
hostname: "env.example.test",
84+
protocol: null,
85+
},
5586
endpoint: { kind: "dns_failed" },
5687
edgeReachable: false,
5788
}),
@@ -67,6 +98,275 @@ describe("classifyReachability", () => {
6798
});
6899
});
69100

101+
const tunnelConfig = {
102+
providerKind: "cloudflare_tunnel" as const,
103+
connectorToken: "token",
104+
tunnelId: "tunnel-1",
105+
};
106+
const metricsHost = "127.0.0.1:20241";
107+
const publicUrl = "https://env.example.test/.well-known/t3/environment";
108+
const connectorStartup = new TextEncoder().encode(
109+
`2026-10-25T00:00:00Z INF Starting metrics server on ${metricsHost}/metrics\n` +
110+
"2026-10-25T00:00:00Z INF Registered tunnel connection connIndex=0 protocol=http2\n",
111+
);
112+
113+
type PublicAnswer = "530" | "200" | "dns_failed" | "network_error";
114+
115+
// Real runtime and reachability; only cloudflared, its metrics server, and the public
116+
// endpoint are fake. The metrics server keeps reporting four ready connections, as a
117+
// connector the edge dropped after a host network change does.
118+
const makeConnectFixture = Effect.fnUntraced(function* () {
119+
const started = yield* Queue.unbounded<number>();
120+
const killed: Array<number> = [];
121+
const publicAnswer = yield* Ref.make<PublicAnswer>("530");
122+
const publicGate = yield* Ref.make<{
123+
readonly entered: Deferred.Deferred<void>;
124+
readonly release: Deferred.Deferred<void>;
125+
} | null>(null);
126+
let nextPid = 800;
127+
const spawner = ChildProcessSpawner.make((command) =>
128+
Effect.gen(function* () {
129+
if (!ChildProcess.isStandardCommand(command) || command.command !== "cloudflared") {
130+
return yield* PlatformError.systemError({
131+
_tag: "NotFound",
132+
module: "ChildProcess",
133+
method: "spawn",
134+
description: "not installed",
135+
});
136+
}
137+
const pid = nextPid++;
138+
const exit = yield* Deferred.make<ChildProcessSpawner.ExitCode>();
139+
let exited = false;
140+
const handle = ChildProcessSpawner.makeHandle({
141+
pid: ChildProcessSpawner.ProcessId(pid),
142+
exitCode: Deferred.await(exit),
143+
isRunning: Effect.sync(() => !exited),
144+
kill: () =>
145+
Effect.sync(() => {
146+
if (exited) return;
147+
exited = true;
148+
killed.push(pid);
149+
Deferred.doneUnsafe(exit, Exit.succeed(ChildProcessSpawner.ExitCode(143)));
150+
}),
151+
unref: Effect.succeed(Effect.void),
152+
stdin: Sink.drain,
153+
stdout: Stream.empty,
154+
stderr: Stream.empty,
155+
all: Stream.concat(Stream.make(connectorStartup), Stream.never),
156+
getInputFd: () => Sink.drain,
157+
getOutputFd: () => Stream.empty,
158+
});
159+
yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
160+
yield* Queue.offer(started, pid);
161+
return handle;
162+
}),
163+
);
164+
const httpClient = HttpClient.make((request) =>
165+
Effect.gen(function* () {
166+
if (request.url === `http://${metricsHost}/ready`) {
167+
return HttpClientResponse.fromWeb(request, Response.json({ readyConnections: 4 }));
168+
}
169+
if (request.url === `http://${metricsHost}/config`) {
170+
return HttpClientResponse.fromWeb(
171+
request,
172+
Response.json({ config: { ingress: [{ hostname: "env.example.test" }, {}] } }),
173+
);
174+
}
175+
if (request.url !== publicUrl) return yield* Effect.die(`unexpected ${request.url}`);
176+
const answer = yield* Ref.get(publicAnswer);
177+
const gate = yield* Ref.getAndSet(publicGate, null);
178+
if (gate) {
179+
yield* Deferred.succeed(gate.entered, undefined);
180+
yield* Deferred.await(gate.release);
181+
}
182+
if (answer === "530" || answer === "200") {
183+
return HttpClientResponse.fromWeb(request, new Response(null, { status: Number(answer) }));
184+
}
185+
return yield* new HttpClientError.HttpClientError({
186+
reason: new HttpClientError.TransportError({
187+
request,
188+
cause: Object.assign(new Error(answer), {
189+
code: answer === "dns_failed" ? "ENOTFOUND" : "ECONNRESET",
190+
}),
191+
}),
192+
});
193+
}),
194+
);
195+
const runtimeLayer = ManagedEndpointRuntime.layer.pipe(
196+
Layer.provide(
197+
Layer.succeed(
198+
RelayClient.RelayClient,
199+
RelayClient.RelayClient.of({
200+
resolve: Effect.succeed({
201+
status: "available",
202+
executablePath: "cloudflared",
203+
source: "path",
204+
version: RelayClient.CLOUDFLARED_VERSION,
205+
}),
206+
install: Effect.die("unused"),
207+
installWithProgress: () => Effect.die("unused"),
208+
pruneManagedVersions: Effect.void,
209+
}),
210+
),
211+
),
212+
);
213+
const context = yield* Layer.build(
214+
ConnectReachability.layer.pipe(
215+
Layer.provideMerge(runtimeLayer),
216+
Layer.provide(
217+
Layer.mergeAll(
218+
Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
219+
Layer.succeed(HttpClient.HttpClient, httpClient),
220+
Layer.mock(ServerSecretStore.ServerSecretStore)({
221+
get: (name) =>
222+
Effect.succeed(
223+
name === CLOUD_ENDPOINT_RUNTIME_CONFIG
224+
? Option.some(new Uint8Array())
225+
: Option.none(),
226+
),
227+
}),
228+
),
229+
),
230+
),
231+
);
232+
const runtime = yield* Effect.service(ManagedEndpointRuntime.CloudManagedEndpointRuntime).pipe(
233+
Effect.provide(context),
234+
);
235+
const reachability = yield* Effect.service(ConnectReachability.ConnectReachability).pipe(
236+
Effect.provide(context),
237+
);
238+
/** Waits for the next cloudflared to start and register, so checks see its metrics. */
239+
const nextConnector = Effect.gen(function* () {
240+
const pid = yield* Queue.take(started);
241+
yield* Stream.runHead(runtime.tunnelConnected);
242+
return pid;
243+
});
244+
/** Starts a check and holds it at the public probe, after it read the connector. */
245+
const holdCheck = Effect.gen(function* () {
246+
const entered = yield* Deferred.make<void>();
247+
const release = yield* Deferred.make<void>();
248+
yield* Ref.set(publicGate, { entered, release });
249+
const fiber = yield* reachability.check.pipe(Effect.forkChild({ startImmediately: true }));
250+
yield* Deferred.await(entered);
251+
return { release: Deferred.succeed(release, undefined), result: Fiber.join(fiber) };
252+
});
253+
return { runtime, reachability, killed, publicAnswer, nextConnector, holdCheck };
254+
});
255+
256+
describe("ConnectReachability stale connector recovery", () => {
257+
it.effect("restarts cloudflared when Cloudflare has no connector but it reports ready", () =>
258+
Effect.gen(function* () {
259+
const fixture = yield* makeConnectFixture();
260+
yield* fixture.runtime.applyConfig(tunnelConfig);
261+
expect(yield* fixture.nextConnector).toBe(800);
262+
263+
expect(yield* fixture.reachability.check).toMatchObject({ cause: "no_connector" });
264+
expect(fixture.killed).toEqual([]);
265+
266+
yield* TestClock.adjust(Duration.minutes(2));
267+
expect(yield* fixture.reachability.check).toMatchObject({ cause: "no_connector" });
268+
expect(fixture.killed).toEqual([800]);
269+
expect(yield* fixture.nextConnector).toBe(801);
270+
271+
yield* Ref.set(fixture.publicAnswer, "200");
272+
expect(yield* fixture.reachability.check).toMatchObject({ status: "reachable" });
273+
}),
274+
);
275+
276+
it.effect("backs off restarts while Cloudflare keeps reporting no connector", () =>
277+
Effect.gen(function* () {
278+
const fixture = yield* makeConnectFixture();
279+
yield* fixture.runtime.applyConfig(tunnelConfig);
280+
yield* fixture.nextConnector;
281+
yield* fixture.reachability.check;
282+
yield* TestClock.adjust(Duration.minutes(2));
283+
yield* fixture.reachability.check;
284+
expect(fixture.killed).toEqual([800]);
285+
expect(yield* fixture.nextConnector).toBe(801);
286+
287+
yield* fixture.reachability.check;
288+
yield* TestClock.adjust(Duration.minutes(2));
289+
yield* fixture.reachability.check;
290+
expect(fixture.killed).toEqual([800]);
291+
292+
yield* TestClock.adjust(Duration.minutes(2));
293+
yield* fixture.reachability.check;
294+
expect(fixture.killed).toEqual([800, 801]);
295+
expect(yield* fixture.nextConnector).toBe(802);
296+
}),
297+
);
298+
299+
it.effect("needs a new streak after a healthy, DNS, or network result", () =>
300+
Effect.gen(function* () {
301+
const fixture = yield* makeConnectFixture();
302+
yield* fixture.runtime.applyConfig(tunnelConfig);
303+
yield* fixture.nextConnector;
304+
for (const answer of ["200", "dns_failed", "network_error"] as const) {
305+
yield* Ref.set(fixture.publicAnswer, "530");
306+
expect(yield* fixture.reachability.check).toMatchObject({ cause: "no_connector" });
307+
yield* TestClock.adjust(Duration.minutes(2));
308+
yield* Ref.set(fixture.publicAnswer, answer);
309+
expect(yield* fixture.reachability.check).toMatchObject(
310+
answer === "200"
311+
? { status: "reachable" }
312+
: { cause: answer === "dns_failed" ? "dns_failed" : "timeout" },
313+
);
314+
}
315+
yield* Ref.set(fixture.publicAnswer, "530");
316+
yield* fixture.reachability.check;
317+
expect(fixture.killed).toEqual([]);
318+
}),
319+
);
320+
321+
it.effect("does not count a check that finishes after a newer one", () =>
322+
Effect.gen(function* () {
323+
const fixture = yield* makeConnectFixture();
324+
yield* fixture.runtime.applyConfig(tunnelConfig);
325+
yield* fixture.nextConnector;
326+
yield* fixture.reachability.check;
327+
yield* TestClock.adjust(Duration.minutes(2));
328+
329+
const older = yield* fixture.holdCheck;
330+
yield* Ref.set(fixture.publicAnswer, "200");
331+
expect(yield* fixture.reachability.check).toMatchObject({ status: "reachable" });
332+
yield* older.release;
333+
expect(yield* older.result).toMatchObject({ cause: "no_connector" });
334+
335+
yield* Ref.set(fixture.publicAnswer, "530");
336+
yield* TestClock.adjust(Duration.minutes(2));
337+
yield* fixture.reachability.check;
338+
expect(fixture.killed).toEqual([]);
339+
}),
340+
);
341+
342+
it.effect("ignores a dead-connector result once its connector is replaced or removed", () =>
343+
Effect.gen(function* () {
344+
for (const nextConfig of [{ ...tunnelConfig, connectorToken: "replacement" }, null]) {
345+
yield* Effect.gen(function* () {
346+
const fixture = yield* makeConnectFixture();
347+
yield* fixture.runtime.applyConfig(tunnelConfig);
348+
yield* fixture.nextConnector;
349+
yield* fixture.reachability.check;
350+
yield* TestClock.adjust(Duration.minutes(2));
351+
352+
const held = yield* fixture.holdCheck;
353+
yield* fixture.runtime.applyConfig(nextConfig);
354+
yield* held.release;
355+
expect(yield* held.result).toMatchObject({ cause: "no_connector" });
356+
357+
expect(fixture.killed).toEqual([800]);
358+
if (nextConfig) {
359+
expect(yield* fixture.nextConnector).toBe(801);
360+
expect(yield* fixture.runtime.inspectConnector).not.toBeNull();
361+
} else {
362+
expect(yield* fixture.runtime.inspectConnector).toBeNull();
363+
}
364+
}).pipe(Effect.scoped);
365+
}
366+
}),
367+
);
368+
});
369+
70370
describe("T3 Connect troubleshooting links", () => {
71371
it("each section names a heading in the user guide", () => {
72372
const guide = NodeFS.readFileSync(

0 commit comments

Comments
 (0)