Skip to content

Commit 3c307d6

Browse files
committed
fix(server): narrow stale tunnel recovery and verify restart guards
1 parent 4b5efd4 commit 3c307d6

3 files changed

Lines changed: 50 additions & 14 deletions

File tree

‎apps/server/src/cloud/ConnectReachability.test.ts‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import { describe, expect, it } from "@effect/vitest";
66
import * as Duration from "effect/Duration";
77
import * as Effect from "effect/Effect";
88
import * as Layer from "effect/Layer";
9-
import * as Option from "effect/Option";
109
import * as Stream from "effect/Stream";
1110
import * as TestClock from "effect/testing/TestClock";
1211
import { HttpClient, HttpClientResponse } from "effect/http";
@@ -92,6 +91,7 @@ describe("ConnectReachability stale connector recovery", () => {
9291
it.effect("restarts cloudflared when Cloudflare has no connector but it reports ready", () =>
9392
Effect.gen(function* () {
9493
const restarted: Array<number> = [];
94+
let status = 530;
9595
const reachability = yield* Effect.service(ConnectReachability.ConnectReachability).pipe(
9696
Effect.provide(
9797
ConnectReachability.layer.pipe(
@@ -103,14 +103,14 @@ describe("ConnectReachability stale connector recovery", () => {
103103
restartConnector: (pid) => Effect.sync(() => restarted.push(pid) > 0),
104104
}),
105105
Layer.mock(ServerSecretStore.ServerSecretStore)({
106-
get: () => Effect.succeed(Option.some(new Uint8Array())),
106+
get: () => Effect.succeedSome(new Uint8Array()),
107107
}),
108108
// Cloudflare answers 530 while cloudflared still reports four ready connections.
109109
Layer.succeed(
110110
HttpClient.HttpClient,
111111
HttpClient.make((request) =>
112112
Effect.succeed(
113-
HttpClientResponse.fromWeb(request, new Response(null, { status: 530 })),
113+
HttpClientResponse.fromWeb(request, new Response(null, { status })),
114114
),
115115
),
116116
),
@@ -132,6 +132,28 @@ describe("ConnectReachability stale connector recovery", () => {
132132
yield* TestClock.adjust(Duration.minutes(1));
133133
yield* reachability.check;
134134
expect(restarted).toEqual([connected.pid]);
135+
136+
yield* reachability.check;
137+
yield* TestClock.adjust(Duration.minutes(2));
138+
yield* reachability.check;
139+
expect(restarted).toEqual([connected.pid]);
140+
yield* TestClock.adjust(Duration.minutes(2));
141+
yield* reachability.check;
142+
expect(restarted).toEqual([connected.pid, connected.pid]);
143+
144+
status = 200;
145+
expect(yield* reachability.check).toMatchObject({ status: "reachable" });
146+
status = 502;
147+
yield* reachability.check;
148+
yield* TestClock.adjust(Duration.minutes(2));
149+
yield* reachability.check;
150+
expect(restarted).toHaveLength(2);
151+
152+
status = 530;
153+
yield* reachability.check;
154+
yield* TestClock.adjust(Duration.minutes(2));
155+
yield* reachability.check;
156+
expect(restarted).toEqual([connected.pid, connected.pid, connected.pid]);
135157
}),
136158
);
137159
});

‎apps/server/src/cloud/ConnectReachability.ts‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -133,9 +133,7 @@ export function classifyReachability(
133133
return fail(
134134
"no_connector",
135135
"Cloudflare has no live connection from this machine for the tunnel.",
136-
(connector.readyConnections ?? 0) > 0
137-
? "T3 Code retries the tunnel automatically. Wait a few minutes. If it stays offline, restart T3 Code."
138-
: restart,
136+
restart,
139137
);
140138
}
141139
if (status !== null) {
@@ -257,20 +255,16 @@ const make = Effect.gen(function* () {
257255
if (connector === null || (endpoint?.kind === "status" && endpoint.status === 200)) {
258256
return facts;
259257
}
260-
// The classification ignores the TCP edge test while cloudflared has ready connections.
261-
const connected = (connector.readyConnections ?? 0) > 0;
262258
const [edgeResults, exitNode] = yield* Effect.all(
263259
[
264-
connected
265-
? Effect.succeed(null)
266-
: Effect.forEach(TUNNEL_EDGE_HOSTS, (host) => canOpenTcp(host, TUNNEL_EDGE_PORT), {
267-
concurrency: "unbounded",
268-
}),
260+
Effect.forEach(TUNNEL_EDGE_HOSTS, (host) => canOpenTcp(host, TUNNEL_EDGE_PORT), {
261+
concurrency: "unbounded",
262+
}),
269263
readExitNode,
270264
],
271265
{ concurrency: 2 },
272266
);
273-
return { ...facts, edgeReachable: edgeResults?.some(Boolean) ?? null, exitNode };
267+
return { ...facts, edgeReachable: edgeResults.some(Boolean), exitNode };
274268
});
275269

276270
// Runs inside the check span; the runtime restart has its own.

‎apps/server/src/cloud/ManagedEndpointRuntime.test.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,26 @@ describe("CloudManagedEndpointRuntime", () => {
288288
return { spawner, spawned, killed, exits, started, emit, observe };
289289
});
290290

291+
it.effect("restarts only the inspected connector and supervises its replacement", () =>
292+
Effect.gen(function* () {
293+
const fixture = yield* makeTransportSpawner();
294+
const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
295+
expect(yield* runtime.restartConnector(800)).toBe(false);
296+
yield* runtime.applyConfig(tunnelConfig);
297+
yield* fixture.observe(runtime, 0);
298+
expect(yield* runtime.inspectConnector).toMatchObject({ pid: 800 });
299+
expect(yield* runtime.restartConnector(799)).toBe(false);
300+
expect(fixture.killed).toEqual([]);
301+
expect(yield* runtime.restartConnector(800)).toBe(true);
302+
expect(yield* Queue.take(fixture.started)).toBe(0);
303+
expect(yield* Queue.take(fixture.started)).toBe(1);
304+
yield* fixture.observe(runtime, 1);
305+
expect(fixture.killed).toContain(800);
306+
expect(yield* runtime.restartConnector(800)).toBe(false);
307+
expect(yield* runtime.inspectConnector).toMatchObject({ pid: 801 });
308+
}),
309+
);
310+
291311
it.effect("preserves an explicit transport choice despite repeated QUIC stalls", () =>
292312
Effect.gen(function* () {
293313
const fixture = yield* makeTransportSpawner({ transport: "quic" });

0 commit comments

Comments
 (0)