Skip to content

Commit c22b91c

Browse files
PR Batch Testerclaude
andcommitted
test(server): keep one regression test for stale tunnel restarts
Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent fefb5fb commit c22b91c

1 file changed

Lines changed: 39 additions & 316 deletions

File tree

‎apps/server/src/cloud/ConnectReachability.test.ts‎

Lines changed: 39 additions & 316 deletions
Original file line numberDiff line numberDiff line change
@@ -2,28 +2,17 @@
22
import * as NodeFS from "node:fs";
33

44
import { T3_CONNECT_TROUBLESHOOTING_SECTIONS } from "@t3tools/contracts";
5-
import * as RelayClient from "@t3tools/shared/relayClient";
65
import { describe, expect, it } from "@effect/vitest";
7-
import * as Clock from "effect/Clock";
8-
import * as Deferred from "effect/Deferred";
96
import * as Duration from "effect/Duration";
107
import * as Effect from "effect/Effect";
11-
import * as Exit from "effect/Exit";
12-
import * as Fiber from "effect/Fiber";
138
import * as Layer from "effect/Layer";
149
import * as Option from "effect/Option";
15-
import * as PlatformError from "effect/PlatformError";
16-
import * as Queue from "effect/Queue";
17-
import * as Ref from "effect/Ref";
18-
import * as Sink from "effect/Sink";
1910
import * as Stream from "effect/Stream";
2011
import * as TestClock from "effect/testing/TestClock";
2112
import { HttpClient, HttpClientResponse } from "effect/http";
22-
import * as HttpClientError from "effect/http/HttpClientError";
23-
import { ChildProcess, ChildProcessSpawner } from "effect/process";
13+
import { ChildProcessSpawner } from "effect/process";
2414

2515
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
26-
import { CLOUD_ENDPOINT_RUNTIME_CONFIG } from "./config.ts";
2716
import * as ConnectReachability from "./ConnectReachability.ts";
2817
import { classifyReachability, type ReachabilityFacts } from "./ConnectReachability.ts";
2918
import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
@@ -99,316 +88,50 @@ describe("classifyReachability", () => {
9988
});
10089
});
10190

102-
const tunnelConfig = {
103-
providerKind: "cloudflare_tunnel" as const,
104-
connectorToken: "token",
105-
tunnelId: "tunnel-1",
106-
};
107-
const metricsHost = "127.0.0.1:20241";
108-
const publicUrl = "https://env.example.test/.well-known/t3/environment";
109-
const connectorStartup = new TextEncoder().encode(
110-
`2026-10-25T00:00:00Z INF Starting metrics server on ${metricsHost}/metrics\n` +
111-
"2026-10-25T00:00:00Z INF Registered tunnel connection connIndex=0 protocol=http2\n",
112-
);
113-
114-
type PublicAnswer = "530" | "200" | "dns_failed" | "network_error";
115-
116-
// Real runtime and reachability; only cloudflared, its metrics server, and the public
117-
// endpoint are fake. The metrics server keeps reporting four ready connections, as a
118-
// connector the edge dropped after a host network change does.
119-
const makeConnectFixture = Effect.fnUntraced(function* () {
120-
const started = yield* Queue.unbounded<number>();
121-
const killed: Array<number> = [];
122-
const publicAnswer = yield* Ref.make<PublicAnswer>("530");
123-
const publicGate = yield* Ref.make<{
124-
readonly entered: Deferred.Deferred<void>;
125-
readonly release: Deferred.Deferred<void>;
126-
} | null>(null);
127-
let nextPid = 800;
128-
const spawner = ChildProcessSpawner.make((command) =>
129-
Effect.gen(function* () {
130-
if (!ChildProcess.isStandardCommand(command) || command.command !== "cloudflared") {
131-
return yield* PlatformError.systemError({
132-
_tag: "NotFound",
133-
module: "ChildProcess",
134-
method: "spawn",
135-
description: "not installed",
136-
});
137-
}
138-
const pid = nextPid++;
139-
const exit = yield* Deferred.make<ChildProcessSpawner.ExitCode>();
140-
let exited = false;
141-
const handle = ChildProcessSpawner.makeHandle({
142-
pid: ChildProcessSpawner.ProcessId(pid),
143-
exitCode: Deferred.await(exit),
144-
isRunning: Effect.sync(() => !exited),
145-
kill: () =>
146-
Effect.sync(() => {
147-
if (exited) return;
148-
exited = true;
149-
killed.push(pid);
150-
Deferred.doneUnsafe(exit, Exit.succeed(ChildProcessSpawner.ExitCode(143)));
151-
}),
152-
unref: Effect.succeed(Effect.void),
153-
stdin: Sink.drain,
154-
stdout: Stream.empty,
155-
stderr: Stream.empty,
156-
all: Stream.concat(Stream.make(connectorStartup), Stream.never),
157-
getInputFd: () => Sink.drain,
158-
getOutputFd: () => Stream.empty,
159-
});
160-
yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
161-
yield* Queue.offer(started, pid);
162-
return handle;
163-
}),
164-
);
165-
const httpClient = HttpClient.make((request) =>
166-
Effect.gen(function* () {
167-
if (request.url === `http://${metricsHost}/ready`) {
168-
return HttpClientResponse.fromWeb(request, Response.json({ readyConnections: 4 }));
169-
}
170-
if (request.url === `http://${metricsHost}/config`) {
171-
return HttpClientResponse.fromWeb(
172-
request,
173-
Response.json({ config: { ingress: [{ hostname: "env.example.test" }, {}] } }),
174-
);
175-
}
176-
if (request.url !== publicUrl) return yield* Effect.die(`unexpected ${request.url}`);
177-
const answer = yield* Ref.get(publicAnswer);
178-
const gate = yield* Ref.getAndSet(publicGate, null);
179-
if (gate) {
180-
yield* Deferred.succeed(gate.entered, undefined);
181-
yield* Deferred.await(gate.release);
182-
}
183-
if (answer === "530" || answer === "200") {
184-
return HttpClientResponse.fromWeb(request, new Response(null, { status: Number(answer) }));
185-
}
186-
return yield* new HttpClientError.HttpClientError({
187-
reason: new HttpClientError.TransportError({
188-
request,
189-
cause: Object.assign(new Error(answer), {
190-
code: answer === "dns_failed" ? "ENOTFOUND" : "ECONNRESET",
191-
}),
192-
}),
193-
});
194-
}),
195-
);
196-
const runtimeLayer = ManagedEndpointRuntime.layer.pipe(
197-
Layer.provide(
198-
Layer.succeed(
199-
RelayClient.RelayClient,
200-
RelayClient.RelayClient.of({
201-
resolve: Effect.succeed({
202-
status: "available",
203-
executablePath: "cloudflared",
204-
source: "path",
205-
version: RelayClient.CLOUDFLARED_VERSION,
206-
}),
207-
install: Effect.die("unused"),
208-
installWithProgress: () => Effect.die("unused"),
209-
pruneManagedVersions: Effect.void,
210-
}),
211-
),
212-
),
213-
);
214-
const context = yield* Layer.build(
215-
ConnectReachability.layer.pipe(
216-
Layer.provideMerge(runtimeLayer),
217-
Layer.provide(
218-
Layer.mergeAll(
219-
Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner),
220-
Layer.succeed(HttpClient.HttpClient, httpClient),
221-
Layer.mock(ServerSecretStore.ServerSecretStore)({
222-
get: (name) =>
223-
Effect.succeed(
224-
name === CLOUD_ENDPOINT_RUNTIME_CONFIG
225-
? Option.some(new Uint8Array())
226-
: Option.none(),
227-
),
228-
}),
229-
),
230-
),
231-
),
232-
);
233-
const runtime = yield* Effect.service(ManagedEndpointRuntime.CloudManagedEndpointRuntime).pipe(
234-
Effect.provide(context),
235-
);
236-
const reachability = yield* Effect.service(ConnectReachability.ConnectReachability).pipe(
237-
Effect.provide(context),
238-
);
239-
/** Waits for the next cloudflared to start and register, so checks see its metrics. */
240-
const nextConnector = Effect.gen(function* () {
241-
const pid = yield* Queue.take(started);
242-
yield* Stream.runHead(runtime.tunnelConnected);
243-
return pid;
244-
});
245-
/** Starts a check and holds it at the public probe, after it read the connector. */
246-
const holdCheck = Effect.gen(function* () {
247-
const entered = yield* Deferred.make<void>();
248-
const release = yield* Deferred.make<void>();
249-
yield* Ref.set(publicGate, { entered, release });
250-
const fiber = yield* reachability.check.pipe(Effect.forkChild({ startImmediately: true }));
251-
yield* Deferred.await(entered);
252-
return { release: Deferred.succeed(release, undefined), result: Fiber.join(fiber) };
253-
});
254-
return { runtime, reachability, killed, publicAnswer, nextConnector, holdCheck };
255-
});
256-
25791
describe("ConnectReachability stale connector recovery", () => {
25892
it.effect("restarts cloudflared when Cloudflare has no connector but it reports ready", () =>
25993
Effect.gen(function* () {
260-
const fixture = yield* makeConnectFixture();
261-
yield* fixture.runtime.applyConfig(tunnelConfig);
262-
expect(yield* fixture.nextConnector).toBe(800);
263-
264-
expect(yield* fixture.reachability.check).toMatchObject({ cause: "no_connector" });
265-
expect(fixture.killed).toEqual([]);
266-
267-
yield* TestClock.adjust(Duration.minutes(2));
268-
expect(yield* fixture.reachability.check).toMatchObject({ cause: "no_connector" });
269-
expect(fixture.killed).toEqual([800]);
270-
expect(yield* fixture.nextConnector).toBe(801);
271-
272-
yield* Ref.set(fixture.publicAnswer, "200");
273-
expect(yield* fixture.reachability.check).toMatchObject({ status: "reachable" });
274-
}),
275-
);
276-
277-
it.effect("backs off restarts while Cloudflare keeps reporting no connector", () =>
278-
Effect.gen(function* () {
279-
const fixture = yield* makeConnectFixture();
280-
yield* fixture.runtime.applyConfig(tunnelConfig);
281-
yield* fixture.nextConnector;
282-
yield* fixture.reachability.check;
283-
yield* TestClock.adjust(Duration.minutes(2));
284-
yield* fixture.reachability.check;
285-
expect(fixture.killed).toEqual([800]);
286-
expect(yield* fixture.nextConnector).toBe(801);
287-
288-
yield* fixture.reachability.check;
289-
yield* TestClock.adjust(Duration.minutes(2));
290-
yield* fixture.reachability.check;
291-
expect(fixture.killed).toEqual([800]);
292-
293-
yield* TestClock.adjust(Duration.minutes(2));
294-
yield* fixture.reachability.check;
295-
expect(fixture.killed).toEqual([800, 801]);
296-
expect(yield* fixture.nextConnector).toBe(802);
297-
}),
298-
);
299-
300-
it.effect("needs a new streak after a healthy, DNS, or network result", () =>
301-
Effect.gen(function* () {
302-
const fixture = yield* makeConnectFixture();
303-
yield* fixture.runtime.applyConfig(tunnelConfig);
304-
yield* fixture.nextConnector;
305-
for (const answer of ["200", "dns_failed", "network_error"] as const) {
306-
yield* Ref.set(fixture.publicAnswer, "530");
307-
expect(yield* fixture.reachability.check).toMatchObject({ cause: "no_connector" });
308-
yield* TestClock.adjust(Duration.minutes(2));
309-
yield* Ref.set(fixture.publicAnswer, answer);
310-
expect(yield* fixture.reachability.check).toMatchObject(
311-
answer === "200"
312-
? { status: "reachable" }
313-
: { cause: answer === "dns_failed" ? "dns_failed" : "timeout" },
314-
);
315-
}
316-
yield* Ref.set(fixture.publicAnswer, "530");
317-
yield* fixture.reachability.check;
318-
expect(fixture.killed).toEqual([]);
319-
}),
320-
);
321-
322-
it.effect("does not count a check that finishes after a newer one", () =>
323-
Effect.gen(function* () {
324-
const fixture = yield* makeConnectFixture();
325-
yield* fixture.runtime.applyConfig(tunnelConfig);
326-
yield* fixture.nextConnector;
327-
yield* fixture.reachability.check;
328-
yield* TestClock.adjust(Duration.minutes(2));
329-
330-
const older = yield* fixture.holdCheck;
331-
yield* Ref.set(fixture.publicAnswer, "200");
332-
expect(yield* fixture.reachability.check).toMatchObject({ status: "reachable" });
333-
yield* older.release;
334-
expect(yield* older.result).toMatchObject({ cause: "no_connector" });
335-
336-
yield* Ref.set(fixture.publicAnswer, "530");
337-
yield* TestClock.adjust(Duration.minutes(2));
338-
yield* fixture.reachability.check;
339-
expect(fixture.killed).toEqual([]);
340-
}),
341-
);
342-
343-
it.effect("does not apply an older failure after a newer check resets recovery", () =>
344-
Effect.gen(function* () {
345-
const fixture = yield* makeConnectFixture();
346-
yield* fixture.runtime.applyConfig(tunnelConfig);
347-
yield* fixture.nextConnector;
348-
const clock = yield* Clock.Clock;
349-
const recoveryReached = yield* Deferred.make<void>();
350-
const releaseRecovery = yield* Deferred.make<void>();
351-
let clockReads = 0;
352-
const older = yield* fixture.reachability.check.pipe(
353-
Effect.provideService(Clock.Clock, {
354-
...clock,
355-
currentTimeMillis: Effect.suspend(() => {
356-
// The first read dates the published result; the second dates recovery.
357-
if (++clockReads !== 2) return clock.currentTimeMillis;
358-
return Deferred.succeed(recoveryReached, undefined).pipe(
359-
Effect.andThen(Deferred.await(releaseRecovery)),
360-
Effect.andThen(clock.currentTimeMillis),
361-
);
362-
}),
363-
}),
364-
Effect.forkChild({ startImmediately: true }),
94+
const restarted: Array<number> = [];
95+
const reachability = yield* Effect.service(ConnectReachability.ConnectReachability).pipe(
96+
Effect.provide(
97+
ConnectReachability.layer.pipe(
98+
Layer.provide(
99+
Layer.mergeAll(
100+
Layer.mock(ManagedEndpointRuntime.CloudManagedEndpointRuntime)({
101+
inspectConnector: Effect.succeed(connected),
102+
readinessChanged: Stream.never,
103+
restartConnector: (pid) => Effect.sync(() => restarted.push(pid) > 0),
104+
}),
105+
Layer.mock(ServerSecretStore.ServerSecretStore)({
106+
get: () => Effect.succeed(Option.some(new Uint8Array())),
107+
}),
108+
// Cloudflare answers 530 while cloudflared still reports four ready connections.
109+
Layer.succeed(
110+
HttpClient.HttpClient,
111+
HttpClient.make((request) =>
112+
Effect.succeed(
113+
HttpClientResponse.fromWeb(request, new Response(null, { status: 530 })),
114+
),
115+
),
116+
),
117+
Layer.succeed(
118+
ChildProcessSpawner.ChildProcessSpawner,
119+
ChildProcessSpawner.make(() => Effect.die("no tailscale")),
120+
),
121+
),
122+
),
123+
),
124+
),
365125
);
366-
yield* Deferred.await(recoveryReached);
367-
expect(Option.getOrThrow(yield* fixture.reachability.latest)).toMatchObject({
368-
cause: "no_connector",
369-
});
370-
yield* Ref.set(fixture.publicAnswer, "200");
371-
expect(yield* fixture.reachability.check).toMatchObject({ status: "reachable" });
372-
yield* TestClock.adjust(Duration.minutes(2));
373-
yield* Deferred.succeed(releaseRecovery, undefined);
374-
expect(yield* Fiber.join(older)).toMatchObject({ cause: "no_connector" });
375-
376-
yield* TestClock.adjust(Duration.minutes(2));
377-
yield* Ref.set(fixture.publicAnswer, "530");
378-
yield* fixture.reachability.check;
379-
expect(fixture.killed).toEqual([]);
380-
}),
381-
);
382-
383-
it.effect("ignores a dead-connector result once its connector is replaced or removed", () =>
384-
Effect.gen(function* () {
385-
for (const nextConfig of [{ ...tunnelConfig, connectorToken: "replacement" }, null]) {
386-
yield* Effect.gen(function* () {
387-
const fixture = yield* makeConnectFixture();
388-
yield* fixture.runtime.applyConfig(tunnelConfig);
389-
yield* fixture.nextConnector;
390-
yield* fixture.reachability.check;
391-
yield* TestClock.adjust(Duration.minutes(2));
392126

393-
const held = yield* fixture.holdCheck;
394-
yield* fixture.runtime.applyConfig(nextConfig);
395-
yield* held.release;
396-
expect(yield* held.result).toMatchObject({ cause: "no_connector" });
127+
expect(yield* reachability.check).toMatchObject({ cause: "no_connector" });
128+
yield* TestClock.adjust(Duration.minutes(1));
129+
yield* reachability.check;
130+
expect(restarted).toEqual([]);
397131

398-
expect(fixture.killed).toEqual([800]);
399-
if (nextConfig) {
400-
expect(yield* fixture.nextConnector).toBe(801);
401-
expect(yield* fixture.runtime.inspectConnector).not.toBeNull();
402-
yield* fixture.reachability.check;
403-
yield* TestClock.adjust(Duration.minutes(2));
404-
yield* fixture.reachability.check;
405-
expect(fixture.killed).toEqual([800, 801]);
406-
expect(yield* fixture.nextConnector).toBe(802);
407-
} else {
408-
expect(yield* fixture.runtime.inspectConnector).toBeNull();
409-
}
410-
}).pipe(Effect.scoped);
411-
}
132+
yield* TestClock.adjust(Duration.minutes(1));
133+
yield* reachability.check;
134+
expect(restarted).toEqual([connected.pid]);
412135
}),
413136
);
414137
});

0 commit comments

Comments
 (0)