|
2 | 2 | import * as NodeFS from "node:fs"; |
3 | 3 |
|
4 | 4 | import { T3_CONNECT_TROUBLESHOOTING_SECTIONS } from "@t3tools/contracts"; |
5 | | -import * as RelayClient from "@t3tools/shared/relayClient"; |
6 | 5 | import { describe, expect, it } from "@effect/vitest"; |
7 | | -import * as Clock from "effect/Clock"; |
8 | | -import * as Deferred from "effect/Deferred"; |
9 | 6 | import * as Duration from "effect/Duration"; |
10 | 7 | import * as Effect from "effect/Effect"; |
11 | | -import * as Exit from "effect/Exit"; |
12 | | -import * as Fiber from "effect/Fiber"; |
13 | 8 | import * as Layer from "effect/Layer"; |
14 | 9 | import * as Option from "effect/Option"; |
15 | | -import * as PlatformError from "effect/PlatformError"; |
16 | | -import * as Queue from "effect/Queue"; |
17 | | -import * as Ref from "effect/Ref"; |
18 | | -import * as Sink from "effect/Sink"; |
19 | 10 | import * as Stream from "effect/Stream"; |
20 | 11 | import * as TestClock from "effect/testing/TestClock"; |
21 | 12 | import { HttpClient, HttpClientResponse } from "effect/http"; |
22 | | -import * as HttpClientError from "effect/http/HttpClientError"; |
23 | | -import { ChildProcess, ChildProcessSpawner } from "effect/process"; |
| 13 | +import { ChildProcessSpawner } from "effect/process"; |
24 | 14 |
|
25 | 15 | import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; |
26 | | -import { CLOUD_ENDPOINT_RUNTIME_CONFIG } from "./config.ts"; |
27 | 16 | import * as ConnectReachability from "./ConnectReachability.ts"; |
28 | 17 | import { classifyReachability, type ReachabilityFacts } from "./ConnectReachability.ts"; |
29 | 18 | import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; |
@@ -99,316 +88,50 @@ describe("classifyReachability", () => { |
99 | 88 | }); |
100 | 89 | }); |
101 | 90 |
|
102 | | -const tunnelConfig = { |
103 | | - providerKind: "cloudflare_tunnel" as const, |
104 | | - connectorToken: "token", |
105 | | - tunnelId: "tunnel-1", |
106 | | -}; |
107 | | -const metricsHost = "127.0.0.1:20241"; |
108 | | -const publicUrl = "https://env.example.test/.well-known/t3/environment"; |
109 | | -const connectorStartup = new TextEncoder().encode( |
110 | | - `2026-10-25T00:00:00Z INF Starting metrics server on ${metricsHost}/metrics\n` + |
111 | | - "2026-10-25T00:00:00Z INF Registered tunnel connection connIndex=0 protocol=http2\n", |
112 | | -); |
113 | | - |
114 | | -type PublicAnswer = "530" | "200" | "dns_failed" | "network_error"; |
115 | | - |
116 | | -// Real runtime and reachability; only cloudflared, its metrics server, and the public |
117 | | -// endpoint are fake. The metrics server keeps reporting four ready connections, as a |
118 | | -// connector the edge dropped after a host network change does. |
119 | | -const makeConnectFixture = Effect.fnUntraced(function* () { |
120 | | - const started = yield* Queue.unbounded<number>(); |
121 | | - const killed: Array<number> = []; |
122 | | - const publicAnswer = yield* Ref.make<PublicAnswer>("530"); |
123 | | - const publicGate = yield* Ref.make<{ |
124 | | - readonly entered: Deferred.Deferred<void>; |
125 | | - readonly release: Deferred.Deferred<void>; |
126 | | - } | null>(null); |
127 | | - let nextPid = 800; |
128 | | - const spawner = ChildProcessSpawner.make((command) => |
129 | | - Effect.gen(function* () { |
130 | | - if (!ChildProcess.isStandardCommand(command) || command.command !== "cloudflared") { |
131 | | - return yield* PlatformError.systemError({ |
132 | | - _tag: "NotFound", |
133 | | - module: "ChildProcess", |
134 | | - method: "spawn", |
135 | | - description: "not installed", |
136 | | - }); |
137 | | - } |
138 | | - const pid = nextPid++; |
139 | | - const exit = yield* Deferred.make<ChildProcessSpawner.ExitCode>(); |
140 | | - let exited = false; |
141 | | - const handle = ChildProcessSpawner.makeHandle({ |
142 | | - pid: ChildProcessSpawner.ProcessId(pid), |
143 | | - exitCode: Deferred.await(exit), |
144 | | - isRunning: Effect.sync(() => !exited), |
145 | | - kill: () => |
146 | | - Effect.sync(() => { |
147 | | - if (exited) return; |
148 | | - exited = true; |
149 | | - killed.push(pid); |
150 | | - Deferred.doneUnsafe(exit, Exit.succeed(ChildProcessSpawner.ExitCode(143))); |
151 | | - }), |
152 | | - unref: Effect.succeed(Effect.void), |
153 | | - stdin: Sink.drain, |
154 | | - stdout: Stream.empty, |
155 | | - stderr: Stream.empty, |
156 | | - all: Stream.concat(Stream.make(connectorStartup), Stream.never), |
157 | | - getInputFd: () => Sink.drain, |
158 | | - getOutputFd: () => Stream.empty, |
159 | | - }); |
160 | | - yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore)); |
161 | | - yield* Queue.offer(started, pid); |
162 | | - return handle; |
163 | | - }), |
164 | | - ); |
165 | | - const httpClient = HttpClient.make((request) => |
166 | | - Effect.gen(function* () { |
167 | | - if (request.url === `http://${metricsHost}/ready`) { |
168 | | - return HttpClientResponse.fromWeb(request, Response.json({ readyConnections: 4 })); |
169 | | - } |
170 | | - if (request.url === `http://${metricsHost}/config`) { |
171 | | - return HttpClientResponse.fromWeb( |
172 | | - request, |
173 | | - Response.json({ config: { ingress: [{ hostname: "env.example.test" }, {}] } }), |
174 | | - ); |
175 | | - } |
176 | | - if (request.url !== publicUrl) return yield* Effect.die(`unexpected ${request.url}`); |
177 | | - const answer = yield* Ref.get(publicAnswer); |
178 | | - const gate = yield* Ref.getAndSet(publicGate, null); |
179 | | - if (gate) { |
180 | | - yield* Deferred.succeed(gate.entered, undefined); |
181 | | - yield* Deferred.await(gate.release); |
182 | | - } |
183 | | - if (answer === "530" || answer === "200") { |
184 | | - return HttpClientResponse.fromWeb(request, new Response(null, { status: Number(answer) })); |
185 | | - } |
186 | | - return yield* new HttpClientError.HttpClientError({ |
187 | | - reason: new HttpClientError.TransportError({ |
188 | | - request, |
189 | | - cause: Object.assign(new Error(answer), { |
190 | | - code: answer === "dns_failed" ? "ENOTFOUND" : "ECONNRESET", |
191 | | - }), |
192 | | - }), |
193 | | - }); |
194 | | - }), |
195 | | - ); |
196 | | - const runtimeLayer = ManagedEndpointRuntime.layer.pipe( |
197 | | - Layer.provide( |
198 | | - Layer.succeed( |
199 | | - RelayClient.RelayClient, |
200 | | - RelayClient.RelayClient.of({ |
201 | | - resolve: Effect.succeed({ |
202 | | - status: "available", |
203 | | - executablePath: "cloudflared", |
204 | | - source: "path", |
205 | | - version: RelayClient.CLOUDFLARED_VERSION, |
206 | | - }), |
207 | | - install: Effect.die("unused"), |
208 | | - installWithProgress: () => Effect.die("unused"), |
209 | | - pruneManagedVersions: Effect.void, |
210 | | - }), |
211 | | - ), |
212 | | - ), |
213 | | - ); |
214 | | - const context = yield* Layer.build( |
215 | | - ConnectReachability.layer.pipe( |
216 | | - Layer.provideMerge(runtimeLayer), |
217 | | - Layer.provide( |
218 | | - Layer.mergeAll( |
219 | | - Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner), |
220 | | - Layer.succeed(HttpClient.HttpClient, httpClient), |
221 | | - Layer.mock(ServerSecretStore.ServerSecretStore)({ |
222 | | - get: (name) => |
223 | | - Effect.succeed( |
224 | | - name === CLOUD_ENDPOINT_RUNTIME_CONFIG |
225 | | - ? Option.some(new Uint8Array()) |
226 | | - : Option.none(), |
227 | | - ), |
228 | | - }), |
229 | | - ), |
230 | | - ), |
231 | | - ), |
232 | | - ); |
233 | | - const runtime = yield* Effect.service(ManagedEndpointRuntime.CloudManagedEndpointRuntime).pipe( |
234 | | - Effect.provide(context), |
235 | | - ); |
236 | | - const reachability = yield* Effect.service(ConnectReachability.ConnectReachability).pipe( |
237 | | - Effect.provide(context), |
238 | | - ); |
239 | | - /** Waits for the next cloudflared to start and register, so checks see its metrics. */ |
240 | | - const nextConnector = Effect.gen(function* () { |
241 | | - const pid = yield* Queue.take(started); |
242 | | - yield* Stream.runHead(runtime.tunnelConnected); |
243 | | - return pid; |
244 | | - }); |
245 | | - /** Starts a check and holds it at the public probe, after it read the connector. */ |
246 | | - const holdCheck = Effect.gen(function* () { |
247 | | - const entered = yield* Deferred.make<void>(); |
248 | | - const release = yield* Deferred.make<void>(); |
249 | | - yield* Ref.set(publicGate, { entered, release }); |
250 | | - const fiber = yield* reachability.check.pipe(Effect.forkChild({ startImmediately: true })); |
251 | | - yield* Deferred.await(entered); |
252 | | - return { release: Deferred.succeed(release, undefined), result: Fiber.join(fiber) }; |
253 | | - }); |
254 | | - return { runtime, reachability, killed, publicAnswer, nextConnector, holdCheck }; |
255 | | -}); |
256 | | - |
257 | 91 | describe("ConnectReachability stale connector recovery", () => { |
258 | 92 | it.effect("restarts cloudflared when Cloudflare has no connector but it reports ready", () => |
259 | 93 | Effect.gen(function* () { |
260 | | - const fixture = yield* makeConnectFixture(); |
261 | | - yield* fixture.runtime.applyConfig(tunnelConfig); |
262 | | - expect(yield* fixture.nextConnector).toBe(800); |
263 | | - |
264 | | - expect(yield* fixture.reachability.check).toMatchObject({ cause: "no_connector" }); |
265 | | - expect(fixture.killed).toEqual([]); |
266 | | - |
267 | | - yield* TestClock.adjust(Duration.minutes(2)); |
268 | | - expect(yield* fixture.reachability.check).toMatchObject({ cause: "no_connector" }); |
269 | | - expect(fixture.killed).toEqual([800]); |
270 | | - expect(yield* fixture.nextConnector).toBe(801); |
271 | | - |
272 | | - yield* Ref.set(fixture.publicAnswer, "200"); |
273 | | - expect(yield* fixture.reachability.check).toMatchObject({ status: "reachable" }); |
274 | | - }), |
275 | | - ); |
276 | | - |
277 | | - it.effect("backs off restarts while Cloudflare keeps reporting no connector", () => |
278 | | - Effect.gen(function* () { |
279 | | - const fixture = yield* makeConnectFixture(); |
280 | | - yield* fixture.runtime.applyConfig(tunnelConfig); |
281 | | - yield* fixture.nextConnector; |
282 | | - yield* fixture.reachability.check; |
283 | | - yield* TestClock.adjust(Duration.minutes(2)); |
284 | | - yield* fixture.reachability.check; |
285 | | - expect(fixture.killed).toEqual([800]); |
286 | | - expect(yield* fixture.nextConnector).toBe(801); |
287 | | - |
288 | | - yield* fixture.reachability.check; |
289 | | - yield* TestClock.adjust(Duration.minutes(2)); |
290 | | - yield* fixture.reachability.check; |
291 | | - expect(fixture.killed).toEqual([800]); |
292 | | - |
293 | | - yield* TestClock.adjust(Duration.minutes(2)); |
294 | | - yield* fixture.reachability.check; |
295 | | - expect(fixture.killed).toEqual([800, 801]); |
296 | | - expect(yield* fixture.nextConnector).toBe(802); |
297 | | - }), |
298 | | - ); |
299 | | - |
300 | | - it.effect("needs a new streak after a healthy, DNS, or network result", () => |
301 | | - Effect.gen(function* () { |
302 | | - const fixture = yield* makeConnectFixture(); |
303 | | - yield* fixture.runtime.applyConfig(tunnelConfig); |
304 | | - yield* fixture.nextConnector; |
305 | | - for (const answer of ["200", "dns_failed", "network_error"] as const) { |
306 | | - yield* Ref.set(fixture.publicAnswer, "530"); |
307 | | - expect(yield* fixture.reachability.check).toMatchObject({ cause: "no_connector" }); |
308 | | - yield* TestClock.adjust(Duration.minutes(2)); |
309 | | - yield* Ref.set(fixture.publicAnswer, answer); |
310 | | - expect(yield* fixture.reachability.check).toMatchObject( |
311 | | - answer === "200" |
312 | | - ? { status: "reachable" } |
313 | | - : { cause: answer === "dns_failed" ? "dns_failed" : "timeout" }, |
314 | | - ); |
315 | | - } |
316 | | - yield* Ref.set(fixture.publicAnswer, "530"); |
317 | | - yield* fixture.reachability.check; |
318 | | - expect(fixture.killed).toEqual([]); |
319 | | - }), |
320 | | - ); |
321 | | - |
322 | | - it.effect("does not count a check that finishes after a newer one", () => |
323 | | - Effect.gen(function* () { |
324 | | - const fixture = yield* makeConnectFixture(); |
325 | | - yield* fixture.runtime.applyConfig(tunnelConfig); |
326 | | - yield* fixture.nextConnector; |
327 | | - yield* fixture.reachability.check; |
328 | | - yield* TestClock.adjust(Duration.minutes(2)); |
329 | | - |
330 | | - const older = yield* fixture.holdCheck; |
331 | | - yield* Ref.set(fixture.publicAnswer, "200"); |
332 | | - expect(yield* fixture.reachability.check).toMatchObject({ status: "reachable" }); |
333 | | - yield* older.release; |
334 | | - expect(yield* older.result).toMatchObject({ cause: "no_connector" }); |
335 | | - |
336 | | - yield* Ref.set(fixture.publicAnswer, "530"); |
337 | | - yield* TestClock.adjust(Duration.minutes(2)); |
338 | | - yield* fixture.reachability.check; |
339 | | - expect(fixture.killed).toEqual([]); |
340 | | - }), |
341 | | - ); |
342 | | - |
343 | | - it.effect("does not apply an older failure after a newer check resets recovery", () => |
344 | | - Effect.gen(function* () { |
345 | | - const fixture = yield* makeConnectFixture(); |
346 | | - yield* fixture.runtime.applyConfig(tunnelConfig); |
347 | | - yield* fixture.nextConnector; |
348 | | - const clock = yield* Clock.Clock; |
349 | | - const recoveryReached = yield* Deferred.make<void>(); |
350 | | - const releaseRecovery = yield* Deferred.make<void>(); |
351 | | - let clockReads = 0; |
352 | | - const older = yield* fixture.reachability.check.pipe( |
353 | | - Effect.provideService(Clock.Clock, { |
354 | | - ...clock, |
355 | | - currentTimeMillis: Effect.suspend(() => { |
356 | | - // The first read dates the published result; the second dates recovery. |
357 | | - if (++clockReads !== 2) return clock.currentTimeMillis; |
358 | | - return Deferred.succeed(recoveryReached, undefined).pipe( |
359 | | - Effect.andThen(Deferred.await(releaseRecovery)), |
360 | | - Effect.andThen(clock.currentTimeMillis), |
361 | | - ); |
362 | | - }), |
363 | | - }), |
364 | | - Effect.forkChild({ startImmediately: true }), |
| 94 | + const restarted: Array<number> = []; |
| 95 | + const reachability = yield* Effect.service(ConnectReachability.ConnectReachability).pipe( |
| 96 | + Effect.provide( |
| 97 | + ConnectReachability.layer.pipe( |
| 98 | + Layer.provide( |
| 99 | + Layer.mergeAll( |
| 100 | + Layer.mock(ManagedEndpointRuntime.CloudManagedEndpointRuntime)({ |
| 101 | + inspectConnector: Effect.succeed(connected), |
| 102 | + readinessChanged: Stream.never, |
| 103 | + restartConnector: (pid) => Effect.sync(() => restarted.push(pid) > 0), |
| 104 | + }), |
| 105 | + Layer.mock(ServerSecretStore.ServerSecretStore)({ |
| 106 | + get: () => Effect.succeed(Option.some(new Uint8Array())), |
| 107 | + }), |
| 108 | + // Cloudflare answers 530 while cloudflared still reports four ready connections. |
| 109 | + Layer.succeed( |
| 110 | + HttpClient.HttpClient, |
| 111 | + HttpClient.make((request) => |
| 112 | + Effect.succeed( |
| 113 | + HttpClientResponse.fromWeb(request, new Response(null, { status: 530 })), |
| 114 | + ), |
| 115 | + ), |
| 116 | + ), |
| 117 | + Layer.succeed( |
| 118 | + ChildProcessSpawner.ChildProcessSpawner, |
| 119 | + ChildProcessSpawner.make(() => Effect.die("no tailscale")), |
| 120 | + ), |
| 121 | + ), |
| 122 | + ), |
| 123 | + ), |
| 124 | + ), |
365 | 125 | ); |
366 | | - yield* Deferred.await(recoveryReached); |
367 | | - expect(Option.getOrThrow(yield* fixture.reachability.latest)).toMatchObject({ |
368 | | - cause: "no_connector", |
369 | | - }); |
370 | | - yield* Ref.set(fixture.publicAnswer, "200"); |
371 | | - expect(yield* fixture.reachability.check).toMatchObject({ status: "reachable" }); |
372 | | - yield* TestClock.adjust(Duration.minutes(2)); |
373 | | - yield* Deferred.succeed(releaseRecovery, undefined); |
374 | | - expect(yield* Fiber.join(older)).toMatchObject({ cause: "no_connector" }); |
375 | | - |
376 | | - yield* TestClock.adjust(Duration.minutes(2)); |
377 | | - yield* Ref.set(fixture.publicAnswer, "530"); |
378 | | - yield* fixture.reachability.check; |
379 | | - expect(fixture.killed).toEqual([]); |
380 | | - }), |
381 | | - ); |
382 | | - |
383 | | - it.effect("ignores a dead-connector result once its connector is replaced or removed", () => |
384 | | - Effect.gen(function* () { |
385 | | - for (const nextConfig of [{ ...tunnelConfig, connectorToken: "replacement" }, null]) { |
386 | | - yield* Effect.gen(function* () { |
387 | | - const fixture = yield* makeConnectFixture(); |
388 | | - yield* fixture.runtime.applyConfig(tunnelConfig); |
389 | | - yield* fixture.nextConnector; |
390 | | - yield* fixture.reachability.check; |
391 | | - yield* TestClock.adjust(Duration.minutes(2)); |
392 | 126 |
|
393 | | - const held = yield* fixture.holdCheck; |
394 | | - yield* fixture.runtime.applyConfig(nextConfig); |
395 | | - yield* held.release; |
396 | | - expect(yield* held.result).toMatchObject({ cause: "no_connector" }); |
| 127 | + expect(yield* reachability.check).toMatchObject({ cause: "no_connector" }); |
| 128 | + yield* TestClock.adjust(Duration.minutes(1)); |
| 129 | + yield* reachability.check; |
| 130 | + expect(restarted).toEqual([]); |
397 | 131 |
|
398 | | - expect(fixture.killed).toEqual([800]); |
399 | | - if (nextConfig) { |
400 | | - expect(yield* fixture.nextConnector).toBe(801); |
401 | | - expect(yield* fixture.runtime.inspectConnector).not.toBeNull(); |
402 | | - yield* fixture.reachability.check; |
403 | | - yield* TestClock.adjust(Duration.minutes(2)); |
404 | | - yield* fixture.reachability.check; |
405 | | - expect(fixture.killed).toEqual([800, 801]); |
406 | | - expect(yield* fixture.nextConnector).toBe(802); |
407 | | - } else { |
408 | | - expect(yield* fixture.runtime.inspectConnector).toBeNull(); |
409 | | - } |
410 | | - }).pipe(Effect.scoped); |
411 | | - } |
| 132 | + yield* TestClock.adjust(Duration.minutes(1)); |
| 133 | + yield* reachability.check; |
| 134 | + expect(restarted).toEqual([connected.pid]); |
412 | 135 | }), |
413 | 136 | ); |
414 | 137 | }); |
|
0 commit comments