22import * as NodeFS from "node:fs" ;
33
44import { T3_CONNECT_TROUBLESHOOTING_SECTIONS } from "@t3tools/contracts" ;
5+ import * as RelayClient from "@t3tools/shared/relayClient" ;
56import { describe , expect , it } from "@effect/vitest" ;
7+ import * as Deferred from "effect/Deferred" ;
8+ import * as Duration from "effect/Duration" ;
9+ import * as Effect from "effect/Effect" ;
10+ import * as Exit from "effect/Exit" ;
11+ import * as Fiber from "effect/Fiber" ;
12+ import * as Layer from "effect/Layer" ;
13+ import * as Option from "effect/Option" ;
14+ import * as PlatformError from "effect/PlatformError" ;
15+ import * as Queue from "effect/Queue" ;
16+ import * as Ref from "effect/Ref" ;
17+ import * as Sink from "effect/Sink" ;
18+ import * as Stream from "effect/Stream" ;
19+ import * as TestClock from "effect/testing/TestClock" ;
20+ import { HttpClient , HttpClientResponse } from "effect/http" ;
21+ import * as HttpClientError from "effect/http/HttpClientError" ;
22+ import { ChildProcess , ChildProcessSpawner } from "effect/process" ;
623
24+ import * as ServerSecretStore from "../auth/ServerSecretStore.ts" ;
25+ import { CLOUD_ENDPOINT_RUNTIME_CONFIG } from "./config.ts" ;
26+ import * as ConnectReachability from "./ConnectReachability.ts" ;
727import { classifyReachability , type ReachabilityFacts } from "./ConnectReachability.ts" ;
28+ import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts" ;
829
930const checkedAt = "2026-10-19T00:00:00.000Z" ;
10- const connected = { readyConnections : 4 , hostname : "env.example.test" , protocol : "quic" } ;
11- const notConnected = { readyConnections : 0 , hostname : null , protocol : null } ;
31+ const connected = { pid : 1 , readyConnections : 4 , hostname : "env.example.test" , protocol : "quic" } ;
32+ const notConnected = { pid : 1 , readyConnections : 0 , hostname : null , protocol : null } ;
1233const facts = ( overrides : Partial < ReachabilityFacts > ) : ReachabilityFacts => ( {
1334 managedTunnelConfigured : true ,
1435 connector : connected ,
@@ -40,7 +61,12 @@ describe("classifyReachability", () => {
4061 expect (
4162 classifyReachability (
4263 facts ( {
43- connector : { readyConnections : null , hostname : "env.example.test" , protocol : null } ,
64+ connector : {
65+ pid : 1 ,
66+ readyConnections : null ,
67+ hostname : "env.example.test" ,
68+ protocol : null ,
69+ } ,
4470 endpoint : { kind : "status" , status : 502 } ,
4571 edgeReachable : false ,
4672 } ) ,
@@ -51,7 +77,12 @@ describe("classifyReachability", () => {
5177 expect (
5278 classifyReachability (
5379 facts ( {
54- connector : { readyConnections : null , hostname : "env.example.test" , protocol : null } ,
80+ connector : {
81+ pid : 1 ,
82+ readyConnections : null ,
83+ hostname : "env.example.test" ,
84+ protocol : null ,
85+ } ,
5586 endpoint : { kind : "dns_failed" } ,
5687 edgeReachable : false ,
5788 } ) ,
@@ -67,6 +98,275 @@ describe("classifyReachability", () => {
6798 } ) ;
6899} ) ;
69100
101+ const tunnelConfig = {
102+ providerKind : "cloudflare_tunnel" as const ,
103+ connectorToken : "token" ,
104+ tunnelId : "tunnel-1" ,
105+ } ;
106+ const metricsHost = "127.0.0.1:20241" ;
107+ const publicUrl = "https://env.example.test/.well-known/t3/environment" ;
108+ const connectorStartup = new TextEncoder ( ) . encode (
109+ `2026-10-25T00:00:00Z INF Starting metrics server on ${ metricsHost } /metrics\n` +
110+ "2026-10-25T00:00:00Z INF Registered tunnel connection connIndex=0 protocol=http2\n" ,
111+ ) ;
112+
113+ type PublicAnswer = "530" | "200" | "dns_failed" | "network_error" ;
114+
115+ // Real runtime and reachability; only cloudflared, its metrics server, and the public
116+ // endpoint are fake. The metrics server keeps reporting four ready connections, as a
117+ // connector the edge dropped after a host network change does.
118+ const makeConnectFixture = Effect . fnUntraced ( function * ( ) {
119+ const started = yield * Queue . unbounded < number > ( ) ;
120+ const killed : Array < number > = [ ] ;
121+ const publicAnswer = yield * Ref . make < PublicAnswer > ( "530" ) ;
122+ const publicGate = yield * Ref . make < {
123+ readonly entered : Deferred . Deferred < void > ;
124+ readonly release : Deferred . Deferred < void > ;
125+ } | null > ( null ) ;
126+ let nextPid = 800 ;
127+ const spawner = ChildProcessSpawner . make ( ( command ) =>
128+ Effect . gen ( function * ( ) {
129+ if ( ! ChildProcess . isStandardCommand ( command ) || command . command !== "cloudflared" ) {
130+ return yield * PlatformError . systemError ( {
131+ _tag : "NotFound" ,
132+ module : "ChildProcess" ,
133+ method : "spawn" ,
134+ description : "not installed" ,
135+ } ) ;
136+ }
137+ const pid = nextPid ++ ;
138+ const exit = yield * Deferred . make < ChildProcessSpawner . ExitCode > ( ) ;
139+ let exited = false ;
140+ const handle = ChildProcessSpawner . makeHandle ( {
141+ pid : ChildProcessSpawner . ProcessId ( pid ) ,
142+ exitCode : Deferred . await ( exit ) ,
143+ isRunning : Effect . sync ( ( ) => ! exited ) ,
144+ kill : ( ) =>
145+ Effect . sync ( ( ) => {
146+ if ( exited ) return ;
147+ exited = true ;
148+ killed . push ( pid ) ;
149+ Deferred . doneUnsafe ( exit , Exit . succeed ( ChildProcessSpawner . ExitCode ( 143 ) ) ) ;
150+ } ) ,
151+ unref : Effect . succeed ( Effect . void ) ,
152+ stdin : Sink . drain ,
153+ stdout : Stream . empty ,
154+ stderr : Stream . empty ,
155+ all : Stream . concat ( Stream . make ( connectorStartup ) , Stream . never ) ,
156+ getInputFd : ( ) => Sink . drain ,
157+ getOutputFd : ( ) => Stream . empty ,
158+ } ) ;
159+ yield * Effect . addFinalizer ( ( ) => handle . kill ( ) . pipe ( Effect . ignore ) ) ;
160+ yield * Queue . offer ( started , pid ) ;
161+ return handle ;
162+ } ) ,
163+ ) ;
164+ const httpClient = HttpClient . make ( ( request ) =>
165+ Effect . gen ( function * ( ) {
166+ if ( request . url === `http://${ metricsHost } /ready` ) {
167+ return HttpClientResponse . fromWeb ( request , Response . json ( { readyConnections : 4 } ) ) ;
168+ }
169+ if ( request . url === `http://${ metricsHost } /config` ) {
170+ return HttpClientResponse . fromWeb (
171+ request ,
172+ Response . json ( { config : { ingress : [ { hostname : "env.example.test" } , { } ] } } ) ,
173+ ) ;
174+ }
175+ if ( request . url !== publicUrl ) return yield * Effect . die ( `unexpected ${ request . url } ` ) ;
176+ const answer = yield * Ref . get ( publicAnswer ) ;
177+ const gate = yield * Ref . getAndSet ( publicGate , null ) ;
178+ if ( gate ) {
179+ yield * Deferred . succeed ( gate . entered , undefined ) ;
180+ yield * Deferred . await ( gate . release ) ;
181+ }
182+ if ( answer === "530" || answer === "200" ) {
183+ return HttpClientResponse . fromWeb ( request , new Response ( null , { status : Number ( answer ) } ) ) ;
184+ }
185+ return yield * new HttpClientError . HttpClientError ( {
186+ reason : new HttpClientError . TransportError ( {
187+ request,
188+ cause : Object . assign ( new Error ( answer ) , {
189+ code : answer === "dns_failed" ? "ENOTFOUND" : "ECONNRESET" ,
190+ } ) ,
191+ } ) ,
192+ } ) ;
193+ } ) ,
194+ ) ;
195+ const runtimeLayer = ManagedEndpointRuntime . layer . pipe (
196+ Layer . provide (
197+ Layer . succeed (
198+ RelayClient . RelayClient ,
199+ RelayClient . RelayClient . of ( {
200+ resolve : Effect . succeed ( {
201+ status : "available" ,
202+ executablePath : "cloudflared" ,
203+ source : "path" ,
204+ version : RelayClient . CLOUDFLARED_VERSION ,
205+ } ) ,
206+ install : Effect . die ( "unused" ) ,
207+ installWithProgress : ( ) => Effect . die ( "unused" ) ,
208+ pruneManagedVersions : Effect . void ,
209+ } ) ,
210+ ) ,
211+ ) ,
212+ ) ;
213+ const context = yield * Layer . build (
214+ ConnectReachability . layer . pipe (
215+ Layer . provideMerge ( runtimeLayer ) ,
216+ Layer . provide (
217+ Layer . mergeAll (
218+ Layer . succeed ( ChildProcessSpawner . ChildProcessSpawner , spawner ) ,
219+ Layer . succeed ( HttpClient . HttpClient , httpClient ) ,
220+ Layer . mock ( ServerSecretStore . ServerSecretStore ) ( {
221+ get : ( name ) =>
222+ Effect . succeed (
223+ name === CLOUD_ENDPOINT_RUNTIME_CONFIG
224+ ? Option . some ( new Uint8Array ( ) )
225+ : Option . none ( ) ,
226+ ) ,
227+ } ) ,
228+ ) ,
229+ ) ,
230+ ) ,
231+ ) ;
232+ const runtime = yield * Effect . service ( ManagedEndpointRuntime . CloudManagedEndpointRuntime ) . pipe (
233+ Effect . provide ( context ) ,
234+ ) ;
235+ const reachability = yield * Effect . service ( ConnectReachability . ConnectReachability ) . pipe (
236+ Effect . provide ( context ) ,
237+ ) ;
238+ /** Waits for the next cloudflared to start and register, so checks see its metrics. */
239+ const nextConnector = Effect . gen ( function * ( ) {
240+ const pid = yield * Queue . take ( started ) ;
241+ yield * Stream . runHead ( runtime . tunnelConnected ) ;
242+ return pid ;
243+ } ) ;
244+ /** Starts a check and holds it at the public probe, after it read the connector. */
245+ const holdCheck = Effect . gen ( function * ( ) {
246+ const entered = yield * Deferred . make < void > ( ) ;
247+ const release = yield * Deferred . make < void > ( ) ;
248+ yield * Ref . set ( publicGate , { entered, release } ) ;
249+ const fiber = yield * reachability . check . pipe ( Effect . forkChild ( { startImmediately : true } ) ) ;
250+ yield * Deferred . await ( entered ) ;
251+ return { release : Deferred . succeed ( release , undefined ) , result : Fiber . join ( fiber ) } ;
252+ } ) ;
253+ return { runtime, reachability, killed, publicAnswer, nextConnector, holdCheck } ;
254+ } ) ;
255+
256+ describe ( "ConnectReachability stale connector recovery" , ( ) => {
257+ it . effect ( "restarts cloudflared when Cloudflare has no connector but it reports ready" , ( ) =>
258+ Effect . gen ( function * ( ) {
259+ const fixture = yield * makeConnectFixture ( ) ;
260+ yield * fixture . runtime . applyConfig ( tunnelConfig ) ;
261+ expect ( yield * fixture . nextConnector ) . toBe ( 800 ) ;
262+
263+ expect ( yield * fixture . reachability . check ) . toMatchObject ( { cause : "no_connector" } ) ;
264+ expect ( fixture . killed ) . toEqual ( [ ] ) ;
265+
266+ yield * TestClock . adjust ( Duration . minutes ( 2 ) ) ;
267+ expect ( yield * fixture . reachability . check ) . toMatchObject ( { cause : "no_connector" } ) ;
268+ expect ( fixture . killed ) . toEqual ( [ 800 ] ) ;
269+ expect ( yield * fixture . nextConnector ) . toBe ( 801 ) ;
270+
271+ yield * Ref . set ( fixture . publicAnswer , "200" ) ;
272+ expect ( yield * fixture . reachability . check ) . toMatchObject ( { status : "reachable" } ) ;
273+ } ) ,
274+ ) ;
275+
276+ it . effect ( "backs off restarts while Cloudflare keeps reporting no connector" , ( ) =>
277+ Effect . gen ( function * ( ) {
278+ const fixture = yield * makeConnectFixture ( ) ;
279+ yield * fixture . runtime . applyConfig ( tunnelConfig ) ;
280+ yield * fixture . nextConnector ;
281+ yield * fixture . reachability . check ;
282+ yield * TestClock . adjust ( Duration . minutes ( 2 ) ) ;
283+ yield * fixture . reachability . check ;
284+ expect ( fixture . killed ) . toEqual ( [ 800 ] ) ;
285+ expect ( yield * fixture . nextConnector ) . toBe ( 801 ) ;
286+
287+ yield * fixture . reachability . check ;
288+ yield * TestClock . adjust ( Duration . minutes ( 2 ) ) ;
289+ yield * fixture . reachability . check ;
290+ expect ( fixture . killed ) . toEqual ( [ 800 ] ) ;
291+
292+ yield * TestClock . adjust ( Duration . minutes ( 2 ) ) ;
293+ yield * fixture . reachability . check ;
294+ expect ( fixture . killed ) . toEqual ( [ 800 , 801 ] ) ;
295+ expect ( yield * fixture . nextConnector ) . toBe ( 802 ) ;
296+ } ) ,
297+ ) ;
298+
299+ it . effect ( "needs a new streak after a healthy, DNS, or network result" , ( ) =>
300+ Effect . gen ( function * ( ) {
301+ const fixture = yield * makeConnectFixture ( ) ;
302+ yield * fixture . runtime . applyConfig ( tunnelConfig ) ;
303+ yield * fixture . nextConnector ;
304+ for ( const answer of [ "200" , "dns_failed" , "network_error" ] as const ) {
305+ yield * Ref . set ( fixture . publicAnswer , "530" ) ;
306+ expect ( yield * fixture . reachability . check ) . toMatchObject ( { cause : "no_connector" } ) ;
307+ yield * TestClock . adjust ( Duration . minutes ( 2 ) ) ;
308+ yield * Ref . set ( fixture . publicAnswer , answer ) ;
309+ expect ( yield * fixture . reachability . check ) . toMatchObject (
310+ answer === "200"
311+ ? { status : "reachable" }
312+ : { cause : answer === "dns_failed" ? "dns_failed" : "timeout" } ,
313+ ) ;
314+ }
315+ yield * Ref . set ( fixture . publicAnswer , "530" ) ;
316+ yield * fixture . reachability . check ;
317+ expect ( fixture . killed ) . toEqual ( [ ] ) ;
318+ } ) ,
319+ ) ;
320+
321+ it . effect ( "does not count a check that finishes after a newer one" , ( ) =>
322+ Effect . gen ( function * ( ) {
323+ const fixture = yield * makeConnectFixture ( ) ;
324+ yield * fixture . runtime . applyConfig ( tunnelConfig ) ;
325+ yield * fixture . nextConnector ;
326+ yield * fixture . reachability . check ;
327+ yield * TestClock . adjust ( Duration . minutes ( 2 ) ) ;
328+
329+ const older = yield * fixture . holdCheck ;
330+ yield * Ref . set ( fixture . publicAnswer , "200" ) ;
331+ expect ( yield * fixture . reachability . check ) . toMatchObject ( { status : "reachable" } ) ;
332+ yield * older . release ;
333+ expect ( yield * older . result ) . toMatchObject ( { cause : "no_connector" } ) ;
334+
335+ yield * Ref . set ( fixture . publicAnswer , "530" ) ;
336+ yield * TestClock . adjust ( Duration . minutes ( 2 ) ) ;
337+ yield * fixture . reachability . check ;
338+ expect ( fixture . killed ) . toEqual ( [ ] ) ;
339+ } ) ,
340+ ) ;
341+
342+ it . effect ( "ignores a dead-connector result once its connector is replaced or removed" , ( ) =>
343+ Effect . gen ( function * ( ) {
344+ for ( const nextConfig of [ { ...tunnelConfig , connectorToken : "replacement" } , null ] ) {
345+ yield * Effect . gen ( function * ( ) {
346+ const fixture = yield * makeConnectFixture ( ) ;
347+ yield * fixture . runtime . applyConfig ( tunnelConfig ) ;
348+ yield * fixture . nextConnector ;
349+ yield * fixture . reachability . check ;
350+ yield * TestClock . adjust ( Duration . minutes ( 2 ) ) ;
351+
352+ const held = yield * fixture . holdCheck ;
353+ yield * fixture . runtime . applyConfig ( nextConfig ) ;
354+ yield * held . release ;
355+ expect ( yield * held . result ) . toMatchObject ( { cause : "no_connector" } ) ;
356+
357+ expect ( fixture . killed ) . toEqual ( [ 800 ] ) ;
358+ if ( nextConfig ) {
359+ expect ( yield * fixture . nextConnector ) . toBe ( 801 ) ;
360+ expect ( yield * fixture . runtime . inspectConnector ) . not . toBeNull ( ) ;
361+ } else {
362+ expect ( yield * fixture . runtime . inspectConnector ) . toBeNull ( ) ;
363+ }
364+ } ) . pipe ( Effect . scoped ) ;
365+ }
366+ } ) ,
367+ ) ;
368+ } ) ;
369+
70370describe ( "T3 Connect troubleshooting links" , ( ) => {
71371 it ( "each section names a heading in the user guide" , ( ) => {
72372 const guide = NodeFS . readFileSync (
0 commit comments