11// @effect -diagnostics nodeBuiltinImport:off - reads the user guide to check its anchors.
22import * as NodeFS from "node:fs" ;
3+ import * as NodeNet from "node:net" ;
34
45import { T3_CONNECT_TROUBLESHOOTING_SECTIONS } from "@t3tools/contracts" ;
56import { describe , expect , it } from "@effect/vitest" ;
7+ import * as Clock from "effect/Clock" ;
8+ import * as Deferred from "effect/Deferred" ;
9+ import * as Duration from "effect/Duration" ;
10+ import * as Effect from "effect/Effect" ;
11+ import * as Fiber from "effect/Fiber" ;
12+ import * as Layer from "effect/Layer" ;
13+ import * as Queue from "effect/Queue" ;
14+ import * as Stream from "effect/Stream" ;
15+ import * as TestClock from "effect/testing/TestClock" ;
16+ import { HttpClient , HttpClientRequest , HttpClientResponse } from "effect/http" ;
17+ import { ChildProcessSpawner } from "effect/process" ;
18+ import { vi } from "vite-plus/test" ;
619
20+ import * as ServerSecretStore from "../auth/ServerSecretStore.ts" ;
21+ import * as ConnectReachability from "./ConnectReachability.ts" ;
722import { classifyReachability , type ReachabilityFacts } from "./ConnectReachability.ts" ;
23+ import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts" ;
24+
25+ vi . mock ( "node:net" , { spy : true } ) ;
826
927const checkedAt = "2026-10-19T00:00:00.000Z" ;
10- const connected = { readyConnections : 4 , hostname : "env.example.test" , protocol : "quic" } ;
11- const notConnected = { readyConnections : 0 , hostname : null , protocol : null } ;
28+ const connected = { pid : 1 , readyConnections : 4 , hostname : "env.example.test" , protocol : "quic" } ;
29+ const notConnected = { pid : 1 , readyConnections : 0 , hostname : null , protocol : null } ;
1230const facts = ( overrides : Partial < ReachabilityFacts > ) : ReachabilityFacts => ( {
1331 managedTunnelConfigured : true ,
1432 connector : connected ,
@@ -22,6 +40,8 @@ describe("classifyReachability", () => {
2240 it . each ( [
2341 [ { kind : "status" , status : 502 } , "origin_unreachable" ] ,
2442 [ { kind : "status" , status : 530 } , "no_connector" ] ,
43+ [ { kind : "status" , status : 530 , cloudflareError : "1033" } , "no_connector" ] ,
44+ [ { kind : "status" , status : 530 , cloudflareError : "1016" } , "unexpected_response" ] ,
2545 [ { kind : "network_error" } , "timeout" ] ,
2646 ] as const ) ( "maps endpoint result %o to %s" , ( endpoint , cause ) => {
2747 expect ( classifyReachability ( facts ( { endpoint, edgeReachable : true } ) , checkedAt ) ) . toMatchObject (
@@ -40,7 +60,12 @@ describe("classifyReachability", () => {
4060 expect (
4161 classifyReachability (
4262 facts ( {
43- connector : { readyConnections : null , hostname : "env.example.test" , protocol : null } ,
63+ connector : {
64+ pid : 1 ,
65+ readyConnections : null ,
66+ hostname : "env.example.test" ,
67+ protocol : null ,
68+ } ,
4469 endpoint : { kind : "status" , status : 502 } ,
4570 edgeReachable : false ,
4671 } ) ,
@@ -51,7 +76,12 @@ describe("classifyReachability", () => {
5176 expect (
5277 classifyReachability (
5378 facts ( {
54- connector : { readyConnections : null , hostname : "env.example.test" , protocol : null } ,
79+ connector : {
80+ pid : 1 ,
81+ readyConnections : null ,
82+ hostname : "env.example.test" ,
83+ protocol : null ,
84+ } ,
5585 endpoint : { kind : "dns_failed" } ,
5686 edgeReachable : false ,
5787 } ) ,
@@ -74,6 +104,280 @@ describe("classifyReachability", () => {
74104 } ) ;
75105} ) ;
76106
107+ interface ServedResponse {
108+ readonly status : number ;
109+ readonly body : string | ReadableStream < Uint8Array > | { readonly cloudflareError : number } ;
110+ }
111+
112+ // What Cloudflare serves for its 530 page: HTML unless the client accepts JSON.
113+ const cloudflareErrorPage = ( request : HttpClientRequest . HttpClientRequest , code : number ) =>
114+ request . headers . accept === "application/json"
115+ ? JSON . stringify ( { status : 530 , error_code : code , error_name : "tunnel_error" } )
116+ : `<!doctype html><h1><span>Error</span> <span>${ code } </span></h1>` ;
117+
118+ const reachabilityLayer = (
119+ respond : Effect . Effect < ServedResponse > ,
120+ restartConnector : ( pid : number , stillDead : Effect . Effect < boolean > ) => Effect . Effect < boolean > ,
121+ ) =>
122+ ConnectReachability . layer . pipe (
123+ Layer . provide (
124+ Layer . mergeAll (
125+ Layer . mock ( ManagedEndpointRuntime . CloudManagedEndpointRuntime ) ( {
126+ inspectConnector : Effect . succeed ( connected ) ,
127+ readinessChanged : Stream . never ,
128+ restartConnector,
129+ } ) ,
130+ Layer . mock ( ServerSecretStore . ServerSecretStore ) ( {
131+ get : ( ) => Effect . succeedSome ( new Uint8Array ( ) ) ,
132+ } ) ,
133+ Layer . succeed (
134+ HttpClient . HttpClient ,
135+ HttpClient . make ( ( request ) =>
136+ respond . pipe (
137+ Effect . map ( ( { status, body } ) =>
138+ HttpClientResponse . fromWeb (
139+ request ,
140+ new Response (
141+ typeof body === "object" && "cloudflareError" in body
142+ ? cloudflareErrorPage ( request , body . cloudflareError )
143+ : body ,
144+ { status } ,
145+ ) ,
146+ ) ,
147+ ) ,
148+ ) ,
149+ ) ,
150+ ) ,
151+ Layer . succeed (
152+ ChildProcessSpawner . ChildProcessSpawner ,
153+ ChildProcessSpawner . make ( ( ) => Effect . die ( "no tailscale" ) ) ,
154+ ) ,
155+ ) ,
156+ ) ,
157+ ) ;
158+
159+ const mockEdgeConnect = ( ) =>
160+ vi . spyOn ( NodeNet , "connect" ) . mockImplementation ( ( ) => {
161+ const socket = new NodeNet . Socket ( ) ;
162+ queueMicrotask ( ( ) => socket . emit ( "connect" ) ) ;
163+ return socket ;
164+ } ) ;
165+
166+ describe ( "ConnectReachability stale connector recovery" , ( ) => {
167+ it . effect ( "restarts cloudflared when Cloudflare has no connector but it reports ready" , ( ) =>
168+ Effect . gen ( function * ( ) {
169+ using _connect = mockEdgeConnect ( ) ;
170+ const restarts = yield * Queue . unbounded < number > ( ) ;
171+ // Cloudflare answers 530 while cloudflared still reports four ready connections.
172+ let status = 530 ;
173+ let body = { cloudflareError : 1033 } ;
174+ const layer = reachabilityLayer (
175+ Effect . sync ( ( ) => ( { status, body } ) ) ,
176+ ( pid , stillDead ) =>
177+ stillDead . pipe ( Effect . tap ( ( dead ) => ( dead ? Queue . offer ( restarts , pid ) : Effect . void ) ) ) ,
178+ ) ;
179+ // The scheduled checks run in the layer's scope, so the assertions run inside it.
180+ yield * Effect . gen ( function * ( ) {
181+ const reachability = yield * ConnectReachability . ConnectReachability ;
182+
183+ // The first sighting schedules its own confirming check two minutes later.
184+ expect ( yield * reachability . check ) . toMatchObject ( { cause : "no_connector" } ) ;
185+ yield * TestClock . adjust ( Duration . minutes ( 1 ) ) ;
186+ expect ( yield * Queue . size ( restarts ) ) . toBe ( 0 ) ;
187+ yield * TestClock . adjust ( Duration . minutes ( 1 ) ) ;
188+ expect ( yield * Queue . take ( restarts ) ) . toBe ( connected . pid ) ;
189+
190+ // A restart that did not help doubles the wait.
191+ yield * reachability . check ;
192+ yield * TestClock . adjust ( Duration . minutes ( 2 ) ) ;
193+ expect ( yield * Queue . size ( restarts ) ) . toBe ( 0 ) ;
194+ yield * TestClock . adjust ( Duration . minutes ( 2 ) ) ;
195+ expect ( yield * Queue . take ( restarts ) ) . toBe ( connected . pid ) ;
196+
197+ // Reaching the endpoint resets the wait, and a 530 for a broken DNS record is left alone.
198+ status = 200 ;
199+ expect ( yield * reachability . check ) . toMatchObject ( { status : "reachable" } ) ;
200+ status = 530 ;
201+ body = { cloudflareError : 1016 } ;
202+ expect ( yield * reachability . check ) . toMatchObject ( { cause : "unexpected_response" } ) ;
203+ yield * TestClock . adjust ( Duration . minutes ( 2 ) ) ;
204+ expect ( yield * Queue . size ( restarts ) ) . toBe ( 0 ) ;
205+
206+ body = { cloudflareError : 1033 } ;
207+ yield * reachability . check ;
208+ yield * TestClock . adjust ( Duration . minutes ( 2 ) ) ;
209+ expect ( yield * Queue . take ( restarts ) ) . toBe ( connected . pid ) ;
210+ } ) . pipe ( Effect . provide ( layer ) ) ;
211+ } ) ,
212+ ) ;
213+
214+ it . effect ( "starts a new streak after a healthy check that a newer check overtook" , ( ) =>
215+ Effect . gen ( function * ( ) {
216+ using _connect = mockEdgeConnect ( ) ;
217+ const restartTimes = yield * Queue . unbounded < number > ( ) ;
218+ const healthyRequested = yield * Deferred . make < void > ( ) ;
219+ const healthyAnswer = yield * Deferred . make < void > ( ) ;
220+ let status = 530 ;
221+ const layer = reachabilityLayer (
222+ Effect . gen ( function * ( ) {
223+ const served = status ;
224+ if ( served === 200 ) {
225+ yield * Deferred . succeed ( healthyRequested , undefined ) ;
226+ yield * Deferred . await ( healthyAnswer ) ;
227+ }
228+ return { status : served , body : { cloudflareError : 1033 } } ;
229+ } ) ,
230+ ( _pid , stillDead ) =>
231+ stillDead . pipe (
232+ Effect . tap ( ( dead ) =>
233+ dead
234+ ? Clock . currentTimeMillis . pipe (
235+ Effect . flatMap ( ( now ) => Queue . offer ( restartTimes , now ) ) ,
236+ )
237+ : Effect . void ,
238+ ) ,
239+ ) ,
240+ ) ;
241+ yield * Effect . gen ( function * ( ) {
242+ const reachability = yield * ConnectReachability . ConnectReachability ;
243+ const startedAt = yield * Clock . currentTimeMillis ;
244+ yield * reachability . check ;
245+ yield * TestClock . adjust ( Duration . minutes ( 1 ) ) ;
246+
247+ // The healthy answer arrives after a newer check already reported the tunnel dead.
248+ status = 200 ;
249+ const healthy = yield * Effect . forkChild ( reachability . check ) ;
250+ yield * Deferred . await ( healthyRequested ) ;
251+ status = 530 ;
252+ yield * reachability . check ;
253+ yield * Deferred . succeed ( healthyAnswer , undefined ) ;
254+ expect ( yield * Fiber . join ( healthy ) ) . toMatchObject ( { status : "reachable" } ) ;
255+
256+ // The grace period starts again at the next dead check, a minute after the first one.
257+ yield * reachability . check ;
258+ yield * TestClock . adjust ( Duration . minutes ( 2 ) ) ;
259+ expect ( ( yield * Queue . take ( restartTimes ) ) - startedAt ) . toBe (
260+ Duration . toMillis ( Duration . minutes ( 3 ) ) ,
261+ ) ;
262+ } ) . pipe ( Effect . provide ( layer ) ) ;
263+ } ) ,
264+ ) ;
265+
266+ it . effect ( "rejects a restart when an overtaken healthy check finishes while it waits" , ( ) =>
267+ Effect . gen ( function * ( ) {
268+ using _connect = mockEdgeConnect ( ) ;
269+ const healthyRequested = yield * Deferred . make < void > ( ) ;
270+ const healthyAnswer = yield * Deferred . make < void > ( ) ;
271+ const restartRequested = yield * Deferred . make < void > ( ) ;
272+ const releaseRestart = yield * Deferred . make < void > ( ) ;
273+ const guardChecks = yield * Queue . unbounded < boolean > ( ) ;
274+ let status = 530 ;
275+ const layer = reachabilityLayer (
276+ Effect . gen ( function * ( ) {
277+ const served = status ;
278+ if ( served === 200 ) {
279+ yield * Deferred . succeed ( healthyRequested , undefined ) ;
280+ yield * Deferred . await ( healthyAnswer ) ;
281+ }
282+ return { status : served , body : { cloudflareError : 1033 } } ;
283+ } ) ,
284+ ( _pid , stillDead ) =>
285+ Effect . gen ( function * ( ) {
286+ yield * Deferred . succeed ( restartRequested , undefined ) ;
287+ yield * Deferred . await ( releaseRestart ) ;
288+ const accepted = yield * stillDead ;
289+ yield * Queue . offer ( guardChecks , accepted ) ;
290+ return accepted ;
291+ } ) ,
292+ ) ;
293+ yield * Effect . gen ( function * ( ) {
294+ const reachability = yield * ConnectReachability . ConnectReachability ;
295+ yield * reachability . check ;
296+ yield * TestClock . adjust ( Duration . seconds ( 119 ) ) ;
297+
298+ status = 200 ;
299+ const healthy = yield * Effect . forkChild ( reachability . check ) ;
300+ yield * Deferred . await ( healthyRequested ) ;
301+ status = 530 ;
302+ yield * TestClock . adjust ( Duration . seconds ( 1 ) ) ;
303+ yield * Deferred . await ( restartRequested ) ;
304+
305+ yield * Deferred . succeed ( healthyAnswer , undefined ) ;
306+ expect ( yield * Fiber . join ( healthy ) ) . toMatchObject ( { status : "reachable" } ) ;
307+ yield * Deferred . succeed ( releaseRestart , undefined ) ;
308+ expect ( yield * Queue . take ( guardChecks ) ) . toBe ( false ) ;
309+
310+ yield * reachability . check ;
311+ yield * TestClock . adjust ( Duration . minutes ( 1 ) ) ;
312+ expect ( yield * Queue . size ( guardChecks ) ) . toBe ( 0 ) ;
313+ yield * TestClock . adjust ( Duration . minutes ( 1 ) ) ;
314+ expect ( yield * Queue . take ( guardChecks ) ) . toBe ( true ) ;
315+ } ) . pipe ( Effect . provide ( layer ) ) ;
316+ } ) ,
317+ ) ;
318+
319+ it . effect ( "drops the confirming check of a streak that already ended" , ( ) =>
320+ Effect . gen ( function * ( ) {
321+ using _connect = mockEdgeConnect ( ) ;
322+ const restarts = yield * Queue . unbounded < number > ( ) ;
323+ let status = 530 ;
324+ let requests = 0 ;
325+ const layer = reachabilityLayer (
326+ Effect . sync ( ( ) => {
327+ requests += 1 ;
328+ return { status, body : { cloudflareError : 1033 } } ;
329+ } ) ,
330+ ( pid , stillDead ) =>
331+ stillDead . pipe ( Effect . tap ( ( dead ) => ( dead ? Queue . offer ( restarts , pid ) : Effect . void ) ) ) ,
332+ ) ;
333+ yield * Effect . gen ( function * ( ) {
334+ const reachability = yield * ConnectReachability . ConnectReachability ;
335+ yield * reachability . check ;
336+ yield * TestClock . adjust ( Duration . minutes ( 1 ) ) ;
337+ status = 200 ;
338+ yield * reachability . check ;
339+ status = 530 ;
340+ yield * reachability . check ;
341+
342+ const before = requests ;
343+ yield * TestClock . adjust ( Duration . minutes ( 1 ) ) ;
344+ expect ( requests ) . toBe ( before ) ;
345+ yield * TestClock . adjust ( Duration . minutes ( 1 ) ) ;
346+ expect ( yield * Queue . take ( restarts ) ) . toBe ( connected . pid ) ;
347+ } ) . pipe ( Effect . provide ( layer ) ) ;
348+ } ) ,
349+ ) ;
350+
351+ it . effect ( "keeps a 530 whose error body never finishes" , ( ) =>
352+ Effect . gen ( function * ( ) {
353+ using _connect = mockEdgeConnect ( ) ;
354+ let markBodyRead = ( ) => { } ;
355+ const bodyRead = new Promise < void > ( ( resolve ) => {
356+ markBodyRead = resolve ;
357+ } ) ;
358+ const layer = reachabilityLayer (
359+ Effect . sync ( ( ) => ( {
360+ status : 530 ,
361+ body : new ReadableStream ( {
362+ pull : ( ) => {
363+ markBodyRead ( ) ;
364+ return new Promise < void > ( ( ) => { } ) ;
365+ } ,
366+ } ) ,
367+ } ) ) ,
368+ ( ) => Effect . succeed ( false ) ,
369+ ) ;
370+ yield * Effect . gen ( function * ( ) {
371+ const reachability = yield * ConnectReachability . ConnectReachability ;
372+ const check = yield * Effect . forkChild ( reachability . check ) ;
373+ yield * Effect . promise ( ( ) => bodyRead ) ;
374+ yield * TestClock . adjust ( Duration . seconds ( 2 ) ) ;
375+ expect ( yield * Fiber . join ( check ) ) . toMatchObject ( { cause : "no_connector" } ) ;
376+ } ) . pipe ( Effect . provide ( layer ) ) ;
377+ } ) ,
378+ ) ;
379+ } ) ;
380+
77381describe ( "T3 Connect troubleshooting links" , ( ) => {
78382 it ( "each section names a heading in the user guide" , ( ) => {
79383 const guide = NodeFS . readFileSync (
0 commit comments