@@ -3,6 +3,7 @@ import { vi } from "vite-plus/test";
33import * as Deferred from "effect/Deferred" ;
44import * as Duration from "effect/Duration" ;
55import * as Effect from "effect/Effect" ;
6+ import * as Exit from "effect/Exit" ;
67import * as Fiber from "effect/Fiber" ;
78import * as Layer from "effect/Layer" ;
89import * as Option from "effect/Option" ;
@@ -12,6 +13,7 @@ import * as Sink from "effect/Sink";
1213import * as Stream from "effect/Stream" ;
1314import * as TestClock from "effect/testing/TestClock" ;
1415import { ChildProcess , ChildProcessSpawner } from "effect/process" ;
16+ import { HttpClient , HttpClientError , HttpClientResponse } from "effect/http" ;
1517import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay" ;
1618import * as RelayClient from "@t3tools/shared/relayClient" ;
1719
@@ -33,12 +35,17 @@ const layerRelayClientAvailable = Layer.succeed(
3335 } ) ,
3436) ;
3537
38+ // Only connectors that log a metrics address are polled for readiness.
39+ const unusedHttpClient = HttpClient . make ( ( ) => Effect . die ( "unused" ) ) ;
40+
3641const runtimeDependencies = (
3742 spawner : ReturnType < typeof ChildProcessSpawner . make > ,
3843 relayClientLayer = layerRelayClientAvailable ,
44+ httpClient = unusedHttpClient ,
3945) =>
4046 Layer . mergeAll (
4147 Layer . succeed ( ChildProcessSpawner . ChildProcessSpawner , spawner ) ,
48+ Layer . succeed ( HttpClient . HttpClient , httpClient ) ,
4249 relayClientLayer ,
4350 Layer . mock ( ServerSecretStore . ServerSecretStore ) ( {
4451 get : ( ) => Effect . succeedNone ,
@@ -48,11 +55,12 @@ const runtimeDependencies = (
4855const buildCloudManagedEndpointRuntime = (
4956 spawner : ReturnType < typeof ChildProcessSpawner . make > ,
5057 relayClientLayer = layerRelayClientAvailable ,
58+ httpClient = unusedHttpClient ,
5159) =>
5260 Effect . gen ( function * ( ) {
5361 const context = yield * Layer . build (
5462 ManagedEndpointRuntime . layer . pipe (
55- Layer . provide ( runtimeDependencies ( spawner , relayClientLayer ) ) ,
63+ Layer . provide ( runtimeDependencies ( spawner , relayClientLayer , httpClient ) ) ,
5664 ) ,
5765 ) ;
5866 return yield * Effect . service ( ManagedEndpointRuntime . CloudManagedEndpointRuntime ) . pipe (
@@ -410,6 +418,97 @@ describe("CloudManagedEndpointRuntime", () => {
410418 } ) . pipe ( Effect . provide ( TestClock . layer ( ) ) ) ,
411419 ) ;
412420
421+ it . effect (
422+ "restarts a running connector whose readiness check keeps reporting no connections" ,
423+ ( ) =>
424+ Effect . gen ( function * ( ) {
425+ const spawned : Array < number > = [ ] ;
426+ const killed : Array < number > = [ ] ;
427+ const exits : Array < Deferred . Deferred < ChildProcessSpawner . ExitCode > > = [ ] ;
428+ const secondSpawned = yield * Deferred . make < void > ( ) ;
429+ const spawner = ChildProcessSpawner . make ( ( ) =>
430+ Effect . gen ( function * ( ) {
431+ const index = spawned . length ;
432+ const pid = 1000 + index ;
433+ const exit = yield * Deferred . make < ChildProcessSpawner . ExitCode > ( ) ;
434+ spawned . push ( pid ) ;
435+ exits . push ( exit ) ;
436+ if ( index === 1 ) yield * Deferred . succeed ( secondSpawned , undefined ) ;
437+ const handle = makeHandle ( {
438+ pid,
439+ exitCode : Deferred . await ( exit ) ,
440+ onKill : ( ) => {
441+ if ( ! killed . includes ( pid ) ) killed . push ( pid ) ;
442+ Deferred . doneUnsafe ( exit , Exit . succeed ( ChildProcessSpawner . ExitCode ( 143 ) ) ) ;
443+ } ,
444+ output : Stream . concat (
445+ Stream . make (
446+ new TextEncoder ( ) . encode (
447+ `2026-10-14T08:49:04Z INF Starting metrics server on 127.0.0.1:${ 40000 + index } /metrics\n` ,
448+ ) ,
449+ ) ,
450+ Stream . never ,
451+ ) ,
452+ } ) ;
453+ yield * Effect . addFinalizer ( ( ) => handle . kill ( ) . pipe ( Effect . ignore ) ) ;
454+ return handle ;
455+ } ) ,
456+ ) ;
457+ // Each readiness check takes the next scripted answer, then 503. It
458+ // records which connectors were killed before it was sent.
459+ const answers : Array < number | "refused" > = [ ] ;
460+ const checks = yield * Queue . unbounded < { readonly url : string ; readonly killed : string } > ( ) ;
461+ const httpClient = HttpClient . make ( ( request ) => {
462+ Queue . offerUnsafe ( checks , { url : request . url , killed : killed . join ( "," ) } ) ;
463+ const status = answers . shift ( ) ?? 503 ;
464+ return status === "refused"
465+ ? Effect . fail (
466+ new HttpClientError . HttpClientError ( {
467+ reason : new HttpClientError . TransportError ( { request } ) ,
468+ } ) ,
469+ )
470+ : Effect . succeed ( HttpClientResponse . fromWeb ( request , new Response ( null , { status } ) ) ) ;
471+ } ) ;
472+ // Advances the test clock until the watchdog checks readiness again.
473+ const nextCheck = Effect . gen ( function * ( ) {
474+ for ( let second = 0 ; second < 60 ; second += 1 ) {
475+ yield * TestClock . adjust ( Duration . seconds ( 1 ) ) ;
476+ const check = yield * Queue . poll ( checks ) ;
477+ if ( Option . isSome ( check ) ) return check . value ;
478+ }
479+ return yield * Effect . die ( "No readiness check within a minute" ) ;
480+ } ) ;
481+ const runtime = yield * buildCloudManagedEndpointRuntime (
482+ spawner ,
483+ layerRelayClientAvailable ,
484+ httpClient ,
485+ ) ;
486+ yield * runtime . applyConfig ( {
487+ providerKind : "cloudflare_tunnel" ,
488+ connectorToken : "token" ,
489+ tunnelId : "tunnel-1" ,
490+ } ) ;
491+
492+ // Five 503s, a failed request that does not count, a ready answer that
493+ // clears the streak, then six 503s in a row.
494+ answers . push ( 503 , 503 , 503 , 503 , 503 , "refused" , 200 ) ;
495+ for ( let check = 0 ; check < 13 ; check += 1 ) {
496+ expect ( yield * nextCheck ) . toEqual ( { url : "http://127.0.0.1:40000/ready" , killed : "" } ) ;
497+ }
498+ yield * Deferred . await ( secondSpawned ) ;
499+ expect ( killed ) . toEqual ( [ 1000 ] ) ;
500+ expect ( spawned ) . toEqual ( [ 1000 , 1001 ] ) ;
501+
502+ // The replacement follows a restart that never got a ready answer, so
503+ // it is killed only after twelve 503s.
504+ for ( let check = 0 ; check < 12 ; check += 1 ) {
505+ expect ( yield * nextCheck ) . toEqual ( { url : "http://127.0.0.1:40001/ready" , killed : "1000" } ) ;
506+ }
507+ yield * Deferred . await ( exits [ 1 ] ! ) ;
508+ expect ( killed ) . toEqual ( [ 1000 , 1001 ] ) ;
509+ } ) . pipe ( Effect . provide ( TestClock . layer ( ) ) ) ,
510+ ) ;
511+
413512 it . effect ( "starts, deduplicates, rotates, and stops the Cloudflare connector" , ( ) =>
414513 Effect . gen ( function * ( ) {
415514 const spawned : Array < ChildProcess . StandardCommand > = [ ] ;
@@ -457,8 +556,28 @@ describe("CloudManagedEndpointRuntime", () => {
457556
458557 expect ( spawned . map ( ( command ) => command . command ) ) . toEqual ( [ "cloudflared" , "cloudflared" ] ) ;
459558 expect ( spawned . map ( ( command ) => command . args ) ) . toEqual ( [
460- [ "tunnel" , "--no-autoupdate" , "--loglevel" , "info" , "--output" , "default" , "run" ] ,
461- [ "tunnel" , "--no-autoupdate" , "--loglevel" , "info" , "--output" , "default" , "run" ] ,
559+ [
560+ "tunnel" ,
561+ "--no-autoupdate" ,
562+ "--loglevel" ,
563+ "info" ,
564+ "--output" ,
565+ "default" ,
566+ "--metrics" ,
567+ "127.0.0.1:0" ,
568+ "run" ,
569+ ] ,
570+ [
571+ "tunnel" ,
572+ "--no-autoupdate" ,
573+ "--loglevel" ,
574+ "info" ,
575+ "--output" ,
576+ "default" ,
577+ "--metrics" ,
578+ "127.0.0.1:0" ,
579+ "run" ,
580+ ] ,
462581 ] ) ;
463582 expect ( spawned . map ( ( command ) => command . options . env ?. TUNNEL_TOKEN ) ) . toEqual ( [
464583 "token-1" ,
0 commit comments