@@ -30,6 +30,7 @@ import {
3030 registerServiceTask ,
3131 serviceLauncherIsRunning ,
3232 serviceLauncherLockPath ,
33+ stateDirOwned ,
3334 writeServiceState ,
3435} from "./serviceLauncher.ts" ;
3536import {
@@ -44,12 +45,27 @@ import {
4445} from "./cloud/serviceProtocol.ts" ;
4546
4647const spawnedChildren = vi . hoisted ( ( ) : Array < NodeChildProcess . ChildProcess > => [ ] ) ;
48+ const spawnHook = vi . hoisted ( ( ) => ( {
49+ current : undefined as ( ( child : NodeChildProcess . ChildProcess ) => void ) | undefined ,
50+ } ) ) ;
51+
52+ const failRm = vi . hoisted ( ( ) => ( { path : undefined as string | undefined } ) ) ;
53+
54+ vi . mock ( "node:fs/promises" , async ( importOriginal ) => {
55+ const actual = await importOriginal < typeof NodeFSP > ( ) ;
56+ const rm : typeof actual . rm = ( path , options ) =>
57+ path === failRm . path
58+ ? Promise . reject ( Object . assign ( new Error ( "EACCES" ) , { code : "EACCES" } ) )
59+ : actual . rm ( path , options ) ;
60+ return { ...actual , rm } ;
61+ } ) ;
4762
4863vi . mock ( "node:child_process" , async ( importOriginal ) => {
4964 const actual = await importOriginal < typeof NodeChildProcess > ( ) ;
5065 const spawn = ( ...args : Parameters < typeof actual . spawn > ) => {
5166 const child = actual . spawn ( ...args ) ;
5267 spawnedChildren . push ( child ) ;
68+ spawnHook . current ?.( child ) ;
5369 return child ;
5470 } ;
5571 return { ...actual , spawn : spawn as typeof actual . spawn } ;
@@ -578,25 +594,36 @@ process.exit(0);
578594 } ) ,
579595 ) ;
580596
581- it . effect ( "stays idle instead of exiting when another server owns the T3 home" , ( ) =>
597+ it . effect ( "waits without exiting and restarts once the other owner releases the T3 home" , ( ) =>
582598 Effect . gen ( function * ( ) {
583599 const fs = yield * FileSystem . FileSystem ;
584600 const path = yield * Path . Path ;
585- const root = yield * fs . makeTempDirectoryScoped ( { prefix : "t3-service-refused -" } ) ;
601+ const root = yield * fs . makeTempDirectoryScoped ( { prefix : "t3-service-reclaim -" } ) ;
586602 const statePath = path . join ( root , "runtime" , "service-state.json" ) ;
603+ const ownerPath = path . join ( root , "userdata" , "server-owner.sqlite" ) ;
604+ yield * fs . makeDirectory ( path . dirname ( ownerPath ) , { recursive : true } ) ;
587605 yield * writeFakeRuntime (
588606 fs ,
589607 path ,
590608 path . join ( root , "runtime" , "versions" , "1.0.0" ) ,
591- `process.exit(${ SERVER_EXIT_CODE_STATE_DIR_OWNED } );\n` ,
609+ [
610+ `const db = new (require("node:sqlite").DatabaseSync)(${ JSON . stringify ( ownerPath ) } );` ,
611+ `try { db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); }` ,
612+ `catch { process.exit(${ SERVER_EXIT_CODE_STATE_DIR_OWNED } ); }` ,
613+ 'process.send("owner");' ,
614+ "setInterval(() => {}, 1_000);" ,
615+ ] . join ( "\n" ) ,
592616 ) ;
593617 yield * Effect . promise ( ( ) =>
594618 writeServiceState ( statePath , {
595619 protocol : SERVICE_LAUNCHER_PROTOCOL ,
596620 activeVersion : "1.0.0" ,
597621 } ) ,
598622 ) ;
623+ using otherOwner = new NodeSqlite . DatabaseSync ( ownerPath ) ;
624+ otherOwner . exec ( "BEGIN EXCLUSIVE;" ) ;
599625 const refused = Promise . withResolvers < void > ( ) ;
626+ const owned = Promise . withResolvers < void > ( ) ;
600627 yield * Effect . acquireRelease (
601628 Effect . sync ( ( ) =>
602629 vi . spyOn ( process . stderr , "write" ) . mockImplementation ( ( chunk ) => {
@@ -606,15 +633,218 @@ process.exit(0);
606633 ) ,
607634 ( spy ) => Effect . sync ( ( ) => spy . mockRestore ( ) ) ,
608635 ) ;
609- const launcher = new Launcher ( root , yield * Effect . promise ( ( ) => readServiceState ( statePath ) ) ) ;
636+ yield * Effect . acquireRelease (
637+ Effect . sync ( ( ) => {
638+ spawnHook . current = ( child ) => child . once ( "message" , ( ) => owned . resolve ( ) ) ;
639+ } ) ,
640+ ( ) =>
641+ Effect . sync ( ( ) => {
642+ spawnHook . current = undefined ;
643+ } ) ,
644+ ) ;
645+ const spawnsBefore = spawnedChildren . length ;
646+ const launcher = new Launcher (
647+ root ,
648+ yield * Effect . promise ( ( ) => readServiceState ( statePath ) ) ,
649+ { stateDirOwnerPollMs : 10 } ,
650+ ) ;
610651 let completed = false ;
611652 const running = launcher . run ( ) . finally ( ( ) => {
612653 completed = true ;
613654 } ) ;
614655 yield * Effect . promise ( ( ) => refused . promise ) ;
615656 assert . isFalse ( completed ) ;
657+ otherOwner . close ( ) ;
658+ yield * Effect . promise ( ( ) => owned . promise ) ;
659+ yield * Effect . promise ( ( ) => launcher . stop ( "SIGTERM" ) ) ;
660+ yield * Effect . promise ( ( ) => running ) ;
661+ assert . strictEqual ( spawnedChildren . length - spawnsBefore , 2 ) ;
662+ } ) ,
663+ ) ;
664+
665+ it . effect (
666+ "keeps a pending update while owned and rolls a failed resumed trial back to the other owner's writes" ,
667+ ( ) =>
668+ Effect . gen ( function * ( ) {
669+ const fs = yield * FileSystem . FileSystem ;
670+ const path = yield * Path . Path ;
671+ const root = yield * fs . makeTempDirectoryScoped ( { prefix : "t3-service-reclaim-update-" } ) ;
672+ const statePath = path . join ( root , "runtime" , "service-state.json" ) ;
673+ const ownerPath = path . join ( root , "userdata" , "server-owner.sqlite" ) ;
674+ const databasePath = path . join ( root , "userdata" , "state.sqlite" ) ;
675+ const startsPath = path . join ( root , "starts.log" ) ;
676+ yield * fs . makeDirectory ( path . dirname ( ownerPath ) , { recursive : true } ) ;
677+ yield * fs . writeFileString ( databasePath , "before update" ) ;
678+ for ( const version of [ "1.0.0" , "1.1.0" ] ) {
679+ yield * writeFakeRuntime (
680+ fs ,
681+ path ,
682+ path . join ( root , "runtime" , "versions" , version ) ,
683+ [
684+ `const db = new (require("node:sqlite").DatabaseSync)(${ JSON . stringify ( ownerPath ) } );` ,
685+ `try { db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); }` ,
686+ `catch { process.exit(${ SERVER_EXIT_CODE_STATE_DIR_OWNED } ); }` ,
687+ "const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);" ,
688+ `require("node:fs").appendFileSync(${ JSON . stringify ( startsPath ) } , context.childVersion + "\\n");` ,
689+ 'if (context.update?.status === "pending") {' ,
690+ ` require("node:fs").writeFileSync(${ JSON . stringify ( databasePath ) } , "migrated by failed trial");` ,
691+ " process.exit(1);" ,
692+ "}" ,
693+ 'process.send("active");' ,
694+ "setInterval(() => {}, 1_000);" ,
695+ ] . join ( "\n" ) ,
696+ ) ;
697+ }
698+ yield * Effect . promise ( ( ) =>
699+ writeServiceState ( statePath , {
700+ protocol : SERVICE_LAUNCHER_PROTOCOL ,
701+ activeVersion : "1.0.0" ,
702+ update : {
703+ id : "update-1" ,
704+ fromVersion : "1.0.0" ,
705+ targetVersion : "1.1.0" ,
706+ dbPath : databasePath ,
707+ status : "pending" ,
708+ } ,
709+ } ) ,
710+ ) ;
711+ using otherOwner = new NodeSqlite . DatabaseSync ( ownerPath ) ;
712+ otherOwner . exec ( "BEGIN EXCLUSIVE;" ) ;
713+ const refused = Promise . withResolvers < void > ( ) ;
714+ const active = Promise . withResolvers < void > ( ) ;
715+ yield * Effect . acquireRelease (
716+ Effect . sync ( ( ) =>
717+ vi . spyOn ( process . stderr , "write" ) . mockImplementation ( ( chunk ) => {
718+ if ( String ( chunk ) . includes ( "owns this T3 home" ) ) refused . resolve ( ) ;
719+ return true ;
720+ } ) ,
721+ ) ,
722+ ( spy ) => Effect . sync ( ( ) => spy . mockRestore ( ) ) ,
723+ ) ;
724+ yield * Effect . acquireRelease (
725+ Effect . sync ( ( ) => {
726+ spawnHook . current = ( child ) =>
727+ child . on ( "message" , ( message ) => message === "active" && active . resolve ( ) ) ;
728+ } ) ,
729+ ( ) =>
730+ Effect . sync ( ( ) => {
731+ spawnHook . current = undefined ;
732+ } ) ,
733+ ) ;
734+ const launcher = new Launcher (
735+ root ,
736+ yield * Effect . promise ( ( ) => readServiceState ( statePath ) ) ,
737+ { stateDirOwnerPollMs : 10 } ,
738+ ) ;
739+ const running = launcher . run ( ) ;
740+ yield * Effect . promise ( ( ) => refused . promise ) ;
741+ const idle = yield * Effect . promise ( ( ) => readServiceState ( statePath ) ) ;
742+ assert . strictEqual ( idle . activeVersion , "1.0.0" ) ;
743+ assert . strictEqual ( idle . update ?. status , "pending" ) ;
744+ assert . isFalse ( yield * fs . exists ( path . join ( root , "runtime" , "db-backup" , "update-1" ) ) ) ;
745+
746+ yield * fs . writeFileString ( databasePath , "written by the other owner" ) ;
747+ otherOwner . close ( ) ;
748+ yield * Effect . promise ( ( ) => active . promise ) ;
749+ yield * Effect . promise ( ( ) => launcher . stop ( "SIGTERM" ) ) ;
750+ yield * Effect . promise ( ( ) => running ) ;
751+
752+ const state = yield * Effect . promise ( ( ) => readServiceState ( statePath ) ) ;
753+ assert . strictEqual ( state . activeVersion , "1.0.0" ) ;
754+ assert . strictEqual ( state . update ?. status , "rolled-back" ) ;
755+ assert . strictEqual ( yield * fs . readFileString ( startsPath ) , "1.1.0\n1.0.0\n" ) ;
756+ assert . strictEqual ( yield * fs . readFileString ( databasePath ) , "written by the other owner" ) ;
757+ } ) ,
758+ ) ;
759+
760+ it . effect ( "fails a pending update when its backup cannot be dropped while owned" , ( ) =>
761+ Effect . gen ( function * ( ) {
762+ const fs = yield * FileSystem . FileSystem ;
763+ const path = yield * Path . Path ;
764+ const root = yield * fs . makeTempDirectoryScoped ( { prefix : "t3-service-stale-backup-" } ) ;
765+ const statePath = path . join ( root , "runtime" , "service-state.json" ) ;
766+ const ownerPath = path . join ( root , "userdata" , "server-owner.sqlite" ) ;
767+ const databasePath = path . join ( root , "userdata" , "state.sqlite" ) ;
768+ yield * fs . makeDirectory ( path . dirname ( ownerPath ) , { recursive : true } ) ;
769+ yield * fs . writeFileString ( databasePath , "before update" ) ;
770+ yield * writeFakeRuntime (
771+ fs ,
772+ path ,
773+ path . join ( root , "runtime" , "versions" , "1.1.0" ) ,
774+ `process.exit(${ SERVER_EXIT_CODE_STATE_DIR_OWNED } );\n` ,
775+ ) ;
776+ yield * Effect . promise ( ( ) =>
777+ writeServiceState ( statePath , {
778+ protocol : SERVICE_LAUNCHER_PROTOCOL ,
779+ activeVersion : "1.0.0" ,
780+ update : {
781+ id : "update-1" ,
782+ fromVersion : "1.0.0" ,
783+ targetVersion : "1.1.0" ,
784+ dbPath : databasePath ,
785+ status : "pending" ,
786+ } ,
787+ } ) ,
788+ ) ;
789+ const refused = Promise . withResolvers < void > ( ) ;
790+ yield * Effect . acquireRelease (
791+ Effect . sync ( ( ) =>
792+ vi . spyOn ( process . stderr , "write" ) . mockImplementation ( ( chunk ) => {
793+ if ( String ( chunk ) . includes ( "owns this T3 home" ) ) refused . resolve ( ) ;
794+ return true ;
795+ } ) ,
796+ ) ,
797+ ( spy ) => Effect . sync ( ( ) => spy . mockRestore ( ) ) ,
798+ ) ;
799+ yield * Effect . acquireRelease (
800+ Effect . sync ( ( ) => {
801+ failRm . path = path . join ( root , "runtime" , "db-backup" , "update-1" ) ;
802+ } ) ,
803+ ( ) =>
804+ Effect . sync ( ( ) => {
805+ failRm . path = undefined ;
806+ } ) ,
807+ ) ;
808+ const launcher = new Launcher ( root , yield * Effect . promise ( ( ) => readServiceState ( statePath ) ) ) ;
809+ const running = launcher . run ( ) ;
810+ yield * Effect . promise ( ( ) => refused . promise ) ;
616811 yield * Effect . promise ( ( ) => launcher . stop ( "SIGTERM" ) ) ;
617812 yield * Effect . promise ( ( ) => running ) ;
813+
814+ const state = yield * Effect . promise ( ( ) => readServiceState ( statePath ) ) ;
815+ assert . strictEqual ( state . activeVersion , "1.0.0" ) ;
816+ assert . strictEqual ( state . update ?. status , "failed" ) ;
817+ } ) ,
818+ ) ;
819+
820+ it . effect ( "treats the T3 home as owned while the lock or a pre-lock server holds it" , ( ) =>
821+ Effect . gen ( function * ( ) {
822+ const fs = yield * FileSystem . FileSystem ;
823+ const path = yield * Path . Path ;
824+ const root = yield * fs . makeTempDirectoryScoped ( { prefix : "t3-service-owned-" } ) ;
825+ const stateDir = path . join ( root , "userdata" ) ;
826+ const recordPath = path . join ( stateDir , "server-runtime.json" ) ;
827+ yield * fs . makeDirectory ( stateDir , { recursive : true } ) ;
828+ assert . isFalse ( yield * Effect . promise ( ( ) => stateDirOwned ( root ) ) ) ;
829+
830+ using owner = new NodeSqlite . DatabaseSync ( path . join ( stateDir , "server-owner.sqlite" ) ) ;
831+ owner . exec ( "BEGIN EXCLUSIVE;" ) ;
832+ assert . isTrue ( yield * Effect . promise ( ( ) => stateDirOwned ( root ) ) ) ;
833+ owner . close ( ) ;
834+
835+ const record = { version : 1 , pid : process . pid , port : 1 , origin : "" , startedAt : "" } ;
836+ yield * fs . writeFileString ( recordPath , JSON . stringify ( record ) ) ;
837+ assert . isTrue ( yield * Effect . promise ( ( ) => stateDirOwned ( root ) ) ) ;
838+ yield * fs . writeFileString ( recordPath , JSON . stringify ( { ...record , ownerId : "lock-aware" } ) ) ;
839+ assert . isFalse ( yield * Effect . promise ( ( ) => stateDirOwned ( root ) ) ) ;
840+ // This process started after 1970, so its PID was reused since that record.
841+ yield * fs . writeFileString (
842+ recordPath ,
843+ JSON . stringify ( { ...record , startedAt : "1970-01-01T00:00:00Z" } ) ,
844+ ) ;
845+ assert . isFalse ( yield * Effect . promise ( ( ) => stateDirOwned ( root ) ) ) ;
846+ yield * fs . writeFileString ( recordPath , JSON . stringify ( { ...record , pid : 2 ** 22 + 1 } ) ) ;
847+ assert . isFalse ( yield * Effect . promise ( ( ) => stateDirOwned ( root ) ) ) ;
618848 } ) ,
619849 ) ;
620850
0 commit comments