@@ -277,6 +277,13 @@ const make = Effect.gen(function*() {
277277 // the active shards are the ones that we have acquired the lock for
278278 const acquiredShards = MutableHashSet . empty < ShardId > ( )
279279 const activeShardsLatch = yield * Latch . make ( false )
280+ // open once shutdown starts handing every shard off
281+ const handoffLatch = Latch . makeUnsafe ( false )
282+ // open once the handoff has released every shard lock
283+ const handedOffLatch = Latch . makeUnsafe ( false )
284+ // serializes runner registration with unregistration, so a registration
285+ // in flight cannot re-advertise a runner that is shutting down
286+ const withRegistrationLock = Semaphore . makeUnsafe ( 1 ) . withPermits ( 1 )
280287
281288 const events = yield * PubSub . unbounded < ShardingRegistrationEvent > ( )
282289 const getRegistrationEvents : Stream . Stream < ShardingRegistrationEvent > = Stream . fromPubSub ( events )
@@ -337,6 +344,8 @@ const make = Effect.gen(function*() {
337344 // allow them to move to another runner.
338345
339346 const releasingShards = MutableHashSet . empty < ShardId > ( )
347+ // Hands every held shard off on shutdown, set when this is a runner
348+ let handoff : Effect . Effect < void > = Effect . void
340349 // Shards whose entities must be force interrupted and locks fully released
341350 // before normal reacquisition.
342351 const forceReleasingShards = MutableHashSet . empty < ShardId > ( )
@@ -348,23 +357,49 @@ const make = Effect.gen(function*() {
348357 return Effect . ignore ( runnerStorage . releaseAll ( selfAddress ) )
349358 } )
350359
360+ // Shard releases fork into a child scope that closes after the shutdown
361+ // handoff, as forks into the closing sharding scope never run.
362+ const releaseScope = yield * Scope . fork ( shardingScope )
363+ // Interrupt the shards' entities on every entity manager that is open
364+ const forkInterruptShards = Effect . fnUntraced ( function * ( shardIds : Iterable < ShardId > , force : boolean ) {
365+ const fibers = Arr . empty < Fiber . Fiber < void > > ( )
366+ for ( const shardId of shardIds ) {
367+ for ( const state of entityManagers . values ( ) ) {
368+ if ( state . status === "closed" ) continue
369+ fibers . push (
370+ yield * Effect . forkIn ( state . manager . interruptShard ( shardId , { force } ) , releaseScope , {
371+ startImmediately : true
372+ } )
373+ )
374+ }
375+ }
376+ return fibers
377+ } )
351378 const releaseShardsMap = yield * FiberMap . make < ShardId > ( )
352379 let forcedShardReleaseRunning = false
380+ // Called from the acquisition loop only, so no `acquire` is in flight
381+ const checkHandedOff = ( ) => {
382+ if (
383+ handoffLatch . isOpen ( ) &&
384+ MutableHashSet . size ( acquiredShards ) === 0 &&
385+ MutableHashSet . size ( releasingShards ) === 0 &&
386+ ! forcedShardReleasePending ( )
387+ ) {
388+ handedOffLatch . openUnsafe ( )
389+ }
390+ }
353391 // Interrupt the shards' entities, wait for lock health, run the storage
354392 // release, then clear the shards' bookkeeping.
355393 const runShardRelease = Effect . fnUntraced ( function * < E > (
356394 shardIds : ReadonlyArray < ShardId > ,
357395 force : boolean ,
358396 release : Effect . Effect < void , E >
359397 ) {
360- const fibers = Arr . empty < Fiber . Fiber < void > > ( )
361- for ( const shardId of shardIds ) {
362- for ( const state of entityManagers . values ( ) ) {
363- if ( state . status === "closed" ) continue
364- fibers . push ( yield * Effect . forkScoped ( state . manager . interruptShard ( shardId , { force } ) ) )
365- }
366- }
367- yield * Fiber . joinAll ( fibers )
398+ yield * Fiber . joinAll ( yield * forkInterruptShards ( shardIds , force ) )
399+ // stop the singletons of shards that are no longer acquired, and wait
400+ // for those on the released shards
401+ yield * startSyncSingletons
402+ yield * awaitStoppingSingletons ( shardIds )
368403 yield * shardLocksHealthyLatch . await
369404 yield * release
370405 for ( const shardId of shardIds ) {
@@ -378,6 +413,8 @@ const make = Effect.gen(function*() {
378413 interrupt : MutableRef . get ( isShutdown )
379414 } )
380415 }
416+ // let the acquisition loop check whether the handoff has finished
417+ if ( handoffLatch . isOpen ( ) ) activeShardsLatch . openUnsafe ( )
381418 } )
382419 const retryShardRelease =
383420 ( annotations : { readonly fiber : string ; readonly shardId ?: ShardId } ) =>
@@ -429,7 +466,7 @@ const make = Effect.gen(function*() {
429466 forcedShardReleaseRunning = false
430467 activeShardsLatch . openUnsafe ( )
431468 } ) ) ,
432- Effect . forkIn ( shardingScope ) ,
469+ Effect . forkIn ( releaseScope ) ,
433470 Effect . asVoid
434471 )
435472 } )
@@ -449,9 +486,11 @@ const make = Effect.gen(function*() {
449486 yield * activeShardsLatch . await
450487 activeShardsLatch . closeUnsafe ( )
451488
452- // if a shard is no longer assigned to this runner, we release it
489+ // if a shard is no longer assigned to this runner, or the runner is
490+ // handing every shard off, we release it
491+ const handingOff = handoffLatch . isOpen ( )
453492 for ( const shardId of acquiredShards ) {
454- if ( MutableHashSet . has ( selfShards , shardId ) ) continue
493+ if ( ! handingOff && MutableHashSet . has ( selfShards , shardId ) ) continue
455494 MutableHashSet . remove ( acquiredShards , shardId )
456495 MutableHashSet . add ( releasingShards , shardId )
457496 }
@@ -460,6 +499,7 @@ const make = Effect.gen(function*() {
460499 yield * Effect . forkIn ( syncSingletons , shardingScope )
461500 yield * releaseShards
462501 }
502+ checkHandedOff ( )
463503
464504 if ( ! shardLocksHealthyLatch . isOpen ( ) ) {
465505 continue
@@ -471,6 +511,11 @@ const make = Effect.gen(function*() {
471511 continue
472512 }
473513
514+ // a runner that is shutting down takes no new shards
515+ if ( isShutdown . current ) {
516+ continue
517+ }
518+
474519 // if a shard has been assigned to this runner, we acquire it
475520 const unacquiredShards = MutableHashSet . empty < ShardId > ( )
476521 for ( const shardId of selfShards ) {
@@ -518,7 +563,8 @@ const make = Effect.gen(function*() {
518563 // update metrics
519564 ClusterMetrics . shards . updateUnsafe ( BigInt ( MutableHashSet . size ( acquiredShards ) ) , Context . empty ( ) )
520565 }
521- yield * Effect . sleep ( 1000 )
566+ // the shutdown handoff releases the shards acquired here without waiting
567+ yield * Effect . raceFirst ( Effect . sleep ( 1000 ) , handoffLatch . await )
522568 activeShardsLatch . openUnsafe ( )
523569 }
524570 } ) . pipe (
@@ -551,16 +597,7 @@ const make = Effect.gen(function*() {
551597 yield * Effect . logError ( "Shard lock storage is unhealthy" , cause )
552598 yield * Effect . forkIn ( syncSingletons , shardingScope , { startImmediately : true } )
553599
554- for ( const shardId of affectedShards ) {
555- for ( const state of entityManagers . values ( ) ) {
556- if ( state . status === "closed" ) continue
557- yield * Effect . forkIn (
558- state . manager . interruptShard ( shardId , { force : true } ) ,
559- shardingScope ,
560- { startImmediately : true }
561- )
562- }
563- }
600+ yield * forkInterruptShards ( affectedShards , true )
564601 activeShardsLatch . openUnsafe ( )
565602 } )
566603 } )
@@ -627,6 +664,18 @@ const make = Effect.gen(function*() {
627664 Effect . forever ,
628665 Effect . forkIn ( shardingScope )
629666 )
667+
668+ // Release every held shard through the per-shard path. Their entities are
669+ // forced to stop, so the handoff does not wait out graceful termination.
670+ // This is best effort: the wait is bounded, and if it times out, the final
671+ // `releaseAll` attempts to release the remaining locks.
672+ handoff = Effect . suspend ( ( ) => {
673+ if ( handoffLatch . openUnsafe ( ) ) activeShardsLatch . openUnsafe ( )
674+ return forkInterruptShards ( [ ...acquiredShards , ...releasingShards ] , true )
675+ } ) . pipe (
676+ Effect . andThen ( Effect . timeoutOption ( handedOffLatch . await , shardLockInterval ) ) ,
677+ Effect . asVoid
678+ )
630679 }
631680
632681 // --- Storage inbox ---
@@ -1294,9 +1343,12 @@ const make = Effect.gen(function*() {
12941343 while ( true ) {
12951344 // Ensure the current runner is registered
12961345 if ( selfRunner && ! isShutdown . current && ! MutableHashMap . has ( allRunners , selfRunner ) ) {
1297- yield * Effect . logDebug ( "Registering runner" , selfRunner )
1298- const machineId = yield * withTimeout ( runnerStorage . register ( selfRunner , true ) )
1299- yield * snowflakeGen . setMachineId ( machineId )
1346+ yield * withRegistrationLock ( Effect . gen ( function * ( ) {
1347+ if ( isShutdown . current ) return
1348+ yield * Effect . logDebug ( "Registering runner" , selfRunner )
1349+ const machineId = yield * withTimeout ( runnerStorage . register ( selfRunner , true ) )
1350+ yield * snowflakeGen . setMachineId ( machineId )
1351+ } ) )
13001352 }
13011353
13021354 const runners = yield * withTimeout ( runnerStorage . getRunners )
@@ -1619,6 +1671,47 @@ const make = Effect.gen(function*() {
16191671 const singletonFibers = yield * FiberMap . make < SingletonAddress > ( )
16201672 const withSingletonLock = Semaphore . makeUnsafe ( 1 ) . withPermits ( 1 )
16211673
1674+ // Singletons that are stopping. They count as running, so they are not
1675+ // restarted, and shard releases wait for them. Their teardown is awaited
1676+ // without the singleton lock, as it can register or stop other singletons.
1677+ const stoppingSingletons = MutableHashMap . empty < SingletonAddress , Fiber . Fiber < void > > ( )
1678+ const isSingletonRegistered = ( address : SingletonAddress ) => {
1679+ const map = singletons . get ( address . shardId )
1680+ return map !== undefined && MutableHashMap . has ( map , address )
1681+ }
1682+
1683+ // Called with the singleton lock held. Starts stopping the singleton and
1684+ // returns the wait for its teardown, to run without the lock.
1685+ const stopSingletonUnsafe = ( address : SingletonAddress , activeTeardown : boolean ) : Effect . Effect < void > => {
1686+ const stopping = MutableHashMap . get ( stoppingSingletons , address )
1687+ if ( Option . isSome ( stopping ) ) return Fiber . join ( stopping . value )
1688+ const fiber = FiberMap . getUnsafe ( singletonFibers , address )
1689+ if ( Option . isNone ( fiber ) ) return Effect . void
1690+ const interrupt = Fiber . interrupt ( fiber . value )
1691+ const stop = Effect . runForkWith ( services ) (
1692+ activeTeardown ? ActiveTeardown . aroundShard ( address . shardId , interrupt ) : interrupt
1693+ )
1694+ MutableHashMap . set ( stoppingSingletons , address , stop )
1695+ stop . addObserver ( ( ) => {
1696+ MutableHashMap . remove ( stoppingSingletons , address )
1697+ // a registration made while the previous singleton was stopping
1698+ if ( isSingletonRegistered ( address ) ) runFork ( syncSingletons )
1699+ } )
1700+ return Fiber . join ( stop )
1701+ }
1702+
1703+ // Waits for the singletons of the given shards that are still stopping
1704+ const awaitStoppingSingletons = ( shardIds : ReadonlyArray < ShardId > ) =>
1705+ Effect . suspend ( ( ) => {
1706+ const stops = Arr . empty < Fiber . Fiber < void > > ( )
1707+ MutableHashMap . forEach ( stoppingSingletons , ( stop , address ) => {
1708+ if ( shardIds . some ( ( shardId ) => Equal . equals ( shardId , address . shardId ) ) ) {
1709+ stops . push ( stop )
1710+ }
1711+ } )
1712+ return Fiber . joinAll ( stops )
1713+ } )
1714+
16221715 const registerSingleton : Sharding [ "Service" ] [ "registerSingleton" ] = Effect . fnUntraced (
16231716 function * ( name , run , options ) {
16241717 const shardGroup = options ?. shardGroup ?? "default"
@@ -1649,32 +1742,38 @@ const make = Effect.gen(function*() {
16491742
16501743 yield * PubSub . publish ( events , SingletonRegistered ( { address } ) )
16511744
1652- // start if we are on the right shard
1653- if ( MutableHashSet . has ( acquiredShards , address . shardId ) ) {
1745+ // start if we are on the right shard, unless a previous registration of
1746+ // this singleton is still stopping
1747+ if (
1748+ MutableHashSet . has ( acquiredShards , address . shardId ) &&
1749+ ! MutableHashMap . has ( stoppingSingletons , address )
1750+ ) {
16541751 yield * Effect . logDebug ( "Starting singleton" , address )
16551752 yield * FiberMap . run ( singletonFibers , address , wrappedRun )
16561753 }
16571754
1658- yield * Effect . addFinalizer ( ( ) => {
1659- const map = singletons . get ( address . shardId ) !
1660- MutableHashMap . remove ( map , address )
1661- return FiberMap . remove ( singletonFibers , address )
1662- } )
1755+ yield * Effect . addFinalizer ( ( ) =>
1756+ withSingletonLock ( Effect . sync ( ( ) => {
1757+ MutableHashMap . remove ( singletons . get ( address . shardId ) ! , address )
1758+ return stopSingletonUnsafe ( address , false )
1759+ } ) ) . pipe ( Effect . flatten )
1760+ )
16631761 } ,
16641762 withSingletonLock
16651763 )
16661764
1667- const syncSingletons = withSingletonLock ( Effect . gen ( function * ( ) {
1765+ // Starts and stops singletons to match the acquired shards, returning the
1766+ // waits for the stopped singletons
1767+ const startSyncSingletons = withSingletonLock ( Effect . gen ( function * ( ) {
1768+ const stops = Arr . empty < Effect . Effect < void > > ( )
16681769 for ( const [ shardId , map ] of singletons ) {
16691770 for ( const [ address , run ] of map ) {
1670- const running = FiberMap . hasUnsafe ( singletonFibers , address )
1771+ const stopping = MutableHashMap . has ( stoppingSingletons , address )
1772+ const running = stopping || FiberMap . hasUnsafe ( singletonFibers , address )
16711773 const shouldBeRunning = MutableHashSet . has ( acquiredShards , shardId )
16721774 if ( running && ! shouldBeRunning ) {
1673- yield * Effect . logDebug ( "Stopping singleton" , address )
1674- yield * ActiveTeardown . aroundShard (
1675- address . shardId ,
1676- FiberMap . remove ( singletonFibers , address )
1677- )
1775+ if ( ! stopping ) yield * Effect . logDebug ( "Stopping singleton" , address )
1776+ stops . push ( stopSingletonUnsafe ( address , true ) )
16781777 } else if ( ! running && shouldBeRunning ) {
16791778 yield * Effect . logDebug ( "Starting singleton" , address )
16801779 yield * FiberMap . run ( singletonFibers , address , run )
@@ -1685,8 +1784,15 @@ const make = Effect.gen(function*() {
16851784 BigInt ( yield * FiberMap . size ( singletonFibers ) ) ,
16861785 Context . empty ( )
16871786 )
1787+ return stops
16881788 } ) )
16891789
1790+ const syncSingletons : Effect . Effect < void > = Effect . flatMap (
1791+ startSyncSingletons ,
1792+ // wait for the stopped singletons without the lock
1793+ ( stops ) => Effect . all ( stops , { discard : true } )
1794+ )
1795+
16901796 // --- Entities ---
16911797
16921798 const reaper = yield * EntityReaper
@@ -1829,11 +1935,12 @@ const make = Effect.gen(function*() {
18291935 MutableRef . set ( isShutdown , true )
18301936 yield * notifyShardLockWaiters
18311937 if ( selfRunner ) {
1832- yield * Effect . ignore ( runnerStorage . unregister ( selfRunner . address ) )
1938+ yield * Effect . ignore ( withRegistrationLock ( runnerStorage . unregister ( selfRunner . address ) ) )
18331939 }
18341940 } )
18351941
1836- yield * Scope . addFinalizerExit ( shardingScope , shutdown )
1942+ // hand the shards off before the loops stop
1943+ yield * Scope . addFinalizerExit ( shardingScope , ( exit ) => Effect . andThen ( shutdown ( exit ) , handoff ) )
18371944
18381945 const activeEntityCount = Effect . gen ( function * ( ) {
18391946 let count = 0
0 commit comments