@@ -295,6 +295,33 @@ const make = Effect.gen(function*() {
295295 return MutableHashSet . has ( acquiredShards , address . shardId )
296296 }
297297
298+ function isAwaitingShardLock ( shardId : ShardId ) : boolean {
299+ return ! MutableRef . get ( isShutdown ) &&
300+ MutableHashSet . has ( selfShards , shardId ) &&
301+ ! MutableHashSet . has ( acquiredShards , shardId )
302+ }
303+
304+ // Requests for a shard assigned to this runner wait for its lock, for at most
305+ // `shardLockExpiration`, instead of bouncing while the previous owner drains.
306+ // Each notification opens the current latch and replaces it, so a waiter that
307+ // checked its shard before a notification cannot miss it.
308+ let shardLocksChanged = Latch . makeUnsafe ( false )
309+ const notifyShardLockWaiters = Effect . suspend ( ( ) => {
310+ const latch = shardLocksChanged
311+ shardLocksChanged = Latch . makeUnsafe ( false )
312+ return latch . open
313+ } )
314+ const awaitShardLock = ( shardId : ShardId ) : Effect . Effect < void > => {
315+ const loop : Effect . Effect < void > = Effect . suspend ( ( ) =>
316+ isAwaitingShardLock ( shardId ) ? Effect . andThen ( shardLocksChanged . await , loop ) : Effect . void
317+ )
318+ return Effect . suspend ( ( ) =>
319+ isAwaitingShardLock ( shardId )
320+ ? Effect . asVoid ( Effect . timeoutOption ( loop , config . shardLockExpiration ) )
321+ : Effect . void
322+ )
323+ }
324+
298325 yield * Scope . addFinalizer (
299326 shardingScope ,
300327 Effect . logDebug ( "Shutdown complete" ) . pipe ( Effect . annotateLogs ( {
@@ -484,6 +511,7 @@ const make = Effect.gen(function*() {
484511 MutableHashSet . add ( acquiredShards , shardId )
485512 }
486513 if ( acquired . length > 0 ) {
514+ yield * notifyShardLockWaiters
487515 yield * storageReadLatch . open
488516 yield * Effect . forkIn ( syncSingletons , shardingScope )
489517
@@ -519,6 +547,7 @@ const make = Effect.gen(function*() {
519547 activeShardsLatch . openUnsafe ( )
520548
521549 return Effect . gen ( function * ( ) {
550+ yield * notifyShardLockWaiters
522551 yield * Effect . logError ( "Shard lock storage is unhealthy" , cause )
523552 yield * Effect . forkIn ( syncSingletons , shardingScope , { startImmediately : true } )
524553
@@ -1036,16 +1065,21 @@ const make = Effect.gen(function*() {
10361065
10371066 // --- Sending messages ---
10381067
1039- const sendLocal = < M extends Message . Outgoing < any > | Message . Incoming < any > > ( message : M ) =>
1040- Effect . suspend ( function loop ( ) : Effect . Effect <
1068+ const sendLocal = < M extends Message . Outgoing < any > | Message . Incoming < any > > ( message : M ) => {
1069+ const address = message . envelope . address
1070+ const isRequest = message . _tag === "IncomingRequest" || message . _tag === "OutgoingRequest"
1071+ const send = Effect . suspend ( function loop ( ) : Effect . Effect <
10411072 void ,
10421073 | EntityNotAssignedToRunner
10431074 | MailboxFull
10441075 | AlreadyProcessingMessage
10451076 | ( M extends Message . Incoming < any > ? never : PersistenceError )
10461077 > {
1047- const address = message . envelope . address
1048- if ( ! isEntityOnLocalShards ( address ) ) {
1078+ if (
1079+ ! isEntityOnLocalShards ( address ) &&
1080+ // Entities draining from a released shard still accept interrupts and acks
1081+ ( isRequest || ! MutableHashSet . has ( releasingShards , address . shardId ) )
1082+ ) {
10491083 return Effect . fail ( new EntityNotAssignedToRunner ( { address } ) )
10501084 }
10511085 const state = entityManagers . get ( address . entityType )
@@ -1064,6 +1098,8 @@ const make = Effect.gen(function*() {
10641098 simulateRemoteSerialization : config . simulateRemoteSerialization
10651099 } ) as any
10661100 } )
1101+ return isRequest ? Effect . andThen ( awaitShardLock ( address . shardId ) , send ) : send
1102+ }
10671103
10681104 type PendingNotification = {
10691105 resume : ( _ : Effect . Effect < void , EntityNotAssignedToRunner > ) => void
@@ -1129,7 +1165,10 @@ const make = Effect.gen(function*() {
11291165 function sendOutgoing (
11301166 message : Message . Outgoing < any > ,
11311167 discard : boolean ,
1132- retries ?: number
1168+ retries ?: number ,
1169+ // The runner serving the request, for its interrupts and acks. Later
1170+ // attempts fall back to the current shard assignment.
1171+ runner ?: RunnerAddress
11331172 ) : Effect . Effect <
11341173 void ,
11351174 MailboxFull | AlreadyProcessingMessage | PersistenceError | EntityNotAssignedToRunner
@@ -1172,14 +1211,19 @@ const make = Effect.gen(function*() {
11721211 if ( shouldFail && MutableRef . get ( isShutdown ) ) {
11731212 return Effect . fail ( new EntityNotAssignedToRunner ( { address } ) )
11741213 }
1175- const maybeRunner = MutableHashMap . get ( shardAssignments , address . shardId )
1214+ const maybeRunner = runner !== undefined
1215+ ? Option . some ( runner )
1216+ : MutableHashMap . get ( shardAssignments , address . shardId )
11761217 const runnerIsLocal = Option . isSome ( maybeRunner ) && isLocalRunner ( maybeRunner . value )
11771218 if ( isPersisted ) {
11781219 return runnerIsLocal
11791220 ? notifyLocal ( message , discard )
11801221 : runnersService . notify ( { address : maybeRunner , message, discard } )
11811222 } else if ( Option . isNone ( maybeRunner ) ) {
11821223 return Effect . fail ( new EntityNotAssignedToRunner ( { address } ) )
1224+ } else if ( message . _tag === "OutgoingRequest" ) {
1225+ const entry = clientRequests . get ( message . envelope . requestId )
1226+ if ( entry ) entry . runner = maybeRunner . value
11831227 }
11841228 return runnerIsLocal
11851229 ? sendLocal ( message )
@@ -1324,6 +1368,7 @@ const make = Effect.gen(function*() {
13241368 } )
13251369 yield * Effect . logDebug ( "New shard assignments" , selfShards )
13261370 activeShardsLatch . openUnsafe ( )
1371+ yield * notifyShardLockWaiters
13271372
13281373 // update metrics
13291374 if ( selfRunner ) {
@@ -1361,6 +1406,8 @@ const make = Effect.gen(function*() {
13611406 readonly context : Context . Context < never >
13621407 readonly message : Message . OutgoingRequest < any >
13631408 lastChunkId ?: Snowflake . Snowflake
1409+ // the runner a volatile request was last sent to
1410+ runner ?: RunnerAddress
13641411 }
13651412 const clientRequests = new Map < Snowflake . Snowflake , ClientRequestEntry > ( )
13661413
@@ -1445,7 +1492,9 @@ const make = Effect.gen(function*() {
14451492 } ) ,
14461493 rpc : entry . rpc
14471494 } ) ,
1448- false
1495+ false ,
1496+ undefined ,
1497+ entry . runner
14491498 )
14501499 }
14511500 case "Interrupt" : {
@@ -1476,7 +1525,8 @@ const make = Effect.gen(function*() {
14761525 rpc : entry . rpc
14771526 } ) ,
14781527 false ,
1479- 3
1528+ 3 ,
1529+ entry . runner
14801530 ) )
14811531 }
14821532 }
@@ -1777,6 +1827,7 @@ const make = Effect.gen(function*() {
17771827
17781828 if ( isShutdown . current ) return
17791829 MutableRef . set ( isShutdown , true )
1830+ yield * notifyShardLockWaiters
17801831 if ( selfRunner ) {
17811832 yield * Effect . ignore ( runnerStorage . unregister ( selfRunner . address ) )
17821833 }
0 commit comments