Skip to content

Commit 66257d2

Browse files
Tanner-Scaddenclaudeeffect-bottim-smart
authored
Hand cluster shards off one at a time on shutdown (#8941)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]> Co-authored-by: Effect Bot <[email protected]> Co-authored-by: Tim Smart <[email protected]>
1 parent f54c34a commit 66257d2

4 files changed

Lines changed: 385 additions & 51 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"effect": patch
3+
---
4+
5+
Hand cluster shards off one at a time when `Sharding` shuts down, after their entities and singletons have stopped, so live runners can take them over sooner. The handoff is best effort: its wait is bounded, and if it times out, `releaseAll` attempts to release the remaining locks. A runner that is shutting down no longer acquires new shards, and releasing a shard no longer waits forever on an entity whose id is also active on another shard.

‎packages/effect/src/cluster/Sharding.ts‎

Lines changed: 148 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,13 @@ const make = Effect.gen(function*() {
277277
// the active shards are the ones that we have acquired the lock for
278278
const acquiredShards = MutableHashSet.empty<ShardId>()
279279
const activeShardsLatch = yield* Latch.make(false)
280+
// open once shutdown starts handing every shard off
281+
const handoffLatch = Latch.makeUnsafe(false)
282+
// open once the handoff has released every shard lock
283+
const handedOffLatch = Latch.makeUnsafe(false)
284+
// serializes runner registration with unregistration, so a registration
285+
// in flight cannot re-advertise a runner that is shutting down
286+
const withRegistrationLock = Semaphore.makeUnsafe(1).withPermits(1)
280287

281288
const events = yield* PubSub.unbounded<ShardingRegistrationEvent>()
282289
const getRegistrationEvents: Stream.Stream<ShardingRegistrationEvent> = Stream.fromPubSub(events)
@@ -337,6 +344,8 @@ const make = Effect.gen(function*() {
337344
// allow them to move to another runner.
338345

339346
const releasingShards = MutableHashSet.empty<ShardId>()
347+
// Hands every held shard off on shutdown, set when this is a runner
348+
let handoff: Effect.Effect<void> = Effect.void
340349
// Shards whose entities must be force interrupted and locks fully released
341350
// before normal reacquisition.
342351
const forceReleasingShards = MutableHashSet.empty<ShardId>()
@@ -348,23 +357,49 @@ const make = Effect.gen(function*() {
348357
return Effect.ignore(runnerStorage.releaseAll(selfAddress))
349358
})
350359

360+
// Shard releases fork into a child scope that closes after the shutdown
361+
// handoff, as forks into the closing sharding scope never run.
362+
const releaseScope = yield* Scope.fork(shardingScope)
363+
// Interrupt the shards' entities on every entity manager that is open
364+
const forkInterruptShards = Effect.fnUntraced(function*(shardIds: Iterable<ShardId>, force: boolean) {
365+
const fibers = Arr.empty<Fiber.Fiber<void>>()
366+
for (const shardId of shardIds) {
367+
for (const state of entityManagers.values()) {
368+
if (state.status === "closed") continue
369+
fibers.push(
370+
yield* Effect.forkIn(state.manager.interruptShard(shardId, { force }), releaseScope, {
371+
startImmediately: true
372+
})
373+
)
374+
}
375+
}
376+
return fibers
377+
})
351378
const releaseShardsMap = yield* FiberMap.make<ShardId>()
352379
let forcedShardReleaseRunning = false
380+
// Called from the acquisition loop only, so no `acquire` is in flight
381+
const checkHandedOff = () => {
382+
if (
383+
handoffLatch.isOpen() &&
384+
MutableHashSet.size(acquiredShards) === 0 &&
385+
MutableHashSet.size(releasingShards) === 0 &&
386+
!forcedShardReleasePending()
387+
) {
388+
handedOffLatch.openUnsafe()
389+
}
390+
}
353391
// Interrupt the shards' entities, wait for lock health, run the storage
354392
// release, then clear the shards' bookkeeping.
355393
const runShardRelease = Effect.fnUntraced(function*<E>(
356394
shardIds: ReadonlyArray<ShardId>,
357395
force: boolean,
358396
release: Effect.Effect<void, E>
359397
) {
360-
const fibers = Arr.empty<Fiber.Fiber<void>>()
361-
for (const shardId of shardIds) {
362-
for (const state of entityManagers.values()) {
363-
if (state.status === "closed") continue
364-
fibers.push(yield* Effect.forkScoped(state.manager.interruptShard(shardId, { force })))
365-
}
366-
}
367-
yield* Fiber.joinAll(fibers)
398+
yield* Fiber.joinAll(yield* forkInterruptShards(shardIds, force))
399+
// stop the singletons of shards that are no longer acquired, and wait
400+
// for those on the released shards
401+
yield* startSyncSingletons
402+
yield* awaitStoppingSingletons(shardIds)
368403
yield* shardLocksHealthyLatch.await
369404
yield* release
370405
for (const shardId of shardIds) {
@@ -378,6 +413,8 @@ const make = Effect.gen(function*() {
378413
interrupt: MutableRef.get(isShutdown)
379414
})
380415
}
416+
// let the acquisition loop check whether the handoff has finished
417+
if (handoffLatch.isOpen()) activeShardsLatch.openUnsafe()
381418
})
382419
const retryShardRelease =
383420
(annotations: { readonly fiber: string; readonly shardId?: ShardId }) =>
@@ -429,7 +466,7 @@ const make = Effect.gen(function*() {
429466
forcedShardReleaseRunning = false
430467
activeShardsLatch.openUnsafe()
431468
})),
432-
Effect.forkIn(shardingScope),
469+
Effect.forkIn(releaseScope),
433470
Effect.asVoid
434471
)
435472
})
@@ -449,9 +486,11 @@ const make = Effect.gen(function*() {
449486
yield* activeShardsLatch.await
450487
activeShardsLatch.closeUnsafe()
451488

452-
// if a shard is no longer assigned to this runner, we release it
489+
// if a shard is no longer assigned to this runner, or the runner is
490+
// handing every shard off, we release it
491+
const handingOff = handoffLatch.isOpen()
453492
for (const shardId of acquiredShards) {
454-
if (MutableHashSet.has(selfShards, shardId)) continue
493+
if (!handingOff && MutableHashSet.has(selfShards, shardId)) continue
455494
MutableHashSet.remove(acquiredShards, shardId)
456495
MutableHashSet.add(releasingShards, shardId)
457496
}
@@ -460,6 +499,7 @@ const make = Effect.gen(function*() {
460499
yield* Effect.forkIn(syncSingletons, shardingScope)
461500
yield* releaseShards
462501
}
502+
checkHandedOff()
463503

464504
if (!shardLocksHealthyLatch.isOpen()) {
465505
continue
@@ -471,6 +511,11 @@ const make = Effect.gen(function*() {
471511
continue
472512
}
473513

514+
// a runner that is shutting down takes no new shards
515+
if (isShutdown.current) {
516+
continue
517+
}
518+
474519
// if a shard has been assigned to this runner, we acquire it
475520
const unacquiredShards = MutableHashSet.empty<ShardId>()
476521
for (const shardId of selfShards) {
@@ -518,7 +563,8 @@ const make = Effect.gen(function*() {
518563
// update metrics
519564
ClusterMetrics.shards.updateUnsafe(BigInt(MutableHashSet.size(acquiredShards)), Context.empty())
520565
}
521-
yield* Effect.sleep(1000)
566+
// the shutdown handoff releases the shards acquired here without waiting
567+
yield* Effect.raceFirst(Effect.sleep(1000), handoffLatch.await)
522568
activeShardsLatch.openUnsafe()
523569
}
524570
}).pipe(
@@ -551,16 +597,7 @@ const make = Effect.gen(function*() {
551597
yield* Effect.logError("Shard lock storage is unhealthy", cause)
552598
yield* Effect.forkIn(syncSingletons, shardingScope, { startImmediately: true })
553599

554-
for (const shardId of affectedShards) {
555-
for (const state of entityManagers.values()) {
556-
if (state.status === "closed") continue
557-
yield* Effect.forkIn(
558-
state.manager.interruptShard(shardId, { force: true }),
559-
shardingScope,
560-
{ startImmediately: true }
561-
)
562-
}
563-
}
600+
yield* forkInterruptShards(affectedShards, true)
564601
activeShardsLatch.openUnsafe()
565602
})
566603
})
@@ -627,6 +664,18 @@ const make = Effect.gen(function*() {
627664
Effect.forever,
628665
Effect.forkIn(shardingScope)
629666
)
667+
668+
// Release every held shard through the per-shard path. Their entities are
669+
// forced to stop, so the handoff does not wait out graceful termination.
670+
// This is best effort: the wait is bounded, and if it times out, the final
671+
// `releaseAll` attempts to release the remaining locks.
672+
handoff = Effect.suspend(() => {
673+
if (handoffLatch.openUnsafe()) activeShardsLatch.openUnsafe()
674+
return forkInterruptShards([...acquiredShards, ...releasingShards], true)
675+
}).pipe(
676+
Effect.andThen(Effect.timeoutOption(handedOffLatch.await, shardLockInterval)),
677+
Effect.asVoid
678+
)
630679
}
631680

632681
// --- Storage inbox ---
@@ -1294,9 +1343,12 @@ const make = Effect.gen(function*() {
12941343
while (true) {
12951344
// Ensure the current runner is registered
12961345
if (selfRunner && !isShutdown.current && !MutableHashMap.has(allRunners, selfRunner)) {
1297-
yield* Effect.logDebug("Registering runner", selfRunner)
1298-
const machineId = yield* withTimeout(runnerStorage.register(selfRunner, true))
1299-
yield* snowflakeGen.setMachineId(machineId)
1346+
yield* withRegistrationLock(Effect.gen(function*() {
1347+
if (isShutdown.current) return
1348+
yield* Effect.logDebug("Registering runner", selfRunner)
1349+
const machineId = yield* withTimeout(runnerStorage.register(selfRunner, true))
1350+
yield* snowflakeGen.setMachineId(machineId)
1351+
}))
13001352
}
13011353

13021354
const runners = yield* withTimeout(runnerStorage.getRunners)
@@ -1619,6 +1671,47 @@ const make = Effect.gen(function*() {
16191671
const singletonFibers = yield* FiberMap.make<SingletonAddress>()
16201672
const withSingletonLock = Semaphore.makeUnsafe(1).withPermits(1)
16211673

1674+
// Singletons that are stopping. They count as running, so they are not
1675+
// restarted, and shard releases wait for them. Their teardown is awaited
1676+
// without the singleton lock, as it can register or stop other singletons.
1677+
const stoppingSingletons = MutableHashMap.empty<SingletonAddress, Fiber.Fiber<void>>()
1678+
const isSingletonRegistered = (address: SingletonAddress) => {
1679+
const map = singletons.get(address.shardId)
1680+
return map !== undefined && MutableHashMap.has(map, address)
1681+
}
1682+
1683+
// Called with the singleton lock held. Starts stopping the singleton and
1684+
// returns the wait for its teardown, to run without the lock.
1685+
const stopSingletonUnsafe = (address: SingletonAddress, activeTeardown: boolean): Effect.Effect<void> => {
1686+
const stopping = MutableHashMap.get(stoppingSingletons, address)
1687+
if (Option.isSome(stopping)) return Fiber.join(stopping.value)
1688+
const fiber = FiberMap.getUnsafe(singletonFibers, address)
1689+
if (Option.isNone(fiber)) return Effect.void
1690+
const interrupt = Fiber.interrupt(fiber.value)
1691+
const stop = Effect.runForkWith(services)(
1692+
activeTeardown ? ActiveTeardown.aroundShard(address.shardId, interrupt) : interrupt
1693+
)
1694+
MutableHashMap.set(stoppingSingletons, address, stop)
1695+
stop.addObserver(() => {
1696+
MutableHashMap.remove(stoppingSingletons, address)
1697+
// a registration made while the previous singleton was stopping
1698+
if (isSingletonRegistered(address)) runFork(syncSingletons)
1699+
})
1700+
return Fiber.join(stop)
1701+
}
1702+
1703+
// Waits for the singletons of the given shards that are still stopping
1704+
const awaitStoppingSingletons = (shardIds: ReadonlyArray<ShardId>) =>
1705+
Effect.suspend(() => {
1706+
const stops = Arr.empty<Fiber.Fiber<void>>()
1707+
MutableHashMap.forEach(stoppingSingletons, (stop, address) => {
1708+
if (shardIds.some((shardId) => Equal.equals(shardId, address.shardId))) {
1709+
stops.push(stop)
1710+
}
1711+
})
1712+
return Fiber.joinAll(stops)
1713+
})
1714+
16221715
const registerSingleton: Sharding["Service"]["registerSingleton"] = Effect.fnUntraced(
16231716
function*(name, run, options) {
16241717
const shardGroup = options?.shardGroup ?? "default"
@@ -1649,32 +1742,38 @@ const make = Effect.gen(function*() {
16491742

16501743
yield* PubSub.publish(events, SingletonRegistered({ address }))
16511744

1652-
// start if we are on the right shard
1653-
if (MutableHashSet.has(acquiredShards, address.shardId)) {
1745+
// start if we are on the right shard, unless a previous registration of
1746+
// this singleton is still stopping
1747+
if (
1748+
MutableHashSet.has(acquiredShards, address.shardId) &&
1749+
!MutableHashMap.has(stoppingSingletons, address)
1750+
) {
16541751
yield* Effect.logDebug("Starting singleton", address)
16551752
yield* FiberMap.run(singletonFibers, address, wrappedRun)
16561753
}
16571754

1658-
yield* Effect.addFinalizer(() => {
1659-
const map = singletons.get(address.shardId)!
1660-
MutableHashMap.remove(map, address)
1661-
return FiberMap.remove(singletonFibers, address)
1662-
})
1755+
yield* Effect.addFinalizer(() =>
1756+
withSingletonLock(Effect.sync(() => {
1757+
MutableHashMap.remove(singletons.get(address.shardId)!, address)
1758+
return stopSingletonUnsafe(address, false)
1759+
})).pipe(Effect.flatten)
1760+
)
16631761
},
16641762
withSingletonLock
16651763
)
16661764

1667-
const syncSingletons = withSingletonLock(Effect.gen(function*() {
1765+
// Starts and stops singletons to match the acquired shards, returning the
1766+
// waits for the stopped singletons
1767+
const startSyncSingletons = withSingletonLock(Effect.gen(function*() {
1768+
const stops = Arr.empty<Effect.Effect<void>>()
16681769
for (const [shardId, map] of singletons) {
16691770
for (const [address, run] of map) {
1670-
const running = FiberMap.hasUnsafe(singletonFibers, address)
1771+
const stopping = MutableHashMap.has(stoppingSingletons, address)
1772+
const running = stopping || FiberMap.hasUnsafe(singletonFibers, address)
16711773
const shouldBeRunning = MutableHashSet.has(acquiredShards, shardId)
16721774
if (running && !shouldBeRunning) {
1673-
yield* Effect.logDebug("Stopping singleton", address)
1674-
yield* ActiveTeardown.aroundShard(
1675-
address.shardId,
1676-
FiberMap.remove(singletonFibers, address)
1677-
)
1775+
if (!stopping) yield* Effect.logDebug("Stopping singleton", address)
1776+
stops.push(stopSingletonUnsafe(address, true))
16781777
} else if (!running && shouldBeRunning) {
16791778
yield* Effect.logDebug("Starting singleton", address)
16801779
yield* FiberMap.run(singletonFibers, address, run)
@@ -1685,8 +1784,15 @@ const make = Effect.gen(function*() {
16851784
BigInt(yield* FiberMap.size(singletonFibers)),
16861785
Context.empty()
16871786
)
1787+
return stops
16881788
}))
16891789

1790+
const syncSingletons: Effect.Effect<void> = Effect.flatMap(
1791+
startSyncSingletons,
1792+
// wait for the stopped singletons without the lock
1793+
(stops) => Effect.all(stops, { discard: true })
1794+
)
1795+
16901796
// --- Entities ---
16911797

16921798
const reaper = yield* EntityReaper
@@ -1829,11 +1935,12 @@ const make = Effect.gen(function*() {
18291935
MutableRef.set(isShutdown, true)
18301936
yield* notifyShardLockWaiters
18311937
if (selfRunner) {
1832-
yield* Effect.ignore(runnerStorage.unregister(selfRunner.address))
1938+
yield* Effect.ignore(withRegistrationLock(runnerStorage.unregister(selfRunner.address)))
18331939
}
18341940
})
18351941

1836-
yield* Scope.addFinalizerExit(shardingScope, shutdown)
1942+
// hand the shards off before the loops stop
1943+
yield* Scope.addFinalizerExit(shardingScope, (exit) => Effect.andThen(shutdown(exit), handoff))
18371944

18381945
const activeEntityCount = Effect.gen(function*() {
18391946
let count = 0

‎packages/effect/src/cluster/internal/entityManager.ts‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,7 @@ export const make = Effect.fnUntraced(function*<
165165
const serverCloseLatches = new Map<EntityAddress, {
166166
readonly closed: Latch.Latch
167167
readonly force: Latch.Latch
168+
closing: boolean
168169
}>()
169170
const processedRequestIds = new Set<Snowflake.Snowflake>()
170171

@@ -191,7 +192,8 @@ export const make = Effect.fnUntraced(function*<
191192
const keepAliveLatch = Latch.makeUnsafe()
192193
const closeLatches = {
193194
closed: Latch.makeUnsafe(),
194-
force: Latch.makeUnsafe()
195+
force: Latch.makeUnsafe(),
196+
closing: false
195197
}
196198

197199
yield* Scope.addFinalizer(
@@ -469,6 +471,7 @@ export const make = Effect.fnUntraced(function*<
469471
Effect.suspend(() => {
470472
activeServers.delete(address.entityId)
471473
retired.openUnsafe()
474+
closeLatches.closing = true
472475
acquireEntity(address)
473476
drainingServers.set(address.entityId, state)
474477
return Effect.raceFirst(
@@ -714,13 +717,13 @@ export const make = Effect.fnUntraced(function*<
714717
}
715718
})
716719
}
717-
activeServers.forEach((state) => {
718-
if (shardId[Equal.symbol](state.address.shardId)) {
719-
fibers.push(runFork(entities.removeIgnore(state.address)))
720-
}
721-
})
720+
// Look entities up by address: `activeServers` is keyed by entity id,
721+
// so an entity with the same id on another shard can hide this one.
722722
serverCloseLatches.forEach((latches, address) => {
723723
if (shardId[Equal.symbol](address.shardId)) {
724+
if (!latches.closing) {
725+
fibers.push(runFork(entities.removeIgnore(address)))
726+
}
724727
fibers.push(runFork(latches.closed.await))
725728
}
726729
})

0 commit comments

Comments
 (0)