Skip to content

Commit cd2ab19

Browse files
Tanner-Scaddenclaudeeffect-bottim-smart
authored
Route shard handoff interrupts to the draining owner and wait for shard locks (#8940)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]> Co-authored-by: Effect Bot <[email protected]> Co-authored-by: Tim Smart <[email protected]>
1 parent 73fa287 commit cd2ab19

7 files changed

Lines changed: 231 additions & 35 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"effect": patch
3+
---
4+
5+
Fix cluster shard handoffs. Interrupts and stream acknowledgements now reach requests still draining on the previous owner, and `RpcServer` accepts acknowledgements for active streams after a client's EOF. Volatile requests sent during a handoff wait, for at most `shardLockExpiration`, until the shard can be served instead of repeatedly failing with `EntityNotAssignedToRunner`.

‎packages/effect/src/cluster/Sharding.ts‎

Lines changed: 59 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,33 @@ const make = Effect.gen(function*() {
295295
return MutableHashSet.has(acquiredShards, address.shardId)
296296
}
297297

298+
function isAwaitingShardLock(shardId: ShardId): boolean {
299+
return !MutableRef.get(isShutdown) &&
300+
MutableHashSet.has(selfShards, shardId) &&
301+
!MutableHashSet.has(acquiredShards, shardId)
302+
}
303+
304+
// Requests for a shard assigned to this runner wait for its lock, for at most
305+
// `shardLockExpiration`, instead of bouncing while the previous owner drains.
306+
// Each notification opens the current latch and replaces it, so a waiter that
307+
// checked its shard before a notification cannot miss it.
308+
let shardLocksChanged = Latch.makeUnsafe(false)
309+
const notifyShardLockWaiters = Effect.suspend(() => {
310+
const latch = shardLocksChanged
311+
shardLocksChanged = Latch.makeUnsafe(false)
312+
return latch.open
313+
})
314+
const awaitShardLock = (shardId: ShardId): Effect.Effect<void> => {
315+
const loop: Effect.Effect<void> = Effect.suspend(() =>
316+
isAwaitingShardLock(shardId) ? Effect.andThen(shardLocksChanged.await, loop) : Effect.void
317+
)
318+
return Effect.suspend(() =>
319+
isAwaitingShardLock(shardId)
320+
? Effect.asVoid(Effect.timeoutOption(loop, config.shardLockExpiration))
321+
: Effect.void
322+
)
323+
}
324+
298325
yield* Scope.addFinalizer(
299326
shardingScope,
300327
Effect.logDebug("Shutdown complete").pipe(Effect.annotateLogs({
@@ -484,6 +511,7 @@ const make = Effect.gen(function*() {
484511
MutableHashSet.add(acquiredShards, shardId)
485512
}
486513
if (acquired.length > 0) {
514+
yield* notifyShardLockWaiters
487515
yield* storageReadLatch.open
488516
yield* Effect.forkIn(syncSingletons, shardingScope)
489517

@@ -519,6 +547,7 @@ const make = Effect.gen(function*() {
519547
activeShardsLatch.openUnsafe()
520548

521549
return Effect.gen(function*() {
550+
yield* notifyShardLockWaiters
522551
yield* Effect.logError("Shard lock storage is unhealthy", cause)
523552
yield* Effect.forkIn(syncSingletons, shardingScope, { startImmediately: true })
524553

@@ -1036,16 +1065,21 @@ const make = Effect.gen(function*() {
10361065

10371066
// --- Sending messages ---
10381067

1039-
const sendLocal = <M extends Message.Outgoing<any> | Message.Incoming<any>>(message: M) =>
1040-
Effect.suspend(function loop(): Effect.Effect<
1068+
const sendLocal = <M extends Message.Outgoing<any> | Message.Incoming<any>>(message: M) => {
1069+
const address = message.envelope.address
1070+
const isRequest = message._tag === "IncomingRequest" || message._tag === "OutgoingRequest"
1071+
const send = Effect.suspend(function loop(): Effect.Effect<
10411072
void,
10421073
| EntityNotAssignedToRunner
10431074
| MailboxFull
10441075
| AlreadyProcessingMessage
10451076
| (M extends Message.Incoming<any> ? never : PersistenceError)
10461077
> {
1047-
const address = message.envelope.address
1048-
if (!isEntityOnLocalShards(address)) {
1078+
if (
1079+
!isEntityOnLocalShards(address) &&
1080+
// Entities draining from a released shard still accept interrupts and acks
1081+
(isRequest || !MutableHashSet.has(releasingShards, address.shardId))
1082+
) {
10491083
return Effect.fail(new EntityNotAssignedToRunner({ address }))
10501084
}
10511085
const state = entityManagers.get(address.entityType)
@@ -1064,6 +1098,8 @@ const make = Effect.gen(function*() {
10641098
simulateRemoteSerialization: config.simulateRemoteSerialization
10651099
}) as any
10661100
})
1101+
return isRequest ? Effect.andThen(awaitShardLock(address.shardId), send) : send
1102+
}
10671103

10681104
type PendingNotification = {
10691105
resume: (_: Effect.Effect<void, EntityNotAssignedToRunner>) => void
@@ -1129,7 +1165,10 @@ const make = Effect.gen(function*() {
11291165
function sendOutgoing(
11301166
message: Message.Outgoing<any>,
11311167
discard: boolean,
1132-
retries?: number
1168+
retries?: number,
1169+
// The runner serving the request, for its interrupts and acks. Later
1170+
// attempts fall back to the current shard assignment.
1171+
runner?: RunnerAddress
11331172
): Effect.Effect<
11341173
void,
11351174
MailboxFull | AlreadyProcessingMessage | PersistenceError | EntityNotAssignedToRunner
@@ -1172,14 +1211,19 @@ const make = Effect.gen(function*() {
11721211
if (shouldFail && MutableRef.get(isShutdown)) {
11731212
return Effect.fail(new EntityNotAssignedToRunner({ address }))
11741213
}
1175-
const maybeRunner = MutableHashMap.get(shardAssignments, address.shardId)
1214+
const maybeRunner = runner !== undefined
1215+
? Option.some(runner)
1216+
: MutableHashMap.get(shardAssignments, address.shardId)
11761217
const runnerIsLocal = Option.isSome(maybeRunner) && isLocalRunner(maybeRunner.value)
11771218
if (isPersisted) {
11781219
return runnerIsLocal
11791220
? notifyLocal(message, discard)
11801221
: runnersService.notify({ address: maybeRunner, message, discard })
11811222
} else if (Option.isNone(maybeRunner)) {
11821223
return Effect.fail(new EntityNotAssignedToRunner({ address }))
1224+
} else if (message._tag === "OutgoingRequest") {
1225+
const entry = clientRequests.get(message.envelope.requestId)
1226+
if (entry) entry.runner = maybeRunner.value
11831227
}
11841228
return runnerIsLocal
11851229
? sendLocal(message)
@@ -1324,6 +1368,7 @@ const make = Effect.gen(function*() {
13241368
})
13251369
yield* Effect.logDebug("New shard assignments", selfShards)
13261370
activeShardsLatch.openUnsafe()
1371+
yield* notifyShardLockWaiters
13271372

13281373
// update metrics
13291374
if (selfRunner) {
@@ -1361,6 +1406,8 @@ const make = Effect.gen(function*() {
13611406
readonly context: Context.Context<never>
13621407
readonly message: Message.OutgoingRequest<any>
13631408
lastChunkId?: Snowflake.Snowflake
1409+
// the runner a volatile request was last sent to
1410+
runner?: RunnerAddress
13641411
}
13651412
const clientRequests = new Map<Snowflake.Snowflake, ClientRequestEntry>()
13661413

@@ -1445,7 +1492,9 @@ const make = Effect.gen(function*() {
14451492
}),
14461493
rpc: entry.rpc
14471494
}),
1448-
false
1495+
false,
1496+
undefined,
1497+
entry.runner
14491498
)
14501499
}
14511500
case "Interrupt": {
@@ -1476,7 +1525,8 @@ const make = Effect.gen(function*() {
14761525
rpc: entry.rpc
14771526
}),
14781527
false,
1479-
3
1528+
3,
1529+
entry.runner
14801530
))
14811531
}
14821532
}
@@ -1777,6 +1827,7 @@ const make = Effect.gen(function*() {
17771827

17781828
if (isShutdown.current) return
17791829
MutableRef.set(isShutdown, true)
1830+
yield* notifyShardLockWaiters
17801831
if (selfRunner) {
17811832
yield* Effect.ignore(runnerStorage.unregister(selfRunner.address))
17821833
}

‎packages/effect/src/cluster/internal/entityManager.ts‎

Lines changed: 39 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,9 @@ export const make = Effect.fnUntraced(function*<
157157
entityRpcs.set(KeepAliveRpc._tag, KeepAliveRpc as any)
158158

159159
const activeServers = new Map<EntityId, EntityState>()
160+
// Entities finishing their in-flight requests after removal, which still
161+
// accept interrupts and acks
162+
const drainingServers = new Map<EntityId, EntityState>()
160163
const serverCloseLatches = new Map<EntityAddress, {
161164
readonly closed: Latch.Latch
162165
readonly force: Latch.Latch
@@ -465,14 +468,19 @@ export const make = Effect.fnUntraced(function*<
465468
activeServers.delete(address.entityId)
466469
retired.openUnsafe()
467470
acquireEntity(address)
471+
drainingServers.set(address.entityId, state)
468472
return Effect.raceFirst(
469473
state.write(0, { _tag: "Eof" }).pipe(
470474
Effect.andThen(endLatch.await),
471475
Effect.timeoutOption(config.entityTerminationTimeout),
472476
Effect.interruptible
473477
),
474478
Effect.interruptible(closeLatches.force.await)
475-
)
479+
).pipe(Effect.ensuring(Effect.sync(() => {
480+
if (drainingServers.get(address.entityId) === state) {
481+
drainingServers.delete(address.entityId)
482+
}
483+
})))
476484
})
477485
)
478486
if (!options.sharding.hasShardId(address.shardId)) {
@@ -507,9 +515,38 @@ export const make = Effect.fnUntraced(function*<
507515
Effect.forkIn(managerScope)
508516
)
509517

518+
function sendEnvelope(server: EntityState, message: Message.IncomingEnvelope): Effect.Effect<void> {
519+
const entry = server.activeRequests.get(message.envelope.requestId)
520+
if (!entry) {
521+
return Effect.void
522+
} else if (
523+
message.envelope._tag === "AckChunk" &&
524+
Option.isSome(entry.lastSentChunk) &&
525+
message.envelope.replyId !== entry.lastSentChunk.value.id
526+
) {
527+
return Effect.void
528+
}
529+
return server.write(
530+
0,
531+
message.envelope._tag === "AckChunk"
532+
? { _tag: "Ack", requestId: message.envelope.requestId as any }
533+
: {
534+
_tag: "Interrupt",
535+
requestId: message.envelope.requestId as any,
536+
interruptors: []
537+
}
538+
)
539+
}
540+
510541
function sendLocal<R extends Rpc.Any>(
511542
message: Message.IncomingLocal<R>
512543
): Effect.Effect<void, EntityNotAssignedToRunner | MailboxFull | AlreadyProcessingMessage> {
544+
if (message._tag === "IncomingEnvelope") {
545+
const draining = drainingServers.get(message.envelope.address.entityId)
546+
if (draining?.activeRequests.has(message.envelope.requestId)) {
547+
return sendEnvelope(draining, message)
548+
}
549+
}
513550
return Effect.provideService(
514551
Effect.flatMap(
515552
entities.get(message.envelope.address),
@@ -606,26 +643,7 @@ export const make = Effect.fnUntraced(function*<
606643
return server.write(0, requestEnvelope(entry), requestWriteOptions(entry))
607644
}
608645
case "IncomingEnvelope": {
609-
const entry = server.activeRequests.get(message.envelope.requestId)
610-
if (!entry) {
611-
return Effect.void
612-
} else if (
613-
message.envelope._tag === "AckChunk" &&
614-
Option.isSome(entry.lastSentChunk) &&
615-
message.envelope.replyId !== entry.lastSentChunk.value.id
616-
) {
617-
return Effect.void
618-
}
619-
return server.write(
620-
0,
621-
message.envelope._tag === "AckChunk"
622-
? { _tag: "Ack", requestId: message.envelope.requestId as any }
623-
: {
624-
_tag: "Interrupt",
625-
requestId: message.envelope.requestId as any,
626-
interruptors: []
627-
}
628-
)
646+
return sendEnvelope(server, message)
629647
}
630648
}
631649
}

‎packages/effect/src/rpc/RpcServer.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ export const makeNoSerialization: <Rpcs extends Rpc.Any>(
188188
clients.set(clientId, client)
189189
} else if (
190190
client.ended &&
191-
(message._tag !== "Interrupt" || !client.fibers.has(message.requestId))
191+
((message._tag !== "Interrupt" && message._tag !== "Ack") || !client.fibers.has(message.requestId))
192192
) {
193193
return Effect.interrupt
194194
}
@@ -199,7 +199,7 @@ export const makeNoSerialization: <Rpcs extends Rpc.Any>(
199199
}
200200
case "Ack": {
201201
const latch = client.latches?.get(message.requestId)
202-
return latch ? latch.open : Effect.void
202+
return latch ? Effect.asVoid(latch.open) : Effect.void
203203
}
204204
case "Interrupt": {
205205
const fiber = client.fibers.get(message.requestId)

0 commit comments

Comments
 (0)