Skip to content

Commit d331f11

Browse files
authored
server: release mutex while waiting for handlers in stop() (#9394)
Fixes #9393 `Stop()` deadlocks when called after `GracefulStop()` while a handler is still running. #### Root cause Both `Stop()` and `GracefulStop()` call `stop()`, which acquires `s.mu` and holds it for the entire function, including across `s.handlersWG.Wait()`. If a handler ignores context cancellation, `GracefulStop()` blocks in that wait while still holding `s.mu`. A concurrent `Stop()` (the documented way to abort a `GracefulStop()` that is taking too long) then blocks forever on `s.mu.Lock()` and can never force the server to shut down. #### Fix Release `s.mu` around `s.handlersWG.Wait()` and reacquire it afterwards, mirroring the connection-drain loop that already releases the lock via `s.cv.Wait()`. This is safe because none of the state guarded by `s.mu` is accessed during the wait: by that point listeners are closed, all transports are drained/closed, `s.conns` is `nil`, and the worker channel is closed. `handlersWG.Wait()` only touches the `WaitGroup`, which has its own synchronization and was never guarded by `s.mu`. A concurrent `Stop()` can now acquire the lock and return promptly. #### Testing Added `TestStopAfterGracefulStopWithRunningHandler`, which starts a handler that ignores context cancellation, closes the client connection so the transport drains, runs `GracefulStop()` in a goroutine, and asserts that a subsequent `Stop()` returns promptly. Verified the test times out (deadlocks) without the fix and passes with it. RELEASE NOTES: * server: fix deadlock where `Stop()` blocks indefinitely when called concurrently with an in-progress `GracefulStop()` waiting on a handler that does not return promptly
1 parent d41d9b3 commit d331f11

2 files changed

Lines changed: 87 additions & 0 deletions

File tree

‎server.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1719,7 +1719,13 @@ func (s *Server) stop(graceful bool) {
17191719
}
17201720

17211721
if graceful || s.opts.waitForHandlers {
1722+
// Release the lock while waiting for handlers to finish. Holding it
1723+
// here would deadlock a concurrent Stop() (the recommended way to abort
1724+
// a GracefulStop that is taking too long), because Stop() must acquire
1725+
// s.mu before it can force the server to shut down.
1726+
s.mu.Unlock()
17221727
s.handlersWG.Wait()
1728+
s.mu.Lock()
17231729
}
17241730

17251731
if s.events != nil {

‎test/gracefulstop_test.go‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,87 @@ func (s) TestStopAbortsBlockingGRPCCall(t *testing.T) {
301301
<-grpcClientCallReturned
302302
}
303303

304+
// TestStopAfterGracefulStopWithRunningHandler verifies that Stop() returns
305+
// promptly when called concurrently with an in-progress GracefulStop() that is
306+
// blocked waiting on a handler which ignores context cancellation. This is the
307+
// recommended way to abort a GracefulStop that takes too long, and it must not
308+
// deadlock behind the server mutex.
309+
func (s) TestStopAfterGracefulStopWithRunningHandler(t *testing.T) {
310+
handlerStarted := make(chan struct{})
311+
unblockHandler := make(chan struct{})
312+
ss := &stubserver.StubServer{
313+
FullDuplexCallF: func(_ testgrpc.TestService_FullDuplexCallServer) error {
314+
close(handlerStarted)
315+
// Ignore context cancellation to simulate a handler that does not
316+
// promptly return when the server shuts down.
317+
<-unblockHandler
318+
return nil
319+
},
320+
}
321+
if err := ss.Start(nil); err != nil {
322+
t.Fatalf("Error starting endpoint server: %v", err)
323+
}
324+
defer ss.Stop()
325+
// The handler blocks until unblockHandler is closed. If the test fails
326+
// before reaching the normal unblock point below, this ensures the handler
327+
// is released so its goroutine does not leak. handlerUnblocked guards
328+
// against closing the channel twice.
329+
handlerUnblocked := false
330+
defer func() {
331+
if !handlerUnblocked {
332+
close(unblockHandler)
333+
}
334+
}()
335+
336+
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
337+
defer cancel()
338+
339+
if _, err := ss.Client.FullDuplexCall(ctx); err != nil {
340+
t.Fatalf("Error starting FullDuplexCall: %v", err)
341+
}
342+
select {
343+
case <-handlerStarted:
344+
case <-ctx.Done():
345+
t.Fatalf("Timed out waiting for the handler to start: %v", ctx.Err())
346+
}
347+
348+
// Close the client connection so the server transport drains while the
349+
// handler is still running. GracefulStop will then remove the connection
350+
// and block waiting on the running handler.
351+
ss.CC.Close()
352+
353+
gracefulStopReturned := make(chan struct{})
354+
go func() {
355+
ss.S.GracefulStop()
356+
close(gracefulStopReturned)
357+
}()
358+
359+
// Give GracefulStop time to drain the connection and start waiting on the
360+
// running handler.
361+
time.Sleep(defaultTestShortTimeout)
362+
363+
stopReturned := make(chan struct{})
364+
go func() {
365+
ss.S.Stop()
366+
close(stopReturned)
367+
}()
368+
369+
select {
370+
case <-stopReturned:
371+
case <-time.After(defaultTestTimeout):
372+
t.Fatal("Timed out waiting for Stop() to return")
373+
}
374+
375+
// Let the handler finish so GracefulStop can also return.
376+
close(unblockHandler)
377+
handlerUnblocked = true
378+
select {
379+
case <-gracefulStopReturned:
380+
case <-time.After(defaultTestTimeout):
381+
t.Fatal("GracefulStop() did not return after the handler completed")
382+
}
383+
}
384+
304385
// TestServeHTTP_GracefulStop verifies that calling GracefulStop on a server
305386
// using ServeHTTP returns properly.
306387
func (s) TestServeHTTP_GracefulStop(t *testing.T) {

0 commit comments

Comments
 (0)