Skip to content

Commit b58f32d

Browse files
authored
server: Set a pprof label on new stream goroutines (#9082)
Fixes #9010 To make stack-traces and some profiles more useful, change sets goroutine labels indicating which gRPC method is being handled. Goroutine labels are inherited by child goroutines, so this provides useful context in profiles and traces for work that's been farmed out to child goroutines. These currently show up in three places: - trace labels in pprof CPU profiles - trace labels in runtime/pprof (and http/pprof) debug=0 goroutine profiles (which are pprof format) - debug=1 aggregated text-format goroutine profiles For Go 1.27, golang/go#76349 adds goroutine labels to tracebacks and by extension debug=2 pprof text-based profiles for go 1.27+ modules. The naming of the goroutine label currently matches the opentelemetry RPC method tag, and has some similarities to the current proposal for goroutine tag naming for tests in golang/go#75047. I.e. this uses `grpc.method`. This change avoids setting anything on the client side due to the lower utility and goroutine lifetime issues. Include a `GRPC_GO_SERVER_GOROUTINE_LABELS` environment variable to allow users to easily opt-in of setting these goroutine labels. The value the form `grpc.method=true` to enable a specific goroutine label, and has special values of `all` and `none` which enable and disable all registered goroutine labels respectively. RELEASE NOTES: * server: Set runtime/pprof goroutine labels for incoming method streams. This may be enabled with the `GRPC_GO_SERVER_GOROUTINE_LABELS=grpc.method=true` environment variable for just the new goroutine label or `GRPC_GO_SERVER_GOROUTINE_LABELS=all` to enable all goroutine labels that might be added later. `none` is available for blanket disabling, as well.
1 parent 6c98be3 commit b58f32d

4 files changed

Lines changed: 279 additions & 0 deletions

File tree

‎internal/envconfig/envconfig.go‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,15 @@ var (
5959
// unconditionally.
6060
XDSEndpointHashKeyBackwardCompat = boolFromEnv("GRPC_XDS_ENDPOINT_HASH_KEY_BACKWARD_COMPAT", false)
6161

62+
// LabelServerGoroutines controls setting [runtime/pprof.Labels] on the
63+
// goroutines spawned by [grpc.Server] type.
64+
// For now, this is limited to the goroutines spawned to handle incoming
65+
// requests on the server.
66+
// Set "GRPC_GO_SERVER_GOROUTINE_LABELS" to "grpc.method=true" to
67+
// enable this grpc.method label, or "all" to enable all valid labels.
68+
// This variable is a bit-field.
69+
LabelServerGoroutines = goroutineLabelsFromEnv("GRPC_GO_SERVER_GOROUTINE_LABELS", 0)
70+
6271
// RingHashSetRequestHashKey is set if the ring hash balancer can get the
6372
// request hash header by setting the "requestHashHeader" field, according
6473
// to gRFC A76. It can be disabled by setting the environment variable
@@ -156,3 +165,52 @@ func uint64FromEnv(envVar string, def, min, max uint64) uint64 {
156165
}
157166
return v
158167
}
168+
169+
// GoroutineLabels is a bitfield indicating which goroutine labels are enabled.
170+
type GoroutineLabels uint16
171+
172+
func goroutineLabelsFromEnv(envVar string, def GoroutineLabels) GoroutineLabels {
173+
val := def
174+
v := os.Getenv(envVar)
175+
if strings.EqualFold(v, "all") {
176+
return AllGoroutineLabels
177+
} else if strings.EqualFold(v, "none") {
178+
return 0
179+
}
180+
for s := range strings.SplitSeq(v, ",") {
181+
s = strings.TrimSpace(s)
182+
if len(s) == 0 {
183+
continue
184+
}
185+
pre, post, ok := strings.Cut(s, "=")
186+
if !ok {
187+
// no equals sign
188+
continue
189+
}
190+
post = strings.TrimSpace(post)
191+
pre = strings.TrimSpace(pre)
192+
bitDesignator := GoroutineLabels(0)
193+
switch {
194+
case strings.EqualFold(pre, "grpc.method"):
195+
bitDesignator = GoroutineLabelServerMethod
196+
default:
197+
continue
198+
}
199+
if strings.EqualFold(post, "true") {
200+
val |= bitDesignator
201+
} else if strings.EqualFold(post, "false") {
202+
val &^= bitDesignator
203+
}
204+
}
205+
return val
206+
}
207+
208+
const (
209+
// GoroutineLabelServerMethod sets the grpc.method label on new
210+
// server-side gRPC streams.
211+
GoroutineLabelServerMethod GoroutineLabels = 1 << iota
212+
)
213+
214+
// AllGoroutineLabels is an or'd together bitfield of all valid GoroutineLabels
215+
// constant values (above).
216+
const AllGoroutineLabels = GoroutineLabelServerMethod

‎internal/envconfig/envconfig_test.go‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,3 +101,118 @@ func (s) TestBoolFromEnv(t *testing.T) {
101101
})
102102
}
103103
}
104+
105+
func (s) TestGoroutineLabelsFromEnv(t *testing.T) {
106+
var testCases = []struct {
107+
name string
108+
val string
109+
def GoroutineLabels
110+
want GoroutineLabels
111+
}{
112+
{
113+
name: "unset_env_non-zero_default",
114+
val: "",
115+
def: GoroutineLabelServerMethod,
116+
want: GoroutineLabelServerMethod,
117+
}, {
118+
name: "unset_env_zero_default",
119+
val: "",
120+
def: 0,
121+
want: 0,
122+
}, {
123+
name: "force-enable_zero_default",
124+
val: "grpc.method=true",
125+
def: 0,
126+
want: GoroutineLabelServerMethod,
127+
}, {
128+
name: "force-enable_zero_default_all_caps",
129+
val: "grpc.method=TRUE",
130+
def: 0,
131+
want: GoroutineLabelServerMethod,
132+
}, {
133+
name: "force-enable_zero_default_with_whitespace",
134+
val: " grpc.method\t= true",
135+
def: 0,
136+
want: GoroutineLabelServerMethod,
137+
}, {
138+
name: "force-enable_zero_default_with_other_garbage",
139+
val: "grpc.method=true,foobar",
140+
def: 0,
141+
want: GoroutineLabelServerMethod,
142+
}, {
143+
name: "force-enable_mixed_case_zero_default_with_other_garbage",
144+
val: "grpc.method=tRuE,foobar",
145+
def: 0,
146+
want: GoroutineLabelServerMethod,
147+
}, {
148+
name: "force-disable_zero_default",
149+
val: "grpc.method=false",
150+
def: 0,
151+
want: 0,
152+
}, {
153+
name: "force-disable_non-zero_default",
154+
val: "grpc.method=false",
155+
def: GoroutineLabelServerMethod,
156+
want: 0,
157+
}, {
158+
name: "force-disable_non-zero_default_all_caps",
159+
val: "grpc.method=FALSE",
160+
def: GoroutineLabelServerMethod,
161+
want: 0,
162+
}, {
163+
name: "force-disable_non-zero_default_mixed_case",
164+
val: "grpc.method=fAlSe",
165+
def: GoroutineLabelServerMethod,
166+
want: 0,
167+
}, {
168+
name: "unknown_val_no_equal",
169+
val: "grpc.unknown.garbage",
170+
def: GoroutineLabelServerMethod,
171+
want: GoroutineLabelServerMethod,
172+
}, {
173+
name: "unknown_val",
174+
val: "grpc.unknown.garbage=fooble",
175+
def: GoroutineLabelServerMethod,
176+
want: GoroutineLabelServerMethod,
177+
}, {
178+
name: "all_with_empty_default",
179+
val: "all",
180+
def: 0,
181+
want: AllGoroutineLabels,
182+
}, {
183+
name: "all_with_server_method_default",
184+
val: "all",
185+
def: GoroutineLabelServerMethod,
186+
want: AllGoroutineLabels,
187+
}, {
188+
name: "none_with_empty_default",
189+
val: "none",
190+
def: 0,
191+
want: 0,
192+
}, {
193+
name: "none_with_server_method_default",
194+
val: "none",
195+
def: GoroutineLabelServerMethod,
196+
want: 0,
197+
}, {
198+
name: "unparseable_rhs",
199+
val: "grpc.method=quux",
200+
def: GoroutineLabelServerMethod,
201+
want: GoroutineLabelServerMethod,
202+
},
203+
}
204+
for _, tc := range testCases {
205+
t.Run(tc.name, func(t *testing.T) {
206+
const testVar = "testvar"
207+
if tc.val == "" {
208+
os.Unsetenv(testVar)
209+
} else {
210+
os.Setenv(testVar, tc.val)
211+
}
212+
if got := goroutineLabelsFromEnv(testVar, tc.def); got != tc.want {
213+
t.Errorf("goroutineLabelsFromEnv(%q(=%q), %v) = %v; want %v", testVar, tc.val, tc.def, got, tc.want)
214+
}
215+
})
216+
}
217+
218+
}

‎server.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"net/http"
2929
"reflect"
3030
"runtime"
31+
"runtime/pprof"
3132
"strings"
3233
"sync"
3334
"sync/atomic"
@@ -42,6 +43,7 @@ import (
4243
"google.golang.org/grpc/internal"
4344
"google.golang.org/grpc/internal/binarylog"
4445
"google.golang.org/grpc/internal/channelz"
46+
"google.golang.org/grpc/internal/envconfig"
4547
"google.golang.org/grpc/internal/grpcsync"
4648
"google.golang.org/grpc/internal/grpcutil"
4749
istats "google.golang.org/grpc/internal/stats"
@@ -1798,6 +1800,12 @@ func (s *Server) handleMalformedMethodName(stream *transport.ServerStream, ti *t
17981800
func (s *Server) handleStream(t transport.ServerTransport, stream *transport.ServerStream) {
17991801
ctx := stream.Context()
18001802
ctx = contextWithServer(ctx, s)
1803+
if envconfig.LabelServerGoroutines&envconfig.GoroutineLabelServerMethod != 0 {
1804+
// This method always runs in its own goroutine, so we can set a
1805+
// goroutine label without needing to restore a previous context.
1806+
ctx = pprof.WithLabels(ctx, pprof.Labels("grpc.method", stream.Method()))
1807+
pprof.SetGoroutineLabels(ctx)
1808+
}
18011809
var ti *traceInfo
18021810
if EnableTracing {
18031811
tr := newTrace("grpc.Recv."+methodFamily(stream.Method()), stream.Method())

‎test/server_test.go‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,15 @@ package test
2121
import (
2222
"context"
2323
"io"
24+
"runtime/pprof"
2425
"sync/atomic"
2526
"testing"
2627

2728
"google.golang.org/grpc"
2829
"google.golang.org/grpc/codes"
30+
"google.golang.org/grpc/internal/envconfig"
2931
"google.golang.org/grpc/internal/stubserver"
32+
"google.golang.org/grpc/internal/testutils"
3033
"google.golang.org/grpc/status"
3134

3235
testgrpc "google.golang.org/grpc/interop/grpc_testing"
@@ -70,6 +73,101 @@ func (s) TestServerReturningContextError(t *testing.T) {
7073

7174
}
7275

76+
func pprofCtxCollectLabels(ctx context.Context) map[string]string {
77+
seenLabels := map[string]string{}
78+
pprof.ForLabels(ctx, func(k, val string) bool {
79+
seenLabels[k] = val
80+
return true
81+
})
82+
return seenLabels
83+
}
84+
85+
// TestServerSetGoroutineLabelsInContext verifies that when enabled, the
86+
// grpc.method runtime/pprof goroutine label gets set in the context that's
87+
// passed to the handlers.
88+
func (s) TestServerSetGoroutineLabelsInContext(t *testing.T) {
89+
testutils.SetEnvConfig(t, &envconfig.LabelServerGoroutines, envconfig.GoroutineLabelServerMethod)
90+
ss := &stubserver.StubServer{
91+
EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
92+
ctxLabels := pprofCtxCollectLabels(ctx)
93+
if val, ok := ctxLabels["grpc.method"]; !ok {
94+
t.Errorf("missing \"grpc.method\" label; found labels: %v", ctxLabels)
95+
} else if wantVal := "/grpc.testing.TestService/EmptyCall"; val != wantVal {
96+
t.Errorf("unexpected value for \"grpc.method\" label %q; want %q", ctxLabels["grpc.method"], wantVal)
97+
}
98+
return &testpb.Empty{}, nil
99+
},
100+
FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
101+
ctxLabels := pprofCtxCollectLabels(stream.Context())
102+
if val, ok := ctxLabels["grpc.method"]; !ok {
103+
t.Errorf("missing \"grpc.method\" label; found labels: %v", ctxLabels)
104+
} else if wantVal := "/grpc.testing.TestService/FullDuplexCall"; val != wantVal {
105+
t.Errorf("unexpected value for \"grpc.method\" label %q; want %q", ctxLabels["grpc.method"], wantVal)
106+
}
107+
return nil
108+
},
109+
}
110+
if err := ss.Start(nil); err != nil {
111+
t.Fatalf("Error starting endpoint server: %v", err)
112+
}
113+
defer ss.Stop()
114+
115+
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
116+
defer cancel()
117+
if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
118+
t.Fatalf("ss.Client.EmptyCall() got error %v; want OK", err)
119+
}
120+
121+
stream, err := ss.Client.FullDuplexCall(ctx)
122+
if err != nil {
123+
t.Fatalf("unexpected error starting the stream: %v", err)
124+
}
125+
if _, err = stream.Recv(); err != io.EOF {
126+
t.Fatalf("ss.Client.FullDuplexCall().Recv() got error %v; want io.EOF", err)
127+
}
128+
}
129+
130+
// TestServerSetGoroutineLabelsInContextEnvVarDisabled verifies that when
131+
// disable, the grpc.method runtime/pprof goroutine label does _not_ get set in
132+
// the context that's passed to the handlers.
133+
func (s) TestServerSetGoroutineLabelsInContextEnvVarDisabled(t *testing.T) {
134+
testutils.SetEnvConfig(t, &envconfig.LabelServerGoroutines, 0)
135+
ss := &stubserver.StubServer{
136+
EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
137+
ctxLabels := pprofCtxCollectLabels(ctx)
138+
if val, ok := ctxLabels["grpc.method"]; ok {
139+
t.Errorf("\"grpc.method\" label set with value %q; found labels: %v", val, ctxLabels)
140+
}
141+
return &testpb.Empty{}, nil
142+
},
143+
FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
144+
ctxLabels := pprofCtxCollectLabels(stream.Context())
145+
if val, ok := ctxLabels["grpc.method"]; ok {
146+
t.Errorf("\"grpc.method\" label set with value %q; found labels: %v", val, ctxLabels)
147+
}
148+
return nil
149+
},
150+
}
151+
if err := ss.Start(nil); err != nil {
152+
t.Fatalf("Error starting endpoint server: %v", err)
153+
}
154+
defer ss.Stop()
155+
156+
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
157+
defer cancel()
158+
if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
159+
t.Fatalf("ss.Client.EmptyCall() got error %v; want OK", err)
160+
}
161+
162+
stream, err := ss.Client.FullDuplexCall(ctx)
163+
if err != nil {
164+
t.Fatalf("unexpected error starting the stream: %v", err)
165+
}
166+
if _, err = stream.Recv(); err != io.EOF {
167+
t.Fatalf("ss.Client.FullDuplexCall().Recv() got error %v; want io.EOF", err)
168+
}
169+
}
170+
73171
func (s) TestChainUnaryServerInterceptor(t *testing.T) {
74172
var (
75173
firstIntKey = ctxKey("firstIntKey")

0 commit comments

Comments
 (0)