Skip to content

Commit 6553ea1

Browse files
authored
stats/otel: Add subchannel metrics (A94) (#8738)
Addresses : https://github.com/grpc/proposal/blob/master/A94-subchannel-otel-metrics.md This PR adds subchannel metrics with applicable labels as per the RFC proposal. `disconnection_reason` will be added as a follow up PR. RELEASE NOTES: * stats/otel: add subchannel metrics (without the disconnection reason) to eventually replace the pickfirst metrics.
1 parent 81a00ce commit 6553ea1

9 files changed

Lines changed: 422 additions & 11 deletions

File tree

‎balancer/pickfirst/metrics_test.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,11 +102,31 @@ func (s) TestPickFirstMetrics(t *testing.T) {
102102
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.disconnections", got, 0)
103103
}
104104

105+
// Checking for subchannel metrics as well
106+
if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_succeeded"); got != 1 {
107+
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_succeeded", got, 1)
108+
}
109+
if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_failed"); got != 0 {
110+
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_failed", got, 0)
111+
}
112+
if got, _ := tmr.Metric("grpc.subchannel.disconnections"); got != 0 {
113+
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.disconnections", got, 0)
114+
}
115+
if got, _ := tmr.Metric("grpc.subchannel.open_connections"); got != 1 {
116+
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.open_connections", got, 1)
117+
}
118+
105119
ss.Stop()
106120
testutils.AwaitState(ctx, t, cc, connectivity.Idle)
107121
if got, _ := tmr.Metric("grpc.lb.pick_first.disconnections"); got != 1 {
108122
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.disconnections", got, 1)
109123
}
124+
if got, _ := tmr.Metric("grpc.subchannel.disconnections"); got != 1 {
125+
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.disconnections", got, 1)
126+
}
127+
if got, _ := tmr.Metric("grpc.subchannel.open_connections"); got != -1 {
128+
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.open_connections", got, -1)
129+
}
110130
}
111131

112132
// TestPickFirstMetricsFailure tests the connection attempts failed metric. It

‎balancer/pickfirst/pickfirst_ext_test.go‎

Lines changed: 114 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1935,6 +1935,7 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TF_AfterEndOfList(t *testing.T) {
19351935
holds[1].Fail(fmt.Errorf("test error"))
19361936
tmr.WaitForInt64CountIncr(ctx, 1)
19371937
testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
1938+
waitForMetric(ctx, t, tmr, "grpc.subchannel.connection_attempts_failed")
19381939

19391940
// Only connection attempt fails in this test.
19401941
if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_succeeded"); got != 0 {
@@ -1946,6 +1947,12 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TF_AfterEndOfList(t *testing.T) {
19461947
if got, _ := tmr.Metric("grpc.lb.pick_first.disconnections"); got != 0 {
19471948
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.disconnections", got, 0)
19481949
}
1950+
if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_failed"); got != 1 {
1951+
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_failed", got, 1)
1952+
}
1953+
if got, _ := tmr.Metric("grpc.lb.subchannel.connection_attempts_succeeded"); got != 0 {
1954+
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.subchannel.connection_attempts_succeeded", got, 0)
1955+
}
19491956
}
19501957

19511958
// Test verifies that pickfirst attempts to connect to the second backend once
@@ -1995,7 +2002,7 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TriggerConnectionDelay(t *testing.T) {
19952002
// that the channel becomes READY.
19962003
holds[1].Resume()
19972004
testutils.AwaitState(ctx, t, cc, connectivity.Ready)
1998-
2005+
waitForMetric(ctx, t, tmr, "grpc.subchannel.connection_attempts_succeeded")
19992006
// Only connection attempt successes in this test.
20002007
if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_succeeded"); got != 1 {
20012008
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.connection_attempts_succeeded", got, 1)
@@ -2006,6 +2013,27 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TriggerConnectionDelay(t *testing.T) {
20062013
if got, _ := tmr.Metric("grpc.lb.pick_first.disconnections"); got != 0 {
20072014
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.disconnections", got, 0)
20082015
}
2016+
2017+
if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_succeeded"); got != 1 {
2018+
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_succeeded", got, 1)
2019+
}
2020+
if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_failed"); got != 0 {
2021+
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_failed", got, 0)
2022+
}
2023+
}
2024+
2025+
func waitForMetric(ctx context.Context, t *testing.T, tmr *stats.TestMetricsRecorder, metricName string) {
2026+
for {
2027+
if _, ok := tmr.Metric(metricName); ok {
2028+
break
2029+
}
2030+
select {
2031+
case <-ctx.Done():
2032+
t.Fatalf("Timeout waiting for metric emission: %s", metricName)
2033+
case <-time.After(10 * time.Millisecond):
2034+
continue
2035+
}
2036+
}
20092037
}
20102038

20112039
// Test tests the pickfirst balancer by causing a SubConn to fail and then
@@ -2057,6 +2085,9 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TF_ThenTimerFires(t *testing.T) {
20572085
if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_failed"); got != 1 {
20582086
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.connection_attempts_failed", got, 1)
20592087
}
2088+
if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_failed"); got != 1 {
2089+
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_failed", got, 1)
2090+
}
20602091
if holds[2].IsStarted() != false {
20612092
t.Fatalf("Server %d with address %q contacted unexpectedly", 2, addrs[2])
20622093
}
@@ -2073,7 +2104,10 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TF_ThenTimerFires(t *testing.T) {
20732104
// that the channel becomes READY.
20742105
holds[1].Resume()
20752106
testutils.AwaitState(ctx, t, cc, connectivity.Ready)
2076-
2107+
waitForMetric(ctx, t, tmr, "grpc.subchannel.connection_attempts_succeeded")
2108+
if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_succeeded"); got != 1 {
2109+
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.subchannel.connection_attempts_succeeded", got, 1)
2110+
}
20772111
if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_succeeded"); got != 1 {
20782112
t.Errorf("Unexpected data for metric %v, got: %v, want: %v", "grpc.lb.pick_first.connection_attempts_succeeded", got, 1)
20792113
}
@@ -2082,6 +2116,84 @@ func (s) TestPickFirstLeaf_HappyEyeballs_TF_ThenTimerFires(t *testing.T) {
20822116
}
20832117
}
20842118

2119+
// Test verifies that when a subchannel is shut down by the LB (because another
2120+
// subchannel won) while its dial is still in-flight, it records exactly one
2121+
// successful attempt.
2122+
func (s) TestPickFirstLeaf_HappyEyeballs_Ignore_Inflight_Cancellations(t *testing.T) {
2123+
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
2124+
defer cancel()
2125+
2126+
originalTimer := pfinternal.TimeAfterFunc
2127+
defer func() {
2128+
pfinternal.TimeAfterFunc = originalTimer
2129+
}()
2130+
triggerTimer, timeAfter := mockTimer()
2131+
pfinternal.TimeAfterFunc = timeAfter
2132+
2133+
tmr := stats.NewTestMetricsRecorder()
2134+
dialer := testutils.NewBlockingDialer()
2135+
opts := []grpc.DialOption{
2136+
grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, pfbalancer.Name)),
2137+
grpc.WithContextDialer(dialer.DialContext),
2138+
grpc.WithStatsHandler(tmr),
2139+
}
2140+
2141+
// Setup 2 backend addresses
2142+
cc, rb, bm := setupPickFirstLeaf(t, 2, opts...)
2143+
addrs := bm.resolverAddrs()
2144+
holds := bm.holds(dialer)
2145+
rb.UpdateState(resolver.State{Addresses: addrs})
2146+
cc.Connect()
2147+
2148+
// Make sure we connect to second subconn
2149+
testutils.AwaitState(ctx, t, cc, connectivity.Connecting)
2150+
if holds[0].Wait(ctx) != true {
2151+
t.Fatalf("Timeout waiting for server %d to be contacted", 0)
2152+
}
2153+
triggerTimer()
2154+
if holds[1].Wait(ctx) != true {
2155+
t.Fatalf("Timeout waiting for server %d to be contacted", 1)
2156+
}
2157+
holds[1].Resume()
2158+
2159+
// Wait for Channel to become READY.
2160+
testutils.AwaitState(ctx, t, cc, connectivity.Ready)
2161+
2162+
// Unblock the First SubConn.
2163+
// Since the LB has already closed this subchannel, the context passed to Dial
2164+
// is canceled. This will lead to an inflight attempt to be cancelled.
2165+
// No success or failure metric should be recorded for this.
2166+
holds[0].Resume()
2167+
2168+
// --- Assertions ---
2169+
2170+
// Wait for the SUCCESS metric to ensure recording logic has processed.
2171+
waitForMetric(ctx, t, tmr, "grpc.subchannel.connection_attempts_succeeded")
2172+
2173+
// Verify Success: Exactly 1 (The Winner).
2174+
if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_succeeded"); got != 1 {
2175+
t.Errorf("Unexpected data for metric %v, got: %v, want: 1", "grpc.subchannel.connection_attempts_succeeded", got)
2176+
}
2177+
2178+
// Verify Failure: Exactly 0 (The Loser was ignored).
2179+
// We poll briefly to ensure no delayed failure metric appears.
2180+
sCtx, sCancel := context.WithTimeout(ctx, 50*time.Millisecond)
2181+
defer sCancel()
2182+
for ; sCtx.Err() == nil; <-time.After(time.Millisecond) {
2183+
if got, _ := tmr.Metric("grpc.subchannel.connection_attempts_failed"); got != 0 {
2184+
t.Fatalf("Unexpected failure recorded for shutdown subchannel, got: %v, want: 0", got)
2185+
}
2186+
}
2187+
2188+
// LB Metrics Check
2189+
if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_succeeded"); got != 1 {
2190+
t.Errorf("Unexpected data for metric %v, got: %v, want: 1", "grpc.lb.pick_first.connection_attempts_succeeded", got)
2191+
}
2192+
if got, _ := tmr.Metric("grpc.lb.pick_first.connection_attempts_failed"); got != 0 {
2193+
t.Errorf("Unexpected data for metric %v, got: %v, want: 0", "grpc.lb.pick_first.connection_attempts_failed", got)
2194+
}
2195+
}
2196+
20852197
func (s) TestPickFirstLeaf_InterleavingIPV4Preferred(t *testing.T) {
20862198
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
20872199
defer cancel()

‎clientconn.go‎

Lines changed: 103 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ import (
3535
"google.golang.org/grpc/balancer/pickfirst"
3636
"google.golang.org/grpc/codes"
3737
"google.golang.org/grpc/connectivity"
38+
"google.golang.org/grpc/credentials"
39+
expstats "google.golang.org/grpc/experimental/stats"
3840
"google.golang.org/grpc/internal"
3941
"google.golang.org/grpc/internal/channelz"
4042
"google.golang.org/grpc/internal/grpcsync"
@@ -98,6 +100,41 @@ var (
98100
errTransportCredentialsMissing = errors.New("grpc: the credentials require transport level security (use grpc.WithTransportCredentials() to set)")
99101
)
100102

103+
var (
104+
disconnectionsMetric = expstats.RegisterInt64Count(expstats.MetricDescriptor{
105+
Name: "grpc.subchannel.disconnections",
106+
Description: "EXPERIMENTAL. Number of times the selected subchannel becomes disconnected.",
107+
Unit: "{disconnection}",
108+
Labels: []string{"grpc.target"},
109+
OptionalLabels: []string{"grpc.lb.backend_service", "grpc.lb.locality", "grpc.disconnect_error"},
110+
Default: false,
111+
})
112+
connectionAttemptsSucceededMetric = expstats.RegisterInt64Count(expstats.MetricDescriptor{
113+
Name: "grpc.subchannel.connection_attempts_succeeded",
114+
Description: "EXPERIMENTAL. Number of successful connection attempts.",
115+
Unit: "{attempt}",
116+
Labels: []string{"grpc.target"},
117+
OptionalLabels: []string{"grpc.lb.backend_service", "grpc.lb.locality"},
118+
Default: false,
119+
})
120+
connectionAttemptsFailedMetric = expstats.RegisterInt64Count(expstats.MetricDescriptor{
121+
Name: "grpc.subchannel.connection_attempts_failed",
122+
Description: "EXPERIMENTAL. Number of failed connection attempts.",
123+
Unit: "{attempt}",
124+
Labels: []string{"grpc.target"},
125+
OptionalLabels: []string{"grpc.lb.backend_service", "grpc.lb.locality"},
126+
Default: false,
127+
})
128+
openConnectionsMetric = expstats.RegisterInt64UpDownCount(expstats.MetricDescriptor{
129+
Name: "grpc.subchannel.open_connections",
130+
Description: "EXPERIMENTAL. Number of open connections.",
131+
Unit: "{attempt}",
132+
Labels: []string{"grpc.target"},
133+
OptionalLabels: []string{"grpc.lb.backend_service", "grpc.security_level", "grpc.lb.locality"},
134+
Default: false,
135+
})
136+
)
137+
101138
const (
102139
defaultClientMaxReceiveMessageSize = 1024 * 1024 * 4
103140
defaultClientMaxSendMessageSize = math.MaxInt32
@@ -875,6 +912,7 @@ func (cc *ClientConn) newAddrConnLocked(addrs []resolver.Address, opts balancer.
875912
channelz: channelz.RegisterSubChannel(cc.channelz, ""),
876913
resetBackoff: make(chan struct{}),
877914
}
915+
ac.updateTelemetryLabelsLocked()
878916
ac.ctx, ac.cancel = context.WithCancel(cc.ctx)
879917
// Start with our address set to the first address; this may be updated if
880918
// we connect to different addresses.
@@ -991,7 +1029,7 @@ func (ac *addrConn) updateAddrs(addrs []resolver.Address) {
9911029
}
9921030

9931031
ac.addrs = addrs
994-
1032+
ac.updateTelemetryLabelsLocked()
9951033
if ac.state == connectivity.Shutdown ||
9961034
ac.state == connectivity.TransientFailure ||
9971035
ac.state == connectivity.Idle {
@@ -1230,13 +1268,28 @@ type addrConn struct {
12301268
resetBackoff chan struct{}
12311269

12321270
channelz *channelz.SubChannel
1271+
1272+
localityLabel string
1273+
backendServiceLabel string
12331274
}
12341275

12351276
// Note: this requires a lock on ac.mu.
12361277
func (ac *addrConn) updateConnectivityState(s connectivity.State, lastErr error) {
12371278
if ac.state == s {
12381279
return
12391280
}
1281+
1282+
// If we are transitioning out of Ready, it means there is a disconnection.
1283+
// A SubConn can also transition from CONNECTING directly to IDLE when
1284+
// a transport is successfully created, but the connection fails
1285+
// before the SubConn can send the notification for READY. We treat
1286+
// this as a successful connection and transition to IDLE.
1287+
// TODO: https://github.com/grpc/grpc-go/issues/7862 - Remove the second
1288+
// part of the if condition below once the issue is fixed.
1289+
if ac.state == connectivity.Ready || (ac.state == connectivity.Connecting && s == connectivity.Idle) {
1290+
disconnectionsMetric.Record(ac.cc.metricsRecorderList, 1, ac.cc.target, ac.backendServiceLabel, ac.localityLabel, "unknown")
1291+
openConnectionsMetric.Record(ac.cc.metricsRecorderList, -1, ac.cc.target, ac.backendServiceLabel, ac.securityLevelLocked(), ac.localityLabel)
1292+
}
12401293
ac.state = s
12411294
ac.channelz.ChannelMetrics.State.Store(&s)
12421295
if lastErr == nil {
@@ -1294,6 +1347,15 @@ func (ac *addrConn) resetTransportAndUnlock() {
12941347
ac.mu.Unlock()
12951348

12961349
if err := ac.tryAllAddrs(acCtx, addrs, connectDeadline); err != nil {
1350+
if !errors.Is(err, context.Canceled) {
1351+
connectionAttemptsFailedMetric.Record(ac.cc.metricsRecorderList, 1, ac.cc.target, ac.backendServiceLabel, ac.localityLabel)
1352+
} else {
1353+
if logger.V(2) {
1354+
// This records cancelled connection attempts which can be later
1355+
// replaced by a metric.
1356+
logger.Infof("Context cancellation detected; not recording this as a failed connection attempt.")
1357+
}
1358+
}
12971359
// TODO: #7534 - Move re-resolution requests into the pick_first LB policy
12981360
// to ensure one resolution request per pass instead of per subconn failure.
12991361
ac.cc.resolveNow(resolver.ResolveNowOptions{})
@@ -1333,10 +1395,50 @@ func (ac *addrConn) resetTransportAndUnlock() {
13331395
}
13341396
// Success; reset backoff.
13351397
ac.mu.Lock()
1398+
connectionAttemptsSucceededMetric.Record(ac.cc.metricsRecorderList, 1, ac.cc.target, ac.backendServiceLabel, ac.localityLabel)
1399+
openConnectionsMetric.Record(ac.cc.metricsRecorderList, 1, ac.cc.target, ac.backendServiceLabel, ac.securityLevelLocked(), ac.localityLabel)
13361400
ac.backoffIdx = 0
13371401
ac.mu.Unlock()
13381402
}
13391403

1404+
// updateTelemetryLabelsLocked calculates and caches the telemetry labels based on the
1405+
// first address in addrConn.
1406+
func (ac *addrConn) updateTelemetryLabelsLocked() {
1407+
labelsFunc, ok := internal.AddressToTelemetryLabels.(func(resolver.Address) map[string]string)
1408+
if !ok || len(ac.addrs) == 0 {
1409+
// Reset defaults
1410+
ac.localityLabel = ""
1411+
ac.backendServiceLabel = ""
1412+
return
1413+
}
1414+
labels := labelsFunc(ac.addrs[0])
1415+
ac.localityLabel = labels["grpc.lb.locality"]
1416+
ac.backendServiceLabel = labels["grpc.lb.backend_service"]
1417+
}
1418+
1419+
type securityLevelKey struct{}
1420+
1421+
func (ac *addrConn) securityLevelLocked() string {
1422+
var secLevel string
1423+
// During disconnection, ac.transport is nil. Fall back to the security level
1424+
// stored in the current address during connection.
1425+
if ac.transport == nil {
1426+
secLevel, _ = ac.curAddr.Attributes.Value(securityLevelKey{}).(string)
1427+
return secLevel
1428+
}
1429+
authInfo := ac.transport.Peer().AuthInfo
1430+
if ci, ok := authInfo.(interface {
1431+
GetCommonAuthInfo() credentials.CommonAuthInfo
1432+
}); ok {
1433+
secLevel = ci.GetCommonAuthInfo().SecurityLevel.String()
1434+
// Store the security level in the current address' attributes so
1435+
// that it remains available for disconnection metrics after the
1436+
// transport is closed.
1437+
ac.curAddr.Attributes = ac.curAddr.Attributes.WithValue(securityLevelKey{}, secLevel)
1438+
}
1439+
return secLevel
1440+
}
1441+
13401442
// tryAllAddrs tries to create a connection to the addresses, and stop when at
13411443
// the first successful one. It returns an error if no address was successfully
13421444
// connected, or updates ac appropriately with the new transport.

‎internal/internal.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,10 @@ var (
244244
// When set, the function will be called before the stream enters
245245
// the blocking state.
246246
NewStreamWaitingForResolver = func() {}
247+
248+
// AddressToTelemetryLabels is an xDS-provided function to extract telemetry
249+
// labels from a resolver.Address. Callers must assert its type before calling.
250+
AddressToTelemetryLabels any // func(addr resolver.Address) map[string]string
247251
)
248252

249253
// HealthChecker defines the signature of the client-side LB channel health

0 commit comments

Comments
 (0)