Skip to content

Commit 16736e9

Browse files
klihubklihub
committed
rdt: implement OpenTelemetry metrics.
Implement OpenTelemetry metric instruments covering the same set of RDT features as the existing Prometheus metrics collector. Notes: Unfortunately we can't get fully identical metrics using otel. Due to how otel translates otel instruments to prometheus metrics, we either end up with different types (gauge vs. counter) or extra metric name suffices derived from types and units. Signed-off-by: Krisztian Litkey <[email protected]>
1 parent acf9094 commit 16736e9

3 files changed

Lines changed: 221 additions & 22 deletions

File tree

‎go.mod‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,16 @@
11
module github.com/intel/goresctrl
22

3-
go 1.23
3+
go 1.23.0
44

55
require (
6-
github.com/google/go-cmp v0.5.9
6+
github.com/google/go-cmp v0.7.0
77
github.com/opencontainers/runtime-spec v1.0.2
88
github.com/prometheus/client_golang v1.16.0
9-
github.com/stretchr/testify v1.8.1
9+
github.com/stretchr/testify v1.11.1
10+
go.opentelemetry.io/otel v1.38.0
11+
go.opentelemetry.io/otel/metric v1.38.0
1012
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611
11-
golang.org/x/sys v0.11.0
13+
golang.org/x/sys v0.21.0
1214
k8s.io/apimachinery v0.27.4
1315
sigs.k8s.io/yaml v1.3.0
1416
)
@@ -20,13 +22,12 @@ require (
2022
github.com/gogo/protobuf v1.3.2 // indirect
2123
github.com/golang/protobuf v1.5.3 // indirect
2224
github.com/google/gofuzz v1.2.0 // indirect
23-
github.com/kr/text v0.2.0 // indirect
2425
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
2526
github.com/pmezard/go-difflib v1.0.0 // indirect
2627
github.com/prometheus/client_model v0.3.0 // indirect
2728
github.com/prometheus/common v0.42.0 // indirect
2829
github.com/prometheus/procfs v0.10.1 // indirect
29-
github.com/rogpeppe/go-internal v1.9.0 // indirect
30+
github.com/rogpeppe/go-internal v1.13.1 // indirect
3031
google.golang.org/protobuf v1.33.0 // indirect
3132
gopkg.in/inf.v0 v0.9.1 // indirect
3233
gopkg.in/yaml.v2 v2.4.0 // indirect

‎go.sum‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
22
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
33
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
44
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
5-
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
6-
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
75
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
86
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
7+
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
8+
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
9+
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
10+
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
911
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
1012
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
1113
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -14,8 +16,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
1416
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
1517
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
1618
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
17-
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
18-
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
19+
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
20+
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
1921
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
2022
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
2123
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -38,19 +40,22 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI
3840
github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
3941
github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
4042
github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
41-
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
42-
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
43+
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
44+
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
4345
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
4446
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
45-
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
46-
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
47-
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
48-
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
49-
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
50-
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
51-
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
47+
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
48+
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
5249
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
5350
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
51+
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
52+
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
53+
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
54+
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
55+
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
56+
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
57+
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
58+
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
5459
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
5560
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
5661
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -69,8 +74,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
6974
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
7075
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
7176
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
72-
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
73-
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
77+
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
78+
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
7479
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
7580
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
7681
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -92,7 +97,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
9297
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
9398
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
9499
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
95-
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
96100
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
97101
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
98102
k8s.io/apimachinery v0.27.4 h1:CdxflD4AF61yewuid0fLl6bM4a3q04jWel0IlP+aYjs=

‎pkg/rdt/otel.go‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
/*
2+
Copyright 2025 Intel Corporation
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package rdt
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"strings"
23+
"sync"
24+
25+
"go.opentelemetry.io/otel/attribute"
26+
"go.opentelemetry.io/otel/metric"
27+
)
28+
29+
func RegisterOpenTelemetryInstruments(meter metric.Meter) error {
30+
for resource, features := range GetMonFeatures() {
31+
switch resource {
32+
case MonResourceL3:
33+
for _, f := range features {
34+
if err := registerInstrument(meter, f); err != nil {
35+
return err
36+
}
37+
}
38+
}
39+
}
40+
41+
return nil
42+
}
43+
44+
func registerInstrument(meter metric.Meter, feature string) error {
45+
instr, err := createInstrument(meter, feature)
46+
if err != nil {
47+
return err
48+
}
49+
50+
_, err = meter.RegisterCallback(
51+
func(ctx context.Context, o metric.Observer) error {
52+
select {
53+
case <-ctx.Done():
54+
return fmt.Errorf("rdt metric collection cancelled: %w", ctx.Err())
55+
default:
56+
observeInstrument(o, instr, feature)
57+
}
58+
return nil
59+
},
60+
instr,
61+
)
62+
if err != nil {
63+
return err
64+
}
65+
66+
return nil
67+
}
68+
69+
func createInstrument(meter metric.Meter, feature string) (metric.Int64Observable, error) {
70+
switch feature {
71+
case "llc_occupancy":
72+
name := "l3.llc.occupancy"
73+
help := "L3 (LLC) occupancy"
74+
return meter.Int64ObservableGauge(
75+
name,
76+
metric.WithDescription(help),
77+
)
78+
79+
case "mbm_local_bytes":
80+
name := "l3.mbm.local"
81+
help := "bytes transferred to/from local memory through LLC"
82+
unit := "bytes"
83+
return meter.Int64ObservableCounter(
84+
name,
85+
metric.WithDescription(help),
86+
metric.WithUnit(unit),
87+
)
88+
89+
case "mbm_total_bytes":
90+
name := "l3.mbm.total"
91+
help := "total bytes transferred to/from memory through LLC"
92+
unit := "bytes"
93+
return meter.Int64ObservableCounter(
94+
name,
95+
metric.WithDescription(help),
96+
metric.WithUnit(unit),
97+
)
98+
}
99+
100+
// an unknown feature, counter for bytes, gauge otherwise
101+
name := ""
102+
help := ""
103+
unit := ""
104+
if strings.HasSuffix(feature, "_bytes") {
105+
name = strings.TrimSuffix(feature, "_bytes")
106+
unit = "bytes"
107+
}
108+
name = "l3." + strings.ReplaceAll(name, "_", ".")
109+
help = "L3 " + feature
110+
111+
if unit == "bytes" {
112+
return meter.Int64ObservableUpDownCounter(
113+
name,
114+
metric.WithDescription(help),
115+
metric.WithUnit(unit),
116+
)
117+
}
118+
119+
return meter.Int64ObservableGauge(
120+
name,
121+
metric.WithDescription(help),
122+
metric.WithUnit(unit),
123+
)
124+
}
125+
126+
func observeInstrument(o metric.Observer, m metric.Int64Observable, feature string) {
127+
var wg sync.WaitGroup
128+
129+
for _, cls := range GetClasses() {
130+
observeGroup(o, m, feature, cls)
131+
for _, monGrp := range cls.GetMonGroups() {
132+
wg.Add(1)
133+
go func(mg MonGroup) {
134+
defer wg.Done()
135+
observeGroup(o, m, feature, mg, getMonGroupAttributes(mg)...)
136+
}(monGrp)
137+
}
138+
}
139+
140+
wg.Wait()
141+
}
142+
143+
func observeGroup(o metric.Observer, m metric.Int64Observable, feature string, g ResctrlGroup, customAttributes ...attribute.KeyValue) {
144+
var (
145+
allData = g.GetMonData()
146+
cgName string
147+
mgName string
148+
)
149+
150+
if mg, ok := g.(MonGroup); ok {
151+
cgName = mg.Parent().Name()
152+
mgName = mg.Name()
153+
} else {
154+
cgName = g.Name()
155+
}
156+
157+
attributes := attribute.NewSet(
158+
attribute.String("rdt.class", cgName),
159+
attribute.String("rdt.mon.group", mgName),
160+
)
161+
162+
for cacheID, data := range allData.L3 {
163+
if value, ok := data[feature]; ok {
164+
o.ObserveInt64(
165+
m,
166+
int64(value),
167+
metric.WithAttributeSet(attributes),
168+
metric.WithAttributes(
169+
append(
170+
[]attribute.KeyValue{
171+
attribute.String("cache.id", fmt.Sprint(cacheID)),
172+
},
173+
customAttributes...,
174+
)...,
175+
),
176+
)
177+
}
178+
}
179+
}
180+
181+
func getMonGroupAttributes(mg MonGroup) []attribute.KeyValue {
182+
var (
183+
annotations = mg.GetAnnotations()
184+
attributes = []attribute.KeyValue{}
185+
)
186+
187+
for _, name := range customLabels {
188+
if value, ok := annotations[name]; ok {
189+
attributes = append(attributes, attribute.String(name, value))
190+
}
191+
}
192+
193+
return attributes
194+
}

0 commit comments

Comments
 (0)