Skip to content

Commit 559f9a5

Browse files
mxpvmxpv
authored
Merge pull request #5744 from mxpv/metrics
Add runtime label to metrics
2 parents ee3272e + efa8ab7 commit 559f9a5

6 files changed

Lines changed: 65 additions & 31 deletions

File tree

‎metrics/cgroups/v1/cgroups.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import (
2929
"github.com/containerd/containerd/namespaces"
3030
"github.com/containerd/containerd/runtime"
3131
"github.com/containerd/containerd/runtime/v1/linux"
32-
metrics "github.com/docker/go-metrics"
32+
"github.com/docker/go-metrics"
3333
"github.com/sirupsen/logrus"
3434
)
3535

@@ -55,8 +55,8 @@ type cgroupsMonitor struct {
5555
publisher events.Publisher
5656
}
5757

58-
func (m *cgroupsMonitor) Monitor(c runtime.Task) error {
59-
if err := m.collector.Add(c); err != nil {
58+
func (m *cgroupsMonitor) Monitor(c runtime.Task, labels map[string]string) error {
59+
if err := m.collector.Add(c, labels); err != nil {
6060
return err
6161
}
6262
t, ok := c.(*linux.Task)

‎metrics/cgroups/v1/metrics.go‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import (
2828
v1 "github.com/containerd/containerd/metrics/types/v1"
2929
"github.com/containerd/containerd/namespaces"
3030
"github.com/containerd/typeurl"
31-
metrics "github.com/docker/go-metrics"
31+
"github.com/docker/go-metrics"
3232
"github.com/gogo/protobuf/types"
3333
"github.com/prometheus/client_golang/prometheus"
3434
)
@@ -53,7 +53,7 @@ func NewCollector(ns *metrics.Namespace) *Collector {
5353
// add machine cpus and memory info
5454
c := &Collector{
5555
ns: ns,
56-
tasks: make(map[string]Statable),
56+
tasks: make(map[string]entry),
5757
}
5858
c.metrics = append(c.metrics, pidMetrics...)
5959
c.metrics = append(c.metrics, cpuMetrics...)
@@ -69,12 +69,19 @@ func taskID(id, namespace string) string {
6969
return fmt.Sprintf("%s-%s", id, namespace)
7070
}
7171

72+
type entry struct {
73+
task Statable
74+
// ns is an optional child namespace that contains additional to parent labels.
75+
// This can be used to append task specific labels to be able to differentiate the different containerd metrics.
76+
ns *metrics.Namespace
77+
}
78+
7279
// Collector provides the ability to collect container stats and export
7380
// them in the prometheus format
7481
type Collector struct {
7582
mu sync.RWMutex
7683

77-
tasks map[string]Statable
84+
tasks map[string]entry
7885
ns *metrics.Namespace
7986
metrics []*metric
8087
storedMetrics chan prometheus.Metric
@@ -109,10 +116,11 @@ storedLoop:
109116
wg.Wait()
110117
}
111118

112-
func (c *Collector) collect(t Statable, ch chan<- prometheus.Metric, block bool, wg *sync.WaitGroup) {
119+
func (c *Collector) collect(entry entry, ch chan<- prometheus.Metric, block bool, wg *sync.WaitGroup) {
113120
if wg != nil {
114121
defer wg.Done()
115122
}
123+
t := entry.task
116124
ctx := namespaces.WithNamespace(context.Background(), t.Namespace())
117125
stats, err := t.Stats(ctx)
118126
if err != nil {
@@ -129,13 +137,17 @@ func (c *Collector) collect(t Statable, ch chan<- prometheus.Metric, block bool,
129137
log.L.WithError(err).Errorf("invalid metric type for %s", t.ID())
130138
return
131139
}
140+
ns := entry.ns
141+
if ns == nil {
142+
ns = c.ns
143+
}
132144
for _, m := range c.metrics {
133-
m.collect(t.ID(), t.Namespace(), s, c.ns, ch, block)
145+
m.collect(t.ID(), t.Namespace(), s, ns, ch, block)
134146
}
135147
}
136148

137149
// Add adds the provided cgroup and id so that metrics are collected and exported
138-
func (c *Collector) Add(t Statable) error {
150+
func (c *Collector) Add(t Statable, labels map[string]string) error {
139151
if c.ns == nil {
140152
return nil
141153
}
@@ -145,7 +157,11 @@ func (c *Collector) Add(t Statable) error {
145157
if _, ok := c.tasks[id]; ok {
146158
return nil // requests to collect metrics should be idempotent
147159
}
148-
c.tasks[id] = t
160+
entry := entry{task: t}
161+
if labels != nil {
162+
entry.ns = c.ns.WithConstLabels(labels)
163+
}
164+
c.tasks[id] = entry
149165
return nil
150166
}
151167

@@ -165,6 +181,6 @@ func (c *Collector) RemoveAll() {
165181
return
166182
}
167183
c.mu.Lock()
168-
c.tasks = make(map[string]Statable)
184+
c.tasks = make(map[string]entry)
169185
c.mu.Unlock()
170186
}

‎metrics/cgroups/v2/cgroups.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import (
2323

2424
"github.com/containerd/containerd/events"
2525
"github.com/containerd/containerd/runtime"
26-
metrics "github.com/docker/go-metrics"
26+
"github.com/docker/go-metrics"
2727
)
2828

2929
// NewTaskMonitor returns a new cgroups monitor
@@ -42,8 +42,8 @@ type cgroupsMonitor struct {
4242
publisher events.Publisher
4343
}
4444

45-
func (m *cgroupsMonitor) Monitor(c runtime.Task) error {
46-
if err := m.collector.Add(c); err != nil {
45+
func (m *cgroupsMonitor) Monitor(c runtime.Task, labels map[string]string) error {
46+
if err := m.collector.Add(c, labels); err != nil {
4747
return err
4848
}
4949
return nil

‎metrics/cgroups/v2/metrics.go‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import (
2727
v2 "github.com/containerd/containerd/metrics/types/v2"
2828
"github.com/containerd/containerd/namespaces"
2929
"github.com/containerd/typeurl"
30-
metrics "github.com/docker/go-metrics"
30+
"github.com/docker/go-metrics"
3131
"github.com/gogo/protobuf/types"
3232
"github.com/prometheus/client_golang/prometheus"
3333
)
@@ -47,7 +47,7 @@ func NewCollector(ns *metrics.Namespace) *Collector {
4747
}
4848
c := &Collector{
4949
ns: ns,
50-
tasks: make(map[string]Statable),
50+
tasks: make(map[string]entry),
5151
}
5252
c.metrics = append(c.metrics, pidMetrics...)
5353
c.metrics = append(c.metrics, cpuMetrics...)
@@ -62,12 +62,19 @@ func taskID(id, namespace string) string {
6262
return fmt.Sprintf("%s-%s", id, namespace)
6363
}
6464

65+
type entry struct {
66+
task Statable
67+
// ns is an optional child namespace that contains additional to parent labels.
68+
// This can be used to append task specific labels to be able to differentiate the different containerd metrics.
69+
ns *metrics.Namespace
70+
}
71+
6572
// Collector provides the ability to collect container stats and export
6673
// them in the prometheus format
6774
type Collector struct {
6875
mu sync.RWMutex
6976

70-
tasks map[string]Statable
77+
tasks map[string]entry
7178
ns *metrics.Namespace
7279
metrics []*metric
7380
storedMetrics chan prometheus.Metric
@@ -102,10 +109,11 @@ storedLoop:
102109
wg.Wait()
103110
}
104111

105-
func (c *Collector) collect(t Statable, ch chan<- prometheus.Metric, block bool, wg *sync.WaitGroup) {
112+
func (c *Collector) collect(entry entry, ch chan<- prometheus.Metric, block bool, wg *sync.WaitGroup) {
106113
if wg != nil {
107114
defer wg.Done()
108115
}
116+
t := entry.task
109117
ctx := namespaces.WithNamespace(context.Background(), t.Namespace())
110118
stats, err := t.Stats(ctx)
111119
if err != nil {
@@ -122,13 +130,17 @@ func (c *Collector) collect(t Statable, ch chan<- prometheus.Metric, block bool,
122130
log.L.WithError(err).Errorf("invalid metric type for %s", t.ID())
123131
return
124132
}
133+
ns := entry.ns
134+
if ns == nil {
135+
ns = c.ns
136+
}
125137
for _, m := range c.metrics {
126-
m.collect(t.ID(), t.Namespace(), s, c.ns, ch, block)
138+
m.collect(t.ID(), t.Namespace(), s, ns, ch, block)
127139
}
128140
}
129141

130142
// Add adds the provided cgroup and id so that metrics are collected and exported
131-
func (c *Collector) Add(t Statable) error {
143+
func (c *Collector) Add(t Statable, labels map[string]string) error {
132144
if c.ns == nil {
133145
return nil
134146
}
@@ -138,7 +150,11 @@ func (c *Collector) Add(t Statable) error {
138150
if _, ok := c.tasks[id]; ok {
139151
return nil // requests to collect metrics should be idempotent
140152
}
141-
c.tasks[id] = t
153+
entry := entry{task: t}
154+
if labels != nil {
155+
entry.ns = c.ns.WithConstLabels(labels)
156+
}
157+
c.tasks[id] = entry
142158
return nil
143159
}
144160

@@ -158,6 +174,6 @@ func (c *Collector) RemoveAll() {
158174
return
159175
}
160176
c.mu.Lock()
161-
c.tasks = make(map[string]Statable)
177+
c.tasks = make(map[string]entry)
162178
c.mu.Unlock()
163179
}

‎runtime/monitor.go‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,11 @@ package runtime
1818

1919
// TaskMonitor provides an interface for monitoring of containers within containerd
2020
type TaskMonitor interface {
21-
// Monitor adds the provided container to the monitor
22-
Monitor(Task) error
21+
// Monitor adds the provided container to the monitor.
22+
// Labels are optional (can be nil) key value pairs to be added to the metrics namespace.
23+
Monitor(task Task, labels map[string]string) error
2324
// Stop stops and removes the provided container from the monitor
24-
Stop(Task) error
25+
Stop(task Task) error
2526
}
2627

2728
// NewMultiTaskMonitor returns a new TaskMonitor broadcasting to the provided monitors
@@ -39,7 +40,7 @@ func NewNoopMonitor() TaskMonitor {
3940
type noopTaskMonitor struct {
4041
}
4142

42-
func (mm *noopTaskMonitor) Monitor(c Task) error {
43+
func (mm *noopTaskMonitor) Monitor(c Task, labels map[string]string) error {
4344
return nil
4445
}
4546

@@ -51,9 +52,9 @@ type multiTaskMonitor struct {
5152
monitors []TaskMonitor
5253
}
5354

54-
func (mm *multiTaskMonitor) Monitor(c Task) error {
55+
func (mm *multiTaskMonitor) Monitor(task Task, labels map[string]string) error {
5556
for _, m := range mm.monitors {
56-
if err := m.Monitor(c); err != nil {
57+
if err := m.Monitor(task, labels); err != nil {
5758
return err
5859
}
5960
}

‎services/tasks/local.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,15 +115,15 @@ func initFunc(ic *plugin.InitContext) (interface{}, error) {
115115
return nil, err
116116
}
117117
for _, t := range tasks {
118-
l.monitor.Monitor(t)
118+
l.monitor.Monitor(t, nil)
119119
}
120120
}
121121
v2Tasks, err := l.v2Runtime.Tasks(ic.Context, true)
122122
if err != nil {
123123
return nil, err
124124
}
125125
for _, t := range v2Tasks {
126-
l.monitor.Monitor(t)
126+
l.monitor.Monitor(t, nil)
127127
}
128128
return l, nil
129129
}
@@ -211,7 +211,8 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc.
211211
if err != nil {
212212
return nil, errdefs.ToGRPC(err)
213213
}
214-
if err := l.monitor.Monitor(c); err != nil {
214+
labels := map[string]string{"runtime": container.Runtime.Name}
215+
if err := l.monitor.Monitor(c, labels); err != nil {
215216
return nil, errors.Wrap(err, "monitor task")
216217
}
217218
return &api.CreateTaskResponse{

0 commit comments

Comments
 (0)