Skip to content

Commit d763e6f

Browse files
ilyam8ilyam8
authored
chore(go.d/snmp): add collection stats (netdata#21409)
1 parent bd3157d commit d763e6f

17 files changed

+543
-72
lines changed

‎src/go/plugin/go.d/collector/snmp/charts.go‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,12 @@ const (
1616
prioProfileChart = module.Priority + iota
1717
prioPingRtt
1818
prioPingStdDev
19+
20+
prioInternalStatsTimings
21+
prioInternalStatsSnmpOps
22+
prioInternalStatsMetrics
23+
prioInternalStatsTableCache
24+
prioInternalStatsErrors
1925
)
2026

2127
var (
@@ -66,6 +72,115 @@ func (c *Collector) addPingCharts() {
6672
}
6773
}
6874

75+
var (
76+
profileStatsChartsTmpl = module.Charts{
77+
profileStatsTimingsChartTmpl.Copy(),
78+
profileStatsSnmpChartTmpl.Copy(),
79+
profileStatsMetricsChartTmpl.Copy(),
80+
profileStatsTableCacheChartTmpl.Copy(),
81+
profileStatsErrorsChartTmpl.Copy(),
82+
}
83+
84+
profileStatsTimingsChartTmpl = module.Chart{
85+
ID: "snmp_device_prof_%s_stats_timings",
86+
Title: "SNMP profile collection timings",
87+
Units: "milliseconds",
88+
Fam: "Internal/Stats",
89+
Ctx: "snmp.device_prof_stats_timings",
90+
Priority: prioInternalStatsTimings,
91+
Dims: module.Dims{
92+
{ID: "snmp_device_prof_%s_stats_timings_scalar", Name: "scalar"},
93+
{ID: "snmp_device_prof_%s_stats_timings_table", Name: "table"},
94+
{ID: "snmp_device_prof_%s_stats_timings_virtual", Name: "virtual"},
95+
},
96+
}
97+
98+
profileStatsSnmpChartTmpl = module.Chart{
99+
ID: "snmp_device_prof_%s_stats_snmp",
100+
Title: "SNMP profile operations",
101+
Units: "operations",
102+
Fam: "Internal/Stats",
103+
Ctx: "snmp.device_prof_stats_snmp",
104+
Priority: prioInternalStatsSnmpOps,
105+
Dims: module.Dims{
106+
{ID: "snmp_device_prof_%s_stats_snmp_get_requests", Name: "get_requests"},
107+
{ID: "snmp_device_prof_%s_stats_snmp_get_oids", Name: "get_oids"},
108+
{ID: "snmp_device_prof_%s_stats_snmp_walk_requests", Name: "walk_requests"},
109+
{ID: "snmp_device_prof_%s_stats_snmp_walk_pdus", Name: "walk_pdus"},
110+
{ID: "snmp_device_prof_%s_stats_snmp_tables_walked", Name: "tables_walked"},
111+
{ID: "snmp_device_prof_%s_stats_snmp_tables_cached", Name: "tables_cached"},
112+
},
113+
}
114+
115+
profileStatsMetricsChartTmpl = module.Chart{
116+
ID: "snmp_device_prof_%s_stats_metrics",
117+
Title: "SNMP profile metric counts",
118+
Units: "metrics",
119+
Fam: "Internal/Stats",
120+
Ctx: "snmp.device_prof_stats_metrics",
121+
Priority: prioInternalStatsMetrics,
122+
Dims: module.Dims{
123+
{ID: "snmp_device_prof_%s_stats_metrics_scalar", Name: "scalar"},
124+
{ID: "snmp_device_prof_%s_stats_metrics_table", Name: "table"},
125+
{ID: "snmp_device_prof_%s_stats_metrics_virtual", Name: "virtual"},
126+
{ID: "snmp_device_prof_%s_stats_metrics_tables", Name: "tables"},
127+
{ID: "snmp_device_prof_%s_stats_metrics_rows", Name: "rows"},
128+
},
129+
}
130+
131+
profileStatsTableCacheChartTmpl = module.Chart{
132+
ID: "snmp_device_prof_%s_stats_table_cache",
133+
Title: "SNMP profile table cache",
134+
Units: "tables",
135+
Fam: "Internal/Stats",
136+
Ctx: "snmp.device_prof_stats_table_cache",
137+
Priority: prioInternalStatsTableCache,
138+
Dims: module.Dims{
139+
{ID: "snmp_device_prof_%s_stats_table_cache_hits", Name: "hits"},
140+
{ID: "snmp_device_prof_%s_stats_table_cache_misses", Name: "misses"},
141+
},
142+
}
143+
144+
profileStatsErrorsChartTmpl = module.Chart{
145+
ID: "snmp_device_prof_%s_stats_errors",
146+
Title: "SNMP profile errors",
147+
Units: "errors",
148+
Fam: "Internal/Stats",
149+
Ctx: "snmp.device_prof_stats_errors",
150+
Priority: prioInternalStatsErrors,
151+
Dims: module.Dims{
152+
{ID: "snmp_device_prof_%s_stats_errors_snmp", Name: "snmp"},
153+
{ID: "snmp_device_prof_%s_stats_errors_processing_scalar", Name: "processing_scalar"},
154+
{ID: "snmp_device_prof_%s_stats_errors_processing_table", Name: "processing_table"},
155+
},
156+
}
157+
)
158+
159+
func (c *Collector) addProfileStatsCharts(name string) {
160+
if name == "" {
161+
return
162+
}
163+
164+
charts := profileStatsChartsTmpl.Copy()
165+
166+
labels := c.chartBaseLabels()
167+
labels["profile"] = name
168+
169+
for _, chart := range *charts {
170+
chart.ID = fmt.Sprintf(chart.ID, name)
171+
for _, dim := range chart.Dims {
172+
dim.ID = fmt.Sprintf(dim.ID, name)
173+
}
174+
for k, v := range labels {
175+
chart.Labels = append(chart.Labels, module.Label{Key: k, Value: v})
176+
}
177+
}
178+
179+
if err := c.Charts().Add(*charts...); err != nil {
180+
c.Warningf("failed to add profile stats charts for %s: %v", name, err)
181+
}
182+
}
183+
69184
func (c *Collector) addProfileScalarMetricChart(m ddsnmp.Metric) {
70185
if m.Name == "" {
71186
return

‎src/go/plugin/go.d/collector/snmp/collect_snmp.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ package snmp
44

55
import (
66
"fmt"
7+
"path/filepath"
78
"sort"
89
"strings"
910

@@ -22,6 +23,7 @@ func (c *Collector) collectSNMP(mx map[string]int64) error {
2223

2324
c.collectProfileScalarMetrics(mx, pms)
2425
c.collectProfileTableMetrics(mx, pms)
26+
c.collectProfileStats(mx, pms)
2527

2628
return nil
2729
}
@@ -92,6 +94,38 @@ func (c *Collector) collectProfileTableMetrics(mx map[string]int64, pms []*ddsnm
9294
}
9395
}
9496

97+
func (c *Collector) collectProfileStats(mx map[string]int64, pms []*ddsnmp.ProfileMetrics) {
98+
for _, pm := range pms {
99+
name := stripFileNameExt(pm.Source)
100+
101+
if !c.seenProfiles[name] {
102+
c.seenProfiles[name] = true
103+
c.addProfileStatsCharts(name)
104+
}
105+
106+
px := fmt.Sprintf("snmp_device_prof_%s_stats_", name)
107+
mx[px+"timings_scalar"] = pm.Stats.Timing.Scalar.Milliseconds()
108+
mx[px+"timings_table"] = pm.Stats.Timing.Table.Milliseconds()
109+
mx[px+"timings_virtual"] = pm.Stats.Timing.VirtualMetrics.Milliseconds()
110+
mx[px+"snmp_get_requests"] = pm.Stats.SNMP.GetRequests
111+
mx[px+"snmp_get_oids"] = pm.Stats.SNMP.GetOIDs
112+
mx[px+"snmp_walk_pdus"] = pm.Stats.SNMP.WalkPDUs
113+
mx[px+"snmp_walk_requests"] = pm.Stats.SNMP.WalkRequests
114+
mx[px+"snmp_tables_walked"] = pm.Stats.SNMP.TablesWalked
115+
mx[px+"snmp_tables_cached"] = pm.Stats.SNMP.TablesCached
116+
mx[px+"metrics_scalar"] = pm.Stats.Metrics.Scalar
117+
mx[px+"metrics_table"] = pm.Stats.Metrics.Table
118+
mx[px+"metrics_virtual"] = pm.Stats.Metrics.Virtual
119+
mx[px+"metrics_tables"] = pm.Stats.Metrics.Tables
120+
mx[px+"metrics_rows"] = pm.Stats.Metrics.Rows
121+
mx[px+"table_cache_hits"] = pm.Stats.TableCache.Hits
122+
mx[px+"table_cache_misses"] = pm.Stats.TableCache.Misses
123+
mx[px+"errors_snmp"] = pm.Stats.Errors.SNMP
124+
mx[px+"errors_processing_scalar"] = pm.Stats.Errors.Processing.Scalar
125+
mx[px+"errors_processing_table"] = pm.Stats.Errors.Processing.Table
126+
}
127+
}
128+
95129
func tableMetricKey(m ddsnmp.Metric) string {
96130
if m.Name == "" {
97131
return ""
@@ -124,3 +158,7 @@ func tableMetricKey(m ddsnmp.Metric) string {
124158

125159
return sb.String()
126160
}
161+
162+
func stripFileNameExt(path string) string {
163+
return strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
164+
}

‎src/go/plugin/go.d/collector/snmp/collector.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ func New() *Collector {
6767
charts: &module.Charts{},
6868
seenScalarMetrics: make(map[string]bool),
6969
seenTableMetrics: make(map[string]bool),
70+
seenProfiles: make(map[string]bool),
7071

7172
newProber: ping.NewProber,
7273
newSnmpClient: gosnmp.NewHandler,
@@ -86,6 +87,7 @@ type (
8687
charts *module.Charts
8788
seenScalarMetrics map[string]bool
8889
seenTableMetrics map[string]bool
90+
seenProfiles map[string]bool
8991

9092
prober ping.Prober
9193
newProber func(ping.ProberConfig, *logger.Logger) ping.Prober

‎src/go/plugin/go.d/collector/snmp/collector_test.go‎

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,7 @@ func TestCollector_Collect(t *testing.T) {
226226
collr.newDdSnmpColl = func(ddsnmpcollector.Config) ddCollector {
227227
return &mockDdSnmpCollector{pms: []*ddsnmp.ProfileMetrics{
228228
{
229+
Source: "test",
229230
Metrics: []ddsnmp.Metric{
230231
{
231232
Name: "uptime",
@@ -243,7 +244,26 @@ func TestCollector_Collect(t *testing.T) {
243244
},
244245
want: map[string]int64{
245246
// scalar → "snmp_device_prof_<name>"
246-
"snmp_device_prof_uptime": 123,
247+
"snmp_device_prof_test_stats_errors_processing_scalar": 0,
248+
"snmp_device_prof_test_stats_errors_processing_table": 0,
249+
"snmp_device_prof_test_stats_errors_snmp": 0,
250+
"snmp_device_prof_test_stats_metrics_rows": 0,
251+
"snmp_device_prof_test_stats_metrics_scalar": 0,
252+
"snmp_device_prof_test_stats_metrics_table": 0,
253+
"snmp_device_prof_test_stats_metrics_tables": 0,
254+
"snmp_device_prof_test_stats_metrics_virtual": 0,
255+
"snmp_device_prof_test_stats_snmp_get_oids": 0,
256+
"snmp_device_prof_test_stats_snmp_get_requests": 0,
257+
"snmp_device_prof_test_stats_snmp_tables_cached": 0,
258+
"snmp_device_prof_test_stats_snmp_tables_walked": 0,
259+
"snmp_device_prof_test_stats_snmp_walk_pdus": 0,
260+
"snmp_device_prof_test_stats_snmp_walk_requests": 0,
261+
"snmp_device_prof_test_stats_table_cache_hits": 0,
262+
"snmp_device_prof_test_stats_table_cache_misses": 0,
263+
"snmp_device_prof_test_stats_timings_scalar": 0,
264+
"snmp_device_prof_test_stats_timings_table": 0,
265+
"snmp_device_prof_test_stats_timings_virtual": 0,
266+
"snmp_device_prof_uptime": 123,
247267
},
248268
},
249269
"collects table multivalue metric": {
@@ -260,6 +280,7 @@ func TestCollector_Collect(t *testing.T) {
260280
collr.newDdSnmpColl = func(ddsnmpcollector.Config) ddCollector {
261281
return &mockDdSnmpCollector{pms: []*ddsnmp.ProfileMetrics{
262282
{
283+
Source: "test",
263284
Metrics: []ddsnmp.Metric{
264285
{
265286
Name: "if_octets",
@@ -281,8 +302,27 @@ func TestCollector_Collect(t *testing.T) {
281302
want: map[string]int64{
282303
// table key: "snmp_device_prof_<name>_<sorted tag values>_<subkey>"
283304
// here tags = {"ifName":"eth0"} → key part becomes "_eth0"
284-
"snmp_device_prof_if_octets_eth0_in": 1,
285-
"snmp_device_prof_if_octets_eth0_out": 2,
305+
"snmp_device_prof_test_stats_errors_processing_scalar": 0,
306+
"snmp_device_prof_test_stats_errors_processing_table": 0,
307+
"snmp_device_prof_test_stats_errors_snmp": 0,
308+
"snmp_device_prof_test_stats_metrics_rows": 0,
309+
"snmp_device_prof_test_stats_metrics_scalar": 0,
310+
"snmp_device_prof_test_stats_metrics_table": 0,
311+
"snmp_device_prof_test_stats_metrics_tables": 0,
312+
"snmp_device_prof_test_stats_metrics_virtual": 0,
313+
"snmp_device_prof_test_stats_snmp_get_oids": 0,
314+
"snmp_device_prof_test_stats_snmp_get_requests": 0,
315+
"snmp_device_prof_test_stats_snmp_tables_cached": 0,
316+
"snmp_device_prof_test_stats_snmp_tables_walked": 0,
317+
"snmp_device_prof_test_stats_snmp_walk_pdus": 0,
318+
"snmp_device_prof_test_stats_snmp_walk_requests": 0,
319+
"snmp_device_prof_test_stats_table_cache_hits": 0,
320+
"snmp_device_prof_test_stats_table_cache_misses": 0,
321+
"snmp_device_prof_test_stats_timings_scalar": 0,
322+
"snmp_device_prof_test_stats_timings_table": 0,
323+
"snmp_device_prof_test_stats_timings_virtual": 0,
324+
"snmp_device_prof_if_octets_eth0_in": 1,
325+
"snmp_device_prof_if_octets_eth0_out": 2,
286326
},
287327
},
288328
}

‎src/go/plugin/go.d/collector/snmp/ddsnmp/ddsnmpcollector/collector.go‎

Lines changed: 33 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -63,18 +63,20 @@ type (
6363
vmetricsCollector *vmetricsCollector
6464
}
6565
profileState struct {
66-
profile *ddsnmp.Profile
67-
initialized bool
68-
globalTags map[string]string
69-
deviceMetadata map[string]ddsnmp.MetaTag
66+
profile *ddsnmp.Profile
67+
initialized bool
68+
cache struct {
69+
globalTags map[string]string
70+
deviceMetadata map[string]ddsnmp.MetaTag
71+
}
7072
}
7173
)
7274

7375
func (c *Collector) CollectDeviceMetadata() (map[string]ddsnmp.MetaTag, error) {
7476
meta := make(map[string]ddsnmp.MetaTag)
7577

7678
for _, prof := range c.profiles {
77-
profDeviceMeta, err := c.deviceMetadataCollector.Collect(prof.profile)
79+
profDeviceMeta, err := c.deviceMetadataCollector.collect(prof.profile)
7880
if err != nil {
7981
return nil, err
8082
}
@@ -91,7 +93,8 @@ func (c *Collector) Collect() ([]*ddsnmp.ProfileMetrics, error) {
9193
var metrics []*ddsnmp.ProfileMetrics
9294
var errs []error
9395

94-
if expired := c.tableCache.clearExpired(); len(expired) > 0 {
96+
expired := c.tableCache.clearExpired()
97+
if len(expired) > 0 {
9598
c.log.Debugf("Cleared %d expired table cache entries", len(expired))
9699
}
97100

@@ -106,13 +109,16 @@ func (c *Collector) Collect() ([]*ddsnmp.ProfileMetrics, error) {
106109

107110
metrics = append(metrics, pm)
108111

109-
if vmetrics := c.vmetricsCollector.Collect(prof.profile.Definition, pm.Metrics); len(vmetrics) > 0 {
112+
now := time.Now()
113+
if vmetrics := c.vmetricsCollector.collect(prof.profile.Definition, pm.Metrics); len(vmetrics) > 0 {
110114
for i := range vmetrics {
111115
vmetrics[i].Profile = pm
112116
}
113117

114118
pm.Metrics = slices.DeleteFunc(pm.Metrics, func(m ddsnmp.Metric) bool { return strings.HasPrefix(m.Name, "_") })
115119
pm.Metrics = append(pm.Metrics, vmetrics...)
120+
pm.Stats.Metrics.Virtual += int64(len(vmetrics))
121+
pm.Stats.Timing.VirtualMetrics = time.Since(now)
116122
}
117123
}
118124

@@ -142,42 +148,47 @@ func (c *Collector) SetSNMPClient(snmpClient gosnmp.Handler) {
142148
}
143149

144150
func (c *Collector) collectProfile(ps *profileState) (*ddsnmp.ProfileMetrics, error) {
151+
pm := &ddsnmp.ProfileMetrics{
152+
Source: ps.profile.SourceFile,
153+
}
154+
145155
if !ps.initialized {
146-
globalTag, err := c.globalTagsCollector.Collect(ps.profile)
156+
globalTag, err := c.globalTagsCollector.collect(ps.profile)
147157
if err != nil {
148158
return nil, fmt.Errorf("failed to collect global tags: %w", err)
149159
}
160+
ps.cache.globalTags = globalTag
150161

151-
deviceMeta, err := c.deviceMetadataCollector.Collect(ps.profile)
162+
deviceMeta, err := c.deviceMetadataCollector.collect(ps.profile)
152163
if err != nil {
153164
return nil, fmt.Errorf("failed to collect device metadata: %w", err)
154165
}
166+
ps.cache.deviceMetadata = deviceMeta
155167

156-
ps.globalTags = globalTag
157-
ps.deviceMetadata = deviceMeta
158168
ps.initialized = true
159169
}
160170

161-
var metrics []ddsnmp.Metric
171+
pm.Tags = maps.Clone(ps.cache.globalTags)
172+
pm.DeviceMetadata = maps.Clone(ps.cache.deviceMetadata)
162173

163-
scalarMetrics, err := c.scalarCollector.Collect(ps.profile)
174+
now := time.Now()
175+
scalarMetrics, err := c.scalarCollector.collect(ps.profile, &pm.Stats)
164176
if err != nil {
165177
return nil, err
166178
}
167-
metrics = append(metrics, scalarMetrics...)
179+
pm.Metrics = append(pm.Metrics, scalarMetrics...)
180+
pm.Stats.Timing.Scalar = time.Since(now)
181+
pm.Stats.Metrics.Scalar += int64(len(scalarMetrics))
168182

169-
tableMetrics, err := c.tableCollector.Collect(ps.profile)
183+
now = time.Now()
184+
tableMetrics, err := c.tableCollector.collect(ps.profile, &pm.Stats)
170185
if err != nil {
171186
return nil, err
172187
}
173-
metrics = append(metrics, tableMetrics...)
188+
pm.Metrics = append(pm.Metrics, tableMetrics...)
189+
pm.Stats.Timing.Table = time.Since(now)
190+
pm.Stats.Metrics.Table += int64(len(tableMetrics))
174191

175-
pm := &ddsnmp.ProfileMetrics{
176-
Source: ps.profile.SourceFile,
177-
DeviceMetadata: maps.Clone(ps.deviceMetadata),
178-
Tags: maps.Clone(ps.globalTags),
179-
Metrics: metrics,
180-
}
181192
for i := range pm.Metrics {
182193
pm.Metrics[i].Profile = pm
183194
}

0 commit comments

Comments
 (0)