Skip to content

Commit 6de80d7

Browse files
ptodevptodev
committed
Allow non-default registry to be used for metrics of SD components
Signed-off-by: Paulin Todev <[email protected]>
1 parent 5082655 commit 6de80d7

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

70 files changed

+1057
-694
lines changed

‎cmd/prometheus/main.go‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -620,14 +620,30 @@ func main() {
620620
discoveryManagerNotify discoveryManager
621621
)
622622

623+
// Register the metrics used by both "scrape" and "notify" discovery managers.
624+
// The same metrics are used for both discovery managers. Hence the registration
625+
// needs to be done here, outside the NewManager() calls, to avoid duplicate
626+
// metric registrations.
627+
discoveryMetrics, err := discovery.NewMetrics(prometheus.DefaultRegisterer)
628+
if err != nil {
629+
level.Error(logger).Log("msg", "failed to create discovery metrics", "err", err)
630+
os.Exit(1)
631+
}
623632
if cfg.enableNewSDManager {
624-
discovery.RegisterMetrics()
625-
discoveryManagerScrape = discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), discovery.Name("scrape"))
626-
discoveryManagerNotify = discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), discovery.Name("notify"))
633+
discoveryManagerScrape = discovery.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, discoveryMetrics, discovery.Name("scrape"))
634+
discoveryManagerNotify = discovery.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, discoveryMetrics, discovery.Name("notify"))
627635
} else {
628-
legacymanager.RegisterMetrics()
629-
discoveryManagerScrape = legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), legacymanager.Name("scrape"))
630-
discoveryManagerNotify = legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), legacymanager.Name("notify"))
636+
discoveryManagerScrape = legacymanager.NewManager(ctxScrape, log.With(logger, "component", "discovery manager scrape"), prometheus.DefaultRegisterer, discoveryMetrics, legacymanager.Name("scrape"))
637+
discoveryManagerNotify = legacymanager.NewManager(ctxNotify, log.With(logger, "component", "discovery manager notify"), prometheus.DefaultRegisterer, discoveryMetrics, legacymanager.Name("notify"))
638+
}
639+
640+
if discoveryManagerScrape == nil {
641+
level.Error(logger).Log("msg", "failed to create a discovery manager scrape")
642+
os.Exit(1)
643+
}
644+
if discoveryManagerNotify == nil {
645+
level.Error(logger).Log("msg", "failed to create a discovery manager notify")
646+
os.Exit(1)
631647
}
632648

633649
scrapeManager, err := scrape.NewManager(

‎cmd/promtool/main.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import (
3636
"github.com/google/pprof/profile"
3737
"github.com/prometheus/client_golang/api"
3838
v1 "github.com/prometheus/client_golang/api/prometheus/v1"
39+
"github.com/prometheus/client_golang/prometheus"
3940
"github.com/prometheus/client_golang/prometheus/promhttp"
4041
"github.com/prometheus/client_golang/prometheus/testutil/promlint"
4142
config_util "github.com/prometheus/common/config"
@@ -317,7 +318,7 @@ func main() {
317318

318319
switch parsedCmd {
319320
case sdCheckCmd.FullCommand():
320-
os.Exit(CheckSD(*sdConfigFile, *sdJobName, *sdTimeout, noDefaultScrapePort))
321+
os.Exit(CheckSD(*sdConfigFile, *sdJobName, *sdTimeout, noDefaultScrapePort, prometheus.DefaultRegisterer))
321322

322323
case checkConfigCmd.FullCommand():
323324
os.Exit(CheckConfig(*agentMode, *checkConfigSyntaxOnly, newLintConfig(*checkConfigLint, *checkConfigLintFatal), *configFiles...))

‎cmd/promtool/sd.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"time"
2323

2424
"github.com/go-kit/log"
25+
"github.com/prometheus/client_golang/prometheus"
2526

2627
"github.com/prometheus/prometheus/config"
2728
"github.com/prometheus/prometheus/discovery"
@@ -37,7 +38,7 @@ type sdCheckResult struct {
3738
}
3839

3940
// CheckSD performs service discovery for the given job name and reports the results.
40-
func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefaultScrapePort bool) int {
41+
func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefaultScrapePort bool, registerer prometheus.Registerer) int {
4142
logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
4243

4344
cfg, err := config.LoadFile(sdConfigFiles, false, false, logger)
@@ -77,7 +78,7 @@ func CheckSD(sdConfigFiles, sdJobName string, sdTimeout time.Duration, noDefault
7778
defer cancel()
7879

7980
for _, cfg := range scrapeConfig.ServiceDiscoveryConfigs {
80-
d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger})
81+
d, err := cfg.NewDiscoverer(discovery.DiscovererOptions{Logger: logger, Registerer: registerer})
8182
if err != nil {
8283
fmt.Fprintln(os.Stderr, "Could not create new discoverer", err)
8384
return failureExitCode

‎discovery/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,11 @@ type Config interface {
234234

235235
type DiscovererOptions struct {
236236
Logger log.Logger
237+
238+
// A registerer for the Discoverer's metrics.
239+
Registerer prometheus.Registerer
240+
241+
HTTPClientOptions []config.HTTPClientOption
237242
}
238243
```
239244

‎discovery/aws/ec2.go‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"github.com/aws/aws-sdk-go/service/ec2"
3131
"github.com/go-kit/log"
3232
"github.com/go-kit/log/level"
33+
"github.com/prometheus/client_golang/prometheus"
3334
"github.com/prometheus/common/config"
3435
"github.com/prometheus/common/model"
3536

@@ -101,7 +102,7 @@ func (*EC2SDConfig) Name() string { return "ec2" }
101102

102103
// NewDiscoverer returns a Discoverer for the EC2 Config.
103104
func (c *EC2SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
104-
return NewEC2Discovery(c, opts.Logger), nil
105+
return NewEC2Discovery(c, opts.Logger, opts.Registerer), nil
105106
}
106107

107108
// UnmarshalYAML implements the yaml.Unmarshaler interface for the EC2 Config.
@@ -147,7 +148,7 @@ type EC2Discovery struct {
147148
}
148149

149150
// NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets.
150-
func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger) *EC2Discovery {
151+
func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger, reg prometheus.Registerer) *EC2Discovery {
151152
if logger == nil {
152153
logger = log.NewNopLogger()
153154
}
@@ -156,10 +157,13 @@ func NewEC2Discovery(conf *EC2SDConfig, logger log.Logger) *EC2Discovery {
156157
cfg: conf,
157158
}
158159
d.Discovery = refresh.NewDiscovery(
159-
logger,
160-
"ec2",
161-
time.Duration(d.cfg.RefreshInterval),
162-
d.refresh,
160+
refresh.Options{
161+
Logger: logger,
162+
Mech: "ec2",
163+
Interval: time.Duration(d.cfg.RefreshInterval),
164+
RefreshF: d.refresh,
165+
Registry: reg,
166+
},
163167
)
164168
return d
165169
}

‎discovery/aws/lightsail.go‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/aws/aws-sdk-go/aws/session"
3030
"github.com/aws/aws-sdk-go/service/lightsail"
3131
"github.com/go-kit/log"
32+
"github.com/prometheus/client_golang/prometheus"
3233
"github.com/prometheus/common/config"
3334
"github.com/prometheus/common/model"
3435

@@ -84,7 +85,7 @@ func (*LightsailSDConfig) Name() string { return "lightsail" }
8485

8586
// NewDiscoverer returns a Discoverer for the Lightsail Config.
8687
func (c *LightsailSDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
87-
return NewLightsailDiscovery(c, opts.Logger), nil
88+
return NewLightsailDiscovery(c, opts.Logger, opts.Registerer), nil
8889
}
8990

9091
// UnmarshalYAML implements the yaml.Unmarshaler interface for the Lightsail Config.
@@ -121,18 +122,21 @@ type LightsailDiscovery struct {
121122
}
122123

123124
// NewLightsailDiscovery returns a new LightsailDiscovery which periodically refreshes its targets.
124-
func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger) *LightsailDiscovery {
125+
func NewLightsailDiscovery(conf *LightsailSDConfig, logger log.Logger, reg prometheus.Registerer) *LightsailDiscovery {
125126
if logger == nil {
126127
logger = log.NewNopLogger()
127128
}
128129
d := &LightsailDiscovery{
129130
cfg: conf,
130131
}
131132
d.Discovery = refresh.NewDiscovery(
132-
logger,
133-
"lightsail",
134-
time.Duration(d.cfg.RefreshInterval),
135-
d.refresh,
133+
refresh.Options{
134+
Logger: logger,
135+
Mech: "lightsail",
136+
Interval: time.Duration(d.cfg.RefreshInterval),
137+
RefreshF: d.refresh,
138+
Registry: reg,
139+
},
136140
)
137141
return d
138142
}

‎discovery/azure/azure.go‎

Lines changed: 33 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -79,17 +79,6 @@ var (
7979
AuthenticationMethod: authMethodOAuth,
8080
HTTPClientConfig: config_util.DefaultHTTPClientConfig,
8181
}
82-
83-
failuresCount = prometheus.NewCounter(
84-
prometheus.CounterOpts{
85-
Name: "prometheus_sd_azure_failures_total",
86-
Help: "Number of Azure service discovery refresh failures.",
87-
})
88-
cacheHitCount = prometheus.NewCounter(
89-
prometheus.CounterOpts{
90-
Name: "prometheus_sd_azure_cache_hit_total",
91-
Help: "Number of cache hit during refresh.",
92-
})
9382
)
9483

9584
var environments = map[string]cloud.Configuration{
@@ -114,8 +103,6 @@ func CloudConfigurationFromName(name string) (cloud.Configuration, error) {
114103

115104
func init() {
116105
discovery.RegisterConfig(&SDConfig{})
117-
prometheus.MustRegister(failuresCount)
118-
prometheus.MustRegister(cacheHitCount)
119106
}
120107

121108
// SDConfig is the configuration for Azure based service discovery.
@@ -138,7 +125,7 @@ func (*SDConfig) Name() string { return "azure" }
138125

139126
// NewDiscoverer returns a Discoverer for the Config.
140127
func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
141-
return NewDiscovery(c, opts.Logger), nil
128+
return NewDiscovery(c, opts.Logger, opts.Registerer)
142129
}
143130

144131
func validateAuthParam(param, name string) error {
@@ -181,14 +168,16 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
181168

182169
type Discovery struct {
183170
*refresh.Discovery
184-
logger log.Logger
185-
cfg *SDConfig
186-
port int
187-
cache *cache.Cache[string, *armnetwork.Interface]
171+
logger log.Logger
172+
cfg *SDConfig
173+
port int
174+
cache *cache.Cache[string, *armnetwork.Interface]
175+
failuresCount prometheus.Counter
176+
cacheHitCount prometheus.Counter
188177
}
189178

190179
// NewDiscovery returns a new AzureDiscovery which periodically refreshes its targets.
191-
func NewDiscovery(cfg *SDConfig, logger log.Logger) *Discovery {
180+
func NewDiscovery(cfg *SDConfig, logger log.Logger, reg prometheus.Registerer) (*Discovery, error) {
192181
if logger == nil {
193182
logger = log.NewNopLogger()
194183
}
@@ -198,16 +187,30 @@ func NewDiscovery(cfg *SDConfig, logger log.Logger) *Discovery {
198187
port: cfg.Port,
199188
logger: logger,
200189
cache: l,
190+
failuresCount: prometheus.NewCounter(
191+
prometheus.CounterOpts{
192+
Name: "prometheus_sd_azure_failures_total",
193+
Help: "Number of Azure service discovery refresh failures.",
194+
}),
195+
cacheHitCount: prometheus.NewCounter(
196+
prometheus.CounterOpts{
197+
Name: "prometheus_sd_azure_cache_hit_total",
198+
Help: "Number of cache hit during refresh.",
199+
}),
201200
}
202201

203202
d.Discovery = refresh.NewDiscovery(
204-
logger,
205-
"azure",
206-
time.Duration(cfg.RefreshInterval),
207-
d.refresh,
203+
refresh.Options{
204+
Logger: logger,
205+
Mech: "azure",
206+
Interval: time.Duration(cfg.RefreshInterval),
207+
RefreshF: d.refresh,
208+
Registry: reg,
209+
Metrics: []prometheus.Collector{d.failuresCount, d.cacheHitCount},
210+
},
208211
)
209212

210-
return d
213+
return d, nil
211214
}
212215

213216
// azureClient represents multiple Azure Resource Manager providers.
@@ -329,14 +332,14 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
329332

330333
client, err := createAzureClient(*d.cfg)
331334
if err != nil {
332-
failuresCount.Inc()
335+
d.failuresCount.Inc()
333336
return nil, fmt.Errorf("could not create Azure client: %w", err)
334337
}
335338
client.logger = d.logger
336339

337340
machines, err := client.getVMs(ctx, d.cfg.ResourceGroup)
338341
if err != nil {
339-
failuresCount.Inc()
342+
d.failuresCount.Inc()
340343
return nil, fmt.Errorf("could not get virtual machines: %w", err)
341344
}
342345

@@ -345,14 +348,14 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
345348
// Load the vms managed by scale sets.
346349
scaleSets, err := client.getScaleSets(ctx, d.cfg.ResourceGroup)
347350
if err != nil {
348-
failuresCount.Inc()
351+
d.failuresCount.Inc()
349352
return nil, fmt.Errorf("could not get virtual machine scale sets: %w", err)
350353
}
351354

352355
for _, scaleSet := range scaleSets {
353356
scaleSetVms, err := client.getScaleSetVMs(ctx, scaleSet)
354357
if err != nil {
355-
failuresCount.Inc()
358+
d.failuresCount.Inc()
356359
return nil, fmt.Errorf("could not get virtual machine scale set vms: %w", err)
357360
}
358361
machines = append(machines, scaleSetVms...)
@@ -403,7 +406,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
403406
var networkInterface *armnetwork.Interface
404407
if v, ok := d.getFromCache(nicID); ok {
405408
networkInterface = v
406-
cacheHitCount.Add(1)
409+
d.cacheHitCount.Add(1)
407410
} else {
408411
networkInterface, err = client.getNetworkInterfaceByID(ctx, nicID)
409412
if err != nil {
@@ -462,7 +465,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
462465
var tg targetgroup.Group
463466
for tgt := range ch {
464467
if tgt.err != nil {
465-
failuresCount.Inc()
468+
d.failuresCount.Inc()
466469
return nil, fmt.Errorf("unable to complete Azure service discovery: %w", tgt.err)
467470
}
468471
if tgt.labelSet != nil {

0 commit comments

Comments
 (0)