Skip to content

Commit a46d59d

Browse files
author
Eric Curtin
committed
atecontroller: address review on the :8080 listener
Name the table rows, make :8080 a const, log the resolved exporters, and mark the /metrics note temporary (#2067).
1 parent a9ea0d7 commit a46d59d

5 files changed

Lines changed: 45 additions & 23 deletions

File tree

‎cmd/atecontroller/main.go‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,14 @@ func init() {
9191
utilruntime.Must(clientv1alpha1.AddToScheme(scheme)) // Register our CRD
9292
}
9393

94-
const serviceName = "atecontroller"
94+
const (
95+
serviceName = "atecontroller"
96+
97+
// metricsPullAddr is the manager's metrics listener, and metricsOffAddr
98+
// turns it off.
99+
metricsPullAddr = ":8080"
100+
metricsOffAddr = "0"
101+
)
95102

96103
// logr verbosity V(n) maps to slog level -n, so V(1) stays below Info until
97104
// --log-level=debug. logr carries no context, so these records have no trace IDs.
@@ -102,9 +109,9 @@ func newControllerRuntimeLogger(h slog.Handler) logr.Logger {
102109
// managerOptions configures the controller-runtime manager. servePull is
103110
// whether OTEL_METRICS_EXPORTER leaves its metrics listener on.
104111
func managerOptions(egressMITMCAPool types.NamespacedName, servePull bool) ctrl.Options {
105-
metricsAddr := "0" // "0" disables the server.
112+
metricsAddr := metricsOffAddr
106113
if servePull {
107-
metricsAddr = metricsserver.DefaultBindAddress
114+
metricsAddr = metricsPullAddr
108115
}
109116
return ctrl.Options{
110117
Scheme: scheme,
@@ -158,6 +165,7 @@ func main() {
158165
serverboot.Fatal(ctx, "Failed to initialize metrics", err)
159166
}
160167
defer serverboot.ShutdownProvider("MeterProvider", mp.Shutdown)
168+
slog.InfoContext(ctx, "Metrics listener", slog.Bool("enabled", servePull), slog.String("addr", metricsPullAddr))
161169

162170
k8sConfig := ctrl.GetConfigOrDie()
163171
k8sClient, err := kubernetes.NewForConfig(k8sConfig)

‎cmd/atecontroller/main_test.go‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -88,19 +88,31 @@ func TestBridgedRegistryProducesBeforeManagerStart(t *testing.T) {
8888
func TestManagerOptionsMetricsListener(t *testing.T) {
8989
t.Parallel()
9090

91-
for _, servePull := range []bool{true, false} {
92-
opts := managerOptions(types.NamespacedName{Namespace: "ate-system", Name: "pool"}, servePull)
91+
tests := []struct {
92+
name string
93+
servePull bool
94+
wantAddr string
95+
wantServed bool
96+
}{
97+
{"serving", true, metricsPullAddr, true},
98+
{"disabled", false, metricsOffAddr, false},
99+
}
100+
for _, tt := range tests {
101+
t.Run(tt.name, func(t *testing.T) {
102+
t.Parallel()
103+
opts := managerOptions(types.NamespacedName{Namespace: "ate-system", Name: "pool"}, tt.servePull)
93104

94-
srv, err := metricsserver.NewServer(opts.Metrics, nil, nil)
95-
if err != nil {
96-
t.Fatalf("servePull=%t: NewServer: %v", servePull, err)
97-
}
98-
if served := srv != nil; served != servePull {
99-
t.Errorf("servePull=%t: the manager serves a metrics listener = %t", servePull, served)
100-
}
101-
if servePull && opts.Metrics.BindAddress != ":8080" {
102-
t.Errorf("metrics address = %q, want :8080", opts.Metrics.BindAddress)
103-
}
105+
if opts.Metrics.BindAddress != tt.wantAddr {
106+
t.Errorf("metrics address = %q, want %q", opts.Metrics.BindAddress, tt.wantAddr)
107+
}
108+
srv, err := metricsserver.NewServer(opts.Metrics, nil, nil)
109+
if err != nil {
110+
t.Fatalf("NewServer: %v", err)
111+
}
112+
if served := srv != nil; served != tt.wantServed {
113+
t.Errorf("serves a metrics listener = %t, want %t", served, tt.wantServed)
114+
}
115+
})
104116
}
105117
}
106118

‎docs/observability.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -333,9 +333,9 @@ A queue that has never processed an item bridges as an exponential histogram wit
333333

334334
### Scraping instead of pushing
335335

336-
ateapi, atelet, atenet-router and the credential provider also serve every instrument on their Prometheus `/metrics` endpoint. A cluster that scrapes those endpoints sets `OTEL_METRICS_EXPORTER=prometheus` on the components, so each series reaches the backend once. With `prometheus` the components install no OTLP metric reader and keep the Prometheus one; traces and logs are unaffected. `none` stops the push too, and leaves their `/metrics` on. The variable takes a comma-separated list, with the same rules as `OTEL_LOGS_EXPORTER`. Unset is `otlp,prometheus`, or `otlp` alone on ateom. `otlp` keeps the push, and `prometheus` or `none` stops it. ateom serves no `/metrics`, so there `prometheus` is skipped as unknown with a warning, and the push stays on unless the variable is `none`.
336+
ateapi, atelet, atenet-router and the credential provider also serve every instrument on their Prometheus `/metrics` endpoint. A cluster that scrapes those endpoints sets `OTEL_METRICS_EXPORTER=prometheus` on the components, so each series reaches the backend once. With `prometheus` the components install no OTLP metric reader and keep the Prometheus one; traces and logs are unaffected. `none` stops the push too. For now `/metrics` stays on for every value, including `otlp` and `none`. This is temporary until the probes move off `:9090`; `otlp` and `none` will then turn it off ([#2067](https://github.com/agent-substrate/substrate/issues/2067)). The variable takes a comma-separated list, with the same rules as `OTEL_LOGS_EXPORTER`. Unset is `otlp,prometheus`, or `otlp` alone on ateom. `otlp` keeps the push, and `prometheus` or `none` stops it. ateom serves no `/metrics`, so there `prometheus` is skipped as unknown with a warning, and the push stays on unless the variable is `none`.
337337

338-
atecontroller's `:8080` listener follows the variable. When the variable is unset or `otlp,prometheus`, atecontroller pushes every metric over OTLP, and `:8080` serves only the controller-runtime families. With `otlp`, atecontroller pushes every metric and closes `:8080`. With `prometheus`, atecontroller does not push, and `:8080` serves the controller-runtime families and the `ate.workerpool.*` instruments. With `none`, atecontroller does not push and closes `:8080`.
338+
atecontroller's `:8080` listener follows the variable. When the variable is unset or `otlp,prometheus`, atecontroller pushes every metric over OTLP, and `:8080` serves only the controller-runtime families ([#2402](https://github.com/agent-substrate/substrate/issues/2402)). With `otlp`, atecontroller pushes every metric and closes `:8080`. With `prometheus`, atecontroller does not push, and `:8080` serves the controller-runtime families and the `ate.workerpool.*` instruments. With `none`, atecontroller does not push and closes `:8080`.
339339

340340
### Local Metrics with Prometheus (Kind Cluster)
341341

‎internal/serverboot/serverboot.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,7 @@ func InitMetricsBridged(ctx context.Context, serviceName string, reg interface {
273273
return nil, false, fmt.Errorf("serviceName is required")
274274
}
275275
exporters := metricsExporters(ctx, true)
276+
slog.InfoContext(ctx, "Metrics exporters resolved", slog.String("exporters", exporters.String()))
276277
servePull = exporters.Has(exporterPrometheus)
277278
switch {
278279
case exporters.Has(ExporterOTLP):

‎internal/serverboot/serverboot_test.go‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -535,18 +535,19 @@ func TestInitMetricsBridgedExporterNoneExportsNothing(t *testing.T) {
535535
// servePull follows the policy table: unset and the list keep both paths.
536536
func TestInitMetricsBridgedServePull(t *testing.T) {
537537
tests := []struct {
538+
name string
538539
value string
539540
wantPush bool
540541
wantPull bool
541542
}{
542-
{"", true, true},
543-
{"otlp", true, false},
544-
{"prometheus", false, true},
545-
{"otlp,prometheus", true, true},
546-
{"none", false, false},
543+
{"unset", "", true, true},
544+
{"otlp", "otlp", true, false},
545+
{"prometheus", "prometheus", false, true},
546+
{"list", "otlp,prometheus", true, true},
547+
{"none", "none", false, false},
547548
}
548549
for _, tt := range tests {
549-
t.Run(tt.value, func(t *testing.T) {
550+
t.Run(tt.name, func(t *testing.T) {
550551
collector := startMetricsCollector(t, tt.value)
551552
mp, servePull, err := InitMetricsBridged(t.Context(), "test-bridged-table", bridgedRegistry(t), nil)
552553
if err != nil {

0 commit comments

Comments
 (0)