Skip to content

Commit 0f7ebfa

Browse files
authored
[bug fix] Export atenet-egress traces and metrics with service.name=atenet-egress (agent-substrate#3417)
`atenet-egress` runs `cmd/atenet router --mode=egress`, which previously initialized OpenTelemetry with the hardcoded service name `"atenet-router"`. - Set OTel `service.name` in `RouterServer.Run` based on `--mode` (`"atenet-egress"` for egress, `"atenet-router"` for ingress) and pass the meter to `extproc.NewRouteDurationHistogram`. - Remove the unused trace span in `handleStatusz`. - Include `atenet-egress` in `ate-setup`'s `otelOverrideDeployments` so `--otlp-endpoint` restarts it too. - Update the metrics registry/docs and E2E metrics test to verify `atenet-egress` telemetry is exported under `atenet-egress` and not `atenet-router`. Fixes agent-substrate#3308 <!-- e.g. "AI-assisted, I reviewed and tested it." No AI co-author trailers. --> AI use: <!-- Breaking change? Describe what users must do on upgrade and ask for the `breaking-change` label. --> - [ ] Tests pass - [ ] Docs updated Signed-off-by: Bo-Cheng Chu <[email protected]>
1 parent 74c6ab3 commit 0f7ebfa

14 files changed

Lines changed: 78 additions & 42 deletions

File tree

‎benchmarking/telemetry/README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,9 @@ go run ./cmd/ate-setup deploy ate-system \
4646

4747
`--otlp-endpoint` patches the `ate-otel-config` ConfigMap and restarts the
4848
workloads that read it. One patch is sufficient for `ateapi`, `ate-controller`,
49-
`atelet`, and `atenet-router`, because each one reads the ConfigMap through
50-
`envFrom`. `ate-controller` also copies the values to the worker pods
51-
that it creates.
49+
`atelet`, `atenet-router`, and `atenet-egress`, because each one reads the
50+
ConfigMap through `envFrom`. `ate-controller` also copies the values to the
51+
worker pods that it creates.
5252

5353
The actor containers are different. Substrate puts no OTLP configuration in
5454
them, thus they use the `env` of the ActorTemplate:

‎cmd/ate-setup/internal/steps/overlay.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -612,7 +612,7 @@ const otelEndpointKey = "OTEL_EXPORTER_OTLP_ENDPOINT"
612612
// otelOverrideDeployments are the control plane Deployments that read
613613
// ate-otel-config. ate-controller additionally copies the values onto the
614614
// worker pods it creates, so one patch reaches the whole system.
615-
var otelOverrideDeployments = []string{"ate-api-server", "ate-controller", "atenet-router"}
615+
var otelOverrideDeployments = []string{"ate-api-server", "ate-controller", "atenet-router", "atenet-egress"}
616616

617617
// applyOtelEndpointOverride points all control plane telemetry at a different
618618
// collector for the duration of a measurement. See

‎cmd/ate-setup/internal/steps/overlay_test.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1427,9 +1427,15 @@ func TestApplyOtelEndpointOverride(t *testing.T) {
14271427
Name: "atelet",
14281428
},
14291429
}
1430+
egress := &appsv1.Deployment{
1431+
ObjectMeta: metav1.ObjectMeta{
1432+
Namespace: NamespaceAteSystem,
1433+
Name: "atenet-egress",
1434+
},
1435+
}
14301436
e := &Env{
14311437
Cfg: &config.Config{OtlpEndpoint: endpoint},
1432-
Kube: fakeKube(t, otelConfig("http://default:4317"), apiServerDeployment(), atelet),
1438+
Kube: fakeKube(t, otelConfig("http://default:4317"), apiServerDeployment(), egress, atelet),
14331439
}
14341440

14351441
if err := e.applyOtelEndpointOverride(t.Context()); err != nil {
@@ -1445,6 +1451,9 @@ func TestApplyOtelEndpointOverride(t *testing.T) {
14451451
if !restartedAt(t, e, "deployment", "ate-api-server") {
14461452
t.Error("ate-api-server was not restarted")
14471453
}
1454+
if !restartedAt(t, e, "deployment", "atenet-egress") {
1455+
t.Error("atenet-egress was not restarted")
1456+
}
14481457
if !restartedAt(t, e, "daemonset", "atelet") {
14491458
t.Error("the atelet DaemonSet was not restarted")
14501459
}

‎cmd/atenet/internal/router/extproc/metrics.go‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,28 +21,20 @@ import (
2121
"time"
2222

2323
envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3"
24-
"go.opentelemetry.io/otel"
2524
"go.opentelemetry.io/otel/metric"
2625
"google.golang.org/grpc/codes"
2726
"google.golang.org/grpc/status"
2827

2928
"go.ate.dev/substrate/internal/ateattr"
3029
)
3130

32-
// ServiceName is the OpenTelemetry service name and instrumentation scope
33-
// shared by every part of the atenet router process — the ext_proc mux here,
34-
// the direction handlers, and the router's own servers. It lives in this
35-
// package because it is the one package all of them already depend on.
36-
const ServiceName = "atenet-router"
37-
3831
// atenet.router.route.duration measures the latency from when the ext_proc handler receives a request
3932
// (dataplane -> EPP) until the target worker endpoint is resolved
4033
const routeDurationMetricName = "atenet.router.route.duration"
4134

42-
// NewRouteDurationHistogram creates the atenet.router.route.duration histogram from
43-
// the global MeterProvider.
44-
func NewRouteDurationHistogram() (metric.Float64Histogram, error) {
45-
h, err := otel.Meter(ServiceName).Float64Histogram(
35+
// NewRouteDurationHistogram creates the atenet.router.route.duration histogram on meter.
36+
func NewRouteDurationHistogram(meter metric.Meter) (metric.Float64Histogram, error) {
37+
h, err := meter.Float64Histogram(
4638
routeDurationMetricName,
4739
metric.WithUnit("s"),
4840
metric.WithDescription(

‎cmd/atenet/internal/router/extproc/metrics_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ func TestClassifyOutcome(t *testing.T) {
145145
func TestRecordRouteDuration_Attributes(t *testing.T) {
146146
reader := sdkmetric.NewManualReader()
147147
mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))
148-
h, err := mp.Meter("atenet-router").Float64Histogram(routeDurationMetricName)
148+
h, err := NewRouteDurationHistogram(mp.Meter("atenet-router"))
149149
if err != nil {
150150
t.Fatalf("failed to create histogram: %v", err)
151151
}
@@ -179,7 +179,7 @@ func TestRecordRouteDuration_Attributes(t *testing.T) {
179179
func TestRecordRouteDuration_NormalizesEmptyTemplateDimensions(t *testing.T) {
180180
reader := sdkmetric.NewManualReader()
181181
mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))
182-
h, err := mp.Meter("atenet-router").Float64Histogram(routeDurationMetricName)
182+
h, err := NewRouteDurationHistogram(mp.Meter("atenet-router"))
183183
if err != nil {
184184
t.Fatalf("failed to create histogram: %v", err)
185185
}

‎cmd/atenet/internal/router/ingress/ingress.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM
8585
// HTTP headers carried inside the ProcessingRequest payload. Extract
8686
// from there so our span links to the gateway's ingress span.
8787
ctx = otel.GetTextMapPropagator().Extract(ctx, md)
88-
ctx, span := otel.Tracer(extproc.ServiceName).Start(ctx, "ExtProc.RequestHeaders")
88+
ctx, span := otel.Tracer(serviceName).Start(ctx, "ExtProc.RequestHeaders")
8989
defer span.End()
9090

9191
actorRef, err := atenet.ParseTargetActor(routingValue(md, atenet.TargetActorHeader, extproc.TargetActorFilterStateAttribute))

‎cmd/atenet/internal/router/ingress/metrics.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,12 @@ import (
2323
"go.opentelemetry.io/otel/attribute"
2424
"go.opentelemetry.io/otel/metric"
2525

26-
"go.ate.dev/substrate/cmd/atenet/internal/router/extproc"
2726
"go.ate.dev/substrate/internal/ateattr"
2827
)
2928

29+
// serviceName is the OpenTelemetry instrumentation scope of the ingress handler.
30+
const serviceName = "atenet-router"
31+
3032
// Request-parking instruments. parking.active is the live count of parked
3133
// requests; parking.wait.duration is how long each request stayed parked
3234
// (labeled by outcome); parking.rejected counts requests shed because the
@@ -49,7 +51,7 @@ type ParkingMetrics struct {
4951
// NewParkingMetrics creates the request-parking instruments from the global
5052
// MeterProvider.
5153
func NewParkingMetrics() (*ParkingMetrics, error) {
52-
meter := otel.Meter(extproc.ServiceName)
54+
meter := otel.Meter(serviceName)
5355

5456
active, err := meter.Int64UpDownCounter(
5557
parkingActiveMetricName,

‎cmd/atenet/internal/router/ingress/resumer.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import (
2222
"time"
2323

2424
"go.ate.dev/substrate/api/ateapi/v1"
25-
"go.ate.dev/substrate/cmd/atenet/internal/router/extproc"
2625
"go.ate.dev/substrate/internal/ateattr"
2726
"go.ate.dev/substrate/internal/resources"
2827
"go.opentelemetry.io/otel"
@@ -182,7 +181,7 @@ func (r *ActorResumer) retryable(err error) bool {
182181
// ResumeActor ensures the actor is running.
183182
// This method will block until the actor is resumed or error.
184183
func (r *ActorResumer) ResumeActor(ctx context.Context, actorRef resources.ActorRef) (*ateapi.Actor, ResumeOutcome, error) {
185-
ctx, span := otel.Tracer(extproc.ServiceName).Start(ctx, "ResumeActor",
184+
ctx, span := otel.Tracer(serviceName).Start(ctx, "ResumeActor",
186185
trace.WithAttributes(ateattr.ActorRefAttributes(actorRef)...))
187186
defer span.End()
188187

‎cmd/atenet/internal/router/router.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/spf13/cobra"
3030
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
3131
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
32+
"go.opentelemetry.io/otel"
3233
"golang.org/x/sync/errgroup"
3334
"google.golang.org/grpc"
3435
"k8s.io/client-go/kubernetes"
@@ -139,17 +140,21 @@ func (s *RouterServer) Run(ctx context.Context) error {
139140
// below, because otelgrpc.NewClientHandler captures the global
140141
// TracerProvider at construction time. Resolved once so the router's SDK
141142
// sampler and Envoy's RandomSampling percent cannot drift.
143+
serviceName := "atenet-router"
144+
if s.cfg.Mode == ModeEgress {
145+
serviceName = "atenet-egress"
146+
}
142147
sampling := serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(dataPlaneTraceRatio))
143148
tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{
144-
ServiceName: extproc.ServiceName,
149+
ServiceName: serviceName,
145150
Sampling: sampling,
146151
})
147152
if err != nil {
148153
return fmt.Errorf("failed to initialize tracing: %w", err)
149154
}
150155
defer serverboot.ShutdownProvider("TracerProvider", tp.Shutdown)
151156

152-
mp, err := serverboot.InitMetrics(ctx, extproc.ServiceName)
157+
mp, err := serverboot.InitMetrics(ctx, serviceName)
153158
if err != nil {
154159
return fmt.Errorf("failed to initialize metrics: %w", err)
155160
}
@@ -252,7 +257,7 @@ func (s *RouterServer) Run(ctx context.Context) error {
252257
}
253258

254259
if s.extprocSrv == nil {
255-
routeDuration, err := extproc.NewRouteDurationHistogram()
260+
routeDuration, err := extproc.NewRouteDurationHistogram(otel.Meter(serviceName))
256261
if err != nil {
257262
return fmt.Errorf("failed to create route-duration histogram: %w", err)
258263
}

‎cmd/atenet/internal/router/status.go‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ import (
2929
"time"
3030

3131
"github.com/spf13/pflag"
32-
"go.opentelemetry.io/otel"
3332
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3433

3534
"go.ate.dev/substrate/cmd/atenet/internal/router/extproc"
@@ -94,10 +93,7 @@ func (s *RouterServer) getRouterIP(ctx context.Context) string {
9493
}
9594

9695
func (s *RouterServer) handleStatusz(w http.ResponseWriter, req *http.Request) {
97-
ctx, span := otel.Tracer(extproc.ServiceName).Start(req.Context(), "handleStatusz")
98-
defer span.End()
99-
100-
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
96+
ctx, cancel := context.WithTimeout(req.Context(), 3*time.Second)
10197
defer cancel()
10298

10399
routerIP := s.getRouterIP(ctx)

0 commit comments

Comments
 (0)