Skip to content

Commit 3597ac8

Browse files
alakeshalakesh
committed
[otel-tracing] Initial opentelemetry support
Add basic intiialization of opentelemetry including minimum support to be able to read open telemetry config from config.toml and initialize exporter. Tracer is initialized and ready to be be used for creating spans, sub spans etc. With no opentelemetry configuration enabled in config file, this patch is a no-op. Basic config stub to be added to use opentelemetry is to add following in config.toml. We use otlp exporter with default port 4317. [otel] exporter_name = "otlp" exporter_endpoint = "0.0.0.1:4317" otel-collector binary needs to run listening at the same port. Signed-off-by: Alakesh Haloi <[email protected]>
1 parent 10824ea commit 3597ac8

4 files changed

Lines changed: 166 additions & 2 deletions

File tree

‎cmd/containerd/command/main.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,12 @@ import (
3434
"github.com/containerd/containerd/services/server"
3535
srvconfig "github.com/containerd/containerd/services/server/config"
3636
"github.com/containerd/containerd/sys"
37+
"github.com/containerd/containerd/tracing"
3738
"github.com/containerd/containerd/version"
3839
"github.com/pkg/errors"
3940
"github.com/sirupsen/logrus"
4041
"github.com/urfave/cli"
42+
"go.opentelemetry.io/otel"
4143
"google.golang.org/grpc/grpclog"
4244
)
4345

@@ -130,6 +132,20 @@ can be used and modified as necessary as a custom configuration.`
130132
return err
131133
}
132134

135+
// Initialize OpenTelemetry tracing
136+
shutdown, err := tracing.InitOpenTelemetry(config)
137+
if err != nil {
138+
errors.Wrap(err, "failed to initialize OpenTelemetry tracing")
139+
}
140+
if shutdown != nil {
141+
defer shutdown()
142+
}
143+
144+
// Get a tracer
145+
ctrdTracer := otel.Tracer("containerd")
146+
ctx, mainCtrdSpan := ctrdTracer.Start(ctx, "containerd-exporter")
147+
defer mainCtrdSpan.End()
148+
133149
// Make sure top-level directories are created early.
134150
if err := server.CreateTopLevelDirectories(config); err != nil {
135151
return err
@@ -243,6 +259,9 @@ can be used and modified as necessary as a custom configuration.`
243259
func serve(ctx gocontext.Context, l net.Listener, serveFunc func(net.Listener) error) {
244260
path := l.Addr().String()
245261
log.G(ctx).WithField("address", path).Info("serving...")
262+
serveSpan, ctx := tracing.StartSpan(ctx, l.Addr().String())
263+
defer tracing.StopSpan(serveSpan)
264+
246265
go func() {
247266
defer l.Close()
248267
if err := serveFunc(l); err != nil {

‎services/server/config/config.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ type Config struct {
6767
Timeouts map[string]string `toml:"timeouts"`
6868
// Imports are additional file path list to config files that can overwrite main config file fields
6969
Imports []string `toml:"imports"`
70+
// OpenTelemetry configuration
71+
OpenTelemetry OpenTelemetryConfig `toml:"otel"`
7072

7173
StreamProcessors map[string]StreamProcessor `toml:"stream_processors"`
7274
}
@@ -165,6 +167,14 @@ type ProxyPlugin struct {
165167
Address string `toml:"address"`
166168
}
167169

170+
// OpenTelemetryConfig provides open telemetry configuration
171+
type OpenTelemetryConfig struct {
172+
ServiceName string `toml:"service_name"`
173+
ExporterName string `toml:"exporter_name"`
174+
ExporterEndpoint string `toml:"exporter_endpoint"`
175+
TraceSamplingRatio float64 `toml:"trace_sampling_ratio"`
176+
}
177+
168178
// BoltConfig defines the configuration values for the bolt plugin, which is
169179
// loaded here, rather than back registered in the metadata package.
170180
type BoltConfig struct {
@@ -203,6 +213,24 @@ func (bc *BoltConfig) Validate() error {
203213
}
204214
}
205215

216+
const (
217+
// ExporterTypeOTLP represents the open telemetry exporter OTLP
218+
ExporterTypeOTLP = "otlp"
219+
)
220+
221+
// Validate OpenTelemetry config
222+
func (cfg *OpenTelemetryConfig) Validate() error {
223+
switch cfg.ExporterName {
224+
case ExporterTypeOTLP:
225+
if cfg.ServiceName == "" {
226+
return errors.Wrapf(errdefs.ErrInvalidArgument, "missing service name in config %+v", cfg)
227+
}
228+
return nil
229+
default:
230+
return errors.Wrapf(errdefs.ErrInvalidArgument, "unsupported exporter: %+v", cfg)
231+
}
232+
}
233+
206234
// Decode unmarshals a plugin specific configuration by plugin id
207235
func (c *Config) Decode(p *plugin.Registration) (interface{}, error) {
208236
id := p.URI()

‎services/server/server.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,11 @@ import (
5151
"github.com/containerd/containerd/sys"
5252
"github.com/containerd/ttrpc"
5353
metrics "github.com/docker/go-metrics"
54+
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
5455
grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
5556
"github.com/pkg/errors"
5657
bolt "go.etcd.io/bbolt"
58+
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
5759
"google.golang.org/grpc"
5860
"google.golang.org/grpc/backoff"
5961
"google.golang.org/grpc/credentials"
@@ -98,8 +100,14 @@ func New(ctx context.Context, config *srvconfig.Config) (*Server, error) {
98100
}
99101

100102
serverOpts := []grpc.ServerOption{
101-
grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
102-
grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
103+
grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(
104+
otelgrpc.StreamServerInterceptor(),
105+
grpc.StreamServerInterceptor(grpc_prometheus.StreamServerInterceptor),
106+
)),
107+
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
108+
otelgrpc.UnaryServerInterceptor(),
109+
grpc.UnaryServerInterceptor(grpc_prometheus.UnaryServerInterceptor),
110+
)),
103111
}
104112
if config.GRPC.MaxRecvMsgSize > 0 {
105113
serverOpts = append(serverOpts, grpc.MaxRecvMsgSize(config.GRPC.MaxRecvMsgSize))

‎tracing/tracing.go‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package tracing
18+
19+
import (
20+
"context"
21+
22+
srvconfig "github.com/containerd/containerd/services/server/config"
23+
"github.com/pkg/errors"
24+
"github.com/sirupsen/logrus"
25+
"go.opentelemetry.io/otel"
26+
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
27+
"go.opentelemetry.io/otel/propagation"
28+
"go.opentelemetry.io/otel/sdk/resource"
29+
sdktrace "go.opentelemetry.io/otel/sdk/trace"
30+
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
31+
"go.opentelemetry.io/otel/trace"
32+
"google.golang.org/grpc"
33+
)
34+
35+
// InitOpenTelemetry reads config and initializes otel middleware, sets the exporter
36+
// propagator and global tracer provider
37+
func InitOpenTelemetry(config *srvconfig.Config) (func(), error) {
38+
ctx := context.Background()
39+
40+
// Check if tracing is configured
41+
if config.OpenTelemetry == (srvconfig.OpenTelemetryConfig{}) {
42+
logrus.Info("OpenTelemetry configuration not found, tracing is disabled")
43+
return nil, nil
44+
}
45+
46+
// Validate configuration
47+
if err := config.OpenTelemetry.Validate(); err != nil {
48+
return nil, errors.Wrap(err, "invalid open telemetry configuration")
49+
}
50+
51+
res, err := resource.New(ctx,
52+
resource.WithAttributes(
53+
// Service name used to displace traces in backends
54+
semconv.ServiceNameKey.String(config.OpenTelemetry.ServiceName),
55+
),
56+
)
57+
if err != nil {
58+
return nil, errors.Wrap(err, "failed to create resource")
59+
}
60+
61+
// Configure OTLP trace exporter and set it up to connect to OpenTelemetry collector
62+
// running on a local host.
63+
ctrdTraceExporter, err := otlptracegrpc.New(ctx,
64+
otlptracegrpc.WithEndpoint(config.OpenTelemetry.ExporterEndpoint),
65+
otlptracegrpc.WithDialOption(grpc.WithBlock()),
66+
)
67+
if err != nil {
68+
return nil, errors.Wrap(err, "failed to create trace exporter")
69+
}
70+
71+
// Register the trace exporter with a TracerProvider, using a batch span
72+
// process to aggregate spans before export.
73+
ctrdBatchSpanProcessor := sdktrace.NewBatchSpanProcessor(ctrdTraceExporter)
74+
ctrdTracerProvider := sdktrace.NewTracerProvider(
75+
// We use TraceIDRatioBased sampling. Ratio read from config translated into following
76+
// if sampling ratio < 0 it is interpreted as 0. If ratio >= 1, it will always sample.
77+
sdktrace.WithSampler(sdktrace.TraceIDRatioBased(config.OpenTelemetry.TraceSamplingRatio)),
78+
sdktrace.WithResource(res),
79+
sdktrace.WithSpanProcessor(ctrdBatchSpanProcessor),
80+
)
81+
otel.SetTracerProvider(ctrdTracerProvider)
82+
83+
// set global propagator to tracecontext
84+
otel.SetTextMapPropagator(propagation.TraceContext{})
85+
86+
return func() {
87+
// Shutdown will flush any remaining spans and shut down the exporter.
88+
err := ctrdTracerProvider.Shutdown(ctx)
89+
if err != nil {
90+
logrus.WithError(err).Errorf("failed to shutdown TracerProvider")
91+
}
92+
}, nil
93+
}
94+
95+
// StartSpan starts child span in a context.
96+
func StartSpan(ctx context.Context, opName string, opts ...trace.SpanStartOption) (trace.Span, context.Context) {
97+
parentSpan := trace.SpanFromContext(ctx)
98+
tracer := trace.NewNoopTracerProvider().Tracer("")
99+
if parentSpan.SpanContext().IsValid() {
100+
tracer = parentSpan.TracerProvider().Tracer("")
101+
}
102+
ctx, span := tracer.Start(ctx, opName, opts...)
103+
return span, ctx
104+
}
105+
106+
// StopSpan ends the span specified
107+
func StopSpan(span trace.Span) {
108+
span.End()
109+
}

0 commit comments

Comments
 (0)