Skip to content

Commit 084387e

Browse files
committed
Move tracing to plugin
This just makes the implementation a little cleaner. It also makes the trace exporter pluggable. Signed-off-by: Brian Goff <[email protected]>
1 parent 6fd80de commit 084387e

6 files changed

Lines changed: 150 additions & 121 deletions

File tree

cmd/containerd/builtins.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,5 @@ import (
3535
_ "github.com/containerd/containerd/services/snapshots"
3636
_ "github.com/containerd/containerd/services/tasks"
3737
_ "github.com/containerd/containerd/services/version"
38+
_ "github.com/containerd/containerd/tracing/plugin"
3839
)

cmd/containerd/command/main.go

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,10 @@ import (
3535
"github.com/containerd/containerd/services/server"
3636
srvconfig "github.com/containerd/containerd/services/server/config"
3737
"github.com/containerd/containerd/sys"
38-
"github.com/containerd/containerd/tracing"
3938
"github.com/containerd/containerd/version"
4039
"github.com/pkg/errors"
4140
"github.com/sirupsen/logrus"
4241
"github.com/urfave/cli"
43-
"go.opentelemetry.io/otel"
4442
"google.golang.org/grpc/grpclog"
4543
)
4644

@@ -135,20 +133,6 @@ can be used and modified as necessary as a custom configuration.`
135133
return err
136134
}
137135

138-
// Initialize OpenTelemetry tracing
139-
shutdown, err := tracing.InitOpenTelemetry(config)
140-
if err != nil {
141-
errors.Wrap(err, "failed to initialize OpenTelemetry tracing")
142-
}
143-
if shutdown != nil {
144-
defer shutdown()
145-
}
146-
147-
// Get a tracer
148-
ctrdTracer := otel.Tracer("containerd")
149-
ctx, mainCtrdSpan := ctrdTracer.Start(ctx, "containerd-exporter")
150-
defer mainCtrdSpan.End()
151-
152136
// Make sure top-level directories are created early.
153137
if err := server.CreateTopLevelDirectories(config); err != nil {
154138
return err
@@ -300,9 +284,6 @@ can be used and modified as necessary as a custom configuration.`
300284
func serve(ctx gocontext.Context, l net.Listener, serveFunc func(net.Listener) error) {
301285
path := l.Addr().String()
302286
log.G(ctx).WithField("address", path).Info("serving...")
303-
serveSpan, ctx := tracing.StartSpan(ctx, l.Addr().String())
304-
defer tracing.StopSpan(serveSpan)
305-
306287
go func() {
307288
defer l.Close()
308289
if err := serveFunc(l); err != nil {

plugin/plugin.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,8 @@ const (
7777
GCPlugin Type = "io.containerd.gc.v1"
7878
// EventPlugin implements event handling
7979
EventPlugin Type = "io.containerd.event.v1"
80+
// TracingProcessorPlugin implements a open telemetry span processor
81+
TracingProcessorPlugin Type = "io.containerd.tracing.processor.v1"
8082
)
8183

8284
const (

services/server/config/config.go

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,6 @@ type Config struct {
6767
Timeouts map[string]string `toml:"timeouts"`
6868
// Imports are additional file path list to config files that can overwrite main config file fields
6969
Imports []string `toml:"imports"`
70-
// OpenTelemetry configuration
71-
OpenTelemetry OpenTelemetryConfig `toml:"otel"`
7270

7371
StreamProcessors map[string]StreamProcessor `toml:"stream_processors"`
7472
}
@@ -167,14 +165,6 @@ type ProxyPlugin struct {
167165
Address string `toml:"address"`
168166
}
169167

170-
// OpenTelemetryConfig provides open telemetry configuration
171-
type OpenTelemetryConfig struct {
172-
ServiceName string `toml:"service_name"`
173-
ExporterName string `toml:"exporter_name"`
174-
ExporterEndpoint string `toml:"exporter_endpoint"`
175-
TraceSamplingRatio float64 `toml:"trace_sampling_ratio"`
176-
}
177-
178168
// BoltConfig defines the configuration values for the bolt plugin, which is
179169
// loaded here, rather than back registered in the metadata package.
180170
type BoltConfig struct {
@@ -213,24 +203,6 @@ func (bc *BoltConfig) Validate() error {
213203
}
214204
}
215205

216-
const (
217-
// ExporterTypeOTLP represents the open telemetry exporter OTLP
218-
ExporterTypeOTLP = "otlp"
219-
)
220-
221-
// Validate OpenTelemetry config
222-
func (cfg *OpenTelemetryConfig) Validate() error {
223-
switch cfg.ExporterName {
224-
case ExporterTypeOTLP:
225-
if cfg.ServiceName == "" {
226-
return errors.Wrapf(errdefs.ErrInvalidArgument, "missing service name in config %+v", cfg)
227-
}
228-
return nil
229-
default:
230-
return errors.Wrapf(errdefs.ErrInvalidArgument, "unsupported exporter: %+v", cfg)
231-
}
232-
}
233-
234206
// Decode unmarshals a plugin specific configuration by plugin id
235207
func (c *Config) Decode(p *plugin.Registration) (interface{}, error) {
236208
id := p.URI()

tracing/plugin/otlp.go

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
/*
2+
Copyright The containerd Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package plugin
18+
19+
import (
20+
"io"
21+
22+
"github.com/containerd/containerd/log"
23+
"github.com/containerd/containerd/plugin"
24+
"github.com/pkg/errors"
25+
"go.opentelemetry.io/otel"
26+
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
27+
"go.opentelemetry.io/otel/propagation"
28+
"go.opentelemetry.io/otel/sdk/resource"
29+
sdktrace "go.opentelemetry.io/otel/sdk/trace"
30+
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
31+
"google.golang.org/grpc"
32+
)
33+
34+
const exporterPlugin = "otlp"
35+
36+
func init() {
37+
plugin.Register(&plugin.Registration{
38+
ID: exporterPlugin,
39+
Type: plugin.TracingProcessorPlugin,
40+
Config: &OTLPConfig{},
41+
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
42+
cfg := ic.Config.(*OTLPConfig)
43+
if cfg.Endpoint == "" {
44+
return nil, errors.Wrap(plugin.ErrSkipPlugin, "otlp endpoint not set")
45+
}
46+
dialOpts := []grpc.DialOption{grpc.WithBlock()}
47+
if cfg.Insecure {
48+
dialOpts = append(dialOpts, grpc.WithInsecure())
49+
}
50+
51+
exp, err := otlptracegrpc.New(ic.Context,
52+
otlptracegrpc.WithEndpoint(cfg.Endpoint),
53+
otlptracegrpc.WithDialOption(dialOpts...),
54+
)
55+
if err != nil {
56+
return nil, errors.Wrap(err, "failed to create otlp exporter")
57+
}
58+
return sdktrace.NewBatchSpanProcessor(exp), nil
59+
},
60+
})
61+
plugin.Register(&plugin.Registration{
62+
ID: "tracing",
63+
Type: plugin.InternalPlugin,
64+
Requires: []plugin.Type{plugin.TracingProcessorPlugin},
65+
Config: &TraceConfig{ServiceName: "containerd"},
66+
InitFn: func(ic *plugin.InitContext) (interface{}, error) {
67+
return newTracer(ic)
68+
},
69+
})
70+
}
71+
72+
// OTLPConfig holds the configurations for the built-in otlp span processor
73+
type OTLPConfig struct {
74+
Endpoint string `toml:"endpoint"`
75+
Insecure bool `toml:"insecure"`
76+
}
77+
78+
// TraceConfig is the common configuration for open telemetry.
79+
type TraceConfig struct {
80+
ServiceName string `toml:"service_name"`
81+
TraceSamplingRatio float64 `toml:"sampling_ratio"`
82+
}
83+
84+
type closer struct {
85+
close func() error
86+
}
87+
88+
func (c *closer) Close() error {
89+
return c.close()
90+
}
91+
92+
// InitOpenTelemetry reads config and initializes otel middleware, sets the exporter
93+
// propagator and global tracer provider
94+
func newTracer(ic *plugin.InitContext) (io.Closer, error) {
95+
ctx := ic.Context
96+
config := ic.Config.(*TraceConfig)
97+
98+
res, err := resource.New(ctx,
99+
resource.WithAttributes(
100+
// Service name used to displace traces in backends
101+
semconv.ServiceNameKey.String(config.ServiceName),
102+
),
103+
)
104+
if err != nil {
105+
return nil, errors.Wrap(err, "failed to create resource")
106+
}
107+
108+
opts := []sdktrace.TracerProviderOption{
109+
sdktrace.WithSampler(sdktrace.TraceIDRatioBased(config.TraceSamplingRatio)),
110+
sdktrace.WithResource(res),
111+
}
112+
113+
ls, err := ic.GetByType(plugin.TracingProcessorPlugin)
114+
if err != nil {
115+
return nil, errors.Wrap(err, "failed to get tracing processors")
116+
}
117+
118+
procs := make([]sdktrace.SpanProcessor, 0, len(ls))
119+
for id, pctx := range ls {
120+
p, err := pctx.Instance()
121+
if err != nil {
122+
log.G(ctx).WithError(err).Errorf("Failed to init tracing processor %q", id)
123+
continue
124+
}
125+
proc := p.(sdktrace.SpanProcessor)
126+
opts = append(opts, sdktrace.WithSpanProcessor(proc))
127+
procs = append(procs, proc)
128+
}
129+
130+
provider := sdktrace.NewTracerProvider(opts...)
131+
132+
otel.SetTracerProvider(provider)
133+
otel.SetTextMapPropagator(propagation.TraceContext{})
134+
135+
return &closer{close: func() error {
136+
for _, p := range procs {
137+
if err := p.Shutdown(ctx); err != nil {
138+
return err
139+
}
140+
}
141+
return nil
142+
}}, nil
143+
}

tracing/tracing.go

Lines changed: 4 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -19,87 +19,17 @@ package tracing
1919
import (
2020
"context"
2121

22-
srvconfig "github.com/containerd/containerd/services/server/config"
23-
"github.com/pkg/errors"
24-
"github.com/sirupsen/logrus"
2522
"go.opentelemetry.io/otel"
26-
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
27-
"go.opentelemetry.io/otel/propagation"
28-
"go.opentelemetry.io/otel/sdk/resource"
29-
sdktrace "go.opentelemetry.io/otel/sdk/trace"
30-
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
3123
"go.opentelemetry.io/otel/trace"
32-
"google.golang.org/grpc"
3324
)
3425

35-
// InitOpenTelemetry reads config and initializes otel middleware, sets the exporter
36-
// propagator and global tracer provider
37-
func InitOpenTelemetry(config *srvconfig.Config) (func(), error) {
38-
ctx := context.Background()
39-
40-
// Check if tracing is configured
41-
if config.OpenTelemetry == (srvconfig.OpenTelemetryConfig{}) {
42-
logrus.Info("OpenTelemetry configuration not found, tracing is disabled")
43-
return nil, nil
44-
}
45-
46-
// Validate configuration
47-
if err := config.OpenTelemetry.Validate(); err != nil {
48-
return nil, errors.Wrap(err, "invalid open telemetry configuration")
49-
}
50-
51-
res, err := resource.New(ctx,
52-
resource.WithAttributes(
53-
// Service name used to displace traces in backends
54-
semconv.ServiceNameKey.String(config.OpenTelemetry.ServiceName),
55-
),
56-
)
57-
if err != nil {
58-
return nil, errors.Wrap(err, "failed to create resource")
59-
}
60-
61-
// Configure OTLP trace exporter and set it up to connect to OpenTelemetry collector
62-
// running on a local host.
63-
ctrdTraceExporter, err := otlptracegrpc.New(ctx,
64-
otlptracegrpc.WithEndpoint(config.OpenTelemetry.ExporterEndpoint),
65-
otlptracegrpc.WithDialOption(grpc.WithBlock()),
66-
)
67-
if err != nil {
68-
return nil, errors.Wrap(err, "failed to create trace exporter")
69-
}
70-
71-
// Register the trace exporter with a TracerProvider, using a batch span
72-
// process to aggregate spans before export.
73-
ctrdBatchSpanProcessor := sdktrace.NewBatchSpanProcessor(ctrdTraceExporter)
74-
ctrdTracerProvider := sdktrace.NewTracerProvider(
75-
// We use TraceIDRatioBased sampling. Ratio read from config translated into following
76-
// if sampling ratio < 0 it is interpreted as 0. If ratio >= 1, it will always sample.
77-
sdktrace.WithSampler(sdktrace.TraceIDRatioBased(config.OpenTelemetry.TraceSamplingRatio)),
78-
sdktrace.WithResource(res),
79-
sdktrace.WithSpanProcessor(ctrdBatchSpanProcessor),
80-
)
81-
otel.SetTracerProvider(ctrdTracerProvider)
82-
83-
// set global propagator to tracecontext
84-
otel.SetTextMapPropagator(propagation.TraceContext{})
85-
86-
return func() {
87-
// Shutdown will flush any remaining spans and shut down the exporter.
88-
err := ctrdTracerProvider.Shutdown(ctx)
89-
if err != nil {
90-
logrus.WithError(err).Errorf("failed to shutdown TracerProvider")
91-
}
92-
}, nil
93-
}
94-
9526
// StartSpan starts child span in a context.
9627
func StartSpan(ctx context.Context, opName string, opts ...trace.SpanStartOption) (trace.Span, context.Context) {
97-
parentSpan := trace.SpanFromContext(ctx)
98-
tracer := trace.NewNoopTracerProvider().Tracer("")
99-
if parentSpan.SpanContext().IsValid() {
100-
tracer = parentSpan.TracerProvider().Tracer("")
28+
if parent := trace.SpanFromContext(ctx); parent != nil && parent.SpanContext().IsValid() {
29+
ctx, span := parent.TracerProvider().Tracer("").Start(ctx, opName, opts...)
30+
return span, ctx
10131
}
102-
ctx, span := tracer.Start(ctx, opName, opts...)
32+
ctx, span := otel.Tracer("").Start(ctx, opName, opts...)
10333
return span, ctx
10434
}
10535

0 commit comments

Comments
 (0)