cmd/dockerd: Add workaround for OTEL meter leak

vvoland · vvoland · commit cca708546415 · 2024-10-18T14:41:54.000+02:00
OTEL meter implementation has a memory leak issue which causes each
meter counter invocation to create a new instrument when the meter
provider is not set.

Also add a test, which will fail once a fixed OTEL is vendored.

Signed-off-by: Paweł Gronowski &lt;pawel.gronowski@docker.com&gt;
diff --git a/cmd/dockerd/daemon_test.go b/cmd/dockerd/daemon_test.go
@@ -1,12 +1,14 @@
 package main
 
 import (
+	"runtime"
 	"testing"
 
 	"github.com/containerd/log"
 	"github.com/docker/docker/daemon/config"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/spf13/pflag"
+	"go.opentelemetry.io/otel"
 	"gotest.tools/v3/assert"
 	is "gotest.tools/v3/assert/cmp"
 	"gotest.tools/v3/fs"
@@ -281,3 +283,29 @@ func TestCDISpecDirs(t *testing.T) {
 		})
 	}
 }
+
+// TestOtelMeterLeak tests for a memory leak in the OTEL meter implementation.
+// Once the fixed OTEL is vendored, this test will fail - the workaround
+// and this test should be removed then.
+func TestOtelMeterLeak(t *testing.T) {
+	meter := otel.Meter("foo")
+
+	var before runtime.MemStats
+	runtime.ReadMemStats(&before)
+
+	const counters = 10 * 1000 * 1000
+	for i := 0; i < counters; i++ {
+		_, _ = meter.Int64Counter("bar")
+	}
+
+	var after runtime.MemStats
+	runtime.ReadMemStats(&after)
+
+	allocs := after.Mallocs - before.Mallocs
+	t.Log("Allocations:", allocs)
+
+	if allocs < 10 {
+		// TODO: Remove Workaround OTEL memory leak in cmd/dockerd/daemon.go
+		t.Fatal("Allocations count decreased. OTEL leak workaround is no longer needed!")
+	}
+}
diff --git a/cmd/dockerd/docker.go b/cmd/dockerd/docker.go
@@ -15,6 +15,9 @@ import (
 	"github.com/moby/buildkit/util/apicaps"
 	"github.com/moby/term"
 	"github.com/spf13/cobra"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/metric/noop"
 )
 
 var honorXDG bool
@@ -89,6 +92,12 @@ func main() {
 	// Fixes https://github.com/docker/docker/issues/19728
 	signal.Ignore(syscall.SIGPIPE)
 
+	// Workaround OTEL memory leak
+	// See: https://github.com/open-telemetry/opentelemetry-go-contrib/issues/5190
+	// The need for this workaround is checked by the TestOtelMeterLeak test
+	// TODO: Remove this workaround after upgrading to v1.30.0
+	otel.SetMeterProvider(noop.MeterProvider{})
+
 	// Set terminal emulation based on platform as required.
 	_, stdout, stderr := term.StdStreams()
 	onError := func(err error) {
diff --git a/vendor.mod b/vendor.mod
@@ -100,6 +100,7 @@ require (
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1
 	go.opentelemetry.io/otel v1.21.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0
+	go.opentelemetry.io/otel/metric v1.21.0
 	go.opentelemetry.io/otel/sdk v1.21.0
 	go.opentelemetry.io/otel/trace v1.21.0
 	golang.org/x/mod v0.21.0
@@ -214,7 +215,6 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect
-	go.opentelemetry.io/otel/metric v1.21.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.21.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
