Skip to content

Commit 9a68034

Browse files
authored
add self observability for stdout exporter (#8263)
ref: #7020 ```txt goos: darwin goarch: arm64 pkg: go.opentelemetry.io/otel/exporters/stdout/stdoutlog │ bench_observability_disabled_6x.txt │ bench_observability_upload_6x.txt │ │ sec/op │ sec/op vs base │ ExporterObservability/Disabled-8 3.968µ ± 1% ExporterObservability/UploadSuccess-8 4.236µ ± 2% geomean 3.967µ 4.235µ ? ¹ ² ¹ benchmark set differs from baseline; geomeans may not be comparable ² ratios must be >0 to compute geomean │ bench_observability_disabled_6x.txt │ bench_observability_upload_6x.txt │ │ B/op │ B/op vs base │ ExporterObservability/Disabled-8 1.419Ki ± 0% ExporterObservability/UploadSuccess-8 1.420Ki ± 0% geomean 1.419Ki 1.420Ki ? ¹ ² ¹ benchmark set differs from baseline; geomeans may not be comparable ² ratios must be >0 to compute geomean │ bench_observability_disabled_6x.txt │ bench_observability_upload_6x.txt │ │ allocs/op │ allocs/op vs base │ ExporterObservability/Disabled-8 33.00 ± 0% ExporterObservability/UploadSuccess-8 33.00 ± 0% geomean 33.00 33.00 ? ¹ ² ¹ benchmark set differs from baseline; geomeans may not be comparable ² ratios must be >0 to compute geomean ```
1 parent 714a196 commit 9a68034

8 files changed

Lines changed: 378 additions & 37 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
4141
- Add `go.opentelemetry.io/otel/semconv/v1.41.0` package.
4242
The package contains semantic conventions from the `v1.41.0` version of the OpenTelemetry Semantic Conventions.
4343
See the [migration documentation](./semconv/v1.41.0/MIGRATION.md) for information on how to upgrade from `go.opentelemetry.io/otel/semconv/v1.40.0`. (#8324)
44+
- Add experimental self-observability metrics in `go.opentelemetry.io/otel/exporters/stdout/stdoutlog`. (#8263)
4445
- Add Observable variants of instruments to `go.opentelemetry.io/otel/semconv/v1.41.0` package. (#8350)
4546
- Generate explicit histogram bucket boundaries from weaver configuration for HTTP and RPC duration instruments in `go.opentelemetry.io/otel/semconv/v1.41.0`. (#8002)
4647

‎exporters/stdout/stdoutlog/exporter.go‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ import (
88
"encoding/json"
99
"sync/atomic"
1010

11+
"go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/counter"
12+
"go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/observ"
13+
1114
"go.opentelemetry.io/otel/sdk/log"
1215
)
1316

@@ -18,6 +21,7 @@ var _ log.Exporter = &Exporter{}
1821
type Exporter struct {
1922
encoder atomic.Pointer[json.Encoder]
2023
timestamps bool
24+
inst *observ.Instrumentation
2125
}
2226

2327
// New creates an [Exporter].
@@ -29,21 +33,31 @@ func New(options ...Option) (*Exporter, error) {
2933
enc.SetIndent("", "\t")
3034
}
3135

32-
e := Exporter{
36+
e := &Exporter{
3337
timestamps: cfg.Timestamps,
3438
}
3539
e.encoder.Store(enc)
3640

37-
return &e, nil
41+
var err error
42+
e.inst, err = observ.NewInstrumentation(counter.NextExporterID())
43+
return e, err
3844
}
3945

4046
// Export exports log records to writer.
41-
func (e *Exporter) Export(ctx context.Context, records []log.Record) error {
47+
func (e *Exporter) Export(ctx context.Context, records []log.Record) (err error) {
4248
enc := e.encoder.Load()
4349
if enc == nil {
4450
return nil
4551
}
4652

53+
var success int64
54+
if e.inst != nil {
55+
op := e.inst.ExportLogs(ctx, int64(len(records)))
56+
defer func() {
57+
op.End(success, err)
58+
}()
59+
}
60+
4761
for _, record := range records {
4862
// Honor context cancellation.
4963
if err := ctx.Err(); err != nil {
@@ -55,6 +69,7 @@ func (e *Exporter) Export(ctx context.Context, records []log.Record) error {
5569
if err := enc.Encode(recordJSON); err != nil {
5670
return err
5771
}
72+
success++
5873
}
5974
return nil
6075
}

‎exporters/stdout/stdoutlog/exporter_test.go‎

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,19 +7,29 @@ import (
77
"bytes"
88
"context"
99
"encoding/json"
10+
"errors"
1011
"sync"
1112
"testing"
1213
"time"
1314

1415
"github.com/stretchr/testify/assert"
1516
"github.com/stretchr/testify/require"
1617

18+
"go.opentelemetry.io/otel"
1719
"go.opentelemetry.io/otel/attribute"
20+
"go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal"
21+
"go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/counter"
22+
"go.opentelemetry.io/otel/exporters/stdout/stdoutlog/internal/observ"
1823
"go.opentelemetry.io/otel/log"
1924
"go.opentelemetry.io/otel/sdk/instrumentation"
2025
sdklog "go.opentelemetry.io/otel/sdk/log"
2126
"go.opentelemetry.io/otel/sdk/log/logtest"
27+
"go.opentelemetry.io/otel/sdk/metric"
28+
"go.opentelemetry.io/otel/sdk/metric/metricdata"
29+
"go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest"
2230
"go.opentelemetry.io/otel/sdk/resource"
31+
semconv "go.opentelemetry.io/otel/semconv/v1.41.0"
32+
"go.opentelemetry.io/otel/semconv/v1.41.0/otelconv"
2333
"go.opentelemetry.io/otel/trace"
2434
)
2535

@@ -458,3 +468,216 @@ func TestValueMarshalJSON(t *testing.T) {
458468
})
459469
}
460470
}
471+
472+
func TestObservability(t *testing.T) {
473+
tests := []struct {
474+
name string
475+
enabled bool
476+
test func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics)
477+
}{
478+
{
479+
name: "Disabled",
480+
enabled: false,
481+
test: func(t *testing.T, _ func() metricdata.ScopeMetrics) {
482+
var buf bytes.Buffer
483+
exporter, err := New(WithWriter(&buf))
484+
require.NoError(t, err)
485+
assert.Nil(t, exporter.inst)
486+
},
487+
},
488+
{
489+
name: "upload success",
490+
enabled: true,
491+
test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) {
492+
ctx := t.Context()
493+
var buf bytes.Buffer
494+
exporter, err := New(WithWriter(&buf))
495+
require.NoError(t, err)
496+
err = exporter.Export(ctx, []sdklog.Record{getRecord(time.Now())})
497+
require.NoError(t, err)
498+
assertStdoutLogObservabilityMetrics(t, scopeMetrics(), 1, 1, nil)
499+
},
500+
},
501+
{
502+
name: "upload failed",
503+
enabled: true,
504+
test: func(t *testing.T, scopeMetrics func() metricdata.ScopeMetrics) {
505+
ctx := t.Context()
506+
writeErr := errors.New("write failed")
507+
exporter, err := New(WithWriter(&failingWriter{err: writeErr}))
508+
require.NoError(t, err)
509+
err = exporter.Export(ctx, []sdklog.Record{getRecord(time.Now())})
510+
require.ErrorIs(t, err, writeErr)
511+
assertStdoutLogObservabilityMetrics(t, scopeMetrics(), 1, 0, writeErr)
512+
},
513+
},
514+
}
515+
516+
for _, tc := range tests {
517+
t.Run(tc.name, func(t *testing.T) {
518+
if tc.enabled {
519+
t.Setenv("OTEL_GO_X_OBSERVABILITY", "true")
520+
_ = counter.SetExporterID(0)
521+
}
522+
provider := otel.GetMeterProvider()
523+
t.Cleanup(func() {
524+
otel.SetMeterProvider(provider)
525+
})
526+
r := metric.NewManualReader()
527+
mp := metric.NewMeterProvider(metric.WithReader(r))
528+
otel.SetMeterProvider(mp)
529+
530+
scopeMetrics := func() metricdata.ScopeMetrics {
531+
var got metricdata.ResourceMetrics
532+
err := r.Collect(t.Context(), &got)
533+
require.NoError(t, err)
534+
require.Len(t, got.ScopeMetrics, 1)
535+
return got.ScopeMetrics[0]
536+
}
537+
tc.test(t, scopeMetrics)
538+
})
539+
}
540+
}
541+
542+
func BenchmarkExporterObservability(b *testing.B) {
543+
ctx := b.Context()
544+
rec := getRecord(time.Date(2020, 1, 1, 12, 0, 0, 0, time.UTC))
545+
records := []sdklog.Record{rec}
546+
547+
b.Run("Disabled", func(b *testing.B) {
548+
b.Setenv("OTEL_GO_X_OBSERVABILITY", "false")
549+
var buf bytes.Buffer
550+
exporter, err := New(WithWriter(&buf))
551+
require.NoError(b, err)
552+
553+
b.ReportAllocs()
554+
b.ResetTimer()
555+
for b.Loop() {
556+
buf.Reset()
557+
_ = exporter.Export(ctx, records)
558+
}
559+
})
560+
561+
setupObservability := func(b *testing.B) {
562+
b.Helper()
563+
b.Setenv("OTEL_GO_X_OBSERVABILITY", "true")
564+
_ = counter.SetExporterID(0)
565+
provider := otel.GetMeterProvider()
566+
b.Cleanup(func() {
567+
otel.SetMeterProvider(provider)
568+
})
569+
r := metric.NewManualReader()
570+
mp := metric.NewMeterProvider(metric.WithReader(r))
571+
otel.SetMeterProvider(mp)
572+
}
573+
574+
b.Run("UploadSuccess", func(b *testing.B) {
575+
setupObservability(b)
576+
var buf bytes.Buffer
577+
exporter, err := New(WithWriter(&buf))
578+
require.NoError(b, err)
579+
b.ReportAllocs()
580+
b.ResetTimer()
581+
for b.Loop() {
582+
buf.Reset()
583+
_ = exporter.Export(ctx, records)
584+
}
585+
})
586+
}
587+
588+
// failingWriter implements [io.Writer] and always returns an error.
589+
type failingWriter struct {
590+
err error
591+
}
592+
593+
func (w *failingWriter) Write([]byte) (int, error) {
594+
return 0, w.err
595+
}
596+
597+
func stdoutObservAttrSet(err error) attribute.Set {
598+
attrs := []attribute.KeyValue{
599+
semconv.OTelComponentName(observ.GetComponentName(0)),
600+
semconv.OTelComponentTypeKey.String(observ.ComponentType),
601+
}
602+
if err != nil {
603+
attrs = append(attrs, semconv.ErrorType(err))
604+
}
605+
return attribute.NewSet(attrs...)
606+
}
607+
608+
func stdoutLogInflightMetric() metricdata.Metrics {
609+
inflight := otelconv.SDKExporterLogInflight{}
610+
return metricdata.Metrics{
611+
Name: inflight.Name(),
612+
Description: inflight.Description(),
613+
Unit: inflight.Unit(),
614+
Data: metricdata.Sum[int64]{
615+
Temporality: metricdata.CumulativeTemporality,
616+
DataPoints: []metricdata.DataPoint[int64]{
617+
{Attributes: stdoutObservAttrSet(nil), Value: 0},
618+
},
619+
},
620+
}
621+
}
622+
623+
func stdoutLogExportedMetric(success, total int64, err error) metricdata.Metrics {
624+
dp := []metricdata.DataPoint[int64]{
625+
{Attributes: stdoutObservAttrSet(nil), Value: success},
626+
}
627+
if err != nil {
628+
dp = append(dp, metricdata.DataPoint[int64]{
629+
Attributes: stdoutObservAttrSet(err),
630+
Value: total - success,
631+
})
632+
}
633+
exported := otelconv.SDKExporterLogExported{}
634+
return metricdata.Metrics{
635+
Name: exported.Name(),
636+
Description: exported.Description(),
637+
Unit: exported.Unit(),
638+
Data: metricdata.Sum[int64]{
639+
Temporality: metricdata.CumulativeTemporality,
640+
IsMonotonic: true,
641+
DataPoints: dp,
642+
},
643+
}
644+
}
645+
646+
func stdoutLogDurationMetric(err error) metricdata.Metrics {
647+
duration := otelconv.SDKExporterOperationDuration{}
648+
return metricdata.Metrics{
649+
Name: duration.Name(),
650+
Description: duration.Description(),
651+
Unit: duration.Unit(),
652+
Data: metricdata.Histogram[float64]{
653+
Temporality: metricdata.CumulativeTemporality,
654+
DataPoints: []metricdata.HistogramDataPoint[float64]{
655+
{Attributes: stdoutObservAttrSet(err)},
656+
},
657+
},
658+
}
659+
}
660+
661+
func assertStdoutLogObservabilityMetrics(
662+
t *testing.T,
663+
got metricdata.ScopeMetrics,
664+
logs int64,
665+
success int64,
666+
err error,
667+
) {
668+
t.Helper()
669+
wantScope := instrumentation.Scope{
670+
Name: observ.ScopeName,
671+
Version: internal.Version,
672+
SchemaURL: semconv.SchemaURL,
673+
}
674+
assert.Equal(t, wantScope, got.Scope)
675+
676+
m := got.Metrics
677+
require.Len(t, m, 3)
678+
679+
o := metricdatatest.IgnoreTimestamp()
680+
metricdatatest.AssertEqual(t, stdoutLogInflightMetric(), m[0], o)
681+
metricdatatest.AssertEqual(t, stdoutLogExportedMetric(success, logs, err), m[1], o)
682+
metricdatatest.AssertEqual(t, stdoutLogDurationMetric(err), m[2], metricdatatest.IgnoreValue(), o)
683+
}

‎exporters/stdout/stdoutlog/internal/counter/counter.go‎

Lines changed: 31 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)