Skip to content

Commit ceeab7d

Browse files
authored
introduce otel logging exporter (#22609)
Similar to what we're doing for tracing, we want to collect the logs in a central place, so we can then forward them to the encrypted log sink. We're using the OTEL specification for that here. The most interesting thing here is probably how we encode property values. Property values are always expected to be sent as attributes. This is so log clients (e.g. various language SDKs), can just marshal property values into their usual protobuf form, and send them, and we do proper JSON formatting in the CLI later. This way we get consistent formatting of property values in the logs, and can later redact them properly. To make sure we always recognize property values correctly, we wrap them in another Protobuf container, that includes a magic string. This makes sure we don't accidentally decode something that is not a property value as a property value. For now we don't do anything with these logs. We will want to forward them to the encrypted log handler after we merge that. For the current API, e.g. `Infof` and friends, we save the arguments as separate structured log fields, in the `pulumi.log.arg<x>` field. `pulumi logs decrypt` is then responsible for filling in those details.
1 parent a7d5f52 commit ceeab7d

7 files changed

Lines changed: 310 additions & 7 deletions

File tree

‎pkg/cmd/pulumi/pulumi.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,7 @@ func NewPulumiCmd() (*cobra.Command, func()) {
313313

314314
cmdutil.InitTracing("pulumi-cli", "pulumi", tracingFlag)
315315

316-
if err := cmdutil.InitOtelReceiver(otelTracesFlag); err != nil {
316+
if err := cmdutil.InitOtelReceiver(otelTracesFlag, nil); err != nil {
317317
logging.V(3).Infof("failed to initialize OTLP receiver: %v", err)
318318
}
319319

‎sdk/go/common/util/cmdutil/trace.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
3434
"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
3535
"github.com/pulumi/pulumi/sdk/v3/go/common/util/otelreceiver"
36+
otellog "github.com/pulumi/pulumi/sdk/v3/go/common/util/otelreceiver/logging"
3637
jaeger "github.com/uber/jaeger-client-go"
3738
"github.com/uber/jaeger-client-go/transport/zipkin"
3839
"go.opentelemetry.io/otel"
@@ -215,8 +216,9 @@ func OTelEndpoint() string {
215216
return otelEndpoint
216217
}
217218

218-
// InitOTelReceiver starts the OTLP receiver with the given endpoint.
219-
func InitOtelReceiver(endpoint string) error {
219+
// InitOtelReceiver starts the OTLP receiver with the given endpoint and
220+
// optional log exporter.
221+
func InitOtelReceiver(endpoint string, logExporter otellog.LogExporter) error {
220222
if endpoint == "" {
221223
return nil
222224
}
@@ -226,7 +228,7 @@ func InitOtelReceiver(endpoint string) error {
226228
return fmt.Errorf("failed to create OTLP exporter: %w", err)
227229
}
228230

229-
receiver, err := otelreceiver.Start(exporter)
231+
receiver, err := otelreceiver.Start(exporter, logExporter)
230232
if err != nil {
231233
_ = exporter.Shutdown(context.Background())
232234
return fmt.Errorf("failed to start OTLP receiver: %w", err)
Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
// Copyright 2026, Pulumi Corporation.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package logging
16+
17+
import (
18+
"encoding/binary"
19+
"errors"
20+
21+
"google.golang.org/protobuf/proto"
22+
"google.golang.org/protobuf/types/known/structpb"
23+
)
24+
25+
// PropertyValueLogMagic is the magic prefix for encoded property
26+
// values: the ASCII string "pulumiPv" as a little-endian uint64.
27+
// This must match the constant used by the Python and Node SDKs.
28+
const PropertyValueLogMagic uint64 = 0x7650696d756c7570
29+
30+
// EncodeStructValueForLog prepends the magic prefix to the
31+
// protobuf-encoded structpb.Value.
32+
func EncodeStructValueForLog(val *structpb.Value) ([]byte, error) {
33+
valBytes, err := proto.Marshal(val)
34+
if err != nil {
35+
return nil, err
36+
}
37+
buf := make([]byte, 8+len(valBytes))
38+
binary.LittleEndian.PutUint64(buf[:8], PropertyValueLogMagic)
39+
copy(buf[8:], valBytes)
40+
return buf, nil
41+
}
42+
43+
// DecodeStructValueFromLog verifies the magic prefix and returns the
44+
// inner protobuf-encoded structpb.Value.
45+
func DecodeStructValueFromLog(data []byte) (*structpb.Value, error) {
46+
if len(data) < 8 {
47+
return nil, errors.New("not a property value log: too short")
48+
}
49+
if binary.LittleEndian.Uint64(data[:8]) != PropertyValueLogMagic {
50+
return nil, errors.New("not a property value log: magic mismatch")
51+
}
52+
val := &structpb.Value{}
53+
if err := proto.Unmarshal(data[8:], val); err != nil {
54+
return nil, err
55+
}
56+
return val, nil
57+
}
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
// Copyright 2026, Pulumi Corporation.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// Package logging implements the OTLP LogsService receiver that
16+
// forwards log records to a LogExporter.
17+
package logging
18+
19+
import (
20+
"context"
21+
22+
collogspb "go.opentelemetry.io/proto/otlp/collector/logs/v1"
23+
"google.golang.org/grpc"
24+
"google.golang.org/grpc/codes"
25+
"google.golang.org/grpc/status"
26+
"google.golang.org/protobuf/proto"
27+
28+
"go.opentelemetry.io/collector/pdata/plog"
29+
)
30+
31+
// LogExporter receives OTLP log records. Property value attributes
32+
// are passed through as raw bytes; the consumer is responsible for
33+
// decoding them (e.g. via logging.DecodeStructValueFromLog and
34+
// plugin.UnmarshalProperties).
35+
type LogExporter interface {
36+
ExportLogs(ctx context.Context, logs plog.Logs) error
37+
Shutdown(ctx context.Context) error
38+
}
39+
40+
// Register registers the OTLP LogsService on a gRPC server,
41+
// forwarding log records to the given exporter.
42+
func Register(s *grpc.Server, exporter LogExporter) {
43+
collogspb.RegisterLogsServiceServer(s, &service{exporter: exporter})
44+
}
45+
46+
type service struct {
47+
collogspb.UnimplementedLogsServiceServer
48+
exporter LogExporter
49+
}
50+
51+
func (s *service) Export(
52+
ctx context.Context,
53+
req *collogspb.ExportLogsServiceRequest,
54+
) (*collogspb.ExportLogsServiceResponse, error) {
55+
if req == nil {
56+
return nil, status.Error(codes.InvalidArgument, "nil request")
57+
}
58+
59+
protoBytes, err := proto.Marshal(req)
60+
if err != nil {
61+
return nil, status.Errorf(codes.Internal, "failed to marshal request: %v", err)
62+
}
63+
64+
var unmarshaler plog.ProtoUnmarshaler
65+
logs, err := unmarshaler.UnmarshalLogs(protoBytes)
66+
if err != nil {
67+
return nil, status.Errorf(codes.Internal, "failed to unmarshal logs: %v", err)
68+
}
69+
70+
if s.exporter != nil {
71+
if err := s.exporter.ExportLogs(ctx, logs); err != nil {
72+
return nil, status.Errorf(codes.Internal, "failed to export logs: %v", err)
73+
}
74+
}
75+
76+
return &collogspb.ExportLogsServiceResponse{}, nil
77+
}
Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
// Copyright 2026, Pulumi Corporation.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package logging
16+
17+
import (
18+
"context"
19+
"sync"
20+
"testing"
21+
"time"
22+
23+
"github.com/stretchr/testify/assert"
24+
"github.com/stretchr/testify/require"
25+
"go.opentelemetry.io/collector/pdata/pcommon"
26+
"go.opentelemetry.io/collector/pdata/plog"
27+
collogspb "go.opentelemetry.io/proto/otlp/collector/logs/v1"
28+
commonpb "go.opentelemetry.io/proto/otlp/common/v1"
29+
logspb "go.opentelemetry.io/proto/otlp/logs/v1"
30+
resourcepb "go.opentelemetry.io/proto/otlp/resource/v1"
31+
)
32+
33+
type mockLogExporter struct {
34+
mu sync.Mutex
35+
logs []plog.Logs
36+
}
37+
38+
func (m *mockLogExporter) ExportLogs(_ context.Context, logs plog.Logs) error {
39+
m.mu.Lock()
40+
defer m.mu.Unlock()
41+
m.logs = append(m.logs, logs)
42+
return nil
43+
}
44+
45+
func (m *mockLogExporter) Shutdown(context.Context) error { return nil }
46+
47+
func (m *mockLogExporter) firstRecord() plog.LogRecord {
48+
m.mu.Lock()
49+
defer m.mu.Unlock()
50+
return m.logs[0].ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0)
51+
}
52+
53+
func TestExportForwardsToExporter(t *testing.T) {
54+
t.Parallel()
55+
56+
exporter := &mockLogExporter{}
57+
svc := &service{exporter: exporter}
58+
now := uint64(time.Now().UnixNano()) //nolint:gosec // test timestamps
59+
60+
req := &collogspb.ExportLogsServiceRequest{
61+
ResourceLogs: []*logspb.ResourceLogs{{
62+
Resource: &resourcepb.Resource{
63+
Attributes: []*commonpb.KeyValue{{
64+
Key: "service.name",
65+
Value: &commonpb.AnyValue{
66+
Value: &commonpb.AnyValue_StringValue{StringValue: "test-svc"},
67+
},
68+
}},
69+
},
70+
ScopeLogs: []*logspb.ScopeLogs{{
71+
LogRecords: []*logspb.LogRecord{{
72+
TimeUnixNano: now,
73+
SeverityNumber: logspb.SeverityNumber_SEVERITY_NUMBER_INFO,
74+
Body: &commonpb.AnyValue{
75+
Value: &commonpb.AnyValue_StringValue{StringValue: "hello"},
76+
},
77+
Attributes: []*commonpb.KeyValue{{
78+
Key: "key",
79+
Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: "value"}},
80+
}},
81+
}},
82+
}},
83+
}},
84+
}
85+
86+
resp, err := svc.Export(t.Context(), req)
87+
require.NoError(t, err)
88+
require.NotNil(t, resp)
89+
90+
require.Len(t, exporter.logs, 1)
91+
lr := exporter.firstRecord()
92+
assert.Equal(t, "hello", lr.Body().AsString())
93+
v, ok := lr.Attributes().Get("key")
94+
require.True(t, ok)
95+
assert.Equal(t, "value", v.Str())
96+
}
97+
98+
func TestExportPassesThroughBytesAttributes(t *testing.T) {
99+
t.Parallel()
100+
101+
// Property value bytes are passed through to the exporter as-is;
102+
// decoding is the consumer's responsibility.
103+
encoded := []byte("pulumiPv" + "some-protobuf-data")
104+
105+
exporter := &mockLogExporter{}
106+
svc := &service{exporter: exporter}
107+
now := uint64(time.Now().UnixNano()) //nolint:gosec // test timestamps
108+
109+
req := &collogspb.ExportLogsServiceRequest{
110+
ResourceLogs: []*logspb.ResourceLogs{{
111+
ScopeLogs: []*logspb.ScopeLogs{{
112+
LogRecords: []*logspb.LogRecord{{
113+
TimeUnixNano: now,
114+
SeverityNumber: logspb.SeverityNumber_SEVERITY_NUMBER_DEBUG,
115+
Body: &commonpb.AnyValue{
116+
Value: &commonpb.AnyValue_StringValue{StringValue: "resource inputs"},
117+
},
118+
Attributes: []*commonpb.KeyValue{{
119+
Key: "inputs",
120+
Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_BytesValue{BytesValue: encoded}},
121+
}},
122+
}},
123+
}},
124+
}},
125+
}
126+
127+
resp, err := svc.Export(t.Context(), req)
128+
require.NoError(t, err)
129+
require.NotNil(t, resp)
130+
131+
lr := exporter.firstRecord()
132+
v, ok := lr.Attributes().Get("inputs")
133+
require.True(t, ok)
134+
require.Equal(t, pcommon.ValueTypeBytes, v.Type())
135+
assert.Equal(t, encoded, v.Bytes().AsRaw())
136+
}
137+
138+
func TestExportNilExporterDoesNotPanic(t *testing.T) {
139+
t.Parallel()
140+
141+
svc := &service{exporter: nil}
142+
now := uint64(time.Now().UnixNano()) //nolint:gosec // test timestamps
143+
144+
req := &collogspb.ExportLogsServiceRequest{
145+
ResourceLogs: []*logspb.ResourceLogs{{
146+
ScopeLogs: []*logspb.ScopeLogs{{
147+
LogRecords: []*logspb.LogRecord{{
148+
TimeUnixNano: now,
149+
SeverityNumber: logspb.SeverityNumber_SEVERITY_NUMBER_INFO,
150+
Body: &commonpb.AnyValue{
151+
Value: &commonpb.AnyValue_StringValue{StringValue: "test"},
152+
},
153+
}},
154+
}},
155+
}},
156+
}
157+
158+
resp, err := svc.Export(t.Context(), req)
159+
require.NoError(t, err)
160+
require.NotNil(t, resp)
161+
}

‎sdk/go/common/util/otelreceiver/receiver.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
2727

2828
"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
29+
otellog "github.com/pulumi/pulumi/sdk/v3/go/common/util/otelreceiver/logging"
2930
)
3031

3132
// Receiver is an OTLP gRPC receiver that can receive traces, metrics, and logs
@@ -47,8 +48,10 @@ type traceService struct {
4748
r *Receiver
4849
}
4950

50-
// Start creates and starts a new OTLP receiver with the given exporter.
51-
func Start(exporter SpanExporter) (*Receiver, error) {
51+
// Start creates and starts a new OTLP receiver with the given span
52+
// and log exporters. Either exporter may be nil if that signal is
53+
// not needed.
54+
func Start(exporter SpanExporter, logExporter otellog.LogExporter) (*Receiver, error) {
5255
addr := "127.0.0.1:0"
5356
listener, err := net.Listen("tcp", addr)
5457
if err != nil {
@@ -67,6 +70,9 @@ func Start(exporter SpanExporter) (*Receiver, error) {
6770
r.server = grpc.NewServer()
6871

6972
coltracepb.RegisterTraceServiceServer(r.server, &traceService{r: r})
73+
if logExporter != nil {
74+
otellog.Register(r.server, logExporter)
75+
}
7076

7177
go func() {
7278
defer close(r.done)

‎sdk/go/common/util/otelreceiver/receiver_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ func TestReceiverWithExporter(t *testing.T) {
7878
// Create a mock exporter to capture spans
7979
exporter := &mockExporter{}
8080

81-
receiver, err := Start(exporter)
81+
receiver, err := Start(exporter, nil)
8282
require.NoError(t, err)
8383
defer func() {
8484
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)

0 commit comments

Comments
 (0)