Skip to content

Commit 5e1b1d0

Browse files
authored
Add more tracing spans around stack references (#23787)
User reported perf issues around stack references. Adding a load of trace spans so we can get them to run a trace run and see what's actually slow.
1 parent 45b7ec7 commit 5e1b1d0

7 files changed

Lines changed: 74 additions & 8 deletions

File tree

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
component: engine
2+
kind: improvement
3+
body: Add OpenTelemetry spans to the StackReference read path so performance traces can pinpoint slow snapshot fetches, deserialization, or secret decryption
4+
time: 2026-07-03T00:00:00+00:00

‎pkg/backend/backend.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ import (
2222
"io"
2323
"time"
2424

25+
"go.opentelemetry.io/otel"
26+
2527
"github.com/pulumi/pulumi/pkg/v3/backend/display"
2628
sdkDisplay "github.com/pulumi/pulumi/pkg/v3/display"
2729
"github.com/pulumi/pulumi/pkg/v3/engine"
@@ -36,6 +38,7 @@ import (
3638
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
3739
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/config"
3840
"github.com/pulumi/pulumi/sdk/v3/go/common/tokens"
41+
"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
3942
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
4043
"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
4144
"github.com/pulumi/pulumi/sdk/v3/go/property"
@@ -397,6 +400,10 @@ func (c *backendClient) GetStackOutputs(
397400
name string,
398401
onDecryptError func(err error) error,
399402
) (property.Map, error) {
403+
tracer := otel.Tracer("pulumi-cli")
404+
ctx, span := cmdutil.StartSpan(ctx, tracer, "backendClient.GetStackOutputs")
405+
defer span.End()
406+
400407
ref, err := c.backend.ParseStackReference(name)
401408
if err != nil {
402409
return property.Map{}, err

‎pkg/backend/diy/state.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"github.com/pulumi/pulumi/sdk/v3/go/common/util/retry"
3333
"github.com/pulumi/pulumi/sdk/v3/go/property"
3434

35+
"go.opentelemetry.io/otel"
3536
"gocloud.dev/blob"
3637
"gocloud.dev/gcerrors"
3738

@@ -43,6 +44,7 @@ import (
4344
"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
4445
"github.com/pulumi/pulumi/sdk/v3/go/common/encoding"
4546
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/config"
47+
"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
4648
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
4749
"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
4850
)
@@ -131,6 +133,10 @@ func (b *diyBackend) stackExists(
131133
func (b *diyBackend) getSnapshot(ctx context.Context,
132134
secretsProvider secrets.Provider, ref *diyBackendReference,
133135
) (*deploy.Snapshot, error) {
136+
tracer := otel.Tracer("pulumi-cli")
137+
ctx, span := cmdutil.StartSpan(ctx, tracer, "diyBackend.getSnapshot")
138+
defer span.End()
139+
134140
contract.Requiref(ref != nil, "ref", "must not be nil")
135141

136142
checkpoint, _, _, err := b.getCheckpoint(ctx, ref)
@@ -169,6 +175,10 @@ func (b *diyBackend) getSnapshot(ctx context.Context,
169175
func (b *diyBackend) getSnapshotStackOutputs(ctx context.Context,
170176
secretsProvider secrets.Provider, ref *diyBackendReference,
171177
) (property.Map, error) {
178+
tracer := otel.Tracer("pulumi-cli")
179+
ctx, span := cmdutil.StartSpan(ctx, tracer, "diyBackend.getSnapshotStackOutputs")
180+
defer span.End()
181+
172182
contract.Requiref(ref != nil, "ref", "must not be nil")
173183

174184
checkpoint, _, _, err := b.getCheckpoint(ctx, ref)

‎pkg/backend/httpstate/state.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,13 @@ import (
2222
"time"
2323

2424
"github.com/jonboulle/clockwork"
25+
"go.opentelemetry.io/otel"
26+
2527
"github.com/pulumi/pulumi/pkg/v3/channel"
2628
"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
2729
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
2830
"github.com/pulumi/pulumi/sdk/v3/go/common/tokens"
31+
"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
2932
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
3033
"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
3134
"github.com/pulumi/pulumi/sdk/v3/go/property"
@@ -190,6 +193,10 @@ func (b *cloudBackend) completeUpdate(
190193
func (b *cloudBackend) getSnapshot(ctx context.Context,
191194
secretsProvider secrets.Provider, stackRef backend.StackReference,
192195
) (*deploy.Snapshot, error) {
196+
tracer := otel.Tracer("pulumi-cli")
197+
ctx, span := cmdutil.StartSpan(ctx, tracer, "cloudBackend.getSnapshot")
198+
defer span.End()
199+
193200
untypedDeployment, err := b.exportDeployment(ctx, stackRef, nil /* get latest */)
194201
if err != nil {
195202
return nil, err
@@ -225,6 +232,10 @@ func (b *cloudBackend) getSnapshot(ctx context.Context,
225232
func (b *cloudBackend) getSnapshotStackOutputs(ctx context.Context,
226233
secretsProvider secrets.Provider, stackRef backend.StackReference,
227234
) (property.Map, error) {
235+
tracer := otel.Tracer("pulumi-cli")
236+
ctx, span := cmdutil.StartSpan(ctx, tracer, "cloudBackend.getSnapshotStackOutputs")
237+
defer span.End()
238+
228239
untypedDeployment, err := b.exportDeployment(ctx, stackRef, nil /* get latest */)
229240
if err != nil {
230241
return property.Map{}, err

‎pkg/resource/deploy/builtins.go‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,12 @@ import (
2121
"sort"
2222

2323
uuid "github.com/gofrs/uuid"
24+
"go.opentelemetry.io/otel"
2425

2526
"github.com/pulumi/pulumi/pkg/v3/resource/plugin"
2627
"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
2728
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
29+
"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
2830
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
2931
"github.com/pulumi/pulumi/sdk/v3/go/pulumi-internal/gsync"
3032
)
@@ -192,12 +194,12 @@ func (p *builtinProvider) Diff(_ context.Context, req plugin.DiffRequest) (plugi
192194
}
193195
}
194196

195-
func (p *builtinProvider) Create(_ context.Context, req plugin.CreateRequest) (plugin.CreateResponse, error) {
197+
func (p *builtinProvider) Create(ctx context.Context, req plugin.CreateRequest) (plugin.CreateResponse, error) {
196198
typ := req.URN.Type()
197199
switch typ { //nolint:exhaustive
198200
case stackReferenceType:
199201

200-
state, err := p.readStackReference(req.Properties)
202+
state, err := p.readStackReference(ctx, req.Properties)
201203
if err != nil {
202204
return plugin.CreateResponse{Status: resource.StatusUnknown}, err
203205
}
@@ -270,7 +272,7 @@ func (p *builtinProvider) List(context.Context, plugin.ListRequest) (*plugin.Lis
270272
return nil, errors.New("the builtin provider does not support List")
271273
}
272274

273-
func (p *builtinProvider) Read(_ context.Context, req plugin.ReadRequest) (plugin.ReadResponse, error) {
275+
func (p *builtinProvider) Read(ctx context.Context, req plugin.ReadRequest) (plugin.ReadResponse, error) {
274276
contract.Requiref(req.URN != "", "urn", "must not be empty")
275277
contract.Requiref(req.ID != "", "id", "must not be empty")
276278

@@ -288,7 +290,7 @@ func (p *builtinProvider) Read(_ context.Context, req plugin.ReadRequest) (plugi
288290
return plugin.ReadResponse{Status: resource.StatusUnknown}, errors.New("stack reference can not be imported")
289291
}
290292

291-
outputs, err := p.readStackReference(req.State)
293+
outputs, err := p.readStackReference(ctx, req.State)
292294
if err != nil {
293295
return plugin.ReadResponse{Status: resource.StatusUnknown}, err
294296
}
@@ -330,12 +332,12 @@ const (
330332
getResource = "pulumi:pulumi:getResource"
331333
)
332334

333-
func (p *builtinProvider) Invoke(_ context.Context, req plugin.InvokeRequest) (plugin.InvokeResponse, error) {
335+
func (p *builtinProvider) Invoke(ctx context.Context, req plugin.InvokeRequest) (plugin.InvokeResponse, error) {
334336
var outs resource.PropertyMap
335337
var err error
336338
switch req.Tok {
337339
case readStackOutputs:
338-
outs, err = p.readStackReference(req.Args)
340+
outs, err = p.readStackReference(ctx, req.Args)
339341
case readStackResourceOutputs:
340342
outs, err = p.readStackResourceOutputs(req.Args)
341343
case getResource:
@@ -363,7 +365,13 @@ func (p *builtinProvider) SignalCancellation(context.Context) error {
363365
return nil
364366
}
365367

366-
func (p *builtinProvider) readStackReference(inputs resource.PropertyMap) (resource.PropertyMap, error) {
368+
func (p *builtinProvider) readStackReference(
369+
ctx context.Context, inputs resource.PropertyMap,
370+
) (resource.PropertyMap, error) {
371+
tracer := otel.Tracer("pulumi-cli")
372+
ctx, span := cmdutil.StartSpan(ctx, tracer, "builtinProvider.readStackReference")
373+
defer span.End()
374+
367375
name, ok := inputs["name"]
368376
contract.Assertf(ok, "missing required property 'name'")
369377
contract.Assertf(name.IsString(), "expected 'name' to be a string")
@@ -376,7 +384,7 @@ func (p *builtinProvider) readStackReference(inputs resource.PropertyMap) (resou
376384
// than failing outright.
377385
var decryptionErr error
378386
outputs, err := p.backendClient.GetStackOutputs(
379-
p.context,
387+
ctx,
380388
name.StringValue(),
381389
func(e error) error { decryptionErr = e; return nil },
382390
)

‎pkg/resource/stack/deployment.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ import (
2727
"strings"
2828

2929
fxs "github.com/pgavlin/fx/v2/slices"
30+
"go.opentelemetry.io/otel"
31+
3032
"github.com/pulumi/pulumi/pkg/v3/resource/deploy"
3133
"github.com/pulumi/pulumi/pkg/v3/resource/stack/migrate"
3234
"github.com/pulumi/pulumi/pkg/v3/secrets"
@@ -37,6 +39,7 @@ import (
3739
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/asset"
3840
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/config"
3941
"github.com/pulumi/pulumi/sdk/v3/go/common/slice"
42+
"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
4043
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
4144
"github.com/pulumi/pulumi/sdk/v3/go/common/util/maputil"
4245
"github.com/santhosh-tekuri/jsonschema/v5"
@@ -404,6 +407,10 @@ func UnmarshalUntypedDeployment(
404407
ctx context.Context,
405408
deployment *apitype.UntypedDeployment,
406409
) (*apitype.DeploymentV3, error) {
410+
tracer := otel.Tracer("pulumi-cli")
411+
_, span := cmdutil.StartSpan(ctx, tracer, "stack.UnmarshalUntypedDeployment")
412+
defer span.End()
413+
407414
contract.Requiref(deployment != nil, "deployment", "must not be nil")
408415
switch {
409416
case deployment.Version > DeploymentSchemaVersionLatest:
@@ -454,6 +461,10 @@ func DeserializeUntypedDeployment(
454461
deployment *apitype.UntypedDeployment,
455462
secretsProv secrets.Provider,
456463
) (*deploy.Snapshot, error) {
464+
tracer := otel.Tracer("pulumi-cli")
465+
ctx, span := cmdutil.StartSpan(ctx, tracer, "stack.DeserializeUntypedDeployment")
466+
defer span.End()
467+
457468
v3deployment, err := UnmarshalUntypedDeployment(ctx, deployment)
458469
if err != nil {
459470
return nil, err
@@ -468,6 +479,10 @@ func DeserializeStackOutputs(
468479
deployment apitype.DeploymentV3,
469480
secretsProv secrets.Provider,
470481
) (resource.PropertyMap, error) {
482+
tracer := otel.Tracer("pulumi-cli")
483+
ctx, span := cmdutil.StartSpan(ctx, tracer, "stack.DeserializeStackOutputs")
484+
defer span.End()
485+
471486
// Find the root stack resource in the deployment.
472487
var stackResource *apitype.ResourceV3
473488
for i := range deployment.Resources {
@@ -501,6 +516,10 @@ func DeserializeDeploymentV3(
501516
deployment apitype.DeploymentV3,
502517
secretsProv secrets.Provider,
503518
) (*deploy.Snapshot, error) {
519+
tracer := otel.Tracer("pulumi-cli")
520+
ctx, span := cmdutil.StartSpan(ctx, tracer, "stack.DeserializeDeploymentV3")
521+
defer span.End()
522+
504523
// Unpack the versions.
505524
manifest, err := deploy.DeserializeManifest(deployment.Manifest)
506525
if err != nil {

‎pkg/resource/stack/secrets.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,15 @@ import (
2222
"sync"
2323
"sync/atomic"
2424

25+
"go.opentelemetry.io/otel"
26+
2527
"github.com/pulumi/pulumi/pkg/v3/secrets"
2628
"github.com/pulumi/pulumi/pkg/v3/secrets/b64"
2729
"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
2830
"github.com/pulumi/pulumi/sdk/v3/go/common/env"
2931
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
3032
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/config"
33+
"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
3134
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
3235
)
3336

@@ -556,6 +559,10 @@ func BatchDecrypt[T any](
556559
sm secrets.Manager,
557560
fn func(context.Context, config.Decrypter) (T, error),
558561
) (result T, err error) {
562+
tracer := otel.Tracer("pulumi-cli")
563+
ctx, span := cmdutil.StartSpan(ctx, tracer, "stack.BatchDecrypt")
564+
defer span.End()
565+
559566
var dec config.Decrypter
560567

561568
if sm != nil {

0 commit comments

Comments
 (0)