// Unless explicitly stated otherwise all files in this repository are licensed

// under the Apache License Version 2.0.

// This product includes software developed at Datadog (https://www.datadoghq.com/).

// Copyright 2016 Datadog, Inc.

package tracer

import (

"bufio"

"cmp"

"context"

"encoding/binary"

"encoding/json"

"errors"

"fmt"

"io"

"maps"

"math"

"net"

"net/http"

"net/url"

"os"

"path/filepath"

"runtime"

"runtime/debug"

"strconv"

"strings"

"sync/atomic"

"time"

"golang.org/x/mod/semver"

pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace"

"github.com/tinylib/msgp/msgp"

"github.com/DataDog/dd-trace-go/v2/ddtrace/ext"

"github.com/DataDog/dd-trace-go/v2/internal"

appsecconfig "github.com/DataDog/dd-trace-go/v2/internal/appsec/config"

"github.com/DataDog/dd-trace-go/v2/internal/civisibility/constants"

internalconfig "github.com/DataDog/dd-trace-go/v2/internal/config"

"github.com/DataDog/dd-trace-go/v2/internal/env"

"github.com/DataDog/dd-trace-go/v2/internal/globalconfig"

llmobsconfig "github.com/DataDog/dd-trace-go/v2/internal/llmobs/config"

"github.com/DataDog/dd-trace-go/v2/internal/locking"

"github.com/DataDog/dd-trace-go/v2/internal/log"

"github.com/DataDog/dd-trace-go/v2/internal/namingschema"

"github.com/DataDog/dd-trace-go/v2/internal/normalizer"

"github.com/DataDog/dd-trace-go/v2/internal/orchestrion"

"github.com/DataDog/dd-trace-go/v2/internal/processtags"

"github.com/DataDog/dd-trace-go/v2/internal/stableconfig"

"github.com/DataDog/dd-trace-go/v2/internal/telemetry"

"github.com/DataDog/dd-trace-go/v2/internal/version"

"github.com/DataDog/datadog-go/v5/statsd"

)

const (

envLLMObsEnabled = "DD_LLMOBS_ENABLED"

envLLMObsMlApp = "DD_LLMOBS_ML_APP"

envLLMObsAgentlessEnabled = "DD_LLMOBS_AGENTLESS_ENABLED"

envLLMObsProjectName = "DD_LLMOBS_PROJECT_NAME"

)

var contribIntegrations = map[string]struct {

name string // user readable name for startup logs

imported bool // true if the user has imported the integration

}{

"github.com/99designs/gqlgen": {"gqlgen", false},

"github.com/aws/aws-sdk-go": {"AWS SDK", false},

"github.com/aws/aws-sdk-go-v2": {"AWS SDK v2", false},

"github.com/bradfitz/gomemcache": {"Memcache", false},

"cloud.google.com/go/pubsub.v1": {"Pub/Sub", false},

"cloud.google.com/go/pubsub/v2": {"Pub/Sub v2", false},

"github.com/confluentinc/confluent-kafka-go": {"Kafka (confluent)", false},

"github.com/confluentinc/confluent-kafka-go/v2": {"Kafka (confluent) v2", false},

"database/sql": {"SQL", false},

"github.com/dimfeld/httptreemux/v5": {"HTTP Treemux", false},

"github.com/elastic/go-elasticsearch/v6": {"Elasticsearch v6", false},

"github.com/emicklei/go-restful/v3": {"go-restful v3", false},

"github.com/gin-gonic/gin": {"Gin", false},

"github.com/globalsign/mgo": {"MongoDB (mgo)", false},

"github.com/go-chi/chi": {"chi", false},

"github.com/go-chi/chi/v5": {"chi v5", false},

"github.com/go-pg/pg/v10": {"go-pg v10", false},

"github.com/go-redis/redis": {"Redis", false},

"github.com/go-redis/redis/v7": {"Redis v7", false},

"github.com/go-redis/redis/v8": {"Redis v8", false},

"go.mongodb.org/mongo-driver": {"MongoDB", false},

"github.com/gocql/gocql": {"Cassandra", false},

"github.com/gofiber/fiber/v2": {"Fiber", false},

"github.com/gomodule/redigo": {"Redigo", false},

"google.golang.org/api": {"Google API", false},

"google.golang.org/grpc": {"gRPC", false},

"github.com/gorilla/mux": {"Gorilla Mux", false},

"gorm.io/gorm.v1": {"Gorm v1", false},

"github.com/graph-gophers/graphql-go": {"Graph Gophers GraphQL", false},

"github.com/graphql-go/graphql": {"GraphQL-Go GraphQL", false},

"github.com/hashicorp/consul/api": {"Consul", false},

"github.com/hashicorp/vault/api": {"Vault", false},

"github.com/jackc/pgx/v5": {"PGX", false},

"github.com/jmoiron/sqlx": {"SQLx", false},

"github.com/julienschmidt/httprouter": {"HTTP Router", false},

"k8s.io/client-go/kubernetes": {"Kubernetes", false},

"github.com/labstack/echo/v4": {"echo v4", false},

"log/slog": {"log/slog", false},

"github.com/mark3labs/mcp-go": {"MCP", false},

"github.com/miekg/dns": {"miekg/dns", false},

"net/http": {"HTTP", false},

"gopkg.in/olivere/elastic.v5": {"Elasticsearch v5", false},

"github.com/redis/go-redis/v9": {"Redis v9", false},

"github.com/redis/rueidis": {"Rueidis", false},

"github.com/rs/zerolog": {"Zerolog", false},

"github.com/segmentio/kafka-go": {"Kafka v0", false},

"github.com/IBM/sarama": {"IBM sarama", false},

"github.com/Shopify/sarama": {"Shopify sarama", false},

"github.com/sirupsen/logrus": {"Logrus", false},

"github.com/syndtr/goleveldb": {"LevelDB", false},

"github.com/tidwall/buntdb": {"BuntDB", false},

"github.com/twitchtv/twirp": {"Twirp", false},

"github.com/twmb/franz-go": {"franz-go", false},

"github.com/uptrace/bun": {"Bun", false},

"github.com/urfave/negroni": {"Negroni", false},

"github.com/valyala/fasthttp": {"FastHTTP", false},

"github.com/valkey-io/valkey-go": {"Valkey", false},

}

var (

// defaultSocketDSD specifies the socket path to use for connecting to the statsd server.

// Replaced in tests

defaultSocketDSD = "/var/run/datadog/dsd.socket"

// defaultStatsdPort specifies the default port to use for connecting to the statsd server.

defaultStatsdPort = "8125"

)

// Supported trace protocols.

const (

traceProtocolV04 = internalconfig.TraceProtocolV04

traceProtocolV1 = internalconfig.TraceProtocolV1

)

// config holds the tracer configuration.

type config struct {

// internalConfig holds a reference to the global configuration singleton.

internalConfig *internalconfig.Config

// appsecStartOptions controls the options used when starting appsec features.

appsecStartOptions []appsecconfig.StartOption

// agent holds the capabilities of the agent and determines some

// of the behaviour of the tracer. It is updated atomically so that

// periodic polling can refresh it without locking the hot path.

agent atomicAgentFeatures

// agentInfoPollInterval overrides the default polling interval for /info.

// A zero value uses defaultAgentInfoPollInterval.

agentInfoPollInterval time.Duration

// integrations reports if the user has instrumented a Datadog integration and

// if they have a version of the library available to integrate.

integrations map[string]integrationConfig

// sendRetries is the number of times a trace or CI Visibility payload send is retried upon

// failure.

sendRetries int

// universalVersion, reports whether span service name and config service name

// should match to set application version tag. False by default

universalVersion bool

// sampler specifies the sampler that will be used for sampling traces.

sampler RateSampler

// globalTags holds a set of tags that will be automatically applied to

// all spans.

globalTags dynamicConfig[map[string]any]

// ddTransport specifies the Datadog transport used to send msgpack traces and stats to the agent.

ddTransport ddTransport

// httpClientTimeout specifies the timeout for the HTTP client.

httpClientTimeout time.Duration

// propagator propagates span context cross-process

propagator Propagator

// httpClient specifies the HTTP client to be used by the agent's transport.

httpClient *http.Client

// logger specifies the logger to use when printing errors. If not specified, the "log" package

// will be used.

logger Logger

// dogstatsdAddr specifies the address to connect for sending metrics to the

// Datadog Agent. If not set, it defaults to "localhost:8125" or to the

// combination of the environment variables DD_AGENT_HOST and DD_DOGSTATSD_PORT.

dogstatsdAddr string

// statsdClient is set when a user provides a custom statsd client for tracking metrics

// associated with the runtime and the tracer.

statsdClient internal.StatsdClient

// spanRules contains user-defined rules to determine the sampling rate to apply

// to a single span without affecting the entire trace

spanRules []SamplingRule

// traceRules contains user-defined rules to determine the sampling rate to apply

// to the entire trace if any spans satisfy the criteria

traceRules []SamplingRule

// tickChan specifies a channel which will receive the time every time the tracer must flush.

// It defaults to time.Ticker; replaced in tests.

tickChan <-chan time.Time

// enabled reports whether tracing is enabled.

enabled dynamicConfig[bool]

// enableHostnameDetection specifies whether the tracer should enable hostname detection.

enableHostnameDetection bool

// orchestrionCfg holds Orchestrion (aka auto-instrumentation) configuration.

// Only used for telemetry currently.

orchestrionCfg orchestrionConfig

// traceSampleRules holds the trace sampling rules

traceSampleRules dynamicConfig[[]SamplingRule]

// headerAsTags holds the header as tags configuration.

headerAsTags dynamicConfig[[]string]

// dynamicInstrumentationEnabled controls whether the target application can

// be modified by Dynamic Instrumentation / Live Debugger. If the value is

// explicitly set to false (as opposed to starting as false by default), then

// it is frozen -- it cannot be overwritten by Remote Config.

dynamicInstrumentationEnabled dynamicConfig[bool]

// ciVisibilityAgentless controls if the tracer is loaded with CI Visibility agentless mode. default false

ciVisibilityAgentless bool

// ciVisibilityNoopTracer controls if CI Visibility must set a wrapper to behave like a noop tracer. default false

ciVisibilityNoopTracer bool

// tracingAsTransport specifies whether the tracer is running in transport-only mode, where traces are only sent when other products request it.

tracingAsTransport bool

// llmobs contains the LLM Observability config

llmobs llmobsconfig.Config

}

// orchestrionConfig contains Orchestrion configuration.

type (

orchestrionConfig struct {

// Enabled indicates whether this tracer was instanciated via Orchestrion.

Enabled bool `json:"enabled"`

// Metadata holds Orchestrion specific metadata (e.g orchestrion version, mode (toolexec or manual) etc..)

Metadata *orchestrionMetadata `json:"metadata,omitempty"`

}

orchestrionMetadata struct {

// Version is the version of the orchestrion tool that was used to instrument the application.

Version string `json:"version,omitempty"`

}

)

// StartOption represents a function that can be provided as a parameter to Start.

type StartOption func(*config)

// newConfig renders the tracer configuration based on defaults, environment variables

// and passed user opts.

func newConfig(opts ...StartOption) (*config, error) {

c := new(config)

c.internalConfig = internalconfig.CreateNew()

// If this was built with a recent-enough version of Orchestrion, force the orchestrion config to

// the baked-in values. We do this early so that opts can be used to override the baked-in values,

// which is necessary for some tests to work properly.

c.orchestrionCfg.Enabled = orchestrion.Enabled()

if orchestrion.Version != "" {

c.orchestrionCfg.Metadata = &orchestrionMetadata{Version: orchestrion.Version}

}

c.sampler = NewAllSampler()

c.httpClientTimeout = time.Second * 10 // 10 seconds

if v := env.Get("OTEL_LOGS_EXPORTER"); v != "" {

log.Warn("OTEL_LOGS_EXPORTER is not supported")

}

if internal.BoolEnv("DD_TRACE_ANALYTICS_ENABLED", false) {

globalconfig.SetAnalyticsRate(1.0)

}

if c.internalConfig.ReportHostname() {

if err := c.internalConfig.HostnameLookupError(); err != nil {

return c, fmt.Errorf("unable to look up hostname: %s", err.Error())

}

}

c.headerAsTags = newDynamicConfig("trace_header_tags", nil, setHeaderTags, equalSlice[string])

if v := env.Get("DD_TRACE_HEADER_TAGS"); v != "" {

c.headerAsTags.update(strings.Split(v, ","), telemetry.OriginEnvVar)

// Required to ensure that the startup header tags are set on reset.

c.headerAsTags.setStartup(c.headerAsTags.get())

}

if v := getDDorOtelConfig("resourceAttributes"); v != "" {

tags := internal.ParseTagString(v)

internal.CleanGitMetadataTags(tags)

for key, val := range tags {

WithGlobalTag(key, val)(c)

}

// TODO: should we track the origin of these tags individually?

c.globalTags.setOrigin(telemetry.OriginEnvVar)

}

c.enabled = newDynamicConfig("tracing_enabled", internal.BoolVal(getDDorOtelConfig("enabled"), true), func(_ bool) bool { return true }, equal[bool])

if _, ok := env.Lookup("DD_TRACE_ENABLED"); ok {

c.enabled.setOrigin(telemetry.OriginEnvVar)

}

if compatMode := env.Get("DD_TRACE_CLIENT_HOSTNAME_COMPAT"); compatMode != "" {

if semver.IsValid(compatMode) {

c.enableHostnameDetection = semver.Compare(semver.MajorMinor(compatMode), "v1.66") <= 0

} else {

log.Warn("ignoring DD_TRACE_CLIENT_HOSTNAME_COMPAT, invalid version %q", compatMode)

}

}

dynamicInstrumentationEnabledDefault, origin, _ := stableconfig.Bool("DD_DYNAMIC_INSTRUMENTATION_ENABLED", false)

c.dynamicInstrumentationEnabled = newDynamicConfig(

"dynamic_instrumentation_enabled",

dynamicInstrumentationEnabledDefault,

func(bool) bool {

// NOTE: the side effects of changes are performed in onRemoteConfigUpdate.

return true

}, /* apply */

equal[bool],

)

c.dynamicInstrumentationEnabled.setOrigin(origin)

namingschema.LoadFromEnv()

// LLM Observability config

c.llmobs = llmobsconfig.Config{

Enabled: internal.BoolEnv(envLLMObsEnabled, false),

MLApp: env.Get(envLLMObsMlApp),

AgentlessEnabled: llmobsAgentlessEnabledFromEnv(),

ProjectName: env.Get(envLLMObsProjectName),

}

for _, fn := range opts {

if fn == nil {

continue

}

fn(c)

}

rawAgentURL := c.internalConfig.RawAgentURL()

if c.httpClient == nil || orchestrion.Enabled() {

if orchestrion.Enabled() && c.httpClient != nil {

// Make sure we don't create http client traces from inside the tracer by using our http client

// TODO(eliott.bouhana): remove once dd:no-span is implemented

log.Debug("Orchestrion is enabled, but a custom HTTP client was provided to tracer.Start. This is not supported and will be ignored.")

}

if rawAgentURL != nil && rawAgentURL.Scheme == "unix" {

// If we're connecting over UDS we can just rely on the agent to provide the hostname

log.Debug("connecting to agent over unix, do not set hostname on any traces")

c.httpClient = internal.UDSClient(rawAgentURL.Path, cmp.Or(c.httpClientTimeout, defaultHTTPTimeout))

} else {

c.httpClient = internal.DefaultHTTPClient(c.httpClientTimeout, false)

}

}

WithGlobalTag(ext.RuntimeID, globalconfig.RuntimeID())(c)

globalTags := c.globalTags.get()

if c.internalConfig.Env() == "" {

if v, ok := globalTags["env"]; ok {

if e, ok := v.(string); ok {

c.internalConfig.SetEnv(e, c.globalTags.Origin(), internalconfig.ProductTracer)

}

}

}

if c.internalConfig.Version() == "" {

if v, ok := globalTags["version"]; ok {

if ver, ok := v.(string); ok {

c.internalConfig.SetVersion(ver, c.globalTags.Origin(), internalconfig.ProductTracer)

}

}

}

svcIsUserDefined := true

if c.internalConfig.ServiceName() == "" {

if v, ok := globalTags["service"]; ok {

if s, ok := v.(string); ok {

c.internalConfig.SetServiceName(s, c.globalTags.Origin(), internalconfig.ProductTracer)

globalconfig.SetServiceName(s)

}

} else {

// There is not an explicit service set, default to binary name.

// In this case, don't set a global service name so the contribs continue using their defaults.

c.internalConfig.SetServiceName(filepath.Base(os.Args[0]), internalconfig.OriginDefault, internalconfig.ProductTracer)

svcIsUserDefined = false

}

} else {

globalconfig.SetServiceName(c.internalConfig.ServiceName())

}

processtags.SetServiceNameTag(c.internalConfig.ServiceName(), svcIsUserDefined)

if c.ddTransport == nil {

agentURL := c.internalConfig.AgentURL().String()

traceURL, headers := resolveTraceTransport(c.internalConfig)

c.ddTransport = newHTTPTransport(traceURL, agentURL+statsAPIPath, c.httpClient, headers)

}

if c.propagator == nil {

c.propagator = NewPropagator(&PropagatorConfig{

MaxTagsHeaderLen: c.internalConfig.MaxTagsHeaderLen(),

})

}

if c.logger != nil {

log.UseLogger(c.logger)

}

if c.internalConfig.Debug() {

log.SetLevel(log.LevelDebug)

}

// Check if CI Visibility mode is enabled

if c.internalConfig.CIVisibilityEnabled() {

c.httpClientTimeout = time.Second * 45 // Increase timeout up to 45 seconds (same as other tracers in CIVis mode)

c.internalConfig.SetLogStartup(false, internalconfig.OriginCalculated) // If we are in CI Visibility mode we don't want to log the startup to stdout to avoid polluting the output

ciTransport := newCiVisibilityTransport(c) // Create a default CI Visibility Transport

c.ddTransport = ciTransport // Replace the default transport with the CI Visibility transport

c.ciVisibilityAgentless = ciTransport.agentless

c.ciVisibilityNoopTracer = internal.BoolEnv(constants.CIVisibilityUseNoopTracer, false)

}

// if using stdout or traces are disabled or we are in ci visibility agentless mode, agent is disabled

agentDisabled := c.internalConfig.LogToStdout() || !c.enabled.get() || c.ciVisibilityAgentless

agentURL := c.internalConfig.AgentURL()

af := loadAgentFeatures(agentDisabled, agentURL, c.httpClient)

c.agent.store(af)

// If the agent doesn't support the v1 protocol, downgrade to v0.4

// Also downgrade if CSS is disabled, as v1 is not compatible without CSS.

if c.internalConfig.TraceProtocol() == traceProtocolV04 || !af.v1ProtocolAvailable || !c.internalConfig.StatsComputationEnabled() {

c.internalConfig.SetTraceProtocol(traceProtocolV04, internalconfig.OriginCalculated)

if t, ok := c.ddTransport.(*httpTransport); ok && t.traceURL == agentURL.String()+tracesAPIPathV1 {

t.traceURL = agentURL.String() + tracesAPIPath

}

}

info, ok := debug.ReadBuildInfo()

if !ok {

c.loadContribIntegrations([]*debug.Module{})

} else {

c.loadContribIntegrations(info.Deps)

}

if c.statsdClient == nil {

// configure statsd client

addr := resolveDogstatsdAddr(c.dogstatsdAddr, af, defaultSocketDSD)

globalconfig.SetDogstatsdAddr(addr)

c.dogstatsdAddr = addr

}

// Re-initialize the globalTags config with the value constructed from the environment and start options

// This allows persisting the initial value of globalTags for future resets and updates.

globalTagsOrigin := c.globalTags.Origin()

c.initGlobalTags(c.globalTags.get(), globalTagsOrigin)

if tracingEnabled, _, _ := stableconfig.Bool("DD_APM_TRACING_ENABLED", true); !tracingEnabled {

apmTracingDisabled(c)

}

// Update the llmobs config with stuff needed from the tracer.

c.llmobs.TracerConfig = llmobsconfig.TracerConfig{

DDTags: c.globalTags.get(),

Env: c.internalConfig.Env(),

Service: c.internalConfig.ServiceName(),

Version: c.internalConfig.Version(),

AgentURL: c.internalConfig.AgentURL(),

APIKey: c.internalConfig.APIKey(),

APPKey: env.Get("DD_APP_KEY"),

HTTPClient: c.httpClient,

Site: env.Get("DD_SITE"),

}

c.llmobs.AgentFeatures = llmobsconfig.AgentFeatures{

EVPProxyV2: af.evpProxyV2,

}

// Set global 128-bits trace ID generation variable

traceID128BitEnabled.Store(c.internalConfig.TraceID128BitEnabled())

return c, nil

}

func llmobsAgentlessEnabledFromEnv() *bool {

v, ok := internal.BoolEnvNoDefault(envLLMObsAgentlessEnabled)

if !ok {

return nil

}

return &v

}

func apmTracingDisabled(c *config) {

// Enable tracing as transport layer mode

// This means to stop sending trace metrics, send one trace per minute and those force-kept by other products

// using the tracer as transport layer for their data. And finally adding the _dd.apm.enabled=0 tag to all traces

// to let the backend know that it needs to keep APM UI disabled.

c.internalConfig.SetGlobalSampleRate(1.0, internalconfig.OriginCalculated)

c.internalConfig.SetTraceRateLimitPerSecond(1.0/60, internalconfig.OriginCalculated)

c.tracingAsTransport = true

WithGlobalTag("_dd.apm.enabled", 0)(c)

// Disable runtime metrics. In `tracingAsTransport` mode, we'll still

// tell the agent we computed them, so it doesn't do it either.

c.internalConfig.SetRuntimeMetricsEnabled(false, internalconfig.OriginCalculated)

c.internalConfig.SetRuntimeMetricsV2Enabled(false, internalconfig.OriginCalculated)

}

// resolveTraceTransport returns the trace URL and headers for the Datadog

// agent transport. In OTLP export mode the ddTransport is not used for trace

// sending (otlpTransport handles that), but it may still be used for stats

// and agent discovery, so it always points at the DD agent.

func resolveTraceTransport(cfg *internalconfig.Config) (traceURL string, headers map[string]string) {

agentURL := cfg.AgentURL().String()

traceURL = agentURL + tracesAPIPath

if cfg.TraceProtocol() == traceProtocolV1 {

traceURL = agentURL + tracesAPIPathV1

}

return traceURL, datadogHeaders()

}

// resolveDogstatsdAddr resolves the Dogstatsd address to use with the following

// priority order:

// 1. Explicitly configured address via WithDogstatsdAddr.

// 2. Environment variables DD_DOGSTATSD_HOST (or DD_AGENT_HOST) and DD_DOGSTATSD_PORT.

// 3. Auto-discovery: UDS socket at /var/run/datadog/dsd.socket, agent-reported port,

// or the default localhost:8125.

func resolveDogstatsdAddr(configAddr string, af agentFeatures, socketDSDPath string) string {

// 1. User explicitly set address via WithDogstatsdAddr; honor it as-is.

if configAddr != "" {

return configAddr

}

// 2. Build address from dogstatsd-specific environment variables.

// DD_AGENT_HOST is only used as a host fallback when DD_DOGSTATSD_HOST or

// DD_DOGSTATSD_PORT is set — on its own it does not trigger this path.

envHost := env.Get("DD_DOGSTATSD_HOST")

envPort := env.Get("DD_DOGSTATSD_PORT")

if envHost != "" || envPort != "" {

host := envHost

if host == "" {

host = env.Get("DD_AGENT_HOST")

}

if host == "" {

host = defaultHostname

}

// For the port, prefer the env var, then the agent-reported port

// (loaded from the trace-agent /info endpoint), then the default.

port := envPort

if port == "" && af.StatsdPort != 0 {

port = strconv.Itoa(af.StatsdPort)

} else if port == "" {

port = defaultStatsdPort

}

return net.JoinHostPort(host, port)

}

// 3. No user configuration at all — auto-discover.

// Check for the UDS socket first; this is the preferred transport when available.

if _, err := os.Stat(socketDSDPath); err == nil {

return "unix://" + socketDSDPath

}

// For TCP, use DD_AGENT_HOST as the hostname if available, otherwise localhost.

host := env.Get("DD_AGENT_HOST")

if host == "" {

host = defaultHostname

}

// Use the agent-reported port if available. Agent features are loaded from

// the trace-agent, which may not be running — in that case StatsdPort is 0.

if af.StatsdPort != 0 {

return net.JoinHostPort(host, strconv.Itoa(af.StatsdPort))

}

// Fall back to default port 8125.

return net.JoinHostPort(host, defaultStatsdPort)

}

func newStatsdClient(c *config) (internal.StatsdClient, error) {

if c.statsdClient != nil {

return c.statsdClient, nil

}

return internal.NewStatsdClient(c.dogstatsdAddr, statsTags(c))

}

type integrationConfig struct {

Instrumented bool `json:"instrumented"` // indicates if the user has imported and used the integration

Available bool `json:"available"` // indicates if the user is using a library that can be used with DataDog integrations

Version string `json:"available_version"` // if available, indicates the version of the library the user has

}

// agentFeatures holds information about the trace-agent's capabilities.

// When running WithLambdaMode, a zero-value of this struct will be used

// as features.

type agentFeatures struct {

// DropP0s reports whether it's ok for the tracer to not send any

// P0 traces to the agent.

DropP0s bool

// Stats reports whether the agent can receive client-computed stats on

// the /v0.6/stats endpoint.

Stats bool

// StatsdPort specifies the Dogstatsd port as provided by the agent.

// If it's the default, it will be 0, which means 8125.

StatsdPort int

// featureFlags specifies all the feature flags reported by the trace-agent.

featureFlags map[string]struct{}

// peerTags specifies precursor tags to aggregate stats on when client stats is enabled

peerTags []string

// defaultEnv is the trace-agent's default env, used for stats calculation if no env override is present

defaultEnv string

// metaStructAvailable reports whether the trace-agent can receive spans with the `meta_struct` field.

metaStructAvailable bool

// obfuscationVersion reports the trace-agent's version of obfuscation logic. A value of 0 means this field wasn't present.

obfuscationVersion int

// spanEvents reports whether the trace-agent can receive spans with the `span_events` field.

spanEventsAvailable bool

// evpProxyV2 reports if the trace-agent can receive payloads on the /evp_proxy/v2 endpoint.

evpProxyV2 bool

// v1ProtocolAvailable reports whether the trace-agent and tracer are configured to use the v1 protocol.

v1ProtocolAvailable bool

// hasTelemetryProxy reports whether the trace-agent exposes the /telemetry/proxy/ endpoint.

// This is only true when the agent has telemetry forwarding enabled (the default).

// Notably, the Datadog Lambda extension does not expose this endpoint.

hasTelemetryProxy bool

// hasRemoteConfig reports whether the trace-agent has remote configuration

// enabled, as indicated by the presence of /v0.7/config in its endpoints.

hasRemoteConfig bool

// reachable reports whether the trace-agent was reachable at startup and

// responded successfully to the /info endpoint. When false, the agent may

// simply be unreachable due to a transient startup issue, so the telemetry

// client should still attempt the agent URL to avoid silently dropping data.

reachable bool

}

// HasFlag reports whether the agent has set the feat feature flag.

func (a *agentFeatures) HasFlag(feat string) bool {

_, ok := a.featureFlags[feat]

return ok

}

// atomicAgentFeatures wraps agentFeatures in an atomic pointer for lock-free

// reads on the hot path. All fields update atomically as a single consistent

// snapshot from one /info response.

type atomicAgentFeatures struct {

val atomic.Pointer[agentFeatures]

}

func (a *atomicAgentFeatures) load() agentFeatures {

if p := a.val.Load(); p != nil {

return *p

}

return agentFeatures{}

}

func (a *atomicAgentFeatures) store(f agentFeatures) {

a.val.Store(&f)

}

// update atomically reads the current snapshot, calls fn with it, and stores

// the result. fn may be called more than once if a concurrent store races the

// CAS; it must therefore be a pure transform with no observable side-effects.

func (a *atomicAgentFeatures) update(fn func(agentFeatures) agentFeatures) {

for {

old := a.val.Load()

var cur agentFeatures

if old != nil {

cur = *old

}

next := fn(cur)

if a.val.CompareAndSwap(old, &next) {

return

}

}

}

// errAgentFeaturesNotSupported is returned by fetchAgentFeatures when the

// agent does not expose the /info endpoint (pre-7.28.0 agents). Callers

// should treat this as "no capabilities known" at startup, or "keep the

// previous snapshot" during a poll.

var errAgentFeaturesNotSupported = errors.New("agent does not support /info")

// fetchAgentFeatures queries the trace-agent's /info endpoint and parses the

// response into an agentFeatures value. It returns an error if the request

// fails or the response cannot be decoded; the caller should retain the

// previous snapshot in that case. A 404 response returns

// errAgentFeaturesNotSupported so callers can distinguish it from a real

// network failure. The request is bound to ctx so callers can cancel it on

// shutdown.

func fetchAgentFeatures(ctx context.Context, agentURL *url.URL, httpClient *http.Client) (agentFeatures, error) {

req, err := http.NewRequestWithContext(ctx, http.MethodGet, agentURL.JoinPath("info").String(), nil)

if err != nil {

return agentFeatures{}, fmt.Errorf("creating /info request: %w", err)

}

resp, err := httpClient.Do(req)

if err != nil {

return agentFeatures{}, fmt.Errorf("loading features: %w", err)

}

defer resp.Body.Close()

if resp.StatusCode == http.StatusNotFound {

// agent is older than 7.28.0; /info not available

io.Copy(io.Discard, resp.Body) //nolint:errcheck // best-effort drain for connection reuse

return agentFeatures{}, errAgentFeaturesNotSupported

}

if resp.StatusCode != http.StatusOK {

return agentFeatures{}, fmt.Errorf("unexpected /info status: %d", resp.StatusCode)

}

type infoResponse struct {

Endpoints []string `json:"endpoints"`

ClientDropP0s bool `json:"client_drop_p0s"`

FeatureFlags []string `json:"feature_flags"`

PeerTags []string `json:"peer_tags"`

SpanMetaStruct bool `json:"span_meta_structs"`

ObfuscationVersion int `json:"obfuscation_version"`

SpanEvents bool `json:"span_events"`

Config struct {

StatsdPort int `json:"statsd_port"`

DefaultEnv string `json:"default_env"`

} `json:"config"`

}

var info infoResponse

if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&info); err != nil {

return agentFeatures{}, fmt.Errorf("decoding features: %w", err)

}

var features agentFeatures

features.DropP0s = info.ClientDropP0s

features.StatsdPort = info.Config.StatsdPort

features.defaultEnv = info.Config.DefaultEnv

features.metaStructAvailable = info.SpanMetaStruct

features.peerTags = info.PeerTags

features.obfuscationVersion = info.ObfuscationVersion

features.spanEventsAvailable = info.SpanEvents

for _, endpoint := range info.Endpoints {

switch endpoint {

case "/v0.6/stats":

features.Stats = true

case "/evp_proxy/v2/":

features.evpProxyV2 = true

case "/v1.0/traces":

features.v1ProtocolAvailable = true

case "/telemetry/proxy/":

features.hasTelemetryProxy = true

case "/v0.7/config":

features.hasRemoteConfig = true

}

}

features.featureFlags = make(map[string]struct{}, len(info.FeatureFlags))

for _, flag := range info.FeatureFlags {

features.featureFlags[flag] = struct{}{}

}

features.reachable = true

return features, nil

}

// loadAgentFeatures queries the trace-agent for its capabilities at startup and

// stores the result. It handles the agentDisabled case and logs errors.

func loadAgentFeatures(agentDisabled bool, agentURL *url.URL, httpClient *http.Client) agentFeatures {

if agentDisabled {

// there is no agent; all features off

return agentFeatures{}

}

features, err := fetchAgentFeatures(context.Background(), agentURL, httpClient)

if err != nil && !errors.Is(err, errAgentFeaturesNotSupported) {

log.Error("%s", err.Error())

}

return features

}

// agentEnabled reports whether the tracer should communicate with the agent.

// The agent is considered disabled in serverless (LogToStdout), when the

// tracer itself is disabled, or in CI visibility agentless mode.

func (c *config) agentEnabled() bool {

return !c.internalConfig.LogToStdout() && c.enabled.get() && !c.ciVisibilityAgentless

}

// MarkIntegrationImported labels the given integration as imported

func MarkIntegrationImported(integration string) bool {

s, ok := contribIntegrations[integration]

if !ok {

return false

}

s.imported = true

contribIntegrations[integration] = s

return true

}

func (c *config) loadContribIntegrations(deps []*debug.Module) {

integrations := map[string]integrationConfig{}

for _, s := range contribIntegrations {

integrations[s.name] = integrationConfig{

Instrumented: s.imported,

}

}

for _, d := range deps {

p := d.Path

s, ok := contribIntegrations[p]

if !ok {

continue

}

conf := integrations[s.name]

conf.Available = true

conf.Version = d.Version

integrations[s.name] = conf

}

c.integrations = integrations

}

// canComputeStats determines whether Client-Side Stats can be computed, which requires:

// - 'client_drop_p0' is enabled

// - Trace Agent exposes 'stats' endpoint

// - Stats Computation is enabled on the tracer (or has 'discovery' FF)

func (c *config) canComputeStats() bool {

a := c.agent.load()

return a.Stats && a.DropP0s && (c.internalConfig.HasFeature("discovery") || c.internalConfig.StatsComputationEnabled())

}

// canDropP0s determines whether P0 spans can be dropped.

// Currently equivalent to canComputeStats() as both capabilities are part

// of the Client-Side Stats feature and cannot be enabled independently.

func (c *config) canDropP0s() bool {

return c.canComputeStats()

}

func statsTags(c *config) []string {

tags := []string{

"lang:go",

"lang_version:" + runtime.Version(),

}

if v := c.internalConfig.Env(); v != "" {

tags = append(tags, "env:"+v)

}

if v := c.internalConfig.Hostname(); v != "" {

tags = append(tags, "host:"+v)

}

for k, v := range c.globalTags.get() {

if vstr, ok := v.(string); ok {

tags = append(tags, k+":"+vstr)

}

}

tags = append(tags, processtags.GlobalTags().Slice()...)

// globalconfig.StatsTags is shared with contrib statsd clients. Process

// tags are shared too; keep only tracer_version and service tracer-only.

globalconfig.SetStatsTags(tags)

tags = append(tags, "tracer_version:"+version.Tag)

if v := c.internalConfig.ServiceName(); v != "" {

tags = append(tags, "service:"+v)

}

return tags

}

// withNoopStats is used for testing to disable statsd client

func withNoopStats() StartOption {

return func(c *config) {

c.statsdClient = &statsd.NoOpClientDirect{}

}

}

// WithAppSecEnabled specifies whether AppSec features should be activated

// or not.

//

// By default, AppSec features are enabled if `DD_APPSEC_ENABLED` is set to a

// truthy value; and may be enabled by remote configuration if

// `DD_APPSEC_ENABLED` is not set at all.

//

// Using this option to explicitly disable appsec also prevents it from being

// remote activated.

func WithAppSecEnabled(enabled bool) StartOption {

mode := appsecconfig.ForcedOff

if enabled {

mode = appsecconfig.ForcedOn

}

return func(c *config) {

c.appsecStartOptions = append(c.appsecStartOptions, appsecconfig.WithEnablementMode(mode))

}

}

// WithFeatureFlags specifies a set of feature flags to enable. Please take into account

// that most, if not all features flags are considered to be experimental and result in

// unexpected bugs.

func WithFeatureFlags(feats ...string) StartOption {

return func(c *config) {

c.internalConfig.SetFeatureFlags(feats, telemetry.OriginCode)

log.Info("FEATURES enabled: %s", feats)

}

}

// WithLogger sets logger as the tracer's error printer.

// Diagnostic and startup tracer logs are prefixed to simplify the search within logs.

// If JSON logging format is required, it's possible to wrap tracer logs using an existing JSON logger with this

// function. To learn more about this possibility, please visit: https://github.com/DataDog/dd-trace-go/issues/2152#issuecomment-1790586933

func WithLogger(logger Logger) StartOption {

return func(c *config) {

c.logger = logger

}

}

// WithDebugStack can be used to globally enable or disable the collection of stack traces when

// spans finish with errors. It is enabled by default. This is a global version of the NoDebugStack

// FinishOption.

func WithDebugStack(enabled bool) StartOption {

return func(c *config) {

c.internalConfig.SetDebugStack(enabled, internalconfig.OriginCode)

}

}

// WithDebugMode enables debug mode on the tracer, resulting in more verbose logging.

func WithDebugMode(enabled bool) StartOption {

return func(c *config) {

c.internalConfig.SetDebug(enabled, telemetry.OriginCode)

}

}

// WithLambdaMode enables lambda mode on the tracer, for use with AWS Lambda.

// This option is only required if the the Datadog Lambda Extension is not

// running.

func WithLambdaMode(enabled bool) StartOption {

return func(c *config) {

c.internalConfig.SetLogToStdout(enabled, telemetry.OriginCode)

}

}

// WithSendRetries enables re-sending payloads that are not successfully

// submitted to the agent. This will cause the tracer to retry the send at

// most `retries` times.

func WithSendRetries(retries int) StartOption {

return func(c *config) {

c.sendRetries = retries

}

}

// WithRetryInterval sets the interval, in seconds, for retrying submitting payloads to the agent.

func WithRetryInterval(interval int) StartOption {

return func(c *config) {

c.internalConfig.SetRetryInterval(time.Duration(interval)*time.Second, telemetry.OriginCode)

}

}

// WithPropagator sets an alternative propagator to be used by the tracer.

func WithPropagator(p Propagator) StartOption {

return func(c *config) {

c.propagator = p

}

}

// WithService sets the default service name for the program.

func WithService(name string) StartOption {

return func(c *config) {

c.internalConfig.SetServiceName(name, internalconfig.OriginCode, internalconfig.ProductTracer)

globalconfig.SetServiceName(name)

}

}

// WithGlobalServiceName causes contrib libraries to use the global service name and not any locally defined service name.

// This is synonymous with `DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED`.

func WithGlobalServiceName(enabled bool) StartOption {

return func(_ *config) {

namingschema.SetRemoveIntegrationServiceNames(enabled)

}

}

// WithAgentAddr sets the address where the agent is located. The default is

// localhost:8126. It should contain both host and port.

func WithAgentAddr(addr string) StartOption {

return func(c *config) {

c.internalConfig.SetAgentURL(&url.URL{

Scheme: "http",

Host: addr,

}, telemetry.OriginCode)

}

}

// WithAgentURL sets the full trace agent URL

func WithAgentURL(agentURL string) StartOption {

return func(c *config) {

u, err := url.Parse(agentURL)

if err != nil {

var urlErr *url.Error

if errors.As(err, &urlErr) {

u, _ = url.Parse(urlErr.URL)

if u != nil {

urlErr.URL = u.Redacted()

log.Warn("Fail to parse Agent URL: %s", urlErr.Err)

return

}

log.Warn("Fail to parse Agent URL: %s", err.Error())

return

}

log.Warn("Fail to parse Agent URL")

return

}

switch u.Scheme {

case "http", "https":

c.internalConfig.SetAgentURL(&url.URL{

Scheme: u.Scheme,

Host: u.Host,

}, telemetry.OriginCode)

case "unix":

c.internalConfig.SetAgentURL(&url.URL{