Skip to content

Commit 412ae10

Browse files
authored
feat: otel process ctxt protobuf encoding (#1651)
Depends #1650 # What does this PR do? Follow-up of #1585 #1640 #1650 Adds the protobuf definition for the OTel process context and associated messages, and make the interface of the publisher higher-level by taking the new structured `ProcessContext` value instead of a raw bytes payload. # Motivation Since libdatadog is already taking care of some protobuf encoding, and it's supposedly faster and simpler to do here rather than on the side of each language runtime, it makes sense to offer an interface with a struct that is encoded by libdatadog. The process context will be a `ProcessContext`-based opaque pointer on the FFI side, with proper setters/getters. # Additional Notes I was not sure where to put the protobuf definition, as there are a bunch of protobuf-dedicated crates in libdatadog. I feel like the OTel process context is about tracing metadata, but I'm happy to move it elsewhere if it makes more sense. # How to test the change? Once the FFI lands, we'll be able to publish the context from a language runtime and check the whole process end-to-end. For now, I feel like an additional test would mostly test `prost`, which isn't very valuable. Co-authored-by: yann.hamdaoui <[email protected]>
1 parent 408a961 commit 412ae10

File tree

11 files changed

+529
-5
lines changed

11 files changed

+529
-5
lines changed

Cargo.lock

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

LICENSE-3rdparty.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
root_name: builder, build_common, tools, libdd-alloc, libdd-crashtracker, libdd-common, libdd-telemetry, libdd-ddsketch, libdd-crashtracker-ffi, libdd-common-ffi, datadog-ffe, datadog-ffe-ffi, datadog-ipc, datadog-ipc-macros, libdd-tinybytes, tarpc, tarpc-plugins, spawn_worker, cc_utils, libdd-library-config, libdd-library-config-ffi, datadog-live-debugger, libdd-data-pipeline, libdd-dogstatsd-client, libdd-trace-protobuf, libdd-trace-stats, libdd-trace-utils, libdd-trace-normalization, libdd-log, datadog-live-debugger-ffi, libdd-profiling, libdd-profiling-protobuf, libdd-profiling-ffi, libdd-data-pipeline-ffi, libdd-ddsketch-ffi, libdd-log-ffi, libdd-telemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, libdd-trace-obfuscation, datadog-tracer-flare, sidecar_mockgen, test_spawn_from_lib, bin_tests
1+
root_name: builder, build_common, tools, libdd-alloc, libdd-crashtracker, libdd-common, libdd-telemetry, libdd-ddsketch, libdd-crashtracker-ffi, libdd-common-ffi, datadog-ffe, datadog-ffe-ffi, datadog-ipc, datadog-ipc-macros, libdd-tinybytes, tarpc, tarpc-plugins, spawn_worker, cc_utils, libdd-library-config, libdd-trace-protobuf, libdd-library-config-ffi, datadog-live-debugger, libdd-data-pipeline, libdd-dogstatsd-client, libdd-trace-stats, libdd-trace-utils, libdd-trace-normalization, libdd-log, datadog-live-debugger-ffi, libdd-profiling, libdd-profiling-protobuf, libdd-profiling-ffi, libdd-data-pipeline-ffi, libdd-ddsketch-ffi, libdd-log-ffi, libdd-telemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, libdd-trace-obfuscation, datadog-tracer-flare, sidecar_mockgen, test_spawn_from_lib, bin_tests
22
third_party_libraries:
33
- package_name: addr2line
44
package_version: 0.24.2

libdd-library-config/Cargo.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,15 @@ bench = false
1717
[dependencies]
1818
serde = { version = "1.0", features = ["derive"] }
1919
serde_yaml = "0.9.34"
20+
prost = "0.14.1"
2021
anyhow = "1.0"
2122

2223
rand = "0.8.3"
2324
rmp = "0.8.14"
2425
rmp-serde = "1.3.0"
2526

27+
libdd-trace-protobuf = { version = "1.0.0", path = "../libdd-trace-protobuf" }
28+
2629
[dev-dependencies]
2730
tempfile = { version = "3.3" }
2831
serial_test = "3.2"

libdd-library-config/src/otel_process_ctx.rs

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ pub mod linux {
3333
process::{getpid, set_virtual_memory_region_name, Pid},
3434
};
3535

36+
use libdd_trace_protobuf::opentelemetry::proto::common::v1::ProcessContext;
37+
use prost::Message;
38+
3639
/// Current version of the process context format
3740
pub const PROCESS_CTX_VERSION: u32 = 2;
3841
/// Signature bytes for identifying process context mappings
@@ -368,7 +371,12 @@ pub mod linux {
368371
/// is Undefined Behavior, for example. We assume that a forking runtime (such as Python or
369372
/// Ruby) that doesn't follow with an immediate `exec` is already "taking that risk", so to
370373
/// speak (typically, if no thread is ever spawned before the fork, things are mostly fine).
371-
pub fn publish(payload: Vec<u8>) -> anyhow::Result<()> {
374+
#[inline]
375+
pub fn publish(context: &ProcessContext) -> anyhow::Result<()> {
376+
publish_raw_payload(context.encode_to_vec())
377+
}
378+
379+
fn publish_raw_payload(payload: Vec<u8>) -> anyhow::Result<()> {
372380
let mut guard = lock_context_handle()?;
373381

374382
match &mut *guard {
@@ -497,7 +505,7 @@ pub mod linux {
497505
let payload_v1 = "example process context payload";
498506
let payload_v2 = "another example process context payload of different size";
499507

500-
super::publish(payload_v1.as_bytes().to_vec())
508+
super::publish_raw_payload(payload_v1.as_bytes().to_vec())
501509
.expect("couldn't publish the process context");
502510

503511
let header = read_process_context().expect("couldn't read back the process context");
@@ -522,7 +530,8 @@ pub mod linux {
522530
let published_at_ns_v1 = header.published_at_ns;
523531
// Ensure the clock advances so the updated timestamp is strictly greater
524532
std::thread::sleep(std::time::Duration::from_nanos(10));
525-
super::publish(payload_v2.as_bytes().to_vec())
533+
534+
super::publish_raw_payload(payload_v2.as_bytes().to_vec())
526535
.expect("couldn't update the process context");
527536

528537
let header = read_process_context().expect("couldn't read back the process context");
@@ -555,7 +564,7 @@ pub mod linux {
555564
fn unpublish_process_context() {
556565
let payload = "example process context payload";
557566

558-
super::publish(payload.as_bytes().to_vec())
567+
super::publish_raw_payload(payload.as_bytes().to_vec())
559568
.expect("couldn't publish the process context");
560569

561570
// The mapping must be discoverable right after publishing

libdd-trace-protobuf/build.rs

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,7 @@ fn generate_protobuf() {
261261
"src/pb/span.proto",
262262
"src/pb/stats.proto",
263263
"src/pb/remoteconfig.proto",
264+
"src/pb/opentelemetry/proto/common/v1/process_context.proto",
264265
"src/pb/idx/tracer_payload.proto",
265266
"src/pb/idx/span.proto",
266267
],
@@ -288,6 +289,23 @@ fn generate_protobuf() {
288289
prepend_to_file(serde_uses, &output_path.join("pb.rs"));
289290
prepend_to_file(serde_uses, &output_path.join("remoteconfig.rs"));
290291
prepend_to_file(serde_uses, &output_path.join("pb.idx.rs"));
292+
293+
// We vendored a few OTel protobuf definitions, which requires their own copyright header,
294+
// although they thankfully have the same Apache license.
295+
let otel_license = "// Copyright 2019, OpenTelemetry Authors
296+
// SPDX-License-Identifier: Apache-2.0
297+
298+
"
299+
.as_bytes();
300+
301+
prepend_to_file(
302+
otel_license,
303+
&output_path.join("opentelemetry.proto.resource.v1.rs"),
304+
);
305+
prepend_to_file(
306+
otel_license,
307+
&output_path.join("opentelemetry.proto.common.v1.rs"),
308+
);
291309
}
292310

293311
#[cfg(feature = "generate-protobuf")]

libdd-trace-protobuf/src/_includes.rs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,20 @@
22
// SPDX-License-Identifier: Apache-2.0
33

44
// This file is @generated by prost-build.
5+
pub mod opentelemetry {
6+
pub mod proto {
7+
pub mod common {
8+
pub mod v1 {
9+
include!("opentelemetry.proto.common.v1.rs");
10+
}
11+
}
12+
pub mod resource {
13+
pub mod v1 {
14+
include!("opentelemetry.proto.resource.v1.rs");
15+
}
16+
}
17+
}
18+
}
519
pub mod pb {
620
include!("pb.rs");
721
pub mod idx {
Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
// Copyright 2019, OpenTelemetry Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
// This file is @generated by prost-build.
5+
/// Represents any type of attribute value. AnyValue may contain a
6+
/// primitive value such as a string or integer or it may contain an arbitrary nested
7+
/// object containing arrays, key-value lists and primitives.
8+
#[derive(Clone, PartialEq, ::prost::Message)]
9+
pub struct AnyValue {
10+
/// The value is one of the listed fields. It is valid for all values to be unspecified
11+
/// in which case this AnyValue is considered to be "empty".
12+
#[prost(oneof = "any_value::Value", tags = "1, 2, 3, 4, 5, 6, 7, 8")]
13+
pub value: ::core::option::Option<any_value::Value>,
14+
}
15+
/// Nested message and enum types in `AnyValue`.
16+
pub mod any_value {
17+
/// The value is one of the listed fields. It is valid for all values to be unspecified
18+
/// in which case this AnyValue is considered to be "empty".
19+
#[derive(Clone, PartialEq, ::prost::Oneof)]
20+
pub enum Value {
21+
#[prost(string, tag = "1")]
22+
StringValue(::prost::alloc::string::String),
23+
#[prost(bool, tag = "2")]
24+
BoolValue(bool),
25+
#[prost(int64, tag = "3")]
26+
IntValue(i64),
27+
#[prost(double, tag = "4")]
28+
DoubleValue(f64),
29+
#[prost(message, tag = "5")]
30+
ArrayValue(super::ArrayValue),
31+
#[prost(message, tag = "6")]
32+
KvlistValue(super::KeyValueList),
33+
#[prost(bytes, tag = "7")]
34+
BytesValue(::prost::alloc::vec::Vec<u8>),
35+
/// Reference to the string value in ProfilesDictionary.string_table.
36+
///
37+
/// Note: This is currently used exclusively in the Profiling signal.
38+
/// Implementers of OTLP receivers for signals other than Profiling should
39+
/// treat the presence of this value as a non-fatal issue.
40+
/// Log an error or warning indicating an unexpected field intended for the
41+
/// Profiling signal and process the data as if this value were absent or
42+
/// empty, ignoring its semantic content for the non-Profiling signal.
43+
///
44+
/// Status: \[Development\]
45+
#[prost(int32, tag = "8")]
46+
StringValueRef(i32),
47+
}
48+
}
49+
/// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message
50+
/// since oneof in AnyValue does not allow repeated fields.
51+
#[derive(Clone, PartialEq, ::prost::Message)]
52+
pub struct ArrayValue {
53+
/// Array of values. The array may be empty (contain 0 elements).
54+
#[prost(message, repeated, tag = "1")]
55+
pub values: ::prost::alloc::vec::Vec<AnyValue>,
56+
}
57+
/// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message
58+
/// since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need
59+
/// a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to
60+
/// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches
61+
/// are semantically equivalent.
62+
#[derive(Clone, PartialEq, ::prost::Message)]
63+
pub struct KeyValueList {
64+
/// A collection of key/value pairs of key-value pairs. The list may be empty (may
65+
/// contain 0 elements).
66+
///
67+
/// The keys MUST be unique (it is not allowed to have more than one
68+
/// value with the same key).
69+
/// The behavior of software that receives duplicated keys can be unpredictable.
70+
#[prost(message, repeated, tag = "1")]
71+
pub values: ::prost::alloc::vec::Vec<KeyValue>,
72+
}
73+
/// Represents a key-value pair that is used to store Span attributes, Link
74+
/// attributes, etc.
75+
#[derive(Clone, PartialEq, ::prost::Message)]
76+
pub struct KeyValue {
77+
/// The key name of the pair.
78+
/// key_ref MUST NOT be set if key is used.
79+
#[prost(string, tag = "1")]
80+
pub key: ::prost::alloc::string::String,
81+
/// The value of the pair.
82+
#[prost(message, optional, tag = "2")]
83+
pub value: ::core::option::Option<AnyValue>,
84+
/// Reference to the string key in ProfilesDictionary.string_table.
85+
/// key MUST NOT be set if key_ref is used.
86+
///
87+
/// Note: This is currently used exclusively in the Profiling signal.
88+
/// Implementers of OTLP receivers for signals other than Profiling should
89+
/// treat the presence of this key as a non-fatal issue.
90+
/// Log an error or warning indicating an unexpected field intended for the
91+
/// Profiling signal and process the data as if this value were absent or
92+
/// empty, ignoring its semantic content for the non-Profiling signal.
93+
///
94+
/// Status: \[Development\]
95+
#[prost(int32, tag = "3")]
96+
pub key_ref: i32,
97+
}
98+
/// InstrumentationScope is a message representing the instrumentation scope information
99+
/// such as the fully qualified name and version.
100+
#[derive(Clone, PartialEq, ::prost::Message)]
101+
pub struct InstrumentationScope {
102+
/// A name denoting the Instrumentation scope.
103+
/// An empty instrumentation scope name means the name is unknown.
104+
#[prost(string, tag = "1")]
105+
pub name: ::prost::alloc::string::String,
106+
/// Defines the version of the instrumentation scope.
107+
/// An empty instrumentation scope version means the version is unknown.
108+
#[prost(string, tag = "2")]
109+
pub version: ::prost::alloc::string::String,
110+
/// Additional attributes that describe the scope. \[Optional\].
111+
/// Attribute keys MUST be unique (it is not allowed to have more than one
112+
/// attribute with the same key).
113+
/// The behavior of software that receives duplicated keys can be unpredictable.
114+
#[prost(message, repeated, tag = "3")]
115+
pub attributes: ::prost::alloc::vec::Vec<KeyValue>,
116+
/// The number of attributes that were discarded. Attributes
117+
/// can be discarded because their keys are too long or because there are too many
118+
/// attributes. If this value is 0, then no attributes were dropped.
119+
#[prost(uint32, tag = "4")]
120+
pub dropped_attributes_count: u32,
121+
}
122+
/// A reference to an Entity.
123+
/// Entity represents an object of interest associated with produced telemetry: e.g spans, metrics, profiles, or logs.
124+
///
125+
/// Status: \[Development\]
126+
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
127+
pub struct EntityRef {
128+
/// The Schema URL, if known. This is the identifier of the Schema that the entity data
129+
/// is recorded in. To learn more about Schema URL see
130+
/// <https://opentelemetry.io/docs/specs/otel/schemas/#schema-url>
131+
///
132+
/// This schema_url applies to the data in this message and to the Resource attributes
133+
/// referenced by id_keys and description_keys.
134+
/// TODO: discuss if we are happy with this somewhat complicated definition of what
135+
/// the schema_url applies to.
136+
///
137+
/// This field obsoletes the schema_url field in ResourceMetrics/ResourceSpans/ResourceLogs.
138+
#[prost(string, tag = "1")]
139+
pub schema_url: ::prost::alloc::string::String,
140+
/// Defines the type of the entity. MUST not change during the lifetime of the entity.
141+
/// For example: "service" or "host". This field is required and MUST not be empty
142+
/// for valid entities.
143+
#[prost(string, tag = "2")]
144+
pub r#type: ::prost::alloc::string::String,
145+
/// Attribute Keys that identify the entity.
146+
/// MUST not change during the lifetime of the entity. The Id must contain at least one attribute.
147+
/// These keys MUST exist in the containing {message}.attributes.
148+
#[prost(string, repeated, tag = "3")]
149+
pub id_keys: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
150+
/// Descriptive (non-identifying) attribute keys of the entity.
151+
/// MAY change over the lifetime of the entity. MAY be empty.
152+
/// These attribute keys are not part of entity's identity.
153+
/// These keys MUST exist in the containing {message}.attributes.
154+
#[prost(string, repeated, tag = "4")]
155+
pub description_keys: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
156+
}
157+
/// ProcessContext represents the payload for the process context sharing mechanism.
158+
///
159+
/// This message is designed to be published by OpenTelemetry SDKs via a memory-mapped
160+
/// region, allowing external readers (such as the OpenTelemetry eBPF Profiler) to
161+
/// discover and read resource attributes from instrumented processes without requiring
162+
/// direct integration or process activity.
163+
#[derive(Clone, PartialEq, ::prost::Message)]
164+
pub struct ProcessContext {
165+
/// The resource attributes describing this process.
166+
///
167+
/// Attribute keys MUST be unique (it is not allowed to have more than one
168+
/// attribute with the same key). The behavior of software that receives
169+
/// duplicated keys can be unpredictable.
170+
///
171+
/// Attributes SHOULD follow OpenTelemetry semantic conventions where applicable.
172+
/// See: <https://opentelemetry.io/docs/specs/semconv/>
173+
#[prost(message, optional, tag = "1")]
174+
pub resource: ::core::option::Option<super::super::resource::v1::Resource>,
175+
/// Additional attributes to share with external readers that are not part of
176+
/// the standard Resource. \[Optional\]
177+
///
178+
/// This field allows publishers to include supplementary key-value pairs that
179+
/// may be useful for external readers but are not part of the SDK's configured
180+
/// Resource.
181+
///
182+
/// Consider adding any keys here to the profiles semantic conventions in
183+
/// <https://opentelemetry.io/docs/specs/semconv/general/profiles/>
184+
#[prost(message, repeated, tag = "2")]
185+
pub extra_attributes: ::prost::alloc::vec::Vec<KeyValue>,
186+
}
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Copyright 2019, OpenTelemetry Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
// This file is @generated by prost-build.
5+
/// Resource information.
6+
#[derive(Clone, PartialEq, ::prost::Message)]
7+
pub struct Resource {
8+
/// Set of attributes that describe the resource.
9+
/// Attribute keys MUST be unique (it is not allowed to have more than one
10+
/// attribute with the same key).
11+
/// The behavior of software that receives duplicated keys can be unpredictable.
12+
#[prost(message, repeated, tag = "1")]
13+
pub attributes: ::prost::alloc::vec::Vec<super::super::common::v1::KeyValue>,
14+
/// The number of dropped attributes. If the value is 0, then
15+
/// no attributes were dropped.
16+
#[prost(uint32, tag = "2")]
17+
pub dropped_attributes_count: u32,
18+
/// Set of entities that participate in this Resource.
19+
///
20+
/// Note: keys in the references MUST exist in attributes of this message.
21+
///
22+
/// Status: \[Development\]
23+
#[prost(message, repeated, tag = "3")]
24+
pub entity_refs: ::prost::alloc::vec::Vec<super::super::common::v1::EntityRef>,
25+
}

0 commit comments

Comments
 (0)