Skip to content

Commit 3559561

Browse files
committed
fix(supervisor): address OTLP relay review feedback
Inject only the generic OTEL_EXPORTER_OTLP_ENDPOINT and OTEL_EXPORTER_OTLP_PROTOCOL as lowest-precedence defaults, and only while the relay is active. The supervisor now starts the relay before attach and signals it to the sandbox runtime through AgentSpec::otlp_relay_active. An endpoint in the creation request, the image ENV, or an exec environment suppresses the defaults, and a provider-supplied traces endpoint is removed. Harden the relay itself: - skip the exit flush and shutdown wait when the last export failed - count batches still pending on Drop as discarded and log them - validate the collector endpoint with the shared openshell-otel helper - accept Content-Encoding: identity - fix the 413 and shutdown 503 messages and the memory bound docs - narrow the attribution claim to resource attributes Document best-effort delivery and the 404 for metrics and logs. Signed-off-by: Roland Huß <[email protected]>
1 parent 53914d0 commit 3559561

19 files changed

Lines changed: 613 additions & 201 deletions

File tree

‎crates/openshell-core/src/sandbox_env.rs‎

Lines changed: 9 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -301,36 +301,23 @@ pub const OCI_IMAGE_USER: &str = "OPENSHELL_OCI_IMAGE_USER";
301301

302302
/// Standard OpenTelemetry environment variable for the OTLP exporter endpoint.
303303
///
304-
/// The sandbox runtime sets it in every workload process to
305-
/// [`OTLP_RELAY_ENDPOINT`] so agent SDKs export to the supervisor's relay,
306-
/// unless the sandbox creation request already names an OTLP endpoint
307-
/// (generic or traces-specific), in which case the caller's variables are
308-
/// left untouched. A per-exec environment is a developer's explicit
309-
/// instruction and still overrides it for that exec only.
304+
/// While the supervisor serves the OTLP relay, the sandbox runtime sets it in
305+
/// workload processes to [`OTLP_RELAY_ENDPOINT`] as a lowest-precedence
306+
/// default, so agent SDKs export to the relay without configuration. It is
307+
/// not set when the creation request, the image, or a per-exec environment
308+
/// names an endpoint, generic or traces-specific, and any signal-specific
309+
/// endpoint the workload sets later takes precedence over it in the SDK.
310310
pub const OTEL_EXPORTER_OTLP_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_ENDPOINT";
311311

312312
/// Standard OpenTelemetry environment variable for the OTLP exporter protocol.
313313
///
314314
/// Set to `http/protobuf` alongside [`OTEL_EXPORTER_OTLP_ENDPOINT`].
315315
pub const OTEL_EXPORTER_OTLP_PROTOCOL: &str = "OTEL_EXPORTER_OTLP_PROTOCOL";
316316

317-
/// Signal-specific endpoint variable. SDKs give it precedence over the
318-
/// generic endpoint, so it is set too (to the full traces URL) or an image
319-
/// that ships one would silently bypass the relay.
317+
/// Signal-specific endpoint variable. Never injected; a layer that sets it
318+
/// has chosen where traces go, so the relay default is left out.
320319
pub const OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT";
321320

322-
/// Signal-specific protocol variable, set alongside
323-
/// [`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`] for the same reason.
324-
pub const OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: &str = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL";
325-
326-
/// Standard OTLP compression variable. The relay refuses compressed bodies,
327-
/// so both this and the traces-specific variant are set to `none`.
328-
pub const OTEL_EXPORTER_OTLP_COMPRESSION: &str = "OTEL_EXPORTER_OTLP_COMPRESSION";
329-
330-
/// Signal-specific compression variable, see
331-
/// [`OTEL_EXPORTER_OTLP_COMPRESSION`].
332-
pub const OTEL_EXPORTER_OTLP_TRACES_COMPRESSION: &str = "OTEL_EXPORTER_OTLP_TRACES_COMPRESSION";
333-
334321
/// Reserved destination agent processes export OTLP to.
335322
///
336323
/// This address is never routed. A workload `connect()` to any non-loopback
@@ -348,10 +335,6 @@ pub const OTLP_RELAY_ADDR: &str = "192.0.0.8:4318";
348335
/// [`OTEL_EXPORTER_OTLP_ENDPOINT`].
349336
pub const OTLP_RELAY_ENDPOINT: &str = "http://192.0.0.8:4318";
350337

351-
/// The traces URL injected through [`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`],
352-
/// which unlike the generic endpoint names the full path.
353-
pub const OTLP_RELAY_TRACES_ENDPOINT: &str = "http://192.0.0.8:4318/v1/traces";
354-
355338
// The corporate upstream-proxy configuration deliberately has no reserved
356339
// environment variables: it travels on the supervisor's argv
357340
// (`--upstream-proxy` and friends), which a sandbox image cannot forge the
@@ -361,7 +344,7 @@ pub const OTLP_RELAY_TRACES_ENDPOINT: &str = "http://192.0.0.8:4318/v1/traces";
361344
mod otlp_relay_address_tests {
362345
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
363346

364-
use super::{OTLP_RELAY_ADDR, OTLP_RELAY_ENDPOINT, OTLP_RELAY_TRACES_ENDPOINT};
347+
use super::{OTLP_RELAY_ADDR, OTLP_RELAY_ENDPOINT};
365348

366349
#[test]
367350
fn relay_address_is_an_unroutable_non_loopback_label() {
@@ -381,10 +364,6 @@ mod otlp_relay_address_tests {
381364
"198.18.0.0/15 is the policy DNS synthetic answer pool"
382365
);
383366
assert_eq!(OTLP_RELAY_ENDPOINT, format!("http://{OTLP_RELAY_ADDR}"));
384-
assert_eq!(
385-
OTLP_RELAY_TRACES_ENDPOINT,
386-
format!("{OTLP_RELAY_ENDPOINT}/v1/traces")
387-
);
388367
}
389368
}
390369

‎crates/openshell-isolation-interface/src/lib.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ pub struct AgentSpec {
4848
pub timeout_secs: u64,
4949
/// Whether the entrypoint runs interactively (inherits the parent pgrp).
5050
pub interactive: bool,
51+
/// Whether the supervisor serves the OTLP relay for this sandbox. The
52+
/// boundary points workload SDKs at the relay only while it is active.
53+
pub otlp_relay_active: bool,
5154
}
5255

5356
pub mod contract;

‎crates/openshell-isolation-interface/tests/backend_conformance.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -354,6 +354,7 @@ fn sandbox_ctx() -> SandboxContext {
354354
workdir: None,
355355
timeout_secs: 0,
356356
interactive: false,
357+
otlp_relay_active: false,
357358
},
358359
identity: workload_identity(),
359360
}

‎crates/openshell-otel/src/lib.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,10 @@ pub fn resource_for(config: &OtlpExportConfig<'_>) -> Resource {
189189
}
190190

191191
/// Trim and validate an OTLP endpoint before handing it to an exporter.
192-
fn validated_endpoint(endpoint: &str) -> Result<(&str, http::Uri), SetupError> {
192+
///
193+
/// Every exporter that reads an operator-supplied endpoint uses this, so a
194+
/// value accepted for one signal is accepted for all of them.
195+
pub fn validated_endpoint(endpoint: &str) -> Result<(&str, http::Uri), SetupError> {
193196
let endpoint = endpoint.trim();
194197
if endpoint.is_empty() {
195198
return Err(SetupError::EmptyEndpoint);

‎crates/openshell-sandbox-backend/src/boundary_protocol.rs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1166,6 +1166,10 @@ pub struct AgentSpecWire {
11661166
pub workdir: Option<String>,
11671167
pub timeout_secs: u64,
11681168
pub interactive: bool,
1169+
/// Omitted while false so the request digest of a sandbox without a
1170+
/// relay is unchanged.
1171+
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
1172+
pub otlp_relay_active: bool,
11691173
}
11701174

11711175
impl From<AgentSpec> for AgentSpecWire {
@@ -1176,6 +1180,7 @@ impl From<AgentSpec> for AgentSpecWire {
11761180
workdir: spec.workdir,
11771181
timeout_secs: spec.timeout_secs,
11781182
interactive: spec.interactive,
1183+
otlp_relay_active: spec.otlp_relay_active,
11791184
}
11801185
}
11811186
}
@@ -1588,6 +1593,30 @@ mod tests {
15881593
);
15891594
}
15901595

1596+
#[test]
1597+
fn agent_spec_omits_an_inactive_relay_flag_on_the_wire() {
1598+
let mut spec = AgentSpecWire {
1599+
program: "/bin/true".to_string(),
1600+
args: Vec::new(),
1601+
workdir: None,
1602+
timeout_secs: 0,
1603+
interactive: false,
1604+
otlp_relay_active: false,
1605+
};
1606+
let encoded = serde_json::to_value(&spec).unwrap();
1607+
assert!(
1608+
encoded.get("otlp_relay_active").is_none(),
1609+
"a false flag keeps the encoding of earlier releases: {encoded}"
1610+
);
1611+
let decoded: AgentSpecWire = serde_json::from_value(encoded).unwrap();
1612+
assert_eq!(decoded, spec);
1613+
1614+
spec.otlp_relay_active = true;
1615+
let decoded: AgentSpecWire =
1616+
serde_json::from_value(serde_json::to_value(&spec).unwrap()).unwrap();
1617+
assert!(decoded.otlp_relay_active);
1618+
}
1619+
15911620
#[test]
15921621
fn request_round_trips_and_redacts_secrets() {
15931622
let request = RequestEnvelope {
@@ -1603,6 +1632,7 @@ mod tests {
16031632
workdir: Some("/sandbox".to_string()),
16041633
timeout_secs: 5,
16051634
interactive: false,
1635+
otlp_relay_active: false,
16061636
},
16071637
policy: Box::new(SandboxPolicyWire::from(SandboxPolicy {
16081638
version: 1,
@@ -1777,6 +1807,7 @@ mod tests {
17771807
workdir: None,
17781808
timeout_secs: 5,
17791809
interactive: false,
1810+
otlp_relay_active: false,
17801811
},
17811812
policy: Box::new(SandboxPolicyWire::from(SandboxPolicy {
17821813
version: 1,

‎crates/openshell-sandbox-backend/src/runtime.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2895,6 +2895,7 @@ mod tests {
28952895
workdir: Some("/sandbox".to_string()),
28962896
timeout_secs: 5,
28972897
interactive: false,
2898+
otlp_relay_active: false,
28982899
},
28992900
identity: openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new(
29002901
10_001,

‎crates/openshell-sandbox/src/boundary_exec.rs‎

Lines changed: 64 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -36,20 +36,23 @@ pub struct LocalBoundaryExec {
3636
provider_credentials: ProviderCredentialState,
3737
user_environment: HashMap<String, String>,
3838
runtime: Arc<crate::boundary_io::BoundaryRuntimeState>,
39+
otlp_relay_active: bool,
3940
#[cfg(target_os = "linux")]
4041
launcher: crate::linux::workload_launcher::WorkloadLauncher,
4142
}
4243

4344
impl LocalBoundaryExec {
4445
/// Construct the executor owned by an active sandbox boundary.
4546
#[must_use]
47+
#[allow(clippy::too_many_arguments)]
4648
pub fn new(
4749
policy: SandboxPolicy,
4850
base_workdir: Option<String>,
4951
ca_file_paths: Option<Arc<(std::path::PathBuf, std::path::PathBuf)>>,
5052
provider_credentials: ProviderCredentialState,
5153
user_environment: HashMap<String, String>,
5254
runtime: Arc<crate::boundary_io::BoundaryRuntimeState>,
55+
otlp_relay_active: bool,
5356
#[cfg(target_os = "linux")] launcher: crate::linux::workload_launcher::WorkloadLauncher,
5457
) -> Self {
5558
Self {
@@ -59,6 +62,7 @@ impl LocalBoundaryExec {
5962
provider_credentials,
6063
user_environment,
6164
runtime,
65+
otlp_relay_active,
6266
#[cfg(target_os = "linux")]
6367
launcher,
6468
}
@@ -182,18 +186,17 @@ impl LocalBoundaryExec {
182186
}
183187
}
184188
crate::process::strip_proxy_env_std(&mut command);
185-
// Point OTel SDKs at the supervisor's trace relay, after the creation
189+
// Point OTel SDKs at the supervisor's trace relay after the creation
186190
// request's environment and the provider variables, matching the main
187-
// process, and only when the creation request named no endpoint of
188-
// its own. A per-exec `spec.env` value applied below still wins for
189-
// that exec only.
190-
if let Some(relay_env) =
191-
crate::child_env::otlp_relay_env_vars_unless_configured(&self.user_environment)
192-
{
193-
for (key, value) in relay_env {
194-
command.env(key, value);
195-
}
196-
}
191+
// process. An endpoint in the creation request, the image, or this
192+
// exec's `spec.env` suppresses it, and any other `spec.env` value
193+
// applied below still wins for this exec only.
194+
crate::child_env::apply_otlp_relay_env(
195+
&mut command,
196+
self.otlp_relay_active,
197+
&self.user_environment,
198+
&spec.env,
199+
);
197200
for (key, value) in &spec.env {
198201
if !key.starts_with(crate::process::RESERVED_ENV_PREFIX) {
199202
command.env(key, value);
@@ -693,6 +696,10 @@ mod tests {
693696
use tokio::io::{AsyncReadExt, AsyncWriteExt};
694697

695698
fn executor() -> LocalBoundaryExec {
699+
executor_with_relay(true)
700+
}
701+
702+
fn executor_with_relay(otlp_relay_active: bool) -> LocalBoundaryExec {
696703
let (launcher, listener) =
697704
crate::linux::workload_launcher::start().expect("start test workload launcher");
698705
std::thread::spawn(move || {
@@ -728,6 +735,7 @@ mod tests {
728735
),
729736
HashMap::new(),
730737
crate::boundary_io::BoundaryRuntimeState::new(),
738+
otlp_relay_active,
731739
launcher,
732740
)
733741
}
@@ -822,12 +830,22 @@ mod tests {
822830
Some("http/protobuf")
823831
);
824832

833+
assert_eq!(
834+
env_of(
835+
&command,
836+
openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
837+
),
838+
None,
839+
"only the generic default is injected"
840+
);
841+
825842
// A per-exec environment is a developer's explicit instruction and
826-
// is applied last (contracts/sandbox-environment.md).
843+
// is applied last (contracts/sandbox-environment.md). An endpoint in
844+
// it suppresses the whole relay default, protocol included.
827845
let command = executor
828846
.command(&spec(vec![(
829847
openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT.to_string(),
830-
"http://127.0.0.1:4318".to_string(),
848+
"http://127.0.0.1:4317".to_string(),
831849
)]))
832850
.expect("resolve exec");
833851
assert_eq!(
@@ -836,8 +854,40 @@ mod tests {
836854
openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT
837855
)
838856
.as_deref(),
839-
Some("http://127.0.0.1:4318")
857+
Some("http://127.0.0.1:4317")
840858
);
859+
assert_eq!(
860+
env_of(
861+
&command,
862+
openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL
863+
),
864+
None
865+
);
866+
}
867+
868+
#[test]
869+
fn exec_children_get_no_relay_endpoint_while_the_relay_is_inactive() {
870+
let command = executor_with_relay(false)
871+
.command(&ExecSpec {
872+
program: "/bin/true".to_string(),
873+
args: Vec::new(),
874+
shell: None,
875+
runtime_helper: None,
876+
env: Vec::new(),
877+
workdir: None,
878+
pty: false,
879+
})
880+
.expect("resolve exec");
881+
for key in [
882+
openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT,
883+
openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL,
884+
] {
885+
assert_eq!(
886+
env_of(&command, key),
887+
None,
888+
"{key} injected without a relay"
889+
);
890+
}
841891
}
842892

843893
#[tokio::test]

‎crates/openshell-sandbox/src/boundary_server.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2995,6 +2995,7 @@ mod linux {
29952995
provider_env,
29962996
ca_file_paths,
29972997
Some(boundary_runtime.clone()),
2998+
spec.otlp_relay_active,
29982999
))
29993000
.map_err(|error| format!("start process supervisor leaf: {error:?}"))?;
30003001
let signaler = spawned.signaler();
@@ -3862,6 +3863,7 @@ mod linux {
38623863
workdir: Some("/sandbox".to_string()),
38633864
timeout_secs: 0,
38643865
interactive: true,
3866+
otlp_relay_active: false,
38653867
})
38663868
.expect("resolve scratch command");
38673869

@@ -4955,6 +4957,7 @@ mod linux {
49554957
workdir: None,
49564958
timeout_secs: 5,
49574959
interactive: false,
4960+
otlp_relay_active: false,
49584961
},
49594962
vm_identity_test_policy(user, group),
49604963
None,
@@ -5409,6 +5412,7 @@ mod linux {
54095412
workdir: None,
54105413
timeout_secs: 60,
54115414
interactive: false,
5415+
otlp_relay_active: false,
54125416
};
54135417

54145418
assert!(matches!(
@@ -5685,6 +5689,7 @@ mod linux {
56855689
workdir: None,
56865690
timeout_secs: 5,
56875691
interactive: false,
5692+
otlp_relay_active: false,
56885693
};
56895694
let wire_policy = SandboxPolicyWire::from(policy.clone());
56905695
let process = Arc::new(

0 commit comments

Comments
 (0)