Skip to content

Commit 3e51e8f

Browse files
committed
feat(gateway): point sandbox agents at the OTLP relay
When the gateway has a relay exporter, `CreateSandbox` sets `OTEL_EXPORTER_OTLP_ENDPOINT` to the reserved relay address and `OTEL_EXPORTER_OTLP_PROTOCOL` to `http/protobuf` in the sandbox spec's environment. A caller-supplied endpoint is left untouched together with its protocol, so an agent pointed at its own collector through policy-approved egress keeps working. The injection runs after the template merge and before the final size recheck, so it is covered by the existing environment cap and reaches every driver through the declared environment path. The values persist in the stored spec: restarts launch from the stored record and inherit them, and `sandbox get` shows where an agent's traces go. This replaces the supervisor-side injection the relay used when it shared the agent's process tree. Tests cover the injection rule and both create outcomes with and without an exporter on the gateway. Refs NVIDIA#2641 Signed-off-by: Roland Huß <[email protected]>
1 parent e9eff28 commit 3e51e8f

2 files changed

Lines changed: 157 additions & 4 deletions

File tree

‎crates/openshell-server/src/grpc/sandbox.rs‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,11 @@ async fn handle_create_sandbox_inner(
434434
*policy = validate_and_canonicalize_policy(policy.clone())?;
435435
}
436436

437+
// Point the workload's OTel SDK at the supervisor relay when this gateway
438+
// can accept relayed telemetry. The values persist in the stored spec, so
439+
// restarts inherit them and `sandbox get` shows where traces go.
440+
inject_otel_relay_environment(&mut spec.environment, state.otel_relay_exporter.is_some());
441+
437442
// Process identity and MCP default materialization can increase the
438443
// protobuf size. Recheck the exact canonical spec before any middleware or
439444
// compute boundary can observe or persist it. The initial check remains
@@ -580,6 +585,31 @@ fn validate_create_sandbox_request_pre_io(
580585
Ok(())
581586
}
582587

588+
/// Set `OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_EXPORTER_OTLP_PROTOCOL` to the
589+
/// supervisor relay when `otel_relay_enabled`, unless the caller already chose
590+
/// an endpoint. A caller-supplied endpoint is left untouched together with its
591+
/// protocol, since both describe their collector rather than ours. Returns
592+
/// whether anything was injected.
593+
fn inject_otel_relay_environment(
594+
environment: &mut HashMap<String, String>,
595+
otel_relay_enabled: bool,
596+
) -> bool {
597+
use openshell_core::sandbox_env::{
598+
OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_PROTOCOL, OTLP_RELAY_ENDPOINT,
599+
};
600+
if !otel_relay_enabled || environment.contains_key(OTEL_EXPORTER_OTLP_ENDPOINT) {
601+
return false;
602+
}
603+
environment.insert(
604+
OTEL_EXPORTER_OTLP_ENDPOINT.to_string(),
605+
OTLP_RELAY_ENDPOINT.to_string(),
606+
);
607+
environment
608+
.entry(OTEL_EXPORTER_OTLP_PROTOCOL.to_string())
609+
.or_insert_with(|| "http/protobuf".to_string());
610+
true
611+
}
612+
583613
fn validate_template_create_governance_spec(spec: &SandboxSpec) -> Result<(), Status> {
584614
if !spec.log_level.is_empty() {
585615
return Err(Status::invalid_argument(
@@ -3246,6 +3276,121 @@ mod tests {
32463276
state
32473277
}
32483278

3279+
// ---- OTLP relay environment ----
3280+
3281+
async fn test_server_state_with_otel_relay() -> Arc<ServerState> {
3282+
let mut state = test_server_state().await;
3283+
Arc::get_mut(&mut state)
3284+
.expect("test server state should be uniquely owned")
3285+
.otel_relay_exporter = Some(Arc::new(
3286+
crate::otel_relay::OtelRelayExporter::lazy_for_test(),
3287+
));
3288+
state
3289+
}
3290+
3291+
fn otel_create_request(
3292+
name: &str,
3293+
environment: HashMap<String, String>,
3294+
) -> CreateSandboxRequest {
3295+
CreateSandboxRequest {
3296+
name: name.to_string(),
3297+
spec: Some(SandboxSpec {
3298+
environment,
3299+
..Default::default()
3300+
}),
3301+
labels: HashMap::new(),
3302+
annotations: HashMap::new(),
3303+
workspace_scope: Some(openshell_core::proto::workspace_selector("default")),
3304+
await_main_process_attachment: false,
3305+
workload_template_name: String::new(),
3306+
}
3307+
}
3308+
3309+
#[test]
3310+
fn inject_otel_relay_environment_sets_endpoint_and_protocol_when_enabled() {
3311+
let mut env = HashMap::from([("HOME".to_string(), "/home/user".to_string())]);
3312+
assert!(inject_otel_relay_environment(&mut env, true));
3313+
assert_eq!(
3314+
env.get("OTEL_EXPORTER_OTLP_ENDPOINT").map(String::as_str),
3315+
Some(openshell_core::sandbox_env::OTLP_RELAY_ENDPOINT)
3316+
);
3317+
assert_eq!(
3318+
env.get("OTEL_EXPORTER_OTLP_PROTOCOL").map(String::as_str),
3319+
Some("http/protobuf")
3320+
);
3321+
assert_eq!(env.get("HOME").map(String::as_str), Some("/home/user"));
3322+
}
3323+
3324+
#[test]
3325+
fn inject_otel_relay_environment_is_a_noop_without_a_relay() {
3326+
let mut env = HashMap::new();
3327+
assert!(!inject_otel_relay_environment(&mut env, false));
3328+
assert!(env.is_empty());
3329+
}
3330+
3331+
#[test]
3332+
fn inject_otel_relay_environment_keeps_a_caller_supplied_endpoint() {
3333+
let mut env = HashMap::from([(
3334+
"OTEL_EXPORTER_OTLP_ENDPOINT".to_string(),
3335+
"https://collector.example:4317".to_string(),
3336+
)]);
3337+
assert!(!inject_otel_relay_environment(&mut env, true));
3338+
assert_eq!(
3339+
env.get("OTEL_EXPORTER_OTLP_ENDPOINT").map(String::as_str),
3340+
Some("https://collector.example:4317")
3341+
);
3342+
assert!(
3343+
!env.contains_key("OTEL_EXPORTER_OTLP_PROTOCOL"),
3344+
"the caller's collector keeps the caller's protocol choice"
3345+
);
3346+
}
3347+
3348+
#[tokio::test]
3349+
async fn create_sandbox_injects_relay_endpoint_when_gateway_has_an_exporter() {
3350+
let state = test_server_state_with_otel_relay().await;
3351+
let created = handle_create_sandbox(
3352+
&state,
3353+
authed_request(otel_create_request("otel-relay", HashMap::new())),
3354+
)
3355+
.await
3356+
.expect("create should succeed")
3357+
.into_inner()
3358+
.sandbox
3359+
.expect("created sandbox");
3360+
3361+
let env = created.spec.expect("resolved sandbox spec").environment;
3362+
assert_eq!(
3363+
env.get("OTEL_EXPORTER_OTLP_ENDPOINT").map(String::as_str),
3364+
Some(openshell_core::sandbox_env::OTLP_RELAY_ENDPOINT)
3365+
);
3366+
assert_eq!(
3367+
env.get("OTEL_EXPORTER_OTLP_PROTOCOL").map(String::as_str),
3368+
Some("http/protobuf")
3369+
);
3370+
}
3371+
3372+
#[tokio::test]
3373+
async fn create_sandbox_leaves_environment_alone_without_an_exporter() {
3374+
let state = test_server_state().await;
3375+
let created = handle_create_sandbox(
3376+
&state,
3377+
authed_request(otel_create_request(
3378+
"no-relay",
3379+
HashMap::from([("HOME".to_string(), "/home/user".to_string())]),
3380+
)),
3381+
)
3382+
.await
3383+
.expect("create should succeed")
3384+
.into_inner()
3385+
.sandbox
3386+
.expect("created sandbox");
3387+
3388+
let env = created.spec.expect("resolved sandbox spec").environment;
3389+
assert_eq!(env.get("HOME").map(String::as_str), Some("/home/user"));
3390+
assert!(!env.contains_key("OTEL_EXPORTER_OTLP_ENDPOINT"));
3391+
assert!(!env.contains_key("OTEL_EXPORTER_OTLP_PROTOCOL"));
3392+
}
3393+
32493394
// ---- shell_escape ----
32503395

32513396
#[test]

‎crates/openshell-server/src/otel_relay.rs‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,17 @@ impl OtelRelayExporter {
3535
})
3636
}
3737

38+
/// An exporter over a lazy channel that performs no I/O until the first
39+
/// RPC, for tests that only need "a relay exporter is configured".
40+
#[cfg(test)]
41+
pub(crate) fn lazy_for_test() -> Self {
42+
Self {
43+
client: TraceServiceClient::new(
44+
Channel::from_static("http://127.0.0.1:1").connect_lazy(),
45+
),
46+
}
47+
}
48+
3849
/// Export raw protobuf-encoded `ExportTraceServiceRequest` bytes.
3950
pub async fn export_raw(&self, trace_data: Vec<u8>) -> Result<(), ExportError> {
4051
let request = ExportTraceServiceRequest::decode(trace_data.as_slice())
@@ -110,10 +121,7 @@ mod tests {
110121
async fn export_raw_rejects_undecodable_trace_bytes() {
111122
// A lazy channel performs no I/O, so the decode failure surfaces
112123
// before any RPC is attempted and no collector is needed.
113-
let channel = Channel::from_static("http://127.0.0.1:1").connect_lazy();
114-
let exporter = OtelRelayExporter {
115-
client: TraceServiceClient::new(channel),
116-
};
124+
let exporter = OtelRelayExporter::lazy_for_test();
117125
let err = exporter
118126
.export_raw(vec![0xff, 0xff, 0xff])
119127
.await

0 commit comments

Comments
 (0)