Skip to content

Commit a6632d4

Browse files
committed
feat(gateway): point sandbox agents at the OTLP relay
When the gateway has a relay exporter, `CreateSandbox` sets `OTEL_EXPORTER_OTLP_ENDPOINT` to the reserved relay address and `OTEL_EXPORTER_OTLP_PROTOCOL` to `http/protobuf` in the sandbox spec's environment. A caller-supplied endpoint is left untouched together with its protocol, so an agent pointed at its own collector through policy-approved egress keeps working. The injection runs after the template merge and before the final size recheck, so it is covered by the existing environment cap and reaches every driver through the declared environment path. The values persist in the stored spec: restarts launch from the stored record and inherit them, and `sandbox get` shows where an agent's traces go. This replaces the supervisor-side injection the relay used when it shared the agent's process tree. Tests cover the injection rule and both create outcomes with and without an exporter on the gateway. Refs NVIDIA#2641 Signed-off-by: Roland Huß <[email protected]>
1 parent 7be6abb commit a6632d4

2 files changed

Lines changed: 158 additions & 4 deletions

File tree

‎crates/openshell-server/src/grpc/sandbox.rs‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -553,6 +553,11 @@ async fn handle_create_sandbox_inner(
553553
*policy = validate_and_canonicalize_policy(policy.clone())?;
554554
}
555555

556+
// Point the workload's OTel SDK at the supervisor relay when this gateway
557+
// can accept relayed telemetry. The values persist in the stored spec, so
558+
// restarts inherit them and `sandbox get` shows where traces go.
559+
inject_otel_relay_environment(&mut spec.environment, state.otel_relay_exporter.is_some());
560+
556561
// Process identity and MCP default materialization can increase the
557562
// protobuf size. Recheck the exact canonical spec before any middleware or
558563
// compute boundary can observe or persist it. The initial check remains
@@ -785,6 +790,31 @@ fn validate_create_sandbox_request_pre_io(
785790
Ok(())
786791
}
787792

793+
/// Set `OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_EXPORTER_OTLP_PROTOCOL` to the
794+
/// supervisor relay when `otel_relay_enabled`, unless the caller already chose
795+
/// an endpoint. A caller-supplied endpoint is left untouched together with its
796+
/// protocol, since both describe their collector rather than ours. Returns
797+
/// whether anything was injected.
798+
fn inject_otel_relay_environment(
799+
environment: &mut HashMap<String, String>,
800+
otel_relay_enabled: bool,
801+
) -> bool {
802+
use openshell_core::sandbox_env::{
803+
OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_PROTOCOL, OTLP_RELAY_ENDPOINT,
804+
};
805+
if !otel_relay_enabled || environment.contains_key(OTEL_EXPORTER_OTLP_ENDPOINT) {
806+
return false;
807+
}
808+
environment.insert(
809+
OTEL_EXPORTER_OTLP_ENDPOINT.to_string(),
810+
OTLP_RELAY_ENDPOINT.to_string(),
811+
);
812+
environment
813+
.entry(OTEL_EXPORTER_OTLP_PROTOCOL.to_string())
814+
.or_insert_with(|| "http/protobuf".to_string());
815+
true
816+
}
817+
788818
fn validate_template_create_governance_spec(spec: &SandboxSpec) -> Result<(), Status> {
789819
if !spec.log_level.is_empty() {
790820
return Err(Status::invalid_argument(
@@ -4053,6 +4083,122 @@ mod tests {
40534083
);
40544084
}
40554085

4086+
// ---- OTLP relay environment ----
4087+
4088+
async fn test_server_state_with_otel_relay() -> Arc<ServerState> {
4089+
let mut state = test_server_state().await;
4090+
Arc::get_mut(&mut state)
4091+
.expect("test server state should be uniquely owned")
4092+
.otel_relay_exporter = Some(Arc::new(
4093+
crate::otel_relay::OtelRelayExporter::lazy_for_test(),
4094+
));
4095+
state
4096+
}
4097+
4098+
fn otel_create_request(
4099+
name: &str,
4100+
environment: HashMap<String, String>,
4101+
) -> CreateSandboxRequest {
4102+
CreateSandboxRequest {
4103+
name: name.to_string(),
4104+
spec: Some(SandboxSpec {
4105+
environment,
4106+
..Default::default()
4107+
}),
4108+
labels: HashMap::new(),
4109+
annotations: HashMap::new(),
4110+
workspace_scope: Some(openshell_core::proto::workspace_selector("default")),
4111+
await_main_process_attachment: false,
4112+
workload_template_name: String::new(),
4113+
request_id: String::new(),
4114+
}
4115+
}
4116+
4117+
#[test]
4118+
fn inject_otel_relay_environment_sets_endpoint_and_protocol_when_enabled() {
4119+
let mut env = HashMap::from([("HOME".to_string(), "/home/user".to_string())]);
4120+
assert!(inject_otel_relay_environment(&mut env, true));
4121+
assert_eq!(
4122+
env.get("OTEL_EXPORTER_OTLP_ENDPOINT").map(String::as_str),
4123+
Some(openshell_core::sandbox_env::OTLP_RELAY_ENDPOINT)
4124+
);
4125+
assert_eq!(
4126+
env.get("OTEL_EXPORTER_OTLP_PROTOCOL").map(String::as_str),
4127+
Some("http/protobuf")
4128+
);
4129+
assert_eq!(env.get("HOME").map(String::as_str), Some("/home/user"));
4130+
}
4131+
4132+
#[test]
4133+
fn inject_otel_relay_environment_is_a_noop_without_a_relay() {
4134+
let mut env = HashMap::new();
4135+
assert!(!inject_otel_relay_environment(&mut env, false));
4136+
assert!(env.is_empty());
4137+
}
4138+
4139+
#[test]
4140+
fn inject_otel_relay_environment_keeps_a_caller_supplied_endpoint() {
4141+
let mut env = HashMap::from([(
4142+
"OTEL_EXPORTER_OTLP_ENDPOINT".to_string(),
4143+
"https://collector.example:4317".to_string(),
4144+
)]);
4145+
assert!(!inject_otel_relay_environment(&mut env, true));
4146+
assert_eq!(
4147+
env.get("OTEL_EXPORTER_OTLP_ENDPOINT").map(String::as_str),
4148+
Some("https://collector.example:4317")
4149+
);
4150+
assert!(
4151+
!env.contains_key("OTEL_EXPORTER_OTLP_PROTOCOL"),
4152+
"the caller's collector keeps the caller's protocol choice"
4153+
);
4154+
}
4155+
4156+
#[tokio::test]
4157+
async fn create_sandbox_injects_relay_endpoint_when_gateway_has_an_exporter() {
4158+
let state = test_server_state_with_otel_relay().await;
4159+
let created = handle_create_sandbox(
4160+
&state,
4161+
authed_request(otel_create_request("otel-relay", HashMap::new())),
4162+
)
4163+
.await
4164+
.expect("create should succeed")
4165+
.into_inner()
4166+
.sandbox
4167+
.expect("created sandbox");
4168+
4169+
let env = created.spec.expect("resolved sandbox spec").environment;
4170+
assert_eq!(
4171+
env.get("OTEL_EXPORTER_OTLP_ENDPOINT").map(String::as_str),
4172+
Some(openshell_core::sandbox_env::OTLP_RELAY_ENDPOINT)
4173+
);
4174+
assert_eq!(
4175+
env.get("OTEL_EXPORTER_OTLP_PROTOCOL").map(String::as_str),
4176+
Some("http/protobuf")
4177+
);
4178+
}
4179+
4180+
#[tokio::test]
4181+
async fn create_sandbox_leaves_environment_alone_without_an_exporter() {
4182+
let state = test_server_state().await;
4183+
let created = handle_create_sandbox(
4184+
&state,
4185+
authed_request(otel_create_request(
4186+
"no-relay",
4187+
HashMap::from([("HOME".to_string(), "/home/user".to_string())]),
4188+
)),
4189+
)
4190+
.await
4191+
.expect("create should succeed")
4192+
.into_inner()
4193+
.sandbox
4194+
.expect("created sandbox");
4195+
4196+
let env = created.spec.expect("resolved sandbox spec").environment;
4197+
assert_eq!(env.get("HOME").map(String::as_str), Some("/home/user"));
4198+
assert!(!env.contains_key("OTEL_EXPORTER_OTLP_ENDPOINT"));
4199+
assert!(!env.contains_key("OTEL_EXPORTER_OTLP_PROTOCOL"));
4200+
}
4201+
40564202
// ---- shell_escape ----
40574203

40584204
#[test]

‎crates/openshell-server/src/otel_relay.rs‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,17 @@ impl OtelRelayExporter {
3535
})
3636
}
3737

38+
/// An exporter over a lazy channel that performs no I/O until the first
39+
/// RPC, for tests that only need "a relay exporter is configured".
40+
#[cfg(test)]
41+
pub(crate) fn lazy_for_test() -> Self {
42+
Self {
43+
client: TraceServiceClient::new(
44+
Channel::from_static("http://127.0.0.1:1").connect_lazy(),
45+
),
46+
}
47+
}
48+
3849
/// Export raw protobuf-encoded `ExportTraceServiceRequest` bytes.
3950
pub async fn export_raw(&self, trace_data: Vec<u8>) -> Result<(), ExportError> {
4051
let request = ExportTraceServiceRequest::decode(trace_data.as_slice())
@@ -110,10 +121,7 @@ mod tests {
110121
async fn export_raw_rejects_undecodable_trace_bytes() {
111122
// A lazy channel performs no I/O, so the decode failure surfaces
112123
// before any RPC is attempted and no collector is needed.
113-
let channel = Channel::from_static("http://127.0.0.1:1").connect_lazy();
114-
let exporter = OtelRelayExporter {
115-
client: TraceServiceClient::new(channel),
116-
};
124+
let exporter = OtelRelayExporter::lazy_for_test();
117125
let err = exporter
118126
.export_raw(vec![0xff, 0xff, 0xff])
119127
.await

0 commit comments

Comments
 (0)