Skip to content

Commit 1dd45f8

Browse files
committed
feat(sandbox,router,cli): honor route timeout on streaming chunk-idle
The L7 inference proxy's per-chunk idle timeout was a hardcoded 120s, ignoring `route.timeout`. Workloads with SSE gaps longer than 120s (large prefills, tool-call-buffering parsers like qwen3_coder) were aborted even when the backend was healthy and the operator had explicitly raised `--timeout`. Derive the effective per-chunk idle timeout from `route.timeout`, floored at 120s to avoid regressing default behavior. Operators raise it with the existing `openshell inference set --timeout <secs>` flag — no new CLI surface. The floor keeps the current default intact when operators don't tune the route. - openshell-router: add `DEFAULT_CHUNK_IDLE_TIMEOUT` + `effective_chunk_idle_timeout()` in config.rs, plus a `chunk_idle_timeout` field on `StreamingProxyResponse`. Populated in both the live and mock response paths. - openshell-sandbox: proxy.rs reads `resp.chunk_idle_timeout` instead of the deleted constant. OCSF truncation event now includes the effective value so operators can see which timeout fired. - openshell-cli: clarify `--timeout` help text on `inference set` / `inference update` — also raises the streaming chunk-idle floor. - Tests: pure-function tests for `effective_chunk_idle_timeout` and integration tests exercising both the floor and the above-floor scaling on live + mock streaming paths. Refs: NVIDIA#866
1 parent 0d301d5 commit 1dd45f8

6 files changed

Lines changed: 210 additions & 11 deletions

File tree

‎crates/openshell-cli/src/main.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -958,6 +958,11 @@ enum InferenceCommands {
958958
no_verify: bool,
959959

960960
/// Request timeout in seconds for inference calls (0 = default 60s).
961+
///
962+
/// For streaming responses, this value also raises the sandbox proxy's
963+
/// per-chunk idle timeout (floored at 120s). Raise it above 120 when
964+
/// targeting long-prefill or tool-call–buffering models where gaps
965+
/// between SSE chunks can legitimately exceed 2 minutes.
961966
#[arg(long, default_value_t = 0)]
962967
timeout: u64,
963968
},
@@ -982,6 +987,11 @@ enum InferenceCommands {
982987
no_verify: bool,
983988

984989
/// Request timeout in seconds for inference calls (0 = default 60s, unchanged if omitted).
990+
///
991+
/// For streaming responses, this value also raises the sandbox proxy's
992+
/// per-chunk idle timeout (floored at 120s). Raise it above 120 when
993+
/// targeting long-prefill or tool-call–buffering models where gaps
994+
/// between SSE chunks can legitimately exceed 2 minutes.
985995
#[arg(long)]
986996
timeout: Option<u64>,
987997
},

‎crates/openshell-router/src/backend.rs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ pub struct ProxyResponse {
5252
pub struct StreamingProxyResponse {
5353
pub status: u16,
5454
pub headers: Vec<(String, String)>,
55+
/// Per-chunk idle timeout the caller should apply when reading from
56+
/// [`StreamingProxyResponse::next_chunk`]. Derived from the matched
57+
/// route's `timeout` via [`crate::config::effective_chunk_idle_timeout`].
58+
pub chunk_idle_timeout: std::time::Duration,
5559
/// Either a live response to stream from, or a pre-buffered body (for mock routes).
5660
body: StreamingBody,
5761
}
@@ -68,10 +72,11 @@ const COMMON_INFERENCE_REQUEST_HEADERS: [&str; 4] =
6872

6973
impl StreamingProxyResponse {
7074
/// Create from a fully-buffered [`ProxyResponse`] (for mock routes).
71-
pub fn from_buffered(resp: ProxyResponse) -> Self {
75+
pub fn from_buffered(resp: ProxyResponse, chunk_idle_timeout: std::time::Duration) -> Self {
7276
Self {
7377
status: resp.status,
7478
headers: resp.headers,
79+
chunk_idle_timeout,
7580
body: StreamingBody::Buffered(Some(resp.body)),
7681
}
7782
}
@@ -508,6 +513,7 @@ pub async fn proxy_to_backend_streaming(
508513
Ok(StreamingProxyResponse {
509514
status,
510515
headers: resp_headers,
516+
chunk_idle_timeout: crate::config::effective_chunk_idle_timeout(route.timeout),
511517
body: StreamingBody::Live(response),
512518
})
513519
}

‎crates/openshell-router/src/config.rs‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,29 @@ use crate::RouterError;
1111

1212
pub const DEFAULT_ROUTE_TIMEOUT: Duration = Duration::from_secs(60);
1313

14+
/// Minimum per-chunk idle timeout for streaming inference responses.
15+
///
16+
/// Streaming requests have no total-duration timeout by design (see
17+
/// `backend::send_backend_request_streaming`). Liveness is enforced purely
18+
/// by the proxy's per-chunk idle check: if no SSE chunk arrives within this
19+
/// window, the proxy closes the stream.
20+
///
21+
/// A route's `timeout` (settable via `openshell inference set --timeout`)
22+
/// is honored as the effective chunk-idle timeout for streaming calls,
23+
/// floored at this value so that lowering the route total-timeout below
24+
/// 120s cannot regress default streaming behavior. Reasoning models
25+
/// (nemotron-3-super, o1, o3) and tool-call–buffering parsers routinely
26+
/// produce gaps longer than 60s between SSE deltas; 120s is the safe
27+
/// default floor.
28+
pub const DEFAULT_CHUNK_IDLE_TIMEOUT: Duration = Duration::from_secs(120);
29+
30+
/// Effective per-chunk idle timeout for a streaming request on this route.
31+
///
32+
/// See [`DEFAULT_CHUNK_IDLE_TIMEOUT`] for the rationale behind the floor.
33+
pub fn effective_chunk_idle_timeout(route_timeout: Duration) -> Duration {
34+
route_timeout.max(DEFAULT_CHUNK_IDLE_TIMEOUT)
35+
}
36+
1437
#[derive(Debug, Clone, Deserialize)]
1538
pub struct RouterConfig {
1639
pub routes: Vec<RouteConfig>,
@@ -161,6 +184,40 @@ mod tests {
161184
use super::*;
162185
use std::io::Write;
163186

187+
#[test]
188+
fn chunk_idle_timeout_is_floored_when_route_timeout_is_low() {
189+
// Default route timeout (60s) is below the 120s chunk-idle floor.
190+
// Keeping the floor avoids regressing default streaming behavior
191+
// when operators don't set --timeout.
192+
assert_eq!(
193+
effective_chunk_idle_timeout(DEFAULT_ROUTE_TIMEOUT),
194+
DEFAULT_CHUNK_IDLE_TIMEOUT
195+
);
196+
assert_eq!(
197+
effective_chunk_idle_timeout(Duration::from_secs(10)),
198+
DEFAULT_CHUNK_IDLE_TIMEOUT
199+
);
200+
assert_eq!(
201+
effective_chunk_idle_timeout(Duration::from_secs(119)),
202+
DEFAULT_CHUNK_IDLE_TIMEOUT
203+
);
204+
}
205+
206+
#[test]
207+
fn chunk_idle_timeout_scales_with_route_timeout_above_floor() {
208+
// Operators who set --timeout above 120s expect streaming chunk-idle
209+
// to scale with their intent (e.g., long-context prefills,
210+
// tool-call-buffering parsers).
211+
assert_eq!(
212+
effective_chunk_idle_timeout(Duration::from_secs(300)),
213+
Duration::from_secs(300)
214+
);
215+
assert_eq!(
216+
effective_chunk_idle_timeout(Duration::from_secs(3600)),
217+
Duration::from_secs(3600)
218+
);
219+
}
220+
164221
#[test]
165222
fn load_from_file_valid_yaml_round_trip() {
166223
let yaml = r#"

‎crates/openshell-router/src/lib.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,10 @@ impl Router {
128128
if mock::is_mock_route(route) {
129129
info!(endpoint = %route.endpoint, "returning mock response (buffered)");
130130
let buffered = mock::mock_response(route, &normalized_source);
131-
return Ok(StreamingProxyResponse::from_buffered(buffered));
131+
return Ok(StreamingProxyResponse::from_buffered(
132+
buffered,
133+
config::effective_chunk_idle_timeout(route.timeout),
134+
));
132135
}
133136

134137
backend::proxy_to_backend_streaming(

‎crates/openshell-router/tests/backend_integration.rs‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -594,6 +594,134 @@ async fn streaming_proxy_completes_despite_exceeding_route_timeout() {
594594
assert!(total_bytes > 0, "should have received body chunks");
595595
}
596596

597+
/// StreamingProxyResponse exposes an effective per-chunk idle timeout derived
598+
/// from the route's total timeout. The sandbox proxy reads this field instead
599+
/// of a hardcoded constant, so honoring `openshell inference set --timeout` on
600+
/// streaming paths lives here.
601+
#[tokio::test]
602+
async fn streaming_response_chunk_idle_is_floored_at_default_for_low_route_timeout() {
603+
use std::time::Duration;
604+
605+
let mock_server = MockServer::start().await;
606+
Mock::given(method("POST"))
607+
.and(path("/v1/chat/completions"))
608+
.respond_with(
609+
ResponseTemplate::new(200)
610+
.append_header("content-type", "text/event-stream")
611+
.set_body_string("data: [DONE]\n\n"),
612+
)
613+
.mount(&mock_server)
614+
.await;
615+
616+
let router = Router::new().unwrap();
617+
let candidates = vec![ResolvedRoute {
618+
name: "inference.local".to_string(),
619+
endpoint: mock_server.uri(),
620+
model: "test-model".to_string(),
621+
api_key: "test-api-key".to_string(),
622+
protocols: vec!["openai_chat_completions".to_string()],
623+
auth: AuthHeader::Bearer,
624+
default_headers: Vec::new(),
625+
passthrough_headers: Vec::new(),
626+
// Route timeout below the 120s floor — chunk_idle must not regress.
627+
timeout: Duration::from_secs(30),
628+
}];
629+
630+
let resp = router
631+
.proxy_with_candidates_streaming(
632+
"openai_chat_completions",
633+
"POST",
634+
"/v1/chat/completions",
635+
vec![],
636+
bytes::Bytes::from(br#"{"model":"test-model","messages":[]}"#.as_slice()),
637+
&candidates,
638+
)
639+
.await
640+
.unwrap();
641+
642+
assert_eq!(
643+
resp.chunk_idle_timeout,
644+
openshell_router::config::DEFAULT_CHUNK_IDLE_TIMEOUT,
645+
"low route timeout must not lower streaming chunk-idle below default"
646+
);
647+
}
648+
649+
#[tokio::test]
650+
async fn streaming_response_chunk_idle_matches_route_timeout_above_floor() {
651+
use std::time::Duration;
652+
653+
let mock_server = MockServer::start().await;
654+
Mock::given(method("POST"))
655+
.and(path("/v1/chat/completions"))
656+
.respond_with(
657+
ResponseTemplate::new(200)
658+
.append_header("content-type", "text/event-stream")
659+
.set_body_string("data: [DONE]\n\n"),
660+
)
661+
.mount(&mock_server)
662+
.await;
663+
664+
let router = Router::new().unwrap();
665+
let candidates = vec![ResolvedRoute {
666+
name: "inference.local".to_string(),
667+
endpoint: mock_server.uri(),
668+
model: "test-model".to_string(),
669+
api_key: "test-api-key".to_string(),
670+
protocols: vec!["openai_chat_completions".to_string()],
671+
auth: AuthHeader::Bearer,
672+
default_headers: Vec::new(),
673+
passthrough_headers: Vec::new(),
674+
// Operators set --timeout 3600 for long-prefill / tool-call-buffering workloads.
675+
timeout: Duration::from_secs(3600),
676+
}];
677+
678+
let resp = router
679+
.proxy_with_candidates_streaming(
680+
"openai_chat_completions",
681+
"POST",
682+
"/v1/chat/completions",
683+
vec![],
684+
bytes::Bytes::from(br#"{"model":"test-model","messages":[]}"#.as_slice()),
685+
&candidates,
686+
)
687+
.await
688+
.unwrap();
689+
690+
assert_eq!(resp.chunk_idle_timeout, Duration::from_secs(3600));
691+
}
692+
693+
#[tokio::test]
694+
async fn streaming_mock_route_carries_chunk_idle_timeout() {
695+
use std::time::Duration;
696+
697+
let router = Router::new().unwrap();
698+
let candidates = vec![ResolvedRoute {
699+
name: "inference.local".to_string(),
700+
endpoint: "mock://test".to_string(),
701+
model: "mock/test-model".to_string(),
702+
api_key: "unused".to_string(),
703+
protocols: vec!["openai_chat_completions".to_string()],
704+
auth: AuthHeader::Bearer,
705+
default_headers: Vec::new(),
706+
passthrough_headers: Vec::new(),
707+
timeout: Duration::from_secs(1800),
708+
}];
709+
710+
let resp = router
711+
.proxy_with_candidates_streaming(
712+
"openai_chat_completions",
713+
"POST",
714+
"/v1/chat/completions",
715+
vec![],
716+
bytes::Bytes::from(br#"{"model":"mock/test-model","messages":[]}"#.as_slice()),
717+
&candidates,
718+
)
719+
.await
720+
.unwrap();
721+
722+
assert_eq!(resp.chunk_idle_timeout, Duration::from_secs(1800));
723+
}
724+
597725
/// Non-streaming (buffered) proxy must still enforce the route timeout.
598726
#[tokio::test]
599727
async fn buffered_proxy_enforces_route_timeout() {

‎crates/openshell-sandbox/src/proxy.rs‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,6 @@ const INFERENCE_LOCAL_PORT: u16 = 443;
3232
/// Maximum total bytes for a streaming inference response body (32 MiB).
3333
const MAX_STREAMING_BODY: usize = 32 * 1024 * 1024;
3434

35-
/// Idle timeout per chunk when relaying streaming inference responses.
36-
///
37-
/// Reasoning models (e.g. nemotron-3-super, o1, o3) can pause for 60+ seconds
38-
/// between "thinking" and output phases. 120s provides headroom while still
39-
/// catching genuinely stuck streams.
40-
const CHUNK_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(120);
41-
4235
/// Result of a proxy CONNECT policy decision.
4336
struct ConnectDecision {
4437
action: NetworkAction,
@@ -1329,8 +1322,9 @@ async fn route_inference_request(
13291322
// write_all call, reducing the number of TLS records per chunk
13301323
// from 3 to 1 while preserving incremental delivery.
13311324
let mut total_bytes: usize = 0;
1325+
let chunk_idle_timeout = resp.chunk_idle_timeout;
13321326
loop {
1333-
match tokio::time::timeout(CHUNK_IDLE_TIMEOUT, resp.next_chunk()).await {
1327+
match tokio::time::timeout(chunk_idle_timeout, resp.next_chunk()).await {
13341328
Ok(Ok(Some(chunk))) => {
13351329
total_bytes += chunk.len();
13361330
if total_bytes > MAX_STREAMING_BODY {
@@ -1366,14 +1360,15 @@ async fn route_inference_request(
13661360
break;
13671361
}
13681362
Err(_) => {
1363+
let timeout_secs = chunk_idle_timeout.as_secs();
13691364
let event = NetworkActivityBuilder::new(crate::ocsf_ctx())
13701365
.activity(ActivityId::Fail)
13711366
.severity(SeverityId::Medium)
13721367
.status(StatusId::Failure)
13731368
.dst_endpoint(Endpoint::from_domain(INFERENCE_LOCAL_HOST, 443))
13741369
.message(format!(
13751370
"streaming response chunk idle timeout after \
1376-
{total_bytes} bytes, closing"
1371+
{total_bytes} bytes (idle_timeout: {timeout_secs}s), closing"
13771372
))
13781373
.build();
13791374
ocsf_emit!(event);

0 commit comments

Comments
 (0)