Skip to content

Commit 5d9b0f0

Browse files
fix(inference): prepend publisher prefix for Vertex non-Anthropic models (NVIDIA#2735)
* fix(inference): prepend publisher prefix for Vertex non-Anthropic models Vertex AI's OpenAI-compatible endpoint requires the request body's model field to carry a publisher prefix (e.g. google/gemini-2.5-flash), but validate_vertex_model_id rejects slash as a path-traversal guard. This created a deadlock: bare model IDs pass validation but are rejected by Vertex with HTTP 400 "Malformed publisher model"; prefixed IDs are rejected at configuration time. Fix: in resolve_vertex_ai_route, compute body_model_id for non-Anthropic routes by prepending the publisher from infer_vertex_publisher() or the explicit VERTEX_AI_PUBLISHER config value. The bare model_id still goes through the path-traversal validator unchanged. Anthropic rawPredict routes encode the model in the URL path, not the body, and are unaffected. Both the project/region path and the base-URL-override path apply the prefix. For unrecognised models with no explicit publisher the bare ID is forwarded unchanged; Vertex's 400 is the correct observable signal in that case. Add an integration test in openshell-router that spins up a mock Vertex endpoint accepting only the publisher-prefixed form and rejecting the bare model name, verifying the body rewrite produces the required format. Closes NVIDIA#2351 Signed-off-by: politerealism <[email protected]> * fix(inference): propagate publisher-prefixed model_id through inference bundle resolve_route_by_name_with_credentials built the ResolvedRoute with config.model_id (the bare stored value) rather than resolved.route.model (the publisher-prefixed value computed by resolve_vertex_ai_route). As a result, the bundle delivered to sandboxes carried e.g. "gemini-2.5-flash" instead of "google/gemini-2.5-flash", so live sandbox requests still hit Vertex AI with the bare model name and received HTTP 400 "Malformed publisher model". Fix: use resolved.route.model in the bundle construction so the publisher prefix survives the bundle boundary and the router sends the correct body to Vertex AI. Update the existing gemini bundle test to assert the prefixed model_id and add a dedicated regression test that verifies the bundle carries the publisher prefix for non-Anthropic Vertex routes. Signed-off-by: politerealism <[email protected]> * style(inference): apply rustfmt to new regression test Signed-off-by: politerealism <[email protected]> * fix(inference): address clippy lints in build_vertex_route - Invert if !is_anthropic to satisfy clippy::if_not_else - Replace match on Option with map_or_else to satisfy clippy::option_if_let_else Signed-off-by: politerealism <[email protected]> --------- Signed-off-by: politerealism <[email protected]>
1 parent 6ebf10e commit 5d9b0f0

2 files changed

Lines changed: 210 additions & 6 deletions

File tree

‎crates/openshell-router/src/backend.rs‎

Lines changed: 86 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -982,8 +982,8 @@ fn is_vertex_anthropic_rawpredict_route(route: &ResolvedRoute) -> bool {
982982
mod tests {
983983
use super::{
984984
ValidationFailure, ValidationFailureKind, build_backend_url, build_provider_url,
985-
parse_bedrock_invocation_path, prepare_backend_request, rewrite_bedrock_path,
986-
route_is_bedrock, verify_backend_endpoint,
985+
parse_bedrock_invocation_path, prepare_backend_request, proxy_to_backend,
986+
rewrite_bedrock_path, route_is_bedrock, verify_backend_endpoint,
987987
};
988988
use crate::RouterError;
989989
use crate::config::{DEFAULT_ROUTE_TIMEOUT, ResolvedRoute};
@@ -2597,6 +2597,90 @@ mod tests {
25972597
);
25982598
}
25992599

2600+
/// Vertex AI's OpenAI-compatible endpoint requires the body `model` field to
2601+
/// carry a publisher prefix (e.g. `google/gemini-2.5-flash`). This test
2602+
/// simulates the fix: `resolve_vertex_ai_route` sets `route.model` to the
2603+
/// prefixed form, and the body rewrite here forwards that value to Vertex.
2604+
///
2605+
/// The mock server only accepts the prefixed form — matching Vertex's
2606+
/// behaviour — and returns 400 "Malformed publisher model" for the bare name.
2607+
#[tokio::test]
2608+
async fn vertex_openai_compat_rewrites_body_model_to_publisher_prefixed_form() {
2609+
let mock_server = MockServer::start().await;
2610+
2611+
// Simulate Vertex accepting only the publisher-prefixed model name.
2612+
Mock::given(method("POST"))
2613+
.and(path("/chat/completions"))
2614+
.and(body_partial_json(
2615+
serde_json::json!({"model": "google/gemini-2.5-flash"}),
2616+
))
2617+
.respond_with(
2618+
ResponseTemplate::new(200).set_body_json(
2619+
serde_json::json!({"choices": [{"message": {"content": "hi"}}]}),
2620+
),
2621+
)
2622+
.expect(1)
2623+
.mount(&mock_server)
2624+
.await;
2625+
2626+
// Simulate Vertex rejecting the bare model name — the pre-fix failure.
2627+
Mock::given(method("POST"))
2628+
.and(path("/chat/completions"))
2629+
.and(body_partial_json(
2630+
serde_json::json!({"model": "gemini-2.5-flash"}),
2631+
))
2632+
.respond_with(ResponseTemplate::new(400).set_body_json(
2633+
serde_json::json!({"error": {"message": "Malformed publisher model"}}),
2634+
))
2635+
.expect(0) // must never be reached after the fix
2636+
.mount(&mock_server)
2637+
.await;
2638+
2639+
// Route as produced by resolve_vertex_ai_route after the fix:
2640+
// route.model carries the publisher prefix.
2641+
let route = ResolvedRoute {
2642+
name: "vertex-gemini".to_string(),
2643+
endpoint: mock_server.uri(),
2644+
model: "google/gemini-2.5-flash".to_string(),
2645+
api_key: "ya29.token".to_string(),
2646+
protocols: vec!["openai_chat_completions".to_string()],
2647+
auth: AuthHeader::Bearer,
2648+
default_headers: vec![],
2649+
passthrough_headers: vec![],
2650+
timeout: DEFAULT_ROUTE_TIMEOUT,
2651+
model_in_path: false,
2652+
request_path_override: Some("/chat/completions".to_string()),
2653+
};
2654+
2655+
// The client sends the bare model name; the body rewrite must replace it
2656+
// with route.model (the publisher-prefixed form) before forwarding.
2657+
let client_body = serde_json::to_vec(&serde_json::json!({
2658+
"model": "gemini-2.5-flash",
2659+
"messages": [{"role": "user", "content": "hello"}]
2660+
}))
2661+
.unwrap();
2662+
2663+
let client = reqwest::Client::new();
2664+
let result = proxy_to_backend(
2665+
&client,
2666+
&route,
2667+
"openai_chat_completions",
2668+
"POST",
2669+
"/chat/completions",
2670+
vec![("content-type".to_string(), "application/json".to_string())],
2671+
bytes::Bytes::from(client_body),
2672+
)
2673+
.await
2674+
.expect("proxy should succeed");
2675+
2676+
assert_eq!(
2677+
result.status, 200,
2678+
"Vertex mock must accept the publisher-prefixed model; \
2679+
got {}: body rewrite did not apply the prefix",
2680+
result.status
2681+
);
2682+
}
2683+
26002684
/// Defense-in-depth: a Bedrock route receiving a non-Bedrock path
26012685
/// is rejected rather than forwarded. The L7 pattern detector
26022686
/// upstream of the router should never produce this combination,

‎crates/openshell-server/src/inference.rs‎

Lines changed: 124 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -698,6 +698,22 @@ fn resolve_vertex_ai_route(
698698
|p| p.eq_ignore_ascii_case("anthropic"),
699699
);
700700

701+
// Vertex's OpenAI-compatible endpoint requires the request body's `model`
702+
// field to carry a publisher prefix: `<publisher>/<model_id>` (e.g.
703+
// `google/gemini-2.5-flash`). The publisher is taken from the explicit
704+
// VERTEX_AI_PUBLISHER config value (when set to a non-Anthropic value) or
705+
// inferred from the model name. For unrecognised models with no explicit
706+
// publisher, the bare model ID is forwarded unchanged; Vertex will return
707+
// a 400 in that case, which is the correct observable signal to the caller.
708+
// Anthropic rawPredict routes encode the model in the URL path, not the
709+
// body, so they are unaffected.
710+
let body_model_id: String = if is_anthropic {
711+
model_id.to_string()
712+
} else {
713+
let publisher = explicit_publisher.or_else(|| infer_vertex_publisher(model_id));
714+
publisher.map_or_else(|| model_id.to_string(), |p| format!("{p}/{model_id}"))
715+
};
716+
701717
// Escape hatch: caller-supplied full base URL still uses the model-derived
702718
// protocol and path contract, but only for the OpenAI-compatible Vertex surface.
703719
// Anthropic-on-Vertex needs model-path shaping and body adaptation that a fully
@@ -721,7 +737,7 @@ fn resolve_vertex_ai_route(
721737
return Ok(build_vertex_route(
722738
route_name,
723739
base_url,
724-
model_id,
740+
&body_model_id,
725741
api_key,
726742
vec!["openai_chat_completions".to_string()],
727743
profile,
@@ -772,7 +788,7 @@ fn resolve_vertex_ai_route(
772788
Ok(build_vertex_route(
773789
route_name,
774790
endpoint,
775-
model_id,
791+
&body_model_id,
776792
api_key,
777793
protocols,
778794
profile,
@@ -1140,7 +1156,7 @@ async fn resolve_route_by_name_with_credentials(
11401156
Ok(Some(ResolvedRoute {
11411157
name: route_name.to_string(),
11421158
base_url: resolved.route.endpoint,
1143-
model_id: config.model_id.clone(),
1159+
model_id: resolved.route.model.clone(),
11441160
api_key: resolved.route.api_key,
11451161
protocols: resolved.route.protocols,
11461162
provider_type: resolved.provider_type,
@@ -1730,13 +1746,51 @@ mod tests {
17301746
route.request_path_override,
17311747
Some("/chat/completions".to_string())
17321748
);
1733-
assert_eq!(route.model_id, "gemini-2.0-flash-001");
1749+
assert_eq!(route.model_id, "google/gemini-2.0-flash-001");
17341750
assert_eq!(
17351751
route.base_url,
17361752
"https://us-central1-aiplatform.googleapis.com/v1beta1/projects/my-gcp-project/locations/us-central1/endpoints/openapi"
17371753
);
17381754
}
17391755

1756+
#[tokio::test]
1757+
async fn bundle_vertex_ai_non_anthropic_model_id_carries_publisher_prefix() {
1758+
// Regression test: the bundle's model_id must carry the publisher prefix
1759+
// so the router sends e.g. "google/gemini-2.5-flash" in the request body,
1760+
// not the bare "gemini-2.5-flash" that Vertex AI rejects with HTTP 400.
1761+
let store = test_store().await;
1762+
let config = [
1763+
(
1764+
"VERTEX_AI_PROJECT_ID".to_string(),
1765+
"my-gcp-project".to_string(),
1766+
),
1767+
("VERTEX_AI_REGION".to_string(), "us-central1".to_string()),
1768+
]
1769+
.into_iter()
1770+
.collect();
1771+
let provider = make_vertex_provider_with_config("vertex-dev", config);
1772+
store
1773+
.put_message(&provider)
1774+
.await
1775+
.expect("persist provider");
1776+
let route = make_route(
1777+
CLUSTER_INFERENCE_ROUTE_NAME,
1778+
"vertex-dev",
1779+
"gemini-2.5-flash",
1780+
);
1781+
store.put_message(&route).await.expect("persist route");
1782+
1783+
let resp = resolve_inference_bundle(&store, "default")
1784+
.await
1785+
.expect("bundle should resolve");
1786+
1787+
assert_eq!(resp.routes.len(), 1);
1788+
assert_eq!(
1789+
resp.routes[0].model_id, "google/gemini-2.5-flash",
1790+
"bundle model_id must carry publisher prefix for non-Anthropic Vertex routes"
1791+
);
1792+
}
1793+
17401794
#[tokio::test]
17411795
async fn bundle_without_cluster_route_returns_empty_routes() {
17421796
let store = test_store().await;
@@ -2613,6 +2667,11 @@ mod tests {
26132667
.contains(&"anthropic_messages".to_string()),
26142668
"must not have anthropic_messages protocol for gemini"
26152669
);
2670+
// Vertex OpenAI-compatible endpoint requires publisher prefix in body model field
2671+
assert_eq!(
2672+
resolved.route.model, "google/gemini-pro",
2673+
"Vertex non-Anthropic body model must carry publisher prefix"
2674+
);
26162675
}
26172676

26182677
#[test]
@@ -2651,6 +2710,67 @@ mod tests {
26512710
.contains(&"anthropic_messages".to_string()),
26522711
"must not have anthropic_messages for unknown model"
26532712
);
2713+
// Unknown models have no inferred publisher; body model ID is unchanged
2714+
assert_eq!(resolved.route.model, "some-unknown-model");
2715+
}
2716+
2717+
#[test]
2718+
fn resolve_vertex_ai_route_non_anthropic_publisher_prefix_gemini() {
2719+
// Gemini models must get `google/<model>` in route.model so the
2720+
// OpenAI-compatible Vertex endpoint accepts the request body.
2721+
let config =
2722+
std::iter::once(("VERTEX_AI_PROJECT_ID".to_string(), "proj-abc".to_string())).collect();
2723+
let provider = make_vertex_provider_with_config("vertex-gemini-flash", config);
2724+
2725+
let resolved =
2726+
resolve_provider_route(&provider, "gemini-2.5-flash").expect("should resolve");
2727+
2728+
assert_eq!(resolved.route.model, "google/gemini-2.5-flash");
2729+
}
2730+
2731+
#[test]
2732+
fn resolve_vertex_ai_route_non_anthropic_publisher_prefix_llama() {
2733+
let config =
2734+
std::iter::once(("VERTEX_AI_PROJECT_ID".to_string(), "proj-abc".to_string())).collect();
2735+
let provider = make_vertex_provider_with_config("vertex-llama", config);
2736+
2737+
let resolved = resolve_provider_route(&provider, "llama-3-70b").expect("should resolve");
2738+
2739+
assert_eq!(resolved.route.model, "meta/llama-3-70b");
2740+
}
2741+
2742+
#[test]
2743+
fn resolve_vertex_ai_route_explicit_publisher_overrides_inference() {
2744+
// VERTEX_AI_PUBLISHER takes precedence over infer_vertex_publisher for
2745+
// unknown model names.
2746+
let config = [
2747+
("VERTEX_AI_PROJECT_ID".to_string(), "proj-abc".to_string()),
2748+
("VERTEX_AI_PUBLISHER".to_string(), "acme".to_string()),
2749+
]
2750+
.into_iter()
2751+
.collect();
2752+
let provider = make_vertex_provider_with_config("vertex-explicit", config);
2753+
2754+
let resolved =
2755+
resolve_provider_route(&provider, "some-acme-model").expect("should resolve");
2756+
2757+
assert_eq!(resolved.route.model, "acme/some-acme-model");
2758+
}
2759+
2760+
#[test]
2761+
fn resolve_vertex_ai_route_base_url_override_gemini_gets_publisher_prefix() {
2762+
// Publisher prefix must also be applied when a base URL override is used.
2763+
let config = std::iter::once((
2764+
"VERTEX_AI_BASE_URL".to_string(),
2765+
"https://us-central1-aiplatform.googleapis.com/v1beta1/projects/my-project/locations/us-central1/endpoints/openapi".to_string(),
2766+
))
2767+
.collect();
2768+
let provider = make_vertex_provider_with_config("vertex-base-url-gemini", config);
2769+
2770+
let resolved =
2771+
resolve_provider_route(&provider, "gemini-2.0-flash").expect("should resolve");
2772+
2773+
assert_eq!(resolved.route.model, "google/gemini-2.0-flash");
26542774
}
26552775

26562776
#[test]

0 commit comments

Comments
 (0)