Skip to content

Commit cb2e243

Browse files
committed
feat(inference): validate endpoints before saving routes
Closes #273 Verify inference endpoints synchronously on the server during set/update, expose a --no-verify escape hatch in the CLI and Python helper, and return actionable failures when validation does not pass.
1 parent 9bda380 commit cb2e243

10 files changed

Lines changed: 511 additions & 21 deletions

File tree

‎Cargo.lock‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎architecture/inference-routing.md‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,9 @@ The gateway implements the `Inference` gRPC service defined in `proto/inference.
6666
1. Validates that both fields are non-empty.
6767
2. Fetches the named provider record from the store.
6868
3. Validates the provider by resolving its route (checking that the provider type is supported and has a usable API key).
69-
4. Builds a managed route spec that stores only `provider_name` and `model_id`. The spec intentionally leaves `base_url`, `api_key`, and `protocols` empty -- these are resolved dynamically at bundle time from the provider record.
70-
5. Upserts the route with name `inference.local`. Version starts at 1 and increments monotonically on each update.
69+
4. Unless `skip_validation` is set, performs a lightweight provider-shaped probe against the resolved upstream endpoint (for example, a tiny chat/messages request with `max_tokens: 1`) to confirm the endpoint is reachable and accepts the expected auth/request shape.
70+
5. Builds a managed route spec that stores only `provider_name` and `model_id`. The spec intentionally leaves `base_url`, `api_key`, and `protocols` empty -- these are resolved dynamically at bundle time from the provider record.
71+
6. Upserts the route with name `inference.local`. Version starts at 1 and increments monotonically on each update.
7172

7273
`GetClusterInference` returns `provider_name`, `model_id`, and `version` for the managed route. Returns `NOT_FOUND` if cluster inference is not configured.
7374

@@ -91,7 +92,7 @@ File: `proto/inference.proto`
9192

9293
Key messages:
9394

94-
- `SetClusterInferenceRequest` -- `provider_name` + `model_id`
95+
- `SetClusterInferenceRequest` -- `provider_name` + `model_id` + optional `skip_validation`
9596
- `SetClusterInferenceResponse` -- `provider_name` + `model_id` + `version`
9697
- `GetInferenceBundleResponse` -- `repeated ResolvedRoute routes` + `revision` + `generated_at_ms`
9798
- `ResolvedRoute` -- `name`, `base_url`, `protocols`, `api_key`, `model_id`, `provider_type`
@@ -296,10 +297,10 @@ The system route is stored as a separate `InferenceRoute` record in the gateway
296297

297298
Cluster inference commands:
298299

299-
- `openshell cluster inference set --provider <name> --model <id>` -- configures user-facing cluster inference
300-
- `openshell cluster inference set --system --provider <name> --model <id>` -- configures system inference
301-
- `openshell cluster inference get` -- displays both user and system inference configuration
302-
- `openshell cluster inference get --system` -- displays only the system inference configuration
300+
- `openshell inference set --provider <name> --model <id>` -- configures user-facing cluster inference
301+
- `openshell inference set --system --provider <name> --model <id>` -- configures system inference
302+
- `openshell inference get` -- displays both user and system inference configuration
303+
- `openshell inference get --system` -- displays only the system inference configuration
303304

304305
The `--provider` flag references a provider record name (not a provider type). The provider must already exist in the cluster and have a supported inference type (`openai`, `anthropic`, or `nvidia`).
305306

‎crates/openshell-cli/src/main.rs‎

Lines changed: 61 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -906,9 +906,6 @@ enum InferenceCommands {
906906
system: bool,
907907

908908
/// Skip endpoint verification before saving the route.
909-
///
910-
/// Accepted now so scripts can opt out explicitly ahead of a future
911-
/// default switch to verification.
912909
#[arg(long)]
913910
no_verify: bool,
914911
},
@@ -929,9 +926,6 @@ enum InferenceCommands {
929926
system: bool,
930927

931928
/// Skip endpoint verification before saving the route.
932-
///
933-
/// Accepted now so scripts can opt out explicitly ahead of a future
934-
/// default switch to verification.
935929
#[arg(long)]
936930
no_verify: bool,
937931
},
@@ -1810,24 +1804,35 @@ async fn main() -> Result<()> {
18101804
provider,
18111805
model,
18121806
system,
1807+
<<<<<<< HEAD
18131808
no_verify: _,
1809+
=======
1810+
no_verify,
1811+
>>>>>>> 7f0504d8 (feat(inference): validate endpoints before saving routes)
18141812
} => {
18151813
let route_name = if system { "sandbox-system" } else { "" };
1816-
run::gateway_inference_set(endpoint, &provider, &model, route_name, &tls)
1817-
.await?;
1814+
run::gateway_inference_set(
1815+
endpoint, &provider, &model, route_name, no_verify, &tls,
1816+
)
1817+
.await?;
18181818
}
18191819
InferenceCommands::Update {
18201820
provider,
18211821
model,
18221822
system,
1823+
<<<<<<< HEAD
18231824
no_verify: _,
1825+
=======
1826+
no_verify,
1827+
>>>>>>> 7f0504d8 (feat(inference): validate endpoints before saving routes)
18241828
} => {
18251829
let route_name = if system { "sandbox-system" } else { "" };
18261830
run::gateway_inference_update(
18271831
endpoint,
18281832
provider.as_deref(),
18291833
model.as_deref(),
18301834
route_name,
1835+
no_verify,
18311836
&tls,
18321837
)
18331838
.await?;
@@ -2559,6 +2564,54 @@ mod tests {
25592564
));
25602565
}
25612566

2567+
#[test]
2568+
fn inference_set_accepts_no_verify_flag() {
2569+
let cli = Cli::try_parse_from([
2570+
"openshell",
2571+
"inference",
2572+
"set",
2573+
"--provider",
2574+
"openai-dev",
2575+
"--model",
2576+
"gpt-4.1",
2577+
"--no-verify",
2578+
])
2579+
.expect("inference set should parse --no-verify");
2580+
2581+
assert!(matches!(
2582+
cli.command,
2583+
Some(Commands::Inference {
2584+
command: Some(InferenceCommands::Set {
2585+
no_verify: true,
2586+
..
2587+
})
2588+
})
2589+
));
2590+
}
2591+
2592+
#[test]
2593+
fn inference_update_accepts_no_verify_flag() {
2594+
let cli = Cli::try_parse_from([
2595+
"openshell",
2596+
"inference",
2597+
"update",
2598+
"--provider",
2599+
"openai-dev",
2600+
"--no-verify",
2601+
])
2602+
.expect("inference update should parse --no-verify");
2603+
2604+
assert!(matches!(
2605+
cli.command,
2606+
Some(Commands::Inference {
2607+
command: Some(InferenceCommands::Update {
2608+
no_verify: true,
2609+
..
2610+
})
2611+
})
2612+
));
2613+
}
2614+
25622615
#[test]
25632616
fn completion_script_uses_openshell_command_name() {
25642617
let script = normalize_completion_script(

‎crates/openshell-cli/src/run.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3390,6 +3390,7 @@ pub async fn gateway_inference_set(
33903390
provider_name: &str,
33913391
model_id: &str,
33923392
route_name: &str,
3393+
skip_validation: bool,
33933394
tls: &TlsOptions,
33943395
) -> Result<()> {
33953396
let mut client = grpc_inference_client(server, tls).await?;
@@ -3398,6 +3399,7 @@ pub async fn gateway_inference_set(
33983399
provider_name: provider_name.to_string(),
33993400
model_id: model_id.to_string(),
34003401
route_name: route_name.to_string(),
3402+
skip_validation,
34013403
})
34023404
.await
34033405
.into_diagnostic()?;
@@ -3422,6 +3424,7 @@ pub async fn gateway_inference_update(
34223424
provider_name: Option<&str>,
34233425
model_id: Option<&str>,
34243426
route_name: &str,
3427+
skip_validation: bool,
34253428
tls: &TlsOptions,
34263429
) -> Result<()> {
34273430
if provider_name.is_none() && model_id.is_none() {
@@ -3449,6 +3452,7 @@ pub async fn gateway_inference_update(
34493452
provider_name: provider.to_string(),
34503453
model_id: model.to_string(),
34513454
route_name: route_name.to_string(),
3455+
skip_validation,
34523456
})
34533457
.await
34543458
.into_diagnostic()?;

‎crates/openshell-server/Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ serde = { workspace = true }
6161
serde_json = { workspace = true }
6262
tokio-stream = { workspace = true }
6363
sqlx = { workspace = true }
64+
reqwest = { workspace = true }
6465
kube = { workspace = true }
6566
kube-runtime = { workspace = true }
6667
k8s-openapi = { workspace = true }
@@ -78,6 +79,7 @@ rcgen = { version = "0.13", features = ["crypto", "pem"] }
7879
tempfile = "3"
7980
tokio-tungstenite = { workspace = true }
8081
futures-util = "0.3"
82+
wiremock = "0.6"
8183

8284
[lints]
8385
workspace = true

0 commit comments

Comments
 (0)