55//! what the supervisor forwards upstream in REST bodies and WebSocket frames.
66//!
77//! Ported from the e2e tests added in NVIDIA/OpenShell#5264. The upstream
8- //! servers run on the test host ([`host_server_support`]), so a gateway on a
9- //! remote cluster can't run this suite .
8+ //! servers run on the test host ([`host_server_support`]), so the suite
9+ //! requires sandboxes to reach the test host as `host.openshell.internal` .
1010
1111use std:: time:: Duration ;
1212
13- use host_server_support:: { HttpRecorder , WebSocketRecorder } ;
13+ use host_server_support:: { HttpRecorder , Request , WebSocketRecorder } ;
1414use testkit:: policy:: { self , Access , Endpoint as _} ;
1515use testkit:: sdk:: raw:: proto;
16- use testkit:: { Context , Error , PYTHON_BINARIES , Poll , Result , ResultExt as _, Sandbox , YamlFile } ;
16+ use testkit:: {
17+ Context , Error , GrpcCode , PYTHON_BINARIES , Poll , Result , ResultExt as _, Sandbox , YamlFile ,
18+ } ;
1719
1820const TEST_HOST : & str = "host.openshell.internal" ;
1921const TOKEN_ENV : & str = "E2E_GATING_TOKEN" ;
@@ -71,15 +73,15 @@ fi"#;
7173async fn profile_rest_endpoint_keeps_body_placeholder ( cx : & mut Context ) -> testkit:: Result {
7274 let server = HttpRecorder :: start ( "ok" ) . await ?;
7375 let websocket = WebSocketRecorder :: start ( ) . await ?;
74- let provider = install_provider ( cx, Some ( ( server. port , websocket. port ) ) ) . await ?;
76+ let provider = install_gating_provider ( cx, Some ( ( server. port , websocket. port ) ) ) . await ?;
7577
7678 let mut policy = policy:: base ( ) ;
7779 allow ( & mut policy, server. port )
7880 . rest ( Access :: Full )
7981 . request_body_credential_rewrite ( false ) ;
8082 let sandbox = create_sandbox ( cx, "rest" , policy, & provider) . await ?;
8183 run_python ( & sandbox, BODY_CLIENT , server. port ) . await ?;
82- expect_one_body ( & server, PLACEHOLDER_PREFIX , TEST_SECRET ) . await
84+ expect_placeholder_body ( & server) . await
8385}
8486
8587/// A provider's WebSocket endpoint completes the handshake but doesn't
@@ -89,7 +91,7 @@ async fn profile_rest_endpoint_keeps_body_placeholder(cx: &mut Context) -> testk
8991async fn profile_websocket_endpoint_denies_binary_frames ( cx : & mut Context ) -> testkit:: Result {
9092 let server = HttpRecorder :: start ( "ok" ) . await ?;
9193 let websocket = WebSocketRecorder :: start ( ) . await ?;
92- let provider = install_provider ( cx, Some ( ( server. port , websocket. port ) ) ) . await ?;
94+ let provider = install_gating_provider ( cx, Some ( ( server. port , websocket. port ) ) ) . await ?;
9395
9496 let mut policy = policy:: base ( ) ;
9597 allow ( & mut policy, websocket. port ) . websocket ( Access :: ReadWrite ) ;
@@ -99,8 +101,9 @@ async fn profile_websocket_endpoint_denies_binary_frames(cx: &mut Context) -> te
99101 . wait_for_handshake ( )
100102 . await
101103 . context ( "upstream should receive the WebSocket handshake" ) ?;
104+ let binary_seen = websocket. wait_until_finished ( ) . await ?;
102105 testkit:: ensure!(
103- !websocket . binary_seen( ) ,
106+ !binary_seen,
104107 "credentialed WebSocket binary frame reached upstream"
105108 ) ;
106109 Ok ( ( ) )
@@ -112,10 +115,11 @@ async fn profile_websocket_endpoint_denies_binary_frames(cx: &mut Context) -> te
112115#[ testkit:: test]
113116async fn policy_binding_requires_inspection_or_opt_in ( cx : & mut Context ) -> testkit:: Result {
114117 let server = HttpRecorder :: start ( "ok" ) . await ?;
115- let provider = install_provider ( cx, None ) . await ?;
118+ let provider = install_gating_provider ( cx, None ) . await ?;
116119
117120 let mut l4 = policy:: base ( ) ;
118121 allow ( & mut l4, server. port ) . credential_binding ( & provider) ;
122+ // The wording checks follow the #5264 tests.
119123 let error = expect_rejected ( cx, "l4" , l4, & provider) . await ?;
120124 testkit:: ensure!( error. contains( "credentialed endpoint" ) , "{error}" ) ;
121125 testkit:: ensure!( error. contains( "L4-only" ) , "{error}" ) ;
@@ -145,7 +149,7 @@ async fn policy_binding_rest_body_keeps_placeholder_without_rewrite(
145149 cx : & mut Context ,
146150) -> testkit:: Result {
147151 let server = HttpRecorder :: start ( "ok" ) . await ?;
148- let provider = install_provider ( cx, None ) . await ?;
152+ let provider = install_gating_provider ( cx, None ) . await ?;
149153
150154 let mut policy = policy:: base ( ) ;
151155 allow ( & mut policy, server. port )
@@ -154,7 +158,7 @@ async fn policy_binding_rest_body_keeps_placeholder_without_rewrite(
154158 . credential_binding ( & provider) ;
155159 let sandbox = create_sandbox ( cx, "body" , policy, & provider) . await ?;
156160 run_python ( & sandbox, BODY_CLIENT , server. port ) . await ?;
157- expect_one_body ( & server, PLACEHOLDER_PREFIX , TEST_SECRET ) . await
161+ expect_placeholder_body ( & server) . await
158162}
159163
160164/// A policy binding with `request_body_credential_rewrite: true` replaces
@@ -164,7 +168,7 @@ async fn policy_binding_rest_body_rewrite_sends_secret_upstream(
164168 cx : & mut Context ,
165169) -> testkit:: Result {
166170 let server = HttpRecorder :: start ( "ok" ) . await ?;
167- let provider = install_provider ( cx, None ) . await ?;
171+ let provider = install_gating_provider ( cx, None ) . await ?;
168172
169173 let mut policy = policy:: base ( ) ;
170174 allow ( & mut policy, server. port )
@@ -173,15 +177,21 @@ async fn policy_binding_rest_body_rewrite_sends_secret_upstream(
173177 . credential_binding ( & provider) ;
174178 let sandbox = create_sandbox ( cx, "rw" , policy, & provider) . await ?;
175179 run_python ( & sandbox, BODY_CLIENT , server. port ) . await ?;
176- expect_one_body ( & server, TEST_SECRET , PLACEHOLDER_PREFIX ) . await
180+ let request = one_request ( & server) . await ?;
181+ testkit:: ensure!(
182+ request. body == format!( "prefix-{TEST_SECRET}-suffix" ) ,
183+ "expected the secret in the body upstream; got {}" ,
184+ redact( & request)
185+ ) ;
186+ Ok ( ( ) )
177187}
178188
179189/// Binding and unbinding a provider with `policy set --wait` adds and
180190/// removes its environment variable in a running sandbox.
181191#[ testkit:: test]
182192async fn policy_binding_updates_provider_environment_live ( cx : & mut Context ) -> testkit:: Result {
183193 let server = HttpRecorder :: start ( "ok" ) . await ?;
184- let provider = install_provider ( cx, None ) . await ?;
194+ let provider = install_gating_provider ( cx, None ) . await ?;
185195
186196 let policy_binding = |bind : bool | {
187197 let mut policy = policy:: base ( ) ;
@@ -223,7 +233,7 @@ fn allow(policy: &mut proto::SandboxPolicy, port: u16) -> &mut proto::NetworkEnd
223233/// Install a run-unique provider holding `TEST_SECRET` in `TOKEN_ENV`. With
224234/// `endpoints`, its profile declares the REST and WebSocket ports as its
225235/// endpoints; without, a policy must bind the provider explicitly.
226- async fn install_provider ( cx : & Context , endpoints : Option < ( u16 , u16 ) > ) -> Result < String > {
236+ async fn install_gating_provider ( cx : & Context , endpoints : Option < ( u16 , u16 ) > ) -> Result < String > {
227237 let name = cx. name ( "cg" ) ;
228238 let endpoints = endpoints. map_or_else ( String :: new, |( rest, websocket) | {
229239 format ! (
@@ -288,7 +298,8 @@ async fn create_sandbox(
288298 Ok ( sandbox)
289299}
290300
291- /// Fail unless the gateway rejects the sandbox, and return the rejection.
301+ /// Fail unless the gateway rejects the sandbox as a failed precondition, and
302+ /// return the rejection.
292303async fn expect_rejected (
293304 cx : & Context ,
294305 suffix : & str ,
@@ -300,7 +311,10 @@ async fn expect_rejected(
300311 "the gateway accepted sandbox {}" ,
301312 sandbox. name( )
302313 ) ) ) ,
303- Err ( error) => Ok ( error. to_string ( ) ) ,
314+ Err ( error) if error. grpc_code ( ) == Some ( GrpcCode :: FailedPrecondition ) => {
315+ Ok ( error. to_string ( ) )
316+ }
317+ Err ( error) => Err ( error. context ( "expected a failed precondition" ) ) ,
304318 }
305319}
306320
@@ -315,23 +329,39 @@ async fn run_python(sandbox: &Sandbox, script: &str, port: u16) -> Result {
315329 Ok ( ( ) )
316330}
317331
318- /// Fail unless `server` saw exactly one request, containing `present` and
319- /// not `absent`.
320- async fn expect_one_body ( server : & HttpRecorder , present : & str , absent : & str ) -> Result {
321- let requests = server. wait_for_requests ( 1 ) . await ?;
332+ /// The one request `server` saw.
333+ async fn one_request ( server : & HttpRecorder ) -> Result < Request > {
334+ let mut requests = server. wait_for_requests ( 1 ) . await ?;
322335 testkit:: ensure!(
323336 requests. len( ) == 1 ,
324337 "expected one request upstream; got {}" ,
325338 requests. len( )
326339 ) ;
327- let request = & requests[ 0 ] ;
340+ Ok ( requests. remove ( 0 ) )
341+ }
342+
343+ /// Fail unless `server` saw one request whose body has the credential
344+ /// placeholder in place of the token, and the secret appears nowhere in it.
345+ async fn expect_placeholder_body ( server : & HttpRecorder ) -> Result {
346+ let request = one_request ( server) . await ?;
328347 testkit:: ensure!(
329- request. contains( present) && !request. contains( absent) ,
330- "expected {present:?} and not {absent:?} upstream"
348+ request
349+ . body
350+ . starts_with( & format!( "prefix-{PLACEHOLDER_PREFIX}" ) )
351+ && request. body. ends_with( "-suffix" )
352+ && !request. head. contains( TEST_SECRET )
353+ && !request. body. contains( TEST_SECRET ) ,
354+ "expected the placeholder in the body upstream and no secret; got {}" ,
355+ redact( & request)
331356 ) ;
332357 Ok ( ( ) )
333358}
334359
360+ /// `request` for a failure message, with the secret replaced.
361+ fn redact ( request : & Request ) -> String {
362+ format ! ( "{}\n \n {}" , request. head, request. body) . replace ( TEST_SECRET , "<secret>" )
363+ }
364+
335365/// Wait until new exec sessions report `expected` from [`TOKEN_PRESENCE`].
336366async fn wait_for_token ( sandbox : & Sandbox , expected : & str ) -> Result {
337367 testkit:: poll_until (
0 commit comments