@@ -63,24 +63,22 @@ where
6363 ) -> LoopResult < ( ) > {
6464 let mut ranges = Vec :: new ( ) ;
6565 let ( mut rx, mut tx) = ( connection. rx , connection. tx ) ;
66- // Note how this loop only exits when the `requests` queue is
67- // closed. A successfully closed stream and unrecoverable errors
68- // return immediately .
66+ // Keep `requests` open until the `ObjectDescriptor` and all readers are
67+ // dropped, even after an unrecoverable stream error. Exiting early would
68+ // close the channel and cause later `read_range()` calls to panic .
6969 let error = loop {
7070 tokio:: select! {
7171 m = rx. next_message( ) => {
7272 match self . handle_response( m) . await {
73- // Successful end of stream, return without error.
74- None => break None ,
7573 // An unrecoverable in the stream or its data, return
7674 // the error.
77- Some ( Err ( e) ) => break Some ( e) ,
75+ Err ( e) => break Some ( e) ,
7876 // New message on the stream handled successfully,
7977 // continue.
80- Some ( Ok ( None ) ) => { } ,
78+ Ok ( None ) => { } ,
8179 // The stream reconnected successfully, update the local
8280 // variables and continue.
83- Some ( Ok ( Some ( connection) ) ) => {
81+ Ok ( Some ( connection) ) => {
8482 ( rx, tx) = ( connection. rx, connection. tx) ;
8583 }
8684 } ;
9694 drop ( rx) ;
9795 drop ( tx) ;
9896 let Some ( e) = error else {
99- // A successfully closed stream *and* there are no more readers.
97+ // The `requests` queue is closed, so there are no more readers.
10098 return Ok ( ( ) ) ;
10199 } ;
102100 // Return errors for any future readers.
@@ -110,15 +108,23 @@ where
110108 pub async fn handle_response (
111109 & mut self ,
112110 message : TonicResult < Option < BidiReadObjectResponse > > ,
113- ) -> Option < LoopResult < Option < Connection < C :: Stream > > > > {
114- let response = match message. transpose ( ) ? {
115- Ok ( r) => r,
111+ ) -> LoopResult < Option < Connection < C :: Stream > > > {
112+ let response = match message {
113+ Ok ( Some ( r) ) => r,
114+ // The server should keep the stream open until `run()` closes the
115+ // send side. Fail all reads instead of reconnecting, as the default
116+ // resume policy would retry `OK` closes without a limit or backoff.
117+ Ok ( None ) => {
118+ let error = Arc :: new ( crate :: Error :: io (
119+ "bidi_read_object stream closed unexpectedly" ,
120+ ) ) ;
121+ self . close_readers ( error. clone ( ) ) . await ;
122+ return Err ( error) ;
123+ }
116124 Err ( status) => return self . reconnect ( status) . await ,
117125 } ;
118- match self . handle_response_success ( response) . await {
119- Err ( e) => Some ( Err ( e) ) ,
120- Ok ( _) => Some ( Ok ( None ) ) ,
121- }
126+ self . handle_response_success ( response) . await ?;
127+ Ok ( None )
122128 }
123129
124130 pub async fn handle_response_success (
@@ -150,10 +156,7 @@ where
150156 result. map_err ( crate :: Error :: io)
151157 }
152158
153- async fn reconnect (
154- & mut self ,
155- status : Status ,
156- ) -> Option < LoopResult < Option < Connection < C :: Stream > > > > {
159+ async fn reconnect ( & mut self , status : Status ) -> LoopResult < Option < Connection < C :: Stream > > > {
157160 let ranges: Vec < _ > = self
158161 . ranges
159162 . lock ( )
@@ -166,7 +169,7 @@ where
166169 Err ( e) => {
167170 let error = Arc :: new ( e) ;
168171 self . close_readers ( error. clone ( ) ) . await ;
169- return Some ( Err ( error) ) ;
172+ return Err ( error) ;
170173 }
171174 Ok ( t) => t,
172175 } ;
@@ -175,9 +178,9 @@ where
175178 // TODO: refactor to handle_ranges().
176179 let error = Arc :: new ( e) ;
177180 self . close_readers ( error. clone ( ) ) . await ;
178- return Some ( Err ( error) ) ;
181+ return Err ( error) ;
179182 }
180- Some ( Ok ( Some ( connection) ) )
183+ Ok ( Some ( connection) )
181184 }
182185
183186 async fn close_readers ( & mut self , error : Arc < crate :: Error > ) {
@@ -259,7 +262,7 @@ mod tests {
259262
260263 #[ tokio:: test]
261264 async fn run_immediately_closed ( ) -> anyhow:: Result < ( ) > {
262- let ( request_tx, _request_rx ) = tokio:: sync:: mpsc:: channel ( 1 ) ;
265+ let ( request_tx, mut request_rx ) = tokio:: sync:: mpsc:: channel ( 1 ) ;
263266 let ( response_tx, response_rx) = mock_stream ( ) ;
264267 let ( tx, rx) = tokio:: sync:: mpsc:: channel ( 1 ) ;
265268 let connection = Connection :: new ( request_tx, response_rx) ;
@@ -272,8 +275,25 @@ mod tests {
272275 let handle = tokio:: spawn ( worker. run ( connection, rx) ) ;
273276 // Closing the stream without an error should not attempt a reconnect.
274277 drop ( response_tx) ;
278+ // Wait until the worker gives up on the stream and drops its sender.
279+ let got = request_rx. recv ( ) . await ;
280+ assert ! ( got. is_none( ) , "{got:?}" ) ;
281+
282+ // Reads sent after the close should fail instead of ending cleanly.
283+ let mut reader = mock_reader ( & tx, ReadRange :: segment ( 100 , 100 ) ) . await ;
284+ let got = reader. recv ( ) . await ;
285+ assert ! (
286+ matches!( got, Some ( Err ( ReadError :: UnrecoverableBidiReadInterrupt ( ref e) ) ) if e. is_io( ) ) ,
287+ "{got:?}"
288+ ) ;
289+ let got = reader. recv ( ) . await ;
290+ assert ! ( got. is_none( ) , "{got:?}" ) ;
291+
292+ // The worker returns the error once the `requests` queue closes.
275293 drop ( tx) ;
276- handle. await ??;
294+ let err = handle. await ?. unwrap_err ( ) ;
295+ assert ! ( err. is_io( ) , "{err:?}" ) ;
296+ assert ! ( err. to_string( ) . contains( "closed unexpectedly" ) , "{err}" ) ;
277297 Ok ( ( ) )
278298 }
279299
@@ -404,6 +424,60 @@ mod tests {
404424 Ok ( ( ) )
405425 }
406426
427+ #[ tokio:: test]
428+ async fn run_closed_with_partial_read ( ) -> anyhow:: Result < ( ) > {
429+ let SuccessfulReadSetup {
430+ join,
431+ response_tx,
432+ request,
433+ mut reader,
434+ tx,
435+ } = set_up_successful_read ( ) . await ;
436+
437+ // 1. Send part of the range, then close the stream with `OK`.
438+ let content = bytes:: Bytes :: from_owner ( String :: from_iter ( ( 0 ..50 ) . map ( |_| 'x' ) ) ) ;
439+ let response = BidiReadObjectResponse {
440+ object_data_ranges : vec ! [ ObjectRangeData {
441+ read_range: Some ( proto_range_id( 100 , content. len( ) as i64 , request. read_id) ) ,
442+ checksummed_data: Some ( ChecksummedData {
443+ content: content. clone( ) ,
444+ ..ChecksummedData :: default ( )
445+ } ) ,
446+ ..ObjectRangeData :: default ( )
447+ } ] ,
448+ ..BidiReadObjectResponse :: default ( )
449+ } ;
450+ response_tx. send ( Ok ( response) ) . await ?;
451+ drop ( response_tx) ;
452+
453+ // 2. The read in progress should fail instead of ending early.
454+ let got = reader. recv ( ) . await ;
455+ assert ! ( matches!( got, Some ( Ok ( ref b) ) if * b == content) , "{got:?}" ) ;
456+ let got = reader. recv ( ) . await ;
457+ assert ! (
458+ matches!( got, Some ( Err ( ReadError :: UnrecoverableBidiReadInterrupt ( ref e) ) ) if e. is_io( ) ) ,
459+ "{got:?}"
460+ ) ;
461+ let got = reader. recv ( ) . await ;
462+ assert ! ( got. is_none( ) , "{got:?}" ) ;
463+
464+ // 3. Reads sent after the close should fail too.
465+ let mut reader = mock_reader ( & tx, ReadRange :: segment ( 200 , 100 ) ) . await ;
466+ let got = reader. recv ( ) . await ;
467+ assert ! (
468+ matches!( got, Some ( Err ( ReadError :: UnrecoverableBidiReadInterrupt ( ref e) ) ) if e. is_io( ) ) ,
469+ "{got:?}"
470+ ) ;
471+ let got = reader. recv ( ) . await ;
472+ assert ! ( got. is_none( ) , "{got:?}" ) ;
473+
474+ // 4. The worker returns the error once the `requests` queue closes.
475+ drop ( tx) ;
476+ let err = join. await ?. unwrap_err ( ) ;
477+ assert ! ( err. is_io( ) , "{err:?}" ) ;
478+ Ok ( ( ) )
479+ }
480+
407481 #[ tokio:: test]
408482 async fn run_full_read ( ) -> anyhow:: Result < ( ) > {
409483 let SuccessfulReadSetup {
0 commit comments