Skip to content

Commit efcb315

Browse files
committed
fix(storage): fail bidi reads on unexpected server OK
Previously, `Worker::run` exited with `Ok(())` when the server closed a `BidiReadObject` stream with `OK` while `ObjectDescriptor` was still live. This silently truncated in-flight range reads and dropped the `requests` receiver, causing later `read_range()` calls to panic. Fix this by treating an unexpected `Ok(None)` in `Worker::handle_response` as an unrecoverable error so in-flight and subsequent reads fail with `ReadError::UnrecoverableBidiReadInterrupt`. Fixes googleapis#6688
1 parent e0e7bfa commit efcb315

3 files changed

Lines changed: 169 additions & 25 deletions

File tree

‎src/storage/src/object_descriptor.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,14 @@ use std::sync::Arc;
5454
/// You should actively read from all concurrent reads: the client library uses
5555
/// separate buffers for each `read_range()` call, but once a buffer is full the
5656
/// library will stop delivering data to **all** the concurrent reads.
57+
///
58+
/// The client library can automatically resume reads interrupted by transient
59+
/// errors. If the underlying stream closes unexpectedly or fails with an
60+
/// unrecoverable error, the reads in progress and any later reads return an
61+
/// error with [ReadError::UnrecoverableBidiReadInterrupt] as its source. Open a
62+
/// new `ObjectDescriptor` to continue reading.
63+
///
64+
/// [ReadError::UnrecoverableBidiReadInterrupt]: crate::error::ReadError::UnrecoverableBidiReadInterrupt
5765
#[derive(Debug, Clone)]
5866
pub struct ObjectDescriptor {
5967
inner: Arc<dyn ObjectDescriptorStub>,

‎src/storage/src/storage/bidi/transport.rs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,68 @@ mod tests {
311311
Ok(())
312312
}
313313

314+
#[tokio::test]
315+
async fn read_range_after_unexpected_close() -> anyhow::Result<()> {
316+
use std::error::Error as _;
317+
const LEN: i64 = 42;
318+
319+
let (connector, test_context) = BidiTestContext::new();
320+
test_context.connect_tx.send(Ok(base_response())).await?;
321+
let (transport, _) = ObjectDescriptorTransport::new(connector, Vec::new()).await?;
322+
let mut connect_rx = test_context.take_receiver();
323+
let request = connect_rx.recv().await.expect("the initial request");
324+
assert!(request.read_object_spec.is_some(), "{request:?}");
325+
326+
// 1. Start a read and wait until the worker sends its request.
327+
let mut existing = transport.read_range(ReadRange::segment(100, 200)).await;
328+
let request = connect_rx.recv().await.expect("the read request");
329+
let range_request = request.read_ranges.first();
330+
assert_eq!(range_request, Some(&proto_range(100, 200)), "{request:?}");
331+
332+
// 2. Send part of the range, then close the stream with `OK`.
333+
let content = bytes::Bytes::from_owner(String::from_iter((0..LEN).map(|_| 'x')));
334+
let response = BidiReadObjectResponse {
335+
object_data_ranges: vec![ObjectRangeData {
336+
checksummed_data: Some(ChecksummedData {
337+
content: content.clone(),
338+
..ChecksummedData::default()
339+
}),
340+
read_range: Some(proto_range_id(100, LEN, 0)),
341+
range_end: false,
342+
}],
343+
..BidiReadObjectResponse::default()
344+
};
345+
test_context.connect_tx.send(Ok(response)).await?;
346+
drop(test_context.connect_tx);
347+
348+
// 3. The read in progress should fail instead of ending early.
349+
let got = existing.next().await.transpose()?;
350+
assert_eq!(got, Some(content));
351+
let err = existing.next().await.transpose().unwrap_err();
352+
assert!(err.is_io(), "{err:?}");
353+
let source = err.source().and_then(|e| e.downcast_ref::<ReadError>());
354+
assert!(
355+
matches!(source, Some(ReadError::UnrecoverableBidiReadInterrupt(e)) if e.is_io()),
356+
"{err:?}"
357+
);
358+
let got = existing.next().await;
359+
assert!(got.is_none(), "{got:?}");
360+
361+
// 4. Later `read_range()` calls should fail instead of panicking.
362+
let mut reader = transport.read_range(ReadRange::segment(300, 100)).await;
363+
let err = reader.next().await.transpose().unwrap_err();
364+
assert!(err.is_io(), "{err:?}");
365+
let source = err.source().and_then(|e| e.downcast_ref::<ReadError>());
366+
assert!(
367+
matches!(source, Some(ReadError::UnrecoverableBidiReadInterrupt(e)) if e.is_io()),
368+
"{err:?}"
369+
);
370+
let got = reader.next().await;
371+
assert!(got.is_none(), "{got:?}");
372+
373+
Ok(())
374+
}
375+
314376
#[tokio::test]
315377
async fn connect_error() -> anyhow::Result<()> {
316378
let mut mock = MockTestClient::new();

‎src/storage/src/storage/bidi/worker.rs‎

Lines changed: 99 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -63,24 +63,22 @@ where
6363
) -> LoopResult<()> {
6464
let mut ranges = Vec::new();
6565
let (mut rx, mut tx) = (connection.rx, connection.tx);
66-
// Note how this loop only exits when the `requests` queue is
67-
// closed. A successfully closed stream and unrecoverable errors
68-
// return immediately.
66+
// Keep `requests` open until the `ObjectDescriptor` and all readers are
67+
// dropped, even after an unrecoverable stream error. Exiting early would
68+
// close the channel and cause later `read_range()` calls to panic.
6969
let error = loop {
7070
tokio::select! {
7171
m = rx.next_message() => {
7272
match self.handle_response(m).await {
73-
// Successful end of stream, return without error.
74-
None => break None,
7573
// An unrecoverable in the stream or its data, return
7674
// the error.
77-
Some(Err(e)) => break Some(e),
75+
Err(e) => break Some(e),
7876
// New message on the stream handled successfully,
7977
// continue.
80-
Some(Ok(None)) => {},
78+
Ok(None) => {},
8179
// The stream reconnected successfully, update the local
8280
// variables and continue.
83-
Some(Ok(Some(connection))) => {
81+
Ok(Some(connection)) => {
8482
(rx, tx) = (connection.rx, connection.tx);
8583
}
8684
};
@@ -96,7 +94,7 @@ where
9694
drop(rx);
9795
drop(tx);
9896
let Some(e) = error else {
99-
// A successfully closed stream *and* there are no more readers.
97+
// The `requests` queue is closed, so there are no more readers.
10098
return Ok(());
10199
};
102100
// Return errors for any future readers.
@@ -110,15 +108,23 @@ where
110108
pub async fn handle_response(
111109
&mut self,
112110
message: TonicResult<Option<BidiReadObjectResponse>>,
113-
) -> Option<LoopResult<Option<Connection<C::Stream>>>> {
114-
let response = match message.transpose()? {
115-
Ok(r) => r,
111+
) -> LoopResult<Option<Connection<C::Stream>>> {
112+
let response = match message {
113+
Ok(Some(r)) => r,
114+
// The server should keep the stream open until `run()` closes the
115+
// send side. Fail all reads instead of reconnecting, as the default
116+
// resume policy would retry `OK` closes without a limit or backoff.
117+
Ok(None) => {
118+
let error = Arc::new(crate::Error::io(
119+
"bidi_read_object stream closed unexpectedly",
120+
));
121+
self.close_readers(error.clone()).await;
122+
return Err(error);
123+
}
116124
Err(status) => return self.reconnect(status).await,
117125
};
118-
match self.handle_response_success(response).await {
119-
Err(e) => Some(Err(e)),
120-
Ok(_) => Some(Ok(None)),
121-
}
126+
self.handle_response_success(response).await?;
127+
Ok(None)
122128
}
123129

124130
pub async fn handle_response_success(
@@ -150,10 +156,7 @@ where
150156
result.map_err(crate::Error::io)
151157
}
152158

153-
async fn reconnect(
154-
&mut self,
155-
status: Status,
156-
) -> Option<LoopResult<Option<Connection<C::Stream>>>> {
159+
async fn reconnect(&mut self, status: Status) -> LoopResult<Option<Connection<C::Stream>>> {
157160
let ranges: Vec<_> = self
158161
.ranges
159162
.lock()
@@ -166,7 +169,7 @@ where
166169
Err(e) => {
167170
let error = Arc::new(e);
168171
self.close_readers(error.clone()).await;
169-
return Some(Err(error));
172+
return Err(error);
170173
}
171174
Ok(t) => t,
172175
};
@@ -175,9 +178,9 @@ where
175178
// TODO: refactor to handle_ranges().
176179
let error = Arc::new(e);
177180
self.close_readers(error.clone()).await;
178-
return Some(Err(error));
181+
return Err(error);
179182
}
180-
Some(Ok(Some(connection)))
183+
Ok(Some(connection))
181184
}
182185

183186
async fn close_readers(&mut self, error: Arc<crate::Error>) {
@@ -259,7 +262,7 @@ mod tests {
259262

260263
#[tokio::test]
261264
async fn run_immediately_closed() -> anyhow::Result<()> {
262-
let (request_tx, _request_rx) = tokio::sync::mpsc::channel(1);
265+
let (request_tx, mut request_rx) = tokio::sync::mpsc::channel(1);
263266
let (response_tx, response_rx) = mock_stream();
264267
let (tx, rx) = tokio::sync::mpsc::channel(1);
265268
let connection = Connection::new(request_tx, response_rx);
@@ -272,8 +275,25 @@ mod tests {
272275
let handle = tokio::spawn(worker.run(connection, rx));
273276
// Closing the stream without an error should not attempt a reconnect.
274277
drop(response_tx);
278+
// Wait until the worker gives up on the stream and drops its sender.
279+
let got = request_rx.recv().await;
280+
assert!(got.is_none(), "{got:?}");
281+
282+
// Reads sent after the close should fail instead of ending cleanly.
283+
let mut reader = mock_reader(&tx, ReadRange::segment(100, 100)).await;
284+
let got = reader.recv().await;
285+
assert!(
286+
matches!(got, Some(Err(ReadError::UnrecoverableBidiReadInterrupt(ref e))) if e.is_io()),
287+
"{got:?}"
288+
);
289+
let got = reader.recv().await;
290+
assert!(got.is_none(), "{got:?}");
291+
292+
// The worker returns the error once the `requests` queue closes.
275293
drop(tx);
276-
handle.await??;
294+
let err = handle.await?.unwrap_err();
295+
assert!(err.is_io(), "{err:?}");
296+
assert!(err.to_string().contains("closed unexpectedly"), "{err}");
277297
Ok(())
278298
}
279299

@@ -404,6 +424,60 @@ mod tests {
404424
Ok(())
405425
}
406426

427+
#[tokio::test]
428+
async fn run_closed_with_partial_read() -> anyhow::Result<()> {
429+
let SuccessfulReadSetup {
430+
join,
431+
response_tx,
432+
request,
433+
mut reader,
434+
tx,
435+
} = set_up_successful_read().await;
436+
437+
// 1. Send part of the range, then close the stream with `OK`.
438+
let content = bytes::Bytes::from_owner(String::from_iter((0..50).map(|_| 'x')));
439+
let response = BidiReadObjectResponse {
440+
object_data_ranges: vec![ObjectRangeData {
441+
read_range: Some(proto_range_id(100, content.len() as i64, request.read_id)),
442+
checksummed_data: Some(ChecksummedData {
443+
content: content.clone(),
444+
..ChecksummedData::default()
445+
}),
446+
..ObjectRangeData::default()
447+
}],
448+
..BidiReadObjectResponse::default()
449+
};
450+
response_tx.send(Ok(response)).await?;
451+
drop(response_tx);
452+
453+
// 2. The read in progress should fail instead of ending early.
454+
let got = reader.recv().await;
455+
assert!(matches!(got, Some(Ok(ref b)) if *b == content), "{got:?}");
456+
let got = reader.recv().await;
457+
assert!(
458+
matches!(got, Some(Err(ReadError::UnrecoverableBidiReadInterrupt(ref e))) if e.is_io()),
459+
"{got:?}"
460+
);
461+
let got = reader.recv().await;
462+
assert!(got.is_none(), "{got:?}");
463+
464+
// 3. Reads sent after the close should fail too.
465+
let mut reader = mock_reader(&tx, ReadRange::segment(200, 100)).await;
466+
let got = reader.recv().await;
467+
assert!(
468+
matches!(got, Some(Err(ReadError::UnrecoverableBidiReadInterrupt(ref e))) if e.is_io()),
469+
"{got:?}"
470+
);
471+
let got = reader.recv().await;
472+
assert!(got.is_none(), "{got:?}");
473+
474+
// 4. The worker returns the error once the `requests` queue closes.
475+
drop(tx);
476+
let err = join.await?.unwrap_err();
477+
assert!(err.is_io(), "{err:?}");
478+
Ok(())
479+
}
480+
407481
#[tokio::test]
408482
async fn run_full_read() -> anyhow::Result<()> {
409483
let SuccessfulReadSetup {

0 commit comments

Comments
 (0)