Skip to content

Commit 89d0475

Browse files
shanielhclaude
andcommitted
fix(pubsub): nack buffered messages when the stream closes
When a `MessageStream` shuts down or is dropped, it drops the gRPC stream. Any responses the server already sent but the client had not read yet were discarded with it. Their messages never reached lease management, so they were neither leased nor nacked, and the service only redelivered them once their ack deadline expired. With the default 60 second deadline, a message published around shutdown could stall for a minute. Now, before the stream is dropped (on shutdown, on a permanent error, or when the `MessageStream` is dropped), the client reads the responses already buffered on the stream, without waiting on the network, and nacks their messages with `ModifyAckDeadline(0)`. The drain stops at the first response that is not immediately available, at the end of the stream, at an error, or after 100 responses. The nacks go directly through the stub, because lease management never knew about these messages and may already have shut down. This applies to both `ShutdownBehavior` values. Messages that are still in flight on the network when the stream closes are not covered. The service still redelivers them after their ack deadline. Fixes googleapis#7412 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent e0e7bfa commit 89d0475

3 files changed

Lines changed: 271 additions & 4 deletions

File tree

‎src/pubsub/src/subscriber/lease_state.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ use tokio_util::task::TaskTracker;
3434
// on the size of an ack ID. We can safely fit 1000 ack IDs into a request.
3535
//
3636
// https://docs.cloud.google.com/pubsub/quotas
37-
const MAX_IDS_PER_RPC: usize = 1000;
37+
pub(super) const MAX_IDS_PER_RPC: usize = 1000;
3838

3939
// How often we extend deadlines for messages under lease
4040
const EXTEND_PERIOD: Duration = Duration::from_secs(3);

‎src/pubsub/src/subscriber/message_stream.rs‎

Lines changed: 174 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,17 @@
1515
use super::builder::Subscribe;
1616
use super::handler::{AckResult, Action, AtLeastOnce, ExactlyOnce, Handler};
1717
use super::lease_loop::LeaseLoop;
18-
use super::lease_state::{AtLeastOnceInfo, ExactlyOnceInfo, LeaseInfo, LeaseOptions, NewMessage};
18+
use super::lease_state::{
19+
AtLeastOnceInfo, ExactlyOnceInfo, LeaseInfo, LeaseOptions, MAX_IDS_PER_RPC, NewMessage,
20+
};
1921
use super::leaser::DefaultLeaser;
20-
use super::retry_policy::StreamRetryPolicy;
22+
use super::retry_policy::{StreamRetryPolicy, settle_options};
2123
use super::shutdown_token::ShutdownToken;
2224
use super::stream::Stream;
25+
use super::stub::Stub as _;
2326
use super::transport::Transport;
2427
use crate::google::pubsub::v1::{StreamingPullRequest, StreamingPullResponse};
25-
use crate::model::Message;
28+
use crate::model::{Message, ModifyAckDeadlineRequest};
2629
use crate::{Error, Result};
2730
use futures::FutureExt;
2831
use futures::future::{BoxFuture, Shared};
@@ -398,12 +401,74 @@ impl MessageStreamImpl {
398401

399402
// Permanently close the stream.
400403
fn close(&mut self) {
404+
self.drain_and_nack();
401405
self.stream = Some(StreamState::Closed);
402406
self.pool.clear();
403407
self.shutdown.cancel();
404408
}
409+
410+
/// Nacks the messages in responses already buffered on the stream.
411+
///
412+
/// When we close the gRPC stream, any responses the server already sent
413+
/// but that we have not read yet are discarded with it. The service
414+
/// considers those messages delivered, so it would only redeliver them
415+
/// once their ack deadline expires. Instead, we read what is already
416+
/// buffered (without waiting on the network) and nack those messages, so
417+
/// the service can redeliver them right away.
418+
///
419+
/// These messages were never added to lease management, so we nack them
420+
/// directly instead of through the lease loop. The lease loop may have
421+
/// already shut down at this point.
422+
///
423+
/// Messages still in flight on the network when the stream closes are not
424+
/// covered. The service redelivers them after their ack deadline.
425+
fn drain_and_nack(&mut self) {
426+
let Some(StreamState::Active(stream)) = self.stream.as_mut() else {
427+
return;
428+
};
429+
let ack_ids: Vec<String> = stream
430+
.drain_buffered(MAX_DRAINED_RESPONSES)
431+
.into_iter()
432+
.flat_map(|r| r.received_messages)
433+
.map(|rm| rm.ack_id)
434+
.collect();
435+
if ack_ids.is_empty() {
436+
return;
437+
}
438+
// We may be called from `Drop`, outside of a runtime.
439+
let Ok(runtime) = tokio::runtime::Handle::try_current() else {
440+
return;
441+
};
442+
let stub = self.stub.clone();
443+
let subscription = self.initial_req.subscription.clone();
444+
runtime.spawn(async move {
445+
for batch in ack_ids.chunks(MAX_IDS_PER_RPC) {
446+
let req = ModifyAckDeadlineRequest::new()
447+
.set_subscription(subscription.clone())
448+
.set_ack_ids(batch.to_vec())
449+
.set_ack_deadline_seconds(0);
450+
let _ = stub.modify_ack_deadline(req, settle_options()).await;
451+
}
452+
});
453+
}
405454
}
406455

456+
impl Drop for MessageStreamImpl {
457+
fn drop(&mut self) {
458+
// The application may drop the stream without calling `next()` after
459+
// a shutdown. Do not lose the responses buffered on the stream.
460+
self.drain_and_nack();
461+
}
462+
}
463+
464+
/// The maximum number of buffered responses we drain from the stream when it
465+
/// closes.
466+
///
467+
/// The drain never waits on the network, so it is already bounded by what the
468+
/// gRPC stream has buffered. This limit is a safeguard in case the server keeps
469+
/// the buffer full while we drain.
470+
const MAX_DRAINED_RESPONSES: usize = 100;
471+
407472
/// A `Handler` without its action `Sender`.
408473
///
409474
/// We only want to create strong `Sender`s for `Handler`s that we yield to the
@@ -2107,4 +2172,110 @@ mod tests {
21072172

21082173
Ok(())
21092174
}
2175+
2176+
/// Waits for nack requests until `want` ack IDs have been nacked.
2177+
async fn recv_nacked(
2178+
nack_rx: &mut tokio::sync::mpsc::UnboundedReceiver<v1::ModifyAckDeadlineRequest>,
2179+
want: usize,
2180+
) -> anyhow::Result<Vec<String>> {
2181+
let mut nacked = Vec::new();
2182+
while nacked.len() < want {
2183+
let req = tokio::time::timeout(Duration::from_secs(10), nack_rx.recv())
2184+
.await?
2185+
.ok_or_else(|| anyhow::anyhow!("nack channel closed"))?;
2186+
assert_eq!(req.subscription, "projects/p/subscriptions/s");
2187+
assert_eq!(req.ack_deadline_seconds, 0);
2188+
nacked.extend(req.ack_ids);
2189+
}
2190+
Ok(sorted(nacked))
2191+
}
2192+
2193+
/// Sets up a stream that has read one message (acked), and has responses
2194+
/// for messages 1..4 buffered, but not yet read.
2195+
async fn stream_with_buffered_responses(
2196+
shutdown_behavior: ShutdownBehavior,
2197+
) -> anyhow::Result<(
2198+
MessageStream,
2199+
tokio::sync::mpsc::UnboundedReceiver<v1::ModifyAckDeadlineRequest>,
2200+
tokio::sync::mpsc::Sender<std::result::Result<v1::StreamingPullResponse, TonicStatus>>,
2201+
tokio::task::JoinHandle<()>,
2202+
)> {
2203+
let (response_tx, response_rx) = channel(10);
2204+
let (nack_tx, nack_rx) = unbounded_channel();
2205+
2206+
let mut mock = MockSubscriber::new();
2207+
mock.expect_streaming_pull()
2208+
.return_once(|_| Ok(TonicResponse::from(response_rx)));
2209+
mock.expect_acknowledge()
2210+
.returning(|_| Ok(TonicResponse::from(())));
2211+
mock.expect_modify_ack_deadline().returning(move |r| {
2212+
let r = r.into_inner();
2213+
if r.ack_deadline_seconds == 0 {
2214+
let _ = nack_tx.send(r);
2215+
}
2216+
Ok(TonicResponse::from(()))
2217+
});
2218+
let (endpoint, server) = start("0.0.0.0:0", mock).await?;
2219+
let client = test_client(endpoint).await?;
2220+
let mut stream = client
2221+
.subscribe("projects/p/subscriptions/s")
2222+
.set_shutdown_behavior(shutdown_behavior)
2223+
.build();
2224+
2225+
response_tx.send(Ok(test_response(0..1))).await?;
2226+
let Some((m, h)) = stream.next().await.transpose()? else {
2227+
anyhow::bail!("expected message 0")
2228+
};
2229+
assert_eq!(m.data, test_data(0));
2230+
h.ack();
2231+
2232+
// The server sends more messages, which the application does not read.
2233+
response_tx.send(Ok(test_response(1..3))).await?;
2234+
response_tx.send(Ok(test_response(3..4))).await?;
2235+
// Give the responses time to reach the client's gRPC buffers.
2236+
tokio::time::sleep(Duration::from_millis(200)).await;
2237+
2238+
Ok((stream, nack_rx, response_tx, server))
2239+
}
2240+
2241+
#[test_case(ShutdownBehavior::NackImmediately)]
2242+
#[test_case(ShutdownBehavior::WaitForProcessing)]
2243+
#[tokio_test_no_panics]
2244+
async fn shutdown_nacks_buffered_responses(
2245+
shutdown_behavior: ShutdownBehavior,
2246+
) -> anyhow::Result<()> {
2247+
let (mut stream, mut nack_rx, _response_tx, _server) =
2248+
stream_with_buffered_responses(shutdown_behavior).await?;
2249+
2250+
stream.shutdown_token().shutdown().await;
2251+
let end = stream.next().await.transpose()?;
2252+
assert!(end.is_none(), "Received extra message: {end:?}");
2253+
2254+
// Messages 1..4 were never read by the application, so lease
2255+
// management never knew about them. We must nack them anyway, instead
2256+
// of leaving them with the service until their ack deadline expires.
2257+
let nacked = recv_nacked(&mut nack_rx, 3).await?;
2258+
assert_eq!(nacked, test_ids(1..4));
2259+
2260+
Ok(())
2261+
}
2262+
2263+
#[test_case(ShutdownBehavior::NackImmediately)]
2264+
#[test_case(ShutdownBehavior::WaitForProcessing)]
2265+
#[tokio_test_no_panics]
2266+
async fn drop_nacks_buffered_responses(
2267+
shutdown_behavior: ShutdownBehavior,
2268+
) -> anyhow::Result<()> {
2269+
let (stream, mut nack_rx, _response_tx, _server) =
2270+
stream_with_buffered_responses(shutdown_behavior).await?;
2271+
let shutdown_token = stream.shutdown_token();
2272+
2273+
drop(stream);
2274+
shutdown_token.wait_for_shutdown().await;
2275+
2276+
let nacked = recv_nacked(&mut nack_rx, 3).await?;
2277+
assert_eq!(nacked, test_ids(1..4));
2278+
2279+
Ok(())
2280+
}
21102281
}

‎src/pubsub/src/subscriber/stream.rs‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,24 @@ where
7979
_ = pong_timeout(&mut self.pong_deadline_rx) => Err(keepalive_timeout()),
8080
}
8181
}
82+
83+
/// Reads the responses already buffered on the stream, without waiting.
84+
///
85+
/// Stops at the first response that is not immediately available, at the
86+
/// end of the stream, at an error, or after `limit` responses, whichever
87+
/// comes first. This never waits on the network, so it returns promptly.
88+
pub(super) fn drain_buffered(&mut self, limit: usize) -> Vec<StreamingPullResponse> {
89+
use futures::FutureExt as _;
90+
let mut responses = Vec::new();
91+
while responses.len() < limit {
92+
match self.stream.next_message().now_or_never() {
93+
Some(Ok(Some(response))) => responses.push(response),
94+
// Not yet available, end of stream, or an error.
95+
_ => break,
96+
}
97+
}
98+
responses
99+
}
82100
}
83101

84102
/// The error returned when a keepalive ping receives no response within
@@ -338,6 +356,84 @@ mod tests {
338356
Ok(())
339357
}
340358

359+
#[tokio_test_no_panics(start_paused = true)]
360+
async fn drain_buffered_stops_when_pending() -> anyhow::Result<()> {
361+
let (response_tx, response_rx) = mpsc::channel(10);
362+
363+
let mut mock = MockStub::new();
364+
mock.expect_streaming_pull()
365+
.times(1)
366+
.return_once(move |_s, _r, _o| Ok(TonicResponse::from(response_rx)));
367+
368+
response_tx.send(Ok(test_response(1..10))).await?;
369+
response_tx.send(Ok(test_response(11..20))).await?;
370+
response_tx.send(Ok(test_response(21..30))).await?;
371+
372+
let mut stream = open_stream(Arc::new(mock), initial_request()).await?;
373+
assert_eq!(stream.next_message().await?, Some(test_response(1..10)));
374+
375+
// The stream is still open, so the drain must not wait for more.
376+
let drained = stream.drain_buffered(100);
377+
assert_eq!(drained, vec![test_response(11..20), test_response(21..30)]);
378+
assert_eq!(stream.drain_buffered(100), Vec::new());
379+
380+
// Responses that arrive later are not lost by an earlier drain.
381+
response_tx.send(Ok(test_response(31..40))).await?;
382+
assert_eq!(stream.drain_buffered(100), vec![test_response(31..40)]);
383+
384+
Ok(())
385+
}
386+
387+
#[tokio_test_no_panics(start_paused = true)]
388+
async fn drain_buffered_respects_limit() -> anyhow::Result<()> {
389+
let (response_tx, response_rx) = mpsc::channel(10);
390+
391+
let mut mock = MockStub::new();
392+
mock.expect_streaming_pull()
393+
.times(1)
394+
.return_once(move |_s, _r, _o| Ok(TonicResponse::from(response_rx)));
395+
396+
for i in 0..5 {
397+
response_tx.send(Ok(test_response(i..i + 1))).await?;
398+
}
399+
400+
let mut stream = open_stream(Arc::new(mock), initial_request()).await?;
401+
assert_eq!(stream.next_message().await?, Some(test_response(0..1)));
402+
403+
let drained = stream.drain_buffered(2);
404+
assert_eq!(drained, vec![test_response(1..2), test_response(2..3)]);
405+
406+
Ok(())
407+
}
408+
409+
#[tokio_test_no_panics(start_paused = true)]
410+
async fn drain_buffered_stops_at_end_and_error() -> anyhow::Result<()> {
411+
let (response_tx, response_rx) = mpsc::channel(10);
412+
413+
let mut mock = MockStub::new();
414+
mock.expect_streaming_pull()
415+
.times(1)
416+
.return_once(move |_s, _r, _o| Ok(TonicResponse::from(response_rx)));
417+
418+
response_tx.send(Ok(test_response(0..1))).await?;
419+
response_tx.send(Ok(test_response(1..2))).await?;
420+
response_tx
421+
.send(Err(gaxi::grpc::tonic::Status::internal("fail")))
422+
.await?;
423+
response_tx.send(Ok(test_response(2..3))).await?;
424+
drop(response_tx);
425+
426+
let mut stream = open_stream(Arc::new(mock), initial_request()).await?;
427+
assert_eq!(stream.next_message().await?, Some(test_response(0..1)));
428+
// The drain stops at the error.
429+
assert_eq!(stream.drain_buffered(100), vec![test_response(1..2)]);
430+
// The drain stops at the end of the stream.
431+
assert_eq!(stream.drain_buffered(100), vec![test_response(2..3)]);
432+
assert_eq!(stream.drain_buffered(100), Vec::new());
433+
434+
Ok(())
435+
}
436+
341437
#[tokio_test_no_panics(start_paused = true)]
342438
async fn keepalives() -> anyhow::Result<()> {
343439
let (response_tx, response_rx) = mpsc::channel(10);

0 commit comments

Comments
 (0)