|
15 | 15 | use super::builder::Subscribe; |
16 | 16 | use super::handler::{AckResult, Action, AtLeastOnce, ExactlyOnce, Handler}; |
17 | 17 | use super::lease_loop::LeaseLoop; |
18 | | -use super::lease_state::{AtLeastOnceInfo, ExactlyOnceInfo, LeaseInfo, LeaseOptions, NewMessage}; |
| 18 | +use super::lease_state::{ |
| 19 | + AtLeastOnceInfo, ExactlyOnceInfo, LeaseInfo, LeaseOptions, MAX_IDS_PER_RPC, NewMessage, |
| 20 | +}; |
19 | 21 | use super::leaser::DefaultLeaser; |
20 | | -use super::retry_policy::StreamRetryPolicy; |
| 22 | +use super::retry_policy::{StreamRetryPolicy, settle_options}; |
21 | 23 | use super::shutdown_token::ShutdownToken; |
22 | 24 | use super::stream::Stream; |
| 25 | +use super::stub::Stub as _; |
23 | 26 | use super::transport::Transport; |
24 | 27 | use crate::google::pubsub::v1::{StreamingPullRequest, StreamingPullResponse}; |
25 | | -use crate::model::Message; |
| 28 | +use crate::model::{Message, ModifyAckDeadlineRequest}; |
26 | 29 | use crate::{Error, Result}; |
27 | 30 | use futures::FutureExt; |
28 | 31 | use futures::future::{BoxFuture, Shared}; |
@@ -398,12 +401,74 @@ impl MessageStreamImpl { |
398 | 401 |
|
399 | 402 | // Permanently close the stream. |
400 | 403 | fn close(&mut self) { |
| 404 | + self.drain_and_nack(); |
401 | 405 | self.stream = Some(StreamState::Closed); |
402 | 406 | self.pool.clear(); |
403 | 407 | self.shutdown.cancel(); |
404 | 408 | } |
| 409 | + |
| 410 | + /// Nacks the messages in responses already buffered on the stream. |
| 411 | + /// |
| 412 | + /// When we close the gRPC stream, any responses the server already sent |
| 413 | + /// but that we have not read yet are discarded with it. The service |
| 414 | + /// considers those messages delivered, so it would only redeliver them |
| 415 | + /// once their ack deadline expires. Instead, we read what is already |
| 416 | + /// buffered (without waiting on the network) and nack those messages, so |
| 417 | + /// the service can redeliver them right away. |
| 418 | + /// |
| 419 | + /// These messages were never added to lease management, so we nack them |
| 420 | + /// directly instead of through the lease loop. The lease loop may have |
| 421 | + /// already shut down at this point. |
| 422 | + /// |
| 423 | + /// Messages still in flight on the network when the stream closes are not |
| 424 | + /// covered. The service redelivers them after their ack deadline. |
| 425 | + fn drain_and_nack(&mut self) { |
| 426 | + let Some(StreamState::Active(stream)) = self.stream.as_mut() else { |
| 427 | + return; |
| 428 | + }; |
| 429 | + let ack_ids: Vec<String> = stream |
| 430 | + .drain_buffered(MAX_DRAINED_RESPONSES) |
| 431 | + .into_iter() |
| 432 | + .flat_map(|r| r.received_messages) |
| 433 | + .map(|rm| rm.ack_id) |
| 434 | + .collect(); |
| 435 | + if ack_ids.is_empty() { |
| 436 | + return; |
| 437 | + } |
| 438 | + // We may be called from `Drop`, outside of a runtime. |
| 439 | + let Ok(runtime) = tokio::runtime::Handle::try_current() else { |
| 440 | + return; |
| 441 | + }; |
| 442 | + let stub = self.stub.clone(); |
| 443 | + let subscription = self.initial_req.subscription.clone(); |
| 444 | + runtime.spawn(async move { |
| 445 | + for batch in ack_ids.chunks(MAX_IDS_PER_RPC) { |
| 446 | + let req = ModifyAckDeadlineRequest::new() |
| 447 | + .set_subscription(subscription.clone()) |
| 448 | + .set_ack_ids(batch.to_vec()) |
| 449 | + .set_ack_deadline_seconds(0); |
| 450 | + let _ = stub.modify_ack_deadline(req, settle_options()).await; |
| 451 | + } |
| 452 | + }); |
| 453 | + } |
405 | 454 | } |
406 | 455 |
|
| 456 | +impl Drop for MessageStreamImpl { |
| 457 | + fn drop(&mut self) { |
| 458 | + // The application may drop the stream without calling `next()` after |
| 459 | + // a shutdown. Do not lose the responses buffered on the stream. |
| 460 | + self.drain_and_nack(); |
| 461 | + } |
| 462 | +} |
| 463 | + |
| 464 | +/// The maximum number of buffered responses we drain from the stream when it |
| 465 | +/// closes. |
| 466 | +/// |
| 467 | +/// The drain never waits on the network, so it is already bounded by what the |
| 468 | +/// gRPC stream has buffered. This limit is a safeguard in case the server keeps |
| 469 | +/// the buffer full while we drain. |
| 470 | +const MAX_DRAINED_RESPONSES: usize = 100; |
| 471 | + |
407 | 472 | /// A `Handler` without its action `Sender`. |
408 | 473 | /// |
409 | 474 | /// We only want to create strong `Sender`s for `Handler`s that we yield to the |
@@ -2107,4 +2172,110 @@ mod tests { |
2107 | 2172 |
|
2108 | 2173 | Ok(()) |
2109 | 2174 | } |
| 2175 | + |
| 2176 | + /// Waits for nack requests until `want` ack IDs have been nacked. |
| 2177 | + async fn recv_nacked( |
| 2178 | + nack_rx: &mut tokio::sync::mpsc::UnboundedReceiver<v1::ModifyAckDeadlineRequest>, |
| 2179 | + want: usize, |
| 2180 | + ) -> anyhow::Result<Vec<String>> { |
| 2181 | + let mut nacked = Vec::new(); |
| 2182 | + while nacked.len() < want { |
| 2183 | + let req = tokio::time::timeout(Duration::from_secs(10), nack_rx.recv()) |
| 2184 | + .await? |
| 2185 | + .ok_or_else(|| anyhow::anyhow!("nack channel closed"))?; |
| 2186 | + assert_eq!(req.subscription, "projects/p/subscriptions/s"); |
| 2187 | + assert_eq!(req.ack_deadline_seconds, 0); |
| 2188 | + nacked.extend(req.ack_ids); |
| 2189 | + } |
| 2190 | + Ok(sorted(nacked)) |
| 2191 | + } |
| 2192 | + |
| 2193 | + /// Sets up a stream that has read one message (acked), and has responses |
| 2194 | + /// for messages 1..4 buffered, but not yet read. |
| 2195 | + async fn stream_with_buffered_responses( |
| 2196 | + shutdown_behavior: ShutdownBehavior, |
| 2197 | + ) -> anyhow::Result<( |
| 2198 | + MessageStream, |
| 2199 | + tokio::sync::mpsc::UnboundedReceiver<v1::ModifyAckDeadlineRequest>, |
| 2200 | + tokio::sync::mpsc::Sender<std::result::Result<v1::StreamingPullResponse, TonicStatus>>, |
| 2201 | + tokio::task::JoinHandle<()>, |
| 2202 | + )> { |
| 2203 | + let (response_tx, response_rx) = channel(10); |
| 2204 | + let (nack_tx, nack_rx) = unbounded_channel(); |
| 2205 | + |
| 2206 | + let mut mock = MockSubscriber::new(); |
| 2207 | + mock.expect_streaming_pull() |
| 2208 | + .return_once(|_| Ok(TonicResponse::from(response_rx))); |
| 2209 | + mock.expect_acknowledge() |
| 2210 | + .returning(|_| Ok(TonicResponse::from(()))); |
| 2211 | + mock.expect_modify_ack_deadline().returning(move |r| { |
| 2212 | + let r = r.into_inner(); |
| 2213 | + if r.ack_deadline_seconds == 0 { |
| 2214 | + let _ = nack_tx.send(r); |
| 2215 | + } |
| 2216 | + Ok(TonicResponse::from(())) |
| 2217 | + }); |
| 2218 | + let (endpoint, server) = start("0.0.0.0:0", mock).await?; |
| 2219 | + let client = test_client(endpoint).await?; |
| 2220 | + let mut stream = client |
| 2221 | + .subscribe("projects/p/subscriptions/s") |
| 2222 | + .set_shutdown_behavior(shutdown_behavior) |
| 2223 | + .build(); |
| 2224 | + |
| 2225 | + response_tx.send(Ok(test_response(0..1))).await?; |
| 2226 | + let Some((m, h)) = stream.next().await.transpose()? else { |
| 2227 | + anyhow::bail!("expected message 0") |
| 2228 | + }; |
| 2229 | + assert_eq!(m.data, test_data(0)); |
| 2230 | + h.ack(); |
| 2231 | + |
| 2232 | + // The server sends more messages, which the application does not read. |
| 2233 | + response_tx.send(Ok(test_response(1..3))).await?; |
| 2234 | + response_tx.send(Ok(test_response(3..4))).await?; |
| 2235 | + // Give the responses time to reach the client's gRPC buffers. |
| 2236 | + tokio::time::sleep(Duration::from_millis(200)).await; |
| 2237 | + |
| 2238 | + Ok((stream, nack_rx, response_tx, server)) |
| 2239 | + } |
| 2240 | + |
| 2241 | + #[test_case(ShutdownBehavior::NackImmediately)] |
| 2242 | + #[test_case(ShutdownBehavior::WaitForProcessing)] |
| 2243 | + #[tokio_test_no_panics] |
| 2244 | + async fn shutdown_nacks_buffered_responses( |
| 2245 | + shutdown_behavior: ShutdownBehavior, |
| 2246 | + ) -> anyhow::Result<()> { |
| 2247 | + let (mut stream, mut nack_rx, _response_tx, _server) = |
| 2248 | + stream_with_buffered_responses(shutdown_behavior).await?; |
| 2249 | + |
| 2250 | + stream.shutdown_token().shutdown().await; |
| 2251 | + let end = stream.next().await.transpose()?; |
| 2252 | + assert!(end.is_none(), "Received extra message: {end:?}"); |
| 2253 | + |
| 2254 | + // Messages 1..4 were never read by the application, so lease |
| 2255 | + // management never knew about them. We must nack them anyway, instead |
| 2256 | + // of leaving them with the service until their ack deadline expires. |
| 2257 | + let nacked = recv_nacked(&mut nack_rx, 3).await?; |
| 2258 | + assert_eq!(nacked, test_ids(1..4)); |
| 2259 | + |
| 2260 | + Ok(()) |
| 2261 | + } |
| 2262 | + |
| 2263 | + #[test_case(ShutdownBehavior::NackImmediately)] |
| 2264 | + #[test_case(ShutdownBehavior::WaitForProcessing)] |
| 2265 | + #[tokio_test_no_panics] |
| 2266 | + async fn drop_nacks_buffered_responses( |
| 2267 | + shutdown_behavior: ShutdownBehavior, |
| 2268 | + ) -> anyhow::Result<()> { |
| 2269 | + let (stream, mut nack_rx, _response_tx, _server) = |
| 2270 | + stream_with_buffered_responses(shutdown_behavior).await?; |
| 2271 | + let shutdown_token = stream.shutdown_token(); |
| 2272 | + |
| 2273 | + drop(stream); |
| 2274 | + shutdown_token.wait_for_shutdown().await; |
| 2275 | + |
| 2276 | + let nacked = recv_nacked(&mut nack_rx, 3).await?; |
| 2277 | + assert_eq!(nacked, test_ids(1..4)); |
| 2278 | + |
| 2279 | + Ok(()) |
| 2280 | + } |
2110 | 2281 | } |
0 commit comments