Skip to content

Commit 7ca2848

Browse files
committed
fix(bigquery): configure write user agents
Add a client-level user-agent setting that applies consistently to all writers and stream operations, including shared streams and retries. Fixes googleapis#7389. AI disclosure: Initially created using gpt-6 astra through codex.
1 parent 1e8d5c7 commit 7ca2848

10 files changed

Lines changed: 190 additions & 30 deletions

File tree

‎src/bigquery/src/write/base.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,11 @@
1313
// limitations under the License.
1414

1515
use super::format::DataFormat;
16-
use crate::Result;
1716
use crate::model::{AppendRowsRequest, FinalizeWriteStreamResponse};
1817
use crate::write::generated::gapic_storage::client::BigQueryWrite;
1918
use crate::write::runner::Runner;
2019
use crate::write::transport::Transport;
20+
use crate::{RequestOptions, Result};
2121
use std::sync::Arc;
2222

2323
/// A shared internal structure for holding common state across different stream types,
@@ -27,6 +27,7 @@ use std::sync::Arc;
2727
/// structs (e.g. `BufferedWriter`, `CommittedWriter`, `PendingWriter`).
2828
#[derive(Debug)]
2929
pub(crate) struct BaseWriter<F> {
30+
pub(crate) request_options: RequestOptions,
3031
pub(crate) runner: Runner,
3132
pub(crate) write_stream: String,
3233
pub(crate) format: F,
@@ -39,8 +40,9 @@ where
3940
{
4041
pub(crate) fn new(inner: Arc<Transport>, write_stream: String, format: F) -> Self {
4142
let runner = Runner::new(inner.clone());
42-
let client = BigQueryWrite::from_stub::<Transport>(inner);
43+
let client = inner.client();
4344
Self {
45+
request_options: inner.options.clone(),
4446
runner,
4547
write_stream,
4648
format,
@@ -55,6 +57,7 @@ where
5557
pub(crate) async fn finalize(&self) -> Result<FinalizeWriteStreamResponse> {
5658
self.client
5759
.finalize_write_stream()
60+
.with_options(self.request_options.clone())
5861
.set_name(&self.write_stream)
5962
.send()
6063
.await

‎src/bigquery/src/write/buffered.rs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ where
8686
self.inner
8787
.client
8888
.flush_rows()
89+
.with_options(self.inner.request_options.clone())
8990
.set_write_stream(&self.inner.write_stream)
9091
.set_offset(offset)
9192
.send()
@@ -112,11 +113,14 @@ mod tests {
112113
let (response_tx, response_rx) = mpsc::channel(10);
113114

114115
let mut mock = MockBigQueryWrite::new();
115-
mock.expect_append_rows()
116-
.return_once(|_| Ok(TonicResponse::from(response_rx)));
116+
mock.expect_append_rows().return_once(|req| {
117+
check_user_agent(req.metadata());
118+
Ok(TonicResponse::from(response_rx))
119+
});
117120

118121
mock.expect_flush_rows()
119122
.return_once(|req| {
123+
check_user_agent(req.metadata());
120124
assert_eq!(req.get_ref().offset, Some(3));
121125
assert_eq!(req.get_ref().write_stream, write_stream());
122126
Ok(TonicResponse::new(
@@ -126,14 +130,15 @@ mod tests {
126130

127131
mock.expect_finalize_write_stream()
128132
.return_once(|req| {
133+
check_user_agent(req.metadata());
129134
assert_eq!(req.get_ref().name, write_stream());
130135
Ok(TonicResponse::new(
131136
bigquery_grpc_mock::google::cloud::bigquery::storage::v1::FinalizeWriteStreamResponse::default()
132137
))
133138
});
134139

135140
let (endpoint, _server) = start("0.0.0.0:0", mock).await?;
136-
let transport = Arc::new(test_transport(endpoint).await?);
141+
let transport = Arc::new(test_transport_with_user_agent(endpoint).await?);
137142

138143
let writer = BufferedWriter::new(transport, write_stream(), format());
139144
assert_eq!(writer.write_stream(), write_stream());

‎src/bigquery/src/write/client.rs‎

Lines changed: 74 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ impl Write {
3838
}
3939

4040
pub(crate) async fn new(builder: ClientBuilder) -> BuilderResult<Self> {
41-
let inner = Arc::new(Transport::new(builder.config).await?);
41+
let inner = Arc::new(Transport::new(builder.config, builder.request_options).await?);
4242
let pools = Arc::new(Mutex::new(HashMap::new()));
4343
Ok(Self {
4444
inner,
@@ -171,9 +171,81 @@ mod tests {
171171
use super::super::error::AppendError;
172172
use super::*;
173173
use crate::model::{ArrowRecordBatch, ArrowSchema, ProtoRows, ProtoSchema};
174+
use crate::write::retry_policy::RetryableErrors;
175+
use crate::write::test::{
176+
NoBackoff, USER_AGENT, check_user_agent, convert, rows, schema, test_response,
177+
};
178+
use bigquery_grpc_mock::google::cloud::bigquery::storage::v1;
174179
use bigquery_grpc_mock::{MockBigQueryWrite, start};
175-
use gaxi::grpc::tonic::Status as TonicStatus;
180+
use gaxi::grpc::tonic::{Response as TonicResponse, Status as TonicStatus};
176181
use google_cloud_auth::credentials::anonymous::Builder as Anonymous;
182+
use google_cloud_gax::retry_policy::RetryPolicyExt;
183+
use test_case::test_case;
184+
use tokio::sync::mpsc;
185+
186+
#[test_case(false; "standalone")]
187+
#[test_case(true; "multiplexed")]
188+
#[tokio::test]
189+
async fn default_stream_user_agent(multiplexing: bool) -> anyhow::Result<()> {
190+
let (response_tx, response_rx) = mpsc::channel(1);
191+
let mut mock = MockBigQueryWrite::new();
192+
if multiplexing {
193+
mock.expect_get_write_stream()
194+
.times(2)
195+
.returning(move |req| {
196+
check_user_agent(req.metadata());
197+
Ok(TonicResponse::new(v1::WriteStream {
198+
name: req.into_inner().name,
199+
location: "us".to_string(),
200+
..Default::default()
201+
}))
202+
});
203+
}
204+
let mut sequence = mockall::Sequence::new();
205+
mock.expect_append_rows()
206+
.times(1)
207+
.in_sequence(&mut sequence)
208+
.return_once(move |req| {
209+
check_user_agent(req.metadata());
210+
Err(TonicStatus::unavailable("retry opening the stream"))
211+
});
212+
mock.expect_append_rows()
213+
.times(1)
214+
.in_sequence(&mut sequence)
215+
.return_once(move |req| {
216+
check_user_agent(req.metadata());
217+
Ok(TonicResponse::from(response_rx))
218+
});
219+
let (endpoint, _server) = start("0.0.0.0:0", mock).await?;
220+
let client = Write::builder()
221+
.with_endpoint(endpoint)
222+
.with_credentials(Anonymous::new().build())
223+
.with_user_agent(USER_AGENT)
224+
.build()
225+
.await?;
226+
let writer = client
227+
.open_default_stream("projects/p/datasets/d/tables/t")
228+
.with_multiplexing(multiplexing)
229+
.with_retry_policy(RetryableErrors.with_attempt_limit(2))
230+
.with_backoff_policy(NoBackoff)
231+
.with_attempt_timeout(std::time::Duration::from_secs(5))
232+
.build_arrow(schema())
233+
.await?;
234+
response_tx.send(Ok(convert(&test_response(1)))).await?;
235+
assert_eq!(writer.append(rows(1)).send().await?.offset, Some(1));
236+
237+
if multiplexing {
238+
let other = client
239+
.open_default_stream("projects/p/datasets/d/tables/other")
240+
.with_multiplexing(true)
241+
.build_arrow(schema())
242+
.await?;
243+
assert!(Arc::ptr_eq(&writer.inner.pool, &other.inner.pool));
244+
response_tx.send(Ok(convert(&test_response(2)))).await?;
245+
assert_eq!(other.append(rows(2)).send().await?.offset, Some(2));
246+
}
247+
Ok(())
248+
}
177249

178250
#[tokio::test]
179251
async fn arrow() -> anyhow::Result<()> {

‎src/bigquery/src/write/client_builder.rs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
use super::pool::StreamPoolOptions;
1616
use super::retry_policy::RetryOptions;
1717
use crate::ClientBuilderResult as BuilderResult;
18+
use crate::RequestOptions;
1819
use crate::client::Write;
1920
use gaxi::options::ClientConfig;
2021
use google_cloud_auth::credentials::Credentials;
@@ -38,6 +39,7 @@ use std::time::Duration;
3839
#[derive(Debug)]
3940
pub struct ClientBuilder {
4041
pub(super) config: ClientConfig,
42+
pub(super) request_options: RequestOptions,
4143
pub(super) pool_options: StreamPoolOptions,
4244
pub(super) retry_options: RetryOptions,
4345
}
@@ -46,6 +48,7 @@ impl ClientBuilder {
4648
pub(super) fn new() -> Self {
4749
Self {
4850
config: ClientConfig::default(),
51+
request_options: RequestOptions::default(),
4952
pool_options: StreamPoolOptions::default(),
5053
retry_options: RetryOptions::default(),
5154
}
@@ -64,6 +67,26 @@ impl ClientBuilder {
6467
Write::new(self).await
6568
}
6669

70+
/// Sets the user agent header for requests made by this client and its writers.
71+
///
72+
/// This includes append requests on shared streams and replacement streams
73+
/// opened when retrying a write.
74+
///
75+
/// # Example
76+
/// ```
77+
/// # use google_cloud_bigquery::client::Write;
78+
/// # async fn sample() -> anyhow::Result<()> {
79+
/// let client = Write::builder()
80+
/// .with_user_agent("my-app/1.2.3")
81+
/// .build()
82+
/// .await?;
83+
/// # Ok(()) }
84+
/// ```
85+
pub fn with_user_agent(mut self, user_agent: impl Into<String>) -> Self {
86+
self.request_options.set_user_agent(user_agent);
87+
self
88+
}
89+
6790
/// Sets the endpoint.
6891
///
6992
/// # Example

‎src/bigquery/src/write/committed.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -76,17 +76,20 @@ mod tests {
7676
let (response_tx, response_rx) = mpsc::channel(10);
7777

7878
let mut mock = MockBigQueryWrite::new();
79-
mock.expect_append_rows()
80-
.return_once(|_| Ok(TonicResponse::from(response_rx)));
79+
mock.expect_append_rows().return_once(|req| {
80+
check_user_agent(req.metadata());
81+
Ok(TonicResponse::from(response_rx))
82+
});
8183

82-
mock.expect_finalize_write_stream().return_once(|_| {
84+
mock.expect_finalize_write_stream().return_once(|req| {
85+
check_user_agent(req.metadata());
8386
Ok(TonicResponse::new(
8487
bigquery_grpc_mock::google::cloud::bigquery::storage::v1::FinalizeWriteStreamResponse::default(),
8588
))
8689
});
8790

8891
let (endpoint, _server) = start("0.0.0.0:0", mock).await?;
89-
let transport = Arc::new(test_transport(endpoint).await?);
92+
let transport = Arc::new(test_transport_with_user_agent(endpoint).await?);
9093

9194
let writer = CommittedWriter::new(transport, write_stream(), format());
9295
assert_eq!(writer.write_stream(), write_stream());

‎src/bigquery/src/write/pending.rs‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ where
8282
.inner
8383
.client
8484
.batch_commit_write_streams()
85+
.with_options(self.inner.request_options.clone())
8586
.set_parent(parent)
8687
.set_write_streams(vec![self.inner.write_stream.clone()])
8788
.send()
@@ -113,16 +114,20 @@ mod tests {
113114
let (response_tx, response_rx) = mpsc::channel(10);
114115

115116
let mut mock = MockBigQueryWrite::new();
116-
mock.expect_append_rows()
117-
.return_once(|_| Ok(TonicResponse::from(response_rx)));
117+
mock.expect_append_rows().return_once(|req| {
118+
check_user_agent(req.metadata());
119+
Ok(TonicResponse::from(response_rx))
120+
});
118121

119-
mock.expect_finalize_write_stream().return_once(|_| {
122+
mock.expect_finalize_write_stream().return_once(|req| {
123+
check_user_agent(req.metadata());
120124
Ok(TonicResponse::new(
121125
v1::FinalizeWriteStreamResponse::default(),
122126
))
123127
});
124128

125-
mock.expect_batch_commit_write_streams().return_once(|_| {
129+
mock.expect_batch_commit_write_streams().return_once(|req| {
130+
check_user_agent(req.metadata());
126131
Ok(TonicResponse::new(v1::BatchCommitWriteStreamsResponse {
127132
commit_time: Some(prost_types::Timestamp {
128133
seconds: 1_700_000_000,
@@ -133,7 +138,7 @@ mod tests {
133138
});
134139

135140
let (endpoint, _server) = start("0.0.0.0:0", mock).await?;
136-
let transport = Arc::new(test_transport(endpoint).await?);
141+
let transport = Arc::new(test_transport_with_user_agent(endpoint).await?);
137142

138143
let writer = PendingWriter::new(transport, write_stream(), format());
139144
assert_eq!(writer.write_stream(), write_stream());

‎src/bigquery/src/write/stream.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ use super::transport::Transport;
1616
use crate::google::cloud::bigquery::storage::v1::{AppendRowsRequest, AppendRowsResponse};
1717
use crate::{Error, Result};
1818
use gaxi::grpc::tonic::Streaming;
19-
use google_cloud_gax::options::RequestOptions;
2019
use std::sync::Arc;
2120
use tokio::sync::mpsc::{Sender, channel};
2221

@@ -43,7 +42,7 @@ async fn open_stream(inner: Arc<Transport>, initial_req: AppendRowsRequest) -> R
4342
request_tx.send(initial_req).await.map_err(Error::io)?;
4443

4544
let stream = inner
46-
.append_rows(&request_params, request_rx, RequestOptions::default())
45+
.append_rows(&request_params, request_rx)
4746
.await?
4847
.into_inner();
4948

‎src/bigquery/src/write/test.rs‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ use crate::google::cloud::bigquery::storage::v1::{AppendRowsRequest, AppendRowsR
2626
use crate::model::{ArrowRecordBatch, ArrowSchema, ProtoSchema};
2727
use bigquery_grpc_mock::google::cloud::bigquery::storage::v1;
2828
use bytes::Bytes;
29+
use gaxi::grpc::tonic::MetadataMap;
2930
use google_cloud_auth::credentials::anonymous::Builder as Anonymous;
3031
use google_cloud_gax::backoff_policy::BackoffPolicy;
3132
use google_cloud_gax::retry_policy::NeverRetry;
@@ -81,7 +82,28 @@ pub(super) async fn test_transport<T: Into<String>>(endpoint: T) -> anyhow::Resu
8182
let mut config = gaxi::options::ClientConfig::default();
8283
config.cred = Some(Anonymous::new().build());
8384
config.endpoint = Some(endpoint.into());
84-
Ok(Transport::new(config).await?)
85+
Ok(Transport::new(config, crate::RequestOptions::default()).await?)
86+
}
87+
88+
pub(super) const USER_AGENT: &str = "my-app/1.2.3";
89+
90+
pub(super) fn check_user_agent(metadata: &MetadataMap) {
91+
let header = metadata.get("user-agent").expect("user-agent header");
92+
// Tonic also includes its own user-agent component.
93+
assert!(
94+
header
95+
.to_str()
96+
.unwrap()
97+
.split_whitespace()
98+
.any(|v| v == USER_AGENT),
99+
"{header:?}"
100+
);
101+
}
102+
103+
pub(super) async fn test_transport_with_user_agent(endpoint: String) -> anyhow::Result<Transport> {
104+
let mut transport = test_transport(endpoint).await?;
105+
transport.options.set_user_agent(USER_AGENT);
106+
Ok(transport)
85107
}
86108

87109
// Both crates have their own copies of the protos. We can just serialize

0 commit comments

Comments
 (0)