Skip to content

Commit b6cff68

Browse files
authored
feat(http-transport-reqwest): allow tls options (#7798)
* feat(http-transport-reqwest): allow tls options * Update reqwest feature * fix reqwest tls backend configuration * fix reqwest rustls webpki feature name * add reqwest rustls ring backend * clarify reqwest tls backend default * Update comment * simplify reqwest tls backend selection * Address issues in feature configurations, dependencies * Set default transport to rustls * Fix compilation error in CI * Remove non TLS backend build * fix Cargo.toml * Bring back default but point to default feature * Fix default feature set * Fix default feature * Simplify reqwest TLS configuration * Document reqwest webpki roots TLS setup * Prettify comment * Remove opendal_http_transport_reqwest export in core * fix linting * Remove default builder
1 parent 311f324 commit b6cff68

8 files changed

Lines changed: 324 additions & 24 deletions

File tree

‎bindings/dotnet/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ doc = false
3434
# this crate won't be published, we always use the local version
3535
opendal = { version = ">=0", path = "../../core", features = [
3636
"blocking",
37-
"reqwest-rustls-tls",
37+
"http-transport-reqwest",
3838
"executors-tokio",
3939

4040
# enabled layers

‎bindings/java/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ jni = { version = "0.22.4" }
3535
# opendal-java won't be published to crates.io, we always use the local version
3636
opendal = { version = ">=0", path = "../../core", default-features = false, features = [
3737
"blocking",
38-
"reqwest-rustls-tls",
38+
"http-transport-reqwest-rustls",
3939
"executors-tokio",
4040

4141
# enabled layers

‎core/Cargo.lock‎

Lines changed: 19 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎core/Cargo.toml‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -87,15 +87,32 @@ blocking = ["opendal-core/blocking"]
8787
default = [
8888
"auto-register-services",
8989
"http-transport-reqwest",
90-
"reqwest-rustls-tls",
9190
"executors-tokio",
9291
"layers-concurrent-limit",
9392
"layers-logging",
9493
"layers-retry",
9594
"layers-timeout",
9695
]
9796
executors-tokio = ["opendal-core/executors-tokio"]
98-
http-transport-reqwest = ["dep:opendal-http-transport-reqwest"]
97+
# Default http-transport-reqwest transport feature, used for:
98+
# - install default http transport
99+
# - feature gate for various of bindings dependency
100+
http-transport-reqwest = ["http-transport-reqwest-rustls"]
101+
# Use reqwest with platform TLS via native-tls.
102+
http-transport-reqwest-native-tls = [
103+
"dep:opendal-http-transport-reqwest",
104+
"opendal-http-transport-reqwest/native-tls",
105+
]
106+
# Use reqwest with Rustls, with default aws-lc-rs crypto provider.
107+
http-transport-reqwest-rustls = [
108+
"dep:opendal-http-transport-reqwest",
109+
"opendal-http-transport-reqwest/rustls",
110+
]
111+
# Use reqwest with Rustls and no built-in crypto provider.
112+
http-transport-reqwest-rustls-no-provider = [
113+
"dep:opendal-http-transport-reqwest",
114+
"opendal-http-transport-reqwest/rustls-no-provider",
115+
]
99116
internal-tokio-rt = ["opendal-core/internal-tokio-rt"]
100117
layers-async-backtrace = ["dep:opendal-layer-async-backtrace"]
101118
layers-await-tree = ["dep:opendal-layer-await-tree"]
@@ -121,14 +138,13 @@ layers-tail-cut = ["dep:opendal-layer-tail-cut"]
121138
layers-throttle = ["dep:opendal-layer-throttle"]
122139
layers-timeout = ["dep:opendal-layer-timeout"]
123140
layers-tracing = ["dep:opendal-layer-tracing"]
141+
# Deprecated since 0.58; Will delete in 0.59.
124142
reqwest-rustls-no-provider-tls = [
125143
"http-transport-reqwest",
126-
"opendal-http-transport-reqwest/rustls-no-provider",
127-
]
128-
reqwest-rustls-tls = [
129-
"http-transport-reqwest",
130-
"opendal-http-transport-reqwest/rustls",
144+
"http-transport-reqwest-rustls-no-provider",
131145
]
146+
# Deprecated since 0.58; Will delete in 0.59.
147+
reqwest-rustls-tls = ["http-transport-reqwest", "http-transport-reqwest-rustls"]
132148
services-aliyun-drive = ["dep:opendal-service-aliyun-drive"]
133149
services-alluxio = ["dep:opendal-service-alluxio"]
134150
services-azblob = ["dep:opendal-service-azblob"]

‎core/http-transports/reqwest/Cargo.toml‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,20 @@ version = { workspace = true }
3131
all-features = true
3232

3333
[features]
34-
default = []
34+
default = ["rustls"]
35+
# Use platform TLS via reqwest's native-tls feature.
36+
native-tls = ["reqwest/native-tls"]
37+
# Use Rustls with reqwest's default crypto provider and platform certificate verification.
3538
rustls = ["reqwest/rustls"]
39+
# Use Rustls without a built-in crypto provider.
40+
#
41+
# `rustls` builds `aws_lc_rs` crypto provider by default. When using other TLS backends,
42+
# consider features like:
43+
# - ring
44+
# - fips
45+
# Also read more about `rustls-native-certs` crate.
46+
#
47+
# To use WebPKI roots, read more about `webpki_roots` crate as well.
3648
rustls-no-provider = ["reqwest/rustls-no-provider"]
3749

3850
[dependencies]
Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
# opendal-http-transport-reqwest
2+
3+
Reqwest-based HTTP transport for [Apache OpenDAL](https://opendal.apache.org).
4+
5+
This crate provides `ReqwestTransport`, an implementation of OpenDAL's
6+
`HttpTransport` trait backed by [reqwest](https://crates.io/crates/reqwest).
7+
8+
## TLS configuration
9+
10+
OpenDAL does not add another TLS backend selector on top of reqwest. Pick the
11+
Cargo feature set you need, build a `reqwest::Client` with reqwest's
12+
`ClientBuilder`, and pass that client to `ReqwestTransport::new`.
13+
14+
The default feature is `rustls`. When `ReqwestTransport::default()` is used on
15+
native targets, this crate explicitly asks reqwest for the Rustls backend if the
16+
`rustls` feature is compiled in.
17+
18+
If this crate is built with only `rustls-no-provider`, install a Rustls default
19+
crypto provider before using `ReqwestTransport::default()`, or build a
20+
preconfigured `reqwest::Client` yourself and pass it to `ReqwestTransport::new`.
21+
22+
### Reqwest TLS resolution
23+
24+
Reqwest has its own TLS resolution rules. For OpenDAL users, the practical
25+
resolution order is:
26+
27+
- Explicit reqwest client configuration wins. Call
28+
`ClientBuilder::tls_backend_rustls()`,
29+
`ClientBuilder::tls_backend_native()`, or
30+
`ClientBuilder::tls_backend_preconfigured()`, but reqwest documents that API
31+
as semver-unstable because the concrete TLS config type must match reqwest's
32+
dependency versions.
33+
- Otherwise, reqwest's automatic TLS backend selection applies. The reqwest
34+
`default-tls` feature takes precedence if any crate in the dependency tree
35+
enables it. Disable reqwest defaults with `default-features = false` when you
36+
need deterministic TLS features.
37+
- Cargo features are additive. If multiple TLS backends are compiled by feature
38+
unification, use explicit client configuration instead of relying on automatic
39+
selection.
40+
41+
See reqwest's TLS documentation:
42+
43+
- [reqwest TLS module](https://docs.rs/reqwest/latest/reqwest/tls/index.html)
44+
- [ClientBuilder::tls_backend_rustls](https://docs.rs/reqwest/latest/reqwest/struct.ClientBuilder.html#method.tls_backend_rustls)
45+
- [ClientBuilder::tls_backend_native](https://docs.rs/reqwest/latest/reqwest/struct.ClientBuilder.html#method.tls_backend_native)
46+
- [ClientBuilder::tls_backend_preconfigured](https://docs.rs/reqwest/latest/reqwest/struct.ClientBuilder.html#method.tls_backend_preconfigured)
47+
48+
### Feature matrix
49+
50+
| Feature | Crypto provider | Certificate roots | Use when |
51+
|---------|-----------------|-------------------|----------|
52+
| `rustls` (default) | reqwest default | Platform verifier | Pure-Rust TLS with the system trust store |
53+
| `rustls-no-provider` | You provide | You provide | BYO crypto provider, roots, or FIPS module |
54+
| `native-tls` | OS library | OS trust store | You want platform TLS |
55+
56+
## Cargo features
57+
58+
Most users depend on the `opendal` facade crate:
59+
60+
```toml
61+
[dependencies]
62+
# Default: reqwest transport with rustls.
63+
opendal = { version = "0.57" }
64+
```
65+
66+
To select one transport TLS feature explicitly, disable default features:
67+
68+
```toml
69+
[dependencies]
70+
opendal = { version = "0.57", default-features = false, features = [
71+
"http-transport-reqwest-rustls",
72+
] }
73+
```
74+
75+
## Build different TLS backends
76+
77+
### Rustls
78+
79+
Use reqwest's Rustls backend explicitly when you build the client:
80+
81+
```toml
82+
[dependencies]
83+
opendal = { version = "0.57", default-features = false, features = [
84+
"http-transport-reqwest-rustls",
85+
] }
86+
reqwest = { version = "0.13.4", default-features = false, features = [
87+
"rustls",
88+
"stream",
89+
] }
90+
```
91+
92+
```rust
93+
use std::time::Duration;
94+
95+
use opendal::HttpTransporter;
96+
use opendal::ReqwestTransport;
97+
98+
fn build_transport() -> Result<HttpTransporter, Box<dyn std::error::Error>> {
99+
let client = reqwest::Client::builder()
100+
.tls_backend_rustls()
101+
.connect_timeout(Duration::from_secs(10))
102+
.build()?;
103+
104+
Ok(HttpTransporter::new(ReqwestTransport::new(client)))
105+
}
106+
```
107+
108+
### Native TLS
109+
110+
Use `native-tls` when you want reqwest to delegate TLS to the platform stack:
111+
112+
```toml
113+
[dependencies]
114+
opendal = { version = "0.57", default-features = false, features = [
115+
"http-transport-reqwest-native-tls",
116+
] }
117+
reqwest = { version = "0.13.4", default-features = false, features = [
118+
"native-tls",
119+
"stream",
120+
] }
121+
```
122+
123+
```rust
124+
use opendal::HttpTransporter;
125+
use opendal::ReqwestTransport;
126+
127+
fn build_transport() -> Result<HttpTransporter, Box<dyn std::error::Error>> {
128+
let client = reqwest::Client::builder()
129+
.tls_backend_native()
130+
.build()?;
131+
132+
Ok(HttpTransporter::new(ReqwestTransport::new(client)))
133+
}
134+
```
135+
136+
### rustls-no-provider
137+
138+
When using `wekpki-roots`, you could use `rustls-no-provider`.
139+
140+
Use `rustls-no-provider` when the application owns the Rustls provider and
141+
certificate roots. This example only configures Mozilla roots from
142+
`webpki-roots`; install or configure the Rustls crypto provider in application
143+
startup.
144+
145+
```toml
146+
[dependencies]
147+
opendal = { version = "0.57", default-features = false, features = [
148+
"http-transport-reqwest-rustls-no-provider",
149+
] }
150+
reqwest = { version = "0.13.4", default-features = false, features = [
151+
"rustls-no-provider",
152+
"stream",
153+
] }
154+
rustls = { version = "0.23", default-features = false, features = ["std"] }
155+
webpki-roots = "1"
156+
```
157+
158+
```rust
159+
use opendal::HttpTransporter;
160+
use opendal::ReqwestTransport;
161+
162+
fn build_transport() -> Result<HttpTransporter, Box<dyn std::error::Error>> {
163+
let root_store =
164+
rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
165+
166+
let tls_config = rustls::ClientConfig::builder()
167+
.with_root_certificates(root_store)
168+
.with_no_client_auth();
169+
170+
let client = reqwest::Client::builder()
171+
.tls_backend_preconfigured(tls_config)
172+
.build()?;
173+
174+
Ok(HttpTransporter::new(ReqwestTransport::new(client)))
175+
}
176+
```
177+
178+
To use `ring` crate, please refer `rustls` documentation.
179+
180+
## WASM targets
181+
182+
On `wasm32-unknown-unknown` and `wasm32-none`, reqwest switches to its WASM
183+
client implementation. It uses the browser or host Fetch API instead of hyper.
184+
TLS is provided by the browser or host environment, so the effective choice is
185+
the system TLS stack exposed to that environment.
186+
187+
This means `native-tls`, `rustls`, `rustls-no-provider`, custom root stores,
188+
and custom Rustls crypto providers do not select the TLS implementation for
189+
wasm requests. See [reqwest's WASM documentation](https://docs.rs/reqwest/latest/reqwest/#wasm)
190+
for the target-specific limitations.
191+
192+
## License and Trademarks
193+
194+
Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0
195+
196+
Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation.

0 commit comments

Comments
 (0)