Skip to content

Commit 0c76e6c

Browse files
committed
fix: keep dotted buckets path-style over HTTPS and read only the credentials file for Hadoop's profile provider
1 parent 3f66db9 commit 0c76e6c

2 files changed

Lines changed: 120 additions & 8 deletions

File tree

‎docs/source/user-guide/latest/datasources.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,9 @@ All configuration options support bucket-specific overrides using the pattern `f
242242
`fs.s3a.path.style.access` selects how the bucket is placed in the request URL: virtual-hosted
243243
addressing (the default, `false`) sends requests to `https://<bucket>.<endpoint>`, while path-style
244244
(`true`) sends them to `https://<endpoint>/<bucket>`, which many S3-compatible services such as MinIO
245-
require. An endpoint whose host is an IP address is always addressed path-style, as the AWS SDK does.
245+
require. An endpoint whose host is an IP address is always addressed path-style, as the AWS SDK does,
246+
and so is a bucket whose name contains a dot over HTTPS, since the dotted host falls outside S3's
247+
wildcard certificate.
246248
Earlier Comet releases addressed every custom `fs.s3a.endpoint` path-style whatever the flag said,
247249
so a MinIO or Ceph RGW deployment that never set the flag now sends requests to `<bucket>.<host>`
248250
and fails with a DNS error; set `fs.s3a.path.style.access=true` to keep the previous behavior.

‎native/core/src/parquet/objectstore/s3.rs‎

Lines changed: 117 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,11 @@ fn extract_s3_config_options(
244244
// and treats non-boolean text as that default. object_store expects the inverse flag.
245245
let path_style_access = get_config_trimmed(configs, bucket, "path.style.access")
246246
.is_some_and(|value| value.eq_ignore_ascii_case("true"));
247-
let mut virtual_hosted_style_request = !path_style_access;
247+
// The AWS SDK addresses a bucket whose name contains a dot path-style over HTTPS, because
248+
// the dotted host does not match S3's wildcard certificate. The default AWS endpoint is
249+
// HTTPS, and normalize_endpoint applies the same rule to a custom one by its scheme.
250+
let mut virtual_hosted_style_request =
251+
!path_style_access && !bucket_needs_path_style_over_https(bucket);
248252

249253
// Extract endpoint configuration and shape it for the selected addressing style. The flag is
250254
// taken from the normalized result so the endpoint and the flag never disagree.
@@ -272,6 +276,12 @@ fn extract_s3_config_options(
272276
s3_configs
273277
}
274278

279+
/// Whether the AWS SDK would refuse to virtual-host `bucket` over HTTPS: a dot in the name
280+
/// makes `bucket.s3.<region>.amazonaws.com` fall outside S3's wildcard certificate.
281+
fn bucket_needs_path_style_over_https(bucket: &str) -> bool {
282+
bucket.contains('.')
283+
}
284+
275285
/// An endpoint shaped for object_store together with the addressing mode it was shaped for.
276286
#[derive(Debug, Clone, PartialEq)]
277287
struct NormalizedEndpoint {
@@ -313,6 +323,9 @@ fn normalize_endpoint(
313323
if !virtual_hosted_style_request {
314324
return path_style(endpoint);
315325
}
326+
if endpoint.starts_with("https://") && bucket_needs_path_style_over_https(bucket) {
327+
return path_style(endpoint);
328+
}
316329

317330
// Fall back to the endpoint as written when it cannot be parsed so object_store reports
318331
// the malformed value instead of a mangled one
@@ -336,6 +349,14 @@ fn normalize_endpoint(
336349
})
337350
}
338351

352+
/// The credentials file Hadoop's profile provider reads when none is configured:
353+
/// `AWS_SHARED_CREDENTIALS_FILE` when set, otherwise `~/.aws/credentials`.
354+
fn default_shared_credentials_file(env_override: Option<String>, home: Option<String>) -> String {
355+
env_override
356+
.filter(|path| !path.trim().is_empty())
357+
.unwrap_or_else(|| format!("{}/.aws/credentials", home.unwrap_or_default()))
358+
}
359+
339360
fn get_config<'a>(
340361
configs: &'a HashMap<String, String>,
341362
bucket: &str,
@@ -532,10 +553,12 @@ fn build_aws_credential_provider_metadata(
532553
HADOOP_PROFILE => Ok(CredentialProviderMetadata::Profile {
533554
name: get_non_empty_config(configs, bucket, "auth.profile.name"),
534555
file: get_non_empty_config(configs, bucket, "auth.profile.file"),
556+
credentials_only: true,
535557
}),
536558
AWS_PROFILE_V1 | AWS_PROFILE => Ok(CredentialProviderMetadata::Profile {
537559
name: None,
538560
file: None,
561+
credentials_only: false,
539562
}),
540563
_ => Err(object_store::Error::Generic {
541564
store: "S3",
@@ -770,6 +793,9 @@ enum CredentialProviderMetadata {
770793
Profile {
771794
name: Option<String>,
772795
file: Option<String>,
796+
// Hadoop's ProfileAWSCredentialsProvider reads only the credentials file, while the
797+
// SDK spellings merge the SDK's config and credentials files.
798+
credentials_only: bool,
773799
},
774800
Static {
775801
is_valid: bool,
@@ -809,7 +835,7 @@ impl CredentialProviderMetadata {
809835
CredentialProviderMetadata::Imds => "Imds".to_string(),
810836
CredentialProviderMetadata::Environment => "Environment".to_string(),
811837
CredentialProviderMetadata::WebIdentity => "WebIdentity".to_string(),
812-
CredentialProviderMetadata::Profile { name, file } => {
838+
CredentialProviderMetadata::Profile { name, file, .. } => {
813839
let overrides: Vec<String> = [("name", name), ("file", file)]
814840
.into_iter()
815841
.filter_map(|(key, value)| value.as_ref().map(|v| format!("{key}: {v}")))
@@ -882,15 +908,28 @@ impl CredentialProviderMetadata {
882908
.build();
883909
Ok(Arc::new(credential_provider))
884910
}
885-
CredentialProviderMetadata::Profile { name, file } => {
911+
CredentialProviderMetadata::Profile {
912+
name,
913+
file,
914+
credentials_only,
915+
} => {
886916
let mut builder = ProfileFileCredentialsProvider::builder()
887917
.configure(&ProviderConfig::with_default_region().await);
888918
if let Some(name) = name {
889919
builder = builder.profile_name(name);
890920
}
891-
if let Some(file) = file {
892-
// Hadoop's ProfileAWSCredentialsProvider loads fs.s3a.auth.profile.file as a
893-
// credentials-format file and reads nothing else, so mirror that here
921+
// Hadoop's ProfileAWSCredentialsProvider loads the configured file, or the
922+
// shared credentials file, as a credentials-format file and reads nothing
923+
// else, so a same-name role profile in the SDK's config file never applies.
924+
let credentials_file = match (file, credentials_only) {
925+
(Some(file), _) => Some(file.clone()),
926+
(None, true) => Some(default_shared_credentials_file(
927+
std::env::var("AWS_SHARED_CREDENTIALS_FILE").ok(),
928+
std::env::var("HOME").ok(),
929+
)),
930+
(None, false) => None,
931+
};
932+
if let Some(file) = credentials_file {
894933
builder = builder.profile_files(
895934
EnvConfigFiles::builder()
896935
.with_file(EnvConfigFileKind::Credentials, file)
@@ -1672,6 +1711,7 @@ mod tests {
16721711
CredentialProviderMetadata::Profile {
16731712
name: expected_name.map(str::to_string),
16741713
file: expected_file.map(str::to_string),
1714+
credentials_only: true,
16751715
},
16761716
"provider {provider_name}, name {name:?}, file {file:?}"
16771717
);
@@ -1701,6 +1741,7 @@ mod tests {
17011741
CredentialProviderMetadata::Profile {
17021742
name: None,
17031743
file: None,
1744+
credentials_only: false,
17041745
},
17051746
"provider {provider_name}"
17061747
);
@@ -1753,6 +1794,7 @@ mod tests {
17531794
CredentialProviderMetadata::Profile {
17541795
name: Some(expected_name.to_string()),
17551796
file: Some(expected_file.to_string()),
1797+
credentials_only: true,
17561798
},
17571799
"bucket {bucket}"
17581800
);
@@ -1783,6 +1825,7 @@ mod tests {
17831825
CredentialProviderMetadata::Profile {
17841826
name: Some("analytics".to_string()),
17851827
file: Some("/etc/aws/credentials".to_string()),
1828+
credentials_only: true,
17861829
},
17871830
CredentialProviderMetadata::Imds,
17881831
])
@@ -2285,15 +2328,79 @@ mod tests {
22852328
);
22862329
}
22872330

2331+
// A dotted bucket over HTTPS stays path-style, as the AWS SDK addresses it, since the
2332+
// dotted host falls outside S3's wildcard certificate; over HTTP it is virtual-hosted.
22882333
assert_eq!(
22892334
normalize_endpoint("custom.endpoint.com", "my.dotted.bucket", true),
22902335
Some(NormalizedEndpoint {
2291-
endpoint: "https://my.dotted.bucket.custom.endpoint.com".to_string(),
2336+
endpoint: "https://custom.endpoint.com".to_string(),
2337+
virtual_hosted_style_request: false,
2338+
})
2339+
);
2340+
assert_eq!(
2341+
normalize_endpoint("http://custom.endpoint.com", "my.dotted.bucket", true),
2342+
Some(NormalizedEndpoint {
2343+
endpoint: "http://my.dotted.bucket.custom.endpoint.com".to_string(),
22922344
virtual_hosted_style_request: true,
22932345
})
22942346
);
22952347
}
22962348

2349+
#[test]
2350+
fn test_extract_s3_config_dotted_bucket_stays_path_style_on_default_endpoint() {
2351+
// No custom endpoint means the HTTPS AWS endpoint, where a dotted bucket must be
2352+
// addressed path-style whatever the flag says.
2353+
let configs = TestConfigBuilder::new().with_region("us-east-1").build();
2354+
let s3_configs = extract_s3_config_options(&configs, "review.dotted.bucket");
2355+
assert_eq!(
2356+
s3_configs.get(&AmazonS3ConfigKey::VirtualHostedStyleRequest),
2357+
Some(&"false".to_string())
2358+
);
2359+
assert!(!s3_configs.contains_key(&AmazonS3ConfigKey::Endpoint));
2360+
2361+
let configs = TestConfigBuilder::new()
2362+
.with_region("us-east-1")
2363+
.with_property("endpoint", "https://s3.us-east-1.amazonaws.com")
2364+
.build();
2365+
let s3_configs = extract_s3_config_options(&configs, "review.dotted.bucket");
2366+
assert_eq!(
2367+
s3_configs.get(&AmazonS3ConfigKey::VirtualHostedStyleRequest),
2368+
Some(&"false".to_string())
2369+
);
2370+
assert_eq!(
2371+
s3_configs.get(&AmazonS3ConfigKey::Endpoint),
2372+
Some(&"https://s3.us-east-1.amazonaws.com".to_string())
2373+
);
2374+
2375+
let s3_configs = extract_s3_config_options(&configs, "plainbucket");
2376+
assert_eq!(
2377+
s3_configs.get(&AmazonS3ConfigKey::VirtualHostedStyleRequest),
2378+
Some(&"true".to_string())
2379+
);
2380+
}
2381+
2382+
#[test]
2383+
fn test_default_shared_credentials_file_matches_hadoop() {
2384+
assert_eq!(
2385+
default_shared_credentials_file(None, Some("/home/comet".to_string())),
2386+
"/home/comet/.aws/credentials"
2387+
);
2388+
assert_eq!(
2389+
default_shared_credentials_file(
2390+
Some("/etc/aws/shared".to_string()),
2391+
Some("/home/comet".to_string())
2392+
),
2393+
"/etc/aws/shared"
2394+
);
2395+
assert_eq!(
2396+
default_shared_credentials_file(
2397+
Some(" ".to_string()),
2398+
Some("/home/comet".to_string())
2399+
),
2400+
"/home/comet/.aws/credentials"
2401+
);
2402+
}
2403+
22972404
#[test]
22982405
fn test_normalize_endpoint_path_style() {
22992406
// Path-style leaves the endpoint as configured apart from the https default, since
@@ -2594,16 +2701,19 @@ mod tests {
25942701
let profile_metadata = CredentialProviderMetadata::Profile {
25952702
name: None,
25962703
file: None,
2704+
credentials_only: false,
25972705
};
25982706
assert_eq!(profile_metadata.simple_string(), "Profile");
25992707
let profile_metadata = CredentialProviderMetadata::Profile {
26002708
name: Some("analytics".to_string()),
26012709
file: None,
2710+
credentials_only: true,
26022711
};
26032712
assert_eq!(profile_metadata.simple_string(), "Profile(name: analytics)");
26042713
let profile_metadata = CredentialProviderMetadata::Profile {
26052714
name: Some("analytics".to_string()),
26062715
file: Some("/etc/aws/credentials".to_string()),
2716+
credentials_only: true,
26072717
};
26082718
assert_eq!(
26092719
profile_metadata.simple_string(),

0 commit comments

Comments
 (0)