@@ -244,7 +244,11 @@ fn extract_s3_config_options(
244244 // and treats non-boolean text as that default. object_store expects the inverse flag.
245245 let path_style_access = get_config_trimmed ( configs, bucket, "path.style.access" )
246246 . is_some_and ( |value| value. eq_ignore_ascii_case ( "true" ) ) ;
247- let mut virtual_hosted_style_request = !path_style_access;
247+ // The AWS SDK addresses a bucket whose name contains a dot path-style over HTTPS, because
248+ // the dotted host does not match S3's wildcard certificate. The default AWS endpoint is
249+ // HTTPS, and normalize_endpoint applies the same rule to a custom one by its scheme.
250+ let mut virtual_hosted_style_request =
251+ !path_style_access && !bucket_needs_path_style_over_https ( bucket) ;
248252
249253 // Extract endpoint configuration and shape it for the selected addressing style. The flag is
250254 // taken from the normalized result so the endpoint and the flag never disagree.
@@ -272,6 +276,12 @@ fn extract_s3_config_options(
272276 s3_configs
273277}
274278
279+ /// Whether the AWS SDK would refuse to virtual-host `bucket` over HTTPS: a dot in the name
280+ /// makes `bucket.s3.<region>.amazonaws.com` fall outside S3's wildcard certificate.
281+ fn bucket_needs_path_style_over_https ( bucket : & str ) -> bool {
282+ bucket. contains ( '.' )
283+ }
284+
275285/// An endpoint shaped for object_store together with the addressing mode it was shaped for.
276286#[ derive( Debug , Clone , PartialEq ) ]
277287struct NormalizedEndpoint {
@@ -313,6 +323,9 @@ fn normalize_endpoint(
313323 if !virtual_hosted_style_request {
314324 return path_style ( endpoint) ;
315325 }
326+ if endpoint. starts_with ( "https://" ) && bucket_needs_path_style_over_https ( bucket) {
327+ return path_style ( endpoint) ;
328+ }
316329
317330 // Fall back to the endpoint as written when it cannot be parsed so object_store reports
318331 // the malformed value instead of a mangled one
@@ -336,6 +349,14 @@ fn normalize_endpoint(
336349 } )
337350}
338351
352+ /// The credentials file Hadoop's profile provider reads when none is configured:
353+ /// `AWS_SHARED_CREDENTIALS_FILE` when set, otherwise `~/.aws/credentials`.
354+ fn default_shared_credentials_file ( env_override : Option < String > , home : Option < String > ) -> String {
355+ env_override
356+ . filter ( |path| !path. trim ( ) . is_empty ( ) )
357+ . unwrap_or_else ( || format ! ( "{}/.aws/credentials" , home. unwrap_or_default( ) ) )
358+ }
359+
339360fn get_config < ' a > (
340361 configs : & ' a HashMap < String , String > ,
341362 bucket : & str ,
@@ -532,10 +553,12 @@ fn build_aws_credential_provider_metadata(
532553 HADOOP_PROFILE => Ok ( CredentialProviderMetadata :: Profile {
533554 name : get_non_empty_config ( configs, bucket, "auth.profile.name" ) ,
534555 file : get_non_empty_config ( configs, bucket, "auth.profile.file" ) ,
556+ credentials_only : true ,
535557 } ) ,
536558 AWS_PROFILE_V1 | AWS_PROFILE => Ok ( CredentialProviderMetadata :: Profile {
537559 name : None ,
538560 file : None ,
561+ credentials_only : false ,
539562 } ) ,
540563 _ => Err ( object_store:: Error :: Generic {
541564 store : "S3" ,
@@ -770,6 +793,9 @@ enum CredentialProviderMetadata {
770793 Profile {
771794 name : Option < String > ,
772795 file : Option < String > ,
796+ // Hadoop's ProfileAWSCredentialsProvider reads only the credentials file, while the
797+ // SDK spellings merge the SDK's config and credentials files.
798+ credentials_only : bool ,
773799 } ,
774800 Static {
775801 is_valid : bool ,
@@ -809,7 +835,7 @@ impl CredentialProviderMetadata {
809835 CredentialProviderMetadata :: Imds => "Imds" . to_string ( ) ,
810836 CredentialProviderMetadata :: Environment => "Environment" . to_string ( ) ,
811837 CredentialProviderMetadata :: WebIdentity => "WebIdentity" . to_string ( ) ,
812- CredentialProviderMetadata :: Profile { name, file } => {
838+ CredentialProviderMetadata :: Profile { name, file, .. } => {
813839 let overrides: Vec < String > = [ ( "name" , name) , ( "file" , file) ]
814840 . into_iter ( )
815841 . filter_map ( |( key, value) | value. as_ref ( ) . map ( |v| format ! ( "{key}: {v}" ) ) )
@@ -882,15 +908,28 @@ impl CredentialProviderMetadata {
882908 . build ( ) ;
883909 Ok ( Arc :: new ( credential_provider) )
884910 }
885- CredentialProviderMetadata :: Profile { name, file } => {
911+ CredentialProviderMetadata :: Profile {
912+ name,
913+ file,
914+ credentials_only,
915+ } => {
886916 let mut builder = ProfileFileCredentialsProvider :: builder ( )
887917 . configure ( & ProviderConfig :: with_default_region ( ) . await ) ;
888918 if let Some ( name) = name {
889919 builder = builder. profile_name ( name) ;
890920 }
891- if let Some ( file) = file {
892- // Hadoop's ProfileAWSCredentialsProvider loads fs.s3a.auth.profile.file as a
893- // credentials-format file and reads nothing else, so mirror that here
921+ // Hadoop's ProfileAWSCredentialsProvider loads the configured file, or the
922+ // shared credentials file, as a credentials-format file and reads nothing
923+ // else, so a same-name role profile in the SDK's config file never applies.
924+ let credentials_file = match ( file, credentials_only) {
925+ ( Some ( file) , _) => Some ( file. clone ( ) ) ,
926+ ( None , true ) => Some ( default_shared_credentials_file (
927+ std:: env:: var ( "AWS_SHARED_CREDENTIALS_FILE" ) . ok ( ) ,
928+ std:: env:: var ( "HOME" ) . ok ( ) ,
929+ ) ) ,
930+ ( None , false ) => None ,
931+ } ;
932+ if let Some ( file) = credentials_file {
894933 builder = builder. profile_files (
895934 EnvConfigFiles :: builder ( )
896935 . with_file ( EnvConfigFileKind :: Credentials , file)
@@ -1672,6 +1711,7 @@ mod tests {
16721711 CredentialProviderMetadata :: Profile {
16731712 name: expected_name. map( str :: to_string) ,
16741713 file: expected_file. map( str :: to_string) ,
1714+ credentials_only: true ,
16751715 } ,
16761716 "provider {provider_name}, name {name:?}, file {file:?}"
16771717 ) ;
@@ -1701,6 +1741,7 @@ mod tests {
17011741 CredentialProviderMetadata :: Profile {
17021742 name: None ,
17031743 file: None ,
1744+ credentials_only: false ,
17041745 } ,
17051746 "provider {provider_name}"
17061747 ) ;
@@ -1753,6 +1794,7 @@ mod tests {
17531794 CredentialProviderMetadata :: Profile {
17541795 name: Some ( expected_name. to_string( ) ) ,
17551796 file: Some ( expected_file. to_string( ) ) ,
1797+ credentials_only: true ,
17561798 } ,
17571799 "bucket {bucket}"
17581800 ) ;
@@ -1783,6 +1825,7 @@ mod tests {
17831825 CredentialProviderMetadata :: Profile {
17841826 name: Some ( "analytics" . to_string( ) ) ,
17851827 file: Some ( "/etc/aws/credentials" . to_string( ) ) ,
1828+ credentials_only: true ,
17861829 } ,
17871830 CredentialProviderMetadata :: Imds ,
17881831 ] )
@@ -2285,15 +2328,79 @@ mod tests {
22852328 ) ;
22862329 }
22872330
2331+ // A dotted bucket over HTTPS stays path-style, as the AWS SDK addresses it, since the
2332+ // dotted host falls outside S3's wildcard certificate; over HTTP it is virtual-hosted.
22882333 assert_eq ! (
22892334 normalize_endpoint( "custom.endpoint.com" , "my.dotted.bucket" , true ) ,
22902335 Some ( NormalizedEndpoint {
2291- endpoint: "https://my.dotted.bucket.custom.endpoint.com" . to_string( ) ,
2336+ endpoint: "https://custom.endpoint.com" . to_string( ) ,
2337+ virtual_hosted_style_request: false ,
2338+ } )
2339+ ) ;
2340+ assert_eq ! (
2341+ normalize_endpoint( "http://custom.endpoint.com" , "my.dotted.bucket" , true ) ,
2342+ Some ( NormalizedEndpoint {
2343+ endpoint: "http://my.dotted.bucket.custom.endpoint.com" . to_string( ) ,
22922344 virtual_hosted_style_request: true ,
22932345 } )
22942346 ) ;
22952347 }
22962348
2349+ #[ test]
2350+ fn test_extract_s3_config_dotted_bucket_stays_path_style_on_default_endpoint ( ) {
2351+ // No custom endpoint means the HTTPS AWS endpoint, where a dotted bucket must be
2352+ // addressed path-style whatever the flag says.
2353+ let configs = TestConfigBuilder :: new ( ) . with_region ( "us-east-1" ) . build ( ) ;
2354+ let s3_configs = extract_s3_config_options ( & configs, "review.dotted.bucket" ) ;
2355+ assert_eq ! (
2356+ s3_configs. get( & AmazonS3ConfigKey :: VirtualHostedStyleRequest ) ,
2357+ Some ( & "false" . to_string( ) )
2358+ ) ;
2359+ assert ! ( !s3_configs. contains_key( & AmazonS3ConfigKey :: Endpoint ) ) ;
2360+
2361+ let configs = TestConfigBuilder :: new ( )
2362+ . with_region ( "us-east-1" )
2363+ . with_property ( "endpoint" , "https://s3.us-east-1.amazonaws.com" )
2364+ . build ( ) ;
2365+ let s3_configs = extract_s3_config_options ( & configs, "review.dotted.bucket" ) ;
2366+ assert_eq ! (
2367+ s3_configs. get( & AmazonS3ConfigKey :: VirtualHostedStyleRequest ) ,
2368+ Some ( & "false" . to_string( ) )
2369+ ) ;
2370+ assert_eq ! (
2371+ s3_configs. get( & AmazonS3ConfigKey :: Endpoint ) ,
2372+ Some ( & "https://s3.us-east-1.amazonaws.com" . to_string( ) )
2373+ ) ;
2374+
2375+ let s3_configs = extract_s3_config_options ( & configs, "plainbucket" ) ;
2376+ assert_eq ! (
2377+ s3_configs. get( & AmazonS3ConfigKey :: VirtualHostedStyleRequest ) ,
2378+ Some ( & "true" . to_string( ) )
2379+ ) ;
2380+ }
2381+
2382+ #[ test]
2383+ fn test_default_shared_credentials_file_matches_hadoop ( ) {
2384+ assert_eq ! (
2385+ default_shared_credentials_file( None , Some ( "/home/comet" . to_string( ) ) ) ,
2386+ "/home/comet/.aws/credentials"
2387+ ) ;
2388+ assert_eq ! (
2389+ default_shared_credentials_file(
2390+ Some ( "/etc/aws/shared" . to_string( ) ) ,
2391+ Some ( "/home/comet" . to_string( ) )
2392+ ) ,
2393+ "/etc/aws/shared"
2394+ ) ;
2395+ assert_eq ! (
2396+ default_shared_credentials_file(
2397+ Some ( " " . to_string( ) ) ,
2398+ Some ( "/home/comet" . to_string( ) )
2399+ ) ,
2400+ "/home/comet/.aws/credentials"
2401+ ) ;
2402+ }
2403+
22972404 #[ test]
22982405 fn test_normalize_endpoint_path_style ( ) {
22992406 // Path-style leaves the endpoint as configured apart from the https default, since
@@ -2594,16 +2701,19 @@ mod tests {
25942701 let profile_metadata = CredentialProviderMetadata :: Profile {
25952702 name : None ,
25962703 file : None ,
2704+ credentials_only : false ,
25972705 } ;
25982706 assert_eq ! ( profile_metadata. simple_string( ) , "Profile" ) ;
25992707 let profile_metadata = CredentialProviderMetadata :: Profile {
26002708 name : Some ( "analytics" . to_string ( ) ) ,
26012709 file : None ,
2710+ credentials_only : true ,
26022711 } ;
26032712 assert_eq ! ( profile_metadata. simple_string( ) , "Profile(name: analytics)" ) ;
26042713 let profile_metadata = CredentialProviderMetadata :: Profile {
26052714 name : Some ( "analytics" . to_string ( ) ) ,
26062715 file : Some ( "/etc/aws/credentials" . to_string ( ) ) ,
2716+ credentials_only : true ,
26072717 } ;
26082718 assert_eq ! (
26092719 profile_metadata. simple_string( ) ,
0 commit comments