Skip to content

Commit 31fafaf

Browse files
committed
fix: decide dotted-bucket addressing by the endpoint scheme and forward Hadoop's default credentials path from the JVM
1 parent 8652a81 commit 31fafaf

3 files changed

Lines changed: 140 additions & 10 deletions

File tree

‎native/core/src/parquet/objectstore/s3.rs‎

Lines changed: 89 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -244,20 +244,24 @@ fn extract_s3_config_options(
244244
// and treats non-boolean text as that default. object_store expects the inverse flag.
245245
let path_style_access = get_config_trimmed(configs, bucket, "path.style.access")
246246
.is_some_and(|value| value.eq_ignore_ascii_case("true"));
247-
// The AWS SDK addresses a bucket whose name contains a dot path-style over HTTPS, because
248-
// the dotted host does not match S3's wildcard certificate. The default AWS endpoint is
249-
// HTTPS, and normalize_endpoint applies the same rule to a custom one by its scheme.
250-
let mut virtual_hosted_style_request =
251-
!path_style_access && !bucket_needs_path_style_over_https(bucket);
247+
let mut virtual_hosted_style_request = !path_style_access;
252248

253249
// Extract endpoint configuration and shape it for the selected addressing style. The flag is
254-
// taken from the normalized result so the endpoint and the flag never disagree.
255-
if let Some(endpoint) = get_config_trimmed(configs, bucket, "endpoint") {
256-
if let Some(normalized) = normalize_endpoint(endpoint, bucket, virtual_hosted_style_request)
257-
{
250+
// taken from the normalized result so the endpoint and the flag never disagree. A custom
251+
// endpoint decides the dotted-bucket rule by its own scheme inside normalize_endpoint; the
252+
// default AWS endpoint is HTTPS, so the rule applies to it here.
253+
let custom_endpoint = get_config_trimmed(configs, bucket, "endpoint")
254+
.and_then(|endpoint| normalize_endpoint(endpoint, bucket, virtual_hosted_style_request));
255+
match custom_endpoint {
256+
Some(normalized) => {
258257
virtual_hosted_style_request = normalized.virtual_hosted_style_request;
259258
s3_configs.insert(AmazonS3ConfigKey::Endpoint, normalized.endpoint);
260259
}
260+
None => {
261+
if bucket_needs_path_style_over_https(bucket) {
262+
virtual_hosted_style_request = false;
263+
}
264+
}
261265
}
262266
s3_configs.insert(
263267
AmazonS3ConfigKey::VirtualHostedStyleRequest,
@@ -550,9 +554,13 @@ fn build_aws_credential_provider_metadata(
550554
// Only Hadoop's own provider reads the profile keys. Hadoop builds the SDK spellings
551555
// through the SDK's static constructor without its configuration, so applying the keys
552556
// to them here would authenticate the native side as a different identity.
557+
// With no configured file, Hadoop reads AWS_SHARED_CREDENTIALS_FILE or the JVM
558+
// user's ~/.aws/credentials; the JVM forwards that resolved path so both sides agree
559+
// even when the native process sees a different HOME.
553560
HADOOP_PROFILE => Ok(CredentialProviderMetadata::Profile {
554561
name: get_non_empty_config(configs, bucket, "auth.profile.name"),
555-
file: get_non_empty_config(configs, bucket, "auth.profile.file"),
562+
file: get_non_empty_config(configs, bucket, "auth.profile.file")
563+
.or_else(|| get_non_empty_config(configs, bucket, "comet.default.profile.file")),
556564
credentials_only: true,
557565
}),
558566
AWS_PROFILE_V1 | AWS_PROFILE => Ok(CredentialProviderMetadata::Profile {
@@ -2377,6 +2385,77 @@ mod tests {
23772385
s3_configs.get(&AmazonS3ConfigKey::VirtualHostedStyleRequest),
23782386
Some(&"true".to_string())
23792387
);
2388+
2389+
// A custom HTTP endpoint keeps virtual hosting for a dotted bucket, as the SDK does,
2390+
// since the certificate rule only applies to HTTPS.
2391+
let configs = TestConfigBuilder::new()
2392+
.with_property("endpoint", "http://storage.example.test")
2393+
.build();
2394+
let s3_configs = extract_s3_config_options(&configs, "review.dotted.bucket");
2395+
assert_eq!(
2396+
s3_configs.get(&AmazonS3ConfigKey::VirtualHostedStyleRequest),
2397+
Some(&"true".to_string())
2398+
);
2399+
assert_eq!(
2400+
s3_configs.get(&AmazonS3ConfigKey::Endpoint),
2401+
Some(&"http://review.dotted.bucket.storage.example.test".to_string())
2402+
);
2403+
}
2404+
2405+
#[tokio::test]
2406+
#[cfg_attr(miri, ignore)] // AWS credential providers call foreign functions
2407+
async fn test_hadoop_profile_provider_takes_the_forwarded_default_file() {
2408+
// With no configured file the JVM-resolved default applies; a configured file wins;
2409+
// the SDK spellings ignore both.
2410+
for (file, expected) in [
2411+
(None, Some("/synthetic/jvm-home/.aws/credentials")),
2412+
(Some("/etc/aws/credentials"), Some("/etc/aws/credentials")),
2413+
] {
2414+
let mut builder = TestConfigBuilder::new()
2415+
.with_credential_provider(HADOOP_PROFILE)
2416+
.with_property(
2417+
"comet.default.profile.file",
2418+
"/synthetic/jvm-home/.aws/credentials",
2419+
);
2420+
if let Some(file) = file {
2421+
builder = builder.with_property("auth.profile.file", file);
2422+
}
2423+
let configs = builder.build();
2424+
let metadata =
2425+
build_credential_provider(&configs, "test-bucket", Duration::from_secs(300))
2426+
.await
2427+
.unwrap()
2428+
.expect("Should return a credential provider")
2429+
.metadata();
2430+
assert_eq!(
2431+
metadata,
2432+
CredentialProviderMetadata::Profile {
2433+
name: None,
2434+
file: expected.map(str::to_string),
2435+
credentials_only: true,
2436+
}
2437+
);
2438+
}
2439+
let configs = TestConfigBuilder::new()
2440+
.with_credential_provider(AWS_PROFILE)
2441+
.with_property(
2442+
"comet.default.profile.file",
2443+
"/synthetic/jvm-home/.aws/credentials",
2444+
)
2445+
.build();
2446+
let metadata = build_credential_provider(&configs, "test-bucket", Duration::from_secs(300))
2447+
.await
2448+
.unwrap()
2449+
.expect("Should return a credential provider")
2450+
.metadata();
2451+
assert_eq!(
2452+
metadata,
2453+
CredentialProviderMetadata::Profile {
2454+
name: None,
2455+
file: None,
2456+
credentials_only: false,
2457+
}
2458+
);
23802459
}
23812460

23822461
#[test]

‎spark/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,21 @@ object NativeConfig {
196196
*
197197
* The result feeds object_store's parse_url_opts natively.
198198
*/
199+
/**
200+
* Where the native side finds the credentials file Hadoop's profile provider would read with no
201+
* `fs.s3a.auth.profile.file` configured: `AWS_SHARED_CREDENTIALS_FILE`, else the JVM user's
202+
* `~/.aws/credentials` (Hadoop resolves the home through `user.home`, not `HOME`).
203+
*/
204+
val COMET_DEFAULT_PROFILE_FILE_KEY = "fs.s3a.comet.default.profile.file"
205+
206+
private[objectstore] def defaultSharedCredentialsFile(
207+
env: Map[String, String] = sys.env,
208+
userHome: String = System.getProperty("user.home")): String =
209+
env
210+
.get("AWS_SHARED_CREDENTIALS_FILE")
211+
.filter(StringUtils.isNotBlank)
212+
.getOrElse(new java.io.File(new java.io.File(userHome, ".aws"), "credentials").getPath)
213+
199214
def extractObjectStoreOptions(hadoopConf: Configuration, uri: URI): Map[String, String] = {
200215
val scheme = Option(uri.getScheme).map(_.toLowerCase(Locale.ROOT)).getOrElse("file")
201216

@@ -231,6 +246,13 @@ object NativeConfig {
231246
val vendorPrefix = if (s3CompliantSchemes.contains(scheme)) s"fs.$scheme." else ""
232247
val vendorEntries = scala.collection.mutable.ArrayBuffer[(String, String)]()
233248

249+
// Hadoop's ProfileAWSCredentialsProvider reads this file when fs.s3a.auth.profile.file is
250+
// unset; native resolves paths against its own environment, so the JVM's answer rides along
251+
// for every scheme that uses the fs.s3a.* surface.
252+
if (prefixes.get.contains("fs.s3a.")) {
253+
options(COMET_DEFAULT_PROFILE_FILE_KEY) = defaultSharedCredentialsFile()
254+
}
255+
234256
hadoopConf.iterator().asScala.foreach { entry =>
235257
val key = entry.getKey
236258
if (prefixes.get.exists(prefix => key.startsWith(prefix))) {

‎spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,35 @@ import org.apache.comet.CometConf.COMET_S3_COMPLIANT_SCHEMES_KEY
3030

3131
class NativeConfigSuite extends AnyFunSuite with Matchers {
3232

33+
test("extractObjectStoreOptions forwards the JVM-resolved default credentials file") {
34+
val hadoopConf = new Configuration()
35+
val options =
36+
NativeConfig.extractObjectStoreOptions(hadoopConf, new URI("s3a://test-bucket/object"))
37+
val expected = new java.io.File(
38+
new java.io.File(System.getProperty("user.home"), ".aws"),
39+
"credentials").getPath
40+
if (sys.env.get("AWS_SHARED_CREDENTIALS_FILE").exists(_.trim.nonEmpty)) {
41+
assert(
42+
options(NativeConfig.COMET_DEFAULT_PROFILE_FILE_KEY) ==
43+
sys.env("AWS_SHARED_CREDENTIALS_FILE"))
44+
} else {
45+
assert(options(NativeConfig.COMET_DEFAULT_PROFILE_FILE_KEY) == expected)
46+
}
47+
val gsOptions =
48+
NativeConfig.extractObjectStoreOptions(hadoopConf, new URI("gs://test-bucket/object"))
49+
assert(!gsOptions.contains(NativeConfig.COMET_DEFAULT_PROFILE_FILE_KEY))
50+
51+
// The env override wins, a blank override falls back to user.home, and HOME is not used.
52+
assert(
53+
NativeConfig.defaultSharedCredentialsFile(
54+
Map("AWS_SHARED_CREDENTIALS_FILE" -> "/etc/aws/shared"),
55+
"/synthetic/jvm-home") == "/etc/aws/shared")
56+
assert(
57+
NativeConfig.defaultSharedCredentialsFile(
58+
Map("AWS_SHARED_CREDENTIALS_FILE" -> " ", "HOME" -> "/synthetic/env-home"),
59+
"/synthetic/jvm-home") == "/synthetic/jvm-home/.aws/credentials")
60+
}
61+
3362
test("extractObjectStoreOptions - multiple cloud provider configurations") {
3463
val hadoopConf = new Configuration()
3564
// S3A configs

0 commit comments

Comments
 (0)