Skip to content

Commit f3758e5

Browse files
committed
fix: resolve Hadoop's default credentials file on the executor at plan creation
1 parent 31fafaf commit f3758e5

7 files changed

Lines changed: 111 additions & 25 deletions

File tree

‎native/core/src/execution/jni_api.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,9 @@ use crate::execution::spark_config::{
116116
COMET_TRACING_ENABLED, SPARK_EXECUTOR_CORES,
117117
};
118118
use crate::parquet::encryption_support::{CometEncryptionFactory, ENCRYPTION_FACTORY_ID};
119+
use crate::parquet::objectstore::s3::{
120+
ExecutorObjectStoreDefaults, COMET_DEFAULT_PROFILE_FILE_KEY,
121+
};
119122
use datafusion_comet_proto::spark_operator::operator::OpStruct;
120123
use log::{info, warn};
121124
use std::sync::OnceLock;
@@ -744,6 +747,12 @@ fn prepare_datafusion_session_context(
744747
session_config.set_str("datafusion.execution.parquet.reorder_filters", "true");
745748
}
746749

750+
// The executor JVM resolves the credentials file Hadoop's profile provider would read on
751+
// this executor; scans overlay it onto their object store options.
752+
session_config = session_config.with_extension(Arc::new(ExecutorObjectStoreDefaults {
753+
default_profile_file: spark_config.get(COMET_DEFAULT_PROFILE_FILE_KEY).cloned(),
754+
}));
755+
747756
// Pass through DataFusion configs from Spark.
748757
// e.g: spark-shell --conf spark.comet.datafusion.sql_parser.parse_float_as_decimal=true
749758
// becomes datafusion.sql_parser.parse_float_as_decimal=true

‎native/core/src/execution/planner.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ use iceberg::expr::Bind;
100100
use crate::execution::operators::ExecutionError::GeneralError;
101101
use crate::execution::shuffle::{CometPartitioning, CompressionCodec};
102102
use crate::execution::spark_plan::SparkPlan;
103+
use crate::parquet::objectstore::s3::apply_executor_object_store_defaults;
103104
use crate::parquet::objectstore::s3_blob_fs_support::normalize_object_store_url;
104105
use crate::parquet::parquet_support::prepare_object_store_with_configs;
105106
use datafusion::common::scalar::ScalarStructBuilder;
@@ -1695,11 +1696,12 @@ impl PhysicalPlanner {
16951696
.map(|f| f.file_path.clone())
16961697
.expect("partition should have files after empty check");
16971698

1698-
let object_store_options: HashMap<String, String> = common
1699+
let mut object_store_options: HashMap<String, String> = common
16991700
.object_store_options
17001701
.iter()
17011702
.map(|(k, v)| (k.clone(), v.clone()))
17021703
.collect();
1704+
apply_executor_object_store_defaults(&self.session_ctx, &mut object_store_options);
17031705
let (object_store_url, _, object_store_backend) =
17041706
prepare_object_store_with_configs(
17051707
self.session_ctx.runtime_env(),
@@ -1747,11 +1749,12 @@ impl PhysicalPlanner {
17471749
convert_spark_types_to_arrow_schema(scan.partition_schema.as_slice());
17481750
let projection_vector: Vec<usize> =
17491751
scan.projection_vector.iter().map(|i| *i as usize).collect();
1750-
let object_store_options: HashMap<String, String> = scan
1752+
let mut object_store_options: HashMap<String, String> = scan
17511753
.object_store_options
17521754
.iter()
17531755
.map(|(k, v)| (k.clone(), v.clone()))
17541756
.collect();
1757+
apply_executor_object_store_defaults(&self.session_ctx, &mut object_store_options);
17551758
let one_file = scan
17561759
.file_partitions
17571760
.first()

‎native/core/src/parquet/objectstore/s3.rs‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,43 @@ fn normalize_endpoint(
353353
})
354354
}
355355

356+
/// Object store defaults the executor JVM resolves at plan creation and native applies to
357+
/// every scan's options, so the native side reads the same files Hadoop does on that executor.
358+
#[derive(Debug, Clone, Default)]
359+
pub struct ExecutorObjectStoreDefaults {
360+
/// The credentials file Hadoop's profile provider reads with no `fs.s3a.auth.profile.file`
361+
/// configured, resolved against the executor JVM's `user.home` and environment.
362+
pub default_profile_file: Option<String>,
363+
}
364+
365+
/// The key the executor JVM and the native provider share for that file.
366+
pub const COMET_DEFAULT_PROFILE_FILE_KEY: &str = "fs.s3a.comet.default.profile.file";
367+
368+
impl ExecutorObjectStoreDefaults {
369+
/// Overlays the executor's defaults onto a scan's forwarded options. The executor value
370+
/// wins over anything the driver serialized, since only the executor knows its own home.
371+
pub fn apply(&self, options: &mut HashMap<String, String>) {
372+
if let Some(file) = &self.default_profile_file {
373+
options.insert(COMET_DEFAULT_PROFILE_FILE_KEY.to_string(), file.clone());
374+
}
375+
}
376+
}
377+
378+
/// Applies the executor defaults registered on `session` (see `ExecutorObjectStoreDefaults`)
379+
/// to a scan's forwarded object store options.
380+
pub fn apply_executor_object_store_defaults(
381+
session: &datafusion::prelude::SessionContext,
382+
options: &mut HashMap<String, String>,
383+
) {
384+
if let Some(defaults) = session
385+
.state()
386+
.config()
387+
.get_extension::<ExecutorObjectStoreDefaults>()
388+
{
389+
defaults.apply(options);
390+
}
391+
}
392+
356393
/// The credentials file Hadoop's profile provider reads when none is configured:
357394
/// `AWS_SHARED_CREDENTIALS_FILE` when set, otherwise `~/.aws/credentials`.
358395
fn default_shared_credentials_file(env_override: Option<String>, home: Option<String>) -> String {
@@ -2458,6 +2495,45 @@ mod tests {
24582495
);
24592496
}
24602497

2498+
#[test]
2499+
fn test_executor_defaults_overlay_the_forwarded_options() {
2500+
use datafusion::prelude::{SessionConfig, SessionContext};
2501+
let mut options = HashMap::from([(
2502+
COMET_DEFAULT_PROFILE_FILE_KEY.to_string(),
2503+
"/synthetic/driver-home/.aws/credentials".to_string(),
2504+
)]);
2505+
// Without executor defaults registered, the options pass through untouched.
2506+
apply_executor_object_store_defaults(&SessionContext::new(), &mut options);
2507+
assert_eq!(
2508+
options
2509+
.get(COMET_DEFAULT_PROFILE_FILE_KEY)
2510+
.map(String::as_str),
2511+
Some("/synthetic/driver-home/.aws/credentials")
2512+
);
2513+
// The executor's own resolution wins over whatever the driver serialized.
2514+
let config = SessionConfig::new().with_extension(Arc::new(ExecutorObjectStoreDefaults {
2515+
default_profile_file: Some("/synthetic/executor-home/.aws/credentials".to_string()),
2516+
}));
2517+
apply_executor_object_store_defaults(
2518+
&SessionContext::new_with_config(config),
2519+
&mut options,
2520+
);
2521+
assert_eq!(
2522+
options
2523+
.get(COMET_DEFAULT_PROFILE_FILE_KEY)
2524+
.map(String::as_str),
2525+
Some("/synthetic/executor-home/.aws/credentials")
2526+
);
2527+
// An executor with nothing resolved leaves the options alone.
2528+
let config =
2529+
SessionConfig::new().with_extension(Arc::new(ExecutorObjectStoreDefaults::default()));
2530+
apply_executor_object_store_defaults(
2531+
&SessionContext::new_with_config(config),
2532+
&mut options,
2533+
);
2534+
assert_eq!(options.len(), 1);
2535+
}
2536+
24612537
#[test]
24622538
fn test_default_shared_credentials_file_matches_hadoop() {
24632539
assert_eq!(

‎spark/src/main/scala/org/apache/comet/CometExecIterator.scala‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import org.apache.spark.util.SerializableConfiguration
3535
import org.apache.comet.CometConf._
3636
import org.apache.comet.Tracing.withTrace
3737
import org.apache.comet.exceptions.CometQueryExecutionException
38+
import org.apache.comet.objectstore.NativeConfig
3839
import org.apache.comet.parquet.CometFileKeyUnwrapper
3940
import org.apache.comet.serde.Config.ConfigMap
4041
import org.apache.comet.shuffle.ShufflePartitionPusher
@@ -358,6 +359,12 @@ object CometExecIterator extends Logging {
358359
CometConf.COMET_PARQUET_ROW_FILTER_PUSHDOWN_ENABLED.key,
359360
CometConf.COMET_PARQUET_ROW_FILTER_PUSHDOWN_ENABLED.get(SQLConf.get).toString)
360361

362+
// This runs on the executor, so the credentials file Hadoop's profile provider would read
363+
// here is resolved against this JVM's user.home, not the driver's or the native process's.
364+
builder.putEntries(
365+
NativeConfig.COMET_DEFAULT_PROFILE_FILE_KEY,
366+
NativeConfig.defaultSharedCredentialsFile())
367+
361368
builder.build().toByteArray
362369
}
363370

‎spark/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -199,11 +199,12 @@ object NativeConfig {
199199
/**
200200
* Where the native side finds the credentials file Hadoop's profile provider would read with no
201201
* `fs.s3a.auth.profile.file` configured: `AWS_SHARED_CREDENTIALS_FILE`, else the JVM user's
202-
* `~/.aws/credentials` (Hadoop resolves the home through `user.home`, not `HOME`).
202+
* `~/.aws/credentials` (Hadoop resolves the home through `user.home`, not `HOME`). Resolved on
203+
* the executor at plan creation, since the executor's home is what its Hadoop provider reads.
203204
*/
204205
val COMET_DEFAULT_PROFILE_FILE_KEY = "fs.s3a.comet.default.profile.file"
205206

206-
private[objectstore] def defaultSharedCredentialsFile(
207+
private[comet] def defaultSharedCredentialsFile(
207208
env: Map[String, String] = sys.env,
208209
userHome: String = System.getProperty("user.home")): String =
209210
env
@@ -246,13 +247,6 @@ object NativeConfig {
246247
val vendorPrefix = if (s3CompliantSchemes.contains(scheme)) s"fs.$scheme." else ""
247248
val vendorEntries = scala.collection.mutable.ArrayBuffer[(String, String)]()
248249

249-
// Hadoop's ProfileAWSCredentialsProvider reads this file when fs.s3a.auth.profile.file is
250-
// unset; native resolves paths against its own environment, so the JVM's answer rides along
251-
// for every scheme that uses the fs.s3a.* surface.
252-
if (prefixes.get.contains("fs.s3a.")) {
253-
options(COMET_DEFAULT_PROFILE_FILE_KEY) = defaultSharedCredentialsFile()
254-
}
255-
256250
hadoopConf.iterator().asScala.foreach { entry =>
257251
val key = entry.getKey
258252
if (prefixes.get.exists(prefix => key.startsWith(prefix))) {

‎spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,14 @@ class CometExecSuite extends CometTestBase {
6767
}
6868
}
6969

70+
test("serialized executor configs carry the executor-resolved default credentials file") {
71+
val protobuf = CometExecIterator.serializeCometSQLConfs()
72+
val entries = org.apache.comet.serde.Config.ConfigMap.parseFrom(protobuf).getEntriesMap
73+
assert(
74+
entries.get(org.apache.comet.objectstore.NativeConfig.COMET_DEFAULT_PROFILE_FILE_KEY) ==
75+
org.apache.comet.objectstore.NativeConfig.defaultSharedCredentialsFile())
76+
}
77+
7078
test("SQLConf serde") {
7179

7280
def roundtrip = {

‎spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -30,23 +30,12 @@ import org.apache.comet.CometConf.COMET_S3_COMPLIANT_SCHEMES_KEY
3030

3131
class NativeConfigSuite extends AnyFunSuite with Matchers {
3232

33-
test("extractObjectStoreOptions forwards the JVM-resolved default credentials file") {
33+
test("the default credentials file is resolved like Hadoop and not frozen at planning") {
34+
// Planning on the driver forwards nothing for it; the executor resolves it at plan creation.
3435
val hadoopConf = new Configuration()
3536
val options =
3637
NativeConfig.extractObjectStoreOptions(hadoopConf, new URI("s3a://test-bucket/object"))
37-
val expected = new java.io.File(
38-
new java.io.File(System.getProperty("user.home"), ".aws"),
39-
"credentials").getPath
40-
if (sys.env.get("AWS_SHARED_CREDENTIALS_FILE").exists(_.trim.nonEmpty)) {
41-
assert(
42-
options(NativeConfig.COMET_DEFAULT_PROFILE_FILE_KEY) ==
43-
sys.env("AWS_SHARED_CREDENTIALS_FILE"))
44-
} else {
45-
assert(options(NativeConfig.COMET_DEFAULT_PROFILE_FILE_KEY) == expected)
46-
}
47-
val gsOptions =
48-
NativeConfig.extractObjectStoreOptions(hadoopConf, new URI("gs://test-bucket/object"))
49-
assert(!gsOptions.contains(NativeConfig.COMET_DEFAULT_PROFILE_FILE_KEY))
38+
assert(!options.contains(NativeConfig.COMET_DEFAULT_PROFILE_FILE_KEY))
5039

5140
// The env override wins, a blank override falls back to user.home, and HOME is not used.
5241
assert(

0 commit comments

Comments
 (0)