@@ -353,6 +353,43 @@ fn normalize_endpoint(
353353 } )
354354}
355355
356+ /// Object store defaults the executor JVM resolves at plan creation and native applies to
357+ /// every scan's options, so the native side reads the same files Hadoop does on that executor.
358+ #[ derive( Debug , Clone , Default ) ]
359+ pub struct ExecutorObjectStoreDefaults {
360+ /// The credentials file Hadoop's profile provider reads with no `fs.s3a.auth.profile.file`
361+ /// configured, resolved against the executor JVM's `user.home` and environment.
362+ pub default_profile_file : Option < String > ,
363+ }
364+
365+ /// The key the executor JVM and the native provider share for that file.
366+ pub const COMET_DEFAULT_PROFILE_FILE_KEY : & str = "fs.s3a.comet.default.profile.file" ;
367+
368+ impl ExecutorObjectStoreDefaults {
369+ /// Overlays the executor's defaults onto a scan's forwarded options. The executor value
370+ /// wins over anything the driver serialized, since only the executor knows its own home.
371+ pub fn apply ( & self , options : & mut HashMap < String , String > ) {
372+ if let Some ( file) = & self . default_profile_file {
373+ options. insert ( COMET_DEFAULT_PROFILE_FILE_KEY . to_string ( ) , file. clone ( ) ) ;
374+ }
375+ }
376+ }
377+
378+ /// Applies the executor defaults registered on `session` (see `ExecutorObjectStoreDefaults`)
379+ /// to a scan's forwarded object store options.
380+ pub fn apply_executor_object_store_defaults (
381+ session : & datafusion:: prelude:: SessionContext ,
382+ options : & mut HashMap < String , String > ,
383+ ) {
384+ if let Some ( defaults) = session
385+ . state ( )
386+ . config ( )
387+ . get_extension :: < ExecutorObjectStoreDefaults > ( )
388+ {
389+ defaults. apply ( options) ;
390+ }
391+ }
392+
356393/// The credentials file Hadoop's profile provider reads when none is configured:
357394/// `AWS_SHARED_CREDENTIALS_FILE` when set, otherwise `~/.aws/credentials`.
358395fn default_shared_credentials_file ( env_override : Option < String > , home : Option < String > ) -> String {
@@ -2458,6 +2495,45 @@ mod tests {
24582495 ) ;
24592496 }
24602497
2498+ #[ test]
2499+ fn test_executor_defaults_overlay_the_forwarded_options ( ) {
2500+ use datafusion:: prelude:: { SessionConfig , SessionContext } ;
2501+ let mut options = HashMap :: from ( [ (
2502+ COMET_DEFAULT_PROFILE_FILE_KEY . to_string ( ) ,
2503+ "/synthetic/driver-home/.aws/credentials" . to_string ( ) ,
2504+ ) ] ) ;
2505+ // Without executor defaults registered, the options pass through untouched.
2506+ apply_executor_object_store_defaults ( & SessionContext :: new ( ) , & mut options) ;
2507+ assert_eq ! (
2508+ options
2509+ . get( COMET_DEFAULT_PROFILE_FILE_KEY )
2510+ . map( String :: as_str) ,
2511+ Some ( "/synthetic/driver-home/.aws/credentials" )
2512+ ) ;
2513+ // The executor's own resolution wins over whatever the driver serialized.
2514+ let config = SessionConfig :: new ( ) . with_extension ( Arc :: new ( ExecutorObjectStoreDefaults {
2515+ default_profile_file : Some ( "/synthetic/executor-home/.aws/credentials" . to_string ( ) ) ,
2516+ } ) ) ;
2517+ apply_executor_object_store_defaults (
2518+ & SessionContext :: new_with_config ( config) ,
2519+ & mut options,
2520+ ) ;
2521+ assert_eq ! (
2522+ options
2523+ . get( COMET_DEFAULT_PROFILE_FILE_KEY )
2524+ . map( String :: as_str) ,
2525+ Some ( "/synthetic/executor-home/.aws/credentials" )
2526+ ) ;
2527+ // An executor with nothing resolved leaves the options alone.
2528+ let config =
2529+ SessionConfig :: new ( ) . with_extension ( Arc :: new ( ExecutorObjectStoreDefaults :: default ( ) ) ) ;
2530+ apply_executor_object_store_defaults (
2531+ & SessionContext :: new_with_config ( config) ,
2532+ & mut options,
2533+ ) ;
2534+ assert_eq ! ( options. len( ) , 1 ) ;
2535+ }
2536+
24612537 #[ test]
24622538 fn test_default_shared_credentials_file_matches_hadoop ( ) {
24632539 assert_eq ! (
0 commit comments