@@ -33,8 +33,10 @@ import org.apache.hadoop.fs.s3a.{S3AFileSystem, S3AUtils}
3333import org .apache .hadoop .security .alias .CredentialProviderFactory
3434import org .apache .spark .sql .comet .CometDeltaNativeScanExec
3535import org .apache .spark .sql .delta .actions .DeletionVectorDescriptor
36+ import org .apache .spark .sql .execution .SparkPlan
3637
3738import org .apache .comet .{CometConf , ExtendedExplainInfo }
39+ import org .apache .comet .hadoop .fs .FakeHDFSFileSystem
3840import org .apache .comet .objectstore .NativeConfig
3941import org .apache .comet .rules .CometScanRule
4042
@@ -252,11 +254,71 @@ class DeltaScanContribSuite extends CometDeltaTestBase {
252254 assert(DeltaScanSupport .s3CompliantAliasSchemeReason(notOptedIn, Seq (blobUri)).isEmpty)
253255 }
254256
255- test(" libhdfsSchemes parses the list exactly like core's scan gate (trim, lowercase, blanks)" ) {
256- withSQLConf(CometConf .COMET_LIBHDFS_SCHEMES .key -> " HDFS , viewfs ,, " ) {
257- assert(DeltaScanSupport .libhdfsSchemes == Set (" hdfs" , " viewfs" ))
257+ test(" scheme gate claims a scheme listed in the Hadoop conf's libhdfs.schemes" ) {
258+ assert(! CometScanRule .isNativelyReadableScheme(new URI (FakeHDFSFileSystem .PREFIX ), Set .empty))
259+ // The bare key reaches the scan's Hadoop conf, where native reads the list. The padding,
260+ // case and empty entries check it is parsed the way native and core's gate parse it.
261+ val plan = planDeltaReadOnFakeScheme(libhdfsConf =
262+ Seq (CometConf .COMET_LIBHDFS_SCHEMES_KEY -> " FAKE , hdfs ,, " ))
263+ assertClaimedOnFakeScheme(plan)
264+ }
265+
266+ test(" scheme gate claims a scheme listed only in a reader option's libhdfs.schemes" ) {
267+ assert(! CometScanRule .isNativelyReadableScheme(new URI (FakeHDFSFileSystem .PREFIX ), Set .empty))
268+ // Delta keeps `fs.` read options in the relation options, which reach the scan's Hadoop conf
269+ // but never SQLConf or the session's Hadoop conf.
270+ val plan = planDeltaReadOnFakeScheme(readOptions =
271+ Map (CometConf .COMET_LIBHDFS_SCHEMES_KEY -> " fake,hdfs" ))
272+ assertClaimedOnFakeScheme(plan)
273+ }
274+
275+ test(" scheme gate ignores a runtime SET of the spark.hadoop. libhdfs key" ) {
276+ // newHadoopConf copies SQLConf keys with their `spark.hadoop.` prefix, so the list never
277+ // reaches the Hadoop conf native reads and `fake` is still not routed through libhdfs.
278+ val plan = planDeltaReadOnFakeScheme(libhdfsConf =
279+ Seq (CometConf .COMET_LIBHDFS_SCHEMES .key -> " fake,hdfs" ))
280+ assert(
281+ collectByName(plan, " CometDeltaNativeScanExec" ).isEmpty,
282+ s " native would not route `fake` through libhdfs; the scan must fall back: \n $plan" )
283+ val reasons = new ExtendedExplainInfo ().getFallbackReasons(plan)
284+ assert(
285+ reasons.exists(_.contains(" Native Delta scan does not support filesystem scheme(s) fake" )),
286+ s " the scan must be declined for its scheme, got reasons: $reasons" )
287+ }
288+
289+ private def assertClaimedOnFakeScheme (plan : SparkPlan ): Unit = {
290+ val reasons = new ExtendedExplainInfo ().getFallbackReasons(plan)
291+ assert(
292+ ! reasons.exists(_.contains(" filesystem scheme" )),
293+ s " `fake` is in the scan's libhdfs list; it must not be declined, got reasons: $reasons" )
294+ assert(
295+ collectByName(plan, " CometDeltaNativeScanExec" ).nonEmpty,
296+ s " expected the native Delta scan to claim the read: \n $plan" )
297+ }
298+
299+ /**
300+ * Writes a Delta table locally and returns the executed plan of a read of it through `fake://`
301+ * (backed by the local filesystem), planned with `libhdfsConf` set and `readOptions` on the
302+ * reader. The plan is not run.
303+ */
304+ private def planDeltaReadOnFakeScheme (
305+ libhdfsConf : Seq [(String , String )] = Nil ,
306+ readOptions : Map [String , String ] = Map .empty): SparkPlan = {
307+ var plan : SparkPlan = null
308+ withTempPath { dir =>
309+ spark.range(0 , 10 ).write.format(" delta" ).save(dir.getAbsolutePath)
310+ val fakePath = s " ${FakeHDFSFileSystem .PREFIX }${dir.getAbsolutePath}"
311+ val sqlConf = (" fs.fake.impl" -> classOf [FakeHDFSFileSystem ].getName) +: libhdfsConf
312+ withSQLConf(sqlConf : _* ) {
313+ plan = spark.read
314+ .format(" delta" )
315+ .options(readOptions)
316+ .load(fakePath)
317+ .queryExecution
318+ .executedPlan
319+ }
258320 }
259- assert( DeltaScanSupport .libhdfsSchemes == Set ( " hdfs " ))
321+ plan
260322 }
261323
262324 test(" unsupportedSchemes passes an all-file: root-path selection (no regression)" ) {
0 commit comments