Skip to content

Commit 335113f

Browse files
committed
fix: read the Delta gate's libhdfs schemes from the scan Hadoop conf
DeltaScanSupport.declineReason still read the list from the SQLConf entry, while the native Delta scan reads fs.comet.libhdfs.schemes from the Hadoop conf the gate builds and hands over. It now uses NativeConfig.resolveLibhdfsSchemes on that conf, as CometScanRule does.
1 parent b8f8231 commit 335113f

3 files changed

Lines changed: 73 additions & 20 deletions

File tree

‎contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ import org.apache.spark.sql.internal.SQLConf
3939
import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType}
4040

4141
import org.apache.comet.CometConf
42-
import org.apache.comet.CometConf.COMET_LIBHDFS_SCHEMES
4342
import org.apache.comet.objectstore.NativeConfig
4443
import org.apache.comet.parquet.CometParquetUtils
4544
import org.apache.comet.rules.{CometScanRule, CometScanTypeChecker}
@@ -293,8 +292,9 @@ object DeltaScanSupport {
293292
}
294293

295294
// Only claim scans whose root paths object_store (or the configured libhdfs schemes) can
296-
// actually read (mirrors core's unsupportedFsSchemes gate).
297-
val libhdfs = libhdfsSchemes
295+
// actually read (mirrors core's unsupportedFsSchemes gate). The libhdfs list comes from the
296+
// scan's Hadoop conf, as native reads it.
297+
val libhdfs = NativeConfig.resolveLibhdfsSchemes(hadoopConf)
298298
val unsupportedRootSchemes = unsupportedSchemes(rootUris, libhdfs)
299299
if (unsupportedRootSchemes.nonEmpty) {
300300
return Some(
@@ -460,16 +460,6 @@ object DeltaScanSupport {
460460
.toSeq
461461
}
462462

463-
/**
464-
* The libhdfs scheme exemption set from [[org.apache.comet.CometConf.COMET_LIBHDFS_SCHEMES]],
465-
* parsed exactly like core's scan gate (`NativeConfig.parseSchemeSet`: split on commas,
466-
* trimmed, lowercased) and defaulting to `Set("hdfs")` when unset.
467-
*/
468-
private[delta] def libhdfsSchemes: Set[String] = COMET_LIBHDFS_SCHEMES.get() match {
469-
case Some(s) => NativeConfig.parseSchemeSet(s)
470-
case None => Set("hdfs")
471-
}
472-
473463
/**
474464
* Decline reason when any of `uris` uses a scheme opted in as an S3-compliant alias through
475465
* `fs.comet.s3Compliant.schemes` (e.g. `blob`), or `None`. Core's native Parquet scan admits

‎contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/DeltaScanContribSuite.scala‎

Lines changed: 66 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,10 @@ import org.apache.hadoop.fs.s3a.{S3AFileSystem, S3AUtils}
3333
import org.apache.hadoop.security.alias.CredentialProviderFactory
3434
import org.apache.spark.sql.comet.CometDeltaNativeScanExec
3535
import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor
36+
import org.apache.spark.sql.execution.SparkPlan
3637

3738
import org.apache.comet.{CometConf, ExtendedExplainInfo}
39+
import org.apache.comet.hadoop.fs.FakeHDFSFileSystem
3840
import org.apache.comet.objectstore.NativeConfig
3941
import org.apache.comet.rules.CometScanRule
4042

@@ -252,11 +254,71 @@ class DeltaScanContribSuite extends CometDeltaTestBase {
252254
assert(DeltaScanSupport.s3CompliantAliasSchemeReason(notOptedIn, Seq(blobUri)).isEmpty)
253255
}
254256

255-
test("libhdfsSchemes parses the list exactly like core's scan gate (trim, lowercase, blanks)") {
256-
withSQLConf(CometConf.COMET_LIBHDFS_SCHEMES.key -> " HDFS , viewfs ,, ") {
257-
assert(DeltaScanSupport.libhdfsSchemes == Set("hdfs", "viewfs"))
257+
test("scheme gate claims a scheme listed in the Hadoop conf's libhdfs.schemes") {
258+
assert(!CometScanRule.isNativelyReadableScheme(new URI(FakeHDFSFileSystem.PREFIX), Set.empty))
259+
// The bare key reaches the scan's Hadoop conf, where native reads the list. The padding,
260+
// case and empty entries check it is parsed the way native and core's gate parse it.
261+
val plan = planDeltaReadOnFakeScheme(libhdfsConf =
262+
Seq(CometConf.COMET_LIBHDFS_SCHEMES_KEY -> " FAKE , hdfs ,, "))
263+
assertClaimedOnFakeScheme(plan)
264+
}
265+
266+
test("scheme gate claims a scheme listed only in a reader option's libhdfs.schemes") {
267+
assert(!CometScanRule.isNativelyReadableScheme(new URI(FakeHDFSFileSystem.PREFIX), Set.empty))
268+
// Delta keeps `fs.` read options in the relation options, which reach the scan's Hadoop conf
269+
// but never SQLConf or the session's Hadoop conf.
270+
val plan = planDeltaReadOnFakeScheme(readOptions =
271+
Map(CometConf.COMET_LIBHDFS_SCHEMES_KEY -> "fake,hdfs"))
272+
assertClaimedOnFakeScheme(plan)
273+
}
274+
275+
test("scheme gate ignores a runtime SET of the spark.hadoop. libhdfs key") {
276+
// newHadoopConf copies SQLConf keys with their `spark.hadoop.` prefix, so the list never
277+
// reaches the Hadoop conf native reads and `fake` is still not routed through libhdfs.
278+
val plan = planDeltaReadOnFakeScheme(libhdfsConf =
279+
Seq(CometConf.COMET_LIBHDFS_SCHEMES.key -> "fake,hdfs"))
280+
assert(
281+
collectByName(plan, "CometDeltaNativeScanExec").isEmpty,
282+
s"native would not route `fake` through libhdfs; the scan must fall back:\n$plan")
283+
val reasons = new ExtendedExplainInfo().getFallbackReasons(plan)
284+
assert(
285+
reasons.exists(_.contains("Native Delta scan does not support filesystem scheme(s) fake")),
286+
s"the scan must be declined for its scheme, got reasons: $reasons")
287+
}
288+
289+
private def assertClaimedOnFakeScheme(plan: SparkPlan): Unit = {
290+
val reasons = new ExtendedExplainInfo().getFallbackReasons(plan)
291+
assert(
292+
!reasons.exists(_.contains("filesystem scheme")),
293+
s"`fake` is in the scan's libhdfs list; it must not be declined, got reasons: $reasons")
294+
assert(
295+
collectByName(plan, "CometDeltaNativeScanExec").nonEmpty,
296+
s"expected the native Delta scan to claim the read:\n$plan")
297+
}
298+
299+
/**
300+
* Writes a Delta table locally and returns the executed plan of a read of it through `fake://`
301+
* (backed by the local filesystem), planned with `libhdfsConf` set and `readOptions` on the
302+
* reader. The plan is not run.
303+
*/
304+
private def planDeltaReadOnFakeScheme(
305+
libhdfsConf: Seq[(String, String)] = Nil,
306+
readOptions: Map[String, String] = Map.empty): SparkPlan = {
307+
var plan: SparkPlan = null
308+
withTempPath { dir =>
309+
spark.range(0, 10).write.format("delta").save(dir.getAbsolutePath)
310+
val fakePath = s"${FakeHDFSFileSystem.PREFIX}${dir.getAbsolutePath}"
311+
val sqlConf = ("fs.fake.impl" -> classOf[FakeHDFSFileSystem].getName) +: libhdfsConf
312+
withSQLConf(sqlConf: _*) {
313+
plan = spark.read
314+
.format("delta")
315+
.options(readOptions)
316+
.load(fakePath)
317+
.queryExecution
318+
.executedPlan
319+
}
258320
}
259-
assert(DeltaScanSupport.libhdfsSchemes == Set("hdfs"))
321+
plan
260322
}
261323

262324
test("unsupportedSchemes passes an all-file: root-path selection (no regression)") {

‎spark/src/main/scala/org/apache/comet/CometConf.scala‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1286,9 +1286,10 @@ object CometConf extends ShimCometConf {
12861286

12871287
val COMET_LIBHDFS_SCHEMES_KEY = "fs.comet.libhdfs.schemes"
12881288

1289-
// Declared so the value appears in the generated config reference and can be set via
1290-
// `spark.hadoop.*`, but never read through this entry: `NativeConfig.resolveLibhdfsSchemes`
1291-
// reads the Hadoop key above instead, so `core-site.xml` is honored too.
1289+
// Declared so the value appears in the generated config reference and can be set at startup
1290+
// via `spark.hadoop.*`, but never read through this entry: the scan gates read the Hadoop key
1291+
// above through `NativeConfig.resolveLibhdfsSchemes`, as native does, so `core-site.xml` is
1292+
// honored too.
12921293
val COMET_LIBHDFS_SCHEMES: OptionalConfigEntry[String] =
12931294
conf(s"spark.hadoop.$COMET_LIBHDFS_SCHEMES_KEY")
12941295
.category(CATEGORY_SCAN)

0 commit comments

Comments
 (0)