@@ -246,77 +246,31 @@ case class CometScanRule(session: SparkSession)
246246 return None
247247 }
248248 // Comet's native readers go through object_store, which only understands a fixed set of URL
249- // schemes. A custom Hadoop FileSystem (e.g. registered via spark.hadoop.fs.<scheme>.impl) would
250- // surface at execution time as `Generic URL error: Unable to recognise URL "..."`. Decline here
251- // so Spark's reader -- which goes through the Hadoop FS API and can resolve custom schemes --
252- // handles the scan. Whether object_store recognizes a scheme is answered by the native layer
253- // itself (`NativeBase.isObjectStoreSchemeSupported`) rather than a hardcoded list, so the
254- // planner can't drift from object_store's actual support.
255- //
256- // EXCEPT schemes the user routes through libhdfs via `spark.hadoop.fs.comet.libhdfs.schemes`
257- // (e.g. `hdfs`, or a test `fake`): those ARE natively readable through the libhdfs object_store
258- // bridge, so they must NOT be declined here. The claim decision is guarded in CI by
259- // CometScanSchemeFallbackSuite; end-to-end execution through libhdfs is guarded by
260- // ParquetReadFromFakeHadoopFsSuite, which is a manual suite (see its scaladoc).
261- //
262- // The default mirrors the native side: when the config is unset, `is_hdfs_scheme`
263- // (native/core/src/parquet/parquet_support.rs) treats `hdfs` as natively readable, and
264- // `create_hdfs_object_store` is in the default build (`default = ["hdfs-opendal"]`). If we
265- // defaulted to an empty set here, a plain `hdfs://` V1 scan would be declined and fall back to
266- // Spark even though native can read it -- a silent regression for HDFS users in the default
267- // configuration. So default to `Set("hdfs")` to stay in lockstep with the native default.
268- val libhdfsSchemes : Set [String ] = COMET_LIBHDFS_SCHEMES .get() match {
269- case Some (s) => NativeConfig .parseSchemeSet(s)
270- case None => Set (" hdfs" )
271- }
272- // Opt-in S3-compliant alias schemes (e.g. `blob`) from `fs.comet.s3Compliant.schemes`. Read
273- // from the Hadoop config rather than SQLConf (unlike COMET_LIBHDFS_SCHEMES above) so
274- // `core-site.xml` is honored. The native object_store gate no longer claims aliases, so admit
275- // them here where the Hadoop config is available.
276- val s3CompliantSchemes = NativeConfig .resolveS3CompliantSchemes(hadoopConf)
277-
278- // Classify each root path once; see RootPathInfo.
279- val roots = CometScanRule .classifyRootPaths(
280- r.location.rootPaths.map(_.toUri),
281- libhdfsSchemes,
282- s3CompliantSchemes)
283-
284- val unsupportedFsSchemes = roots.iterator.flatMap { root =>
285- root.scheme.filter(_ =>
286- ! root.isLibhdfs && ! CometScanRule .isNativelyReadableScheme(root.uri, s3CompliantSchemes))
287- }.toSet
288- if (unsupportedFsSchemes.nonEmpty) {
249+ // schemes; a custom Hadoop FileSystem would hard-fail at execution. The shared helper
250+ // classifies the root paths (honoring libhdfs and S3-compliant aliases) and reports
251+ // unsupported schemes, multi-bucket alias scans, and object_store-rejected paths. The scheme
252+ // claim decision is guarded in CI by CometScanSchemeFallbackSuite; libhdfs end-to-end by
253+ // ParquetReadFromFakeHadoopFsSuite.
254+ val rootIssues =
255+ CometScanRule .checkObjectStoreRootPaths(r.location.rootPaths.map(_.toUri), hadoopConf)
256+ if (rootIssues.unsupportedSchemes.nonEmpty) {
289257 withFallbackReason(
290258 scanExec,
291- s " Unsupported filesystem schemes: ${unsupportedFsSchemes .mkString(" , " )}" )
259+ s " Unsupported filesystem schemes: ${rootIssues.unsupportedSchemes .mkString(" , " )}" )
292260 return None
293261 }
294- // More than one bucket cannot be served by the single object store native planning registers
295- // per FilePartition; see aliasScanBuckets. Scoped to alias scans: plain multi-bucket `s3://`
296- // has the same flaw today and silently declining it would newly fall back scans that work by
297- // luck, so that widening is left as a separate decision. Note the sibling Iceberg guard
298- // (dataFileBuckets, below) is NOT so scoped -- it declines multi-bucket `s3a://` too.
299- val scanBuckets = CometScanRule .aliasScanBuckets(roots)
300- if (scanBuckets.size > 1 ) {
262+ if (rootIssues.multiBucket.nonEmpty) {
301263 withFallbackReason(
302264 scanExec,
303265 " Native Parquet scan reads S3-compliant alias paths across multiple buckets " +
304- s " ( ${scanBuckets. toSeq.sorted.mkString(" , " )}); Comet registers one object store " +
305- " per file partition and would read every file from the first file's bucket" )
266+ s " ( ${rootIssues.multiBucket. toSeq.sorted.mkString(" , " )}); Comet registers one object " +
267+ " store per file partition and would read every file from the first file's bucket" )
306268 return None
307269 }
308- // A scheme object_store recognizes can still carry a path it rejects (e.g. a directory whose
309- // name contains a newline -> `%0A` in the URI), which native execution would hard-fail on.
310- // Only object_store-native schemes are probed: aliases (normalized to s3:// natively) and
311- // libhdfs schemes route elsewhere. Root paths only -- a rejected character deeper in the tree
312- // still fails at execution.
313- val rejectedPath = roots.find(root =>
314- root.scheme.isDefined && ! root.isLibhdfs && ! root.isAlias &&
315- ! CometScanRule .objectStoreAcceptsPath(root.uri))
316- if (rejectedPath.nonEmpty) {
270+ if (rootIssues.rejectedPath.nonEmpty) {
317271 withFallbackReason(
318272 scanExec,
319- s " Native Parquet scan cannot open path ' ${rejectedPath.get.uri }': object_store " +
273+ s " Native Parquet scan cannot open path ' ${rootIssues. rejectedPath.get}': object_store " +
320274 " rejects it (e.g. an unsupported character in the path)" )
321275 return None
322276 }
@@ -577,46 +531,25 @@ case class CometScanRule(session: SparkSession)
577531 }
578532 }
579533 // Comet's native reader opens files through object_store, which only understands a fixed
580- // set of URL schemes. A text file on a custom Hadoop scheme (viewfs://, oss://, ...) would
581- // be claimed here then hard-fail at execution, whereas Spark reads it via the Hadoop FS
582- // API. Decline such schemes (same gate as the native Parquet path); hdfs:// routes through
583- // libhdfs and is left to native.
584- val libhdfsSchemes : Set [String ] = COMET_LIBHDFS_SCHEMES .get() match {
585- case Some (s) => NativeConfig .parseSchemeSet(s)
586- case None => Set (" hdfs" )
587- }
588- val s3CompliantSchemes = NativeConfig .resolveS3CompliantSchemes(hadoopConf)
589- val roots = CometScanRule .classifyRootPaths(
534+ // set of URL schemes. A text file on a custom Hadoop scheme (viewfs://, oss://, ...), a
535+ // multi-bucket S3-compliant alias scan, or a path object_store rejects would be claimed
536+ // here then hard-fail at execution, whereas Spark reads it via the Hadoop FS API. Use the
537+ // same shared gate as the native Parquet path; hdfs:// routes through libhdfs.
538+ val rootIssues = CometScanRule .checkObjectStoreRootPaths(
590539 scan.fileIndex.rootPaths.map(_.toUri),
591- libhdfsSchemes,
592- s3CompliantSchemes)
593- val unsupportedFsSchemes = roots.iterator.flatMap { root =>
594- root.scheme.filter(_ =>
595- ! root.isLibhdfs && ! CometScanRule .isNativelyReadableScheme(
596- root.uri,
597- s3CompliantSchemes))
598- }.toSet
599- if (unsupportedFsSchemes.nonEmpty) {
540+ hadoopConf)
541+ if (rootIssues.unsupportedSchemes.nonEmpty) {
600542 fallbackReasons +=
601- s " Unsupported filesystem schemes: ${unsupportedFsSchemes.mkString(" , " )}"
602- }
603- // A recognized scheme can still carry a path object_store rejects (e.g. a newline -> %0A),
604- // which native execution would hard-fail on. Root paths only.
605- val rejectedPath = roots.find(root =>
606- root.scheme.isDefined && ! root.isLibhdfs && ! root.isAlias &&
607- ! CometScanRule .objectStoreAcceptsPath(root.uri))
608- if (rejectedPath.nonEmpty) {
609- fallbackReasons += s " Native Text scan cannot open path ' ${rejectedPath.get.uri}': " +
610- " object_store rejects it (e.g. an unsupported character in the path)"
543+ s " Unsupported filesystem schemes: ${rootIssues.unsupportedSchemes.mkString(" , " )}"
611544 }
612- // Native planning registers one object store per partition and strips the bucket from every
613- // key, so an S3-compliant alias scan spanning multiple buckets would read every file from
614- // the first bucket. Decline multi-bucket alias scans (same hazard the Parquet gate guards).
615- val scanBuckets = CometScanRule .aliasScanBuckets(roots)
616- if (scanBuckets.size > 1 ) {
545+ if (rootIssues.multiBucket.nonEmpty) {
617546 fallbackReasons += " Native Text scan reads S3-compliant alias paths across multiple " +
618- s " buckets ( ${scanBuckets.toSeq.sorted.mkString(" , " )}); Comet registers one object " +
619- " store per file partition and would read every file from the first file's bucket"
547+ s " buckets ( ${rootIssues.multiBucket.toSeq.sorted.mkString(" , " )}); Comet registers " +
548+ " one object store per file partition and would read every file from the first bucket"
549+ }
550+ rootIssues.rejectedPath.foreach { uri =>
551+ fallbackReasons += s " Native Text scan cannot open path ' $uri': " +
552+ " object_store rejects it (e.g. an unsupported character in the path)"
620553 }
621554 if (fallbackReasons.isEmpty) {
622555 CometBatchScanExec (
@@ -1416,6 +1349,49 @@ object CometScanRule extends Logging {
14161349 bucket = NativeConfig .bucketForUri(uri, s3CompliantSchemes))
14171350 }
14181351
1352+ /**
1353+ * The object_store scheme/bucket/path problems that make a set of root paths unreadable by
1354+ * Comet's native file scans (shared by the Parquet V1 gate and the Text V2 gate). Each caller
1355+ * formats its own fallback message from these findings.
1356+ *
1357+ * - `unsupportedSchemes`: schemes object_store does not natively read and that are not routed
1358+ * through libhdfs (a custom Hadoop FileSystem would hard-fail at execution).
1359+ * - `multiBucket`: the S3-compliant alias buckets when a scan spans more than one; native
1360+ * planning registers one object store per FilePartition and would read every file from the
1361+ * first file's bucket. Empty when a single bucket (or no alias) is used.
1362+ * - `rejectedPath`: a root whose recognized scheme carries a path object_store rejects (e.g.
1363+ * a newline -> `%0A`), which native execution cannot open.
1364+ */
1365+ private [rules] case class RootPathIssues (
1366+ unsupportedSchemes : Set [String ],
1367+ multiBucket : Set [String ],
1368+ rejectedPath : Option [URI ])
1369+
1370+ private [rules] def checkObjectStoreRootPaths (
1371+ rootUris : Seq [URI ],
1372+ hadoopConf : Configuration ): RootPathIssues = {
1373+ // Schemes routed through libhdfs are natively readable; default to Set("hdfs") to match the
1374+ // native side (see the Parquet gate's comment). S3-compliant aliases come from the Hadoop
1375+ // config so core-site.xml is honored.
1376+ val libhdfsSchemes : Set [String ] = COMET_LIBHDFS_SCHEMES .get() match {
1377+ case Some (s) => NativeConfig .parseSchemeSet(s)
1378+ case None => Set (" hdfs" )
1379+ }
1380+ val s3CompliantSchemes = NativeConfig .resolveS3CompliantSchemes(hadoopConf)
1381+ val roots = classifyRootPaths(rootUris, libhdfsSchemes, s3CompliantSchemes)
1382+ val unsupportedSchemes = roots.iterator.flatMap { root =>
1383+ root.scheme.filter(_ =>
1384+ ! root.isLibhdfs && ! isNativelyReadableScheme(root.uri, s3CompliantSchemes))
1385+ }.toSet
1386+ val buckets = aliasScanBuckets(roots)
1387+ val rejectedPath = roots
1388+ .find(root =>
1389+ root.scheme.isDefined && ! root.isLibhdfs && ! root.isAlias &&
1390+ ! objectStoreAcceptsPath(root.uri))
1391+ .map(_.uri)
1392+ RootPathIssues (unsupportedSchemes, if (buckets.size > 1 ) buckets else Set .empty, rejectedPath)
1393+ }
1394+
14191395 /**
14201396 * The distinct buckets this scan's root paths address, or empty when none of them uses an
14211397 * opt-in S3-compliant alias. More than one bucket means the Parquet gate must fall back: native
0 commit comments