Skip to content

Commit 23bba3e

Browse files
committed
more comments addressed
1 parent d12b913 commit 23bba3e

2 files changed

Lines changed: 77 additions & 96 deletions

File tree

‎spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala‎

Lines changed: 72 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -246,77 +246,31 @@ case class CometScanRule(session: SparkSession)
246246
return None
247247
}
248248
// Comet's native readers go through object_store, which only understands a fixed set of URL
249-
// schemes. A custom Hadoop FileSystem (e.g. registered via spark.hadoop.fs.<scheme>.impl) would
250-
// surface at execution time as `Generic URL error: Unable to recognise URL "..."`. Decline here
251-
// so Spark's reader -- which goes through the Hadoop FS API and can resolve custom schemes --
252-
// handles the scan. Whether object_store recognizes a scheme is answered by the native layer
253-
// itself (`NativeBase.isObjectStoreSchemeSupported`) rather than a hardcoded list, so the
254-
// planner can't drift from object_store's actual support.
255-
//
256-
// EXCEPT schemes the user routes through libhdfs via `spark.hadoop.fs.comet.libhdfs.schemes`
257-
// (e.g. `hdfs`, or a test `fake`): those ARE natively readable through the libhdfs object_store
258-
// bridge, so they must NOT be declined here. The claim decision is guarded in CI by
259-
// CometScanSchemeFallbackSuite; end-to-end execution through libhdfs is guarded by
260-
// ParquetReadFromFakeHadoopFsSuite, which is a manual suite (see its scaladoc).
261-
//
262-
// The default mirrors the native side: when the config is unset, `is_hdfs_scheme`
263-
// (native/core/src/parquet/parquet_support.rs) treats `hdfs` as natively readable, and
264-
// `create_hdfs_object_store` is in the default build (`default = ["hdfs-opendal"]`). If we
265-
// defaulted to an empty set here, a plain `hdfs://` V1 scan would be declined and fall back to
266-
// Spark even though native can read it -- a silent regression for HDFS users in the default
267-
// configuration. So default to `Set("hdfs")` to stay in lockstep with the native default.
268-
val libhdfsSchemes: Set[String] = COMET_LIBHDFS_SCHEMES.get() match {
269-
case Some(s) => NativeConfig.parseSchemeSet(s)
270-
case None => Set("hdfs")
271-
}
272-
// Opt-in S3-compliant alias schemes (e.g. `blob`) from `fs.comet.s3Compliant.schemes`. Read
273-
// from the Hadoop config rather than SQLConf (unlike COMET_LIBHDFS_SCHEMES above) so
274-
// `core-site.xml` is honored. The native object_store gate no longer claims aliases, so admit
275-
// them here where the Hadoop config is available.
276-
val s3CompliantSchemes = NativeConfig.resolveS3CompliantSchemes(hadoopConf)
277-
278-
// Classify each root path once; see RootPathInfo.
279-
val roots = CometScanRule.classifyRootPaths(
280-
r.location.rootPaths.map(_.toUri),
281-
libhdfsSchemes,
282-
s3CompliantSchemes)
283-
284-
val unsupportedFsSchemes = roots.iterator.flatMap { root =>
285-
root.scheme.filter(_ =>
286-
!root.isLibhdfs && !CometScanRule.isNativelyReadableScheme(root.uri, s3CompliantSchemes))
287-
}.toSet
288-
if (unsupportedFsSchemes.nonEmpty) {
249+
// schemes; a custom Hadoop FileSystem would hard-fail at execution. The shared helper
250+
// classifies the root paths (honoring libhdfs and S3-compliant aliases) and reports
251+
// unsupported schemes, multi-bucket alias scans, and object_store-rejected paths. The scheme
252+
// claim decision is guarded in CI by CometScanSchemeFallbackSuite; libhdfs end-to-end by
253+
// ParquetReadFromFakeHadoopFsSuite.
254+
val rootIssues =
255+
CometScanRule.checkObjectStoreRootPaths(r.location.rootPaths.map(_.toUri), hadoopConf)
256+
if (rootIssues.unsupportedSchemes.nonEmpty) {
289257
withFallbackReason(
290258
scanExec,
291-
s"Unsupported filesystem schemes: ${unsupportedFsSchemes.mkString(", ")}")
259+
s"Unsupported filesystem schemes: ${rootIssues.unsupportedSchemes.mkString(", ")}")
292260
return None
293261
}
294-
// More than one bucket cannot be served by the single object store native planning registers
295-
// per FilePartition; see aliasScanBuckets. Scoped to alias scans: plain multi-bucket `s3://`
296-
// has the same flaw today and silently declining it would newly fall back scans that work by
297-
// luck, so that widening is left as a separate decision. Note the sibling Iceberg guard
298-
// (dataFileBuckets, below) is NOT so scoped -- it declines multi-bucket `s3a://` too.
299-
val scanBuckets = CometScanRule.aliasScanBuckets(roots)
300-
if (scanBuckets.size > 1) {
262+
if (rootIssues.multiBucket.nonEmpty) {
301263
withFallbackReason(
302264
scanExec,
303265
"Native Parquet scan reads S3-compliant alias paths across multiple buckets " +
304-
s"(${scanBuckets.toSeq.sorted.mkString(", ")}); Comet registers one object store " +
305-
"per file partition and would read every file from the first file's bucket")
266+
s"(${rootIssues.multiBucket.toSeq.sorted.mkString(", ")}); Comet registers one object " +
267+
"store per file partition and would read every file from the first file's bucket")
306268
return None
307269
}
308-
// A scheme object_store recognizes can still carry a path it rejects (e.g. a directory whose
309-
// name contains a newline -> `%0A` in the URI), which native execution would hard-fail on.
310-
// Only object_store-native schemes are probed: aliases (normalized to s3:// natively) and
311-
// libhdfs schemes route elsewhere. Root paths only -- a rejected character deeper in the tree
312-
// still fails at execution.
313-
val rejectedPath = roots.find(root =>
314-
root.scheme.isDefined && !root.isLibhdfs && !root.isAlias &&
315-
!CometScanRule.objectStoreAcceptsPath(root.uri))
316-
if (rejectedPath.nonEmpty) {
270+
if (rootIssues.rejectedPath.nonEmpty) {
317271
withFallbackReason(
318272
scanExec,
319-
s"Native Parquet scan cannot open path '${rejectedPath.get.uri}': object_store " +
273+
s"Native Parquet scan cannot open path '${rootIssues.rejectedPath.get}': object_store " +
320274
"rejects it (e.g. an unsupported character in the path)")
321275
return None
322276
}
@@ -577,46 +531,25 @@ case class CometScanRule(session: SparkSession)
577531
}
578532
}
579533
// Comet's native reader opens files through object_store, which only understands a fixed
580-
// set of URL schemes. A text file on a custom Hadoop scheme (viewfs://, oss://, ...) would
581-
// be claimed here then hard-fail at execution, whereas Spark reads it via the Hadoop FS
582-
// API. Decline such schemes (same gate as the native Parquet path); hdfs:// routes through
583-
// libhdfs and is left to native.
584-
val libhdfsSchemes: Set[String] = COMET_LIBHDFS_SCHEMES.get() match {
585-
case Some(s) => NativeConfig.parseSchemeSet(s)
586-
case None => Set("hdfs")
587-
}
588-
val s3CompliantSchemes = NativeConfig.resolveS3CompliantSchemes(hadoopConf)
589-
val roots = CometScanRule.classifyRootPaths(
534+
// set of URL schemes. A text file on a custom Hadoop scheme (viewfs://, oss://, ...), a
535+
// multi-bucket S3-compliant alias scan, or a path object_store rejects would be claimed
536+
// here then hard-fail at execution, whereas Spark reads it via the Hadoop FS API. Use the
537+
// same shared gate as the native Parquet path; hdfs:// routes through libhdfs.
538+
val rootIssues = CometScanRule.checkObjectStoreRootPaths(
590539
scan.fileIndex.rootPaths.map(_.toUri),
591-
libhdfsSchemes,
592-
s3CompliantSchemes)
593-
val unsupportedFsSchemes = roots.iterator.flatMap { root =>
594-
root.scheme.filter(_ =>
595-
!root.isLibhdfs && !CometScanRule.isNativelyReadableScheme(
596-
root.uri,
597-
s3CompliantSchemes))
598-
}.toSet
599-
if (unsupportedFsSchemes.nonEmpty) {
540+
hadoopConf)
541+
if (rootIssues.unsupportedSchemes.nonEmpty) {
600542
fallbackReasons +=
601-
s"Unsupported filesystem schemes: ${unsupportedFsSchemes.mkString(", ")}"
602-
}
603-
// A recognized scheme can still carry a path object_store rejects (e.g. a newline -> %0A),
604-
// which native execution would hard-fail on. Root paths only.
605-
val rejectedPath = roots.find(root =>
606-
root.scheme.isDefined && !root.isLibhdfs && !root.isAlias &&
607-
!CometScanRule.objectStoreAcceptsPath(root.uri))
608-
if (rejectedPath.nonEmpty) {
609-
fallbackReasons += s"Native Text scan cannot open path '${rejectedPath.get.uri}': " +
610-
"object_store rejects it (e.g. an unsupported character in the path)"
543+
s"Unsupported filesystem schemes: ${rootIssues.unsupportedSchemes.mkString(", ")}"
611544
}
612-
// Native planning registers one object store per partition and strips the bucket from every
613-
// key, so an S3-compliant alias scan spanning multiple buckets would read every file from
614-
// the first bucket. Decline multi-bucket alias scans (same hazard the Parquet gate guards).
615-
val scanBuckets = CometScanRule.aliasScanBuckets(roots)
616-
if (scanBuckets.size > 1) {
545+
if (rootIssues.multiBucket.nonEmpty) {
617546
fallbackReasons += "Native Text scan reads S3-compliant alias paths across multiple " +
618-
s"buckets (${scanBuckets.toSeq.sorted.mkString(", ")}); Comet registers one object " +
619-
"store per file partition and would read every file from the first file's bucket"
547+
s"buckets (${rootIssues.multiBucket.toSeq.sorted.mkString(", ")}); Comet registers " +
548+
"one object store per file partition and would read every file from the first bucket"
549+
}
550+
rootIssues.rejectedPath.foreach { uri =>
551+
fallbackReasons += s"Native Text scan cannot open path '$uri': " +
552+
"object_store rejects it (e.g. an unsupported character in the path)"
620553
}
621554
if (fallbackReasons.isEmpty) {
622555
CometBatchScanExec(
@@ -1416,6 +1349,49 @@ object CometScanRule extends Logging {
14161349
bucket = NativeConfig.bucketForUri(uri, s3CompliantSchemes))
14171350
}
14181351

1352+
/**
1353+
* The object_store scheme/bucket/path problems that make a set of root paths unreadable by
1354+
* Comet's native file scans (shared by the Parquet V1 gate and the Text V2 gate). Each caller
1355+
* formats its own fallback message from these findings.
1356+
*
1357+
* - `unsupportedSchemes`: schemes object_store does not natively read and that are not routed
1358+
* through libhdfs (a custom Hadoop FileSystem would hard-fail at execution).
1359+
* - `multiBucket`: the S3-compliant alias buckets when a scan spans more than one; native
1360+
* planning registers one object store per FilePartition and would read every file from the
1361+
* first file's bucket. Empty when a single bucket (or no alias) is used.
1362+
* - `rejectedPath`: a root whose recognized scheme carries a path object_store rejects (e.g.
1363+
* a newline -> `%0A`), which native execution cannot open.
1364+
*/
1365+
private[rules] case class RootPathIssues(
1366+
unsupportedSchemes: Set[String],
1367+
multiBucket: Set[String],
1368+
rejectedPath: Option[URI])
1369+
1370+
private[rules] def checkObjectStoreRootPaths(
1371+
rootUris: Seq[URI],
1372+
hadoopConf: Configuration): RootPathIssues = {
1373+
// Schemes routed through libhdfs are natively readable; default to Set("hdfs") to match the
1374+
// native side (see the Parquet gate's comment). S3-compliant aliases come from the Hadoop
1375+
// config so core-site.xml is honored.
1376+
val libhdfsSchemes: Set[String] = COMET_LIBHDFS_SCHEMES.get() match {
1377+
case Some(s) => NativeConfig.parseSchemeSet(s)
1378+
case None => Set("hdfs")
1379+
}
1380+
val s3CompliantSchemes = NativeConfig.resolveS3CompliantSchemes(hadoopConf)
1381+
val roots = classifyRootPaths(rootUris, libhdfsSchemes, s3CompliantSchemes)
1382+
val unsupportedSchemes = roots.iterator.flatMap { root =>
1383+
root.scheme.filter(_ =>
1384+
!root.isLibhdfs && !isNativelyReadableScheme(root.uri, s3CompliantSchemes))
1385+
}.toSet
1386+
val buckets = aliasScanBuckets(roots)
1387+
val rejectedPath = roots
1388+
.find(root =>
1389+
root.scheme.isDefined && !root.isLibhdfs && !root.isAlias &&
1390+
!objectStoreAcceptsPath(root.uri))
1391+
.map(_.uri)
1392+
RootPathIssues(unsupportedSchemes, if (buckets.size > 1) buckets else Set.empty, rejectedPath)
1393+
}
1394+
14191395
/**
14201396
* The distinct buckets this scan's root paths address, or empty when none of them uses an
14211397
* opt-in S3-compliant alias. More than one bucket means the Parquet gate must fall back: native

‎spark/src/main/scala/org/apache/spark/sql/comet/CometCsvNativeScanExec.scala‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,11 @@ case class CometCsvNativeScanExec(
5050

5151
override val nodeName: String = "CometCsvNativeScan"
5252

53+
// Show only the semantic output in EXPLAIN / the SQL tab / the event log; the default would
54+
// print the whole `nativeOp` protobuf, which carries every input file path. Mirrors
55+
// CometNativeScanExec.
56+
override def stringArgs: Iterator[Any] = Iterator(output)
57+
5358
override def outputPartitioning: Partitioning = UnknownPartitioning(
5459
originalPlan.inputPartitions.length)
5560

0 commit comments

Comments
 (0)