Skip to content

Commit e2d6deb

Browse files
committed
Merge remote-tracking branch 'origin/main' into feature/delta-native-scan
# Conflicts: # pom.xml # spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala
2 parents 977cd31 + cf66738 commit e2d6deb

95 files changed

Lines changed: 10411 additions & 1307 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -560,6 +560,7 @@ jobs:
560560
org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite
561561
org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite
562562
org.apache.spark.sql.comet.CometTaskMetricsSuite
563+
org.apache.spark.sql.comet.CometUnsafeProjectionSuite
563564
org.apache.spark.sql.comet.CometDppFallbackRepro3949Suite
564565
org.apache.spark.sql.comet.CometShuffleFallbackStickinessSuite
565566
org.apache.spark.sql.comet.PlanDataInjectorSuite
@@ -573,6 +574,9 @@ jobs:
573574
org.apache.comet.vector.CometVectorUtilsSuite
574575
org.apache.comet.udf.CometUdfBridgeSuite
575576
org.apache.comet.objectstore.NativeConfigSuite
577+
org.apache.comet.objectstore.AbfsAuthResolverSuite
578+
org.apache.comet.objectstore.AbfsAuthParitySuite
579+
org.apache.comet.objectstore.AbfsScanFallbackSuite
576580
org.apache.comet.serde.operator.CometIcebergNativeScanSuite
577581
org.apache.comet.serde.operator.CometNativeScanSuite
578582
org.apache.spark.sql.CometToPrettyStringSuite

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,7 @@ jobs:
264264
org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite
265265
org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite
266266
org.apache.spark.sql.comet.CometTaskMetricsSuite
267+
org.apache.spark.sql.comet.CometUnsafeProjectionSuite
267268
org.apache.spark.sql.comet.CometDppFallbackRepro3949Suite
268269
org.apache.spark.sql.comet.CometShuffleFallbackStickinessSuite
269270
org.apache.spark.sql.comet.PlanDataInjectorSuite
@@ -277,6 +278,9 @@ jobs:
277278
org.apache.comet.vector.CometVectorUtilsSuite
278279
org.apache.comet.udf.CometUdfBridgeSuite
279280
org.apache.comet.objectstore.NativeConfigSuite
281+
org.apache.comet.objectstore.AbfsAuthResolverSuite
282+
org.apache.comet.objectstore.AbfsAuthParitySuite
283+
org.apache.comet.objectstore.AbfsScanFallbackSuite
280284
org.apache.comet.serde.operator.CometIcebergNativeScanSuite
281285
org.apache.comet.serde.operator.CometNativeScanSuite
282286
org.apache.spark.sql.CometToPrettyStringSuite

‎contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/CometDeltaNativeScan.scala‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import org.apache.spark.sql.execution.{FileSourceScanExec, ScalarSubquery => Exe
3232
import org.apache.spark.sql.execution.datasources.{FilePartition, PartitionedFile}
3333
import org.apache.spark.sql.types.{ByteType, LongType, MetadataBuilder, StructField, StructType}
3434

35+
import org.apache.comet.CometSparkSessionExtensions.withFallbackReason
3536
import org.apache.comet.objectstore.NativeConfig
3637
import org.apache.comet.serde.OperatorOuterClass
3738
import org.apache.comet.serde.OperatorOuterClass.Operator
@@ -117,7 +118,8 @@ object CometDeltaNativeScan
117118
/**
118119
* Build the planning-time `DeltaScan` operator (common data only; file partitions are injected
119120
* lazily at execution). Returns None when an output data type cannot be serialized or the plan
120-
* shape is not one we can translate faithfully. `memo` is the same claim-memo instance
121+
* shape is not one we can translate faithfully, and tags `scanExec` with the reason when the
122+
* data files need more than one ABFS store. `memo` is the same claim-memo instance
121123
* [[DeltaScanSupport.declineReason]] populated on this claim; its `hadoopConf` and
122124
* `dvDescriptors` are reused here rather than recomputed.
123125
*/
@@ -127,13 +129,21 @@ object CometDeltaNativeScan
127129
memo: DeltaScanSupport.DeltaClaimMemo): Option[Operator] = {
128130
val relation = scanExec.relation
129131

130-
val firstFileUri = scanHelper.selectedPartitions
131-
.flatMap(_.files.headOption)
132-
.headOption
133-
.map(_.getPath.toUri)
132+
val dataFileUris = scanHelper.selectedPartitions.view.flatMap(_.files).map(_.getPath.toUri)
133+
val firstFileUri = dataFileUris.headOption
134134

135135
val hadoopConf = memo.hadoopConf
136136

137+
val objectStoreOptions =
138+
NativeConfig.extractScanObjectStoreOptions(hadoopConf, dataFileUris) match {
139+
case Right(options) => options
140+
case Left(reason) =>
141+
// declineReason already declines data files on more than one store; this only
142+
// catches ABFS authorities that differ in case or percent-encoding alone.
143+
withFallbackReason(scanExec, reason)
144+
return None
145+
}
146+
137147
val tableRootPath = relation.location.rootPaths.head
138148
val tableRoot = tableRootPath.toString
139149

@@ -148,11 +158,11 @@ object CometDeltaNativeScan
148158
partitionSchema = toPhysical(scanExec, relation.partitionSchema),
149159
fileConstantMetadataColumns = scanExec.fileConstantMetadataColumns,
150160
dataFilters = scanHelper.supportedDataFilters,
151-
firstFileUri = firstFileUri,
161+
objectStoreOptions = objectStoreOptions,
152162
hadoopConf = hadoopConf,
153163
conf = scanExec.conf)
154164
} else {
155-
buildDvScanCommon(scanExec, scanHelper, firstFileUri, hadoopConf)
165+
buildDvScanCommon(scanExec, scanHelper, objectStoreOptions, hadoopConf)
156166
}
157167

158168
commonOpt.map { commonBuilder =>
@@ -221,9 +231,12 @@ object CometDeltaNativeScan
221231

222232
/**
223233
* Unions `NativeConfig.extractObjectStoreOptions` over every `uris` authority. Safe to union
224-
* rather than pick one: extracted keys are scheme-disjoint prefixes (`fs.s3a.*` vs
225-
* `fs.azure.*`, ...), so options for different schemes never collide, and re-extracting the
226-
* same scheme from two URIs is idempotent.
234+
* rather than pick one: S3 and GCS options sit under their own scheme's prefix, and
235+
* re-extracting the same scheme from two URIs is idempotent. Azure options are not scoped to an
236+
* account and overlap (WASB copies `fs.azure.*`; ABFS forwards `comet.azure.*` markers plus
237+
* global `fs.azure.*` keys), but only the table root can bring an Azure authority here:
238+
* `DeltaScanSupport.declineReason` declines Azure data files and deletion vectors, whose
239+
* container is userinfo.
227240
*/
228241
private[delta] def mergedObjectStoreOptions(
229242
hadoopConf: org.apache.hadoop.conf.Configuration,
@@ -388,7 +401,7 @@ object CometDeltaNativeScan
388401
private def buildDvScanCommon(
389402
scanExec: FileSourceScanExec,
390403
scanHelper: CometScanExec,
391-
firstFileUri: Option[java.net.URI],
404+
objectStoreOptions: Map[String, String],
392405
hadoopConf: org.apache.hadoop.conf.Configuration)
393406
: Option[OperatorOuterClass.NativeScanCommon.Builder] = {
394407
val relation = scanExec.relation
@@ -482,7 +495,7 @@ object CometDeltaNativeScan
482495
CometNativeScan.populateScanConfFlags(
483496
commonBuilder,
484497
physicalRequired,
485-
firstFileUri,
498+
objectStoreOptions,
486499
hadoopConf,
487500
scanExec.conf)
488501

‎contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanContrib.scala‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan}
2626
import org.apache.spark.sql.execution.datasources.HadoopFsRelation
2727

2828
import org.apache.comet.CometConf.COMET_EXEC_ENABLED
29-
import org.apache.comet.CometSparkSessionExtensions.withFallbackReason
29+
import org.apache.comet.CometSparkSessionExtensions.{hasFallbackReason, withFallbackReason}
3030
import org.apache.comet.rules.CometScanContrib
3131

3232
/**
@@ -93,6 +93,9 @@ class DeltaScanContrib extends CometScanContrib with Logging {
9393
val subqueryDataFilters =
9494
CometDeltaNativeScan.subqueryFiltersFromParent(plan, scanExec)
9595
Some(CometDeltaNativeScanExec(nativeOp, scanExec, subqueryDataFilters))
96+
// convert tags its own reason when it declines the scan's object stores.
97+
case None if hasFallbackReason(scanExec) =>
98+
Some(scanExec)
9699
case None =>
97100
Some(
98101
withFallbackReason(

‎contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/DeltaScanContribSuite.scala‎

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,17 @@ import java.nio.file.Files
2525
import java.util.{Locale, UUID}
2626
import java.util.concurrent.atomic.AtomicBoolean
2727

28+
import scala.jdk.CollectionConverters._
29+
2830
import org.apache.hadoop.conf.Configuration
2931
import org.apache.hadoop.fs.{Path, RawLocalFileSystem}
3032
import org.apache.hadoop.fs.s3a.{S3AFileSystem, S3AUtils}
3133
import org.apache.hadoop.security.alias.CredentialProviderFactory
34+
import org.apache.spark.sql.comet.CometDeltaNativeScanExec
3235
import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor
3336

3437
import org.apache.comet.{CometConf, ExtendedExplainInfo}
38+
import org.apache.comet.objectstore.NativeConfig
3539
import org.apache.comet.rules.CometScanRule
3640

3741
/**
@@ -376,6 +380,25 @@ class DeltaScanContribSuite extends CometDeltaTestBase {
376380
assert(DeltaScanSupport.userInfoBearingAuthorityReason(sameContainer).isDefined)
377381
}
378382

383+
test("multiStoreReason declines data files on more than one ABFS store or beside s3a") {
384+
// Core's extractScanObjectStoreOptions needs one ABFS container and account per scan and no
385+
// other scheme beside ABFS. This gate declines all three shapes before convert reaches it.
386+
val shapes = Seq(
387+
"two accounts" -> Seq(
388+
new URI("abfss://[email protected]/t/part-0.parquet"),
389+
new URI("abfss://[email protected]/t/part-1.parquet")),
390+
"two containers" -> Seq(
391+
new URI("abfss://[email protected]/t/part-0.parquet"),
392+
new URI("abfss://[email protected]/t/part-1.parquet")),
393+
"abfss and s3a" -> Seq(
394+
new URI("abfss://[email protected]/t/part-0.parquet"),
395+
new URI("s3a://bucket/t/part-1.parquet")))
396+
shapes.foreach { case (shape, uris) =>
397+
val reason = DeltaScanSupport.multiStoreReason(uris)
398+
assert(reason.exists(_.contains("multiple object stores")), s"$shape: $reason")
399+
}
400+
}
401+
379402
test(
380403
"multiStoreReason declines distinct underscore-bearing GCS buckets " +
381404
"(URI#getHost null-collapse)") {
@@ -591,15 +614,16 @@ class DeltaScanContribSuite extends CometDeltaTestBase {
591614
CometDeltaNativeScan.mergedObjectStoreOptions(hadoopConf, Seq(s3Uri, abfssUri))
592615
assert(merged.get("fs.s3a.access.key").contains("s3-access-key"))
593616
assert(merged.get("fs.s3a.secret.key").contains("s3-secret-key"))
594-
assert(
595-
merged
596-
.get("fs.azure.account.key.acct.dfs.core.windows.net")
597-
.contains("azure-account-key"))
617+
// ABFS options are what Hadoop resolves for the account, not the fs.azure.* namespace.
618+
val abfsOptions = NativeConfig.extractObjectStoreOptions(hadoopConf, abfssUri)
619+
assert(abfsOptions.nonEmpty)
620+
abfsOptions.foreach { case (key, value) => assert(merged.get(key).contains(value), key) }
598621

599-
// s3-only input must not leak the azure credentials into the merged map.
622+
// s3-only input must not leak the azure options into the merged map.
600623
val s3Only = CometDeltaNativeScan.mergedObjectStoreOptions(hadoopConf, Seq(s3Uri))
601624
assert(s3Only.get("fs.s3a.access.key").contains("s3-access-key"))
602-
assert(!s3Only.keys.exists(_.startsWith("fs.azure.")))
625+
assert(
626+
!s3Only.keys.exists(key => key.startsWith("fs.azure.") || key.startsWith("comet.azure.")))
603627
}
604628

605629
test(
@@ -659,6 +683,30 @@ class DeltaScanContribSuite extends CometDeltaTestBase {
659683
Seq(tableRootPath.toUri))
660684
}
661685

686+
test("a scan that selects no files keeps the table root's object-store options") {
687+
withTempPath { dir =>
688+
val path = dir.getAbsolutePath
689+
spark
690+
.range(0, 10)
691+
.selectExpr("id", "id % 2 AS p")
692+
.write
693+
.format("delta")
694+
.partitionBy("p")
695+
.save(path)
696+
// The scheme list rides along in every extracted map, so a local root's options are not
697+
// empty.
698+
withSQLConf(CometConf.COMET_S3_COMPLIANT_SCHEMES_KEY -> "minio") {
699+
val df = spark.read.format("delta").load(path).where("p = 5")
700+
assert(df.collect().isEmpty)
701+
val scans = deltaNativeScans(df).collect { case s: CometDeltaNativeScanExec => s }
702+
assert(scans.size == 1, s"expected one native Delta scan:\n${df.queryExecution}")
703+
val options =
704+
DeltaSparkScanEnvelope.unpack(scans.head.nativeOp).getCommon.getObjectStoreOptionsMap
705+
assert(options.asScala.toMap == Map(CometConf.COMET_S3_COMPLIANT_SCHEMES_KEY -> "minio"))
706+
}
707+
}
708+
}
709+
662710
test("user guide documents every native Delta scan config verbatim, with its default") {
663711
// The config table on the user-guide page is hand-maintained: the doc build cannot see
664712
// DeltaSparkConfigProvider with the current module layout, so this is the only check tying

‎dev/diffs/3.5.9.diff‎

Lines changed: 17 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -2461,7 +2461,7 @@ index 2c73622739a..5d0efeb263a 100644
24612461

24622462
val d1 = spark.range(1).withColumn("key", expr("id % 100"))
24632463
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala
2464-
index 885286843a1..f0f805a6cd9 100644
2464+
index 885286843a1..6e9d6690374 100644
24652465
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala
24662466
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala
24672467
@@ -17,6 +17,7 @@
@@ -2472,60 +2472,22 @@ index 885286843a1..f0f805a6cd9 100644
24722472
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
24732473
import org.apache.spark.sql.internal.SQLConf
24742474
import org.apache.spark.sql.test.SharedSparkSession
2475-
@@ -181,11 +182,16 @@ class PartitionBatchPruningSuite extends SharedSparkSession with AdaptiveSparkPl
2476-
val result = df.collect().map(_(0)).toArray
2477-
assert(result.length === 1)
2478-
2479-
- val (readPartitions, readBatches) = collect(df.queryExecution.executedPlan) {
2480-
- case in: InMemoryTableScanExec => (in.readPartitions.value, in.readBatches.value)
2481-
- }.head
2482-
- assert(readPartitions === 5)
2483-
- assert(readBatches === 10)
2484-
+ val scans = collect(df.queryExecution.executedPlan) { case in: InMemoryTableScanExec => in }
2485-
+ // Comet's cache scan has none of these test-only accumulators, so there is nothing to count.
2486-
+ if (scans.isEmpty) {
2487-
+ assert(collect(df.queryExecution.executedPlan) {
2488-
+ case c: CometInMemoryTableScanExec => c
2489-
+ }.nonEmpty)
2490-
+ } else {
2491-
+ assert(scans.head.readPartitions.value === 5)
2492-
+ assert(scans.head.readBatches.value === 10)
2493-
+ }
2494-
}
2495-
2496-
def checkBatchPruning(
2497-
@@ -202,14 +208,23 @@ class PartitionBatchPruningSuite extends SharedSparkSession with AdaptiveSparkPl
2498-
df.collect().map(_(0)).toArray
2499-
}
2500-
2501-
- val (readPartitions, readBatches) = collect(df.queryExecution.executedPlan) {
2502-
- case in: InMemoryTableScanExec => (in.readPartitions.value, in.readBatches.value)
2503-
- }.head
2504-
-
2505-
- assert(readBatches === expectedReadBatches, s"Wrong number of read batches: $queryExecution")
2506-
- assert(
2507-
- readPartitions === expectedReadPartitions,
2508-
- s"Wrong number of read partitions: $queryExecution")
2509-
+ val scans = collect(df.queryExecution.executedPlan) { case in: InMemoryTableScanExec => in }
2510-
+ // Comet's cache scan prunes on the same statistics, but has none of these test-only
2511-
+ // accumulators to read, so only the answer above is checked for it.
2512-
+ if (scans.isEmpty) {
2513-
+ assert(collect(df.queryExecution.executedPlan) {
2514-
+ case c: CometInMemoryTableScanExec => c
2515-
+ }.nonEmpty)
2516-
+ } else {
2517-
+ val readPartitions = scans.head.readPartitions.value
2518-
+ val readBatches = scans.head.readBatches.value
2519-
+ assert(
2520-
+ readBatches === expectedReadBatches,
2521-
+ s"Wrong number of read batches: $queryExecution")
2522-
+ assert(
2523-
+ readPartitions === expectedReadPartitions,
2524-
+ s"Wrong number of read partitions: $queryExecution")
2525-
+ }
2526-
}
2527-
}
2528-
}
2475+
@@ -183,6 +184,7 @@ class PartitionBatchPruningSuite extends SharedSparkSession with AdaptiveSparkPl
2476+
2477+
val (readPartitions, readBatches) = collect(df.queryExecution.executedPlan) {
2478+
case in: InMemoryTableScanExec => (in.readPartitions.value, in.readBatches.value)
2479+
+ case in: CometInMemoryTableScanExec => (in.readPartitions.value, in.readBatches.value)
2480+
}.head
2481+
assert(readPartitions === 5)
2482+
assert(readBatches === 10)
2483+
@@ -204,6 +206,7 @@ class PartitionBatchPruningSuite extends SharedSparkSession with AdaptiveSparkPl
2484+
2485+
val (readPartitions, readBatches) = collect(df.queryExecution.executedPlan) {
2486+
case in: InMemoryTableScanExec => (in.readPartitions.value, in.readBatches.value)
2487+
+ case in: CometInMemoryTableScanExec => (in.readPartitions.value, in.readBatches.value)
2488+
}.head
2489+
2490+
assert(readBatches === expectedReadBatches, s"Wrong number of read batches: $queryExecution")
25292491
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
25302492
index fd52d038ca6..154c800be67 100644
25312493
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala

0 commit comments

Comments
 (0)