Skip to content

Commit 2cf7063

Browse files
adivaishCopilot
andcommitted
feat(delta): end-to-end native Delta scan from Spark
Wires the native DeltaScan operator through to Spark execution: - CometDeltaNativeScanExec (CometLeafExec): a single-partition leaf that emits the self-contained DeltaScan operator; no PlanDataInjector is needed. - CometScanRule detects an eligible Delta FileSourceScanExec and replaces it, behind spark.comet.scan.deltaNative.enabled (default off). v1 is conservative: non-partitioned tables read with all columns and no data filters; everything else falls back to Spark. - Native isFeatureEnabled("delta") so JVM tests self-skip when Comet is built without the delta cargo feature. - Per-Spark-profile delta test deps (delta-core 2.4.0 for 3.4; delta-spark 3.3.2/4.0.0/4.1.0/4.2.0). - CometDeltaNativeReadSuite (registered in the CI workflows) verifies the native scan node appears and results match Spark, plus fallback when disabled. Validated end to end on Spark 4.1 + Delta 4.1.0 with a delta-feature native build (2 tests pass). Part of apache#174. Co-authored-by: Copilot <[email protected]>
1 parent dbb7d1d commit 2cf7063

9 files changed

Lines changed: 264 additions & 1 deletion

File tree

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,7 @@ jobs:
318318
org.apache.comet.CometFuzzTestSuite
319319
org.apache.comet.CometFuzzIcebergSuite
320320
org.apache.comet.DataGeneratorSuite
321+
org.apache.comet.CometDeltaNativeReadSuite
321322
- name: "shuffle"
322323
value: |
323324
org.apache.comet.exec.CometShuffleSuite

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ jobs:
134134
org.apache.comet.CometFuzzTestSuite
135135
org.apache.comet.CometFuzzIcebergSuite
136136
org.apache.comet.DataGeneratorSuite
137+
org.apache.comet.CometDeltaNativeReadSuite
137138
- name: "shuffle"
138139
value: |
139140
org.apache.comet.exec.CometShuffleSuite

‎native/core/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ pub extern "system" fn Java_org_apache_comet_NativeBase_isFeatureEnabled(
152152
"jemalloc" => cfg!(feature = "jemalloc"),
153153
"hdfs" => cfg!(feature = "hdfs"),
154154
"hdfs-opendal" => cfg!(feature = "hdfs-opendal"),
155+
"delta" => cfg!(feature = "delta"),
155156
_ => false, // Unknown features return false
156157
};
157158

‎pom.xml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,10 @@ under the License.
7272
<spark.version>4.1.2</spark.version>
7373
<spark.version.short>4.1</spark.version.short>
7474
<spark.maven.scope>provided</spark.maven.scope>
75+
<!-- Delta Lake version used only by Comet's Delta scan tests. Overridden per Spark profile.
76+
Delta >= 3.0 publishes "delta-spark"; Delta < 3.0 (Spark 3.4) publishes "delta-core". -->
77+
<delta.version>4.1.0</delta.version>
78+
<delta.artifact.name>delta-spark</delta.artifact.name>
7579
<protobuf.version>3.25.5</protobuf.version>
7680
<parquet.version>1.16.0</parquet.version>
7781
<parquet.maven.scope>provided</parquet.maven.scope>
@@ -654,6 +658,8 @@ under the License.
654658
<scala.binary.version>2.12</scala.binary.version>
655659
<spark.version>3.4.3</spark.version>
656660
<spark.version.short>3.4</spark.version.short>
661+
<delta.version>2.4.0</delta.version>
662+
<delta.artifact.name>delta-core</delta.artifact.name>
657663
<parquet.version>1.13.1</parquet.version>
658664
<semanticdb.version>4.8.8</semanticdb.version>
659665
<slf4j.version>2.0.6</slf4j.version>
@@ -672,6 +678,7 @@ under the License.
672678
<scala.binary.version>2.12</scala.binary.version>
673679
<spark.version>3.5.8</spark.version>
674680
<spark.version.short>3.5</spark.version.short>
681+
<delta.version>3.3.2</delta.version>
675682
<parquet.version>1.13.1</parquet.version>
676683
<semanticdb.version>4.8.8</semanticdb.version>
677684
<slf4j.version>2.0.7</slf4j.version>
@@ -690,6 +697,7 @@ under the License.
690697
<scala.binary.version>2.13</scala.binary.version>
691698
<spark.version>4.0.2</spark.version>
692699
<spark.version.short>4.0</spark.version.short>
700+
<delta.version>4.0.0</delta.version>
693701
<parquet.version>1.15.2</parquet.version>
694702
<semanticdb.version>4.13.6</semanticdb.version>
695703
<slf4j.version>2.0.16</slf4j.version>
@@ -732,6 +740,7 @@ under the License.
732740
<scala.binary.version>2.13</scala.binary.version>
733741
<spark.version>4.2.0-preview4</spark.version>
734742
<spark.version.short>4.2</spark.version.short>
743+
<delta.version>4.2.0</delta.version>
735744
<parquet.version>1.17.0</parquet.version>
736745
<semanticdb.version>4.13.6</semanticdb.version>
737746
<slf4j.version>2.0.17</slf4j.version>

‎spark/pom.xml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,20 @@ under the License.
8787
<artifactId>junit-4-13_${scala.binary.version}</artifactId>
8888
<scope>test</scope>
8989
</dependency>
90+
<!-- Used by CometDeltaNativeReadSuite to exercise the native Delta scan. Version/artifact are
91+
set per Spark profile in the root pom (delta-spark for Delta >= 3.0, delta-core for < 3.0). -->
92+
<dependency>
93+
<groupId>io.delta</groupId>
94+
<artifactId>${delta.artifact.name}_${scala.binary.version}</artifactId>
95+
<version>${delta.version}</version>
96+
<scope>test</scope>
97+
</dependency>
98+
<dependency>
99+
<groupId>com.google.guava</groupId>
100+
<artifactId>failureaccess</artifactId>
101+
<version>1.0.2</version>
102+
<scope>test</scope>
103+
</dependency>
90104
<dependency>
91105
<groupId>org.apache.spark</groupId>
92106
<artifactId>spark-core_${scala.binary.version}</artifactId>

‎spark/src/main/scala/org/apache/comet/CometConf.scala‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,18 @@ object CometConf extends ShimCometConf {
132132
.checkValue(v => v > 0, "Data file concurrency limit must be positive")
133133
.createWithDefault(1)
134134

135+
val COMET_DELTA_NATIVE_ENABLED: ConfigEntry[Boolean] =
136+
conf("spark.comet.scan.deltaNative.enabled")
137+
.category(CATEGORY_SCAN)
138+
.doc(
139+
"Whether to enable native Delta Lake table scan using delta-kernel-rs. Experimental and " +
140+
"requires Comet to be built with the `delta` cargo feature. When enabled, eligible " +
141+
"Delta scans are read directly through native execution. Currently limited to " +
142+
"non-partitioned tables read without projection or data filters; other cases fall back " +
143+
"to Spark.")
144+
.booleanConf
145+
.createWithDefault(false)
146+
135147
val COMET_CSV_V2_NATIVE_ENABLED: ConfigEntry[Boolean] =
136148
conf("spark.comet.scan.csv.v2.enabled")
137149
.category(CATEGORY_TESTING)

‎spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, DynamicPruningExpre
3232
import org.apache.spark.sql.catalyst.rules.Rule
3333
import org.apache.spark.sql.catalyst.util.{sideBySide, ArrayBasedMapData, GenericArrayData, MetadataColumnHelper}
3434
import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getExistenceDefaultValues
35-
import org.apache.spark.sql.comet.{CometBatchScanExec, CometScanExec}
35+
import org.apache.spark.sql.comet.{CometBatchScanExec, CometDeltaNativeScanExec, CometScanExec, SerializedPlan}
3636
import org.apache.spark.sql.execution.{FileSourceScanExec, InSubqueryExec, SparkPlan, SubqueryAdaptiveBroadcastExec}
3737
import org.apache.spark.sql.execution.datasources.HadoopFsRelation
3838
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
@@ -47,6 +47,7 @@ import org.apache.comet.DataTypeSupport.isComplexType
4747
import org.apache.comet.iceberg.{CometIcebergNativeScanMetadata, IcebergReflection}
4848
import org.apache.comet.objectstore.NativeConfig
4949
import org.apache.comet.parquet.CometParquetUtils.{encryptionEnabled, isEncryptionConfigSupported}
50+
import org.apache.comet.serde.OperatorOuterClass
5051
import org.apache.comet.serde.operator.{CometIcebergNativeScan, CometNativeScan}
5152
import org.apache.comet.shims.{CometTypeShim, ShimCometStreaming, ShimFileFormat, ShimSubqueryBroadcast}
5253

@@ -162,6 +163,10 @@ case class CometScanRule(session: SparkSession)
162163

163164
scanExec.relation match {
164165
case r: HadoopFsRelation =>
166+
tryDeltaNativeScan(scanExec, r) match {
167+
case Some(deltaScan) => return deltaScan
168+
case None => // not an eligible native Delta scan; continue with normal handling
169+
}
165170
if (!CometScanExec.isFileFormatSupported(r.fileFormat)) {
166171
return withFallbackReason(scanExec, s"Unsupported file format ${r.fileFormat}")
167172
}
@@ -189,6 +194,35 @@ case class CometScanRule(session: SparkSession)
189194
}
190195
}
191196

197+
/**
198+
* Attempt to read a Delta scan natively via delta-kernel-rs. Returns `Some` when this scan is
199+
* an eligible Delta scan and `spark.comet.scan.deltaNative.enabled` is on. v1 is intentionally
200+
* conservative: non-partitioned tables read with all columns (no projection) and no data
201+
* filters, so the kernel's full-table read matches Spark's output exactly. Everything else
202+
* returns `None` and falls through to the normal scan handling.
203+
*/
204+
private def tryDeltaNativeScan(
205+
scanExec: FileSourceScanExec,
206+
r: HadoopFsRelation): Option[SparkPlan] = {
207+
if (!CometConf.COMET_DELTA_NATIVE_ENABLED.get()) {
208+
return None
209+
}
210+
if (r.fileFormat.getClass.getName != "org.apache.spark.sql.delta.DeltaParquetFileFormat") {
211+
return None
212+
}
213+
val readsAllColumns =
214+
scanExec.requiredSchema.fieldNames.toSeq == r.dataSchema.fieldNames.toSeq
215+
val rootPaths = r.location.rootPaths
216+
if (r.partitionSchema.nonEmpty || !readsAllColumns || scanExec.dataFilters.nonEmpty ||
217+
rootPaths.length != 1) {
218+
return None
219+
}
220+
val tableUri = rootPaths.head.toUri.toString
221+
val deltaScan = OperatorOuterClass.DeltaScan.newBuilder().setTableUri(tableUri).build()
222+
val op = OperatorOuterClass.Operator.newBuilder().setDeltaScan(deltaScan).build()
223+
Some(CometDeltaNativeScanExec(op, scanExec.output, tableUri, scanExec, SerializedPlan(None)))
224+
}
225+
192226
private def nativeScan(
193227
plan: SparkPlan,
194228
session: SparkSession,
Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.spark.sql.comet
21+
22+
import scala.jdk.CollectionConverters._
23+
24+
import org.apache.spark.rdd.RDD
25+
import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder}
26+
import org.apache.spark.sql.catalyst.plans.QueryPlan
27+
import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
28+
import org.apache.spark.sql.execution.FileSourceScanExec
29+
import org.apache.spark.sql.vectorized.ColumnarBatch
30+
31+
import com.google.common.base.Objects
32+
33+
import org.apache.comet.serde.OperatorOuterClass.Operator
34+
35+
/**
36+
* Native Delta Lake scan that reads a table through delta-kernel-rs (behind the `delta` cargo
37+
* feature). The native `DeltaScan` operator is self-contained - it carries the table URI and the
38+
* kernel performs log replay, deletion-vector and column-mapping application, and partition-value
39+
* injection - so this leaf needs no per-partition planning data and reads the table as a single
40+
* partition. Projection, partition pruning, and predicate pushdown are follow-ups.
41+
*/
42+
case class CometDeltaNativeScanExec(
43+
override val nativeOp: Operator,
44+
override val output: Seq[Attribute],
45+
tableUri: String,
46+
@transient originalPlan: FileSourceScanExec,
47+
override val serializedPlanOpt: SerializedPlan)
48+
extends CometLeafExec {
49+
50+
override val supportsColumnar: Boolean = true
51+
52+
override val nodeName: String = s"CometDeltaNativeScan $tableUri"
53+
54+
override lazy val outputPartitioning: Partitioning = UnknownPartitioning(1)
55+
56+
override lazy val outputOrdering: Seq[SortOrder] = Nil
57+
58+
override def doExecuteColumnar(): RDD[ColumnarBatch] = {
59+
val nativeMetrics = CometMetricNode.fromCometPlan(this)
60+
val serializedPlan = CometExec.serializeNativePlan(nativeOp)
61+
CometExecRDD(
62+
sparkContext,
63+
inputRDDs = Seq.empty,
64+
commonByKey = Map.empty,
65+
perPartitionByKey = Map.empty,
66+
serializedPlan = serializedPlan,
67+
numPartitions = 1,
68+
numOutputCols = output.length,
69+
nativeMetrics = nativeMetrics,
70+
subqueries = Seq.empty)
71+
}
72+
73+
override def convertBlock(): CometDeltaNativeScanExec = {
74+
val newSerializedPlan = if (serializedPlanOpt.isEmpty) {
75+
SerializedPlan(Some(CometExec.serializeNativePlan(nativeOp)))
76+
} else {
77+
serializedPlanOpt
78+
}
79+
CometDeltaNativeScanExec(nativeOp, output, tableUri, originalPlan, newSerializedPlan)
80+
}
81+
82+
override protected def doCanonicalize(): CometDeltaNativeScanExec = {
83+
CometDeltaNativeScanExec(
84+
nativeOp,
85+
output.map(QueryPlan.normalizeExpressions(_, output)),
86+
tableUri,
87+
null,
88+
SerializedPlan(None))
89+
}
90+
91+
override def stringArgs: Iterator[Any] = Iterator(output, tableUri)
92+
93+
override def equals(obj: Any): Boolean = obj match {
94+
case other: CometDeltaNativeScanExec =>
95+
tableUri == other.tableUri && output == other.output &&
96+
serializedPlanOpt == other.serializedPlanOpt
97+
case _ => false
98+
}
99+
100+
override def hashCode(): Int = Objects.hashCode(tableUri, output.asJava, serializedPlanOpt)
101+
}
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.comet
21+
22+
import org.apache.spark.SparkConf
23+
import org.apache.spark.sql.CometTestBase
24+
import org.apache.spark.sql.internal.SQLConf
25+
26+
/**
27+
* End-to-end tests for the native Delta Lake scan (delta-kernel-rs). These require Comet to be
28+
* built with the `delta` cargo feature; when it is not, the suite self-cancels (the native
29+
* `isFeatureEnabled("delta")` check returns false), so it is safe to run in CI against a Comet
30+
* built without the feature.
31+
*/
32+
class CometDeltaNativeReadSuite extends CometTestBase {
33+
34+
override protected def sparkConf: SparkConf = {
35+
val conf = super.sparkConf
36+
conf.set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
37+
conf.set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
38+
conf
39+
}
40+
41+
private def assumeDeltaFeature(): Unit =
42+
assume(isFeatureEnabled("delta"), "Comet was not built with the `delta` cargo feature")
43+
44+
test("native Delta scan reads a plain non-partitioned table") {
45+
assumeDeltaFeature()
46+
withSQLConf(
47+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
48+
CometConf.COMET_DELTA_NATIVE_ENABLED.key -> "true") {
49+
withTempPath { dir =>
50+
val path = dir.getCanonicalPath
51+
spark
52+
.range(0, 50)
53+
.selectExpr("id", "cast(id as double) * 1.5 as score")
54+
.write
55+
.format("delta")
56+
.save(path)
57+
58+
val df = spark.read.format("delta").load(path)
59+
df.collect()
60+
val names = stripAQEPlan(df.queryExecution.executedPlan).collect { case p =>
61+
p.getClass.getSimpleName
62+
}
63+
info(s"plan nodes: ${names.mkString(", ")}")
64+
assert(
65+
names.contains("CometDeltaNativeScanExec"),
66+
s"expected a native Delta scan, got: ${names.mkString(", ")}")
67+
checkSparkAnswer(df)
68+
}
69+
}
70+
}
71+
72+
test("native Delta scan falls back to Spark when disabled") {
73+
assumeDeltaFeature()
74+
withSQLConf(
75+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
76+
CometConf.COMET_DELTA_NATIVE_ENABLED.key -> "false") {
77+
withTempPath { dir =>
78+
val path = dir.getCanonicalPath
79+
spark.range(0, 10).toDF("id").write.format("delta").save(path)
80+
val df = spark.read.format("delta").load(path)
81+
df.collect()
82+
val names = stripAQEPlan(df.queryExecution.executedPlan).collect { case p =>
83+
p.getClass.getSimpleName
84+
}
85+
assert(!names.contains("CometDeltaNativeScanExec"))
86+
checkSparkAnswer(df)
87+
}
88+
}
89+
}
90+
}

0 commit comments

Comments
 (0)