Skip to content

Commit dc35e97

Browse files
adivaishCopilot
andcommitted
feat(delta): partitioned-table support for native Delta scan
Reads partitioned Delta tables natively: the projection now uses the scan's full output schema (data + partition columns, in output order), and delta-kernel-rs injects partition values during the scan. tryDeltaNativeScan no longer bails on partitioned tables, but stays correct by bailing when the scan carries partition filters (Spark consumes those during file pruning with no Filter node above, so a full kernel read would return pruned-away rows) or data filters. CometDeltaNativeReadSuite gains two tests: a full read of a table partitioned by an int and a string column (values match Spark via checkSparkAnswer), and a partition-filter query that must fall back to Spark. Validated end to end on Spark 4.1 + Delta 4.1.0 with a delta-feature native build (5/5 tests pass). Part of apache#174. Co-authored-by: Copilot <[email protected]>
1 parent c03ccf7 commit dc35e97

2 files changed

Lines changed: 77 additions & 7 deletions

File tree

‎spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -197,10 +197,12 @@ case class CometScanRule(session: SparkSession)
197197
/**
198198
* Attempt to read a Delta scan natively via delta-kernel-rs. Returns `Some` when this scan is
199199
* an eligible Delta scan and `spark.comet.scan.deltaNative.enabled` is on. v1 supports column
200-
* projection (the kernel reads only the required columns, in order) but is otherwise
201-
* conservative: non-partitioned tables with no data filters, so the kernel's read matches
202-
* Spark's output exactly. Everything else returns `None` and falls through to the normal scan
203-
* handling.
200+
* projection (the kernel reads only the output columns, in order) and partitioned tables (the
201+
* kernel injects partition values). It is otherwise conservative: it bails when the scan
202+
* carries partition filters (Spark consumes those during file pruning with no Filter node
203+
* above, so a full kernel read would return pruned-away rows) or data filters (kept on Spark to
204+
* preserve Parquet row-group skipping until kernel predicate pushdown lands). Everything else
205+
* returns `None` and falls through to the normal scan handling.
204206
*/
205207
private def tryDeltaNativeScan(
206208
scanExec: FileSourceScanExec,
@@ -212,13 +214,16 @@ case class CometScanRule(session: SparkSession)
212214
return None
213215
}
214216
val rootPaths = r.location.rootPaths
215-
if (r.partitionSchema.nonEmpty || scanExec.dataFilters.nonEmpty || rootPaths.length != 1) {
217+
if (scanExec.partitionFilters.nonEmpty || scanExec.dataFilters.nonEmpty ||
218+
rootPaths.length != 1) {
216219
return None
217220
}
218221
val tableUri = rootPaths.head.toUri.toString
219-
val requiredSchema = schema2Proto(scanExec.requiredSchema.fields)
222+
// Project by the scan's full output (data + partition columns, in output order) so the kernel
223+
// selects exactly those columns and injects partition values.
224+
val outputSchema = schema2Proto(scanExec.schema.fields)
220225
val deltaScanBuilder = OperatorOuterClass.DeltaScan.newBuilder().setTableUri(tableUri)
221-
requiredSchema.foreach(deltaScanBuilder.addRequiredSchema)
226+
outputSchema.foreach(deltaScanBuilder.addRequiredSchema)
222227
val op =
223228
OperatorOuterClass.Operator.newBuilder().setDeltaScan(deltaScanBuilder.build()).build()
224229
Some(CometDeltaNativeScanExec(op, scanExec.output, tableUri, scanExec, SerializedPlan(None)))

‎spark/src/test/scala/org/apache/comet/CometDeltaNativeReadSuite.scala‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,71 @@ class CometDeltaNativeReadSuite extends CometTestBase {
9898
}
9999
}
100100

101+
test("native Delta scan reads a partitioned table") {
102+
assumeDeltaFeature()
103+
withSQLConf(
104+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
105+
CometConf.COMET_DELTA_NATIVE_ENABLED.key -> "true") {
106+
withTempPath { dir =>
107+
val path = dir.getCanonicalPath
108+
spark
109+
.range(0, 60)
110+
.selectExpr(
111+
"id",
112+
"cast(id as double) * 1.5 as score",
113+
"cast(id % 3 as int) as part_i",
114+
"concat('p', cast(id % 2 as string)) as part_s")
115+
.write
116+
.format("delta")
117+
.partitionBy("part_i", "part_s")
118+
.save(path)
119+
120+
// Full read of all partitions (no partition filter): partition values are injected by the
121+
// kernel and must match Spark, including the partition columns.
122+
val df = spark.read.format("delta").load(path)
123+
df.collect()
124+
val names = stripAQEPlan(df.queryExecution.executedPlan).collect { case p =>
125+
p.getClass.getSimpleName
126+
}
127+
info(s"plan nodes: ${names.mkString(", ")}")
128+
assert(
129+
names.contains("CometDeltaNativeScanExec"),
130+
s"expected a native Delta scan, got: ${names.mkString(", ")}")
131+
checkSparkAnswer(df)
132+
}
133+
}
134+
}
135+
136+
test("native Delta scan falls back to Spark for a partition filter") {
137+
assumeDeltaFeature()
138+
withSQLConf(
139+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
140+
CometConf.COMET_DELTA_NATIVE_ENABLED.key -> "true") {
141+
withTempPath { dir =>
142+
val path = dir.getCanonicalPath
143+
spark
144+
.range(0, 30)
145+
.selectExpr("id", "cast(id % 3 as int) as part_i")
146+
.write
147+
.format("delta")
148+
.partitionBy("part_i")
149+
.save(path)
150+
151+
// A partition predicate is consumed by file pruning with no Filter node above, so the
152+
// native full-read path must not fire (it would return pruned-away rows).
153+
val df = spark.read.format("delta").load(path).where("part_i = 1")
154+
df.collect()
155+
val names = stripAQEPlan(df.queryExecution.executedPlan).collect { case p =>
156+
p.getClass.getSimpleName
157+
}
158+
assert(
159+
!names.contains("CometDeltaNativeScanExec"),
160+
s"native Delta scan must not fire with a partition filter, got: ${names.mkString(", ")}")
161+
checkSparkAnswer(df)
162+
}
163+
}
164+
}
165+
101166
test("native Delta scan falls back to Spark when disabled") {
102167
assumeDeltaFeature()
103168
withSQLConf(

0 commit comments

Comments
 (0)