Skip to content

Commit 29361d6

Browse files
schenksjclaude
andcommitted
fix: distinguish CometDeltaNativeScan instances across snapshot versions
CometDeltaNativeScanExec's equality and findAllPlanData key were both tableRoot-only, so two scans of the same Delta path at different time-travel versions (e.g. v0 and v1 in a self-join) collided: - findAllPlanData's `.toMap` silently dropped one of the two entries, leaving only one scan's file list available to the injector - Spark's ReuseExchangeAndSubquery rule considered the two exchanges equal via the scan's equals/hashCode, replacing v1's exchange with ReusedExchange of v0's — so both sides of a full-outer join on the same key read v0's data and "unmatched" rows (keys 5..9) vanished Introduce `CometDeltaNativeScanExec.computeSourceKey(op)` derived from the DeltaScanCommon proto (table root, snapshot version, schemas, filters, projection, column mappings) — mirrors CometNativeScanExec's sourceKey pattern. Use it: - as the key in commonByKey / perPartitionByKey maps - as the key in findAllPlanData results - as the lookup key in DeltaPlanDataInjector.getKey - in equals/hashCode so two scans at different versions are not equal Surfaced by running Delta's own DeltaTimeTravelSuite under the Comet regression diff: `scans on different versions of same table are executed correctly` was producing 0 rows where `a.key IS NULL` (should be 5). All 24 tests in that suite now pass. Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
1 parent 3e4b6a0 commit 29361d6

2 files changed

Lines changed: 51 additions & 8 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/CometDeltaNativeScanExec.scala‎

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,18 @@ case class CometDeltaNativeScanExec(
144144
def commonData: Array[Byte] = serializedPartitionData._1
145145
def perPartitionData: Array[Array[Byte]] = serializedPartitionData._2
146146

147+
/**
148+
* Unique key for matching this scan's common/per-partition data to its operator in the native
149+
* plan. Must be distinct across multiple Delta scans in the same plan tree -- e.g. a self-join
150+
* reading two snapshot versions of the same table, where `tableRoot` alone is not unique.
151+
*
152+
* Derived identically in `DeltaPlanDataInjector.getKey` from the serialized `DeltaScanCommon`
153+
* proto so the driver-side map and the executor-side lookup agree.
154+
*
155+
* Mirrors the pattern used by `CometNativeScanExec.sourceKey`.
156+
*/
157+
def sourceKey: String = CometDeltaNativeScanExec.computeSourceKey(nativeOp)
158+
147159
def numPartitions: Int = perPartitionData.length
148160

149161
override lazy val outputPartitioning: Partitioning =
@@ -191,8 +203,8 @@ case class CometDeltaNativeScanExec(
191203
CometExecRDD(
192204
sparkContext,
193205
inputRDDs = Seq.empty,
194-
commonByKey = Map(tableRoot -> commonData),
195-
perPartitionByKey = Map(tableRoot -> perPartitionData),
206+
commonByKey = Map(sourceKey -> commonData),
207+
perPartitionByKey = Map(sourceKey -> perPartitionData),
196208
serializedPlan = serializedPlan,
197209
numPartitions = perPartitionData.length,
198210
numOutputCols = output.length,
@@ -244,12 +256,41 @@ case class CometDeltaNativeScanExec(
244256

245257
override def equals(obj: Any): Boolean = obj match {
246258
case other: CometDeltaNativeScanExec =>
259+
// Include `sourceKey` so two scans of the same table at different snapshot versions
260+
// are NOT considered equal. Without this, Spark's ReuseExchangeAndSubquery rule
261+
// collapses a self-join across versions into a single exchange and reuses v0's
262+
// shuffle output for both sides of the join.
247263
tableRoot == other.tableRoot &&
248264
output == other.output &&
249-
serializedPlanOpt == other.serializedPlanOpt
265+
serializedPlanOpt == other.serializedPlanOpt &&
266+
sourceKey == other.sourceKey
250267
case _ => false
251268
}
252269

253270
override def hashCode(): Int =
254-
Objects.hashCode(tableRoot, output.asJava, serializedPlanOpt)
271+
Objects.hashCode(tableRoot, output.asJava, serializedPlanOpt, sourceKey)
272+
}
273+
274+
object CometDeltaNativeScanExec {
275+
276+
/**
277+
* Compute a stable, per-scan unique key from a `DeltaScan` operator proto. Must be
278+
* deterministic and identical between the driver side (`CometDeltaNativeScanExec.sourceKey`)
279+
* and the injector side (`DeltaPlanDataInjector.getKey`).
280+
*
281+
* Includes `snapshot_version` so that two scans of the same table at different time-travel
282+
* versions produce distinct keys -- otherwise `findAllPlanData` collapses their per-partition
283+
* data into a single map entry and one scan inherits the other's file list.
284+
*/
285+
def computeSourceKey(nativeOp: Operator): String = {
286+
val common = nativeOp.getDeltaScan.getCommon
287+
val components = Seq(
288+
common.getTableRoot,
289+
common.getSnapshotVersion.toString,
290+
common.getRequiredSchemaList.toString,
291+
common.getDataFiltersList.toString,
292+
common.getProjectionVectorList.toString,
293+
common.getColumnMappingsList.toString)
294+
s"${common.getSource}_${components.mkString("|").hashCode}"
295+
}
255296
}

‎spark/src/main/scala/org/apache/spark/sql/comet/operators.scala‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,7 @@ private[comet] object DeltaPlanDataInjector extends PlanDataInjector {
253253
op.getDeltaScan.hasCommon
254254

255255
override def getKey(op: Operator): Option[String] =
256-
Some(op.getDeltaScan.getCommon.getTableRoot)
256+
Some(CometDeltaNativeScanExec.computeSourceKey(op))
257257

258258
override def inject(
259259
op: Operator,
@@ -721,12 +721,14 @@ abstract class CometNativeExec extends CometExec {
721721
Map(nativeScan.sourceKey -> nativeScan.commonData),
722722
Map(nativeScan.sourceKey -> nativeScan.perPartitionData))
723723

724-
// Found a Delta scan with planning data
724+
// Found a Delta scan with planning data. Keyed by `sourceKey` (not `tableRoot`) so
725+
// two scans of the same table at different snapshot versions don't collide when
726+
// `.toMap` below flattens results from multiple children.
725727
case deltaScan: CometDeltaNativeScanExec
726728
if deltaScan.commonData != null && deltaScan.perPartitionData != null =>
727729
(
728-
Map(deltaScan.tableRoot -> deltaScan.commonData),
729-
Map(deltaScan.tableRoot -> deltaScan.perPartitionData))
730+
Map(deltaScan.sourceKey -> deltaScan.commonData),
731+
Map(deltaScan.sourceKey -> deltaScan.perPartitionData))
730732

731733
// Broadcast stages are boundaries - don't collect per-partition data from inside them.
732734
// After DPP filtering, broadcast scans may have different partition counts than the

0 commit comments

Comments
 (0)