@@ -129,6 +129,30 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
129129 lc.startsWith(" _row-id-col-" ) || lc.startsWith(" _row-commit-version-col-" )
130130 }
131131
132+ // Kernel's marker for a metadata column: field metadata `delta.metadataSpec` whose value is the
133+ // spec's text (e.g. "row_id"). Matches `ColumnMetadataKey::MetadataSpec` + `MetadataColumnSpec`
134+ // in delta-kernel-rs; it rides through Spark's `StructType.json` into kernel's `StructType` serde.
135+ private val KernelMetadataSpecKey = " delta.metadataSpec"
136+ private val KernelRowIdSpec = " row_id"
137+
138+ /**
139+ * If `f` is a materialised row-ID column (`_row-id-col-*`), return it re-marked as kernel's RowId
140+ * metadata column so kernel resolves it from `delta.rowTracking.materializedRowIdColumnName` by
141+ * name (no column-mapping id/physicalName needed) and generates the value. Otherwise return `f`
142+ * unchanged. Only invoked under active column mapping (see caller).
143+ */
144+ private def asKernelRowIdMetadataColumnIfMaterialized (f : StructField ): StructField = {
145+ if (f.name.toLowerCase(Locale .ROOT ).startsWith(" _row-id-col-" )) {
146+ val md = new org.apache.spark.sql.types.MetadataBuilder ()
147+ .withMetadata(f.metadata)
148+ .putString(KernelMetadataSpecKey , KernelRowIdSpec )
149+ .build()
150+ StructField (f.name, org.apache.spark.sql.types.LongType , nullable = true , md)
151+ } else {
152+ f
153+ }
154+ }
155+
132156 /**
133157 * Translate Delta's `delta.columnMapping.id` metadata key to Spark+parquet's standard
134158 * `parquet.field.id` key on every StructField at every level of nesting. Required for
@@ -189,16 +213,58 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
189213 */
190214 private [delta] def dataReadSchemaJson (
191215 annotatedSource : Option [StructType ],
192- requiredSchema : StructType ): String = {
216+ requiredSchema : StructType ,
217+ partitionSchema : StructType = new StructType (),
218+ rowTrackingActive : Boolean = false ): String = {
219+ // `row_id` / `row_commit_version` are SYNTHETIC (kernel doesn't read them; we synthesise from
220+ // baseRowId + row_index) ONLY when row tracking is enabled. With it disabled they are ordinary
221+ // user data columns -- a table may legitimately have a column named `row_id`. The proto
222+ // `required_schema` keeps them in that case (the emit flags are gated on row tracking too), so
223+ // the read schema MUST keep them as well, else the executor sees `required_schema` data columns
224+ // with no kernel schema shipped ("missing kernel data-column schemas"). Mirror the emit gating.
225+ val stripNames =
226+ if (rowTrackingActive) SyntheticReadFieldNames
227+ else
228+ SyntheticReadFieldNames -
229+ DeltaReflection .RowIdColumnName .toLowerCase(Locale .ROOT ) -
230+ DeltaReflection .RowCommitVersionColumnName .toLowerCase(Locale .ROOT )
193231 val dataFields = requiredSchema.fields.filterNot(f =>
194- SyntheticReadFieldNames .contains(f.name.toLowerCase(Locale .ROOT )))
232+ stripNames .contains(f.name.toLowerCase(Locale .ROOT )))
195233 if (dataFields.isEmpty) {
234+ // Zero data columns (partition-only / synthetic-only reads): no kernel read schema; the
235+ // executor drives the row count without a parquet read and the partition columns are filled
236+ // separately. (Kernel can't drive a zero-column scan, so we don't project partitions here.)
196237 " "
197238 } else {
198239 val byName =
199240 annotatedSource.map(_.fields.map(f => f.name.toLowerCase(Locale .ROOT ) -> f).toMap)
200241 .getOrElse(Map .empty)
201- val projected = dataFields.map(f => byName.getOrElse(f.name.toLowerCase(Locale .ROOT ), f))
242+ val pick = (f : StructField ) => byName.getOrElse(f.name.toLowerCase(Locale .ROOT ), f)
243+ // Partition columns aren't in `requiredSchema` (Spark hands us the data half), so append them
244+ // when a `partitionSchema` is supplied -- then kernel's per-file transform INJECTS them (the
245+ // max-kernel path) instead of Comet appending them. Sourced from the annotated schema by name
246+ // so column-mapping physical names / field-ids ride along. The AddFiles route passes an empty
247+ // `partitionSchema` (its identity transform can't inject partitions, so partitions stay
248+ // Comet-appended there until that route also moves to kernel enumeration).
249+ val projected0 = dataFields.map(pick) ++ partitionSchema.fields.map(pick)
250+ // Materialised row-id columns (`_row-id-col-*`, added by OPTIMIZE/UPDATE/MERGE) are matched by
251+ // NAME and carry NO column-mapping annotation. Under ACTIVE column mapping kernel's logical
252+ // with_schema requires both physicalName AND id on every regular field, so shipping the
253+ // materialised column as a plain data field fails ("lacks delta.columnMapping.physicalName/id").
254+ // The kernel-intended way is to request the RowId METADATA column: mark the field with kernel's
255+ // `delta.metadataSpec` = `row_id` (ColumnMetadataKey::MetadataSpec). Kernel then reads
256+ // `delta.rowTracking.materializedRowIdColumnName` by name (bypassing CM make_physical), adds a
257+ // row_index helper, and emits `GenerateRowId` (coalesce(materialised, baseRowId+row_index)) on
258+ // the per-file transform -- so row_id comes from kernel, correct even under CM-id. Only needed
259+ // under active CM (detected from a real data field carrying a physicalName); plain tables read
260+ // the materialised column fine as a data field, so leave them untouched. RowCommitVersion has
261+ // no kernel metadata-column support (Error::unsupported), so `_row-commit-version-col-*` is left
262+ // as-is. See state_info.rs RowId handling + CometDeltaRowTrackingMaterializedSuite (M3).
263+ val columnMappingActive =
264+ projected0.exists(_.metadata.contains(DeltaReflection .PhysicalNameMetadataKey ))
265+ val projected =
266+ if (columnMappingActive) projected0.map(asKernelRowIdMetadataColumnIfMaterialized)
267+ else projected0
202268 StructType (projected).json
203269 }
204270 }
@@ -525,17 +591,23 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
525591 // --- 1. Get the active file list. ---
526592 //
527593 // Two code paths:
528- // (a) Pre-materialized FileIndex (`TahoeBatchFileIndex`, `CdcAddFileIndex`):
529- // Delta's streaming micro-batch reads AND MERGE / UPDATE / DELETE
530- // post-join rewrites both carry an exact `addFiles: Seq[AddFile]` on
531- // the FileIndex. Kernel log replay against the snapshot would return a
532- // DIFFERENT file set (the whole snapshot, or a version's deltas), which
533- // is a correctness hazard -- empty streaming batches, MERGE rewrites
534- // that see the whole table instead of only touched files. Build the
535- // DeltaScanTaskList proto directly from those AddFiles, skipping kernel.
536- // (b) Regular scan against a snapshot: call kernel for log replay as before.
594+ // (a) Exact-subset FileIndex (`TahoeBatchFileIndex`, `CdcAddFileIndex`,
595+ // `TahoeRemoveFileIndex`, `TahoeChangeFileIndex`): Delta's streaming
596+ // micro-batch reads AND MERGE / UPDATE / DELETE post-join rewrites carry
597+ // an exact `addFiles: Seq[AddFile]` on the FileIndex. Kernel log replay
598+ // against the snapshot would return a DIFFERENT file set (the whole
599+ // snapshot, or a version's deltas), which is a correctness hazard --
600+ // empty streaming batches, MERGE rewrites that see the whole table
601+ // instead of only touched files. Build the DeltaScanTaskList proto
602+ // directly from those AddFiles, skipping kernel.
603+ // (b) Regular scan against a snapshot (`PreparedDeltaFileIndex` /
604+ // `TahoeLogFileIndex` -- the vast majority): call kernel for log replay.
605+ // Kernel reproduces the pruned active file set from the pinned snapshot +
606+ // the shipped data predicate, and ships its OWN per-file transform so
607+ // partition injection / column-mapping relabel / row-tracking come from
608+ // kernel rather than Comet-side reconstruction.
537609 val taskListBytes =
538- if (DeltaReflection .isBatchFileIndex (relation.location)) {
610+ if (DeltaReflection .isSubsetFileIndex (relation.location)) {
539611 // Pass BOTH the scan's partition filters AND data filters through
540612 // so `refreshedSnapshotFiles` (which queries
541613 // `snapshot.filesForScan(filters, ...)`) re-applies the same
@@ -595,22 +667,25 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
595667 return None
596668 }
597669 } else {
598- // Non-batch indexes (TahoeLogFileIndex, ...). DV-bearing
599- // PreparedDeltaFileIndex is now classified as a batch index above
600- // (see `isBatchFileIndex`), so its DV-fallback case is already
601- // handled by the `case Some(_)` arm at the top of this match. For
602- // remaining non-batch indexes the Delta-PreprocessTableWithDVs
603- // wrapper detection upstream in `CometScanRule.scanBelowFallsBackForDvs`
604- // is responsible for keeping DV-aware internal reads on vanilla.
670+ // Regular reads (`PreparedDeltaFileIndex` / `TahoeLogFileIndex`): kernel enumerates the
671+ // pinned snapshot (reproducing Delta's pruned active file set via the shipped data predicate)
672+ // and ships its OWN per-file transform, so partition injection / column-mapping relabel /
673+ // row-tracking come from kernel. DV-aware INTERNAL reads (Delta-PreprocessTableWithDVs with
674+ // inverted row-index-filter semantics) are kept on vanilla upstream by
675+ // `CometScanRule.scanBelowFallsBackForDvs`.
605676 try {
606677 // The driver's `with_schema` gets the ANALYSIS-TIME read schema (Delta JSON, carrying
607678 // column-mapping physicalName/id) so kernel resolves the names the query was planned with
608679 // -> correct under schema-change-since-analysis. Fall back to the live snapshot schema when
609- // there's no stashed reference schema; both carry the annotations kernel needs.
680+ // there's no stashed reference schema; both carry the annotations kernel needs. Include the
681+ // partition schema so kernel's transform INJECTS partition columns (max-kernel) rather than
682+ // Comet appending them.
610683 val projJson = CometDeltaNativeScan .dataReadSchemaJson(
611684 scan.deltaMetadata.analyzedSchema.orElse(
612685 DeltaReflection .extractSnapshotSchema(relation)),
613- scan.requiredSchema)
686+ scan.requiredSchema,
687+ relation.partitionSchema,
688+ rowTrackingActive = rowTrackingEnabled)
614689 nativeLib.planDeltaScan(
615690 tableRoot,
616691 snapshotVersion,
@@ -856,7 +931,8 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
856931 // zero data columns => no kernel schemas needed.
857932 val projJson = CometDeltaNativeScan .dataReadSchemaJson(
858933 scan.deltaMetadata.analyzedSchema.orElse(DeltaReflection .extractSnapshotSchema(relation)),
859- scan.requiredSchema)
934+ scan.requiredSchema,
935+ rowTrackingActive = rowTrackingEnabled)
860936 if (projJson.isEmpty) {
861937 taskList
862938 } else {
@@ -965,7 +1041,25 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
9651041 */
9661042 private def serializeSupportedDataFilters (scan : CometDeltaScanMarker ): Array [Byte ] = {
9671043 val protoFilters = new ListBuffer [Expr ]()
968- scan.supportedDataFilters.foreach { filter =>
1044+ // Kernel's stats-based file pruning evaluates the predicate against DATA-column statistics during
1045+ // log replay, so it can only reference real data columns. Exclude any filter that touches:
1046+ // - a SYNTHETIC column (`__delta_internal_is_row_deleted`, `_tmp_metadata_row_index`,
1047+ // `_metadata.*`, ...): not a table column at all -- kernel errors "Predicate references
1048+ // unknown column". These are Spark-level filters applied ABOVE the scan, never file-pruning
1049+ // predicates.
1050+ // - a PARTITION column: partition pruning is done separately in `prunePartitions`; pushing a
1051+ // partition predicate into the data-stats predicate also hits kernel's stricter type checks
1052+ // (e.g. a generated partition column compared against a literal -> "Timestamp < Int64").
1053+ // Dropping an unpushable filter only forgoes data skipping for it (Spark still applies it); it
1054+ // never affects correctness.
1055+ val partitionNamesLc =
1056+ scan.relation.partitionSchema.fields.map(_.name.toLowerCase(Locale .ROOT )).toSet
1057+ def kernelPushable (filter : org.apache.spark.sql.catalyst.expressions.Expression ): Boolean =
1058+ filter.references.forall { a =>
1059+ val lc = a.name.toLowerCase(Locale .ROOT )
1060+ ! SyntheticReadFieldNames .contains(lc) && ! partitionNamesLc.contains(lc)
1061+ }
1062+ scan.supportedDataFilters.filter(kernelPushable).foreach { filter =>
9691063 exprToProto(filter, scan.output) match {
9701064 case Some (proto) => protoFilters += proto
9711065 case _ =>
0 commit comments