Skip to content

Commit 18ea6d8

Browse files
schenksjclaude
andcommitted
refactor(delta): drop the column-mapping physicalisation + its dead plumbing (apache#76)
Now that the kernel-read path ships kernel's own `scan.physical_schema()` / `scan.logical_schema()` (physical names + field-ids resolved at every nesting level), the Comet-side column-mapping reimplementation and everything that fed it are dead. Removed: - `physicalise_field` / `physicalise_type` / `descend_elem` in core's `delta_scan.rs` (the recursive logical->physical rename). The kernel-read planner now requires the shipped kernel schemas for any data-column read (zero-data reads need none); a missing schema is a driver bug, surfaced as a clear error rather than papered over by re-deriving it (which got nested column mapping wrong). - `DeltaScanCommon.column_mappings` (proto field 13) and `DeltaColumnMapping`'s recursive `children` -- the executor's only consumer was `physicalise_field`. - `DeltaScanTaskList.column_mappings` (proto field 5) and the whole `DeltaColumnMapping` proto message: the only remaining use is the DRIVER-side partition-value key translation, which keeps a plain `(logical, physical)` list internal to `plan_delta_scan` -- nothing goes over the wire. - The JVM re-derivation of a column-mapping tree for the add-files path (`CometDeltaNativeScan`) and the two `addAllColumnMappings` relays. All contrib Delta suites stay green (CometDeltaNativeSuite 19/19, ColumnMapping 5/5, Coverage 24/24, NestedArrayStruct 4/4, TypeRoundTrip 10/0). Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent d42a9dc commit 18ea6d8

7 files changed

Lines changed: 82 additions & 264 deletions

File tree

‎contrib/delta/native/src/jni.rs‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ pub unsafe extern "system" fn Java_org_apache_comet_contrib_delta_Native_planDel
155155
let physical_to_logical: std::collections::HashMap<String, String> = plan
156156
.column_mappings
157157
.iter()
158-
.map(|cm| (cm.physical_name.clone(), cm.logical_name.clone()))
158+
.map(|(logical, physical)| (physical.clone(), logical.clone()))
159159
.collect();
160160

161161
let tasks: Vec<DeltaScanTask> = plan
@@ -206,15 +206,15 @@ pub unsafe extern "system" fn Java_org_apache_comet_contrib_delta_Native_planDel
206206
})
207207
.collect();
208208

209-
// Already the recursive proto form (built from the kernel schema in `scan.rs`).
210-
let column_mappings = plan.column_mappings;
209+
// `plan.column_mappings` is consumed above (partition-value key translation) and is NOT put
210+
// on the wire: column mapping is fully resolved by the kernel schemas shipped on
211+
// `DeltaScanCommon`, so the executor needs no `column_mappings`.
211212

212213
let msg = DeltaScanTaskList {
213214
snapshot_version: plan.version,
214215
table_root: url_str,
215216
tasks,
216217
unsupported_features: plan.unsupported_features,
217-
column_mappings,
218218
// Kernel-built projected schemas (Arrow IPC) -- Scala copies these onto
219219
// `DeltaScanCommon.kernel_physical_schema` / `kernel_logical_schema`. Empty when no
220220
// projection was supplied.
@@ -280,7 +280,6 @@ pub unsafe extern "system" fn Java_org_apache_comet_contrib_delta_Native_planDel
280280
table_root: url_str,
281281
tasks: Vec::new(),
282282
unsupported_features: Vec::new(),
283-
column_mappings: Vec::new(),
284283
physical_schema,
285284
logical_schema,
286285
};

‎contrib/delta/native/src/lib.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ pub mod synthetic_columns;
5151
/// access to the typed variants).
5252
pub mod proto {
5353
pub use datafusion_comet_proto::spark_operator::{
54-
DeltaColumnMapping, DeltaDvDescriptor, DeltaPartitionValue, DeltaScan, DeltaScanCommon,
55-
DeltaScanTask, DeltaScanTaskList,
54+
DeltaDvDescriptor, DeltaPartitionValue, DeltaScan, DeltaScanCommon, DeltaScanTask,
55+
DeltaScanTaskList,
5656
};
5757
}
5858

‎contrib/delta/native/src/scan.rs‎

Lines changed: 19 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,11 @@ pub struct DeltaScanPlan {
8888
pub entries: Vec<DeltaFileEntry>,
8989
pub version: u64,
9090
pub unsupported_features: Vec<String>,
91-
/// Logical→physical column name mapping for column-mapped tables, recursive (struct fields
92-
/// carry nested `children`). Empty when column_mapping_mode is None.
93-
pub column_mappings: Vec<crate::proto::DeltaColumnMapping>,
91+
/// Top-level `(logical_name, physical_name)` pairs for column-mapped tables. Driver-internal
92+
/// only -- used to translate kernel's physical-keyed partition values back to logical names.
93+
/// Empty when column_mapping_mode is None. (Nested column mapping is handled by the kernel-built
94+
/// schemas, not this list.)
95+
pub column_mappings: Vec<(String, String)>,
9496
/// Kernel-built data-column schemas (Arrow IPC schema messages) for the kernel-read path,
9597
/// produced from the projected `Scan` (`scan.physical_schema()` / `scan.logical_schema()`).
9698
/// The physical schema carries correct physical names + field-ids at every nesting level
@@ -172,54 +174,27 @@ pub fn plan_delta_read_schemas(
172174
scan_schemas_to_ipc(&scan)
173175
}
174176

175-
/// Build the recursive Delta column-mapping tree for a struct: one entry per field that is renamed
176-
/// (`delta.columnMapping.physicalName`) or contains nested renamed fields. Arrays/maps are descended
177-
/// through transparently (Delta column-maps only struct fields). Drives the native planner's
178-
/// `physicalise_field`.
179-
fn build_struct_column_mappings(
180-
st: &delta_kernel::schema::StructType,
181-
) -> Vec<crate::proto::DeltaColumnMapping> {
177+
/// Build the top-level Delta column mappings (`logical_name` -> `physical_name`) for a column-mapped
178+
/// table. Used driver-side only, to translate kernel's physical-keyed partition values back to
179+
/// logical names (partition columns are top-level). Nested column mapping no longer needs a recursive
180+
/// tree here -- the executor reads with kernel's own `scan.physical_schema()` / `logical_schema()`
181+
/// (shipped on `DeltaScanCommon`), which resolve physical names + field-ids at every nesting level.
182+
fn build_struct_column_mappings(st: &delta_kernel::schema::StructType) -> Vec<(String, String)> {
182183
use delta_kernel::schema::{ColumnMetadataKey, MetadataValue};
183184
st.fields()
184185
.filter_map(|field| {
185186
let physical = match field
186187
.metadata
187188
.get(ColumnMetadataKey::ColumnMappingPhysicalName.as_ref())
188189
{
189-
Some(MetadataValue::String(p)) => Some(p.clone()),
190-
_ => None,
190+
Some(MetadataValue::String(p)) => p.clone(),
191+
_ => return None,
191192
};
192-
let children = build_type_column_mappings(&field.data_type);
193-
if physical.is_none() && children.is_empty() {
194-
return None;
195-
}
196-
Some(crate::proto::DeltaColumnMapping {
197-
logical_name: field.name().clone(),
198-
physical_name: physical.unwrap_or_default(),
199-
children,
200-
})
193+
Some((field.name().clone(), physical))
201194
})
202195
.collect()
203196
}
204197

205-
/// Descend a data type for nested column mappings: structs yield their field mappings; arrays/maps
206-
/// descend into their element / key+value types transparently.
207-
fn build_type_column_mappings(
208-
dt: &delta_kernel::schema::DataType,
209-
) -> Vec<crate::proto::DeltaColumnMapping> {
210-
use delta_kernel::schema::DataType;
211-
match dt {
212-
DataType::Struct(st) => build_struct_column_mappings(st),
213-
DataType::Array(at) => build_type_column_mappings(at.element_type()),
214-
DataType::Map(mt) => {
215-
let mut v = build_type_column_mappings(mt.key_type());
216-
v.extend(build_type_column_mappings(mt.value_type()));
217-
v
218-
}
219-
_ => Vec::new(),
220-
}
221-
}
222-
223198
/// List every active parquet file in a Delta table at the given version.
224199
///
225200
/// Returns `(entries, actual_version)` where `actual_version` is the
@@ -314,12 +289,11 @@ pub fn plan_delta_scan_with_predicate(
314289
// For column_mapping_mode = id or name, each StructField carries a
315290
// `delta.columnMapping.physicalName` metadata entry that tells us what the
316291
// parquet file's column name actually is.
317-
let column_mappings: Vec<crate::proto::DeltaColumnMapping> =
318-
if props.column_mapping_mode.is_some() {
319-
build_struct_column_mappings(snapshot.schema().as_ref())
320-
} else {
321-
Vec::new()
322-
};
292+
let column_mappings: Vec<(String, String)> = if props.column_mapping_mode.is_some() {
293+
build_struct_column_mappings(snapshot.schema().as_ref())
294+
} else {
295+
Vec::new()
296+
};
323297

324298
// `Snapshot::build()` returns `Arc<Snapshot>`, and `scan_builder` consumes
325299
// it. Clone the Arc so we keep a stable handle through scan construction

‎contrib/delta/src/main/scala/org/apache/comet/contrib/delta/CometDeltaNativeScan.scala‎

Lines changed: 22 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -605,70 +605,12 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
605605
return None
606606
}
607607
}
608-
val taskList0 = DeltaScanTaskList.parseFrom(taskListBytes)
609-
// The kernel path populates `column_mappings` from kernel's schema metadata.
610-
// The pre-materialised-index path (`buildTaskListFromAddFiles`) doesn't have
611-
// that information yet, so re-derive the mapping from the relation's data
612-
// + partition schema -- each StructField carries
613-
// `delta.columnMapping.physicalName` in its metadata when the table uses
614-
// column mapping. Without this the native scan can't translate logical
615-
// column references to physical parquet column names and returns nulls.
616-
// Fetch the schema used to resolve column-mapping physical names / field-ids. Prefer the
617-
// ANALYSIS-TIME schema that DeltaScanRule stashed from the original
618-
// DeltaParquetFileFormat.referenceSchema (relation.options): the FileIndex / live snapshot
619-
// may have moved on (RENAME COLUMN, overwriteSchema) since the query was analyzed, and the
620-
// query must read against the schema it was planned with -- otherwise a column whose
621-
// physical name / field-id changed reads the new data instead of NULL (DeltaColumnMapping
622-
// Suite "physical name changes" / "explicit id matching"). Fall back to the live snapshot
623-
// schema when the option is absent (e.g. the Delta-FileIndex-over-plain-ParquetFileFormat
624-
// shape, where referenceSchema was never available).
625-
val snapshotSchemaEarly: Option[StructType] =
626-
scan.deltaMetadata.analyzedSchema.orElse(DeltaReflection.extractSnapshotSchema(relation))
627-
// Only honour physicalName metadata when the table actually has column mapping
628-
// mode enabled. Some Delta test helpers (e.g. `DeltaSourceSuiteBase.withMetadata`)
629-
// call `DeltaColumnMapping.assignColumnIdAndPhysicalName` unconditionally, which
630-
// attaches `delta.columnMapping.physicalName` to every StructField even when the
631-
// table's `delta.columnMapping.mode` is unset / `none`. In that case the writer
632-
// still uses LOGICAL names in the parquet file, so physicalising our scan would
633-
// look up non-existent physical column names and return empty rows.
634-
val tableColumnMappingMode = DeltaReflection
635-
.extractMetadataConfiguration(relation)
636-
.flatMap(_.get("delta.columnMapping.mode"))
637-
.filter(m => m != null && !m.equalsIgnoreCase("none"))
638-
val taskList =
639-
if (!taskList0.getColumnMappingsList.isEmpty || tableColumnMappingMode.isEmpty) {
640-
taskList0
641-
} else {
642-
// `relation.dataSchema.fields[*].metadata` is stripped of Delta's column-mapping
643-
// metadata by HadoopFsRelation, so the lookup here nearly always returns empty.
644-
// Use the Snapshot schema we extracted (which preserves physical names at every
645-
// level) for the data-column mappings, and `relation.partitionSchema` only for
646-
// partition columns (whose metadata isn't stripped).
647-
val dataFieldsSource: Array[StructField] =
648-
snapshotSchemaEarly.map(_.fields).getOrElse(relation.dataSchema.fields)
649-
val allFields = dataFieldsSource ++ relation.partitionSchema.fields
650-
val logicalToPhysical = allFields.flatMap { f =>
651-
if (f.metadata.contains(DeltaReflection.PhysicalNameMetadataKey)) {
652-
Some(f.name -> f.metadata.getString(DeltaReflection.PhysicalNameMetadataKey))
653-
} else {
654-
None
655-
}
656-
}
657-
if (logicalToPhysical.isEmpty) {
658-
taskList0
659-
} else {
660-
val b = DeltaScanTaskList.newBuilder(taskList0)
661-
logicalToPhysical.foreach { case (logical, physical) =>
662-
b.addColumnMappings(
663-
OperatorOuterClass.DeltaColumnMapping
664-
.newBuilder()
665-
.setLogicalName(logical)
666-
.setPhysicalName(physical)
667-
.build())
668-
}
669-
b.build()
670-
}
671-
}
608+
val taskList = DeltaScanTaskList.parseFrom(taskListBytes)
609+
// Column mapping no longer needs any executor-side plumbing: the kernel-read path ships kernel's
610+
// own `scan.physical_schema()` / `logical_schema()` (physical names + field-ids resolved at every
611+
// nesting level), and kernel returns partition values already translated to logical names
612+
// (driver-side, in `planDeltaScan` / `buildTaskListFromAddFiles`). The former re-derivation of a
613+
// `column_mappings` tree here existed only to feed the removed `physicalise_field` schema rebuild.
672614

673615
// Phase 6 reader-feature gate. Kernel reports any Delta reader features that
674616
// are currently in use in this snapshot and that Comet's native path does NOT
@@ -881,29 +823,31 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
881823
commonBuilder.addAllPartitionSchema(partitionSchema.toIterable.asJava)
882824
// Kernel-built projected schemas (`scan.physical_schema()` / `scan.logical_schema()`, Arrow
883825
// IPC) -- correct physical names + field-ids at EVERY nesting level. The executor's kernel-read
884-
// planner uses them verbatim instead of physicalising `required_schema`. The kernel-driver
885-
// `planDeltaScan` path returns them inline; the batch-file-index path (file list from AddFiles)
886-
// fetches them via the schema-only `planDeltaReadSchemas`. Empty => executor falls back to the
887-
// `column_mappings` physicalisation (top-level only -- wrong for nested column mapping).
826+
// planner uses them verbatim. The kernel-driver `planDeltaScan` path returns them inline; the
827+
// batch-file-index path (file list from AddFiles) fetches them via the schema-only
828+
// `planDeltaReadSchemas`. (For a read with zero data columns there are none, and none are
829+
// needed -- the executor drives the row count without a parquet read.)
888830
val kernelSchemaSource: DeltaScanTaskList =
889-
if (!taskList0.getPhysicalSchema.isEmpty) {
890-
taskList0
831+
if (!taskList.getPhysicalSchema.isEmpty) {
832+
taskList
891833
} else {
892834
val projIpc =
893835
CometDeltaNativeScan.dataReadSchemaIpc(scan.requiredSchema, scan.conf.sessionLocalTimeZone)
894836
if (projIpc.isEmpty) {
895-
taskList0
837+
taskList
896838
} else {
897839
try {
898840
DeltaScanTaskList.parseFrom(
899841
nativeLib.planDeltaReadSchemas(tableRoot, snapshotVersion, storageOptions, projIpc))
900842
} catch {
901843
case scala.util.control.NonFatal(e) =>
902-
logWarning(
903-
s"CometDeltaNativeScan: kernel read-schema build failed for $tableRoot; " +
904-
s"falling back to column-mapping physicalisation",
905-
e)
906-
taskList0
844+
// The kernel-read path has no Comet-side physicalisation fallback; if kernel can't
845+
// build the read schemas, decline to native and let Spark's reader handle it.
846+
import org.apache.comet.CometSparkSessionExtensions.withFallbackReason
847+
withFallbackReason(
848+
scan,
849+
s"Native Delta scan could not build kernel read schemas for $tableRoot: $e")
850+
return None
907851
}
908852
}
909853
}
@@ -950,10 +894,8 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
950894
commonBuilder.putObjectStoreOptions(key, value)
951895
}
952896

953-
// Phase 4: pass column mapping from kernel through to the native planner. Relayed as-is so the
954-
// recursive `children` (nested struct-field mappings, #47) are preserved; the native planner
955-
// physicalises required_schema at every nesting level from this tree.
956-
commonBuilder.addAllColumnMappings(taskList.getColumnMappingsList)
897+
// (Column mapping is fully resolved by kernel: the executor reads with the shipped
898+
// `kernel_physical_schema` / `kernel_logical_schema`, so no `column_mappings` tree is sent.)
957899

958900
// --- 3. Pack into a DeltaScan with COMMON ONLY (split-mode, Phase 5).
959901
// Tasks are NOT included in the proto at planning time. They'll be
@@ -978,7 +920,6 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
978920
.setSnapshotVersion(taskList.getSnapshotVersion)
979921
.setTableRoot(taskList.getTableRoot)
980922
.addAllTasks(filteredTasks.asJava)
981-
.addAllColumnMappings(taskList.getColumnMappingsList)
982923
.addAllUnsupportedFeatures(taskList.getUnsupportedFeaturesList)
983924
.build()
984925
lastTaskListBytes.set(filteredTaskList.toByteArray)

‎contrib/delta/src/main/scala/org/apache/spark/sql/comet/CometDeltaNativeScanExec.scala‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -567,8 +567,7 @@ object CometDeltaNativeScanExec {
567567
val components = Seq(
568568
common.getTableRoot,
569569
common.getSnapshotVersion.toString,
570-
common.getRequiredSchemaList.toString,
571-
common.getColumnMappingsList.toString)
570+
common.getRequiredSchemaList.toString)
572571
s"${common.getSource}_${components.mkString("|").hashCode}"
573572
}
574573
}

0 commit comments

Comments
 (0)