@@ -605,70 +605,12 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
605605 return None
606606 }
607607 }
608- val taskList0 = DeltaScanTaskList .parseFrom(taskListBytes)
609- // The kernel path populates `column_mappings` from kernel's schema metadata.
610- // The pre-materialised-index path (`buildTaskListFromAddFiles`) doesn't have
611- // that information yet, so re-derive the mapping from the relation's data
612- // + partition schema -- each StructField carries
613- // `delta.columnMapping.physicalName` in its metadata when the table uses
614- // column mapping. Without this the native scan can't translate logical
615- // column references to physical parquet column names and returns nulls.
616- // Fetch the schema used to resolve column-mapping physical names / field-ids. Prefer the
617- // ANALYSIS-TIME schema that DeltaScanRule stashed from the original
618- // DeltaParquetFileFormat.referenceSchema (relation.options): the FileIndex / live snapshot
619- // may have moved on (RENAME COLUMN, overwriteSchema) since the query was analyzed, and the
620- // query must read against the schema it was planned with -- otherwise a column whose
621- // physical name / field-id changed reads the new data instead of NULL (DeltaColumnMapping
622- // Suite "physical name changes" / "explicit id matching"). Fall back to the live snapshot
623- // schema when the option is absent (e.g. the Delta-FileIndex-over-plain-ParquetFileFormat
624- // shape, where referenceSchema was never available).
625- val snapshotSchemaEarly : Option [StructType ] =
626- scan.deltaMetadata.analyzedSchema.orElse(DeltaReflection .extractSnapshotSchema(relation))
627- // Only honour physicalName metadata when the table actually has column mapping
628- // mode enabled. Some Delta test helpers (e.g. `DeltaSourceSuiteBase.withMetadata`)
629- // call `DeltaColumnMapping.assignColumnIdAndPhysicalName` unconditionally, which
630- // attaches `delta.columnMapping.physicalName` to every StructField even when the
631- // table's `delta.columnMapping.mode` is unset / `none`. In that case the writer
632- // still uses LOGICAL names in the parquet file, so physicalising our scan would
633- // look up non-existent physical column names and return empty rows.
634- val tableColumnMappingMode = DeltaReflection
635- .extractMetadataConfiguration(relation)
636- .flatMap(_.get(" delta.columnMapping.mode" ))
637- .filter(m => m != null && ! m.equalsIgnoreCase(" none" ))
638- val taskList =
639- if (! taskList0.getColumnMappingsList.isEmpty || tableColumnMappingMode.isEmpty) {
640- taskList0
641- } else {
642- // `relation.dataSchema.fields[*].metadata` is stripped of Delta's column-mapping
643- // metadata by HadoopFsRelation, so the lookup here nearly always returns empty.
644- // Use the Snapshot schema we extracted (which preserves physical names at every
645- // level) for the data-column mappings, and `relation.partitionSchema` only for
646- // partition columns (whose metadata isn't stripped).
647- val dataFieldsSource : Array [StructField ] =
648- snapshotSchemaEarly.map(_.fields).getOrElse(relation.dataSchema.fields)
649- val allFields = dataFieldsSource ++ relation.partitionSchema.fields
650- val logicalToPhysical = allFields.flatMap { f =>
651- if (f.metadata.contains(DeltaReflection .PhysicalNameMetadataKey )) {
652- Some (f.name -> f.metadata.getString(DeltaReflection .PhysicalNameMetadataKey ))
653- } else {
654- None
655- }
656- }
657- if (logicalToPhysical.isEmpty) {
658- taskList0
659- } else {
660- val b = DeltaScanTaskList .newBuilder(taskList0)
661- logicalToPhysical.foreach { case (logical, physical) =>
662- b.addColumnMappings(
663- OperatorOuterClass .DeltaColumnMapping
664- .newBuilder()
665- .setLogicalName(logical)
666- .setPhysicalName(physical)
667- .build())
668- }
669- b.build()
670- }
671- }
608+ val taskList = DeltaScanTaskList .parseFrom(taskListBytes)
609+ // Column mapping no longer needs any executor-side plumbing: the kernel-read path ships kernel's
610+ // own `scan.physical_schema()` / `logical_schema()` (physical names + field-ids resolved at every
611+ // nesting level), and kernel returns partition values already translated to logical names
612+ // (driver-side, in `planDeltaScan` / `buildTaskListFromAddFiles`). The former re-derivation of a
613+ // `column_mappings` tree here existed only to feed the removed `physicalise_field` schema rebuild.
672614
673615 // Phase 6 reader-feature gate. Kernel reports any Delta reader features that
674616 // are currently in use in this snapshot and that Comet's native path does NOT
@@ -881,29 +823,31 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
881823 commonBuilder.addAllPartitionSchema(partitionSchema.toIterable.asJava)
882824 // Kernel-built projected schemas (`scan.physical_schema()` / `scan.logical_schema()`, Arrow
883825 // IPC) -- correct physical names + field-ids at EVERY nesting level. The executor's kernel-read
884- // planner uses them verbatim instead of physicalising `required_schema` . The kernel-driver
885- // `planDeltaScan` path returns them inline; the batch-file-index path (file list from AddFiles)
886- // fetches them via the schema-only `planDeltaReadSchemas`. Empty => executor falls back to the
887- // `column_mappings` physicalisation (top-level only -- wrong for nested column mapping).
826+ // planner uses them verbatim. The kernel-driver `planDeltaScan` path returns them inline; the
827+ // batch-file-index path (file list from AddFiles) fetches them via the schema-only
828+ // `planDeltaReadSchemas`. (For a read with zero data columns there are none, and none are
829+ // needed -- the executor drives the row count without a parquet read.)
888830 val kernelSchemaSource : DeltaScanTaskList =
889- if (! taskList0 .getPhysicalSchema.isEmpty) {
890- taskList0
831+ if (! taskList .getPhysicalSchema.isEmpty) {
832+ taskList
891833 } else {
892834 val projIpc =
893835 CometDeltaNativeScan .dataReadSchemaIpc(scan.requiredSchema, scan.conf.sessionLocalTimeZone)
894836 if (projIpc.isEmpty) {
895- taskList0
837+ taskList
896838 } else {
897839 try {
898840 DeltaScanTaskList .parseFrom(
899841 nativeLib.planDeltaReadSchemas(tableRoot, snapshotVersion, storageOptions, projIpc))
900842 } catch {
901843 case scala.util.control.NonFatal (e) =>
902- logWarning(
903- s " CometDeltaNativeScan: kernel read-schema build failed for $tableRoot; " +
904- s " falling back to column-mapping physicalisation " ,
905- e)
906- taskList0
844+ // The kernel-read path has no Comet-side physicalisation fallback; if kernel can't
845+ // build the read schemas, decline to native and let Spark's reader handle it.
846+ import org .apache .comet .CometSparkSessionExtensions .withFallbackReason
847+ withFallbackReason(
848+ scan,
849+ s " Native Delta scan could not build kernel read schemas for $tableRoot: $e" )
850+ return None
907851 }
908852 }
909853 }
@@ -950,10 +894,8 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
950894 commonBuilder.putObjectStoreOptions(key, value)
951895 }
952896
953- // Phase 4: pass column mapping from kernel through to the native planner. Relayed as-is so the
954- // recursive `children` (nested struct-field mappings, #47) are preserved; the native planner
955- // physicalises required_schema at every nesting level from this tree.
956- commonBuilder.addAllColumnMappings(taskList.getColumnMappingsList)
897+ // (Column mapping is fully resolved by kernel: the executor reads with the shipped
898+ // `kernel_physical_schema` / `kernel_logical_schema`, so no `column_mappings` tree is sent.)
957899
958900 // --- 3. Pack into a DeltaScan with COMMON ONLY (split-mode, Phase 5).
959901 // Tasks are NOT included in the proto at planning time. They'll be
@@ -978,7 +920,6 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
978920 .setSnapshotVersion(taskList.getSnapshotVersion)
979921 .setTableRoot(taskList.getTableRoot)
980922 .addAllTasks(filteredTasks.asJava)
981- .addAllColumnMappings(taskList.getColumnMappingsList)
982923 .addAllUnsupportedFeatures(taskList.getUnsupportedFeaturesList)
983924 .build()
984925 lastTaskListBytes.set(filteredTaskList.toByteArray)
0 commit comments