@@ -709,11 +709,14 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
709709 // Needs kernel's per-file transform (row_id GenerateRowId + partition injection), which only the
710710 // kernel-enumeration path ships. Regular reads always take it; DML rewrites (TahoeBatchFileIndex)
711711 // get it via kernel-enumerate + path-filter (set below, only if every touched file matched).
712- // Config-gated while validated.
713- val synthesizeConfig : Boolean =
714- DeltaConf .COMET_DELTA_SYNTHESIZE_IN_WORKER_ENABLED .get(scan.conf)
712+ // In-worker synthesis is now the ONLY native synthesis path; the legacy stacked
713+ // DeltaSyntheticColumnsExec is removed (#82). Regular reads always synthesize in-worker; subset
714+ // reads (DML rewrites) do too when every touched file matched kernel enumeration, otherwise they
715+ // decline to vanilla Spark (the `case None` branch below). A read that would have needed the old
716+ // exec therefore either synthesizes in-worker or declines -- it never reaches a non-synthesize
717+ // native path.
715718 var synthesizeInWorker : Boolean =
716- synthesizeConfig && ! DeltaReflection .isSubsetFileIndex(relation.location)
719+ ! DeltaReflection .isSubsetFileIndex(relation.location)
717720
718721 // Column name list for resolving BoundReference indices to kernel column
719722 // names. Must match the order of scan.output because exprToProto binds
@@ -754,24 +757,6 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
754757 relation.location,
755758 scan.partitionFilters ++ scan.dataFilters) match {
756759 case Some (addFiles) =>
757- // Under column mapping, Delta stores partition values in AddFile keyed by the
758- // PHYSICAL column name. `relation.partitionSchema.fields[*].metadata` has had
759- // Delta's columnMapping metadata stripped by HadoopFsRelation, so look in the
760- // authoritative Snapshot schema (via reflection) and restrict to fields that
761- // appear in the relation's partition schema.
762- val partitionNames = relation.partitionSchema.fields.map(_.name).toSet
763- val snapshotFields = DeltaReflection
764- .extractSnapshotSchema(relation)
765- .map(_.fields)
766- .getOrElse(Array .empty[StructField ])
767- val physToLogical = snapshotFields.flatMap { f =>
768- if (partitionNames.contains(f.name) &&
769- f.metadata.contains(DeltaReflection .PhysicalNameMetadataKey )) {
770- Some (f.metadata.getString(DeltaReflection .PhysicalNameMetadataKey ) -> f.name)
771- } else {
772- None
773- }
774- }.toMap
775760 // DV handling: the driver only ships a DV DESCRIPTOR per AddFile
776761 // (storage type / path / offset / size, KB-scale). The executor decodes
777762 // via `dv_reader::read_dv_indexes` on first poll. Pre-#218 we called
@@ -789,7 +774,7 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
789774 // enumeration (safe: the legacy DeltaSyntheticColumnsExec path still works -- never wrong
790775 // files). CDC indexes (files outside the snapshot) are excluded by isDmlRewriteFileIndex.
791776 val dmlSynthBytes : Option [Array [Byte ]] =
792- if (synthesizeConfig && DeltaReflection .isDmlRewriteFileIndex(relation.location)) {
777+ if (DeltaReflection .isDmlRewriteFileIndex(relation.location)) {
793778 try {
794779 val annotated = scan.deltaMetadata.analyzedSchema.orElse(
795780 DeltaReflection .extractSnapshotSchema(relation))
@@ -846,21 +831,19 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
846831 synthesizeInWorker = true
847832 bytes
848833 case None =>
849- // Residual reads kept on the legacy DeltaSyntheticColumnsExec path FOR NOW: the CDC
850- // family (CdcAddFileIndex / TahoeRemove / TahoeChange) and the rare DML declines
851- // (CM-id materialised row_commit_version; OPTIMIZE file-not-found race). NOTE: kernel
852- // DOES support CDC -- via the separate `delta_kernel::table_changes::TableChanges` API
853- // (emits `_change_type` / `_commit_version` / `_commit_timestamp`), which reuses the
854- // same per-file read + transform primitives as the regular scan. Routing CDC through
855- // it (a TableChanges-based read path) is the path to deleting this exec; until that's
856- // wired, CDC stays here.
857- buildTaskListFromAddFiles(
858- tableRoot,
859- snapshotVersion,
860- addFiles,
861- nativeOp = null ,
862- columnNames,
863- physicalToLogicalPartitionNames = physToLogical).toByteArray
834+ // #82: the legacy buildTaskListFromAddFiles + DeltaSyntheticColumnsExec path is
835+ // retired. The only reads that reached it were CDC-family subset indexes (now read
836+ // natively via kernel TableChanges -- CometDeltaCdfScanExec, #84 -- so they never
837+ // become a marker here) and the rare DML declines that can't synthesize in-worker
838+ // (CM-id materialised row_commit_version; OPTIMIZE file-not-found race). Decline the
839+ // latter to vanilla Spark with the same withFallbackReason mechanism the
840+ // reflection-failure branch below uses (proven to cleanly drop the Comet boundary).
841+ import org .apache .comet .CometSparkSessionExtensions .withFallbackReason
842+ withFallbackReason(
843+ scan,
844+ s " Native Delta scan declines a subset-file-index read that cannot synthesize " +
845+ s " in-worker ( ${relation.location.getClass.getName}); falling back to Spark. " )
846+ return None
864847 }
865848 case None =>
866849 // Reflection failed; fall back conservatively.
@@ -1560,67 +1543,6 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
15601543 }
15611544 }
15621545
1563- /**
1564- * Build a kernel-independent `DeltaScanTaskList` from a caller-provided AddFile list. Used when
1565- * the Delta scan has a pre-materialized FileIndex (streaming micro-batch, MERGE/UPDATE/DELETE
1566- * post-join) so we can honour its exact file list instead of re-running log replay (which would
1567- * return a different set).
1568- *
1569- * Each AddFile becomes one `DeltaScanTask`. Absolute path resolution mirrors
1570- * `DeltaFileOperations.absolutePath`: if `AddFile.path` is already absolute (has a URI scheme),
1571- * keep it verbatim; otherwise join against `tableRoot`.
1572- */
1573- private def buildTaskListFromAddFiles (
1574- tableRoot : String ,
1575- snapshotVersion : Long ,
1576- addFiles : Seq [DeltaReflection .ExtractedAddFile ],
1577- nativeOp : AnyRef ,
1578- columnNames : Array [String ],
1579- physicalToLogicalPartitionNames : Map [String , String ] = Map .empty)
1580- : OperatorOuterClass .DeltaScanTaskList = {
1581- val tlBuilder = OperatorOuterClass .DeltaScanTaskList .newBuilder()
1582- tlBuilder.setTableRoot(tableRoot)
1583- if (snapshotVersion >= 0 ) tlBuilder.setSnapshotVersion(snapshotVersion)
1584-
1585- addFiles.foreach { af =>
1586- val absPath =
1587- if (af.path.contains(" :/" )) af.path
1588- else {
1589- val sep = if (tableRoot.endsWith(" /" )) " " else " /"
1590- tableRoot + sep + af.path
1591- }
1592- val taskBuilder = OperatorOuterClass .DeltaScanTask .newBuilder()
1593- taskBuilder.setFilePath(absPath)
1594- taskBuilder.setFileSize(af.size)
1595- DeltaReflection .parseNumRecords(af.statsJson).foreach(taskBuilder.setRecordCount)
1596- af.partitionValues.foreach { case (k, v) =>
1597- // Under column mapping, Delta stores partition values keyed by the
1598- // PHYSICAL column name (e.g. `col-<uuid>-part`). Our partition_schema
1599- // on the wire uses LOGICAL names and the native kernel-read path matches
1600- // by name, so translate when we have a physical->logical map (the
1601- // kernel-path jni.rs performs the same translation for its own extraction).
1602- val logicalName = physicalToLogicalPartitionNames.getOrElse(k, k)
1603- val pvBuilder =
1604- OperatorOuterClass .DeltaPartitionValue .newBuilder().setName(logicalName)
1605- if (v != null ) pvBuilder.setValue(v)
1606- taskBuilder.addPartitionValues(pvBuilder.build())
1607- }
1608- af.baseRowId.foreach(taskBuilder.setBaseRowId)
1609- af.defaultRowCommitVersion.foreach(taskBuilder.setDefaultRowCommitVersion)
1610- af.modificationTime.foreach(taskBuilder.setModificationTime)
1611- // Ship the DV descriptor (KB-scale) instead of the materialised row-index
1612- // list (was up to 1 GB per file -- the #218 dominator). Executor decodes
1613- // via dv_reader::read_dv_indexes on first poll of the DV-bearing partition.
1614- if (af.hasDeletionVector) {
1615- // Pass tableRoot so addFileDvToProto can pre-resolve "u" -> "p" via Delta's
1616- // own absolutePath (handles Delta's test-only filename prefix; see helper).
1617- DeltaReflection .addFileDvToProto(af.dvDescriptor, tableRoot)
1618- .foreach(taskBuilder.setDv)
1619- }
1620- tlBuilder.addTasks(taskBuilder.build())
1621- }
1622- tlBuilder.build()
1623- }
16241546
16251547 def createExec (nativeOp : Operator , op : CometDeltaScanMarker ): CometNativeExec = {
16261548 val tableRoot = DeltaReflection .extractTableRoot(op.relation).getOrElse(" unknown" )
0 commit comments