@@ -269,6 +269,66 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
269269 }
270270 }
271271
272+ /**
273+ * Kernel read schema for the in-worker synthesis path (`synthesize_in_worker`): data + partitions,
274+ * plus `row_index` as a kernel `RowIndex` metadata column and `row_id` as a kernel `RowId` metadata
275+ * column (kernel injects/generates them). The WORKER-only synthetics -- `is_row_deleted`,
276+ * `row_commit_version`, and Spark `_metadata.*` per-file constants -- are EXCLUDED (kernel doesn't
277+ * read them; the executor produces them). Returns "" when nothing is read from parquet.
278+ */
279+ private [delta] def synthesizeReadSchemaJson (
280+ annotatedSource : Option [StructType ],
281+ requiredSchema : StructType ,
282+ partitionSchema : StructType ): String = {
283+ val byName =
284+ annotatedSource.map(_.fields.map(f => f.name.toLowerCase(Locale .ROOT ) -> f).toMap)
285+ .getOrElse(Map .empty)
286+ val pick = (f : StructField ) => byName.getOrElse(f.name.toLowerCase(Locale .ROOT ), f)
287+ // Synthetics the executor produces itself (NOT read from kernel).
288+ val workerOnly : Set [String ] = Set (
289+ DeltaReflection .IsRowDeletedColumnName ,
290+ DeltaReflection .RowCommitVersionColumnName ,
291+ " file_path" ,
292+ " file_name" ,
293+ " file_size" ,
294+ " file_block_start" ,
295+ " file_block_length" ,
296+ " file_modification_time" ,
297+ " base_row_id" ,
298+ " default_row_commit_version" ).map(_.toLowerCase(Locale .ROOT ))
299+ def isRowIndex (n : String ): Boolean =
300+ n.equalsIgnoreCase(DeltaReflection .RowIndexColumnName ) ||
301+ n.equalsIgnoreCase(DeltaReflection .TmpMetadataRowIndexColumnName )
302+ def isRowId (n : String ): Boolean =
303+ n.equalsIgnoreCase(DeltaReflection .RowIdColumnName ) ||
304+ n.toLowerCase(Locale .ROOT ).startsWith(" _row-id-col-" )
305+ val kept : Array [StructField ] = requiredSchema.fields.flatMap { f =>
306+ val lc = f.name.toLowerCase(Locale .ROOT )
307+ if (workerOnly.contains(lc)) {
308+ None // worker-side constant; not read from kernel
309+ } else if (isRowIndex(f.name)) {
310+ Some (asKernelMetadataColumn(f.name, " row_index" ))
311+ } else if (isRowId(f.name)) {
312+ Some (asKernelMetadataColumn(f.name, KernelRowIdSpec ))
313+ } else {
314+ // Real data column -- includes the MATERIALISED `_row-commit-version-col-*` (kernel has no
315+ // RowCommitVersion metadata column, so it's read from parquet by name, null-filled when a
316+ // file lacks it).
317+ Some (pick(f))
318+ }
319+ }
320+ val all = kept ++ partitionSchema.fields.map(pick)
321+ if (all.isEmpty) " " else StructType (all).json
322+ }
323+
324+ /** A LONG field marked as kernel's `<spec>` metadata column (`delta.metadataSpec`). */
325+ private def asKernelMetadataColumn (name : String , spec : String ): StructField = {
326+ val md = new org.apache.spark.sql.types.MetadataBuilder ()
327+ .putString(KernelMetadataSpecKey , spec)
328+ .build()
329+ StructField (name, org.apache.spark.sql.types.LongType , nullable = true , md)
330+ }
331+
272332 private [delta] def translateDeltaFieldIdToParquet (field : StructField ): StructField = {
273333 val newDataType = translateDataTypeFieldIds(field.dataType)
274334 val newMetadata =
@@ -583,6 +643,15 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
583643 // Serialize the data filters so kernel can apply stats-based file pruning during log replay.
584644 val predicateBytes : Array [Byte ] = serializeSupportedDataFilters(scan)
585645
646+ // Stage B/C: produce synthetic columns inside DeltaKernelScanExec (kernel metadata columns +
647+ // engine-side DV-invert / per-file constants), retiring the stacked DeltaSyntheticColumnsExec.
648+ // Only on the kernel-enumeration read path (NOT the AddFiles/subset path), which ships kernel's
649+ // per-file transform (needed for row_id GenerateRowId + partition injection). Config-gated while
650+ // validated.
651+ val synthesizeInWorker : Boolean =
652+ DeltaConf .COMET_DELTA_SYNTHESIZE_IN_WORKER_ENABLED .get(scan.conf) &&
653+ ! DeltaReflection .isSubsetFileIndex(relation.location)
654+
586655 // Column name list for resolving BoundReference indices to kernel column
587656 // names. Must match the order of scan.output because exprToProto binds
588657 // attribute references by position in that schema.
@@ -680,12 +749,23 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
680749 // there's no stashed reference schema; both carry the annotations kernel needs. Include the
681750 // partition schema so kernel's transform INJECTS partition columns (max-kernel) rather than
682751 // Comet appending them.
683- val projJson = CometDeltaNativeScan .dataReadSchemaJson(
684- scan.deltaMetadata.analyzedSchema.orElse(
685- DeltaReflection .extractSnapshotSchema(relation)),
686- scan.requiredSchema,
687- relation.partitionSchema,
688- rowTrackingActive = rowTrackingEnabled)
752+ val annotated = scan.deltaMetadata.analyzedSchema.orElse(
753+ DeltaReflection .extractSnapshotSchema(relation))
754+ val projJson =
755+ if (synthesizeInWorker) {
756+ // Kernel read = data + partitions + row_index/row_id as kernel metadata columns; the
757+ // executor synthesises is_row_deleted / row_commit_version / _metadata.* itself.
758+ CometDeltaNativeScan .synthesizeReadSchemaJson(
759+ annotated,
760+ scan.requiredSchema,
761+ relation.partitionSchema)
762+ } else {
763+ CometDeltaNativeScan .dataReadSchemaJson(
764+ annotated,
765+ scan.requiredSchema,
766+ relation.partitionSchema,
767+ rowTrackingActive = rowTrackingEnabled)
768+ }
689769 nativeLib.planDeltaScan(
690770 tableRoot,
691771 snapshotVersion,
@@ -899,19 +979,32 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
899979 // row_index, is_row_deleted, row_id, row_commit_version). The native dispatcher
900980 // applies a final ProjectionExec to reorder columns to match Spark's expected
901981 // output layout. Empty when synthetics ARE a suffix -- already in the right order.
902- val finalOutputIndices : Seq [Int ] = computeFinalOutputIndices(
903- needsSyntheticEmit,
904- requiredSchemaForProto,
905- isSynthetic,
906- emitRowIndex,
907- emitIsRowDeleted,
908- emitRowId,
909- emitRowCommitVersion,
910- rowIndexColumnAlias,
911- metadataColumnNamesEmitted)
982+ // In synthesize mode the executor assembles the full output BY NAME, so no positional reorder is
983+ // needed and `required_schema` IS the full output (= scan.output) -- synthetics are NOT stripped.
984+ val finalOutputIndices : Seq [Int ] =
985+ if (synthesizeInWorker) Seq .empty
986+ else
987+ computeFinalOutputIndices(
988+ needsSyntheticEmit,
989+ requiredSchemaForProto,
990+ isSynthetic,
991+ emitRowIndex,
992+ emitIsRowDeleted,
993+ emitRowId,
994+ emitRowCommitVersion,
995+ rowIndexColumnAlias,
996+ metadataColumnNamesEmitted)
912997 val requiredSchemaForProtoStripped =
913- if (needsSyntheticEmit) requiredSchemaForProto.filterNot(isSynthetic)
914- else requiredSchemaForProto
998+ if (synthesizeInWorker) {
999+ // Full output the executor must emit (data + partitions + ALL synthetics) in scan.output
1000+ // order; the by-name assembler places each column. Logical names (no id-translation -- field
1001+ // ids ride on the kernel READ schema, not the output).
1002+ scan.output.map(a => StructField (a.name, a.dataType, a.nullable)).toArray
1003+ } else if (needsSyntheticEmit) {
1004+ requiredSchemaForProto.filterNot(isSynthetic)
1005+ } else {
1006+ requiredSchemaForProto
1007+ }
9151008
9161009 val requiredSchema = schema2Proto(requiredSchemaForProtoStripped)
9171010 val partitionSchema = schema2Proto(partitionSchemaForProto)
@@ -987,6 +1080,7 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
9871080 // the native exec drives the row count from record_count / the parquet footer (#48).
9881081 val kernelReadEligible = DeltaConf .COMET_DELTA_KERNEL_READ_ENABLED .get(scan.conf)
9891082 commonBuilder.setKernelRead(kernelReadEligible)
1083+ commonBuilder.setSynthesizeInWorker(synthesizeInWorker)
9901084
9911085 // (Data-filter pushdown belonged to the removed ParquetSource path; the kernel-read path does
9921086 // its own stats-based file pruning during log replay, so no pushed predicate is shipped.)
0 commit comments