Skip to content

Commit ffdda0f

Browse files
schenksjclaude
andcommitted
feat(delta): synthesize synthetic columns in-worker for regular reads (apache#76 Stage B)
Wire the kernel-enumeration read path to produce synthetic columns inside DeltaKernelScanExec instead of stacking DeltaSyntheticColumnsExec, gated by `spark.comet.delta.synthesizeInWorker.enabled` (default ON; takes effect only on the kernel-enumeration path, which ships kernel's per-file transform). - synthesizeReadSchemaJson: kernel read schema = data + partitions + row_index as a kernel RowIndex metadata column + row_id as a kernel RowId metadata column (delta.metadataSpec marker), EXCLUDING the worker-only synthetics (is_row_deleted / row_commit_version / Spark _metadata.* per-file constants). The materialised `_row-commit-version-col-*` stays a data column (kernel has no RowCommitVersion metadata column; read by name, null-filled). - convert(): on the kernel-enumeration path, ship the FULL output (= scan.output) as required_schema (synthetics not stripped), drop final_output_indices (by-name assembly), and set synthesize_in_worker. - read_all_synthesize: handle the zero-data-column case (literal projection / partition-only count) -- drive the row count from record_count/footer, emit empty-data batches + per-file constants, DV flag-or-drop -- so those reads work without a parquet read. Full contrib Delta suite with synthesize on: 148 passed, 0 failed. DeltaSyntheticColumnsExec remains for the AddFiles/subset (DML/CDC) route until that path also ships transforms. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent e404910 commit ffdda0f

3 files changed

Lines changed: 167 additions & 22 deletions

File tree

‎contrib/delta/native/src/kernel_scan.rs‎

Lines changed: 41 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -588,10 +588,6 @@ impl DeltaKernelScanExec {
588588
.map_err(|e| DataFusionError::Execution(format!("kernel scan table root: {e}")))?;
589589
let engine = get_or_create_engine(&table_root_url, &self.storage_config)
590590
.map_err(|e| DataFusionError::Execution(format!("kernel scan engine: {e}")))?;
591-
let logical_schema = arrow_to_kernel_schema(&self.read_logical_schema)
592-
.map_err(|e| DataFusionError::Execution(format!("kernel scan logical schema: {e}")))?;
593-
let physical_schema = arrow_to_kernel_schema(&self.physical_schema)
594-
.map_err(|e| DataFusionError::Execution(format!("kernel scan physical schema: {e}")))?;
595591

596592
// Does the query want `is_row_deleted` (keep all rows, flag them) vs a plain drop of DV rows?
597593
let emit_is_row_deleted = self
@@ -600,6 +596,47 @@ impl DeltaKernelScanExec {
600596
.iter()
601597
.any(|f| f.name() == IS_ROW_DELETED_NAME);
602598

599+
// Zero data columns (literal projection, partition-only count): nothing to read from parquet,
600+
// so kernel produces no row_index either. Drive the row count from `record_count` / the
601+
// parquet footer, emit an empty-data batch of that length, and let `assemble_synthesized`
602+
// fill partition + per-file-constant synthetics by name. DV: flag all rows when is_row_deleted
603+
// is requested, else shorten the count to the live rows.
604+
if self.physical_schema.fields().is_empty() {
605+
let empty_schema = Arc::new(arrow::datatypes::Schema::empty());
606+
let mut out = Vec::new();
607+
for file in &self.files {
608+
let n = self.file_row_count(engine.as_ref(), &table_root_url, file)?;
609+
let mut deleted: Vec<u64> = Vec::new();
610+
if let Some(desc) = &file.dv {
611+
deleted = read_dv_indexes(desc, &table_root_url, &self.storage_config)
612+
.map_err(|e| map_dv_error_to_datafusion(e, desc))?;
613+
}
614+
let (rows, is_deleted): (usize, Option<Int8Array>) = if emit_is_row_deleted {
615+
let dset: std::collections::HashSet<usize> =
616+
deleted.iter().map(|&i| i as usize).collect();
617+
let flags: Vec<i8> = (0..n).map(|i| i8::from(dset.contains(&i))).collect();
618+
(n, Some(Int8Array::from(flags)))
619+
} else {
620+
let live = n - deleted.iter().filter(|&&i| (i as usize) < n).count();
621+
(live, None)
622+
};
623+
let opts = arrow::array::RecordBatchOptions::new().with_row_count(Some(rows));
624+
let batch = arrow::array::RecordBatch::try_new_with_options(
625+
Arc::clone(&empty_schema),
626+
vec![],
627+
&opts,
628+
)
629+
.map_err(DataFusionError::from)?;
630+
out.push(self.assemble_synthesized(batch, file, is_deleted)?);
631+
}
632+
return Ok(out);
633+
}
634+
635+
let logical_schema = arrow_to_kernel_schema(&self.read_logical_schema)
636+
.map_err(|e| DataFusionError::Execution(format!("kernel scan logical schema: {e}")))?;
637+
let physical_schema = arrow_to_kernel_schema(&self.physical_schema)
638+
.map_err(|e| DataFusionError::Execution(format!("kernel scan physical schema: {e}")))?;
639+
603640
let mut out: Vec<arrow::array::RecordBatch> = Vec::new();
604641
for file in &self.files {
605642
// Decode the DV once per file (sorted physical row indexes), only when we flag or drop.

‎contrib/delta/src/main/scala/org/apache/comet/contrib/delta/CometDeltaNativeScan.scala‎

Lines changed: 112 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,66 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
269269
}
270270
}
271271

272+
/**
273+
* Kernel read schema for the in-worker synthesis path (`synthesize_in_worker`): data + partitions,
274+
* plus `row_index` as a kernel `RowIndex` metadata column and `row_id` as a kernel `RowId` metadata
275+
* column (kernel injects/generates them). The WORKER-only synthetics -- `is_row_deleted`,
276+
* `row_commit_version`, and Spark `_metadata.*` per-file constants -- are EXCLUDED (kernel doesn't
277+
* read them; the executor produces them). Returns "" when nothing is read from parquet.
278+
*/
279+
private[delta] def synthesizeReadSchemaJson(
280+
annotatedSource: Option[StructType],
281+
requiredSchema: StructType,
282+
partitionSchema: StructType): String = {
283+
val byName =
284+
annotatedSource.map(_.fields.map(f => f.name.toLowerCase(Locale.ROOT) -> f).toMap)
285+
.getOrElse(Map.empty)
286+
val pick = (f: StructField) => byName.getOrElse(f.name.toLowerCase(Locale.ROOT), f)
287+
// Synthetics the executor produces itself (NOT read from kernel).
288+
val workerOnly: Set[String] = Set(
289+
DeltaReflection.IsRowDeletedColumnName,
290+
DeltaReflection.RowCommitVersionColumnName,
291+
"file_path",
292+
"file_name",
293+
"file_size",
294+
"file_block_start",
295+
"file_block_length",
296+
"file_modification_time",
297+
"base_row_id",
298+
"default_row_commit_version").map(_.toLowerCase(Locale.ROOT))
299+
def isRowIndex(n: String): Boolean =
300+
n.equalsIgnoreCase(DeltaReflection.RowIndexColumnName) ||
301+
n.equalsIgnoreCase(DeltaReflection.TmpMetadataRowIndexColumnName)
302+
def isRowId(n: String): Boolean =
303+
n.equalsIgnoreCase(DeltaReflection.RowIdColumnName) ||
304+
n.toLowerCase(Locale.ROOT).startsWith("_row-id-col-")
305+
val kept: Array[StructField] = requiredSchema.fields.flatMap { f =>
306+
val lc = f.name.toLowerCase(Locale.ROOT)
307+
if (workerOnly.contains(lc)) {
308+
None // worker-side constant; not read from kernel
309+
} else if (isRowIndex(f.name)) {
310+
Some(asKernelMetadataColumn(f.name, "row_index"))
311+
} else if (isRowId(f.name)) {
312+
Some(asKernelMetadataColumn(f.name, KernelRowIdSpec))
313+
} else {
314+
// Real data column -- includes the MATERIALISED `_row-commit-version-col-*` (kernel has no
315+
// RowCommitVersion metadata column, so it's read from parquet by name, null-filled when a
316+
// file lacks it).
317+
Some(pick(f))
318+
}
319+
}
320+
val all = kept ++ partitionSchema.fields.map(pick)
321+
if (all.isEmpty) "" else StructType(all).json
322+
}
323+
324+
/** A LONG field marked as kernel's `<spec>` metadata column (`delta.metadataSpec`). */
325+
private def asKernelMetadataColumn(name: String, spec: String): StructField = {
326+
val md = new org.apache.spark.sql.types.MetadataBuilder()
327+
.putString(KernelMetadataSpecKey, spec)
328+
.build()
329+
StructField(name, org.apache.spark.sql.types.LongType, nullable = true, md)
330+
}
331+
272332
private[delta] def translateDeltaFieldIdToParquet(field: StructField): StructField = {
273333
val newDataType = translateDataTypeFieldIds(field.dataType)
274334
val newMetadata =
@@ -583,6 +643,15 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
583643
// Serialize the data filters so kernel can apply stats-based file pruning during log replay.
584644
val predicateBytes: Array[Byte] = serializeSupportedDataFilters(scan)
585645

646+
// Stage B/C: produce synthetic columns inside DeltaKernelScanExec (kernel metadata columns +
647+
// engine-side DV-invert / per-file constants), retiring the stacked DeltaSyntheticColumnsExec.
648+
// Only on the kernel-enumeration read path (NOT the AddFiles/subset path), which ships kernel's
649+
// per-file transform (needed for row_id GenerateRowId + partition injection). Config-gated while
650+
// validated.
651+
val synthesizeInWorker: Boolean =
652+
DeltaConf.COMET_DELTA_SYNTHESIZE_IN_WORKER_ENABLED.get(scan.conf) &&
653+
!DeltaReflection.isSubsetFileIndex(relation.location)
654+
586655
// Column name list for resolving BoundReference indices to kernel column
587656
// names. Must match the order of scan.output because exprToProto binds
588657
// attribute references by position in that schema.
@@ -680,12 +749,23 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
680749
// there's no stashed reference schema; both carry the annotations kernel needs. Include the
681750
// partition schema so kernel's transform INJECTS partition columns (max-kernel) rather than
682751
// Comet appending them.
683-
val projJson = CometDeltaNativeScan.dataReadSchemaJson(
684-
scan.deltaMetadata.analyzedSchema.orElse(
685-
DeltaReflection.extractSnapshotSchema(relation)),
686-
scan.requiredSchema,
687-
relation.partitionSchema,
688-
rowTrackingActive = rowTrackingEnabled)
752+
val annotated = scan.deltaMetadata.analyzedSchema.orElse(
753+
DeltaReflection.extractSnapshotSchema(relation))
754+
val projJson =
755+
if (synthesizeInWorker) {
756+
// Kernel read = data + partitions + row_index/row_id as kernel metadata columns; the
757+
// executor synthesises is_row_deleted / row_commit_version / _metadata.* itself.
758+
CometDeltaNativeScan.synthesizeReadSchemaJson(
759+
annotated,
760+
scan.requiredSchema,
761+
relation.partitionSchema)
762+
} else {
763+
CometDeltaNativeScan.dataReadSchemaJson(
764+
annotated,
765+
scan.requiredSchema,
766+
relation.partitionSchema,
767+
rowTrackingActive = rowTrackingEnabled)
768+
}
689769
nativeLib.planDeltaScan(
690770
tableRoot,
691771
snapshotVersion,
@@ -899,19 +979,32 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
899979
// row_index, is_row_deleted, row_id, row_commit_version). The native dispatcher
900980
// applies a final ProjectionExec to reorder columns to match Spark's expected
901981
// output layout. Empty when synthetics ARE a suffix -- already in the right order.
902-
val finalOutputIndices: Seq[Int] = computeFinalOutputIndices(
903-
needsSyntheticEmit,
904-
requiredSchemaForProto,
905-
isSynthetic,
906-
emitRowIndex,
907-
emitIsRowDeleted,
908-
emitRowId,
909-
emitRowCommitVersion,
910-
rowIndexColumnAlias,
911-
metadataColumnNamesEmitted)
982+
// In synthesize mode the executor assembles the full output BY NAME, so no positional reorder is
983+
// needed and `required_schema` IS the full output (= scan.output) -- synthetics are NOT stripped.
984+
val finalOutputIndices: Seq[Int] =
985+
if (synthesizeInWorker) Seq.empty
986+
else
987+
computeFinalOutputIndices(
988+
needsSyntheticEmit,
989+
requiredSchemaForProto,
990+
isSynthetic,
991+
emitRowIndex,
992+
emitIsRowDeleted,
993+
emitRowId,
994+
emitRowCommitVersion,
995+
rowIndexColumnAlias,
996+
metadataColumnNamesEmitted)
912997
val requiredSchemaForProtoStripped =
913-
if (needsSyntheticEmit) requiredSchemaForProto.filterNot(isSynthetic)
914-
else requiredSchemaForProto
998+
if (synthesizeInWorker) {
999+
// Full output the executor must emit (data + partitions + ALL synthetics) in scan.output
1000+
// order; the by-name assembler places each column. Logical names (no id-translation -- field
1001+
// ids ride on the kernel READ schema, not the output).
1002+
scan.output.map(a => StructField(a.name, a.dataType, a.nullable)).toArray
1003+
} else if (needsSyntheticEmit) {
1004+
requiredSchemaForProto.filterNot(isSynthetic)
1005+
} else {
1006+
requiredSchemaForProto
1007+
}
9151008

9161009
val requiredSchema = schema2Proto(requiredSchemaForProtoStripped)
9171010
val partitionSchema = schema2Proto(partitionSchemaForProto)
@@ -987,6 +1080,7 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
9871080
// the native exec drives the row count from record_count / the parquet footer (#48).
9881081
val kernelReadEligible = DeltaConf.COMET_DELTA_KERNEL_READ_ENABLED.get(scan.conf)
9891082
commonBuilder.setKernelRead(kernelReadEligible)
1083+
commonBuilder.setSynthesizeInWorker(synthesizeInWorker)
9901084

9911085
// (Data-filter pushdown belonged to the removed ParquetSource path; the kernel-read path does
9921086
// its own stats-based file pruning during log replay, so no pushed predicate is shipped.)

‎contrib/delta/src/main/scala/org/apache/comet/contrib/delta/DeltaConf.scala‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,4 +80,18 @@ object DeltaConf {
8080
"by default; set to false to fall back to the legacy reader.")
8181
.booleanConf
8282
.createWithDefault(true)
83+
84+
val COMET_DELTA_SYNTHESIZE_IN_WORKER_ENABLED: ConfigEntry[Boolean] =
85+
ConfigBuilder("spark.comet.delta.synthesizeInWorker.enabled")
86+
.category(CATEGORY)
87+
.doc(
88+
"Produce Delta synthetic columns (row_index / row_id via kernel metadata columns; " +
89+
"is_row_deleted by inverting the deletion vector; row_commit_version and Spark " +
90+
"_metadata.* as per-file constants) INSIDE DeltaKernelScanExec and assemble the output " +
91+
"by name, instead of stacking the separate DeltaSyntheticColumnsExec. Only takes effect " +
92+
"on the kernel-enumeration read path (regular reads), which ships kernel's per-file " +
93+
"transform. Off by default while the path is validated; flipping it on lets the standalone " +
94+
"synthetic exec be retired.")
95+
.booleanConf
96+
.createWithDefault(true)
8397
}

0 commit comments

Comments
 (0)