Skip to content

Commit bcc1e29

Browse files
schenksjclaude
andcommitted
feat(delta): ship kernel's per-file transform to the executor (apache#76 Stage A foundation)
Wire kernel's own fully-resolved physical->logical transform from the driver to the executor so partition injection, column-mapping relabel, and row-tracking come from delta-kernel-rs rather than Comet-side reconstruction -- the kernel-intended distributed read pattern. Kernel resolves the transform per file (partition values + baseRowId baked in as literals) and exposes it on `scan_file.transform`; the driver was discarding it and rebuilding an identity patch on the executor. - proto: add `DeltaScanTask.transform_json` (serde JSON of kernel's `Expression`). - driver (scan.rs): capture `scan_file.transform`, serialize per file, surface serialization errors after the visit walk. - jni/core: thread `transform_json` through to `KernelScanFile`. - executor (kernel_scan.rs): deserialize the per-file transform and apply kernel's `transform_to_logical` instead of the identity patch. Replace the `append_partition_columns` + positional-reconcile pair with one BY-NAME output assembler: take each output column from the kernel batch when present (the transform injects partitions when the projected schema carries them), else fill partition constants -- robust whether or not the transform injected partitions. - test: `shipped_kernel_transform_injects_partition_column` builds a real partitioned table, pulls kernel's transform, round-trips it through serde JSON exactly as the wire does, and asserts the partition column is injected by kernel. Additive: `transform_json` is populated only on the kernel-enumeration path and the assembler handles both paths, so existing reads are unchanged. All 108 contrib Rust tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent 2df9acc commit bcc1e29

6 files changed

Lines changed: 275 additions & 49 deletions

File tree

‎contrib/delta/native/Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎contrib/delta/native/src/jni.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,11 @@ pub unsafe extern "system" fn Java_org_apache_comet_contrib_delta_Native_planDel
201201
// scan-file metadata; the BatchFileIndex path sets it from
202202
// AddFile.modificationTime equivalently.
203203
modification_time: Some(entry.modification_time),
204+
// Kernel's fully-resolved physical->logical transform for this file (serde JSON),
205+
// extracted in `plan_delta_scan` from `scan_file.transform`. The executor applies it
206+
// via `transform_to_logical` (partition injection / column-mapping relabel /
207+
// row-tracking baked in). Empty = identity (plain pass-through).
208+
transform_json: entry.transform_json,
204209
})
205210
.collect();
206211

‎contrib/delta/native/src/kernel_scan.rs‎

Lines changed: 226 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -186,9 +186,17 @@ pub struct KernelScanFile {
186186
/// Deletion-vector descriptor (absent = no DV). Decoded executor-side via `dv_reader`.
187187
pub dv: Option<DeltaDvDescriptor>,
188188
/// Partition values for this file: `(column name, value)` where value is `None` for a NULL
189-
/// partition. Empty for unpartitioned tables. These are injected as constant columns after
190-
/// the parquet read (partition columns aren't stored in the data files).
189+
/// partition. Empty for unpartitioned tables. Used only by the legacy `append_partition_columns`
190+
/// path (when `transform_json` is empty); when kernel's transform is shipped it bakes partition
191+
/// values in as literals, so this is redundant.
191192
pub partition_values: Vec<(String, Option<String>)>,
193+
/// Kernel's fully-resolved physical->logical transform for this file (serde JSON of
194+
/// `delta_kernel::expressions::Expression`), or empty for no transform. When non-empty the
195+
/// executor applies it via `transform_to_logical` -- partition injection / column-mapping
196+
/// relabel / row-tracking all come from kernel -- and the read targets the kernel LOGICAL schema
197+
/// (data + partitions), reconciled to `output_schema` by name. Empty falls back to the legacy
198+
/// identity-transform + `append_partition_columns` path.
199+
pub transform_json: Vec<u8>,
192200
}
193201

194202
/// Iceberg-style "kernel reads" scan operator. Reads each Delta data file through
@@ -315,7 +323,7 @@ impl DeltaKernelScanExec {
315323
&opts,
316324
)
317325
.map_err(DataFusionError::from)?;
318-
out.push(self.append_partition_columns(batch, &file.partition_values)?);
326+
out.push(self.assemble_output_batch(batch, &file.partition_values)?);
319327
}
320328
return Ok(out);
321329
}
@@ -366,23 +374,46 @@ impl DeltaKernelScanExec {
366374
_ => None,
367375
};
368376

377+
// Kernel's own fully-resolved transform for this file (partition literals + baseRowId for
378+
// row_id baked in) takes precedence; the executor applies it via `transform_to_logical`
379+
// so partition injection / column-mapping relabel / row-tracking all come from kernel.
380+
// Empty falls back to the shared identity transform + Comet-side `append_partition_columns`
381+
// (legacy path, kept until the driver ships transforms for every read).
382+
let use_shipped_transform = !file.transform_json.is_empty();
383+
let file_transform: Option<ExpressionRef> = if use_shipped_transform {
384+
Some(Arc::new(
385+
serde_json::from_slice::<delta_kernel::expressions::Expression>(
386+
&file.transform_json,
387+
)
388+
.map_err(|e| {
389+
DataFusionError::Execution(format!(
390+
"kernel scan: deserialize transform for {}: {e}",
391+
file.path
392+
))
393+
})?,
394+
))
395+
} else {
396+
transform.clone()
397+
};
398+
369399
let kernel_batches = read_file_via_kernel(
370400
engine.as_ref(),
371401
&table_root_url,
372402
&file.path,
373403
file.size,
374404
selection_vector,
375-
transform.clone(),
405+
file_transform,
376406
physical_schema.clone(),
377407
logical_schema.clone(),
378408
)
379409
.map_err(|e| map_file_read_error(e, &file.path))?;
380410

381411
for batch in &kernel_batches {
382412
// Kernel pins the same arrow version as Comet (arrow-58), so the kernel batch IS a
383-
// Comet batch -- no bridge.
384-
let data_batch = batch.clone();
385-
out.push(self.append_partition_columns(data_batch, &file.partition_values)?);
413+
// Comet batch -- no bridge. Assemble the `output_schema` by name: data (+ partitions
414+
// when the shipped transform injected them) come from the kernel batch; any partition
415+
// column the transform did NOT inject is appended as a constant.
416+
out.push(self.assemble_output_batch(batch.clone(), &file.partition_values)?);
386417
}
387418
}
388419
Ok(out)
@@ -418,75 +449,88 @@ impl DeltaKernelScanExec {
418449
Ok(meta.metadata().file_metadata().num_rows().max(0) as usize)
419450
}
420451

421-
/// Append partition columns as constants (partition values are stored in the Add action, not
422-
/// the data file). Produces a batch matching `output_schema` (data ++ partition). No-op for
423-
/// unpartitioned tables.
424-
fn append_partition_columns(
452+
/// Assemble the final `output_schema` batch (data ++ partition, Spark order) from a kernel-read
453+
/// batch, BY NAME. For each output field:
454+
/// - present in the kernel batch (data column, or a partition column the shipped transform
455+
/// already injected) -> take it, casting to the output field's EXACT type incl. NESTED field
456+
/// names (list element, struct fields, map key/value -- kernel names nested fields by Spark
457+
/// convention, e.g. a list element "element", but `output_schema` from the proto may carry
458+
/// empty/different nested names; the cast is a metadata-only relabel, a no-op when types match);
459+
/// - otherwise a partition column (legacy path: transform didn't inject it) -> a constant from
460+
/// `partition_values` (typed NULL when absent / explicitly NULL);
461+
/// - otherwise -> error (a data column kernel was asked for but didn't produce).
462+
///
463+
/// This unifies the old `append_partition_columns` (legacy identity-transform path) and the
464+
/// shipped-transform path: partitions come from kernel when the transform injects them, else from
465+
/// the Add action's values.
466+
fn assemble_output_batch(
425467
&self,
426-
data_batch: arrow::array::RecordBatch,
468+
kernel_batch: arrow::array::RecordBatch,
427469
partition_values: &[(String, Option<String>)],
428470
) -> DFResult<arrow::array::RecordBatch> {
429-
let num_rows = data_batch.num_rows();
430-
let mut columns: Vec<arrow::array::ArrayRef> = data_batch.columns().to_vec();
431-
if !self.partition_schema.fields().is_empty() {
432-
let parsed_tz = SessionTimezone::parse(&self.session_timezone);
433-
for pf in self.partition_schema.fields() {
471+
let num_rows = kernel_batch.num_rows();
472+
let kernel_schema = kernel_batch.schema();
473+
let parsed_tz = SessionTimezone::parse(&self.session_timezone);
474+
let is_partition =
475+
|name: &str| self.partition_schema.fields().iter().any(|f| f.name() == name);
476+
477+
let mut columns: Vec<arrow::array::ArrayRef> =
478+
Vec::with_capacity(self.output_schema.fields().len());
479+
for out_field in self.output_schema.fields() {
480+
if let Some(idx) = kernel_schema
481+
.fields()
482+
.iter()
483+
.position(|f| f.name() == out_field.name())
484+
{
485+
let col = kernel_batch.column(idx);
486+
columns.push(if col.data_type() == out_field.data_type() {
487+
Arc::clone(col)
488+
} else {
489+
arrow_cast(col, out_field.data_type()).map_err(|e| {
490+
DataFusionError::Execution(format!(
491+
"kernel scan: reconciling column '{}' to output schema: {e}",
492+
out_field.name()
493+
))
494+
})?
495+
});
496+
} else if is_partition(out_field.name()) {
434497
let raw = partition_values
435498
.iter()
436-
.find(|(name, _)| name == pf.name())
499+
.find(|(name, _)| name == out_field.name())
437500
.and_then(|(_, v)| v.as_ref());
438501
let scalar = match raw {
439502
Some(v) => parse_delta_partition_scalar(
440503
v,
441-
pf.data_type(),
504+
out_field.data_type(),
442505
&parsed_tz,
443506
&self.session_timezone,
444507
)
445508
.map_err(|e| {
446509
DataFusionError::Execution(format!(
447510
"kernel scan partition value parse for '{}': {e}",
448-
pf.name()
511+
out_field.name()
449512
))
450513
})?,
451514
// Absent or explicitly-NULL partition value -> a typed NULL constant.
452-
None => ScalarValue::try_from(pf.data_type())?,
515+
None => ScalarValue::try_from(out_field.data_type())?,
453516
};
454517
columns.push(scalar.to_array_of_size(num_rows)?);
518+
} else {
519+
return Err(DataFusionError::Execution(format!(
520+
"kernel scan: output column '{}' was not produced by the kernel read and is \
521+
not a partition column",
522+
out_field.name()
523+
)));
455524
}
456525
}
457-
// Reconcile each column to `output_schema`'s EXACT field type, including NESTED field names
458-
// (list element, struct fields, map key/value). The kernel-read batch names nested fields by
459-
// Spark convention (e.g. a list element "element"), but `output_schema` -- derived from the
460-
// proto `required_schema` -- can carry empty/different nested names. Without this, the emitted
461-
// batch's schema != `output_schema` and DataFusion rejects it ("column types must match schema
462-
// types" on a partitioned try_new, or an `assert_eq!` / coalesce panic for the unpartitioned
463-
// pass-through). The data is already in logical (output) order from the kernel transform, so
464-
// this is a metadata-only relabel via `arrow_cast` -- a no-op whenever the types already match.
465-
let reconciled: Vec<arrow::array::ArrayRef> = columns
466-
.iter()
467-
.zip(self.output_schema.fields())
468-
.map(|(col, field)| {
469-
if col.data_type() == field.data_type() {
470-
Ok(Arc::clone(col))
471-
} else {
472-
arrow_cast(col, field.data_type()).map_err(|e| {
473-
DataFusionError::Execution(format!(
474-
"kernel scan: reconciling column '{}' to output schema: {e}",
475-
field.name()
476-
))
477-
})
478-
}
479-
})
480-
.collect::<DFResult<Vec<_>>>()?;
481526
// Carry the row count explicitly: a zero-data-column read (e.g. a MERGE UPDATE that only
482-
// touches a struct field, or `count(*)`) yields an empty `reconciled` with an empty
527+
// touches a struct field, or `count(*)`) yields an empty `columns` with an empty
483528
// `output_schema`, and plain `try_new` cannot infer the row count from zero columns ("must
484-
// either specify a row count or at least one column"). `num_rows` comes from the input batch
485-
// (the zero-data path builds it via `with_row_count`).
529+
// either specify a row count or at least one column").
486530
let opts = arrow::array::RecordBatchOptions::new().with_row_count(Some(num_rows));
487531
arrow::array::RecordBatch::try_new_with_options(
488532
Arc::clone(&self.output_schema),
489-
reconciled,
533+
columns,
490534
&opts,
491535
)
492536
.map_err(DataFusionError::from)
@@ -711,6 +755,7 @@ mod tests {
711755
record_count: Some(7),
712756
dv: None,
713757
partition_values: vec![],
758+
transform_json: Vec::new(),
714759
}];
715760

716761
let exec = DeltaKernelScanExec::new(
@@ -786,6 +831,7 @@ mod tests {
786831
record_count: Some(3),
787832
dv: None,
788833
partition_values: vec![],
834+
transform_json: Vec::new(),
789835
}];
790836

791837
let exec = DeltaKernelScanExec::new(
@@ -905,4 +951,135 @@ mod tests {
905951
vec![100, 200, 300]
906952
);
907953
}
954+
955+
// Builds a minimal single-file PARTITIONED Delta table in a tempdir: schema (id: long, part:
956+
// string) with partitionColumns=["part"], one add file in partition part="A" whose parquet
957+
// holds ONLY the data column `id` (partition columns aren't stored in the data files). Returns
958+
// (tempdir, table url, parquet size). Mirrors `build_single_file_table` but partitioned.
959+
fn build_partitioned_table() -> (tempfile::TempDir, Url, i64) {
960+
let tmp = tempfile::tempdir().unwrap();
961+
let table_dir = tmp.path().join("pscan_table");
962+
let delta_log = table_dir.join("_delta_log");
963+
// Partition data lives under part=A/ by Delta convention.
964+
let part_dir = table_dir.join("part=A");
965+
std::fs::create_dir_all(&delta_log).unwrap();
966+
std::fs::create_dir_all(&part_dir).unwrap();
967+
968+
// Parquet holds only the data column `id` (0..=4).
969+
let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
970+
"id",
971+
ArrowDataType::Int64,
972+
true,
973+
)]));
974+
let batch = RecordBatch::try_new(
975+
arrow_schema.clone(),
976+
vec![Arc::new(Int64Array::from(vec![0i64, 1, 2, 3, 4]))],
977+
)
978+
.unwrap();
979+
let parquet_rel = "part=A/part-00000.parquet";
980+
let parquet_path = table_dir.join(parquet_rel);
981+
let file = std::fs::File::create(&parquet_path).unwrap();
982+
let mut writer = ArrowWriter::try_new(file, arrow_schema, None).unwrap();
983+
writer.write(&batch).unwrap();
984+
writer.close().unwrap();
985+
let size = std::fs::metadata(&parquet_path).unwrap().len() as i64;
986+
987+
let commit0 = [
988+
r#"{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}"#.to_string(),
989+
r#"{"metaData":{"id":"pscan-id","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part"],"configuration":{},"createdTime":1700000000000}}"#.to_string(),
990+
format!(
991+
r#"{{"add":{{"path":"{parquet_rel}","partitionValues":{{"part":"A"}},"size":{size},"modificationTime":1700000000000,"dataChange":true,"stats":"{{\"numRecords\":5}}"}}}}"#
992+
),
993+
]
994+
.join("\n");
995+
std::fs::write(delta_log.join("00000000000000000000.json"), commit0).unwrap();
996+
997+
let url = normalize_url(table_dir.to_str().unwrap()).unwrap();
998+
(tmp, url, size)
999+
}
1000+
1001+
// The Stage-A mechanism end to end at the kernel level: build a partitioned table, get kernel's
1002+
// OWN per-file transform from the scan (the value the driver ships), round-trip it through serde
1003+
// JSON exactly as the driver->executor wire does, then apply it via `read_file_via_kernel` with
1004+
// kernel's physical/logical schemas. The partition column `part` must be INJECTED by the kernel
1005+
// transform (it isn't in the parquet) with value "A" -- i.e. partitions come from kernel, not
1006+
// Comet's `append_partition_columns`. This is the foundation the whole synthetic-columns rewrite
1007+
// builds on, so it guards the serde round-trip + transform application directly.
1008+
#[test]
1009+
fn shipped_kernel_transform_injects_partition_column() {
1010+
use delta_kernel::arrow::array::StringArray;
1011+
use delta_kernel::expressions::Expression;
1012+
1013+
let (_tmp, url, size) = build_partitioned_table();
1014+
let engine = get_or_create_engine(&url, &DeltaStorageConfig::default()).unwrap();
1015+
let snapshot = Snapshot::builder_for(url.clone())
1016+
.build(engine.as_ref())
1017+
.unwrap();
1018+
1019+
// Project the scan to the FULL logical schema (data + partition), exactly as the Stage-A
1020+
// driver does so kernel emits a partition-injecting transform.
1021+
let scan = snapshot.scan_builder().build().unwrap();
1022+
let physical_schema = scan.physical_schema().clone();
1023+
let logical_schema = scan.logical_schema().clone();
1024+
// Physical schema is data-only (`id`); logical includes the partition column (`part`).
1025+
assert_eq!(physical_schema.fields().len(), 1, "physical = data only");
1026+
assert_eq!(
1027+
logical_schema.fields().len(),
1028+
2,
1029+
"logical = data + partition"
1030+
);
1031+
1032+
// Pull kernel's resolved per-file transform (the value shipped on DeltaScanTask).
1033+
let mut transforms: Vec<Option<ExpressionRef>> = Vec::new();
1034+
for meta in scan.scan_metadata(engine.as_ref()).unwrap() {
1035+
let meta = meta.unwrap();
1036+
for t in &meta.scan_file_transforms {
1037+
if t.is_some() {
1038+
transforms.push(t.clone());
1039+
}
1040+
}
1041+
}
1042+
assert_eq!(transforms.len(), 1, "one selected file");
1043+
let transform = transforms.into_iter().next().unwrap().unwrap();
1044+
1045+
// Round-trip the transform through serde JSON exactly as driver -> executor does.
1046+
let json = serde_json::to_vec(transform.as_ref()).unwrap();
1047+
let restored: Expression = serde_json::from_slice(&json).unwrap();
1048+
assert_eq!(&*transform, &restored, "transform survives serde round-trip");
1049+
1050+
let batches = read_file_via_kernel(
1051+
engine.as_ref(),
1052+
&url,
1053+
"part=A/part-00000.parquet",
1054+
size,
1055+
None,
1056+
Some(Arc::new(restored)),
1057+
physical_schema,
1058+
logical_schema,
1059+
)
1060+
.unwrap();
1061+
1062+
// Output carries data + the kernel-injected partition column.
1063+
let schema = batches[0].schema();
1064+
assert_eq!(schema.field(0).name(), "id");
1065+
assert_eq!(schema.field(1).name(), "part");
1066+
let ids = batches[0]
1067+
.column(0)
1068+
.as_any()
1069+
.downcast_ref::<Int64Array>()
1070+
.expect("id Int64");
1071+
assert_eq!(
1072+
(0..ids.len()).map(|i| ids.value(i)).collect::<Vec<_>>(),
1073+
vec![0, 1, 2, 3, 4]
1074+
);
1075+
let parts = batches[0]
1076+
.column(1)
1077+
.as_any()
1078+
.downcast_ref::<StringArray>()
1079+
.expect("kernel transform should inject the `part` partition column as Utf8");
1080+
assert!(
1081+
(0..parts.len()).all(|i| parts.value(i) == "A"),
1082+
"partition column injected by kernel transform must equal the file's partition value"
1083+
);
1084+
}
9081085
}

0 commit comments

Comments
 (0)