Skip to content

Commit e404910

Browse files
schenksjclaude
andcommitted
feat(delta): worker-inline synthetic-column synthesis behind a flag (apache#76 Stage B/C, gated off)
Foundation for replacing DeltaSyntheticColumnsExec with in-worker synthesis. Adds a `synthesize_in_worker` proto flag; when set, DeltaKernelScanExec produces ALL synthetic columns itself and assembles the full output_schema BY NAME (so no final_output_indices reorder and no stacked synthetic exec are needed): - row_index / row_id arrive as kernel metadata columns carried in the read schemas + per-file transform (driver wiring lands next), so they appear as kernel-batch columns. - is_row_deleted: invert the decoded DV (single forward pass over sorted indexes), computed against PHYSICAL rows (read with no kernel drop so positions survive). - row_commit_version + Spark `_metadata.*` (file_path/file_name/file_size/file_block_*/ file_modification_time/base_row_id/default_row_commit_version): per-file constants, values matching DeltaSyntheticColumnsExec exactly. - drop deleted rows (when is_row_deleted not requested) via filter_record_batch. KernelScanFile gains base_row_id / default_row_commit_version / modification_time / byte range (all already on DeltaScanTask). Core reads the flag, gates the legacy DeltaSyntheticColumnsExec wrap off when set, and passes `synthesize` to the exec. Gated off (Scala doesn't set the flag yet): existing path unchanged. Contrib Rust tests 109/0; core builds with --features contrib-delta. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent 17380bc commit e404910

3 files changed

Lines changed: 323 additions & 5 deletions

File tree

‎contrib/delta/native/src/kernel_scan.rs‎

Lines changed: 296 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,20 @@ use crate::engine::{get_or_create_engine, DeltaStorageConfig};
6666
use crate::planner::{parse_delta_partition_scalar, SessionTimezone};
6767
use crate::proto::DeltaDvDescriptor;
6868

69+
// Engine-side synthetic / Spark `_metadata.*` column names produced in `synthesize` mode. row_index
70+
// and row_id are NOT here -- they come from kernel metadata columns in the read. These mirror
71+
// `DeltaSyntheticColumnsExec` so the two paths emit identical values (kernel pins arrow-58 = Comet's).
72+
const IS_ROW_DELETED_NAME: &str = "__delta_internal_is_row_deleted";
73+
const ROW_COMMIT_VERSION_NAME: &str = "row_commit_version";
74+
const META_FILE_PATH: &str = "file_path";
75+
const META_FILE_NAME: &str = "file_name";
76+
const META_FILE_SIZE: &str = "file_size";
77+
const META_FILE_BLOCK_START: &str = "file_block_start";
78+
const META_FILE_BLOCK_LENGTH: &str = "file_block_length";
79+
const META_FILE_MODIFICATION_TIME: &str = "file_modification_time";
80+
const META_BASE_ROW_ID: &str = "base_row_id";
81+
const META_DEFAULT_ROW_COMMIT_VERSION: &str = "default_row_commit_version";
82+
6983
/// Convert a Comet arrow `Schema` into a kernel `StructType`. Since delta-kernel pins the same arrow
7084
/// version as Comet (arrow-58), this is kernel's own `try_from_arrow` directly -- no cross-version
7185
/// FFI bridge. `try_from_arrow` DOES preserve `PARQUET:field_id` (and nested column-mapping ids)
@@ -197,6 +211,20 @@ pub struct KernelScanFile {
197211
/// (data + partitions), reconciled to `output_schema` by name. Empty falls back to the legacy
198212
/// identity-transform + `append_partition_columns` path.
199213
pub transform_json: Vec<u8>,
214+
/// `AddFile.baseRowId` for row-tracking tables (`None` otherwise). Per-file constant surfaced as
215+
/// `_metadata.base_row_id`. (row_id itself comes from kernel's RowId metadata column.)
216+
pub base_row_id: Option<i64>,
217+
/// `AddFile.defaultRowCommitVersion` for row-tracking tables (`None` otherwise). Per-file
218+
/// constant surfaced as `row_commit_version` / `_metadata.default_row_commit_version` (kernel has
219+
/// no RowCommitVersion metadata-column support, so this stays engine-side).
220+
pub default_row_commit_version: Option<i64>,
221+
/// File modification time, epoch millis (`AddFile.modificationTime`). Surfaced as
222+
/// `_metadata.file_modification_time` (converted to micros).
223+
pub modification_time: i64,
224+
/// File-split byte range (`None`/`None` = whole file). Surfaces `_metadata.file_block_start` /
225+
/// `file_block_length`.
226+
pub byte_range_start: Option<i64>,
227+
pub byte_range_end: Option<i64>,
200228
}
201229

202230
/// Iceberg-style "kernel reads" scan operator. Reads each Delta data file through
@@ -239,6 +267,13 @@ pub struct DeltaKernelScanExec {
239267
storage_config: DeltaStorageConfig,
240268
/// Files this scan reads (whole-table, single partition for now).
241269
files: Vec<KernelScanFile>,
270+
/// Stage B/C: when true, this exec produces ALL synthetic columns itself (row_index / row_id via
271+
/// kernel metadata columns carried in the read; is_row_deleted by inverting the decoded DV;
272+
/// row_commit_version + Spark `_metadata.*` as per-file constants) and assembles the full
273+
/// `output_schema` by name -- no DeltaSyntheticColumnsExec wraps it. When false, the legacy path:
274+
/// the wrapping exec appends synthetics and this exec only reads data (+ partitions) and applies
275+
/// the DV per `apply_dv`.
276+
synthesize: bool,
242277
plan_properties: Arc<PlanProperties>,
243278
metrics: ExecutionPlanMetricsSet,
244279
}
@@ -257,6 +292,7 @@ impl DeltaKernelScanExec {
257292
table_root: String,
258293
storage_config: DeltaStorageConfig,
259294
files: Vec<KernelScanFile>,
295+
synthesize: bool,
260296
) -> Self {
261297
// Single DataFusion partition: the Spark side (CometDeltaNativeScanExec) already splits
262298
// files across Spark partitions and injects each partition's file subset, consuming one
@@ -280,6 +316,7 @@ impl DeltaKernelScanExec {
280316
table_root,
281317
storage_config,
282318
files,
319+
synthesize,
283320
plan_properties,
284321
metrics: ExecutionPlanMetricsSet::new(),
285322
}
@@ -292,6 +329,9 @@ impl DeltaKernelScanExec {
292329
/// the wrapping DeltaSyntheticColumnsExec computes row positions against a single file. Eager
293330
/// (collects into a `Vec`); the blocking kernel reads run on the calling task thread.
294331
fn read_all(&self) -> DFResult<Vec<arrow::array::RecordBatch>> {
332+
if self.synthesize {
333+
return self.read_all_synthesize();
334+
}
295335
let table_root_url = normalize_table_root(&self.table_root)
296336
.map_err(|e| DataFusionError::Execution(format!("kernel scan table root: {e}")))?;
297337
let engine = get_or_create_engine(&table_root_url, &self.storage_config)
@@ -535,6 +575,250 @@ impl DeltaKernelScanExec {
535575
)
536576
.map_err(DataFusionError::from)
537577
}
578+
579+
/// Stage B/C synthesis path. Reads each file via kernel (row_index / row_id arrive as kernel
580+
/// metadata columns carried in `physical_schema`/`read_logical_schema` + the per-file transform),
581+
/// then produces the engine-side synthetics -- `is_row_deleted` by inverting the decoded DV,
582+
/// `row_commit_version` + Spark `_metadata.*` as per-file constants -- dropping DV rows when
583+
/// `is_row_deleted` isn't requested, and assembles the FULL `output_schema` by name. Replaces the
584+
/// stacked DeltaSyntheticColumnsExec (no `final_output_indices` needed -- by-name handles order).
585+
fn read_all_synthesize(&self) -> DFResult<Vec<arrow::array::RecordBatch>> {
586+
use arrow::array::Int8Array;
587+
let table_root_url = normalize_table_root(&self.table_root)
588+
.map_err(|e| DataFusionError::Execution(format!("kernel scan table root: {e}")))?;
589+
let engine = get_or_create_engine(&table_root_url, &self.storage_config)
590+
.map_err(|e| DataFusionError::Execution(format!("kernel scan engine: {e}")))?;
591+
let logical_schema = arrow_to_kernel_schema(&self.read_logical_schema)
592+
.map_err(|e| DataFusionError::Execution(format!("kernel scan logical schema: {e}")))?;
593+
let physical_schema = arrow_to_kernel_schema(&self.physical_schema)
594+
.map_err(|e| DataFusionError::Execution(format!("kernel scan physical schema: {e}")))?;
595+
596+
// Does the query want `is_row_deleted` (keep all rows, flag them) vs a plain drop of DV rows?
597+
let emit_is_row_deleted = self
598+
.output_schema
599+
.fields()
600+
.iter()
601+
.any(|f| f.name() == IS_ROW_DELETED_NAME);
602+
603+
let mut out: Vec<arrow::array::RecordBatch> = Vec::new();
604+
for file in &self.files {
605+
// Decode the DV once per file (sorted physical row indexes), only when we flag or drop.
606+
let mut deleted: Vec<u64> = Vec::new();
607+
if let Some(desc) = &file.dv {
608+
deleted = read_dv_indexes(desc, &table_root_url, &self.storage_config)
609+
.map_err(|e| map_dv_error_to_datafusion(e, desc))?;
610+
deleted.sort_unstable();
611+
}
612+
let has_dv = !deleted.is_empty();
613+
let drop_deleted = has_dv && !emit_is_row_deleted;
614+
615+
let transform: Option<ExpressionRef> = if file.transform_json.is_empty() {
616+
None
617+
} else {
618+
Some(Arc::new(
619+
serde_json::from_slice::<delta_kernel::expressions::Expression>(
620+
&file.transform_json,
621+
)
622+
.map_err(|e| {
623+
DataFusionError::Execution(format!(
624+
"kernel scan: deserialize transform for {}: {e}",
625+
file.path
626+
))
627+
})?,
628+
))
629+
};
630+
631+
// Read ALL physical rows (no kernel DV drop) so row_index stays physical and we control
632+
// flag-vs-drop ourselves -- surviving rows keep their physical row_index after a drop.
633+
let kernel_batches = read_file_via_kernel(
634+
engine.as_ref(),
635+
&table_root_url,
636+
&file.path,
637+
file.size,
638+
None,
639+
transform,
640+
physical_schema.clone(),
641+
logical_schema.clone(),
642+
)
643+
.map_err(|e| map_file_read_error(e, &file.path))?;
644+
645+
let mut file_offset: i64 = 0;
646+
let mut del_ptr: usize = 0;
647+
for kb in &kernel_batches {
648+
let len = kb.num_rows();
649+
let batch_end = file_offset + len as i64;
650+
// is_row_deleted (Int8) over physical rows [file_offset, batch_end) via a single
651+
// forward pass through the sorted DV indexes.
652+
let is_deleted_arr: Option<Int8Array> =
653+
if has_dv && (emit_is_row_deleted || drop_deleted) {
654+
while del_ptr < deleted.len() && (deleted[del_ptr] as i64) < file_offset {
655+
del_ptr += 1;
656+
}
657+
let mut flags = vec![0i8; len];
658+
let mut p = del_ptr;
659+
while p < deleted.len() && (deleted[p] as i64) < batch_end {
660+
flags[(deleted[p] as i64 - file_offset) as usize] = 1;
661+
p += 1;
662+
}
663+
Some(Int8Array::from(flags))
664+
} else {
665+
None
666+
};
667+
668+
let assembled = self.assemble_synthesized(
669+
kb.clone(),
670+
file,
671+
if emit_is_row_deleted {
672+
is_deleted_arr.clone()
673+
} else {
674+
None
675+
},
676+
)?;
677+
678+
let final_batch = if drop_deleted {
679+
let keep: Vec<bool> = match &is_deleted_arr {
680+
Some(a) => (0..len).map(|i| a.value(i) == 0).collect(),
681+
None => vec![true; len],
682+
};
683+
arrow::compute::filter_record_batch(&assembled, &arrow::array::BooleanArray::from(keep))
684+
.map_err(DataFusionError::from)?
685+
} else {
686+
assembled
687+
};
688+
out.push(final_batch);
689+
file_offset = batch_end;
690+
}
691+
}
692+
Ok(out)
693+
}
694+
695+
/// Assemble the FULL `output_schema` by name from a kernel-read batch plus the engine-side
696+
/// synthetics. For each output field: take it from the kernel batch if present (data, partitions,
697+
/// row_index, row_id); else build the known synthetic (`is_row_deleted`, `row_commit_version`,
698+
/// Spark `_metadata.*` per-file constants); else a partition constant; else error.
699+
fn assemble_synthesized(
700+
&self,
701+
kernel_batch: arrow::array::RecordBatch,
702+
file: &KernelScanFile,
703+
is_deleted: Option<arrow::array::Int8Array>,
704+
) -> DFResult<arrow::array::RecordBatch> {
705+
use arrow::array::{Int64Array, StringArray, TimestampMicrosecondArray};
706+
let num_rows = kernel_batch.num_rows();
707+
let kernel_schema = kernel_batch.schema();
708+
let parsed_tz = SessionTimezone::parse(&self.session_timezone);
709+
let cast_to = |arr: arrow::array::ArrayRef,
710+
field: &arrow::datatypes::Field|
711+
-> DFResult<arrow::array::ArrayRef> {
712+
if arr.data_type() == field.data_type() {
713+
Ok(arr)
714+
} else {
715+
arrow_cast(&arr, field.data_type()).map_err(|e| {
716+
DataFusionError::Execution(format!(
717+
"kernel scan (synthesize): reconciling column '{}': {e}",
718+
field.name()
719+
))
720+
})
721+
}
722+
};
723+
724+
let mut columns: Vec<arrow::array::ArrayRef> =
725+
Vec::with_capacity(self.output_schema.fields().len());
726+
for out_field in self.output_schema.fields() {
727+
let name = out_field.name();
728+
// 1. produced by the kernel read (data, partitions, row_index, row_id).
729+
if let Some(idx) = kernel_schema.fields().iter().position(|f| f.name() == name) {
730+
columns.push(cast_to(Arc::clone(kernel_batch.column(idx)), out_field)?);
731+
continue;
732+
}
733+
// 2. engine-side synthetics + Spark `_metadata.*` per-file constants.
734+
let lc = name.to_ascii_lowercase();
735+
let arr: arrow::array::ArrayRef = match lc.as_str() {
736+
IS_ROW_DELETED_NAME => match &is_deleted {
737+
Some(a) => Arc::new(a.clone()),
738+
None => Arc::new(arrow::array::Int8Array::from(vec![0i8; num_rows])),
739+
},
740+
ROW_COMMIT_VERSION_NAME | META_DEFAULT_ROW_COMMIT_VERSION => {
741+
match file.default_row_commit_version {
742+
Some(v) => Arc::new(Int64Array::from(vec![Some(v); num_rows])),
743+
None => Arc::new(Int64Array::from(vec![None as Option<i64>; num_rows])),
744+
}
745+
}
746+
META_FILE_PATH => Arc::new(StringArray::from(vec![file.path.clone(); num_rows])),
747+
META_FILE_NAME => {
748+
let base = match file.path.rfind('/') {
749+
Some(i) => file.path[i + 1..].to_string(),
750+
None => file.path.clone(),
751+
};
752+
Arc::new(StringArray::from(vec![base; num_rows]))
753+
}
754+
META_FILE_SIZE => Arc::new(Int64Array::from(vec![file.size; num_rows])),
755+
META_FILE_BLOCK_START => {
756+
Arc::new(Int64Array::from(vec![file.byte_range_start.unwrap_or(0); num_rows]))
757+
}
758+
META_FILE_BLOCK_LENGTH => {
759+
let v = match (file.byte_range_start, file.byte_range_end) {
760+
(Some(s), Some(e)) => e - s,
761+
_ => file.size,
762+
};
763+
Arc::new(Int64Array::from(vec![v; num_rows]))
764+
}
765+
META_FILE_MODIFICATION_TIME => {
766+
let micros = file.modification_time.saturating_mul(1000);
767+
Arc::new(
768+
TimestampMicrosecondArray::from(vec![micros; num_rows]).with_timezone("UTC"),
769+
)
770+
}
771+
META_BASE_ROW_ID => {
772+
Arc::new(Int64Array::from(vec![file.base_row_id.unwrap_or(0); num_rows]))
773+
}
774+
other
775+
if other.starts_with("_row-id-col-")
776+
|| other.starts_with("_row-commit-version-col-") =>
777+
{
778+
// Materialised row-tracking column absent from this file -> all nulls (kernel's
779+
// RowId metadata column already surfaced a present one as a data column above).
780+
Arc::new(Int64Array::from(vec![None as Option<i64>; num_rows]))
781+
}
782+
_ if self.partition_schema.fields().iter().any(|f| f.name() == name) => {
783+
// Partition column the transform didn't inject -> constant from the Add action.
784+
let raw = file
785+
.partition_values
786+
.iter()
787+
.find(|(n, _)| n == name)
788+
.and_then(|(_, v)| v.as_ref());
789+
let scalar = match raw {
790+
Some(v) => parse_delta_partition_scalar(
791+
v,
792+
out_field.data_type(),
793+
&parsed_tz,
794+
&self.session_timezone,
795+
)
796+
.map_err(|e| {
797+
DataFusionError::Execution(format!(
798+
"kernel scan (synthesize) partition '{name}': {e}"
799+
))
800+
})?,
801+
None => ScalarValue::try_from(out_field.data_type())?,
802+
};
803+
scalar.to_array_of_size(num_rows)?
804+
}
805+
_ => {
806+
return Err(DataFusionError::Execution(format!(
807+
"kernel scan (synthesize): output column '{name}' was not produced by the \
808+
kernel read and is not a known synthetic / partition column"
809+
)));
810+
}
811+
};
812+
columns.push(cast_to(arr, out_field)?);
813+
}
814+
let opts = arrow::array::RecordBatchOptions::new().with_row_count(Some(num_rows));
815+
arrow::array::RecordBatch::try_new_with_options(
816+
Arc::clone(&self.output_schema),
817+
columns,
818+
&opts,
819+
)
820+
.map_err(DataFusionError::from)
821+
}
538822
}
539823

540824
impl fmt::Debug for DeltaKernelScanExec {
@@ -756,6 +1040,11 @@ mod tests {
7561040
dv: None,
7571041
partition_values: vec![],
7581042
transform_json: Vec::new(),
1043+
base_row_id: None,
1044+
default_row_commit_version: None,
1045+
modification_time: 0,
1046+
byte_range_start: None,
1047+
byte_range_end: None,
7591048
}];
7601049

7611050
let exec = DeltaKernelScanExec::new(
@@ -769,6 +1058,7 @@ mod tests {
7691058
url.as_str().to_string(),
7701059
DeltaStorageConfig::default(),
7711060
files,
1061+
false, // legacy path (no in-worker synthesis)
7721062
);
7731063

7741064
let ctx = Arc::new(TaskContext::default());
@@ -832,6 +1122,11 @@ mod tests {
8321122
dv: None,
8331123
partition_values: vec![],
8341124
transform_json: Vec::new(),
1125+
base_row_id: None,
1126+
default_row_commit_version: None,
1127+
modification_time: 0,
1128+
byte_range_start: None,
1129+
byte_range_end: None,
8351130
}];
8361131

8371132
let exec = DeltaKernelScanExec::new(
@@ -845,6 +1140,7 @@ mod tests {
8451140
url.as_str().to_string(),
8461141
DeltaStorageConfig::default(),
8471142
files,
1143+
false, // legacy path (no in-worker synthesis)
8481144
);
8491145
let stream = exec.execute(0, Arc::new(TaskContext::default())).unwrap();
8501146
let batches = datafusion::physical_plan::common::collect(stream)

0 commit comments

Comments
 (0)