Skip to content

Commit dc60b71

Browse files
schenksjclaude
andcommitted
feat(delta): kernel-native CDF read mechanism via TableChanges (apache#76/apache#84, Rust foundation)
Add the executor-side Change Data Feed read using delta-kernel-rs's TableChanges API, the kernel-native way to read change data (corrects the earlier mistaken "kernel can't do CDC"). - read_cdf_via_kernel: TableChanges::try_new(url, start, Option<end>) -> into_scan_builder(). build().execute() -> Arrow batches with data + _change_type / _commit_version / _commit_timestamp (column mapping / partitions / DVs resolved by kernel). Single-task: kernel's CDF per-file API (scan_metadata / CdfScanFile / get_cdf_transform_expr) is pub(crate), so only the all-in-one execute() is usable -- one partition reads the whole bounded version range. - DeltaKernelScanExec gains `cdf: Option<(start, end)>`; read_all routes to read_all_cdf, which reuses the by-name assembler to place the CDF columns in scan.output order. - proto: DeltaScanCommon.cdf_read / cdf_start_version / cdf_end_version. core maps them and bypasses the kernel data-column schema requirement (CDF ships no per-file tasks/schemas). Additive + dormant: the Scala side doesn't set cdf_read yet, so existing reads are unchanged (contrib Rust tests 109/0). Remaining (apache#84): Scala CDF detection/routing -- intercept DeltaCDFRelation (path readChangeFeed) + CdcAddFileIndex (streaming), extract start/end versions + the CDF output schema -- then CometDeltaCdcSuite, then retire DeltaSyntheticColumnsExec. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent 976a4e4 commit dc60b71

3 files changed

Lines changed: 96 additions & 1 deletion

File tree

‎contrib/delta/native/src/kernel_scan.rs‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,40 @@ pub fn read_file_via_kernel(
185185
Ok(out)
186186
}
187187

188+
/// Read a Delta Change Data Feed (CDF / readChangeFeed) over `[start_version, end_version]` via
189+
/// kernel's `TableChanges` API and return Arrow `RecordBatch`es in the CDF LOGICAL schema: data
190+
/// columns + `_change_type` (Utf8) + `_commit_version` (Int64) + `_commit_timestamp`
191+
/// (Timestamp(us, UTC)), with column mapping / partitions / deletion vectors all resolved by kernel.
192+
///
193+
/// Kernel's CDF per-file API (`scan_metadata` / `CdfScanFile` / `get_cdf_transform_expr`) is
194+
/// `pub(crate)`; only `execute()` is public. So unlike the regular read, CDF can't be distributed
195+
/// per-file -- this reconstructs `TableChanges` and reads the WHOLE range on one task (the caller
196+
/// runs it as a single DataFusion partition). For CDF's typically-bounded version ranges that's fine.
197+
pub fn read_cdf_via_kernel(
198+
engine: Arc<dyn Engine>,
199+
table_root: &Url,
200+
start_version: u64,
201+
end_version: Option<u64>,
202+
) -> DeltaResult<Vec<RecordBatch>> {
203+
let table_changes = delta_kernel::table_changes::TableChanges::try_new(
204+
table_root.clone(),
205+
engine.as_ref(),
206+
start_version,
207+
end_version,
208+
)?;
209+
let scan = table_changes.into_scan_builder().build()?;
210+
let mut out: Vec<RecordBatch> = Vec::new();
211+
for batch in scan.execute(engine.clone())? {
212+
let batch = batch?;
213+
let arrow = batch
214+
.any_ref()
215+
.downcast_ref::<ArrowEngineData>()
216+
.ok_or_else(|| Error::generic("CDF read returned non-Arrow EngineData"))?;
217+
out.push(arrow.record_batch().clone());
218+
}
219+
Ok(out)
220+
}
221+
188222
/// One Delta data file's worth of per-file inputs the driver ships to the executor: enough to
189223
/// read + DV-filter the file via kernel without reconstructing the `Snapshot`.
190224
#[derive(Debug, Clone)]
@@ -274,6 +308,12 @@ pub struct DeltaKernelScanExec {
274308
/// the wrapping exec appends synthetics and this exec only reads data (+ partitions) and applies
275309
/// the DV per `apply_dv`.
276310
synthesize: bool,
311+
/// Change Data Feed read: `Some((start, end))` reads table changes via kernel's
312+
/// `TableChanges::execute` over the inclusive version range (end `None` = latest), single
313+
/// partition, emitting data + `_change_type` / `_commit_version` / `_commit_timestamp`. `None` =
314+
/// normal scan. When set, `files` is empty and the kernel data-column schemas are unused (kernel
315+
/// enumerates + reads the whole range itself).
316+
cdf: Option<(u64, Option<u64>)>,
277317
plan_properties: Arc<PlanProperties>,
278318
metrics: ExecutionPlanMetricsSet,
279319
}
@@ -293,6 +333,7 @@ impl DeltaKernelScanExec {
293333
storage_config: DeltaStorageConfig,
294334
files: Vec<KernelScanFile>,
295335
synthesize: bool,
336+
cdf: Option<(u64, Option<u64>)>,
296337
) -> Self {
297338
// Single DataFusion partition: the Spark side (CometDeltaNativeScanExec) already splits
298339
// files across Spark partitions and injects each partition's file subset, consuming one
@@ -317,6 +358,7 @@ impl DeltaKernelScanExec {
317358
storage_config,
318359
files,
319360
synthesize,
361+
cdf,
320362
plan_properties,
321363
metrics: ExecutionPlanMetricsSet::new(),
322364
}
@@ -329,6 +371,9 @@ impl DeltaKernelScanExec {
329371
/// the wrapping DeltaSyntheticColumnsExec computes row positions against a single file. Eager
330372
/// (collects into a `Vec`); the blocking kernel reads run on the calling task thread.
331373
fn read_all(&self) -> DFResult<Vec<arrow::array::RecordBatch>> {
374+
if let Some((start, end)) = self.cdf {
375+
return self.read_all_cdf(start, end);
376+
}
332377
if self.synthesize {
333378
return self.read_all_synthesize();
334379
}
@@ -459,6 +504,33 @@ impl DeltaKernelScanExec {
459504
Ok(out)
460505
}
461506

507+
/// Change Data Feed read (single partition): read the whole version range via kernel's
508+
/// `TableChanges::execute`, then reconcile each batch to `output_schema` BY NAME (kernel emits
509+
/// data + partitions + `_change_type` / `_commit_version` / `_commit_timestamp`; the by-name
510+
/// assembler places them in Spark's expected order).
511+
fn read_all_cdf(
512+
&self,
513+
start_version: u64,
514+
end_version: Option<u64>,
515+
) -> DFResult<Vec<arrow::array::RecordBatch>> {
516+
let table_root_url = normalize_table_root(&self.table_root)
517+
.map_err(|e| DataFusionError::Execution(format!("kernel CDF table root: {e}")))?;
518+
let engine = get_or_create_engine(&table_root_url, &self.storage_config)
519+
.map_err(|e| DataFusionError::Execution(format!("kernel CDF engine: {e}")))?;
520+
let engine_dyn: Arc<dyn Engine> = engine;
521+
let batches = read_cdf_via_kernel(engine_dyn, &table_root_url, start_version, end_version)
522+
.map_err(|e| {
523+
DataFusionError::Execution(format!("kernel CDF read for {}: {e}", self.table_root))
524+
})?;
525+
let mut out = Vec::with_capacity(batches.len());
526+
for batch in batches {
527+
// Reuse the by-name assembler: CDF output carries data + partitions + the 3 CDF columns,
528+
// all produced by kernel, so no partition fallback is exercised.
529+
out.push(self.assemble_output_batch(batch, &[])?);
530+
}
531+
Ok(out)
532+
}
533+
462534
/// Row count for a file with no data columns to read: Delta's `numRecords` stat when present,
463535
/// else the parquet footer's row count (always available, one metadata fetch via the storage
464536
/// handler). Used only by the zero-data-column path.
@@ -1096,6 +1168,7 @@ mod tests {
10961168
DeltaStorageConfig::default(),
10971169
files,
10981170
false, // legacy path (no in-worker synthesis)
1171+
None, // not a CDF read
10991172
);
11001173

11011174
let ctx = Arc::new(TaskContext::default());
@@ -1178,6 +1251,7 @@ mod tests {
11781251
DeltaStorageConfig::default(),
11791252
files,
11801253
false, // legacy path (no in-worker synthesis)
1254+
None, // not a CDF read
11811255
);
11821256
let stream = exec.execute(0, Arc::new(TaskContext::default())).unwrap();
11831257
let batches = datafusion::physical_plan::common::collect(stream)

‎native/core/src/execution/planner/delta_scan.rs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,17 @@ fn plan_delta_kernel_scan(
106106
// `record_count`). Otherwise the kernel schemas are required -- their absence is a driver bug,
107107
// not something to paper over by re-deriving them (which is what the old `physicalise_field`
108108
// path did, getting nested column mapping wrong).
109+
// Change Data Feed read: the executor reconstructs TableChanges + execute()s the whole version
110+
// range itself (single partition), so it ships no per-file tasks and uses no kernel data-column
111+
// schemas -- bypass the schema-selection below.
112+
let cdf: Option<(u64, Option<u64>)> = if common.cdf_read {
113+
Some((common.cdf_start_version, common.cdf_end_version))
114+
} else {
115+
None
116+
};
117+
109118
let (physical_schema, read_logical_schema, needs_transform): (SchemaRef, SchemaRef, bool) =
110-
if data_fields.is_empty() {
119+
if cdf.is_some() || data_fields.is_empty() {
111120
let empty: SchemaRef = Arc::new(Schema::empty());
112121
(Arc::clone(&empty), empty, false)
113122
} else if !common.kernel_physical_schema.is_empty() {
@@ -212,6 +221,7 @@ fn plan_delta_kernel_scan(
212221
storage_config.clone(),
213222
files,
214223
synthesize,
224+
cdf,
215225
));
216226

217227
let scan_exec: Arc<dyn datafusion::physical_plan::ExecutionPlan> = if need_synthetics {

‎native/proto/src/proto/operator.proto‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,17 @@ message DeltaScanCommon {
388388
// synthesis inputs ride on DeltaScanTask (base_row_id / default_row_commit_version /
389389
// modification_time / byte range / dv).
390390
bool synthesize_in_worker = 28;
391+
// Change Data Feed (CDF / readChangeFeed): when true, DeltaKernelScanExec reads table changes via
392+
// delta-kernel-rs's `TableChanges` API (single partition reconstructs TableChanges(start, end) and
393+
// calls `execute()`), emitting the data columns plus `_change_type` / `_commit_version` /
394+
// `_commit_timestamp`. The version range is below; `required_schema` is the full CDF output
395+
// (assembled by name). No per-file tasks -- the worker enumerates + reads the whole range itself
396+
// (kernel's CDF per-file API is pub(crate); only execute() is public). Default false.
397+
bool cdf_read = 29;
398+
// Inclusive start version of the CDF range.
399+
uint64 cdf_start_version = 30;
400+
// Inclusive end version of the CDF range. `has_cdf_end_version` distinguishes "latest" (unset).
401+
optional uint64 cdf_end_version = 31;
391402
}
392403

393404
// Operator payload for a Delta scan.

0 commit comments

Comments
 (0)