@@ -66,6 +66,20 @@ use crate::engine::{get_or_create_engine, DeltaStorageConfig};
6666use crate :: planner:: { parse_delta_partition_scalar, SessionTimezone } ;
6767use crate :: proto:: DeltaDvDescriptor ;
6868
69+ // Engine-side synthetic / Spark `_metadata.*` column names produced in `synthesize` mode. row_index
70+ // and row_id are NOT here -- they come from kernel metadata columns in the read. These mirror
71+ // `DeltaSyntheticColumnsExec` so the two paths emit identical values (kernel pins arrow-58 = Comet's).
72+ const IS_ROW_DELETED_NAME : & str = "__delta_internal_is_row_deleted" ;
73+ const ROW_COMMIT_VERSION_NAME : & str = "row_commit_version" ;
74+ const META_FILE_PATH : & str = "file_path" ;
75+ const META_FILE_NAME : & str = "file_name" ;
76+ const META_FILE_SIZE : & str = "file_size" ;
77+ const META_FILE_BLOCK_START : & str = "file_block_start" ;
78+ const META_FILE_BLOCK_LENGTH : & str = "file_block_length" ;
79+ const META_FILE_MODIFICATION_TIME : & str = "file_modification_time" ;
80+ const META_BASE_ROW_ID : & str = "base_row_id" ;
81+ const META_DEFAULT_ROW_COMMIT_VERSION : & str = "default_row_commit_version" ;
82+
6983/// Convert a Comet arrow `Schema` into a kernel `StructType`. Since delta-kernel pins the same arrow
7084/// version as Comet (arrow-58), this is kernel's own `try_from_arrow` directly -- no cross-version
7185/// FFI bridge. `try_from_arrow` DOES preserve `PARQUET:field_id` (and nested column-mapping ids)
@@ -197,6 +211,20 @@ pub struct KernelScanFile {
197211 /// (data + partitions), reconciled to `output_schema` by name. Empty falls back to the legacy
198212 /// identity-transform + `append_partition_columns` path.
199213 pub transform_json : Vec < u8 > ,
214+ /// `AddFile.baseRowId` for row-tracking tables (`None` otherwise). Per-file constant surfaced as
215+ /// `_metadata.base_row_id`. (row_id itself comes from kernel's RowId metadata column.)
216+ pub base_row_id : Option < i64 > ,
217+ /// `AddFile.defaultRowCommitVersion` for row-tracking tables (`None` otherwise). Per-file
218+ /// constant surfaced as `row_commit_version` / `_metadata.default_row_commit_version` (kernel has
219+ /// no RowCommitVersion metadata-column support, so this stays engine-side).
220+ pub default_row_commit_version : Option < i64 > ,
221+ /// File modification time, epoch millis (`AddFile.modificationTime`). Surfaced as
222+ /// `_metadata.file_modification_time` (converted to micros).
223+ pub modification_time : i64 ,
224+ /// File-split byte range (`None`/`None` = whole file). Surfaces `_metadata.file_block_start` /
225+ /// `file_block_length`.
226+ pub byte_range_start : Option < i64 > ,
227+ pub byte_range_end : Option < i64 > ,
200228}
201229
202230/// Iceberg-style "kernel reads" scan operator. Reads each Delta data file through
@@ -239,6 +267,13 @@ pub struct DeltaKernelScanExec {
239267 storage_config : DeltaStorageConfig ,
240268 /// Files this scan reads (whole-table, single partition for now).
241269 files : Vec < KernelScanFile > ,
270+ /// Stage B/C: when true, this exec produces ALL synthetic columns itself (row_index / row_id via
271+ /// kernel metadata columns carried in the read; is_row_deleted by inverting the decoded DV;
272+ /// row_commit_version + Spark `_metadata.*` as per-file constants) and assembles the full
273+ /// `output_schema` by name -- no DeltaSyntheticColumnsExec wraps it. When false, the legacy path:
274+ /// the wrapping exec appends synthetics and this exec only reads data (+ partitions) and applies
275+ /// the DV per `apply_dv`.
276+ synthesize : bool ,
242277 plan_properties : Arc < PlanProperties > ,
243278 metrics : ExecutionPlanMetricsSet ,
244279}
@@ -257,6 +292,7 @@ impl DeltaKernelScanExec {
257292 table_root : String ,
258293 storage_config : DeltaStorageConfig ,
259294 files : Vec < KernelScanFile > ,
295+ synthesize : bool ,
260296 ) -> Self {
261297 // Single DataFusion partition: the Spark side (CometDeltaNativeScanExec) already splits
262298 // files across Spark partitions and injects each partition's file subset, consuming one
@@ -280,6 +316,7 @@ impl DeltaKernelScanExec {
280316 table_root,
281317 storage_config,
282318 files,
319+ synthesize,
283320 plan_properties,
284321 metrics : ExecutionPlanMetricsSet :: new ( ) ,
285322 }
@@ -292,6 +329,9 @@ impl DeltaKernelScanExec {
292329 /// the wrapping DeltaSyntheticColumnsExec computes row positions against a single file. Eager
293330 /// (collects into a `Vec`); the blocking kernel reads run on the calling task thread.
294331 fn read_all ( & self ) -> DFResult < Vec < arrow:: array:: RecordBatch > > {
332+ if self . synthesize {
333+ return self . read_all_synthesize ( ) ;
334+ }
295335 let table_root_url = normalize_table_root ( & self . table_root )
296336 . map_err ( |e| DataFusionError :: Execution ( format ! ( "kernel scan table root: {e}" ) ) ) ?;
297337 let engine = get_or_create_engine ( & table_root_url, & self . storage_config )
@@ -535,6 +575,250 @@ impl DeltaKernelScanExec {
535575 )
536576 . map_err ( DataFusionError :: from)
537577 }
578+
579+ /// Stage B/C synthesis path. Reads each file via kernel (row_index / row_id arrive as kernel
580+ /// metadata columns carried in `physical_schema`/`read_logical_schema` + the per-file transform),
581+ /// then produces the engine-side synthetics -- `is_row_deleted` by inverting the decoded DV,
582+ /// `row_commit_version` + Spark `_metadata.*` as per-file constants -- dropping DV rows when
583+ /// `is_row_deleted` isn't requested, and assembles the FULL `output_schema` by name. Replaces the
584+ /// stacked DeltaSyntheticColumnsExec (no `final_output_indices` needed -- by-name handles order).
585+ fn read_all_synthesize ( & self ) -> DFResult < Vec < arrow:: array:: RecordBatch > > {
586+ use arrow:: array:: Int8Array ;
587+ let table_root_url = normalize_table_root ( & self . table_root )
588+ . map_err ( |e| DataFusionError :: Execution ( format ! ( "kernel scan table root: {e}" ) ) ) ?;
589+ let engine = get_or_create_engine ( & table_root_url, & self . storage_config )
590+ . map_err ( |e| DataFusionError :: Execution ( format ! ( "kernel scan engine: {e}" ) ) ) ?;
591+ let logical_schema = arrow_to_kernel_schema ( & self . read_logical_schema )
592+ . map_err ( |e| DataFusionError :: Execution ( format ! ( "kernel scan logical schema: {e}" ) ) ) ?;
593+ let physical_schema = arrow_to_kernel_schema ( & self . physical_schema )
594+ . map_err ( |e| DataFusionError :: Execution ( format ! ( "kernel scan physical schema: {e}" ) ) ) ?;
595+
596+ // Does the query want `is_row_deleted` (keep all rows, flag them) vs a plain drop of DV rows?
597+ let emit_is_row_deleted = self
598+ . output_schema
599+ . fields ( )
600+ . iter ( )
601+ . any ( |f| f. name ( ) == IS_ROW_DELETED_NAME ) ;
602+
603+ let mut out: Vec < arrow:: array:: RecordBatch > = Vec :: new ( ) ;
604+ for file in & self . files {
605+ // Decode the DV once per file (sorted physical row indexes), only when we flag or drop.
606+ let mut deleted: Vec < u64 > = Vec :: new ( ) ;
607+ if let Some ( desc) = & file. dv {
608+ deleted = read_dv_indexes ( desc, & table_root_url, & self . storage_config )
609+ . map_err ( |e| map_dv_error_to_datafusion ( e, desc) ) ?;
610+ deleted. sort_unstable ( ) ;
611+ }
612+ let has_dv = !deleted. is_empty ( ) ;
613+ let drop_deleted = has_dv && !emit_is_row_deleted;
614+
615+ let transform: Option < ExpressionRef > = if file. transform_json . is_empty ( ) {
616+ None
617+ } else {
618+ Some ( Arc :: new (
619+ serde_json:: from_slice :: < delta_kernel:: expressions:: Expression > (
620+ & file. transform_json ,
621+ )
622+ . map_err ( |e| {
623+ DataFusionError :: Execution ( format ! (
624+ "kernel scan: deserialize transform for {}: {e}" ,
625+ file. path
626+ ) )
627+ } ) ?,
628+ ) )
629+ } ;
630+
631+ // Read ALL physical rows (no kernel DV drop) so row_index stays physical and we control
632+ // flag-vs-drop ourselves -- surviving rows keep their physical row_index after a drop.
633+ let kernel_batches = read_file_via_kernel (
634+ engine. as_ref ( ) ,
635+ & table_root_url,
636+ & file. path ,
637+ file. size ,
638+ None ,
639+ transform,
640+ physical_schema. clone ( ) ,
641+ logical_schema. clone ( ) ,
642+ )
643+ . map_err ( |e| map_file_read_error ( e, & file. path ) ) ?;
644+
645+ let mut file_offset: i64 = 0 ;
646+ let mut del_ptr: usize = 0 ;
647+ for kb in & kernel_batches {
648+ let len = kb. num_rows ( ) ;
649+ let batch_end = file_offset + len as i64 ;
650+ // is_row_deleted (Int8) over physical rows [file_offset, batch_end) via a single
651+ // forward pass through the sorted DV indexes.
652+ let is_deleted_arr: Option < Int8Array > =
653+ if has_dv && ( emit_is_row_deleted || drop_deleted) {
654+ while del_ptr < deleted. len ( ) && ( deleted[ del_ptr] as i64 ) < file_offset {
655+ del_ptr += 1 ;
656+ }
657+ let mut flags = vec ! [ 0i8 ; len] ;
658+ let mut p = del_ptr;
659+ while p < deleted. len ( ) && ( deleted[ p] as i64 ) < batch_end {
660+ flags[ ( deleted[ p] as i64 - file_offset) as usize ] = 1 ;
661+ p += 1 ;
662+ }
663+ Some ( Int8Array :: from ( flags) )
664+ } else {
665+ None
666+ } ;
667+
668+ let assembled = self . assemble_synthesized (
669+ kb. clone ( ) ,
670+ file,
671+ if emit_is_row_deleted {
672+ is_deleted_arr. clone ( )
673+ } else {
674+ None
675+ } ,
676+ ) ?;
677+
678+ let final_batch = if drop_deleted {
679+ let keep: Vec < bool > = match & is_deleted_arr {
680+ Some ( a) => ( 0 ..len) . map ( |i| a. value ( i) == 0 ) . collect ( ) ,
681+ None => vec ! [ true ; len] ,
682+ } ;
683+ arrow:: compute:: filter_record_batch ( & assembled, & arrow:: array:: BooleanArray :: from ( keep) )
684+ . map_err ( DataFusionError :: from) ?
685+ } else {
686+ assembled
687+ } ;
688+ out. push ( final_batch) ;
689+ file_offset = batch_end;
690+ }
691+ }
692+ Ok ( out)
693+ }
694+
695+ /// Assemble the FULL `output_schema` by name from a kernel-read batch plus the engine-side
696+ /// synthetics. For each output field: take it from the kernel batch if present (data, partitions,
697+ /// row_index, row_id); else build the known synthetic (`is_row_deleted`, `row_commit_version`,
698+ /// Spark `_metadata.*` per-file constants); else a partition constant; else error.
699+ fn assemble_synthesized (
700+ & self ,
701+ kernel_batch : arrow:: array:: RecordBatch ,
702+ file : & KernelScanFile ,
703+ is_deleted : Option < arrow:: array:: Int8Array > ,
704+ ) -> DFResult < arrow:: array:: RecordBatch > {
705+ use arrow:: array:: { Int64Array , StringArray , TimestampMicrosecondArray } ;
706+ let num_rows = kernel_batch. num_rows ( ) ;
707+ let kernel_schema = kernel_batch. schema ( ) ;
708+ let parsed_tz = SessionTimezone :: parse ( & self . session_timezone ) ;
709+ let cast_to = |arr : arrow:: array:: ArrayRef ,
710+ field : & arrow:: datatypes:: Field |
711+ -> DFResult < arrow:: array:: ArrayRef > {
712+ if arr. data_type ( ) == field. data_type ( ) {
713+ Ok ( arr)
714+ } else {
715+ arrow_cast ( & arr, field. data_type ( ) ) . map_err ( |e| {
716+ DataFusionError :: Execution ( format ! (
717+ "kernel scan (synthesize): reconciling column '{}': {e}" ,
718+ field. name( )
719+ ) )
720+ } )
721+ }
722+ } ;
723+
724+ let mut columns: Vec < arrow:: array:: ArrayRef > =
725+ Vec :: with_capacity ( self . output_schema . fields ( ) . len ( ) ) ;
726+ for out_field in self . output_schema . fields ( ) {
727+ let name = out_field. name ( ) ;
728+ // 1. produced by the kernel read (data, partitions, row_index, row_id).
729+ if let Some ( idx) = kernel_schema. fields ( ) . iter ( ) . position ( |f| f. name ( ) == name) {
730+ columns. push ( cast_to ( Arc :: clone ( kernel_batch. column ( idx) ) , out_field) ?) ;
731+ continue ;
732+ }
733+ // 2. engine-side synthetics + Spark `_metadata.*` per-file constants.
734+ let lc = name. to_ascii_lowercase ( ) ;
735+ let arr: arrow:: array:: ArrayRef = match lc. as_str ( ) {
736+ IS_ROW_DELETED_NAME => match & is_deleted {
737+ Some ( a) => Arc :: new ( a. clone ( ) ) ,
738+ None => Arc :: new ( arrow:: array:: Int8Array :: from ( vec ! [ 0i8 ; num_rows] ) ) ,
739+ } ,
740+ ROW_COMMIT_VERSION_NAME | META_DEFAULT_ROW_COMMIT_VERSION => {
741+ match file. default_row_commit_version {
742+ Some ( v) => Arc :: new ( Int64Array :: from ( vec ! [ Some ( v) ; num_rows] ) ) ,
743+ None => Arc :: new ( Int64Array :: from ( vec ! [ None as Option <i64 >; num_rows] ) ) ,
744+ }
745+ }
746+ META_FILE_PATH => Arc :: new ( StringArray :: from ( vec ! [ file. path. clone( ) ; num_rows] ) ) ,
747+ META_FILE_NAME => {
748+ let base = match file. path . rfind ( '/' ) {
749+ Some ( i) => file. path [ i + 1 ..] . to_string ( ) ,
750+ None => file. path . clone ( ) ,
751+ } ;
752+ Arc :: new ( StringArray :: from ( vec ! [ base; num_rows] ) )
753+ }
754+ META_FILE_SIZE => Arc :: new ( Int64Array :: from ( vec ! [ file. size; num_rows] ) ) ,
755+ META_FILE_BLOCK_START => {
756+ Arc :: new ( Int64Array :: from ( vec ! [ file. byte_range_start. unwrap_or( 0 ) ; num_rows] ) )
757+ }
758+ META_FILE_BLOCK_LENGTH => {
759+ let v = match ( file. byte_range_start , file. byte_range_end ) {
760+ ( Some ( s) , Some ( e) ) => e - s,
761+ _ => file. size ,
762+ } ;
763+ Arc :: new ( Int64Array :: from ( vec ! [ v; num_rows] ) )
764+ }
765+ META_FILE_MODIFICATION_TIME => {
766+ let micros = file. modification_time . saturating_mul ( 1000 ) ;
767+ Arc :: new (
768+ TimestampMicrosecondArray :: from ( vec ! [ micros; num_rows] ) . with_timezone ( "UTC" ) ,
769+ )
770+ }
771+ META_BASE_ROW_ID => {
772+ Arc :: new ( Int64Array :: from ( vec ! [ file. base_row_id. unwrap_or( 0 ) ; num_rows] ) )
773+ }
774+ other
775+ if other. starts_with ( "_row-id-col-" )
776+ || other. starts_with ( "_row-commit-version-col-" ) =>
777+ {
778+ // Materialised row-tracking column absent from this file -> all nulls (kernel's
779+ // RowId metadata column already surfaced a present one as a data column above).
780+ Arc :: new ( Int64Array :: from ( vec ! [ None as Option <i64 >; num_rows] ) )
781+ }
782+ _ if self . partition_schema . fields ( ) . iter ( ) . any ( |f| f. name ( ) == name) => {
783+ // Partition column the transform didn't inject -> constant from the Add action.
784+ let raw = file
785+ . partition_values
786+ . iter ( )
787+ . find ( |( n, _) | n == name)
788+ . and_then ( |( _, v) | v. as_ref ( ) ) ;
789+ let scalar = match raw {
790+ Some ( v) => parse_delta_partition_scalar (
791+ v,
792+ out_field. data_type ( ) ,
793+ & parsed_tz,
794+ & self . session_timezone ,
795+ )
796+ . map_err ( |e| {
797+ DataFusionError :: Execution ( format ! (
798+ "kernel scan (synthesize) partition '{name}': {e}"
799+ ) )
800+ } ) ?,
801+ None => ScalarValue :: try_from ( out_field. data_type ( ) ) ?,
802+ } ;
803+ scalar. to_array_of_size ( num_rows) ?
804+ }
805+ _ => {
806+ return Err ( DataFusionError :: Execution ( format ! (
807+ "kernel scan (synthesize): output column '{name}' was not produced by the \
808+ kernel read and is not a known synthetic / partition column"
809+ ) ) ) ;
810+ }
811+ } ;
812+ columns. push ( cast_to ( arr, out_field) ?) ;
813+ }
814+ let opts = arrow:: array:: RecordBatchOptions :: new ( ) . with_row_count ( Some ( num_rows) ) ;
815+ arrow:: array:: RecordBatch :: try_new_with_options (
816+ Arc :: clone ( & self . output_schema ) ,
817+ columns,
818+ & opts,
819+ )
820+ . map_err ( DataFusionError :: from)
821+ }
538822}
539823
540824impl fmt:: Debug for DeltaKernelScanExec {
@@ -756,6 +1040,11 @@ mod tests {
7561040 dv: None ,
7571041 partition_values: vec![ ] ,
7581042 transform_json: Vec :: new( ) ,
1043+ base_row_id: None ,
1044+ default_row_commit_version: None ,
1045+ modification_time: 0 ,
1046+ byte_range_start: None ,
1047+ byte_range_end: None ,
7591048 } ] ;
7601049
7611050 let exec = DeltaKernelScanExec :: new (
@@ -769,6 +1058,7 @@ mod tests {
7691058 url. as_str ( ) . to_string ( ) ,
7701059 DeltaStorageConfig :: default ( ) ,
7711060 files,
1061+ false , // legacy path (no in-worker synthesis)
7721062 ) ;
7731063
7741064 let ctx = Arc :: new ( TaskContext :: default ( ) ) ;
@@ -832,6 +1122,11 @@ mod tests {
8321122 dv: None ,
8331123 partition_values: vec![ ] ,
8341124 transform_json: Vec :: new( ) ,
1125+ base_row_id: None ,
1126+ default_row_commit_version: None ,
1127+ modification_time: 0 ,
1128+ byte_range_start: None ,
1129+ byte_range_end: None ,
8351130 } ] ;
8361131
8371132 let exec = DeltaKernelScanExec :: new (
@@ -845,6 +1140,7 @@ mod tests {
8451140 url. as_str ( ) . to_string ( ) ,
8461141 DeltaStorageConfig :: default ( ) ,
8471142 files,
1143+ false , // legacy path (no in-worker synthesis)
8481144 ) ;
8491145 let stream = exec. execute ( 0 , Arc :: new ( TaskContext :: default ( ) ) ) . unwrap ( ) ;
8501146 let batches = datafusion:: physical_plan:: common:: collect ( stream)
0 commit comments