@@ -186,9 +186,17 @@ pub struct KernelScanFile {
186186 /// Deletion-vector descriptor (absent = no DV). Decoded executor-side via `dv_reader`.
187187 pub dv : Option < DeltaDvDescriptor > ,
188188 /// Partition values for this file: `(column name, value)` where value is `None` for a NULL
189- /// partition. Empty for unpartitioned tables. These are injected as constant columns after
190- /// the parquet read (partition columns aren't stored in the data files).
189+ /// partition. Empty for unpartitioned tables. Used only by the legacy `append_partition_columns`
190+ /// path (when `transform_json` is empty); when kernel's transform is shipped it bakes partition
191+ /// values in as literals, so this is redundant.
191192 pub partition_values : Vec < ( String , Option < String > ) > ,
193+ /// Kernel's fully-resolved physical->logical transform for this file (serde JSON of
194+ /// `delta_kernel::expressions::Expression`), or empty for no transform. When non-empty the
195+ /// executor applies it via `transform_to_logical` -- partition injection / column-mapping
196+ /// relabel / row-tracking all come from kernel -- and the read targets the kernel LOGICAL schema
197+ /// (data + partitions), reconciled to `output_schema` by name. Empty falls back to the legacy
198+ /// identity-transform + `append_partition_columns` path.
199+ pub transform_json : Vec < u8 > ,
192200}
193201
194202/// Iceberg-style "kernel reads" scan operator. Reads each Delta data file through
@@ -315,7 +323,7 @@ impl DeltaKernelScanExec {
315323 & opts,
316324 )
317325 . map_err ( DataFusionError :: from) ?;
318- out. push ( self . append_partition_columns ( batch, & file. partition_values ) ?) ;
326+ out. push ( self . assemble_output_batch ( batch, & file. partition_values ) ?) ;
319327 }
320328 return Ok ( out) ;
321329 }
@@ -366,23 +374,46 @@ impl DeltaKernelScanExec {
366374 _ => None ,
367375 } ;
368376
377+ // Kernel's own fully-resolved transform for this file (partition literals + baseRowId for
378+ // row_id baked in) takes precedence; the executor applies it via `transform_to_logical`
379+ // so partition injection / column-mapping relabel / row-tracking all come from kernel.
380+ // Empty falls back to the shared identity transform + Comet-side `append_partition_columns`
381+ // (legacy path, kept until the driver ships transforms for every read).
382+ let use_shipped_transform = !file. transform_json . is_empty ( ) ;
383+ let file_transform: Option < ExpressionRef > = if use_shipped_transform {
384+ Some ( Arc :: new (
385+ serde_json:: from_slice :: < delta_kernel:: expressions:: Expression > (
386+ & file. transform_json ,
387+ )
388+ . map_err ( |e| {
389+ DataFusionError :: Execution ( format ! (
390+ "kernel scan: deserialize transform for {}: {e}" ,
391+ file. path
392+ ) )
393+ } ) ?,
394+ ) )
395+ } else {
396+ transform. clone ( )
397+ } ;
398+
369399 let kernel_batches = read_file_via_kernel (
370400 engine. as_ref ( ) ,
371401 & table_root_url,
372402 & file. path ,
373403 file. size ,
374404 selection_vector,
375- transform . clone ( ) ,
405+ file_transform ,
376406 physical_schema. clone ( ) ,
377407 logical_schema. clone ( ) ,
378408 )
379409 . map_err ( |e| map_file_read_error ( e, & file. path ) ) ?;
380410
381411 for batch in & kernel_batches {
382412 // Kernel pins the same arrow version as Comet (arrow-58), so the kernel batch IS a
383- // Comet batch -- no bridge.
384- let data_batch = batch. clone ( ) ;
385- out. push ( self . append_partition_columns ( data_batch, & file. partition_values ) ?) ;
413+ // Comet batch -- no bridge. Assemble the `output_schema` by name: data (+ partitions
414+ // when the shipped transform injected them) come from the kernel batch; any partition
415+ // column the transform did NOT inject is appended as a constant.
416+ out. push ( self . assemble_output_batch ( batch. clone ( ) , & file. partition_values ) ?) ;
386417 }
387418 }
388419 Ok ( out)
@@ -418,75 +449,88 @@ impl DeltaKernelScanExec {
418449 Ok ( meta. metadata ( ) . file_metadata ( ) . num_rows ( ) . max ( 0 ) as usize )
419450 }
420451
421- /// Append partition columns as constants (partition values are stored in the Add action, not
422- /// the data file). Produces a batch matching `output_schema` (data ++ partition). No-op for
423- /// unpartitioned tables.
424- fn append_partition_columns (
452+ /// Assemble the final `output_schema` batch (data ++ partition, Spark order) from a kernel-read
453+ /// batch, BY NAME. For each output field:
454+ /// - present in the kernel batch (data column, or a partition column the shipped transform
455+ /// already injected) -> take it, casting to the output field's EXACT type incl. NESTED field
456+ /// names (list element, struct fields, map key/value -- kernel names nested fields by Spark
457+ /// convention, e.g. a list element "element", but `output_schema` from the proto may carry
458+ /// empty/different nested names; the cast is a metadata-only relabel, a no-op when types match);
459+ /// - otherwise a partition column (legacy path: transform didn't inject it) -> a constant from
460+ /// `partition_values` (typed NULL when absent / explicitly NULL);
461+ /// - otherwise -> error (a data column kernel was asked for but didn't produce).
462+ ///
463+ /// This unifies the old `append_partition_columns` (legacy identity-transform path) and the
464+ /// shipped-transform path: partitions come from kernel when the transform injects them, else from
465+ /// the Add action's values.
466+ fn assemble_output_batch (
425467 & self ,
426- data_batch : arrow:: array:: RecordBatch ,
468+ kernel_batch : arrow:: array:: RecordBatch ,
427469 partition_values : & [ ( String , Option < String > ) ] ,
428470 ) -> DFResult < arrow:: array:: RecordBatch > {
429- let num_rows = data_batch. num_rows ( ) ;
430- let mut columns: Vec < arrow:: array:: ArrayRef > = data_batch. columns ( ) . to_vec ( ) ;
431- if !self . partition_schema . fields ( ) . is_empty ( ) {
432- let parsed_tz = SessionTimezone :: parse ( & self . session_timezone ) ;
433- for pf in self . partition_schema . fields ( ) {
471+ let num_rows = kernel_batch. num_rows ( ) ;
472+ let kernel_schema = kernel_batch. schema ( ) ;
473+ let parsed_tz = SessionTimezone :: parse ( & self . session_timezone ) ;
474+ let is_partition =
475+ |name : & str | self . partition_schema . fields ( ) . iter ( ) . any ( |f| f. name ( ) == name) ;
476+
477+ let mut columns: Vec < arrow:: array:: ArrayRef > =
478+ Vec :: with_capacity ( self . output_schema . fields ( ) . len ( ) ) ;
479+ for out_field in self . output_schema . fields ( ) {
480+ if let Some ( idx) = kernel_schema
481+ . fields ( )
482+ . iter ( )
483+ . position ( |f| f. name ( ) == out_field. name ( ) )
484+ {
485+ let col = kernel_batch. column ( idx) ;
486+ columns. push ( if col. data_type ( ) == out_field. data_type ( ) {
487+ Arc :: clone ( col)
488+ } else {
489+ arrow_cast ( col, out_field. data_type ( ) ) . map_err ( |e| {
490+ DataFusionError :: Execution ( format ! (
491+ "kernel scan: reconciling column '{}' to output schema: {e}" ,
492+ out_field. name( )
493+ ) )
494+ } ) ?
495+ } ) ;
496+ } else if is_partition ( out_field. name ( ) ) {
434497 let raw = partition_values
435498 . iter ( )
436- . find ( |( name, _) | name == pf . name ( ) )
499+ . find ( |( name, _) | name == out_field . name ( ) )
437500 . and_then ( |( _, v) | v. as_ref ( ) ) ;
438501 let scalar = match raw {
439502 Some ( v) => parse_delta_partition_scalar (
440503 v,
441- pf . data_type ( ) ,
504+ out_field . data_type ( ) ,
442505 & parsed_tz,
443506 & self . session_timezone ,
444507 )
445508 . map_err ( |e| {
446509 DataFusionError :: Execution ( format ! (
447510 "kernel scan partition value parse for '{}': {e}" ,
448- pf . name( )
511+ out_field . name( )
449512 ) )
450513 } ) ?,
451514 // Absent or explicitly-NULL partition value -> a typed NULL constant.
452- None => ScalarValue :: try_from ( pf . data_type ( ) ) ?,
515+ None => ScalarValue :: try_from ( out_field . data_type ( ) ) ?,
453516 } ;
454517 columns. push ( scalar. to_array_of_size ( num_rows) ?) ;
518+ } else {
519+ return Err ( DataFusionError :: Execution ( format ! (
520+ "kernel scan: output column '{}' was not produced by the kernel read and is \
521+ not a partition column",
522+ out_field. name( )
523+ ) ) ) ;
455524 }
456525 }
457- // Reconcile each column to `output_schema`'s EXACT field type, including NESTED field names
458- // (list element, struct fields, map key/value). The kernel-read batch names nested fields by
459- // Spark convention (e.g. a list element "element"), but `output_schema` -- derived from the
460- // proto `required_schema` -- can carry empty/different nested names. Without this, the emitted
461- // batch's schema != `output_schema` and DataFusion rejects it ("column types must match schema
462- // types" on a partitioned try_new, or an `assert_eq!` / coalesce panic for the unpartitioned
463- // pass-through). The data is already in logical (output) order from the kernel transform, so
464- // this is a metadata-only relabel via `arrow_cast` -- a no-op whenever the types already match.
465- let reconciled: Vec < arrow:: array:: ArrayRef > = columns
466- . iter ( )
467- . zip ( self . output_schema . fields ( ) )
468- . map ( |( col, field) | {
469- if col. data_type ( ) == field. data_type ( ) {
470- Ok ( Arc :: clone ( col) )
471- } else {
472- arrow_cast ( col, field. data_type ( ) ) . map_err ( |e| {
473- DataFusionError :: Execution ( format ! (
474- "kernel scan: reconciling column '{}' to output schema: {e}" ,
475- field. name( )
476- ) )
477- } )
478- }
479- } )
480- . collect :: < DFResult < Vec < _ > > > ( ) ?;
481526 // Carry the row count explicitly: a zero-data-column read (e.g. a MERGE UPDATE that only
482- // touches a struct field, or `count(*)`) yields an empty `reconciled ` with an empty
527+ // touches a struct field, or `count(*)`) yields an empty `columns ` with an empty
483528 // `output_schema`, and plain `try_new` cannot infer the row count from zero columns ("must
484- // either specify a row count or at least one column"). `num_rows` comes from the input batch
485- // (the zero-data path builds it via `with_row_count`).
529+ // either specify a row count or at least one column").
486530 let opts = arrow:: array:: RecordBatchOptions :: new ( ) . with_row_count ( Some ( num_rows) ) ;
487531 arrow:: array:: RecordBatch :: try_new_with_options (
488532 Arc :: clone ( & self . output_schema ) ,
489- reconciled ,
533+ columns ,
490534 & opts,
491535 )
492536 . map_err ( DataFusionError :: from)
@@ -711,6 +755,7 @@ mod tests {
711755 record_count: Some ( 7 ) ,
712756 dv: None ,
713757 partition_values: vec![ ] ,
758+ transform_json: Vec :: new( ) ,
714759 } ] ;
715760
716761 let exec = DeltaKernelScanExec :: new (
@@ -786,6 +831,7 @@ mod tests {
786831 record_count: Some ( 3 ) ,
787832 dv: None ,
788833 partition_values: vec![ ] ,
834+ transform_json: Vec :: new( ) ,
789835 } ] ;
790836
791837 let exec = DeltaKernelScanExec :: new (
@@ -905,4 +951,135 @@ mod tests {
905951 vec![ 100 , 200 , 300 ]
906952 ) ;
907953 }
954+
955+ // Builds a minimal single-file PARTITIONED Delta table in a tempdir: schema (id: long, part:
956+ // string) with partitionColumns=["part"], one add file in partition part="A" whose parquet
957+ // holds ONLY the data column `id` (partition columns aren't stored in the data files). Returns
958+ // (tempdir, table url, parquet size). Mirrors `build_single_file_table` but partitioned.
959+ fn build_partitioned_table ( ) -> ( tempfile:: TempDir , Url , i64 ) {
960+ let tmp = tempfile:: tempdir ( ) . unwrap ( ) ;
961+ let table_dir = tmp. path ( ) . join ( "pscan_table" ) ;
962+ let delta_log = table_dir. join ( "_delta_log" ) ;
963+ // Partition data lives under part=A/ by Delta convention.
964+ let part_dir = table_dir. join ( "part=A" ) ;
965+ std:: fs:: create_dir_all ( & delta_log) . unwrap ( ) ;
966+ std:: fs:: create_dir_all ( & part_dir) . unwrap ( ) ;
967+
968+ // Parquet holds only the data column `id` (0..=4).
969+ let arrow_schema = Arc :: new ( ArrowSchema :: new ( vec ! [ Field :: new(
970+ "id" ,
971+ ArrowDataType :: Int64 ,
972+ true ,
973+ ) ] ) ) ;
974+ let batch = RecordBatch :: try_new (
975+ arrow_schema. clone ( ) ,
976+ vec ! [ Arc :: new( Int64Array :: from( vec![ 0i64 , 1 , 2 , 3 , 4 ] ) ) ] ,
977+ )
978+ . unwrap ( ) ;
979+ let parquet_rel = "part=A/part-00000.parquet" ;
980+ let parquet_path = table_dir. join ( parquet_rel) ;
981+ let file = std:: fs:: File :: create ( & parquet_path) . unwrap ( ) ;
982+ let mut writer = ArrowWriter :: try_new ( file, arrow_schema, None ) . unwrap ( ) ;
983+ writer. write ( & batch) . unwrap ( ) ;
984+ writer. close ( ) . unwrap ( ) ;
985+ let size = std:: fs:: metadata ( & parquet_path) . unwrap ( ) . len ( ) as i64 ;
986+
987+ let commit0 = [
988+ r#"{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}"# . to_string ( ) ,
989+ r#"{"metaData":{"id":"pscan-id","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part"],"configuration":{},"createdTime":1700000000000}}"# . to_string ( ) ,
990+ format ! (
991+ r#"{{"add":{{"path":"{parquet_rel}","partitionValues":{{"part":"A"}},"size":{size},"modificationTime":1700000000000,"dataChange":true,"stats":"{{\"numRecords\":5}}"}}}}"#
992+ ) ,
993+ ]
994+ . join ( "\n " ) ;
995+ std:: fs:: write ( delta_log. join ( "00000000000000000000.json" ) , commit0) . unwrap ( ) ;
996+
997+ let url = normalize_url ( table_dir. to_str ( ) . unwrap ( ) ) . unwrap ( ) ;
998+ ( tmp, url, size)
999+ }
1000+
1001+ // The Stage-A mechanism end to end at the kernel level: build a partitioned table, get kernel's
1002+ // OWN per-file transform from the scan (the value the driver ships), round-trip it through serde
1003+ // JSON exactly as the driver->executor wire does, then apply it via `read_file_via_kernel` with
1004+ // kernel's physical/logical schemas. The partition column `part` must be INJECTED by the kernel
1005+ // transform (it isn't in the parquet) with value "A" -- i.e. partitions come from kernel, not
1006+ // Comet's `append_partition_columns`. This is the foundation the whole synthetic-columns rewrite
1007+ // builds on, so it guards the serde round-trip + transform application directly.
1008+ #[ test]
1009+ fn shipped_kernel_transform_injects_partition_column ( ) {
1010+ use delta_kernel:: arrow:: array:: StringArray ;
1011+ use delta_kernel:: expressions:: Expression ;
1012+
1013+ let ( _tmp, url, size) = build_partitioned_table ( ) ;
1014+ let engine = get_or_create_engine ( & url, & DeltaStorageConfig :: default ( ) ) . unwrap ( ) ;
1015+ let snapshot = Snapshot :: builder_for ( url. clone ( ) )
1016+ . build ( engine. as_ref ( ) )
1017+ . unwrap ( ) ;
1018+
1019+ // Project the scan to the FULL logical schema (data + partition), exactly as the Stage-A
1020+ // driver does so kernel emits a partition-injecting transform.
1021+ let scan = snapshot. scan_builder ( ) . build ( ) . unwrap ( ) ;
1022+ let physical_schema = scan. physical_schema ( ) . clone ( ) ;
1023+ let logical_schema = scan. logical_schema ( ) . clone ( ) ;
1024+ // Physical schema is data-only (`id`); logical includes the partition column (`part`).
1025+ assert_eq ! ( physical_schema. fields( ) . len( ) , 1 , "physical = data only" ) ;
1026+ assert_eq ! (
1027+ logical_schema. fields( ) . len( ) ,
1028+ 2 ,
1029+ "logical = data + partition"
1030+ ) ;
1031+
1032+ // Pull kernel's resolved per-file transform (the value shipped on DeltaScanTask).
1033+ let mut transforms: Vec < Option < ExpressionRef > > = Vec :: new ( ) ;
1034+ for meta in scan. scan_metadata ( engine. as_ref ( ) ) . unwrap ( ) {
1035+ let meta = meta. unwrap ( ) ;
1036+ for t in & meta. scan_file_transforms {
1037+ if t. is_some ( ) {
1038+ transforms. push ( t. clone ( ) ) ;
1039+ }
1040+ }
1041+ }
1042+ assert_eq ! ( transforms. len( ) , 1 , "one selected file" ) ;
1043+ let transform = transforms. into_iter ( ) . next ( ) . unwrap ( ) . unwrap ( ) ;
1044+
1045+ // Round-trip the transform through serde JSON exactly as driver -> executor does.
1046+ let json = serde_json:: to_vec ( transform. as_ref ( ) ) . unwrap ( ) ;
1047+ let restored: Expression = serde_json:: from_slice ( & json) . unwrap ( ) ;
1048+ assert_eq ! ( & * transform, & restored, "transform survives serde round-trip" ) ;
1049+
1050+ let batches = read_file_via_kernel (
1051+ engine. as_ref ( ) ,
1052+ & url,
1053+ "part=A/part-00000.parquet" ,
1054+ size,
1055+ None ,
1056+ Some ( Arc :: new ( restored) ) ,
1057+ physical_schema,
1058+ logical_schema,
1059+ )
1060+ . unwrap ( ) ;
1061+
1062+ // Output carries data + the kernel-injected partition column.
1063+ let schema = batches[ 0 ] . schema ( ) ;
1064+ assert_eq ! ( schema. field( 0 ) . name( ) , "id" ) ;
1065+ assert_eq ! ( schema. field( 1 ) . name( ) , "part" ) ;
1066+ let ids = batches[ 0 ]
1067+ . column ( 0 )
1068+ . as_any ( )
1069+ . downcast_ref :: < Int64Array > ( )
1070+ . expect ( "id Int64" ) ;
1071+ assert_eq ! (
1072+ ( 0 ..ids. len( ) ) . map( |i| ids. value( i) ) . collect:: <Vec <_>>( ) ,
1073+ vec![ 0 , 1 , 2 , 3 , 4 ]
1074+ ) ;
1075+ let parts = batches[ 0 ]
1076+ . column ( 1 )
1077+ . as_any ( )
1078+ . downcast_ref :: < StringArray > ( )
1079+ . expect ( "kernel transform should inject the `part` partition column as Utf8" ) ;
1080+ assert ! (
1081+ ( 0 ..parts. len( ) ) . all( |i| parts. value( i) == "A" ) ,
1082+ "partition column injected by kernel transform must equal the file's partition value"
1083+ ) ;
1084+ }
9081085}
0 commit comments