@@ -39,14 +39,14 @@ use crate::file_index::file_index_result::FileIndexResult;
3939use crate :: io:: FileIO ;
4040use crate :: spec:: {
4141 is_variant_extraction_row, BigIntType , CoreOptions , DataField , DataFileMeta ,
42- DataType as PaimonDataType , MergeEngine , PartialUpdateConfig , Predicate , TinyIntType ,
42+ DataType as PaimonDataType , MergeEngine , PartialUpdateConfig , Predicate , RowKind , TinyIntType ,
4343 KEY_FIELD_ID_START , SEQUENCE_NUMBER_FIELD_ID , SEQUENCE_NUMBER_FIELD_NAME , VALUE_KIND_FIELD_ID ,
4444 VALUE_KIND_FIELD_NAME ,
4545} ;
4646use crate :: table:: schema_manager:: SchemaManager ;
4747use crate :: table:: ArrowRecordBatchStream ;
4848use crate :: { DataSplit , DeletionFile , Error } ;
49- use arrow_array:: { RecordBatch , RecordBatchOptions } ;
49+ use arrow_array:: { Array , BooleanArray , Int8Array , RecordBatch , RecordBatchOptions } ;
5050
5151use async_stream:: try_stream;
5252use futures:: StreamExt ;
@@ -184,6 +184,43 @@ fn ensure_merge_input_limit(input_stream_count: usize, limit: Option<usize>) ->
184184 Ok ( ( ) )
185185}
186186
187+ /// Handles retract rows of an unmerged single run the way the merge path
188+ /// handles a lone version of a key: first-row without `ignore-delete` rejects
189+ /// them, otherwise they are omitted.
190+ fn drop_lone_retracts (
191+ batch : RecordBatch ,
192+ value_kind_index : usize ,
193+ reject : bool ,
194+ ) -> crate :: Result < RecordBatch > {
195+ // The merge cursor reads a missing or NULL `_VALUE_KIND` as INSERT.
196+ let Some ( kinds) = batch
197+ . column ( value_kind_index)
198+ . as_any ( )
199+ . downcast_ref :: < Int8Array > ( )
200+ else {
201+ return Ok ( batch) ;
202+ } ;
203+ if kinds. null_count ( ) == 0 && kinds. values ( ) . iter ( ) . all ( |& kind| kind == 0 || kind == 2 ) {
204+ return Ok ( batch) ;
205+ }
206+ let mut keep = Vec :: with_capacity ( kinds. len ( ) ) ;
207+ for kind in kinds. iter ( ) {
208+ let is_add = RowKind :: from_value ( kind. unwrap_or ( 0 ) ) ?. is_add ( ) ;
209+ if !is_add && reject {
210+ return Err ( Error :: Unsupported {
211+ message : "merge-engine=first-row does not support DELETE or UPDATE_BEFORE rows; set ignore-delete=true to ignore them" . to_string ( ) ,
212+ } ) ;
213+ }
214+ keep. push ( is_add) ;
215+ }
216+ arrow_select:: filter:: filter_record_batch ( & batch, & BooleanArray :: from ( keep) ) . map_err ( |e| {
217+ Error :: DataInvalid {
218+ message : format ! ( "Failed to drop retract rows: {e}" ) ,
219+ source : Some ( Box :: new ( e) ) ,
220+ }
221+ } )
222+ }
223+
187224/// Java's `KeyValueFieldsExtractor` builds the physical file layout from the
188225/// schema that wrote the file, including its historical key names and types.
189226pub ( super ) fn key_value_fields_extractor (
@@ -397,15 +434,28 @@ impl KeyValueFileReader {
397434 }
398435
399436 pub fn read ( self , data_splits : & [ DataSplit ] ) -> crate :: Result < ArrowRecordBatchStream > {
400- self . read_with_merge_function ( data_splits, Self :: new_merge_function)
437+ self . read_merged ( data_splits, Self :: new_merge_function, true )
401438 }
402439
440+ /// Always merges, so a custom merge function (e.g. the audit log, which
441+ /// keeps retract rows) sees every row.
403442 pub ( super ) fn read_with_merge_function (
404443 self ,
405444 data_splits : & [ DataSplit ] ,
406445 merge_function : impl Fn ( & KeyValueReadConfig , & [ DataField ] ) -> crate :: Result < Box < dyn MergeFunction > >
407446 + Send
408447 + ' static ,
448+ ) -> crate :: Result < ArrowRecordBatchStream > {
449+ self . read_merged ( data_splits, merge_function, false )
450+ }
451+
452+ fn read_merged (
453+ self ,
454+ data_splits : & [ DataSplit ] ,
455+ merge_function : impl Fn ( & KeyValueReadConfig , & [ DataField ] ) -> crate :: Result < Box < dyn MergeFunction > >
456+ + Send
457+ + ' static ,
458+ skip_single_runs : bool ,
409459 ) -> crate :: Result < ArrowRecordBatchStream > {
410460 // A projected `_ROW_ID` is synthesized as all-nulls here, so the residual
411461 // would silently drop every row rather than hit its missing-column guard.
@@ -711,6 +761,18 @@ impl KeyValueFileReader {
711761 . map( |budget| Arc :: new( budget. without_prefetch( ) ) )
712762 } ;
713763 let mut file_streams: Vec <ArrowRecordBatchStream > = Vec :: new( ) ;
764+ // One sorted run never repeats a key, so under these engines
765+ // every key is a lone version and needs no merge. Retract
766+ // rows are still dropped per batch below: older Rust writers
767+ // recorded `delete_row_count = 0` for files holding them.
768+ let skip_merge = skip_single_runs
769+ && input_stream_count == 1
770+ && matches!(
771+ config. merge_engine,
772+ MergeEngine :: Deduplicate | MergeEngine :: FirstRow
773+ ) ;
774+ let reject_retracts = config. merge_engine == MergeEngine :: FirstRow
775+ && !CoreOptions :: new( & config. table_options) . ignore_delete( ) ;
714776
715777 for MergeRun { files } in merge_group {
716778 let reader = DataFileReader :: new(
@@ -814,25 +876,43 @@ impl KeyValueFileReader {
814876 file_streams. push( run_stream) ;
815877 }
816878
817- // Always go through sort-merge even for a single file: files
818- // written before the writer merged key groups at flush may
819- // still contain duplicate keys.
820- let mut merge_stream = SortMergeReaderBuilder :: new(
821- file_streams,
822- internal_schema. clone( ) ,
823- key_indices. clone( ) ,
824- seq_index,
825- value_kind_index,
826- user_sequence_indices. clone( ) ,
827- value_indices. clone( ) ,
828- merge_output_schema. clone( ) ,
829- merge_function( & config, & merge_output_fields) ?,
830- )
831- . with_user_sequence_descending(
832- !CoreOptions :: new( & config. table_options)
833- . sequence_field_sort_order_is_ascending( ) ,
834- )
835- . build( ) ?;
879+ let mut merge_stream = if skip_merge {
880+ let input = file_streams. pop( ) . expect( "one input stream" ) ;
881+ let columns: Vec <usize > =
882+ key_indices. iter( ) . chain( & value_indices) . copied( ) . collect( ) ;
883+ let schema = merge_output_schema. clone( ) ;
884+ input
885+ . map( move |batch| {
886+ let batch =
887+ drop_lone_retracts( batch?, value_kind_index, reject_retracts) ?;
888+ let columns =
889+ columns. iter( ) . map( |& i| batch. column( i) . clone( ) ) . collect( ) ;
890+ RecordBatch :: try_new( schema. clone( ) , columns) . map_err( |e| {
891+ Error :: UnexpectedError {
892+ message: format!( "Failed to project unmerged batch: {e}" ) ,
893+ source: Some ( Box :: new( e) ) ,
894+ }
895+ } )
896+ } )
897+ . boxed( )
898+ } else {
899+ SortMergeReaderBuilder :: new(
900+ file_streams,
901+ internal_schema. clone( ) ,
902+ key_indices. clone( ) ,
903+ seq_index,
904+ value_kind_index,
905+ user_sequence_indices. clone( ) ,
906+ value_indices. clone( ) ,
907+ merge_output_schema. clone( ) ,
908+ merge_function( & config, & merge_output_fields) ?,
909+ )
910+ . with_user_sequence_descending(
911+ !CoreOptions :: new( & config. table_options)
912+ . sequence_field_sort_order_is_ascending( ) ,
913+ )
914+ . build( ) ?
915+ } ;
836916
837917 while let Some ( batch) = merge_stream. next( ) . await {
838918 let batch = batch?;
@@ -2011,7 +2091,9 @@ mod tests {
20112091 . await
20122092 . expect ( "the first output batch" )
20132093 . expect ( "a late deletion vector must be loaded lazily" ) ;
2014- assert_eq ! ( first. num_rows( ) , 128 ) ;
2094+ // These delete-free files form one run, which streams without the
2095+ // merge, so the first batch is one input batch.
2096+ assert ! ( first. num_rows( ) > 0 ) ;
20152097
20162098 let err = stream. try_collect :: < Vec < _ > > ( ) . await . unwrap_err ( ) ;
20172099 assert ! (
@@ -2693,6 +2775,121 @@ mod tests {
26932775 ) ;
26942776 }
26952777
2778+ /// Writes one KV file with the given keys and row kinds, recorded with
2779+ /// `delete_row_count = Some(0)` as Rust writers did before #318.
2780+ async fn write_legacy_kv_file (
2781+ file_io : & FileIO ,
2782+ table_path : & str ,
2783+ file_name : & str ,
2784+ rows : & [ ( i32 , i8 ) ] ,
2785+ ) -> DataFileMeta {
2786+ let schema = crate :: arrow:: build_target_arrow_schema ( & [
2787+ DataField :: new (
2788+ SEQUENCE_NUMBER_FIELD_ID ,
2789+ SEQUENCE_NUMBER_FIELD_NAME . to_string ( ) ,
2790+ DataType :: BigInt ( BigIntType :: new ( ) ) ,
2791+ ) ,
2792+ DataField :: new (
2793+ VALUE_KIND_FIELD_ID ,
2794+ VALUE_KIND_FIELD_NAME . to_string ( ) ,
2795+ DataType :: TinyInt ( TinyIntType :: new ( ) ) ,
2796+ ) ,
2797+ DataField :: new ( 0 , "id" . to_string ( ) , DataType :: Int ( IntType :: new ( ) ) ) ,
2798+ DataField :: new ( 1 , "value" . to_string ( ) , DataType :: Int ( IntType :: new ( ) ) ) ,
2799+ ] )
2800+ . unwrap ( ) ;
2801+ let batch = RecordBatch :: try_new (
2802+ schema. clone ( ) ,
2803+ vec ! [
2804+ Arc :: new( Int64Array :: from_value( 0 , rows. len( ) ) ) ,
2805+ Arc :: new( Int8Array :: from_iter_values( rows. iter( ) . map( |row| row. 1 ) ) ) ,
2806+ Arc :: new( Int32Array :: from_iter_values( rows. iter( ) . map( |row| row. 0 ) ) ) ,
2807+ Arc :: new( Int32Array :: from_value( 0 , rows. len( ) ) ) ,
2808+ ] ,
2809+ )
2810+ . unwrap ( ) ;
2811+ let mut bytes = Vec :: new ( ) ;
2812+ {
2813+ let mut writer = AsyncArrowWriter :: try_new ( & mut bytes, schema, None ) . unwrap ( ) ;
2814+ writer. write ( & batch) . await . unwrap ( ) ;
2815+ writer. close ( ) . await . unwrap ( ) ;
2816+ }
2817+ let bucket_path = format ! ( "{table_path}/bucket-0" ) ;
2818+ file_io. mkdirs ( & format ! ( "{bucket_path}/" ) ) . await . unwrap ( ) ;
2819+ file_io
2820+ . new_output ( & format ! ( "{bucket_path}/{file_name}" ) )
2821+ . unwrap ( )
2822+ . write ( bytes:: Bytes :: from ( bytes. clone ( ) ) )
2823+ . await
2824+ . unwrap ( ) ;
2825+
2826+ let mut file = dummy_data_file ( file_name. to_string ( ) ) ;
2827+ file. file_size = bytes. len ( ) as i64 ;
2828+ file. row_count = rows. len ( ) as i64 ;
2829+ file. min_key = int_key ( rows[ 0 ] . 0 ) ;
2830+ file. max_key = int_key ( rows[ rows. len ( ) - 1 ] . 0 ) ;
2831+ file
2832+ }
2833+
2834+ #[ tokio:: test]
2835+ async fn single_run_read_drops_retracts_hidden_by_legacy_metadata ( ) {
2836+ let file_io = test_file_io ( ) ;
2837+ let adds = [ ( 1 , 0 ) , ( 2 , 0 ) ] ;
2838+ let retract = [ ( 3 , 3 ) ] ;
2839+ for ( options, expected) in [
2840+ ( vec ! [ ] , Some ( vec ! [ 1 , 2 ] ) ) ,
2841+ ( vec ! [ ( "merge-engine" , "first-row" ) ] , None ) ,
2842+ (
2843+ vec ! [ ( "merge-engine" , "first-row" ) , ( "ignore-delete" , "true" ) ] ,
2844+ Some ( vec ! [ 1 , 2 ] ) ,
2845+ ) ,
2846+ ] {
2847+ let table_path = "memory:/kv_legacy_retract" ;
2848+ let table = pk_table ( & file_io, table_path, & options) ;
2849+ let a = write_legacy_kv_file ( & file_io, table_path, "a.parquet" , & adds) . await ;
2850+ let b = write_legacy_kv_file ( & file_io, table_path, "b.parquet" , & retract) . await ;
2851+ let split = DataSplitBuilder :: new ( )
2852+ . with_snapshot ( 1 )
2853+ . with_partition ( BinaryRow :: new ( 0 ) )
2854+ . with_bucket ( 0 )
2855+ . with_bucket_path ( format ! ( "{table_path}/bucket-0" ) )
2856+ . with_total_buckets ( 1 )
2857+ . with_data_files ( vec ! [ a, b] )
2858+ . build ( )
2859+ . unwrap ( ) ;
2860+ let core_options = table. schema ( ) . core_options ( ) ;
2861+ let result = KeyValueFileReader :: new (
2862+ table. file_io ( ) . clone ( ) ,
2863+ KeyValueReadConfig {
2864+ table_name : table. identifier ( ) . full_name ( ) ,
2865+ table_options : table. schema ( ) . options ( ) . clone ( ) ,
2866+ schema_manager : table. schema_manager ( ) . clone ( ) ,
2867+ table_schema_id : table. schema ( ) . id ( ) ,
2868+ table_fields : table. schema ( ) . fields ( ) . to_vec ( ) ,
2869+ read_type : table. schema ( ) . fields ( ) . to_vec ( ) ,
2870+ predicates : Vec :: new ( ) ,
2871+ primary_keys : table. schema ( ) . trimmed_primary_keys ( ) ,
2872+ table_primary_keys : table. schema ( ) . primary_keys ( ) . to_vec ( ) ,
2873+ merge_engine : core_options. merge_engine ( ) . unwrap ( ) ,
2874+ sequence_fields : Vec :: new ( ) ,
2875+ read_batch_size : core_options. read_batch_size ( ) . unwrap ( ) ,
2876+ merge_splits : false ,
2877+ max_merge_input_streams : None ,
2878+ parquet_read_budget : None ,
2879+ mosaic_prefetch : MosaicPrefetchOptions :: default ( ) ,
2880+ } ,
2881+ )
2882+ . read ( & [ split] )
2883+ . unwrap ( )
2884+ . try_collect :: < Vec < _ > > ( )
2885+ . await ;
2886+ match expected {
2887+ Some ( ids) => assert_eq ! ( int_column( & result. unwrap( ) , "id" ) , ids, "{options:?}" ) ,
2888+ None => assert ! ( result. is_err( ) , "{options:?}" ) ,
2889+ }
2890+ }
2891+ }
2892+
26962893 #[ tokio:: test]
26972894 async fn sorted_run_read_matches_per_file_fan_out ( ) {
26982895 let file_io = test_file_io ( ) ;
0 commit comments