@@ -765,12 +765,36 @@ impl KeyValueFileReader {
765765 // every key is a lone version and needs no merge. Retract
766766 // rows are still dropped per batch below: older Rust writers
767767 // recorded `delete_row_count = 0` for files holding them.
768- let skip_merge = skip_single_runs
768+ let mut skip_merge = skip_single_runs
769769 && input_stream_count == 1
770770 && matches!(
771771 config. merge_engine,
772772 MergeEngine :: Deduplicate | MergeEngine :: FirstRow
773773 ) ;
774+ // A file written under another engine may hold several
775+ // versions of a key (the aggregation writer kept them before
776+ // #839), and ALTER can switch the engine afterwards.
777+ if skip_merge {
778+ ' files: for run in & merge_group {
779+ for file in & run. files {
780+ let schema_id = file. file. schema_id;
781+ if schema_id == table_schema_id {
782+ continue ;
783+ }
784+ let schema =
785+ config. schema_manager. schema( schema_id) . await ?;
786+ let engine =
787+ CoreOptions :: new( schema. options( ) ) . merge_engine( ) ?;
788+ if !matches!(
789+ engine,
790+ MergeEngine :: Deduplicate | MergeEngine :: FirstRow
791+ ) {
792+ skip_merge = false ;
793+ break ' files;
794+ }
795+ }
796+ }
797+ }
774798 let reject_retracts = config. merge_engine == MergeEngine :: FirstRow
775799 && !CoreOptions :: new( & config. table_options) . ignore_delete( ) ;
776800
@@ -2890,6 +2914,127 @@ mod tests {
28902914 }
28912915 }
28922916
2917+ /// Writes one KV file with `(id, sequence, value)` rows, under schema 0.
2918+ async fn write_kv_versions (
2919+ file_io : & FileIO ,
2920+ table_path : & str ,
2921+ file_name : & str ,
2922+ rows : & [ ( i32 , i64 , i32 ) ] ,
2923+ ) -> DataFileMeta {
2924+ let schema = crate :: arrow:: build_target_arrow_schema ( & [
2925+ DataField :: new (
2926+ SEQUENCE_NUMBER_FIELD_ID ,
2927+ SEQUENCE_NUMBER_FIELD_NAME . to_string ( ) ,
2928+ DataType :: BigInt ( BigIntType :: new ( ) ) ,
2929+ ) ,
2930+ DataField :: new (
2931+ VALUE_KIND_FIELD_ID ,
2932+ VALUE_KIND_FIELD_NAME . to_string ( ) ,
2933+ DataType :: TinyInt ( TinyIntType :: new ( ) ) ,
2934+ ) ,
2935+ DataField :: new ( 0 , "id" . to_string ( ) , DataType :: Int ( IntType :: new ( ) ) ) ,
2936+ DataField :: new ( 1 , "value" . to_string ( ) , DataType :: Int ( IntType :: new ( ) ) ) ,
2937+ ] )
2938+ . unwrap ( ) ;
2939+ let batch = RecordBatch :: try_new (
2940+ schema. clone ( ) ,
2941+ vec ! [
2942+ Arc :: new( Int64Array :: from_iter_values( rows. iter( ) . map( |row| row. 1 ) ) ) ,
2943+ Arc :: new( Int8Array :: from_value( 0 , rows. len( ) ) ) ,
2944+ Arc :: new( Int32Array :: from_iter_values( rows. iter( ) . map( |row| row. 0 ) ) ) ,
2945+ Arc :: new( Int32Array :: from_iter_values( rows. iter( ) . map( |row| row. 2 ) ) ) ,
2946+ ] ,
2947+ )
2948+ . unwrap ( ) ;
2949+ let mut bytes = Vec :: new ( ) ;
2950+ {
2951+ let mut writer = AsyncArrowWriter :: try_new ( & mut bytes, schema, None ) . unwrap ( ) ;
2952+ writer. write ( & batch) . await . unwrap ( ) ;
2953+ writer. close ( ) . await . unwrap ( ) ;
2954+ }
2955+ let bucket_path = format ! ( "{table_path}/bucket-0" ) ;
2956+ file_io. mkdirs ( & format ! ( "{bucket_path}/" ) ) . await . unwrap ( ) ;
2957+ file_io
2958+ . new_output ( & format ! ( "{bucket_path}/{file_name}" ) )
2959+ . unwrap ( )
2960+ . write ( bytes:: Bytes :: from ( bytes. clone ( ) ) )
2961+ . await
2962+ . unwrap ( ) ;
2963+
2964+ let mut file = dummy_data_file ( file_name. to_string ( ) ) ;
2965+ file. file_size = bytes. len ( ) as i64 ;
2966+ file. row_count = rows. len ( ) as i64 ;
2967+ file. min_key = int_key ( rows[ 0 ] . 0 ) ;
2968+ file. max_key = int_key ( rows[ rows. len ( ) - 1 ] . 0 ) ;
2969+ file. min_sequence_number = rows. iter ( ) . map ( |row| row. 1 ) . min ( ) . unwrap ( ) ;
2970+ file. max_sequence_number = rows. iter ( ) . map ( |row| row. 1 ) . max ( ) . unwrap ( ) ;
2971+ file
2972+ }
2973+
2974+ #[ tokio:: test]
2975+ async fn single_run_read_merges_files_written_under_aggregation ( ) {
2976+ // Before #839 the aggregation writer kept several versions of a key in
2977+ // one file. After ALTER to deduplicate, such a file still needs merging.
2978+ let file_io = test_file_io ( ) ;
2979+ let table_path = "memory:/kv_engine_change" ;
2980+ let old = pk_table (
2981+ & file_io,
2982+ table_path,
2983+ & [
2984+ ( "merge-engine" , "aggregation" ) ,
2985+ ( "fields.value.aggregate-function" , "sum" ) ,
2986+ ] ,
2987+ ) ;
2988+ write_schema_file ( & old, old. schema ( ) ) . await ;
2989+ let mut json = serde_json:: to_value ( old. schema ( ) ) . unwrap ( ) ;
2990+ json[ "id" ] = 1 . into ( ) ;
2991+ json[ "options" ] [ "merge-engine" ] = "deduplicate" . into ( ) ;
2992+ let schema: TableSchema = serde_json:: from_value ( json) . unwrap ( ) ;
2993+ write_schema_file ( & old, & schema) . await ;
2994+ let table = old. copy_with_resolved_schema ( schema, "main" ) . unwrap ( ) ;
2995+ let a =
2996+ write_kv_versions ( & file_io, table_path, "a.parquet" , & [ ( 1 , 0 , 10 ) , ( 1 , 1 , 11 ) ] ) . await ;
2997+ let b = write_kv_versions ( & file_io, table_path, "b.parquet" , & [ ( 2 , 2 , 20 ) ] ) . await ;
2998+ let split = DataSplitBuilder :: new ( )
2999+ . with_snapshot ( 1 )
3000+ . with_partition ( BinaryRow :: new ( 0 ) )
3001+ . with_bucket ( 0 )
3002+ . with_bucket_path ( format ! ( "{table_path}/bucket-0" ) )
3003+ . with_total_buckets ( 1 )
3004+ . with_data_files ( vec ! [ a, b] )
3005+ . build ( )
3006+ . unwrap ( ) ;
3007+ let core_options = table. schema ( ) . core_options ( ) ;
3008+ let batches = KeyValueFileReader :: new (
3009+ table. file_io ( ) . clone ( ) ,
3010+ KeyValueReadConfig {
3011+ table_name : table. identifier ( ) . full_name ( ) ,
3012+ table_options : table. schema ( ) . options ( ) . clone ( ) ,
3013+ schema_manager : table. schema_manager ( ) . clone ( ) ,
3014+ table_schema_id : table. schema ( ) . id ( ) ,
3015+ table_fields : table. schema ( ) . fields ( ) . to_vec ( ) ,
3016+ read_type : table. schema ( ) . fields ( ) . to_vec ( ) ,
3017+ predicates : Vec :: new ( ) ,
3018+ primary_keys : table. schema ( ) . trimmed_primary_keys ( ) ,
3019+ table_primary_keys : table. schema ( ) . primary_keys ( ) . to_vec ( ) ,
3020+ merge_engine : core_options. merge_engine ( ) . unwrap ( ) ,
3021+ sequence_fields : Vec :: new ( ) ,
3022+ read_batch_size : core_options. read_batch_size ( ) . unwrap ( ) ,
3023+ merge_splits : false ,
3024+ max_merge_input_streams : None ,
3025+ parquet_read_budget : None ,
3026+ mosaic_prefetch : MosaicPrefetchOptions :: default ( ) ,
3027+ } ,
3028+ )
3029+ . read ( & [ split] )
3030+ . unwrap ( )
3031+ . try_collect :: < Vec < _ > > ( )
3032+ . await
3033+ . unwrap ( ) ;
3034+ assert_eq ! ( int_column( & batches, "id" ) , vec![ 1 , 2 ] ) ;
3035+ assert_eq ! ( int_column( & batches, "value" ) , vec![ 11 , 20 ] ) ;
3036+ }
3037+
28933038 #[ tokio:: test]
28943039 async fn sorted_run_read_matches_per_file_fan_out ( ) {
28953040 let file_io = test_file_io ( ) ;
0 commit comments