@@ -81,6 +81,9 @@ pub(crate) struct KeyValueReadConfig {
8181 /// Full table keys also protect partition-PK fields from aggregation.
8282 pub table_primary_keys : Vec < String > ,
8383 pub merge_engine : MergeEngine ,
84+ /// `merge-engine` was set by dynamic options, so files written under
85+ /// `table_schema_id` may still hold another engine's output.
86+ pub merge_engine_overridden : bool ,
8487 pub sequence_fields : Vec < String > ,
8588 pub read_batch_size : usize ,
8689 /// Merge files from all supplied splits into one globally key-sorted stream.
@@ -777,12 +780,15 @@ impl KeyValueFileReader {
777780 if skip_merge {
778781 ' files: for run in & merge_group {
779782 for file in & run. files {
780- let schema_id = file. file. schema_id;
781- if schema_id == table_schema_id {
783+ // Dynamic options can override the engine but keep
784+ // the schema ID, so only then check the stored one.
785+ if file. file. schema_id == table_schema_id
786+ && !config. merge_engine_overridden
787+ {
782788 continue ;
783789 }
784790 let schema =
785- config. schema_manager. schema( schema_id) . await ?;
791+ config. schema_manager. schema( file . file . schema_id) . await ?;
786792 let engine =
787793 CoreOptions :: new( schema. options( ) ) . merge_engine( ) ?;
788794 if !matches!(
@@ -2207,6 +2213,7 @@ mod tests {
22072213 table_options : table. schema ( ) . options ( ) . clone ( ) ,
22082214 schema_manager : table. schema_manager ( ) . clone ( ) ,
22092215 table_schema_id : table. schema ( ) . id ( ) ,
2216+ merge_engine_overridden : table. merge_engine_overridden ( ) ,
22102217 table_fields : table. schema ( ) . fields ( ) . to_vec ( ) ,
22112218 read_type : table. schema ( ) . fields ( ) . to_vec ( ) ,
22122219 predicates : Vec :: new ( ) ,
@@ -2328,6 +2335,7 @@ mod tests {
23282335 table_options : table. schema ( ) . options ( ) . clone ( ) ,
23292336 schema_manager : table. schema_manager ( ) . clone ( ) ,
23302337 table_schema_id : table. schema ( ) . id ( ) ,
2338+ merge_engine_overridden : table. merge_engine_overridden ( ) ,
23312339 table_fields : table. schema ( ) . fields ( ) . to_vec ( ) ,
23322340 read_type : table. schema ( ) . fields ( ) . to_vec ( ) ,
23332341 predicates : Vec :: new ( ) ,
@@ -2533,6 +2541,7 @@ mod tests {
25332541 table_options : table. schema ( ) . options ( ) . clone ( ) ,
25342542 schema_manager : table. schema_manager ( ) . clone ( ) ,
25352543 table_schema_id : table. schema ( ) . id ( ) ,
2544+ merge_engine_overridden : table. merge_engine_overridden ( ) ,
25362545 table_fields : table. schema ( ) . fields ( ) . to_vec ( ) ,
25372546 read_type : table. schema ( ) . fields ( ) . to_vec ( ) ,
25382547 predicates : Vec :: new ( ) ,
@@ -2622,6 +2631,7 @@ mod tests {
26222631 table_options : table. schema ( ) . options ( ) . clone ( ) ,
26232632 schema_manager : table. schema_manager ( ) . clone ( ) ,
26242633 table_schema_id : table. schema ( ) . id ( ) ,
2634+ merge_engine_overridden : table. merge_engine_overridden ( ) ,
26252635 table_fields : table. schema ( ) . fields ( ) . to_vec ( ) ,
26262636 read_type : table. schema ( ) . fields ( ) . to_vec ( ) ,
26272637 predicates : Vec :: new ( ) ,
@@ -2889,6 +2899,7 @@ mod tests {
28892899 table_options : table. schema ( ) . options ( ) . clone ( ) ,
28902900 schema_manager : table. schema_manager ( ) . clone ( ) ,
28912901 table_schema_id : table. schema ( ) . id ( ) ,
2902+ merge_engine_overridden : table. merge_engine_overridden ( ) ,
28922903 table_fields : table. schema ( ) . fields ( ) . to_vec ( ) ,
28932904 read_type : table. schema ( ) . fields ( ) . to_vec ( ) ,
28942905 predicates : Vec :: new ( ) ,
@@ -2971,6 +2982,37 @@ mod tests {
29712982 file
29722983 }
29732984
2985+ async fn read_kv_split ( table : & Table , split : DataSplit ) -> Vec < RecordBatch > {
2986+ let core_options = table. schema ( ) . core_options ( ) ;
2987+ KeyValueFileReader :: new (
2988+ table. file_io ( ) . clone ( ) ,
2989+ KeyValueReadConfig {
2990+ table_name : table. identifier ( ) . full_name ( ) ,
2991+ table_options : table. schema ( ) . options ( ) . clone ( ) ,
2992+ schema_manager : table. schema_manager ( ) . clone ( ) ,
2993+ table_schema_id : table. schema ( ) . id ( ) ,
2994+ merge_engine_overridden : table. merge_engine_overridden ( ) ,
2995+ table_fields : table. schema ( ) . fields ( ) . to_vec ( ) ,
2996+ read_type : table. schema ( ) . fields ( ) . to_vec ( ) ,
2997+ predicates : Vec :: new ( ) ,
2998+ primary_keys : table. schema ( ) . trimmed_primary_keys ( ) ,
2999+ table_primary_keys : table. schema ( ) . primary_keys ( ) . to_vec ( ) ,
3000+ merge_engine : core_options. merge_engine ( ) . unwrap ( ) ,
3001+ sequence_fields : Vec :: new ( ) ,
3002+ read_batch_size : core_options. read_batch_size ( ) . unwrap ( ) ,
3003+ merge_splits : false ,
3004+ max_merge_input_streams : None ,
3005+ parquet_read_budget : None ,
3006+ mosaic_prefetch : MosaicPrefetchOptions :: default ( ) ,
3007+ } ,
3008+ )
3009+ . read ( & [ split] )
3010+ . unwrap ( )
3011+ . try_collect :: < Vec < _ > > ( )
3012+ . await
3013+ . unwrap ( )
3014+ }
3015+
29743016 #[ tokio:: test]
29753017 async fn single_run_read_merges_files_written_under_aggregation ( ) {
29763018 // Before #839 the aggregation writer kept several versions of a key in
@@ -3004,33 +3046,48 @@ mod tests {
30043046 . with_data_files ( vec ! [ a, b] )
30053047 . build ( )
30063048 . unwrap ( ) ;
3007- let core_options = table. schema ( ) . core_options ( ) ;
3008- let batches = KeyValueFileReader :: new (
3009- table. file_io ( ) . clone ( ) ,
3010- KeyValueReadConfig {
3011- table_name : table. identifier ( ) . full_name ( ) ,
3012- table_options : table. schema ( ) . options ( ) . clone ( ) ,
3013- schema_manager : table. schema_manager ( ) . clone ( ) ,
3014- table_schema_id : table. schema ( ) . id ( ) ,
3015- table_fields : table. schema ( ) . fields ( ) . to_vec ( ) ,
3016- read_type : table. schema ( ) . fields ( ) . to_vec ( ) ,
3017- predicates : Vec :: new ( ) ,
3018- primary_keys : table. schema ( ) . trimmed_primary_keys ( ) ,
3019- table_primary_keys : table. schema ( ) . primary_keys ( ) . to_vec ( ) ,
3020- merge_engine : core_options. merge_engine ( ) . unwrap ( ) ,
3021- sequence_fields : Vec :: new ( ) ,
3022- read_batch_size : core_options. read_batch_size ( ) . unwrap ( ) ,
3023- merge_splits : false ,
3024- max_merge_input_streams : None ,
3025- parquet_read_budget : None ,
3026- mosaic_prefetch : MosaicPrefetchOptions :: default ( ) ,
3027- } ,
3028- )
3029- . read ( & [ split] )
3030- . unwrap ( )
3031- . try_collect :: < Vec < _ > > ( )
3032- . await
3033- . unwrap ( ) ;
3049+ let batches = read_kv_split ( & table, split) . await ;
3050+ assert_eq ! ( int_column( & batches, "id" ) , vec![ 1 , 2 ] ) ;
3051+ assert_eq ! ( int_column( & batches, "value" ) , vec![ 11 , 20 ] ) ;
3052+ }
3053+
3054+ #[ tokio:: test]
3055+ async fn single_run_read_merges_when_options_override_the_engine ( ) {
3056+ // copy_with_options keeps the schema ID, so the files and the read
3057+ // schema share it while the stored engine is still aggregation.
3058+ let file_io = test_file_io ( ) ;
3059+ let table_path = "memory:/kv_engine_override" ;
3060+ let old = pk_table (
3061+ & file_io,
3062+ table_path,
3063+ & [
3064+ ( "merge-engine" , "aggregation" ) ,
3065+ ( "fields.value.aggregate-function" , "sum" ) ,
3066+ ] ,
3067+ ) ;
3068+ write_schema_file ( & old, old. schema ( ) ) . await ;
3069+ let table = old. copy_with_options ( HashMap :: from ( [ (
3070+ "merge-engine" . to_string ( ) ,
3071+ "deduplicate" . to_string ( ) ,
3072+ ) ] ) ) ;
3073+ assert_eq ! ( table. schema( ) . id( ) , old. schema( ) . id( ) ) ;
3074+ assert_eq ! (
3075+ table. schema( ) . core_options( ) . merge_engine( ) . unwrap( ) ,
3076+ MergeEngine :: Deduplicate
3077+ ) ;
3078+ let a =
3079+ write_kv_versions ( & file_io, table_path, "a.parquet" , & [ ( 1 , 0 , 10 ) , ( 1 , 1 , 11 ) ] ) . await ;
3080+ let b = write_kv_versions ( & file_io, table_path, "b.parquet" , & [ ( 2 , 2 , 20 ) ] ) . await ;
3081+ let split = DataSplitBuilder :: new ( )
3082+ . with_snapshot ( 1 )
3083+ . with_partition ( BinaryRow :: new ( 0 ) )
3084+ . with_bucket ( 0 )
3085+ . with_bucket_path ( format ! ( "{table_path}/bucket-0" ) )
3086+ . with_total_buckets ( 1 )
3087+ . with_data_files ( vec ! [ a, b] )
3088+ . build ( )
3089+ . unwrap ( ) ;
3090+ let batches = read_kv_split ( & table, split) . await ;
30343091 assert_eq ! ( int_column( & batches, "id" ) , vec![ 1 , 2 ] ) ;
30353092 assert_eq ! ( int_column( & batches, "value" ) , vec![ 11 , 20 ] ) ;
30363093 }
@@ -3067,6 +3124,7 @@ mod tests {
30673124 table_options : table. schema ( ) . options ( ) . clone ( ) ,
30683125 schema_manager : table. schema_manager ( ) . clone ( ) ,
30693126 table_schema_id : table. schema ( ) . id ( ) ,
3127+ merge_engine_overridden : table. merge_engine_overridden ( ) ,
30703128 table_fields : table. schema ( ) . fields ( ) . to_vec ( ) ,
30713129 read_type : table. schema ( ) . fields ( ) . to_vec ( ) ,
30723130 predicates : Vec :: new ( ) ,
@@ -3136,6 +3194,7 @@ mod tests {
31363194 table_options : table. schema ( ) . options ( ) . clone ( ) ,
31373195 schema_manager : table. schema_manager ( ) . clone ( ) ,
31383196 table_schema_id : table. schema ( ) . id ( ) ,
3197+ merge_engine_overridden : table. merge_engine_overridden ( ) ,
31393198 table_fields : table. schema ( ) . fields ( ) . to_vec ( ) ,
31403199 read_type : table. schema ( ) . fields ( ) . to_vec ( ) ,
31413200 predicates : Vec :: new ( ) ,
0 commit comments