@@ -81,9 +81,6 @@ pub(crate) struct KeyValueReadConfig {
8181 /// Full table keys also protect partition-PK fields from aggregation.
8282 pub table_primary_keys: Vec<String>,
8383 pub merge_engine: MergeEngine,
84- /// `merge-engine` was set by dynamic options, so files written under
85- /// `table_schema_id` may still hold another engine's output.
86- pub merge_engine_overridden: bool,
8784 pub sequence_fields: Vec<String>,
8885 pub read_batch_size: usize,
8986 /// Merge files from all supplied splits into one globally key-sorted stream.
@@ -780,15 +777,26 @@ impl KeyValueFileReader {
780777 if skip_merge {
781778 'files: for run in &merge_group {
782779 for file in &run.files {
783- // Dynamic options can override the engine but keep
784- // the schema ID, so only then check the stored one.
785- if file.file.schema_id == table_schema_id
786- && !config.merge_engine_overridden
780+ // Always check the stored schema: dynamic options and
781+ // time travel can change the read schema's engine
782+ // without changing its ID. Merging is always correct,
783+ // so a missing schema file under the read schema's ID
784+ // only costs the skip.
785+ let schema = match config
786+ .schema_manager
787+ .schema(file.file.schema_id)
788+ .await
787789 {
788- continue;
789- }
790- let schema =
791- config.schema_manager.schema(file.file.schema_id).await?;
790+ Ok(schema) => schema,
791+ Err(crate::Error::IoUnexpected { source, .. })
792+ if source.kind() == opendal::ErrorKind::NotFound
793+ && file.file.schema_id == table_schema_id =>
794+ {
795+ skip_merge = false;
796+ break 'files;
797+ }
798+ Err(err) => Err(err)?,
799+ };
792800 let engine =
793801 CoreOptions::new(schema.options()).merge_engine()?;
794802 if !matches!(
@@ -2076,6 +2084,7 @@ mod tests {
20762084 ("deletion-vectors.merge-on-read", "true"),
20772085 ],
20782086 );
2087+ write_schema_file(&table, table.schema()).await;
20792088
20802089 let mut files = Vec::new();
20812090 for i in 0..10 {
@@ -2213,7 +2222,6 @@ mod tests {
22132222 table_options: table.schema().options().clone(),
22142223 schema_manager: table.schema_manager().clone(),
22152224 table_schema_id: table.schema().id(),
2216- merge_engine_overridden: table.merge_engine_overridden(),
22172225 table_fields: table.schema().fields().to_vec(),
22182226 read_type: table.schema().fields().to_vec(),
22192227 predicates: Vec::new(),
@@ -2335,7 +2343,6 @@ mod tests {
23352343 table_options: table.schema().options().clone(),
23362344 schema_manager: table.schema_manager().clone(),
23372345 table_schema_id: table.schema().id(),
2338- merge_engine_overridden: table.merge_engine_overridden(),
23392346 table_fields: table.schema().fields().to_vec(),
23402347 read_type: table.schema().fields().to_vec(),
23412348 predicates: Vec::new(),
@@ -2541,7 +2548,6 @@ mod tests {
25412548 table_options: table.schema().options().clone(),
25422549 schema_manager: table.schema_manager().clone(),
25432550 table_schema_id: table.schema().id(),
2544- merge_engine_overridden: table.merge_engine_overridden(),
25452551 table_fields: table.schema().fields().to_vec(),
25462552 read_type: table.schema().fields().to_vec(),
25472553 predicates: Vec::new(),
@@ -2631,7 +2637,6 @@ mod tests {
26312637 table_options: table.schema().options().clone(),
26322638 schema_manager: table.schema_manager().clone(),
26332639 table_schema_id: table.schema().id(),
2634- merge_engine_overridden: table.merge_engine_overridden(),
26352640 table_fields: table.schema().fields().to_vec(),
26362641 read_type: table.schema().fields().to_vec(),
26372642 predicates: Vec::new(),
@@ -2691,6 +2696,7 @@ mod tests {
26912696 ("read.parquet.row-group.parallelism", "1"),
26922697 ],
26932698 );
2699+ write_schema_file(&table, table.schema()).await;
26942700 let low =
26952701 write_multi_row_group_kv_file(&file_io, table_path, "low.parquet", 0, 0, 10).await;
26962702 let high =
@@ -2747,6 +2753,7 @@ mod tests {
27472753 ("read.parquet.row-group.parallelism", "1"),
27482754 ],
27492755 );
2756+ write_schema_file(&table, table.schema()).await;
27502757 let first_low =
27512758 write_multi_row_group_kv_file(&file_io, table_path, "first-low.parquet", 0, 0, 10)
27522759 .await;
@@ -2880,6 +2887,7 @@ mod tests {
28802887 ] {
28812888 let table_path = "memory:/kv_legacy_retract";
28822889 let table = pk_table(&file_io, table_path, &options);
2890+ write_schema_file(&table, table.schema()).await;
28832891 let a = write_legacy_kv_file(&file_io, table_path, "a.parquet", &adds).await;
28842892 let b = write_legacy_kv_file(&file_io, table_path, "b.parquet", &retract).await;
28852893 let split = DataSplitBuilder::new()
@@ -2899,7 +2907,6 @@ mod tests {
28992907 table_options: table.schema().options().clone(),
29002908 schema_manager: table.schema_manager().clone(),
29012909 table_schema_id: table.schema().id(),
2902- merge_engine_overridden: table.merge_engine_overridden(),
29032910 table_fields: table.schema().fields().to_vec(),
29042911 read_type: table.schema().fields().to_vec(),
29052912 predicates: Vec::new(),
@@ -2991,7 +2998,6 @@ mod tests {
29912998 table_options: table.schema().options().clone(),
29922999 schema_manager: table.schema_manager().clone(),
29933000 table_schema_id: table.schema().id(),
2994- merge_engine_overridden: table.merge_engine_overridden(),
29953001 table_fields: table.schema().fields().to_vec(),
29963002 read_type: table.schema().fields().to_vec(),
29973003 predicates: Vec::new(),
@@ -3052,11 +3058,11 @@ mod tests {
30523058 }
30533059
30543060 #[tokio::test]
3055- async fn single_run_read_merges_when_options_override_the_engine () {
3056- // copy_with_options keeps the schema ID, so the files and the read
3057- // schema share it while the stored engine is still aggregation .
3061+ async fn single_run_read_merges_after_time_travel_past_an_engine_alter () {
3062+ // aggregation ( schema 0) -> ALTER to deduplicate (schema 1). Reading
3063+ // snapshot 1 resolves schema 0 but keeps the current engine option .
30583064 let file_io = test_file_io();
3059- let table_path = "memory:/kv_engine_override ";
3065+ let table_path = "memory:/kv_engine_time_travel ";
30603066 let old = pk_table(
30613067 &file_io,
30623068 table_path,
@@ -3066,15 +3072,45 @@ mod tests {
30663072 ],
30673073 );
30683074 write_schema_file(&old, old.schema()).await;
3069- let table = old.copy_with_options(HashMap::from([(
3070- "merge-engine".to_string(),
3071- "deduplicate".to_string(),
3072- )]));
3073- assert_eq!(table.schema().id(), old.schema().id());
3075+ let mut json = serde_json::to_value(old.schema()).unwrap();
3076+ json["id"] = 1.into();
3077+ json["options"]["merge-engine"] = "deduplicate".into();
3078+ let schema: TableSchema = serde_json::from_value(json).unwrap();
3079+ write_schema_file(&old, &schema).await;
3080+ let table = old.copy_with_resolved_schema(schema, "main").unwrap();
3081+ file_io
3082+ .mkdirs(&format!("{table_path}/snapshot/"))
3083+ .await
3084+ .unwrap();
3085+ let snapshot = crate::spec::Snapshot::builder()
3086+ .version(3)
3087+ .id(1)
3088+ .schema_id(0)
3089+ .base_manifest_list("base-list".to_string())
3090+ .delta_manifest_list("delta-list".to_string())
3091+ .commit_user("test-user".to_string())
3092+ .commit_identifier(0)
3093+ .commit_kind(crate::spec::CommitKind::APPEND)
3094+ .time_millis(1000)
3095+ .build();
3096+ assert!(table
3097+ .snapshot_manager()
3098+ .commit_snapshot(&snapshot)
3099+ .await
3100+ .unwrap());
3101+ let historical = table
3102+ .copy_with_time_travel_strict(HashMap::from([(
3103+ "scan.version".to_string(),
3104+ "1".to_string(),
3105+ )]))
3106+ .await
3107+ .unwrap();
3108+ assert_eq!(historical.schema().id(), 0);
30743109 assert_eq!(
3075- table .schema().core_options().merge_engine().unwrap(),
3110+ historical .schema().core_options().merge_engine().unwrap(),
30763111 MergeEngine::Deduplicate
30773112 );
3113+
30783114 let a =
30793115 write_kv_versions(&file_io, table_path, "a.parquet", &[(1, 0, 10), (1, 1, 11)]).await;
30803116 let b = write_kv_versions(&file_io, table_path, "b.parquet", &[(2, 2, 20)]).await;
@@ -3087,16 +3123,64 @@ mod tests {
30873123 .with_data_files(vec![a, b])
30883124 .build()
30893125 .unwrap();
3090- let batches = read_kv_split(&table , split).await;
3126+ let batches = read_kv_split(&historical , split).await;
30913127 assert_eq!(int_column(&batches, "id"), vec![1, 2]);
30923128 assert_eq!(int_column(&batches, "value"), vec![11, 20]);
30933129 }
30943130
3131+ #[tokio::test]
3132+ async fn single_run_read_merges_when_the_read_schema_names_another_engine() {
3133+ // Dynamic options and time travel can give the read schema another
3134+ // engine under the files' schema ID; the stored engine is aggregation.
3135+ let file_io = test_file_io();
3136+ let table_path = "memory:/kv_engine_override";
3137+ let old = pk_table(
3138+ &file_io,
3139+ table_path,
3140+ &[
3141+ ("merge-engine", "aggregation"),
3142+ ("fields.value.aggregate-function", "sum"),
3143+ ],
3144+ );
3145+ write_schema_file(&old, old.schema()).await;
3146+ let overridden = old.copy_with_options(HashMap::from([(
3147+ "merge-engine".to_string(),
3148+ "deduplicate".to_string(),
3149+ )]));
3150+ // Same read schema as a historical read after an ALTER to deduplicate.
3151+ let rewrapped = old
3152+ .copy_with_resolved_schema(overridden.schema().clone(), "main")
3153+ .unwrap();
3154+ let a =
3155+ write_kv_versions(&file_io, table_path, "a.parquet", &[(1, 0, 10), (1, 1, 11)]).await;
3156+ let b = write_kv_versions(&file_io, table_path, "b.parquet", &[(2, 2, 20)]).await;
3157+ let split = DataSplitBuilder::new()
3158+ .with_snapshot(1)
3159+ .with_partition(BinaryRow::new(0))
3160+ .with_bucket(0)
3161+ .with_bucket_path(format!("{table_path}/bucket-0"))
3162+ .with_total_buckets(1)
3163+ .with_data_files(vec![a, b])
3164+ .build()
3165+ .unwrap();
3166+ for table in [&overridden, &rewrapped] {
3167+ assert_eq!(table.schema().id(), old.schema().id());
3168+ assert_eq!(
3169+ table.schema().core_options().merge_engine().unwrap(),
3170+ MergeEngine::Deduplicate
3171+ );
3172+ let batches = read_kv_split(table, split.clone()).await;
3173+ assert_eq!(int_column(&batches, "id"), vec![1, 2]);
3174+ assert_eq!(int_column(&batches, "value"), vec![11, 20]);
3175+ }
3176+ }
3177+
30953178 #[tokio::test]
30963179 async fn sorted_run_read_matches_per_file_fan_out() {
30973180 let file_io = test_file_io();
30983181 let table_path = "memory:/kv_sorted_run";
30993182 let table = pk_table(&file_io, table_path, &[]);
3183+ write_schema_file(&table, table.schema()).await;
31003184 let low =
31013185 write_multi_row_group_kv_file(&file_io, table_path, "low.parquet", 0, 0, 10).await;
31023186 let high =
@@ -3124,7 +3208,6 @@ mod tests {
31243208 table_options: table.schema().options().clone(),
31253209 schema_manager: table.schema_manager().clone(),
31263210 table_schema_id: table.schema().id(),
3127- merge_engine_overridden: table.merge_engine_overridden(),
31283211 table_fields: table.schema().fields().to_vec(),
31293212 read_type: table.schema().fields().to_vec(),
31303213 predicates: Vec::new(),
@@ -3194,7 +3277,6 @@ mod tests {
31943277 table_options: table.schema().options().clone(),
31953278 schema_manager: table.schema_manager().clone(),
31963279 table_schema_id: table.schema().id(),
3197- merge_engine_overridden: table.merge_engine_overridden(),
31983280 table_fields: table.schema().fields().to_vec(),
31993281 read_type: table.schema().fields().to_vec(),
32003282 predicates: Vec::new(),
0 commit comments