Skip to content

Commit 9b8cd4b

Browse files
committed
[core] Check the stored engine even when the schema ID matches
Table::copy_with_options keeps the schema ID while overriding merge-engine, so files written under aggregation share the read schema's ID.
1 parent 37219af commit 9b8cd4b

4 files changed

Lines changed: 98 additions & 30 deletions

File tree

‎crates/paimon/src/table/audit_log_table/read.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ impl<'a> AuditLogRead<'a> {
125125
table_options: self.read.table.schema().options().clone(),
126126
schema_manager: self.read.table.schema_manager().clone(),
127127
table_schema_id: self.read.table.schema().id(),
128+
merge_engine_overridden: self.read.table.merge_engine_overridden(),
128129
table_fields: self.read.table.schema.fields().to_vec(),
129130
read_type,
130131
predicates: self.read.data_predicates.clone(),

‎crates/paimon/src/table/kv_file_reader.rs‎

Lines changed: 89 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,9 @@ pub(crate) struct KeyValueReadConfig {
8181
/// Full table keys also protect partition-PK fields from aggregation.
8282
pub table_primary_keys: Vec<String>,
8383
pub merge_engine: MergeEngine,
84+
/// `merge-engine` was set by dynamic options, so files written under
85+
/// `table_schema_id` may still hold another engine's output.
86+
pub merge_engine_overridden: bool,
8487
pub sequence_fields: Vec<String>,
8588
pub read_batch_size: usize,
8689
/// Merge files from all supplied splits into one globally key-sorted stream.
@@ -777,12 +780,15 @@ impl KeyValueFileReader {
777780
if skip_merge {
778781
'files: for run in &merge_group {
779782
for file in &run.files {
780-
let schema_id = file.file.schema_id;
781-
if schema_id == table_schema_id {
783+
// Dynamic options can override the engine but keep
784+
// the schema ID, so only then check the stored one.
785+
if file.file.schema_id == table_schema_id
786+
&& !config.merge_engine_overridden
787+
{
782788
continue;
783789
}
784790
let schema =
785-
config.schema_manager.schema(schema_id).await?;
791+
config.schema_manager.schema(file.file.schema_id).await?;
786792
let engine =
787793
CoreOptions::new(schema.options()).merge_engine()?;
788794
if !matches!(
@@ -2207,6 +2213,7 @@ mod tests {
22072213
table_options: table.schema().options().clone(),
22082214
schema_manager: table.schema_manager().clone(),
22092215
table_schema_id: table.schema().id(),
2216+
merge_engine_overridden: table.merge_engine_overridden(),
22102217
table_fields: table.schema().fields().to_vec(),
22112218
read_type: table.schema().fields().to_vec(),
22122219
predicates: Vec::new(),
@@ -2328,6 +2335,7 @@ mod tests {
23282335
table_options: table.schema().options().clone(),
23292336
schema_manager: table.schema_manager().clone(),
23302337
table_schema_id: table.schema().id(),
2338+
merge_engine_overridden: table.merge_engine_overridden(),
23312339
table_fields: table.schema().fields().to_vec(),
23322340
read_type: table.schema().fields().to_vec(),
23332341
predicates: Vec::new(),
@@ -2533,6 +2541,7 @@ mod tests {
25332541
table_options: table.schema().options().clone(),
25342542
schema_manager: table.schema_manager().clone(),
25352543
table_schema_id: table.schema().id(),
2544+
merge_engine_overridden: table.merge_engine_overridden(),
25362545
table_fields: table.schema().fields().to_vec(),
25372546
read_type: table.schema().fields().to_vec(),
25382547
predicates: Vec::new(),
@@ -2622,6 +2631,7 @@ mod tests {
26222631
table_options: table.schema().options().clone(),
26232632
schema_manager: table.schema_manager().clone(),
26242633
table_schema_id: table.schema().id(),
2634+
merge_engine_overridden: table.merge_engine_overridden(),
26252635
table_fields: table.schema().fields().to_vec(),
26262636
read_type: table.schema().fields().to_vec(),
26272637
predicates: Vec::new(),
@@ -2889,6 +2899,7 @@ mod tests {
28892899
table_options: table.schema().options().clone(),
28902900
schema_manager: table.schema_manager().clone(),
28912901
table_schema_id: table.schema().id(),
2902+
merge_engine_overridden: table.merge_engine_overridden(),
28922903
table_fields: table.schema().fields().to_vec(),
28932904
read_type: table.schema().fields().to_vec(),
28942905
predicates: Vec::new(),
@@ -2971,6 +2982,37 @@ mod tests {
29712982
file
29722983
}
29732984

2985+
async fn read_kv_split(table: &Table, split: DataSplit) -> Vec<RecordBatch> {
2986+
let core_options = table.schema().core_options();
2987+
KeyValueFileReader::new(
2988+
table.file_io().clone(),
2989+
KeyValueReadConfig {
2990+
table_name: table.identifier().full_name(),
2991+
table_options: table.schema().options().clone(),
2992+
schema_manager: table.schema_manager().clone(),
2993+
table_schema_id: table.schema().id(),
2994+
merge_engine_overridden: table.merge_engine_overridden(),
2995+
table_fields: table.schema().fields().to_vec(),
2996+
read_type: table.schema().fields().to_vec(),
2997+
predicates: Vec::new(),
2998+
primary_keys: table.schema().trimmed_primary_keys(),
2999+
table_primary_keys: table.schema().primary_keys().to_vec(),
3000+
merge_engine: core_options.merge_engine().unwrap(),
3001+
sequence_fields: Vec::new(),
3002+
read_batch_size: core_options.read_batch_size().unwrap(),
3003+
merge_splits: false,
3004+
max_merge_input_streams: None,
3005+
parquet_read_budget: None,
3006+
mosaic_prefetch: MosaicPrefetchOptions::default(),
3007+
},
3008+
)
3009+
.read(&[split])
3010+
.unwrap()
3011+
.try_collect::<Vec<_>>()
3012+
.await
3013+
.unwrap()
3014+
}
3015+
29743016
#[tokio::test]
29753017
async fn single_run_read_merges_files_written_under_aggregation() {
29763018
// Before #839 the aggregation writer kept several versions of a key in
@@ -3004,33 +3046,48 @@ mod tests {
30043046
.with_data_files(vec![a, b])
30053047
.build()
30063048
.unwrap();
3007-
let core_options = table.schema().core_options();
3008-
let batches = KeyValueFileReader::new(
3009-
table.file_io().clone(),
3010-
KeyValueReadConfig {
3011-
table_name: table.identifier().full_name(),
3012-
table_options: table.schema().options().clone(),
3013-
schema_manager: table.schema_manager().clone(),
3014-
table_schema_id: table.schema().id(),
3015-
table_fields: table.schema().fields().to_vec(),
3016-
read_type: table.schema().fields().to_vec(),
3017-
predicates: Vec::new(),
3018-
primary_keys: table.schema().trimmed_primary_keys(),
3019-
table_primary_keys: table.schema().primary_keys().to_vec(),
3020-
merge_engine: core_options.merge_engine().unwrap(),
3021-
sequence_fields: Vec::new(),
3022-
read_batch_size: core_options.read_batch_size().unwrap(),
3023-
merge_splits: false,
3024-
max_merge_input_streams: None,
3025-
parquet_read_budget: None,
3026-
mosaic_prefetch: MosaicPrefetchOptions::default(),
3027-
},
3028-
)
3029-
.read(&[split])
3030-
.unwrap()
3031-
.try_collect::<Vec<_>>()
3032-
.await
3033-
.unwrap();
3049+
let batches = read_kv_split(&table, split).await;
3050+
assert_eq!(int_column(&batches, "id"), vec![1, 2]);
3051+
assert_eq!(int_column(&batches, "value"), vec![11, 20]);
3052+
}
3053+
3054+
#[tokio::test]
3055+
async fn single_run_read_merges_when_options_override_the_engine() {
3056+
// copy_with_options keeps the schema ID, so the files and the read
3057+
// schema share it while the stored engine is still aggregation.
3058+
let file_io = test_file_io();
3059+
let table_path = "memory:/kv_engine_override";
3060+
let old = pk_table(
3061+
&file_io,
3062+
table_path,
3063+
&[
3064+
("merge-engine", "aggregation"),
3065+
("fields.value.aggregate-function", "sum"),
3066+
],
3067+
);
3068+
write_schema_file(&old, old.schema()).await;
3069+
let table = old.copy_with_options(HashMap::from([(
3070+
"merge-engine".to_string(),
3071+
"deduplicate".to_string(),
3072+
)]));
3073+
assert_eq!(table.schema().id(), old.schema().id());
3074+
assert_eq!(
3075+
table.schema().core_options().merge_engine().unwrap(),
3076+
MergeEngine::Deduplicate
3077+
);
3078+
let a =
3079+
write_kv_versions(&file_io, table_path, "a.parquet", &[(1, 0, 10), (1, 1, 11)]).await;
3080+
let b = write_kv_versions(&file_io, table_path, "b.parquet", &[(2, 2, 20)]).await;
3081+
let split = DataSplitBuilder::new()
3082+
.with_snapshot(1)
3083+
.with_partition(BinaryRow::new(0))
3084+
.with_bucket(0)
3085+
.with_bucket_path(format!("{table_path}/bucket-0"))
3086+
.with_total_buckets(1)
3087+
.with_data_files(vec![a, b])
3088+
.build()
3089+
.unwrap();
3090+
let batches = read_kv_split(&table, split).await;
30343091
assert_eq!(int_column(&batches, "id"), vec![1, 2]);
30353092
assert_eq!(int_column(&batches, "value"), vec![11, 20]);
30363093
}
@@ -3067,6 +3124,7 @@ mod tests {
30673124
table_options: table.schema().options().clone(),
30683125
schema_manager: table.schema_manager().clone(),
30693126
table_schema_id: table.schema().id(),
3127+
merge_engine_overridden: table.merge_engine_overridden(),
30703128
table_fields: table.schema().fields().to_vec(),
30713129
read_type: table.schema().fields().to_vec(),
30723130
predicates: Vec::new(),
@@ -3136,6 +3194,7 @@ mod tests {
31363194
table_options: table.schema().options().clone(),
31373195
schema_manager: table.schema_manager().clone(),
31383196
table_schema_id: table.schema().id(),
3197+
merge_engine_overridden: table.merge_engine_overridden(),
31393198
table_fields: table.schema().fields().to_vec(),
31403199
read_type: table.schema().fields().to_vec(),
31413200
predicates: Vec::new(),

‎crates/paimon/src/table/mod.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -718,6 +718,12 @@ impl Table {
718718
PostponeFixedBucketWriteBuilder::new(self)
719719
}
720720

721+
/// Whether dynamic options set `merge-engine`. They keep the schema ID, so
722+
/// the stored schema of that ID may name another engine.
723+
pub(crate) fn merge_engine_overridden(&self) -> bool {
724+
self.applied_dynamic_option_keys.contains("merge-engine")
725+
}
726+
721727
/// Create a copy of this table with extra options merged into the schema.
722728
///
723729
/// This never switches the schema version; it corresponds to Java

‎crates/paimon/src/table/table_read.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1320,6 +1320,7 @@ impl<'a> PaimonTableRead<'a> {
13201320
table_options: self.table.schema().options().clone(),
13211321
schema_manager: self.table.schema_manager().clone(),
13221322
table_schema_id: self.table.schema().id(),
1323+
merge_engine_overridden: self.table.merge_engine_overridden(),
13231324
table_fields: self.table.schema.fields().to_vec(),
13241325
read_type: read_type.to_vec(),
13251326
predicates: self.data_predicates.clone(),
@@ -2007,6 +2008,7 @@ impl<'a> PaimonTableRead<'a> {
20072008
table_options: self.table.schema().options().clone(),
20082009
schema_manager: self.table.schema_manager().clone(),
20092010
table_schema_id: self.table.schema().id(),
2011+
merge_engine_overridden: self.table.merge_engine_overridden(),
20102012
table_fields: self.table.schema.fields().to_vec(),
20112013
read_type: self.read_type().to_vec(),
20122014
predicates: self.data_predicates.clone(),

0 commit comments

Comments
 (0)