Skip to content

Commit 66abe34

Browse files
committed
[core] Always check the writer engine in the stored schema
Dynamic options and time travel can both give the read schema another merge engine under the files' schema ID, so the ID match is not enough to trust the read schema's engine. Drop the override flag and look up the stored schema for every file; the schema manager caches it.
1 parent 3515f60 commit 66abe34

4 files changed

Lines changed: 112 additions & 39 deletions

File tree

‎crates/paimon/src/table/audit_log_table/read.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,6 @@ impl<'a> AuditLogRead<'a> {
125125
table_options: self.read.table.schema().options().clone(),
126126
schema_manager: self.read.table.schema_manager().clone(),
127127
table_schema_id: self.read.table.schema().id(),
128-
merge_engine_overridden: self.read.table.merge_engine_overridden(),
129128
table_fields: self.read.table.schema.fields().to_vec(),
130129
read_type,
131130
predicates: self.read.data_predicates.clone(),

‎crates/paimon/src/table/kv_file_reader.rs‎

Lines changed: 112 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -81,9 +81,6 @@ pub(crate) struct KeyValueReadConfig {
8181
/// Full table keys also protect partition-PK fields from aggregation.
8282
pub table_primary_keys: Vec<String>,
8383
pub merge_engine: MergeEngine,
84-
/// `merge-engine` was set by dynamic options, so files written under
85-
/// `table_schema_id` may still hold another engine's output.
86-
pub merge_engine_overridden: bool,
8784
pub sequence_fields: Vec<String>,
8885
pub read_batch_size: usize,
8986
/// Merge files from all supplied splits into one globally key-sorted stream.
@@ -780,15 +777,26 @@ impl KeyValueFileReader {
780777
if skip_merge {
781778
'files: for run in &merge_group {
782779
for file in &run.files {
783-
// Dynamic options can override the engine but keep
784-
// the schema ID, so only then check the stored one.
785-
if file.file.schema_id == table_schema_id
786-
&& !config.merge_engine_overridden
780+
// Always check the stored schema: dynamic options and
781+
// time travel can change the read schema's engine
782+
// without changing its ID. Merging is always correct,
783+
// so a missing schema file under the read schema's ID
784+
// only costs the skip.
785+
let schema = match config
786+
.schema_manager
787+
.schema(file.file.schema_id)
788+
.await
787789
{
788-
continue;
789-
}
790-
let schema =
791-
config.schema_manager.schema(file.file.schema_id).await?;
790+
Ok(schema) => schema,
791+
Err(crate::Error::IoUnexpected { source, .. })
792+
if source.kind() == opendal::ErrorKind::NotFound
793+
&& file.file.schema_id == table_schema_id =>
794+
{
795+
skip_merge = false;
796+
break 'files;
797+
}
798+
Err(err) => Err(err)?,
799+
};
792800
let engine =
793801
CoreOptions::new(schema.options()).merge_engine()?;
794802
if !matches!(
@@ -2076,6 +2084,7 @@ mod tests {
20762084
("deletion-vectors.merge-on-read", "true"),
20772085
],
20782086
);
2087+
write_schema_file(&table, table.schema()).await;
20792088

20802089
let mut files = Vec::new();
20812090
for i in 0..10 {
@@ -2213,7 +2222,6 @@ mod tests {
22132222
table_options: table.schema().options().clone(),
22142223
schema_manager: table.schema_manager().clone(),
22152224
table_schema_id: table.schema().id(),
2216-
merge_engine_overridden: table.merge_engine_overridden(),
22172225
table_fields: table.schema().fields().to_vec(),
22182226
read_type: table.schema().fields().to_vec(),
22192227
predicates: Vec::new(),
@@ -2335,7 +2343,6 @@ mod tests {
23352343
table_options: table.schema().options().clone(),
23362344
schema_manager: table.schema_manager().clone(),
23372345
table_schema_id: table.schema().id(),
2338-
merge_engine_overridden: table.merge_engine_overridden(),
23392346
table_fields: table.schema().fields().to_vec(),
23402347
read_type: table.schema().fields().to_vec(),
23412348
predicates: Vec::new(),
@@ -2541,7 +2548,6 @@ mod tests {
25412548
table_options: table.schema().options().clone(),
25422549
schema_manager: table.schema_manager().clone(),
25432550
table_schema_id: table.schema().id(),
2544-
merge_engine_overridden: table.merge_engine_overridden(),
25452551
table_fields: table.schema().fields().to_vec(),
25462552
read_type: table.schema().fields().to_vec(),
25472553
predicates: Vec::new(),
@@ -2631,7 +2637,6 @@ mod tests {
26312637
table_options: table.schema().options().clone(),
26322638
schema_manager: table.schema_manager().clone(),
26332639
table_schema_id: table.schema().id(),
2634-
merge_engine_overridden: table.merge_engine_overridden(),
26352640
table_fields: table.schema().fields().to_vec(),
26362641
read_type: table.schema().fields().to_vec(),
26372642
predicates: Vec::new(),
@@ -2691,6 +2696,7 @@ mod tests {
26912696
("read.parquet.row-group.parallelism", "1"),
26922697
],
26932698
);
2699+
write_schema_file(&table, table.schema()).await;
26942700
let low =
26952701
write_multi_row_group_kv_file(&file_io, table_path, "low.parquet", 0, 0, 10).await;
26962702
let high =
@@ -2747,6 +2753,7 @@ mod tests {
27472753
("read.parquet.row-group.parallelism", "1"),
27482754
],
27492755
);
2756+
write_schema_file(&table, table.schema()).await;
27502757
let first_low =
27512758
write_multi_row_group_kv_file(&file_io, table_path, "first-low.parquet", 0, 0, 10)
27522759
.await;
@@ -2880,6 +2887,7 @@ mod tests {
28802887
] {
28812888
let table_path = "memory:/kv_legacy_retract";
28822889
let table = pk_table(&file_io, table_path, &options);
2890+
write_schema_file(&table, table.schema()).await;
28832891
let a = write_legacy_kv_file(&file_io, table_path, "a.parquet", &adds).await;
28842892
let b = write_legacy_kv_file(&file_io, table_path, "b.parquet", &retract).await;
28852893
let split = DataSplitBuilder::new()
@@ -2899,7 +2907,6 @@ mod tests {
28992907
table_options: table.schema().options().clone(),
29002908
schema_manager: table.schema_manager().clone(),
29012909
table_schema_id: table.schema().id(),
2902-
merge_engine_overridden: table.merge_engine_overridden(),
29032910
table_fields: table.schema().fields().to_vec(),
29042911
read_type: table.schema().fields().to_vec(),
29052912
predicates: Vec::new(),
@@ -2991,7 +2998,6 @@ mod tests {
29912998
table_options: table.schema().options().clone(),
29922999
schema_manager: table.schema_manager().clone(),
29933000
table_schema_id: table.schema().id(),
2994-
merge_engine_overridden: table.merge_engine_overridden(),
29953001
table_fields: table.schema().fields().to_vec(),
29963002
read_type: table.schema().fields().to_vec(),
29973003
predicates: Vec::new(),
@@ -3052,11 +3058,11 @@ mod tests {
30523058
}
30533059

30543060
#[tokio::test]
3055-
async fn single_run_read_merges_when_options_override_the_engine() {
3056-
// copy_with_options keeps the schema ID, so the files and the read
3057-
// schema share it while the stored engine is still aggregation.
3061+
async fn single_run_read_merges_after_time_travel_past_an_engine_alter() {
3062+
// aggregation (schema 0) -> ALTER to deduplicate (schema 1). Reading
3063+
// snapshot 1 resolves schema 0 but keeps the current engine option.
30583064
let file_io = test_file_io();
3059-
let table_path = "memory:/kv_engine_override";
3065+
let table_path = "memory:/kv_engine_time_travel";
30603066
let old = pk_table(
30613067
&file_io,
30623068
table_path,
@@ -3066,15 +3072,45 @@ mod tests {
30663072
],
30673073
);
30683074
write_schema_file(&old, old.schema()).await;
3069-
let table = old.copy_with_options(HashMap::from([(
3070-
"merge-engine".to_string(),
3071-
"deduplicate".to_string(),
3072-
)]));
3073-
assert_eq!(table.schema().id(), old.schema().id());
3075+
let mut json = serde_json::to_value(old.schema()).unwrap();
3076+
json["id"] = 1.into();
3077+
json["options"]["merge-engine"] = "deduplicate".into();
3078+
let schema: TableSchema = serde_json::from_value(json).unwrap();
3079+
write_schema_file(&old, &schema).await;
3080+
let table = old.copy_with_resolved_schema(schema, "main").unwrap();
3081+
file_io
3082+
.mkdirs(&format!("{table_path}/snapshot/"))
3083+
.await
3084+
.unwrap();
3085+
let snapshot = crate::spec::Snapshot::builder()
3086+
.version(3)
3087+
.id(1)
3088+
.schema_id(0)
3089+
.base_manifest_list("base-list".to_string())
3090+
.delta_manifest_list("delta-list".to_string())
3091+
.commit_user("test-user".to_string())
3092+
.commit_identifier(0)
3093+
.commit_kind(crate::spec::CommitKind::APPEND)
3094+
.time_millis(1000)
3095+
.build();
3096+
assert!(table
3097+
.snapshot_manager()
3098+
.commit_snapshot(&snapshot)
3099+
.await
3100+
.unwrap());
3101+
let historical = table
3102+
.copy_with_time_travel_strict(HashMap::from([(
3103+
"scan.version".to_string(),
3104+
"1".to_string(),
3105+
)]))
3106+
.await
3107+
.unwrap();
3108+
assert_eq!(historical.schema().id(), 0);
30743109
assert_eq!(
3075-
table.schema().core_options().merge_engine().unwrap(),
3110+
historical.schema().core_options().merge_engine().unwrap(),
30763111
MergeEngine::Deduplicate
30773112
);
3113+
30783114
let a =
30793115
write_kv_versions(&file_io, table_path, "a.parquet", &[(1, 0, 10), (1, 1, 11)]).await;
30803116
let b = write_kv_versions(&file_io, table_path, "b.parquet", &[(2, 2, 20)]).await;
@@ -3087,16 +3123,64 @@ mod tests {
30873123
.with_data_files(vec![a, b])
30883124
.build()
30893125
.unwrap();
3090-
let batches = read_kv_split(&table, split).await;
3126+
let batches = read_kv_split(&historical, split).await;
30913127
assert_eq!(int_column(&batches, "id"), vec![1, 2]);
30923128
assert_eq!(int_column(&batches, "value"), vec![11, 20]);
30933129
}
30943130

3131+
#[tokio::test]
3132+
async fn single_run_read_merges_when_the_read_schema_names_another_engine() {
3133+
// Dynamic options and time travel can give the read schema another
3134+
// engine under the files' schema ID; the stored engine is aggregation.
3135+
let file_io = test_file_io();
3136+
let table_path = "memory:/kv_engine_override";
3137+
let old = pk_table(
3138+
&file_io,
3139+
table_path,
3140+
&[
3141+
("merge-engine", "aggregation"),
3142+
("fields.value.aggregate-function", "sum"),
3143+
],
3144+
);
3145+
write_schema_file(&old, old.schema()).await;
3146+
let overridden = old.copy_with_options(HashMap::from([(
3147+
"merge-engine".to_string(),
3148+
"deduplicate".to_string(),
3149+
)]));
3150+
// Same read schema as a historical read after an ALTER to deduplicate.
3151+
let rewrapped = old
3152+
.copy_with_resolved_schema(overridden.schema().clone(), "main")
3153+
.unwrap();
3154+
let a =
3155+
write_kv_versions(&file_io, table_path, "a.parquet", &[(1, 0, 10), (1, 1, 11)]).await;
3156+
let b = write_kv_versions(&file_io, table_path, "b.parquet", &[(2, 2, 20)]).await;
3157+
let split = DataSplitBuilder::new()
3158+
.with_snapshot(1)
3159+
.with_partition(BinaryRow::new(0))
3160+
.with_bucket(0)
3161+
.with_bucket_path(format!("{table_path}/bucket-0"))
3162+
.with_total_buckets(1)
3163+
.with_data_files(vec![a, b])
3164+
.build()
3165+
.unwrap();
3166+
for table in [&overridden, &rewrapped] {
3167+
assert_eq!(table.schema().id(), old.schema().id());
3168+
assert_eq!(
3169+
table.schema().core_options().merge_engine().unwrap(),
3170+
MergeEngine::Deduplicate
3171+
);
3172+
let batches = read_kv_split(table, split.clone()).await;
3173+
assert_eq!(int_column(&batches, "id"), vec![1, 2]);
3174+
assert_eq!(int_column(&batches, "value"), vec![11, 20]);
3175+
}
3176+
}
3177+
30953178
#[tokio::test]
30963179
async fn sorted_run_read_matches_per_file_fan_out() {
30973180
let file_io = test_file_io();
30983181
let table_path = "memory:/kv_sorted_run";
30993182
let table = pk_table(&file_io, table_path, &[]);
3183+
write_schema_file(&table, table.schema()).await;
31003184
let low =
31013185
write_multi_row_group_kv_file(&file_io, table_path, "low.parquet", 0, 0, 10).await;
31023186
let high =
@@ -3124,7 +3208,6 @@ mod tests {
31243208
table_options: table.schema().options().clone(),
31253209
schema_manager: table.schema_manager().clone(),
31263210
table_schema_id: table.schema().id(),
3127-
merge_engine_overridden: table.merge_engine_overridden(),
31283211
table_fields: table.schema().fields().to_vec(),
31293212
read_type: table.schema().fields().to_vec(),
31303213
predicates: Vec::new(),
@@ -3194,7 +3277,6 @@ mod tests {
31943277
table_options: table.schema().options().clone(),
31953278
schema_manager: table.schema_manager().clone(),
31963279
table_schema_id: table.schema().id(),
3197-
merge_engine_overridden: table.merge_engine_overridden(),
31983280
table_fields: table.schema().fields().to_vec(),
31993281
read_type: table.schema().fields().to_vec(),
32003282
predicates: Vec::new(),

‎crates/paimon/src/table/mod.rs‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -718,12 +718,6 @@ impl Table {
718718
PostponeFixedBucketWriteBuilder::new(self)
719719
}
720720

721-
/// Whether dynamic options set `merge-engine`. They keep the schema ID, so
722-
/// the stored schema of that ID may name another engine.
723-
pub(crate) fn merge_engine_overridden(&self) -> bool {
724-
self.applied_dynamic_option_keys.contains("merge-engine")
725-
}
726-
727721
/// Create a copy of this table with extra options merged into the schema.
728722
///
729723
/// This never switches the schema version; it corresponds to Java

‎crates/paimon/src/table/table_read.rs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1320,7 +1320,6 @@ impl<'a> PaimonTableRead<'a> {
13201320
table_options: self.table.schema().options().clone(),
13211321
schema_manager: self.table.schema_manager().clone(),
13221322
table_schema_id: self.table.schema().id(),
1323-
merge_engine_overridden: self.table.merge_engine_overridden(),
13241323
table_fields: self.table.schema.fields().to_vec(),
13251324
read_type: read_type.to_vec(),
13261325
predicates: self.data_predicates.clone(),
@@ -2008,7 +2007,6 @@ impl<'a> PaimonTableRead<'a> {
20082007
table_options: self.table.schema().options().clone(),
20092008
schema_manager: self.table.schema_manager().clone(),
20102009
table_schema_id: self.table.schema().id(),
2011-
merge_engine_overridden: self.table.merge_engine_overridden(),
20122010
table_fields: self.table.schema.fields().to_vec(),
20132011
read_type: self.read_type().to_vec(),
20142012
predicates: self.data_predicates.clone(),

0 commit comments

Comments
 (0)