Skip to content

Commit ca5eb6f

Browse files
committed
[core] Keep merging files written under another merge engine
A file written under aggregation could hold several versions of a key (before #839), and ALTER can switch the table to deduplicate afterwards. Skip the sort merge only when every file was written under deduplicate or first-row.
1 parent 9ed7a1e commit ca5eb6f

1 file changed

Lines changed: 146 additions & 1 deletion

File tree

‎crates/paimon/src/table/kv_file_reader.rs‎

Lines changed: 146 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -765,12 +765,36 @@ impl KeyValueFileReader {
765765
// every key is a lone version and needs no merge. Retract
766766
// rows are still dropped per batch below: older Rust writers
767767
// recorded `delete_row_count = 0` for files holding them.
768-
let skip_merge = skip_single_runs
768+
let mut skip_merge = skip_single_runs
769769
&& input_stream_count == 1
770770
&& matches!(
771771
config.merge_engine,
772772
MergeEngine::Deduplicate | MergeEngine::FirstRow
773773
);
774+
// A file written under another engine may hold several
775+
// versions of a key (the aggregation writer kept them before
776+
// #839), and ALTER can switch the engine afterwards.
777+
if skip_merge {
778+
'files: for run in &merge_group {
779+
for file in &run.files {
780+
let schema_id = file.file.schema_id;
781+
if schema_id == table_schema_id {
782+
continue;
783+
}
784+
let schema =
785+
config.schema_manager.schema(schema_id).await?;
786+
let engine =
787+
CoreOptions::new(schema.options()).merge_engine()?;
788+
if !matches!(
789+
engine,
790+
MergeEngine::Deduplicate | MergeEngine::FirstRow
791+
) {
792+
skip_merge = false;
793+
break 'files;
794+
}
795+
}
796+
}
797+
}
774798
let reject_retracts = config.merge_engine == MergeEngine::FirstRow
775799
&& !CoreOptions::new(&config.table_options).ignore_delete();
776800

@@ -2890,6 +2914,127 @@ mod tests {
28902914
}
28912915
}
28922916

2917+
/// Writes one KV file with `(id, sequence, value)` rows, under schema 0.
2918+
async fn write_kv_versions(
2919+
file_io: &FileIO,
2920+
table_path: &str,
2921+
file_name: &str,
2922+
rows: &[(i32, i64, i32)],
2923+
) -> DataFileMeta {
2924+
let schema = crate::arrow::build_target_arrow_schema(&[
2925+
DataField::new(
2926+
SEQUENCE_NUMBER_FIELD_ID,
2927+
SEQUENCE_NUMBER_FIELD_NAME.to_string(),
2928+
DataType::BigInt(BigIntType::new()),
2929+
),
2930+
DataField::new(
2931+
VALUE_KIND_FIELD_ID,
2932+
VALUE_KIND_FIELD_NAME.to_string(),
2933+
DataType::TinyInt(TinyIntType::new()),
2934+
),
2935+
DataField::new(0, "id".to_string(), DataType::Int(IntType::new())),
2936+
DataField::new(1, "value".to_string(), DataType::Int(IntType::new())),
2937+
])
2938+
.unwrap();
2939+
let batch = RecordBatch::try_new(
2940+
schema.clone(),
2941+
vec![
2942+
Arc::new(Int64Array::from_iter_values(rows.iter().map(|row| row.1))),
2943+
Arc::new(Int8Array::from_value(0, rows.len())),
2944+
Arc::new(Int32Array::from_iter_values(rows.iter().map(|row| row.0))),
2945+
Arc::new(Int32Array::from_iter_values(rows.iter().map(|row| row.2))),
2946+
],
2947+
)
2948+
.unwrap();
2949+
let mut bytes = Vec::new();
2950+
{
2951+
let mut writer = AsyncArrowWriter::try_new(&mut bytes, schema, None).unwrap();
2952+
writer.write(&batch).await.unwrap();
2953+
writer.close().await.unwrap();
2954+
}
2955+
let bucket_path = format!("{table_path}/bucket-0");
2956+
file_io.mkdirs(&format!("{bucket_path}/")).await.unwrap();
2957+
file_io
2958+
.new_output(&format!("{bucket_path}/{file_name}"))
2959+
.unwrap()
2960+
.write(bytes::Bytes::from(bytes.clone()))
2961+
.await
2962+
.unwrap();
2963+
2964+
let mut file = dummy_data_file(file_name.to_string());
2965+
file.file_size = bytes.len() as i64;
2966+
file.row_count = rows.len() as i64;
2967+
file.min_key = int_key(rows[0].0);
2968+
file.max_key = int_key(rows[rows.len() - 1].0);
2969+
file.min_sequence_number = rows.iter().map(|row| row.1).min().unwrap();
2970+
file.max_sequence_number = rows.iter().map(|row| row.1).max().unwrap();
2971+
file
2972+
}
2973+
2974+
#[tokio::test]
2975+
async fn single_run_read_merges_files_written_under_aggregation() {
2976+
// Before #839 the aggregation writer kept several versions of a key in
2977+
// one file. After ALTER to deduplicate, such a file still needs merging.
2978+
let file_io = test_file_io();
2979+
let table_path = "memory:/kv_engine_change";
2980+
let old = pk_table(
2981+
&file_io,
2982+
table_path,
2983+
&[
2984+
("merge-engine", "aggregation"),
2985+
("fields.value.aggregate-function", "sum"),
2986+
],
2987+
);
2988+
write_schema_file(&old, old.schema()).await;
2989+
let mut json = serde_json::to_value(old.schema()).unwrap();
2990+
json["id"] = 1.into();
2991+
json["options"]["merge-engine"] = "deduplicate".into();
2992+
let schema: TableSchema = serde_json::from_value(json).unwrap();
2993+
write_schema_file(&old, &schema).await;
2994+
let table = old.copy_with_resolved_schema(schema, "main").unwrap();
2995+
let a =
2996+
write_kv_versions(&file_io, table_path, "a.parquet", &[(1, 0, 10), (1, 1, 11)]).await;
2997+
let b = write_kv_versions(&file_io, table_path, "b.parquet", &[(2, 2, 20)]).await;
2998+
let split = DataSplitBuilder::new()
2999+
.with_snapshot(1)
3000+
.with_partition(BinaryRow::new(0))
3001+
.with_bucket(0)
3002+
.with_bucket_path(format!("{table_path}/bucket-0"))
3003+
.with_total_buckets(1)
3004+
.with_data_files(vec![a, b])
3005+
.build()
3006+
.unwrap();
3007+
let core_options = table.schema().core_options();
3008+
let batches = KeyValueFileReader::new(
3009+
table.file_io().clone(),
3010+
KeyValueReadConfig {
3011+
table_name: table.identifier().full_name(),
3012+
table_options: table.schema().options().clone(),
3013+
schema_manager: table.schema_manager().clone(),
3014+
table_schema_id: table.schema().id(),
3015+
table_fields: table.schema().fields().to_vec(),
3016+
read_type: table.schema().fields().to_vec(),
3017+
predicates: Vec::new(),
3018+
primary_keys: table.schema().trimmed_primary_keys(),
3019+
table_primary_keys: table.schema().primary_keys().to_vec(),
3020+
merge_engine: core_options.merge_engine().unwrap(),
3021+
sequence_fields: Vec::new(),
3022+
read_batch_size: core_options.read_batch_size().unwrap(),
3023+
merge_splits: false,
3024+
max_merge_input_streams: None,
3025+
parquet_read_budget: None,
3026+
mosaic_prefetch: MosaicPrefetchOptions::default(),
3027+
},
3028+
)
3029+
.read(&[split])
3030+
.unwrap()
3031+
.try_collect::<Vec<_>>()
3032+
.await
3033+
.unwrap();
3034+
assert_eq!(int_column(&batches, "id"), vec![1, 2]);
3035+
assert_eq!(int_column(&batches, "value"), vec![11, 20]);
3036+
}
3037+
28933038
#[tokio::test]
28943039
async fn sorted_run_read_matches_per_file_fan_out() {
28953040
let file_io = test_file_io();

0 commit comments

Comments
 (0)