Skip to content

Commit c8b5893

Browse files
committed
[core] Read single-run primary-key sections without sort merge
A section with one sorted run of files that hold no retract rows, under the deduplicate or first-row engine, has no key to merge: the writer keeps keys unique within a file and the run's files don't overlap. Planning already reads such a split raw when it has one file. KeyValueFileReader now streams these sections directly instead of running the per-row merge, so small non-overlapping files packed into one split no longer pay for it.
1 parent 46e6871 commit c8b5893

2 files changed

Lines changed: 182 additions & 20 deletions

File tree

‎crates/paimon/src/table/kv_file_reader.rs‎

Lines changed: 53 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -676,6 +676,20 @@ impl KeyValueFileReader {
676676
.map(|budget| Arc::new(budget.without_prefetch()))
677677
};
678678
let mut file_streams: Vec<ArrowRecordBatchStream> = Vec::new();
679+
// One sorted run of files without retract rows, under an
680+
// engine whose files never repeat a key, has nothing to
681+
// merge. Planning trusts the same invariant when it marks a
682+
// single-file split raw convertible.
683+
let skip_merge = input_stream_count == 1
684+
&& matches!(
685+
config.merge_engine,
686+
MergeEngine::Deduplicate | MergeEngine::FirstRow
687+
)
688+
&& merge_group.iter().all(|run| {
689+
run.files
690+
.iter()
691+
.all(|file| file.file.delete_row_count == Some(0))
692+
});
679693

680694
for MergeRun { files } in merge_group {
681695
let reader = DataFileReader::new(
@@ -778,25 +792,42 @@ impl KeyValueFileReader {
778792
file_streams.push(run_stream);
779793
}
780794

781-
// Always go through sort-merge even for a single file: files
782-
// written before the writer merged key groups at flush may
783-
// still contain duplicate keys.
784-
let mut merge_stream = SortMergeReaderBuilder::new(
785-
file_streams,
786-
internal_schema.clone(),
787-
key_indices.clone(),
788-
seq_index,
789-
value_kind_index,
790-
user_sequence_indices.clone(),
791-
value_indices.clone(),
792-
merge_output_schema.clone(),
793-
merge_function(&config, &merge_output_fields)?,
794-
)
795-
.with_user_sequence_descending(
796-
!CoreOptions::new(&config.table_options)
797-
.sequence_field_sort_order_is_ascending(),
798-
)
799-
.build()?;
795+
let mut merge_stream = if skip_merge {
796+
let input = file_streams.pop().expect("one input stream");
797+
let columns: Vec<usize> =
798+
key_indices.iter().chain(&value_indices).copied().collect();
799+
let schema = merge_output_schema.clone();
800+
input
801+
.map(move |batch| {
802+
let batch = batch?;
803+
let columns =
804+
columns.iter().map(|&i| batch.column(i).clone()).collect();
805+
RecordBatch::try_new(schema.clone(), columns).map_err(|e| {
806+
Error::UnexpectedError {
807+
message: format!("Failed to project unmerged batch: {e}"),
808+
source: Some(Box::new(e)),
809+
}
810+
})
811+
})
812+
.boxed()
813+
} else {
814+
SortMergeReaderBuilder::new(
815+
file_streams,
816+
internal_schema.clone(),
817+
key_indices.clone(),
818+
seq_index,
819+
value_kind_index,
820+
user_sequence_indices.clone(),
821+
value_indices.clone(),
822+
merge_output_schema.clone(),
823+
merge_function(&config, &merge_output_fields)?,
824+
)
825+
.with_user_sequence_descending(
826+
!CoreOptions::new(&config.table_options)
827+
.sequence_field_sort_order_is_ascending(),
828+
)
829+
.build()?
830+
};
800831

801832
while let Some(batch) = merge_stream.next().await {
802833
let batch = batch?;
@@ -1868,7 +1899,9 @@ mod tests {
18681899
.await
18691900
.expect("the first output batch")
18701901
.expect("a late deletion vector must be loaded lazily");
1871-
assert_eq!(first.num_rows(), 128);
1902+
// These delete-free files form one run, which streams without the
1903+
// merge, so the first batch is one input batch.
1904+
assert!(first.num_rows() > 0);
18721905

18731906
let err = stream.try_collect::<Vec<_>>().await.unwrap_err();
18741907
assert!(
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Primary-key reads of a split whose sections each hold one sorted run.
19+
20+
mod common;
21+
22+
use std::sync::Arc;
23+
24+
use arrow_array::{ArrayRef, Int32Array, RecordBatch};
25+
use common::incremental_helpers::{memory_table, persist_table_schema, setup_dirs, write_batch};
26+
use futures::TryStreamExt;
27+
use paimon::spec::{DataType, Datum, IntType, PredicateBuilder, Schema, TableSchema};
28+
use paimon::table::Table;
29+
30+
async fn pk_table() -> Table {
31+
let schema = Schema::builder()
32+
.column("id", DataType::Int(IntType::new()))
33+
.column("v", DataType::Int(IntType::new()))
34+
.primary_key(["id"])
35+
.option("bucket", "1")
36+
.build()
37+
.unwrap();
38+
let path = "memory:/pk_single_run_read";
39+
let (io, table) = memory_table(path, TableSchema::new(0, &schema));
40+
setup_dirs(&io, path).await;
41+
persist_table_schema(&io, path, table.schema()).await;
42+
table
43+
}
44+
45+
async fn write(table: &Table, ids: std::ops::Range<i32>, offset: i32) {
46+
let ids: Vec<i32> = ids.collect();
47+
let values: Vec<i32> = ids.iter().map(|id| id + offset).collect();
48+
write_batch(
49+
table,
50+
&RecordBatch::try_from_iter([
51+
("id", Arc::new(Int32Array::from(ids)) as ArrayRef),
52+
("v", Arc::new(Int32Array::from(values)) as ArrayRef),
53+
])
54+
.unwrap(),
55+
)
56+
.await;
57+
}
58+
59+
async fn read(
60+
table: &Table,
61+
projection: &[&str],
62+
filter: Option<paimon::spec::Predicate>,
63+
) -> Vec<Vec<i32>> {
64+
let mut builder = table.new_read_builder();
65+
builder.with_projection(projection).unwrap();
66+
if let Some(filter) = filter {
67+
builder.with_filter(filter);
68+
}
69+
let plan = builder.new_scan().plan().await.unwrap();
70+
let batches: Vec<RecordBatch> = builder
71+
.new_read()
72+
.unwrap()
73+
.to_arrow(plan.splits())
74+
.unwrap()
75+
.try_collect()
76+
.await
77+
.unwrap();
78+
let mut rows: Vec<Vec<i32>> = batches
79+
.iter()
80+
.flat_map(|batch| {
81+
(0..batch.num_rows()).map(move |row| {
82+
(0..batch.num_columns())
83+
.map(|col| {
84+
batch
85+
.column(col)
86+
.as_any()
87+
.downcast_ref::<Int32Array>()
88+
.unwrap()
89+
.value(row)
90+
})
91+
.collect()
92+
})
93+
})
94+
.collect();
95+
rows.sort_unstable();
96+
rows
97+
}
98+
99+
#[tokio::test]
100+
async fn disjoint_files_read_without_merge_and_overlaps_still_merge() {
101+
let table = pk_table().await;
102+
// Three files with disjoint keys land in one split, one run per section.
103+
write(&table, 0..100, 1000).await;
104+
write(&table, 100..200, 1000).await;
105+
write(&table, 200..300, 1000).await;
106+
let plan = table.new_read_builder().new_scan().plan().await.unwrap();
107+
assert_eq!(plan.splits().len(), 1);
108+
109+
let expected: Vec<Vec<i32>> = (0..300).map(|id| vec![id, id + 1000]).collect();
110+
assert_eq!(read(&table, &["id", "v"], None).await, expected);
111+
// Value-only projection and a residual filter on the unmerged path.
112+
let pb = PredicateBuilder::new(table.schema().fields());
113+
assert_eq!(
114+
read(
115+
&table,
116+
&["v"],
117+
Some(pb.greater_or_equal("v", Datum::Int(1250)).unwrap())
118+
)
119+
.await,
120+
(1250..1300).map(|v| vec![v]).collect::<Vec<_>>()
121+
);
122+
123+
// Overlapping keys make a two-run section, which still merges.
124+
write(&table, 150..160, 2000).await;
125+
let expected: Vec<Vec<i32>> = (0..300)
126+
.map(|id| vec![id, id + if (150..160).contains(&id) { 2000 } else { 1000 }])
127+
.collect();
128+
assert_eq!(read(&table, &["id", "v"], None).await, expected);
129+
}

0 commit comments

Comments
 (0)