Skip to content

Commit 476312c

Browse files
committed
[core] Read single-run primary-key sections without sort merge
A section with one sorted run of files that hold no retract rows, under the deduplicate or first-row engine, has no key to merge: the writer keeps keys unique within a file and the run's files don't overlap. Planning already reads such a split raw when it has one file. KeyValueFileReader now streams these sections directly instead of running the per-row merge, so small non-overlapping files packed into one split no longer pay for it.
1 parent cea0caa commit 476312c

2 files changed

Lines changed: 349 additions & 23 deletions

File tree

‎crates/paimon/src/table/kv_file_reader.rs‎

Lines changed: 220 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,14 @@ use crate::file_index::file_index_result::FileIndexResult;
3939
use crate::io::FileIO;
4040
use crate::spec::{
4141
is_variant_extraction_row, BigIntType, CoreOptions, DataField, DataFileMeta,
42-
DataType as PaimonDataType, MergeEngine, PartialUpdateConfig, Predicate, TinyIntType,
42+
DataType as PaimonDataType, MergeEngine, PartialUpdateConfig, Predicate, RowKind, TinyIntType,
4343
KEY_FIELD_ID_START, SEQUENCE_NUMBER_FIELD_ID, SEQUENCE_NUMBER_FIELD_NAME, VALUE_KIND_FIELD_ID,
4444
VALUE_KIND_FIELD_NAME,
4545
};
4646
use crate::table::schema_manager::SchemaManager;
4747
use crate::table::ArrowRecordBatchStream;
4848
use crate::{DataSplit, DeletionFile, Error};
49-
use arrow_array::{RecordBatch, RecordBatchOptions};
49+
use arrow_array::{Array, BooleanArray, Int8Array, RecordBatch, RecordBatchOptions};
5050

5151
use async_stream::try_stream;
5252
use futures::StreamExt;
@@ -184,6 +184,43 @@ fn ensure_merge_input_limit(input_stream_count: usize, limit: Option<usize>) ->
184184
Ok(())
185185
}
186186

187+
/// Handles retract rows of an unmerged single run the way the merge path
188+
/// handles a lone version of a key: first-row without `ignore-delete` rejects
189+
/// them, otherwise they are omitted.
190+
fn drop_lone_retracts(
191+
batch: RecordBatch,
192+
value_kind_index: usize,
193+
reject: bool,
194+
) -> crate::Result<RecordBatch> {
195+
// The merge cursor reads a missing or NULL `_VALUE_KIND` as INSERT.
196+
let Some(kinds) = batch
197+
.column(value_kind_index)
198+
.as_any()
199+
.downcast_ref::<Int8Array>()
200+
else {
201+
return Ok(batch);
202+
};
203+
if kinds.null_count() == 0 && kinds.values().iter().all(|&kind| kind == 0 || kind == 2) {
204+
return Ok(batch);
205+
}
206+
let mut keep = Vec::with_capacity(kinds.len());
207+
for kind in kinds.iter() {
208+
let is_add = RowKind::from_value(kind.unwrap_or(0))?.is_add();
209+
if !is_add && reject {
210+
return Err(Error::Unsupported {
211+
message: "merge-engine=first-row does not support DELETE or UPDATE_BEFORE rows; set ignore-delete=true to ignore them".to_string(),
212+
});
213+
}
214+
keep.push(is_add);
215+
}
216+
arrow_select::filter::filter_record_batch(&batch, &BooleanArray::from(keep)).map_err(|e| {
217+
Error::DataInvalid {
218+
message: format!("Failed to drop retract rows: {e}"),
219+
source: Some(Box::new(e)),
220+
}
221+
})
222+
}
223+
187224
/// Java's `KeyValueFieldsExtractor` builds the physical file layout from the
188225
/// schema that wrote the file, including its historical key names and types.
189226
pub(super) fn key_value_fields_extractor(
@@ -397,15 +434,28 @@ impl KeyValueFileReader {
397434
}
398435

399436
pub fn read(self, data_splits: &[DataSplit]) -> crate::Result<ArrowRecordBatchStream> {
400-
self.read_with_merge_function(data_splits, Self::new_merge_function)
437+
self.read_merged(data_splits, Self::new_merge_function, true)
401438
}
402439

440+
/// Always merges, so a custom merge function (e.g. the audit log, which
441+
/// keeps retract rows) sees every row.
403442
pub(super) fn read_with_merge_function(
404443
self,
405444
data_splits: &[DataSplit],
406445
merge_function: impl Fn(&KeyValueReadConfig, &[DataField]) -> crate::Result<Box<dyn MergeFunction>>
407446
+ Send
408447
+ 'static,
448+
) -> crate::Result<ArrowRecordBatchStream> {
449+
self.read_merged(data_splits, merge_function, false)
450+
}
451+
452+
fn read_merged(
453+
self,
454+
data_splits: &[DataSplit],
455+
merge_function: impl Fn(&KeyValueReadConfig, &[DataField]) -> crate::Result<Box<dyn MergeFunction>>
456+
+ Send
457+
+ 'static,
458+
skip_single_runs: bool,
409459
) -> crate::Result<ArrowRecordBatchStream> {
410460
// A projected `_ROW_ID` is synthesized as all-nulls here, so the residual
411461
// would silently drop every row rather than hit its missing-column guard.
@@ -711,6 +761,18 @@ impl KeyValueFileReader {
711761
.map(|budget| Arc::new(budget.without_prefetch()))
712762
};
713763
let mut file_streams: Vec<ArrowRecordBatchStream> = Vec::new();
764+
// One sorted run never repeats a key, so under these engines
765+
// every key is a lone version and needs no merge. Retract
766+
// rows are still dropped per batch below: older Rust writers
767+
// recorded `delete_row_count = 0` for files holding them.
768+
let skip_merge = skip_single_runs
769+
&& input_stream_count == 1
770+
&& matches!(
771+
config.merge_engine,
772+
MergeEngine::Deduplicate | MergeEngine::FirstRow
773+
);
774+
let reject_retracts = config.merge_engine == MergeEngine::FirstRow
775+
&& !CoreOptions::new(&config.table_options).ignore_delete();
714776

715777
for MergeRun { files } in merge_group {
716778
let reader = DataFileReader::new(
@@ -814,25 +876,43 @@ impl KeyValueFileReader {
814876
file_streams.push(run_stream);
815877
}
816878

817-
// Always go through sort-merge even for a single file: files
818-
// written before the writer merged key groups at flush may
819-
// still contain duplicate keys.
820-
let mut merge_stream = SortMergeReaderBuilder::new(
821-
file_streams,
822-
internal_schema.clone(),
823-
key_indices.clone(),
824-
seq_index,
825-
value_kind_index,
826-
user_sequence_indices.clone(),
827-
value_indices.clone(),
828-
merge_output_schema.clone(),
829-
merge_function(&config, &merge_output_fields)?,
830-
)
831-
.with_user_sequence_descending(
832-
!CoreOptions::new(&config.table_options)
833-
.sequence_field_sort_order_is_ascending(),
834-
)
835-
.build()?;
879+
let mut merge_stream = if skip_merge {
880+
let input = file_streams.pop().expect("one input stream");
881+
let columns: Vec<usize> =
882+
key_indices.iter().chain(&value_indices).copied().collect();
883+
let schema = merge_output_schema.clone();
884+
input
885+
.map(move |batch| {
886+
let batch =
887+
drop_lone_retracts(batch?, value_kind_index, reject_retracts)?;
888+
let columns =
889+
columns.iter().map(|&i| batch.column(i).clone()).collect();
890+
RecordBatch::try_new(schema.clone(), columns).map_err(|e| {
891+
Error::UnexpectedError {
892+
message: format!("Failed to project unmerged batch: {e}"),
893+
source: Some(Box::new(e)),
894+
}
895+
})
896+
})
897+
.boxed()
898+
} else {
899+
SortMergeReaderBuilder::new(
900+
file_streams,
901+
internal_schema.clone(),
902+
key_indices.clone(),
903+
seq_index,
904+
value_kind_index,
905+
user_sequence_indices.clone(),
906+
value_indices.clone(),
907+
merge_output_schema.clone(),
908+
merge_function(&config, &merge_output_fields)?,
909+
)
910+
.with_user_sequence_descending(
911+
!CoreOptions::new(&config.table_options)
912+
.sequence_field_sort_order_is_ascending(),
913+
)
914+
.build()?
915+
};
836916

837917
while let Some(batch) = merge_stream.next().await {
838918
let batch = batch?;
@@ -2011,7 +2091,9 @@ mod tests {
20112091
.await
20122092
.expect("the first output batch")
20132093
.expect("a late deletion vector must be loaded lazily");
2014-
assert_eq!(first.num_rows(), 128);
2094+
// These delete-free files form one run, which streams without the
2095+
// merge, so the first batch is one input batch.
2096+
assert!(first.num_rows() > 0);
20152097

20162098
let err = stream.try_collect::<Vec<_>>().await.unwrap_err();
20172099
assert!(
@@ -2693,6 +2775,121 @@ mod tests {
26932775
);
26942776
}
26952777

2778+
/// Writes one KV file with the given keys and row kinds, recorded with
2779+
/// `delete_row_count = Some(0)` as Rust writers did before #318.
2780+
async fn write_legacy_kv_file(
2781+
file_io: &FileIO,
2782+
table_path: &str,
2783+
file_name: &str,
2784+
rows: &[(i32, i8)],
2785+
) -> DataFileMeta {
2786+
let schema = crate::arrow::build_target_arrow_schema(&[
2787+
DataField::new(
2788+
SEQUENCE_NUMBER_FIELD_ID,
2789+
SEQUENCE_NUMBER_FIELD_NAME.to_string(),
2790+
DataType::BigInt(BigIntType::new()),
2791+
),
2792+
DataField::new(
2793+
VALUE_KIND_FIELD_ID,
2794+
VALUE_KIND_FIELD_NAME.to_string(),
2795+
DataType::TinyInt(TinyIntType::new()),
2796+
),
2797+
DataField::new(0, "id".to_string(), DataType::Int(IntType::new())),
2798+
DataField::new(1, "value".to_string(), DataType::Int(IntType::new())),
2799+
])
2800+
.unwrap();
2801+
let batch = RecordBatch::try_new(
2802+
schema.clone(),
2803+
vec![
2804+
Arc::new(Int64Array::from_value(0, rows.len())),
2805+
Arc::new(Int8Array::from_iter_values(rows.iter().map(|row| row.1))),
2806+
Arc::new(Int32Array::from_iter_values(rows.iter().map(|row| row.0))),
2807+
Arc::new(Int32Array::from_value(0, rows.len())),
2808+
],
2809+
)
2810+
.unwrap();
2811+
let mut bytes = Vec::new();
2812+
{
2813+
let mut writer = AsyncArrowWriter::try_new(&mut bytes, schema, None).unwrap();
2814+
writer.write(&batch).await.unwrap();
2815+
writer.close().await.unwrap();
2816+
}
2817+
let bucket_path = format!("{table_path}/bucket-0");
2818+
file_io.mkdirs(&format!("{bucket_path}/")).await.unwrap();
2819+
file_io
2820+
.new_output(&format!("{bucket_path}/{file_name}"))
2821+
.unwrap()
2822+
.write(bytes::Bytes::from(bytes.clone()))
2823+
.await
2824+
.unwrap();
2825+
2826+
let mut file = dummy_data_file(file_name.to_string());
2827+
file.file_size = bytes.len() as i64;
2828+
file.row_count = rows.len() as i64;
2829+
file.min_key = int_key(rows[0].0);
2830+
file.max_key = int_key(rows[rows.len() - 1].0);
2831+
file
2832+
}
2833+
2834+
#[tokio::test]
2835+
async fn single_run_read_drops_retracts_hidden_by_legacy_metadata() {
2836+
let file_io = test_file_io();
2837+
let adds = [(1, 0), (2, 0)];
2838+
let retract = [(3, 3)];
2839+
for (options, expected) in [
2840+
(vec![], Some(vec![1, 2])),
2841+
(vec![("merge-engine", "first-row")], None),
2842+
(
2843+
vec![("merge-engine", "first-row"), ("ignore-delete", "true")],
2844+
Some(vec![1, 2]),
2845+
),
2846+
] {
2847+
let table_path = "memory:/kv_legacy_retract";
2848+
let table = pk_table(&file_io, table_path, &options);
2849+
let a = write_legacy_kv_file(&file_io, table_path, "a.parquet", &adds).await;
2850+
let b = write_legacy_kv_file(&file_io, table_path, "b.parquet", &retract).await;
2851+
let split = DataSplitBuilder::new()
2852+
.with_snapshot(1)
2853+
.with_partition(BinaryRow::new(0))
2854+
.with_bucket(0)
2855+
.with_bucket_path(format!("{table_path}/bucket-0"))
2856+
.with_total_buckets(1)
2857+
.with_data_files(vec![a, b])
2858+
.build()
2859+
.unwrap();
2860+
let core_options = table.schema().core_options();
2861+
let result = KeyValueFileReader::new(
2862+
table.file_io().clone(),
2863+
KeyValueReadConfig {
2864+
table_name: table.identifier().full_name(),
2865+
table_options: table.schema().options().clone(),
2866+
schema_manager: table.schema_manager().clone(),
2867+
table_schema_id: table.schema().id(),
2868+
table_fields: table.schema().fields().to_vec(),
2869+
read_type: table.schema().fields().to_vec(),
2870+
predicates: Vec::new(),
2871+
primary_keys: table.schema().trimmed_primary_keys(),
2872+
table_primary_keys: table.schema().primary_keys().to_vec(),
2873+
merge_engine: core_options.merge_engine().unwrap(),
2874+
sequence_fields: Vec::new(),
2875+
read_batch_size: core_options.read_batch_size().unwrap(),
2876+
merge_splits: false,
2877+
max_merge_input_streams: None,
2878+
parquet_read_budget: None,
2879+
mosaic_prefetch: MosaicPrefetchOptions::default(),
2880+
},
2881+
)
2882+
.read(&[split])
2883+
.unwrap()
2884+
.try_collect::<Vec<_>>()
2885+
.await;
2886+
match expected {
2887+
Some(ids) => assert_eq!(int_column(&result.unwrap(), "id"), ids, "{options:?}"),
2888+
None => assert!(result.is_err(), "{options:?}"),
2889+
}
2890+
}
2891+
}
2892+
26962893
#[tokio::test]
26972894
async fn sorted_run_read_matches_per_file_fan_out() {
26982895
let file_io = test_file_io();

0 commit comments

Comments
 (0)