Skip to content

Commit ebd3ecb

Browse files
Rich-T-kidJefffrey
andauthored
Optimize PLAIN parquet string column to dictionary array (#10614)
# Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. --> - Closes #10590 - builds on top of #10591 - works towards apache/datafusion#24111 # Rationale for this change - see #10590 <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> # What changes are included in this PR? Optimizes plain-encoded string/binary columns read into Dictionary Arrow arrays by eliminating the intermediate StringArray materialization. Previously, all values were decoded into a flat array first, then iterated again to build the dictionary. Now, hashes are computed upfront in a single batch pass and used to deduplicate values directly into the dictionary output buffers, reducing the work to a single pass over the data. The hash scratch buffer is also reused across batches, so allocation overhead amortizes as batch sizes grow. <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> # Are these changes tested? yes existing test cover this functionality. nothing logically has changed <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? If this PR claims a performance improvement, please include evidence such as benchmark results. --> # Are there any user-facing changes? ~~yes, `DictionaryBuffer::into_array()` now takes in scratch vector. this is a breaking change~~ EDIT: no breaking changes, `DictionaryBuffer` isn't part of public API --------- Co-authored-by: Jeffrey Vo <[email protected]>
1 parent 432fe12 commit ebd3ecb

2 files changed

Lines changed: 342 additions & 60 deletions

File tree

‎parquet/src/arrow/array_reader/byte_array_dictionary.rs‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ use std::any::Any;
1919
use std::marker::PhantomData;
2020

2121
use arrow_array::{Array, ArrayRef, OffsetSizeTrait, new_empty_array};
22-
use arrow_buffer::ArrowNativeType;
22+
use arrow_buffer::{ArrowNativeType, MutableBuffer};
2323
use arrow_schema::DataType as ArrowType;
2424
use bytes::Bytes;
2525

@@ -128,6 +128,8 @@ struct ByteArrayDictionaryReader<K: ArrowNativeType, V: OffsetSizeTrait> {
128128
def_levels_buffer: Option<Vec<i16>>,
129129
rep_levels_buffer: Option<Vec<i16>>,
130130
record_reader: GenericRecordReader<DictionaryBuffer<K, V>, DictionaryDecoder<K, V>>,
131+
/// Reusable scratch space for hashing byte slices when building dictionaries from plain-encoded values.
132+
hash_scratch: MutableBuffer,
131133
}
132134

133135
impl<K, V> ByteArrayDictionaryReader<K, V>
@@ -146,6 +148,7 @@ where
146148
def_levels_buffer: None,
147149
rep_levels_buffer: None,
148150
record_reader,
151+
hash_scratch: MutableBuffer::new(0),
149152
}
150153
}
151154
}
@@ -181,7 +184,7 @@ where
181184

182185
let buffer = self.record_reader.consume_record_data();
183186
let null_buffer = self.record_reader.consume_compact_bitmap();
184-
let array = buffer.into_array(null_buffer, &self.data_type)?;
187+
let array = buffer.into_array(null_buffer, &self.data_type, &mut self.hash_scratch)?;
185188
self.record_reader.reset();
186189

187190
Ok(array)
@@ -459,7 +462,9 @@ mod tests {
459462

460463
assert!(matches!(output, DictionaryBuffer::Dict { .. }));
461464

462-
let array = output.into_array(Some(valid_buffer), &data_type).unwrap();
465+
let array = output
466+
.into_array(Some(valid_buffer), &data_type, &mut MutableBuffer::new(0))
467+
.unwrap();
463468
assert_eq!(array.data_type(), &data_type);
464469

465470
let array = cast(&array, &ArrowType::Utf8).unwrap();
@@ -530,7 +535,9 @@ mod tests {
530535

531536
assert!(matches!(output, DictionaryBuffer::Dict { .. }));
532537

533-
let array = output.into_array(Some(valid_buffer), &data_type).unwrap();
538+
let array = output
539+
.into_array(Some(valid_buffer), &data_type, &mut MutableBuffer::new(0))
540+
.unwrap();
534541
assert_eq!(array.data_type(), &data_type);
535542

536543
let array = cast(&array, &ArrowType::Utf8).unwrap();
@@ -565,7 +572,9 @@ mod tests {
565572
decoder.set_data(encoding, page, 4, Some(4)).unwrap();
566573
assert_eq!(decoder.read(&mut output, 1024).unwrap(), 4);
567574
}
568-
let array = output.into_array(None, &data_type).unwrap();
575+
let array = output
576+
.into_array(None, &data_type, &mut MutableBuffer::new(0))
577+
.unwrap();
569578
assert_eq!(array.data_type(), &data_type);
570579

571580
let array = cast(&array, &ArrowType::Utf8).unwrap();
@@ -609,7 +618,9 @@ mod tests {
609618
decoder.skip_values(2).expect("skipping two values");
610619
assert_eq!(decoder.read(&mut output, 1024).unwrap(), 2);
611620
}
612-
let array = output.into_array(None, &data_type).unwrap();
621+
let array = output
622+
.into_array(None, &data_type, &mut MutableBuffer::new(0))
623+
.unwrap();
613624
assert_eq!(array.data_type(), &data_type);
614625

615626
let array = cast(&array, &ArrowType::Utf8).unwrap();
@@ -672,7 +683,11 @@ mod tests {
672683

673684
output.pad_nulls(0, 0, 8, &[0]).unwrap();
674685
let array = output
675-
.into_array(Some(Buffer::from(&[0])), &data_type)
686+
.into_array(
687+
Some(Buffer::from(&[0])),
688+
&data_type,
689+
&mut MutableBuffer::new(0),
690+
)
676691
.unwrap();
677692

678693
assert_eq!(array.len(), 8);
@@ -687,7 +702,11 @@ mod tests {
687702

688703
output.pad_nulls(0, 0, 8, &[0]).unwrap();
689704
let array = output
690-
.into_array(Some(Buffer::from(&[0])), &data_type)
705+
.into_array(
706+
Some(Buffer::from(&[0])),
707+
&data_type,
708+
&mut MutableBuffer::new(0),
709+
)
691710
.unwrap();
692711

693712
assert_eq!(array.len(), 8);

0 commit comments

Comments
 (0)