Skip to content

Commit fc09271

Browse files
fix: Fix panic on scan_parquet filter of fixed-size binary column (#28122)
1 parent 3dce32b commit fc09271

5 files changed

Lines changed: 63 additions & 6 deletions

File tree

‎crates/polars-arrow/src/array/fixed_size_binary/mutable.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,12 @@ impl MutableFixedSizeBinaryArray {
110110
Some(bytes) => {
111111
let bytes = bytes.as_ref();
112112
if self.size != bytes.len() {
113-
polars_bail!(ComputeError: "FixedSizeBinaryArray requires every item to be of its length")
113+
polars_bail!(
114+
ComputeError:
115+
"MutableFixedSizeBinaryArray(row_width: {}): attempted to push \
116+
row with {} bytes",
117+
self.size, bytes.len(),
118+
)
114119
}
115120
self.values.extend_from_slice(bytes);
116121

‎crates/polars-parquet/src/arrow/read/deserialize/fixed_size_binary.rs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ use crate::parquet::encoding::hybrid_rle::{HybridRleChunk, HybridRleDecoder};
1717
use crate::parquet::encoding::{Encoding, hybrid_rle};
1818
use crate::parquet::error::{ParquetError, ParquetResult};
1919
use crate::parquet::page::{DataPage, DictPage, split_buffer};
20+
use crate::read::PredicateFilter;
2021
use crate::read::deserialize::dictionary_encoded::constrain_page_validity;
2122
use crate::read::deserialize::utils::{self, Decoded};
2223
use crate::read::expr::{ParquetScalar, SpecializedParquetColumnExpr};
@@ -544,6 +545,22 @@ impl Decoder for BinaryDecoder {
544545
))
545546
}
546547

548+
fn evaluate_dict_predicate(
549+
&self,
550+
_dict: &Self::Dict,
551+
_predicate: &PredicateFilter,
552+
) -> ParquetResult<Bitmap> {
553+
// Dispatch to this codepath disabled from crates/polars-stream/src/nodes/io_sources/parquet/init.rs
554+
//
555+
// This would panic attempting to create a polars DataType::Binary Series from a
556+
// dyn Array of type ArrowDataType::FixedSizeBinary(_) (via Series::from_chunk_and_dtype
557+
// in predicate.evaluate_mut()).
558+
//
559+
// TODO: Performant fix would involve refactoring this (BinaryDecoder) to also store Views
560+
// in the intermediate state, and only when a predicate is being applied on this column.
561+
unimplemented!("parquet: fixed-size binary prefilter");
562+
}
563+
547564
fn evaluate_predicate(
548565
&mut self,
549566
_state: &utils::State<'_, Self>,

‎crates/polars-parquet/src/arrow/read/statistics.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -523,10 +523,21 @@ pub fn deserialize_all(
523523
(D::Utf8View, _) => rmap!(@string),
524524

525525
(D::FixedSizeBinary(width), _) => {
526+
struct FixedSizeBinaryArray2;
527+
528+
impl FixedSizeBinaryArray2 {
529+
fn with_capacity(
530+
row_groups_len: usize,
531+
row_width: usize,
532+
) -> MutableFixedSizeBinaryArray {
533+
MutableFixedSizeBinaryArray::with_capacity(row_width, row_groups_len)
534+
}
535+
}
536+
526537
rmap!(
527538
expect_fixedlen,
528539
|x: Option<Vec<u8>>| ParquetResult::Ok(x),
529-
MutableFixedSizeBinaryArray,
540+
FixedSizeBinaryArray2,
530541
*width
531542
)
532543
},

‎crates/polars-stream/src/nodes/io_sources/parquet/init.rs‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
use std::sync::Arc;
22

3+
use arrow::datatypes::ArrowDataType;
34
use polars_async::executor;
45
use polars_core::frame::DataFrame;
56
use polars_core::runtime::ASYNC;
@@ -353,10 +354,13 @@ impl ParquetReadImpl {
353354
)
354355
}
355356

356-
let allow_column_predicates = predicate
357-
.as_ref()
358-
.is_some_and(|x| x.column_predicates.is_sumwise_complete)
359-
&& row_index.is_none()
357+
let allow_column_predicates = predicate.as_ref().is_some_and(|p| {
358+
p.column_predicates.is_sumwise_complete
359+
&& !projected_arrow_fields.iter().any(|f| {
360+
matches!(f.arrow_field().dtype(), ArrowDataType::FixedSizeBinary(_))
361+
&& p.column_predicates.predicates.contains_key(f.output_name())
362+
})
363+
}) && row_index.is_none()
360364
&& !projected_arrow_fields.iter().any(|x| {
361365
x.arrow_field().dtype().is_nested()
362366
|| matches!(x, ArrowFieldProjection::Mapped { .. })

‎py-polars/tests/unit/io/test_parquet.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4416,3 +4416,23 @@ def test_resolve_metadata_sampled_extrapolates(tmp_path: Path) -> None:
44164416
timeout=30,
44174417
)
44184418
assert out == b"OK"
4419+
4420+
4421+
def test_parquet_prefilter_fixed_size_binary_27781() -> None:
4422+
val = b"0x004521bdf6bf838e71c0b977678adae368c3ac5d5c665cef09cbd61a9d591d3f"
4423+
4424+
table = pa.table(
4425+
{
4426+
"market": pa.array([val, val], type=pa.binary(66)),
4427+
"asset_id": ["abc", "def"],
4428+
}
4429+
)
4430+
4431+
f = io.BytesIO()
4432+
pq.write_table(table, f)
4433+
f.seek(0)
4434+
4435+
assert_frame_equal(
4436+
pl.scan_parquet(f).filter(pl.col("market") == val).collect(),
4437+
pl.DataFrame(table),
4438+
)

0 commit comments

Comments
 (0)