Skip to content

Commit 3c252ca

Browse files
committed
fix: preserve Parquet conversion failures before row filtering
1 parent c1dba19 commit 3c252ca

5 files changed

Lines changed: 505 additions & 25 deletions

File tree

‎docs/source/user-guide/latest/compatibility/scans.md‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -125,12 +125,11 @@ and `INT32 → DOUBLE` widening that Spark 4.0+ accepts unconditionally; `Timest
125125
is rejected by Spark 3.x but accepted by Spark 4.0+). Comet aims to follow the per-version Spark
126126
behavior.
127127

128-
- **A pushed row filter can skip the mismatch error**. Like Spark, Comet raises most mismatches only
129-
while it decodes a file's rows, so an empty file, or one whose row groups are all pruned, reads
130-
without error. With `spark.comet.parquet.rowFilterPushdown.enabled=true` (off by default), the scan
131-
decodes a column that the filter doesn't read only for the rows that pass the filter, so a file in
132-
which no row passes also reads without error. Spark decodes every row of a row group it doesn't
133-
prune, so it raises the error for that file.
128+
- **Conversion errors precede pushed row filters**. Empty files and files whose row groups or
129+
pages are all pruned read without decode-time conversion errors. When row-filter pushdown is
130+
enabled, Comet checks rejected conversions before reading surviving data pages, so a row filter
131+
cannot suppress the error by discarding every row. Legacy LIST shape mismatches that Spark cannot
132+
clip are rejected when the file is opened, including for empty and fully pruned files.
134133
- **List conversion error paths assume Spark's standard encoding**. Comet inserts `list`
135134
before the element name when reporting a rejected array element conversion. Arrow's schema
136135
omits the repeated group name, so paths for legacy LIST encodings or custom group names may

‎native/core/src/parquet/eager_page_index_reader_factory.rs‎

Lines changed: 133 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,7 @@
4545
//!
4646
//! Filed upstream as apache/datafusion#23978. Once the opener merges its deferred page-index
4747
//! load back into `FileMetadataCache` instead of bypassing it, the eager policy can go, but the
48-
//! factory cannot: `get_metadata` is the one per-file hook that sees the raw footer, and two
49-
//! other things hang off it.
48+
//! factory cannot: `get_metadata` is the per-file hook that sees the raw footer.
5049
//!
5150
//! The first is Spark's missing field id check. `ParquetReadSupport` refuses to open a file
5251
//! whose Parquet schema carries no field id when the requested schema carries one, unless
@@ -59,8 +58,17 @@
5958
//!
6059
//! The second is the Variant footer rewrite, `with_spark_arrow_schema`, which replaces the
6160
//! Arrow schema hint in the footer for scans that project Variant.
62-
63-
use arrow::datatypes::{DataType, FieldRef, Schema};
61+
//!
62+
//! Conversion checks also use the raw footer to preserve legacy LIST clipping failures. With
63+
//! row filters enabled, a deferred conversion failure is raised on the first data-page request.
64+
//! Footer, Bloom-filter and page-index reads still proceed, so format pruning can discard a file
65+
//! without decoding it, but row selection cannot hide the failure.
66+
67+
use crate::parquet::parquet_support::SparkParquetOptions;
68+
use crate::parquet::schema_adapter::{
69+
check_file_conversions, RejectOnNonEmpty, REPEATED_PRIMITIVE_KEY,
70+
};
71+
use arrow::datatypes::{DataType, FieldRef, Schema, SchemaRef};
6472
use async_trait::async_trait;
6573
use bytes::Bytes;
6674
use datafusion::common::Result as DFResult;
@@ -93,10 +101,11 @@ use parquet::file::metadata::{
93101
FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader,
94102
};
95103
use parquet::schema::types::{ColumnDescPtr, SchemaDescriptor, Type as ParquetType};
104+
use std::collections::HashMap;
96105
use std::fmt::{Debug, Display, Formatter};
97106
use std::ops::Range;
98107
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
99-
use std::sync::{Arc, OnceLock};
108+
use std::sync::{Arc, Mutex, OnceLock};
100109

101110
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
102111
pub(crate) enum ScanIoSource {
@@ -181,6 +190,8 @@ pub struct EagerPageIndexReaderFactory {
181190
// Refuse a file whose Parquet schema carries no field id, as Spark's `ParquetReadSupport`
182191
// does when the requested schema carries one and `ignoreMissing` is not set.
183192
require_field_ids: bool,
193+
conversion_check: Option<(SchemaRef, SparkParquetOptions, bool)>,
194+
deferred_rejections: Arc<Mutex<HashMap<Path, (ObjectMeta, RejectOnNonEmpty)>>>,
184195
}
185196

186197
impl EagerPageIndexReaderFactory {
@@ -210,9 +221,23 @@ impl EagerPageIndexReaderFactory {
210221
scan_io_metrics,
211222
spark_variant_schema: false,
212223
require_field_ids: false,
224+
conversion_check: None,
225+
deferred_rejections: Arc::new(Mutex::new(HashMap::new())),
213226
}
214227
}
215228

229+
/// Retain raw LIST encoding for open-time validation. When row filters are enabled,
230+
/// enforce deferred failures at the first data-page read, after format pruning.
231+
pub(crate) fn with_conversion_check(
232+
mut self,
233+
required: SchemaRef,
234+
options: SparkParquetOptions,
235+
row_filters: bool,
236+
) -> Self {
237+
self.conversion_check = Some((required, options, row_filters));
238+
self
239+
}
240+
216241
pub fn with_spark_variant_schema(mut self, enabled: bool) -> Self {
217242
self.spark_variant_schema = enabled;
218243
self
@@ -254,6 +279,15 @@ impl ParquetFileReaderFactory for EagerPageIndexReaderFactory {
254279
metrics,
255280
);
256281

282+
// DataFusion replaces the metadata reader after Bloom-filter pruning. The replacement
283+
// receives already-loaded metadata, so retain deferred failures in this scan's factory.
284+
let deferred_rejection = self
285+
.deferred_rejections
286+
.lock()
287+
.unwrap()
288+
.get(&partitioned_file.object_meta.location)
289+
.filter(|(meta, _)| meta == &partitioned_file.object_meta)
290+
.map(|(_, rejection)| rejection.clone());
257291
Ok(Box::new(EagerPageIndexReader {
258292
file_metrics,
259293
scan_io_metrics: Arc::clone(&self.scan_io_metrics),
@@ -263,6 +297,9 @@ impl ParquetFileReaderFactory for EagerPageIndexReaderFactory {
263297
metadata_size_hint,
264298
spark_variant_schema: self.spark_variant_schema,
265299
require_field_ids: self.require_field_ids,
300+
conversion_check: self.conversion_check.clone(),
301+
deferred_rejections: Arc::clone(&self.deferred_rejections),
302+
deferred_rejection,
266303
}))
267304
}
268305
}
@@ -279,6 +316,9 @@ struct EagerPageIndexReader {
279316
metadata_size_hint: Option<usize>,
280317
spark_variant_schema: bool,
281318
require_field_ids: bool,
319+
conversion_check: Option<(SchemaRef, SparkParquetOptions, bool)>,
320+
deferred_rejections: Arc<Mutex<HashMap<Path, (ObjectMeta, RejectOnNonEmpty)>>>,
321+
deferred_rejection: Option<RejectOnNonEmpty>,
282322
}
283323

284324
// Arrow infers ENUM as Binary, losing the distinction from raw binary that Spark needs.
@@ -411,6 +451,53 @@ fn with_spark_arrow_schema(metadata: Arc<ParquetMetaData>) -> ParquetResult<Arc<
411451
))
412452
}
413453

454+
/// Arrow normalizes both standard and legacy LISTs to the same type. Carry the repeated
455+
/// primitive distinction in an ephemeral schema used by the conversion check, never in the
456+
/// shared footer cache or in the schema given to the decoder.
457+
fn mark_repeated_primitives(schema: &Schema, parquet: &SchemaDescriptor) -> Schema {
458+
fn mark(field: &FieldRef, columns: &[ColumnDescPtr], index: &mut usize) -> FieldRef {
459+
let data_type = match field.data_type() {
460+
DataType::Struct(fields) => DataType::Struct(
461+
fields
462+
.iter()
463+
.map(|f| mark(f, columns, index))
464+
.collect::<Vec<_>>()
465+
.into(),
466+
),
467+
DataType::List(item) => DataType::List(mark(item, columns, index)),
468+
DataType::LargeList(item) => DataType::LargeList(mark(item, columns, index)),
469+
DataType::FixedSizeList(item, size) => {
470+
DataType::FixedSizeList(mark(item, columns, index), *size)
471+
}
472+
DataType::Map(entries, sorted) => DataType::Map(mark(entries, columns, index), *sorted),
473+
_ => {
474+
let repeated = columns.get(*index).is_some_and(|column| {
475+
column.self_type().get_basic_info().repetition()
476+
== parquet::basic::Repetition::REPEATED
477+
});
478+
*index += 1;
479+
let mut field = field.as_ref().clone();
480+
if repeated {
481+
let mut metadata = field.metadata().clone();
482+
metadata.insert(REPEATED_PRIMITIVE_KEY.to_string(), "true".to_string());
483+
field = field.with_metadata(metadata);
484+
}
485+
return Arc::new(field);
486+
}
487+
};
488+
Arc::new(field.as_ref().clone().with_data_type(data_type))
489+
}
490+
let mut index = 0;
491+
Schema::new_with_metadata(
492+
schema
493+
.fields()
494+
.iter()
495+
.map(|f| mark(f, parquet.columns(), &mut index))
496+
.collect::<Vec<_>>(),
497+
schema.metadata().clone(),
498+
)
499+
}
500+
414501
impl AsyncFileReader for EagerPageIndexReader {
415502
/// Reads a metadata range, counting its requested size before I/O and its returned
416503
/// bytes only on success. The returned future borrows this reader; store errors retain
@@ -449,6 +536,12 @@ impl AsyncFileReader for EagerPageIndexReader {
449536
where
450537
Self: Send,
451538
{
539+
if ranges.iter().any(|range| !range.is_empty()) {
540+
if let Some(rejection) = &self.deferred_rejection {
541+
let error = ParquetError::External(Box::new(rejection.error()));
542+
return async move { Err(error) }.boxed();
543+
}
544+
}
452545
let requested = ranges_bytes(&ranges);
453546
self.file_metrics.bytes_scanned.add(requested);
454547
let scan_io_metrics = Arc::clone(&self.scan_io_metrics);
@@ -550,11 +643,43 @@ impl AsyncFileReader for EagerPageIndexReader {
550643
},
551644
)));
552645
}
553-
if spark_variant_schema {
554-
with_spark_arrow_schema(metadata)
646+
let metadata = if spark_variant_schema {
647+
with_spark_arrow_schema(metadata)?
555648
} else {
556-
Ok(metadata)
649+
metadata
650+
};
651+
if let Some((required, conversion_options, row_filters)) =
652+
self.conversion_check.as_ref()
653+
{
654+
let file = metadata.file_metadata();
655+
let needs_check = *row_filters
656+
|| file.schema_descr().columns().iter().any(|column| {
657+
column.self_type().get_basic_info().repetition()
658+
== parquet::basic::Repetition::REPEATED
659+
});
660+
if !needs_check {
661+
return Ok(metadata);
662+
}
663+
let schema =
664+
parquet_to_arrow_schema(file.schema_descr(), file.key_value_metadata())?;
665+
let schema = mark_repeated_primitives(&schema, file.schema_descr());
666+
let rejection = check_file_conversions(
667+
Arc::clone(required),
668+
Arc::new(schema),
669+
conversion_options.clone(),
670+
)
671+
.map_err(|e| ParquetError::External(Box::new(e)))?;
672+
if *row_filters {
673+
if let Some(rejection) = &rejection {
674+
self.deferred_rejections.lock().unwrap().insert(
675+
object_meta.location.clone(),
676+
(object_meta.clone(), rejection.clone()),
677+
);
678+
}
679+
self.deferred_rejection = rejection;
680+
}
557681
}
682+
Ok(metadata)
558683
}
559684
.boxed()
560685
}

‎native/core/src/parquet/parquet_exec.rs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,15 @@ pub(crate) fn init_datasource_exec(
196196
parquet_source.metrics(),
197197
)
198198
.with_spark_variant_schema(projects_variant)
199-
.with_require_field_ids(require_field_ids),
199+
.with_require_field_ids(require_field_ids)
200+
.with_conversion_check(
201+
Arc::clone(&required_schema),
202+
spark_parquet_options.clone(),
203+
session_config.options().execution.parquet.pushdown_filters
204+
&& data_filters
205+
.as_ref()
206+
.is_some_and(|filters| !filters.is_empty()),
207+
),
200208
);
201209
parquet_source = parquet_source.with_parquet_file_reader_factory(reader_factory);
202210

0 commit comments

Comments
 (0)