Skip to content

Commit d9b754d

Browse files
committed
fix: preserve structured Spark errors in footer validation
1 parent 3c252ca commit d9b754d

2 files changed

Lines changed: 92 additions & 2 deletions

File tree

‎native/core/src/parquet/eager_page_index_reader_factory.rs‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ use crate::parquet::schema_adapter::{
7171
use arrow::datatypes::{DataType, FieldRef, Schema, SchemaRef};
7272
use async_trait::async_trait;
7373
use bytes::Bytes;
74-
use datafusion::common::Result as DFResult;
74+
use datafusion::common::{DataFusionError, Result as DFResult};
7575
use datafusion::datasource::physical_plan::parquet::metadata::DFParquetMetadata;
7676
use datafusion::datasource::physical_plan::parquet::{
7777
ParquetFileMetrics, ParquetFileReaderFactory,
@@ -668,7 +668,12 @@ impl AsyncFileReader for EagerPageIndexReader {
668668
Arc::new(schema),
669669
conversion_options.clone(),
670670
)
671-
.map_err(|e| ParquetError::External(Box::new(e)))?;
671+
.map_err(|e| match e {
672+
// Keep structured Spark errors directly inside the Parquet wrapper so
673+
// the JNI error converter preserves Spark's exception type and class.
674+
DataFusionError::External(source) => ParquetError::External(source),
675+
other => ParquetError::External(Box::new(other)),
676+
})?;
672677
if *row_filters {
673678
if let Some(rejection) = &rejection {
674679
self.deferred_rejections.lock().unwrap().insert(
@@ -1308,6 +1313,64 @@ mod tests {
13081313
assert!(metadata_for(false, without_ids).await.is_ok());
13091314
}
13101315

1316+
#[tokio::test]
1317+
async fn conversion_check_preserves_structured_spark_errors() {
1318+
let store = Arc::new(InMemory::new());
1319+
let schema = Arc::new(Schema::new(vec![
1320+
arrow::datatypes::Field::new("B", DataType::Int32, false),
1321+
arrow::datatypes::Field::new("b", DataType::Int32, false),
1322+
]));
1323+
let values: arrow::array::ArrayRef = Arc::new(Int32Array::from(vec![1, 2]));
1324+
let batch =
1325+
RecordBatch::try_new(Arc::clone(&schema), vec![Arc::clone(&values), values]).unwrap();
1326+
let mut bytes = Vec::new();
1327+
let mut writer = ArrowWriter::try_new(&mut bytes, schema, None).unwrap();
1328+
writer.write(&batch).unwrap();
1329+
writer.close().unwrap();
1330+
let size = bytes.len() as u64;
1331+
let location = "duplicates.parquet";
1332+
store
1333+
.put(&Path::from(location), Bytes::from(bytes).into())
1334+
.await
1335+
.unwrap();
1336+
let runtime = datafusion::execution::runtime_env::RuntimeEnv::default();
1337+
let metrics = ExecutionPlanMetricsSet::new();
1338+
let required = Arc::new(Schema::new(vec![arrow::datatypes::Field::new(
1339+
"B",
1340+
DataType::Int32,
1341+
false,
1342+
)]));
1343+
let mut options =
1344+
SparkParquetOptions::new(datafusion_comet_spark_expr::EvalMode::Legacy, "UTC", false);
1345+
options.case_sensitive = false;
1346+
let factory = EagerPageIndexReaderFactory::new(
1347+
store,
1348+
runtime.cache_manager.get_file_metadata_cache(),
1349+
ScanIoSource::Local,
1350+
&metrics,
1351+
)
1352+
.with_conversion_check(required, options, true);
1353+
let mut reader = factory
1354+
.create_reader(
1355+
0,
1356+
PartitionedFile::new(location.to_string(), size),
1357+
None,
1358+
&metrics,
1359+
)
1360+
.unwrap();
1361+
let error = reader.get_metadata(None).await.unwrap_err();
1362+
let ParquetError::External(source) = error else {
1363+
panic!("expected structured Spark error, got {error}");
1364+
};
1365+
assert!(
1366+
matches!(
1367+
source.downcast_ref::<SparkError>(),
1368+
Some(SparkError::DuplicateFieldCaseInsensitive { .. })
1369+
),
1370+
"unexpected error: {source}"
1371+
);
1372+
}
1373+
13111374
#[test]
13121375
fn variant_policy_preserves_footer_metadata_and_indexes() {
13131376
let schema = Arc::new(Schema::new_with_metadata(

‎spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1730,6 +1730,33 @@ abstract class ParquetReadSuite extends CometTestBase {
17301730
}
17311731
}
17321732

1733+
test("native scan preserves duplicate field error types with row filter pushdown") {
1734+
withTempPath { dir =>
1735+
val path = dir.getCanonicalPath
1736+
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
1737+
spark.range(5).selectExpr("id as A", "id as B", "id as b").write.parquet(path)
1738+
}
1739+
for (pushdown <- Seq(false, true)) {
1740+
withSQLConf(
1741+
SQLConf.USE_V1_SOURCE_LIST.key -> "parquet",
1742+
SQLConf.CASE_SENSITIVE.key -> "false",
1743+
CometConf.COMET_PARQUET_ROW_FILTER_PUSHDOWN_ENABLED.key -> pushdown.toString) {
1744+
val df = spark.read.schema("A long, B long").parquet(path).where("A = 1")
1745+
checkCometOperators(stripAQEPlan(df.queryExecution.executedPlan))
1746+
val (sparkError, cometError) = checkSparkAnswerMaybeThrows(df)
1747+
Seq("Spark" -> sparkError, "Comet" -> cometError).foreach { case (engine, error) =>
1748+
val chain = error.toSeq.flatMap(causeChain)
1749+
assert(
1750+
chain.exists(e =>
1751+
e.getClass.getName == "org.apache.spark.SparkRuntimeException" &&
1752+
e.getMessage.contains("Found duplicate field")),
1753+
s"$engine: ${chain.mkString("\n")}")
1754+
}
1755+
}
1756+
}
1757+
}
1758+
}
1759+
17331760
test("native scan preserves conversion errors with row filter pushdown") {
17341761
for (pushdown <- Seq(false, true); nested <- Seq(false, true)) {
17351762
withSQLConf(

0 commit comments

Comments
 (0)