Skip to content

Commit b1130d3

Browse files
committed
fix: reject BINARY-as-DECIMAL, re-ignore parquet-mr permissive tests (apache#4351, apache#4352)
Two follow-ups to 80836b1, which unignored four `ParquetTypeWideningSuite` test groups in `dev/diffs/4.1.1.diff` and surfaced a remaining schema-adapter gap on top. Schema adapter (apache#4351): a Parquet BINARY column without a `DecimalLogicalTypeAnnotation` was silently allowed through when the requested schema was `DecimalType`. The cast then fell through and Arrow's `RecordBatch::try_new` raised a generic `column types must match schema types` error as `CometNativeException` instead of the Spark-equivalent `SchemaColumnConvertNotSupportedException`. Drop `Decimal128/Decimal256` from the BINARY/STRING allowed-target match in `replace_with_spark_cast`, and add the same rejection in `wrap_all_type_mismatches` so the fallback path doesn't construct a `CometCastColumnExpr` for a cast it can't actually perform. New unit test `parquet_binary_read_as_decimal_errors`. With this fix, Spark's SPARK-34212 in `ParquetQuerySuite` passes without an ignore tag. Test diffs (apache#4352): re-add `IgnoreCometNativeDataFusion` to the five test groups in `dev/diffs/4.1.1.diff`'s `ParquetTypeWideningSuite` whose failure isn't a rejection-logic bug but a parquet-mr-only behavior assertion (`expectError = vectorized` paths and the explicit `overflows with parquet-mr` test). Same annotation added to the previously-unannotated overflow test in `dev/diffs/4.0.2.diff`. The schema-adapter rejection is correct; Comet just has no parquet-mr-equivalent backend that produces silent overflow on the non-vectorized path. Docs: new `Schema Mismatch Handling` subsection in the parquet scan compat guide. Explains that these gaps only apply to explicit schemas or schema evolution (not plain `spark.read.parquet(path)`), notes cross-version differences, and lists the one remaining gap (apache#4316: missing file path in `ParquetSchemaConvert` errors).
1 parent 80836b1 commit b1130d3

4 files changed

Lines changed: 195 additions & 17 deletions

File tree

‎dev/diffs/4.0.2.diff‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2895,7 +2895,7 @@ index 0acb21f3e6f..e7c65429119 100644
28952895
val e = testSchemaMismatch(dir.getCanonicalPath, vectorizedReaderEnabled = true)
28962896
assert(e.getCause.isInstanceOf[SchemaColumnConvertNotSupportedException])
28972897
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala
2898-
index 09ed6955a51..5cd856ff7b6 100644
2898+
index 09ed6955a51..dd7b39573a2 100644
28992899
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala
29002900
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala
29012901
@@ -24,7 +24,7 @@ import org.apache.parquet.format.converter.ParquetMetadataConverter
@@ -2987,6 +2987,16 @@ index 09ed6955a51..5cd856ff7b6 100644
29872987
) {
29882988
checkAllParquetReaders(
29892989
values = Seq("1.23", "10.34"),
2990+
@@ -336,7 +344,8 @@ class ParquetTypeWideningSuite
2991+
)
2992+
}
2993+
2994+
- test("parquet decimal type change Decimal(5, 2) -> Decimal(3, 2) overflows with parquet-mr") {
2995+
+ test("parquet decimal type change Decimal(5, 2) -> Decimal(3, 2) overflows with parquet-mr",
2996+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
2997+
withTempDir { dir =>
2998+
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
2999+
writeParquetFiles(
29903000
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
29913001
index b8f3ea3c6f3..bbd44221288 100644
29923002
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala

‎dev/diffs/4.1.1.diff‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3036,9 +3036,18 @@ index 56076175d60..5872d9962cc 100644
30363036
val e = testSchemaMismatch(dir.getCanonicalPath, vectorizedReaderEnabled = true)
30373037
assert(e.getCause.isInstanceOf[SchemaColumnConvertNotSupportedException])
30383038
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala
3039-
index 09ed6955a51..236a4e99824 100644
3039+
index 09ed6955a51..82924c83eb5 100644
30403040
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala
30413041
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala
3042+
@@ -24,7 +24,7 @@ import org.apache.parquet.format.converter.ParquetMetadataConverter
3043+
import org.apache.parquet.hadoop.{ParquetFileReader, ParquetOutputFormat}
3044+
3045+
import org.apache.spark.SparkException
3046+
-import org.apache.spark.sql.{DataFrame, QueryTest, Row}
3047+
+import org.apache.spark.sql.{DataFrame, IgnoreCometNativeDataFusion, QueryTest, Row}
3048+
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
3049+
import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException
3050+
import org.apache.spark.sql.functions.col
30423051
@@ -65,7 +65,9 @@ class ParquetTypeWideningSuite
30433052
withClue(
30443053
s"with dictionary encoding '$dictionaryEnabled' with timestamp rebase mode " +
@@ -3069,6 +3078,66 @@ index 09ed6955a51..236a4e99824 100644
30693078
)
30703079
}
30713080
test(s"parquet widening conversion $fromType -> $toType") {
3081+
@@ -231,7 +234,8 @@ class ParquetTypeWideningSuite
3082+
(Seq("2020-01-01", "2020-01-02", "1312-02-27"), DateType, TimestampType)
3083+
)
3084+
}
3085+
- test(s"unsupported parquet conversion $fromType -> $toType") {
3086+
+ test(s"unsupported parquet conversion $fromType -> $toType",
3087+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
3088+
checkAllParquetReaders(values, fromType, toType, expectError = true)
3089+
}
3090+
3091+
@@ -257,7 +261,8 @@ class ParquetTypeWideningSuite
3092+
(Seq("1", "2"), LongType, DecimalType(LongDecimal.precision, 1))
3093+
)
3094+
}
3095+
- test(s"unsupported parquet conversion $fromType -> $toType") {
3096+
+ test(s"unsupported parquet conversion $fromType -> $toType",
3097+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
3098+
checkAllParquetReaders(values, fromType, toType,
3099+
expectError =
3100+
// parquet-mr allows reading decimals into a smaller precision decimal type without
3101+
@@ -271,7 +276,8 @@ class ParquetTypeWideningSuite
3102+
(Seq("2020-01-01", "2020-01-02", "1312-02-27"), TimestampNTZType, DateType))
3103+
outputTimestampType <- ParquetOutputTimestampType.values
3104+
}
3105+
- test(s"unsupported parquet timestamp conversion $fromType ($outputTimestampType) -> $toType") {
3106+
+ test(s"unsupported parquet timestamp conversion $fromType ($outputTimestampType) -> $toType",
3107+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
3108+
withSQLConf(
3109+
SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> outputTimestampType.toString,
3110+
SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key -> LegacyBehaviorPolicy.CORRECTED.toString
3111+
@@ -291,7 +297,8 @@ class ParquetTypeWideningSuite
3112+
Seq(7 -> 5, 10 -> 5, 20 -> 5, 12 -> 10, 20 -> 10, 22 -> 20)
3113+
}
3114+
test(
3115+
- s"parquet decimal precision change Decimal($fromPrecision, 2) -> Decimal($toPrecision, 2)") {
3116+
+ s"parquet decimal precision change Decimal($fromPrecision, 2) -> Decimal($toPrecision, 2)",
3117+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
3118+
checkAllParquetReaders(
3119+
values = Seq("1.23", "10.34"),
3120+
fromType = DecimalType(fromPrecision, 2),
3121+
@@ -322,7 +329,8 @@ class ParquetTypeWideningSuite
3122+
Seq((5, 2) -> (6, 4), (10, 4) -> (12, 7), (20, 5) -> (22, 8))
3123+
}
3124+
test(s"parquet decimal precision and scale change Decimal($fromPrecision, $fromScale) -> " +
3125+
- s"Decimal($toPrecision, $toScale)"
3126+
+ s"Decimal($toPrecision, $toScale)",
3127+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")
3128+
) {
3129+
checkAllParquetReaders(
3130+
values = Seq("1.23", "10.34"),
3131+
@@ -336,7 +344,8 @@ class ParquetTypeWideningSuite
3132+
)
3133+
}
3134+
3135+
- test("parquet decimal type change Decimal(5, 2) -> Decimal(3, 2) overflows with parquet-mr") {
3136+
+ test("parquet decimal type change Decimal(5, 2) -> Decimal(3, 2) overflows with parquet-mr",
3137+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
3138+
withTempDir { dir =>
3139+
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
3140+
writeParquetFiles(
30723141
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala
30733142
index 1cc6d3afbee..8275727fbb4 100644
30743143
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala

‎docs/source/user-guide/latest/compatibility/scans.md‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,36 @@ without falling back to Spark:
9191
([SPARK-47447](https://issues.apache.org/jira/browse/SPARK-47447)) and Comet matches Spark's behavior.
9292
See [#4219](https://github.com/apache/datafusion-comet/issues/4219).
9393

94+
### Schema Mismatch Handling
95+
96+
The issues in this subsection apply only when the requested read schema differs from the schema written
97+
to the Parquet file. They do **not** affect a plain `spark.read.parquet(path)` that infers the schema
98+
from file metadata, because in that case the requested schema and file schema match by construction.
99+
Schema mismatch happens in two real-world scenarios:
100+
101+
1. The user provides an explicit read schema: `spark.read.schema(<schema>).parquet(path)` (or the
102+
equivalent DataFrame API).
103+
2. **Schema evolution / partitioned reads** where files in a single dataset were written at different
104+
times with different types, or a table-format catalog (Iceberg, Delta) records a logical schema
105+
that has evolved past one or more underlying Parquet files. Spark coerces the file types to the
106+
table types at read time.
107+
108+
Spark's vectorized Parquet reader fully validates these conversions in `ParquetVectorUpdaterFactory.getUpdater`
109+
and throws `SchemaColumnConvertNotSupportedException` for unsupported pairs. `native_datafusion` mirrors
110+
that validation in its schema adapter; the entries below are the remaining gaps.
111+
112+
Note that the exact set of accepted conversions has changed between Spark versions
113+
(for example, Spark 3.x's `schemaEvolution.enabled` flag gates `INT32 → INT64`, `FLOAT → DOUBLE`,
114+
and `INT32 → DOUBLE` widening that Spark 4.0+ accepts unconditionally; `TimestampLTZ → TimestampNTZ`
115+
is rejected by Spark 3.x but accepted by Spark 4.0+). Comet aims to follow the per-version Spark
116+
behavior.
117+
118+
- **`ParquetSchemaConvert` errors do not include the file path**. The mismatch itself is detected and
119+
rejected correctly, but the resulting Spark error message reads
120+
`Encountered error while reading file . Data type mismatches…` (note the empty path). Behavior is
121+
consistent across Spark versions. See
122+
[#4316](https://github.com/apache/datafusion-comet/issues/4316).
123+
94124
## `native_iceberg_compat` Limitations
95125

96126
The `native_iceberg_compat` scan has the following additional limitation that may produce incorrect results

‎native/core/src/parquet/schema_adapter.rs‎

Lines changed: 84 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,39 @@ impl SparkPhysicalExprAdapter {
548548
};
549549

550550
if logical_field.data_type() != physical_field.data_type() {
551+
// Reject string/binary -> non-string/binary at planning
552+
// time. Mirrors the same check in `replace_with_spark_cast`;
553+
// applies here because the default adapter rejected the
554+
// cast and we'd otherwise fall through to a CometCastColumnExpr
555+
// that can't actually convert (e.g. BINARY -> DECIMAL with
556+
// no `DecimalLogicalTypeAnnotation` on the file column).
557+
// See #4088 and #4351.
558+
let physical_type = physical_field.data_type();
559+
let target_type = logical_field.data_type();
560+
if matches!(
561+
physical_type,
562+
DataType::Utf8
563+
| DataType::LargeUtf8
564+
| DataType::Binary
565+
| DataType::LargeBinary
566+
) && !matches!(
567+
target_type,
568+
DataType::Utf8
569+
| DataType::LargeUtf8
570+
| DataType::Binary
571+
| DataType::LargeBinary
572+
) {
573+
return Err(DataFusionError::External(Box::new(
574+
SparkError::ParquetSchemaConvert {
575+
file_path: String::new(),
576+
column: format!("[{}]", physical_field.name()),
577+
physical_type: parquet_primitive_name(physical_type)
578+
.to_string(),
579+
spark_type: spark_catalog_name(target_type),
580+
},
581+
)));
582+
}
583+
551584
let cast_expr: Arc<dyn PhysicalExpr> = Arc::new(
552585
CometCastColumnExpr::new(
553586
remapped,
@@ -584,29 +617,32 @@ impl SparkPhysicalExprAdapter {
584617
let target_type = cast.target_field().data_type();
585618

586619
// Reject reading a string/binary Parquet column as anything other
587-
// than string, binary, or a binary-encoded decimal. This mirrors
588-
// Spark's TypeUtil.checkParquetType for the BINARY case (lines
589-
// 208-221): a BINARY (or UTF8-annotated BINARY) physical column is
590-
// only readable as StringType, BinaryType, or a binary-encoded
591-
// decimal; every other target type (numeric, boolean, date,
592-
// timestamp, ...) raises SchemaColumnConvertNotSupportedException.
620+
// than string or binary. Mirrors Spark's
621+
// `ParquetVectorUpdaterFactory.getUpdater` `BINARY` case
622+
// (lines 199-205): a Parquet BINARY (or UTF8-annotated BINARY)
623+
// column is readable as StringType / BinaryType, or as DecimalType
624+
// only when the column carries a `DecimalLogicalTypeAnnotation`
625+
// (`canReadAsDecimal` / `canReadAsBinaryDecimal`). Arrow already
626+
// surfaces decimal-annotated BINARY as `DataType::Decimal128`, so
627+
// observing `physical_type == Binary` here unambiguously means a
628+
// non-decimal source — which Spark rejects with
629+
// `SchemaColumnConvertNotSupportedException`.
593630
//
594631
// Without this guard, Spark's Cast below (in is_adapting_schema
595632
// mode) falls through to DataFusion's cast, which silently parses
596633
// the bytes (returning nulls for non-numeric strings, parsing
597634
// date/timestamp/boolean strings, or in some paths reinterpreting
598-
// raw bytes). See issue #4088.
635+
// raw bytes). For numeric / decimal targets the cast can't run at
636+
// all and Arrow's `RecordBatch::try_new` raises a generic
637+
// `column types must match schema types` error instead of the
638+
// Spark-equivalent `SchemaColumnConvertNotSupportedException`.
639+
// See issues #4088 and #4351.
599640
if matches!(
600641
physical_type,
601642
DataType::Utf8 | DataType::LargeUtf8 | DataType::Binary | DataType::LargeBinary
602643
) && !matches!(
603644
target_type,
604-
DataType::Utf8
605-
| DataType::LargeUtf8
606-
| DataType::Binary
607-
| DataType::LargeBinary
608-
| DataType::Decimal128(_, _)
609-
| DataType::Decimal256(_, _)
645+
DataType::Utf8 | DataType::LargeUtf8 | DataType::Binary | DataType::LargeBinary
610646
) {
611647
return Err(DataFusionError::External(Box::new(
612648
SparkError::ParquetSchemaConvert {
@@ -1104,8 +1140,8 @@ mod test {
11041140
use crate::parquet::schema_adapter::SparkPhysicalExprAdapterFactory;
11051141
use arrow::array::UInt32Array;
11061142
use arrow::array::{
1107-
Date32Array, Decimal128Array, Float32Array, Float64Array, Int32Array, Int64Array,
1108-
StringArray, TimestampMicrosecondArray,
1143+
BinaryArray, Date32Array, Decimal128Array, Float32Array, Float64Array, Int32Array,
1144+
Int64Array, StringArray, TimestampMicrosecondArray,
11091145
};
11101146
use arrow::datatypes::SchemaRef;
11111147
use arrow::datatypes::{DataType, Field, Schema};
@@ -1173,6 +1209,39 @@ mod test {
11731209
Ok(())
11741210
}
11751211

1212+
/// Reading a plain BINARY Parquet column (no `DecimalLogicalTypeAnnotation`)
1213+
/// as `DecimalType` must raise a Spark-compatible `ParquetSchemaConvert`
1214+
/// error. Spark's `ParquetVectorUpdaterFactory.getUpdater` BINARY case
1215+
/// only allows the read when `canReadAsDecimal` / `canReadAsBinaryDecimal`
1216+
/// returns true, both of which require the column to carry a
1217+
/// `DecimalLogicalTypeAnnotation`. See #4351.
1218+
#[tokio::test]
1219+
async fn parquet_binary_read_as_decimal_errors() -> Result<(), DataFusionError> {
1220+
let file_schema =
1221+
Arc::new(Schema::new(vec![Field::new("a", DataType::Binary, false)]));
1222+
let values = Arc::new(BinaryArray::from_vec(vec![b"1.2", b"3.4"]))
1223+
as Arc<dyn arrow::array::Array>;
1224+
let batch = RecordBatch::try_new(Arc::clone(&file_schema), vec![values])?;
1225+
1226+
let required_schema = Arc::new(Schema::new(vec![Field::new(
1227+
"a",
1228+
DataType::Decimal128(37, 1),
1229+
false,
1230+
)]));
1231+
1232+
let err = roundtrip(&batch, required_schema)
1233+
.await
1234+
.expect_err("expected ParquetSchemaConvert error for BINARY -> decimal");
1235+
let msg = err.to_string();
1236+
assert!(
1237+
msg.contains("Column: [[a]]")
1238+
&& msg.contains("Expected: decimal(37,1)")
1239+
&& msg.contains("Found: BINARY"),
1240+
"unexpected error: {msg}"
1241+
);
1242+
Ok(())
1243+
}
1244+
11761245
/// Reading an INT32 Parquet column as a DECIMAL whose integer-precision
11771246
/// (`precision - scale`) cannot represent the full INT32 range must raise
11781247
/// a Spark-compatible `ParquetSchemaConvert` error. Spark's

0 commit comments

Comments
 (0)