Skip to content

Commit f81c8f4

Browse files
committed
test: re-ignore ParquetTypeWideningSuite tests after fallback revert
Commit 2ddece0 reverted the parquet-mr fallback added in apache#4352 but did not restore the IgnoreCometNativeDataFusion annotations that apache#4352 had removed. Without the fallback, those tests run against Comet's native scan and fail because Comet rejects conversions that parquet-mr would silently overflow/null.
1 parent b402205 commit f81c8f4

2 files changed

Lines changed: 140 additions & 2 deletions

File tree

‎dev/diffs/4.0.2.diff‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2877,9 +2877,18 @@ index 0acb21f3e6f..e7c65429119 100644
28772877
val e = testSchemaMismatch(dir.getCanonicalPath, vectorizedReaderEnabled = true)
28782878
assert(e.getCause.isInstanceOf[SchemaColumnConvertNotSupportedException])
28792879
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala
2880-
index 09ed6955a51..236a4e99824 100644
2880+
index 09ed6955a51..82924c83eb5 100644
28812881
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala
28822882
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala
2883+
@@ -24,7 +24,7 @@ import org.apache.parquet.format.converter.ParquetMetadataConverter
2884+
import org.apache.parquet.hadoop.{ParquetFileReader, ParquetOutputFormat}
2885+
2886+
import org.apache.spark.SparkException
2887+
-import org.apache.spark.sql.{DataFrame, QueryTest, Row}
2888+
+import org.apache.spark.sql.{DataFrame, IgnoreCometNativeDataFusion, QueryTest, Row}
2889+
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
2890+
import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException
2891+
import org.apache.spark.sql.functions.col
28832892
@@ -65,7 +65,9 @@ class ParquetTypeWideningSuite
28842893
withClue(
28852894
s"with dictionary encoding '$dictionaryEnabled' with timestamp rebase mode " +
@@ -2910,6 +2919,66 @@ index 09ed6955a51..236a4e99824 100644
29102919
)
29112920
}
29122921
test(s"parquet widening conversion $fromType -> $toType") {
2922+
@@ -231,7 +234,8 @@ class ParquetTypeWideningSuite
2923+
(Seq("2020-01-01", "2020-01-02", "1312-02-27"), DateType, TimestampType)
2924+
)
2925+
}
2926+
- test(s"unsupported parquet conversion $fromType -> $toType") {
2927+
+ test(s"unsupported parquet conversion $fromType -> $toType",
2928+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
2929+
checkAllParquetReaders(values, fromType, toType, expectError = true)
2930+
}
2931+
2932+
@@ -257,7 +261,8 @@ class ParquetTypeWideningSuite
2933+
(Seq("1", "2"), LongType, DecimalType(LongDecimal.precision, 1))
2934+
)
2935+
}
2936+
- test(s"unsupported parquet conversion $fromType -> $toType") {
2937+
+ test(s"unsupported parquet conversion $fromType -> $toType",
2938+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
2939+
checkAllParquetReaders(values, fromType, toType,
2940+
expectError =
2941+
// parquet-mr allows reading decimals into a smaller precision decimal type without
2942+
@@ -271,7 +276,8 @@ class ParquetTypeWideningSuite
2943+
(Seq("2020-01-01", "2020-01-02", "1312-02-27"), TimestampNTZType, DateType))
2944+
outputTimestampType <- ParquetOutputTimestampType.values
2945+
}
2946+
- test(s"unsupported parquet timestamp conversion $fromType ($outputTimestampType) -> $toType") {
2947+
+ test(s"unsupported parquet timestamp conversion $fromType ($outputTimestampType) -> $toType",
2948+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
2949+
withSQLConf(
2950+
SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> outputTimestampType.toString,
2951+
SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key -> LegacyBehaviorPolicy.CORRECTED.toString
2952+
@@ -291,7 +297,8 @@ class ParquetTypeWideningSuite
2953+
Seq(7 -> 5, 10 -> 5, 20 -> 5, 12 -> 10, 20 -> 10, 22 -> 20)
2954+
}
2955+
test(
2956+
- s"parquet decimal precision change Decimal($fromPrecision, 2) -> Decimal($toPrecision, 2)") {
2957+
+ s"parquet decimal precision change Decimal($fromPrecision, 2) -> Decimal($toPrecision, 2)",
2958+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
2959+
checkAllParquetReaders(
2960+
values = Seq("1.23", "10.34"),
2961+
fromType = DecimalType(fromPrecision, 2),
2962+
@@ -322,7 +329,8 @@ class ParquetTypeWideningSuite
2963+
Seq((5, 2) -> (6, 4), (10, 4) -> (12, 7), (20, 5) -> (22, 8))
2964+
}
2965+
test(s"parquet decimal precision and scale change Decimal($fromPrecision, $fromScale) -> " +
2966+
- s"Decimal($toPrecision, $toScale)"
2967+
+ s"Decimal($toPrecision, $toScale)",
2968+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")
2969+
) {
2970+
checkAllParquetReaders(
2971+
values = Seq("1.23", "10.34"),
2972+
@@ -336,7 +344,8 @@ class ParquetTypeWideningSuite
2973+
)
2974+
}
2975+
2976+
- test("parquet decimal type change Decimal(5, 2) -> Decimal(3, 2) overflows with parquet-mr") {
2977+
+ test("parquet decimal type change Decimal(5, 2) -> Decimal(3, 2) overflows with parquet-mr",
2978+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
2979+
withTempDir { dir =>
2980+
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
2981+
writeParquetFiles(
29132982
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
29142983
index b8f3ea3c6f3..bbd44221288 100644
29152984
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala

‎dev/diffs/4.1.1.diff‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3036,9 +3036,18 @@ index 56076175d60..5872d9962cc 100644
30363036
val e = testSchemaMismatch(dir.getCanonicalPath, vectorizedReaderEnabled = true)
30373037
assert(e.getCause.isInstanceOf[SchemaColumnConvertNotSupportedException])
30383038
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala
3039-
index 09ed6955a51..236a4e99824 100644
3039+
index 09ed6955a51..82924c83eb5 100644
30403040
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala
30413041
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala
3042+
@@ -24,7 +24,7 @@ import org.apache.parquet.format.converter.ParquetMetadataConverter
3043+
import org.apache.parquet.hadoop.{ParquetFileReader, ParquetOutputFormat}
3044+
3045+
import org.apache.spark.SparkException
3046+
-import org.apache.spark.sql.{DataFrame, QueryTest, Row}
3047+
+import org.apache.spark.sql.{DataFrame, IgnoreCometNativeDataFusion, QueryTest, Row}
3048+
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
3049+
import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException
3050+
import org.apache.spark.sql.functions.col
30423051
@@ -65,7 +65,9 @@ class ParquetTypeWideningSuite
30433052
withClue(
30443053
s"with dictionary encoding '$dictionaryEnabled' with timestamp rebase mode " +
@@ -3069,6 +3078,66 @@ index 09ed6955a51..236a4e99824 100644
30693078
)
30703079
}
30713080
test(s"parquet widening conversion $fromType -> $toType") {
3081+
@@ -231,7 +234,8 @@ class ParquetTypeWideningSuite
3082+
(Seq("2020-01-01", "2020-01-02", "1312-02-27"), DateType, TimestampType)
3083+
)
3084+
}
3085+
- test(s"unsupported parquet conversion $fromType -> $toType") {
3086+
+ test(s"unsupported parquet conversion $fromType -> $toType",
3087+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
3088+
checkAllParquetReaders(values, fromType, toType, expectError = true)
3089+
}
3090+
3091+
@@ -257,7 +261,8 @@ class ParquetTypeWideningSuite
3092+
(Seq("1", "2"), LongType, DecimalType(LongDecimal.precision, 1))
3093+
)
3094+
}
3095+
- test(s"unsupported parquet conversion $fromType -> $toType") {
3096+
+ test(s"unsupported parquet conversion $fromType -> $toType",
3097+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
3098+
checkAllParquetReaders(values, fromType, toType,
3099+
expectError =
3100+
// parquet-mr allows reading decimals into a smaller precision decimal type without
3101+
@@ -271,7 +276,8 @@ class ParquetTypeWideningSuite
3102+
(Seq("2020-01-01", "2020-01-02", "1312-02-27"), TimestampNTZType, DateType))
3103+
outputTimestampType <- ParquetOutputTimestampType.values
3104+
}
3105+
- test(s"unsupported parquet timestamp conversion $fromType ($outputTimestampType) -> $toType") {
3106+
+ test(s"unsupported parquet timestamp conversion $fromType ($outputTimestampType) -> $toType",
3107+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
3108+
withSQLConf(
3109+
SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> outputTimestampType.toString,
3110+
SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key -> LegacyBehaviorPolicy.CORRECTED.toString
3111+
@@ -291,7 +297,8 @@ class ParquetTypeWideningSuite
3112+
Seq(7 -> 5, 10 -> 5, 20 -> 5, 12 -> 10, 20 -> 10, 22 -> 20)
3113+
}
3114+
test(
3115+
- s"parquet decimal precision change Decimal($fromPrecision, 2) -> Decimal($toPrecision, 2)") {
3116+
+ s"parquet decimal precision change Decimal($fromPrecision, 2) -> Decimal($toPrecision, 2)",
3117+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
3118+
checkAllParquetReaders(
3119+
values = Seq("1.23", "10.34"),
3120+
fromType = DecimalType(fromPrecision, 2),
3121+
@@ -322,7 +329,8 @@ class ParquetTypeWideningSuite
3122+
Seq((5, 2) -> (6, 4), (10, 4) -> (12, 7), (20, 5) -> (22, 8))
3123+
}
3124+
test(s"parquet decimal precision and scale change Decimal($fromPrecision, $fromScale) -> " +
3125+
- s"Decimal($toPrecision, $toScale)"
3126+
+ s"Decimal($toPrecision, $toScale)",
3127+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")
3128+
) {
3129+
checkAllParquetReaders(
3130+
values = Seq("1.23", "10.34"),
3131+
@@ -336,7 +344,8 @@ class ParquetTypeWideningSuite
3132+
)
3133+
}
3134+
3135+
- test("parquet decimal type change Decimal(5, 2) -> Decimal(3, 2) overflows with parquet-mr") {
3136+
+ test("parquet decimal type change Decimal(5, 2) -> Decimal(3, 2) overflows with parquet-mr",
3137+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4352")) {
3138+
withTempDir { dir =>
3139+
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
3140+
writeParquetFiles(
30723141
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala
30733142
index 1cc6d3afbee..8275727fbb4 100644
30743143
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala

0 commit comments

Comments
 (0)