@@ -2486,26 +2486,30 @@ class ParquetIOSuite extends ParquetTest with SharedSparkSession {
24862486 }
24872487
24882488 /**
2489- * Writes 100 non-null values of a nullable INT32 column to a single uncompressed PLAIN page,
2490- * lets `corrupt` modify the bytes of its definition levels, and returns the error of reading
2491- * the file with the vectorized reader.
2489+ * Writes 100 rows of `column` to a single uncompressed data page, lets `corrupt` modify the
2490+ * RLE section that starts with `03 00 00 00 C8 01 01` (the 4-byte length 3, then one RLE run
2491+ * with header 100 << 1 and value 1), and returns the error of reading the file with the
2492+ * vectorized reader. For a nullable INT32 column that is the definition levels; for a
2493+ * BOOLEAN column written as a V2 page it is the RLE-encoded values.
24922494 */
2493- private def readCorruptedDefinitionLevels (corrupt : (Array [Byte ], Int ) => Unit ): Throwable = {
2495+ private def readCorruptedRleSection (
2496+ column : String ,
2497+ writerVersion : String = " PARQUET_1_0" )(
2498+ corrupt : (Array [Byte ], Int ) => Unit ): Throwable = {
24942499 var error : Throwable = null
24952500 withTempPath { dir =>
24962501 val path = dir.getCanonicalPath
2497- spark.range(100 ).selectExpr(" if(id < 0, null, cast(id as int)) as a " ).coalesce(1 )
2502+ spark.range(100 ).selectExpr(column ).coalesce(1 )
24982503 .write
24992504 .option(ParquetOutputFormat .ENABLE_DICTIONARY , " false" )
2505+ .option(ParquetOutputFormat .WRITER_VERSION , writerVersion)
25002506 .option(" compression" , " uncompressed" )
25012507 .parquet(path)
25022508 val file = dir.listFiles().filter(_.getName.endsWith(" .parquet" )).head
25032509 val bytes = Files .readAllBytes(file.toPath)
2504- // The definition levels of the only data page: the 4-byte length 3, then one RLE run
2505- // with header 100 << 1 (varint 0xC8 0x01) and value 1.
2506- val levels = Array [Byte ](3 , 0 , 0 , 0 , 0xC8 .toByte, 0x01 , 1 )
2510+ val section = Array [Byte ](3 , 0 , 0 , 0 , 0xC8 .toByte, 0x01 , 1 )
25072511 val offsets = bytes.indices.filter { i =>
2508- bytes.slice(i, i + levels .length).sameElements(levels )
2512+ bytes.slice(i, i + section .length).sameElements(section )
25092513 }
25102514 assert(offsets.size == 1 )
25112515 corrupt(bytes, offsets.head)
@@ -2528,28 +2532,44 @@ class ParquetIOSuite extends ParquetTest with SharedSparkSession {
25282532 private def causes (e : Throwable ): Seq [Throwable ] =
25292533 Iterator .iterate(e)(_.getCause).takeWhile(_ != null ).toSeq
25302534
2535+ /** Sets the 4-byte length at `offset` to Int.MaxValue, far more than the rest of the page. */
2536+ private def setMaxLength (bytes : Array [Byte ], offset : Int ): Unit =
2537+ Array (0xFF , 0xFF , 0xFF , 0x7F ).zipWithIndex.foreach { case (b, i) =>
2538+ bytes(offset + i) = b.toByte
2539+ }
2540+
2541+ /**
2542+ * Checks that the page error is reported once with the column, directly on top of the
2543+ * `ParquetDecodingException` about the invalid length.
2544+ */
2545+ private def assertInvalidLengthReportsColumn (e : Throwable , column : String ): Unit = {
2546+ val pageErrors = causes(e).filter(_.getMessage.startsWith(" could not read page" ))
2547+ assert(pageErrors.size == 1 , e)
2548+ assert(pageErrors.head.getMessage.contains(s " in col [ $column] " ), e)
2549+ assert(pageErrors.head.getCause.isInstanceOf [ParquetDecodingException ], e)
2550+ assert(pageErrors.head.getCause.getMessage.contains(
2551+ s " Corrupted RLE data: invalid length ${Int .MaxValue }" ), e)
2552+ }
2553+
25312554 test(" SPARK-59832: vectorized reader fails on truncated definition levels" ) {
25322555 // Shorten the run to 64 values (header 64 << 1 = varint 0x80 0x01).
2533- val e = readCorruptedDefinitionLevels((bytes, offset) => bytes(offset + 4 ) = 0x80 .toByte)
2556+ val e = readCorruptedRleSection(" if(id < 0, null, cast(id as int)) as a" ) {
2557+ (bytes, offset) => bytes(offset + 4 ) = 0x80 .toByte
2558+ }
25342559 assert(causes(e).exists {
25352560 case c : ParquetDecodingException => c.getMessage.contains(" Corrupted RLE data" )
25362561 case _ => false
25372562 }, e)
25382563 }
25392564
25402565 test(" SPARK-59832: invalid definition level length reports the column" ) {
2541- val e = readCorruptedDefinitionLevels { (bytes, offset) =>
2542- // Length Int.MaxValue, far more than the rest of the page.
2543- Array (0xFF , 0xFF , 0xFF , 0x7F ).zipWithIndex.foreach { case (b, i) =>
2544- bytes(offset + i) = b.toByte
2545- }
2546- }
2547- val pageError = causes(e).collectFirst {
2548- case c : java.io.IOException if c.getMessage.startsWith(" could not read page" ) => c
2549- }
2550- assert(pageError.exists(_.getMessage.contains(" [a]" )), e)
2551- assert(pageError.exists(_.getCause.getMessage.contains(
2552- s " Corrupted RLE data: invalid length ${Int .MaxValue }" )), e)
2566+ val e = readCorruptedRleSection(" if(id < 0, null, cast(id as int)) as a" )(setMaxLength)
2567+ assertInvalidLengthReportsColumn(e, " a" )
2568+ }
2569+
2570+ test(" SPARK-59832: invalid length of RLE booleans in a V2 page reports the column" ) {
2571+ val e = readCorruptedRleSection(" id >= 0 as b" , writerVersion = " PARQUET_2_0" )(setMaxLength)
2572+ assertInvalidLengthReportsColumn(e, " b" )
25532573 }
25542574}
25552575
0 commit comments