@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.datasources.parquet
1919
2020import java .nio .ByteBuffer
2121import java .util .PrimitiveIterator
22- import java .util .concurrent .{Callable , ExecutionException , TimeUnit }
22+ import java .util .concurrent .{Callable , ExecutionException , TimeoutException , TimeUnit }
2323
2424import scala .jdk .CollectionConverters ._
2525
@@ -50,34 +50,60 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
5050
5151 import VectorizedRleValuesReaderSuite ._
5252
53- // Runs the reads that may never return before SPARK-59832. A thread per read, so that a read
54- // stuck in a loop does not block the following ones.
53+ // Runs the reads that may never return before SPARK-59832, a thread per read, so that a read
54+ // stuck in a loop does not block the following ones. The decode loops do not check for
55+ // interruption, so a stuck read keeps its thread busy until the JVM exits. After the first
56+ // timeout, later reads fail without running, so that a regression leaves one busy thread.
5557 private lazy val executor = ThreadUtils .newDaemonCachedThreadPool(" rle-reader-suite" )
58+ @ volatile private var timedOut = false
5659
5760 override def afterAll (): Unit = {
5861 try {
5962 executor.shutdownNow()
63+ executor.awaitTermination(10 , TimeUnit .SECONDS )
6064 } finally {
6165 super .afterAll()
6266 }
6367 }
6468
6569 /**
66- * Runs `f` in another thread, so that a reader that never returns fails the test, and checks
67- * that it fails with a `ParquetDecodingException` whose message contains `expected`.
70+ * Checks that `f` fails with a `ParquetDecodingException` whose message contains
71+ * "Corrupted RLE data: " followed by `expected`.
72+ */
73+ private def assertCorrupted (expected : String )(f : => Any ): Unit = {
74+ val e = intercept[ParquetDecodingException ](f)
75+ assert(e.getMessage.contains(" Corrupted RLE data: " + expected), e.getMessage)
76+ }
77+
78+ /**
79+ * Like `assertCorrupted`, but runs `f` on another thread with a timeout, so that a read that
80+ * never returns fails the test.
6881 */
6982 private def interceptCorrupted (expected : String )(f : => Any ): Unit = {
83+ assert(! timedOut, " Not run, since an earlier read did not return" )
7084 val future = executor.submit(new Callable [Any ] {
7185 override def call (): Any = f
7286 })
73- val e = intercept[ ParquetDecodingException ] {
87+ assertCorrupted(expected) {
7488 try {
75- future.get(30 , TimeUnit .SECONDS )
89+ future.get(10 , TimeUnit .SECONDS )
7690 } catch {
7791 case e : ExecutionException => throw e.getCause
92+ case e : TimeoutException =>
93+ timedOut = true
94+ throw e
7895 }
7996 }
80- assert(e.getMessage.contains(" Corrupted RLE data: " + expected), e.getMessage)
97+ }
98+
99+ /** Checks that reading `n` dictionary ids from `page` fails, on every stream of `page`. */
100+ private def assertReadIntegersFails (page : Array [Byte ], n : Int , expected : String ): Unit = {
101+ streams(page).foreach { in =>
102+ val reader = new VectorizedRleValuesReader ()
103+ reader.initFromPage(n, in())
104+ val c = new OnHeapColumnVector (n, IntegerType )
105+ interceptCorrupted(expected)(reader.readIntegers(n, c, 0 ))
106+ }
81107 }
82108
83109 test(" PACKED: alternating null/non-null (many single-element runs)" ) {
@@ -240,18 +266,14 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
240266
241267 test(" SPARK-59832: truncated dictionary ids fail instead of being partially read" ) {
242268 // A dictionary id page with 10 ids, read as if it had 20.
243- streams(dictIdPage(Array .fill(10 )(3 ), bitWidth = 4 )).foreach { in =>
244- val reader = new VectorizedRleValuesReader ()
245- reader.initFromPage(20 , in())
246- val c = new OnHeapColumnVector (20 , IntegerType )
247- interceptCorrupted(PastEnd )(reader.readIntegers(20 , c, 0 ))
248- }
249- streams(dictIdPage(Array .fill(10 )(3 ), bitWidth = 4 )).foreach { in =>
269+ val page = dictIdPage(Array .fill(10 )(3 ), bitWidth = 4 )
270+ assertReadIntegersFails(page, 20 , PastEnd )
271+ streams(page).foreach { in =>
250272 val reader = new VectorizedRleValuesReader ()
251273 reader.initFromPage(20 , in())
252274 interceptCorrupted(PastEnd )(reader.skipIntegers(20 ))
253275 }
254- streams(dictIdPage( Array .fill( 10 )( 3 ), bitWidth = 4 ) ).foreach { in =>
276+ streams(page ).foreach { in =>
255277 val reader = new VectorizedRleValuesReader ()
256278 reader.initFromPage(20 , in())
257279 (0 until 10 ).foreach(_ => assert(reader.readInteger() == 3 ))
@@ -355,23 +377,15 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
355377 Seq (- 1 , - 4 , - 5 , payload.length + 1 , Int .MaxValue ).foreach { length =>
356378 streams(BytesUtils .intToBytes(length) ++ payload).foreach { in =>
357379 val reader = new VectorizedRleValuesReader (1 )
358- val e = intercept[ParquetDecodingException ](reader.initFromPage(8 , in()))
359- assert(e.getMessage.contains(s " Corrupted RLE data: invalid length $length" ),
360- e.getMessage)
380+ assertCorrupted(s " invalid length $length" )(reader.initFromPage(8 , in()))
361381 }
362382 }
363383 }
364384
365385 test(" SPARK-59832: invalid bit-packed run is rejected before allocating its buffer" ) {
366- def check (bitWidth : Int , numGroups : Long , data : Array [Byte ])(expected : String ): Unit = {
367- val page = Array (bitWidth.toByte) ++ varint((numGroups << 1 ) | 1 ) ++ data
368- streams(page).foreach { in =>
369- val reader = new VectorizedRleValuesReader ()
370- reader.initFromPage(10 , in())
371- val c = new OnHeapColumnVector (10 , IntegerType )
372- interceptCorrupted(expected)(reader.readIntegers(10 , c, 0 ))
373- }
374- }
386+ def check (bitWidth : Int , numGroups : Long , data : Array [Byte ])(expected : String ): Unit =
387+ assertReadIntegersFails(
388+ Array (bitWidth.toByte) ++ varint((numGroups << 1 ) | 1 ) ++ data, 10 , expected)
375389 // Truncated last group. This was already rejected before (by `in.slice`), even when the
376390 // bytes cover every value read. parquet-java and Arrow accept it, for compatibility with
377391 // writers that do not pad the last group.
@@ -392,32 +406,24 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
392406 // The continuation byte of the run header is missing, and an RLE run of 10 without its
393407 // value.
394408 Seq (Array [Byte ](4 , 0x80 .toByte), Array [Byte ](4 , 0x14 )).foreach { page =>
395- streams(page).foreach { in =>
396- val reader = new VectorizedRleValuesReader ()
397- reader.initFromPage(10 , in())
398- val c = new OnHeapColumnVector (10 , IntegerType )
399- interceptCorrupted(" failed to read from input stream" )(reader.readIntegers(10 , c, 0 ))
400- }
409+ assertReadIntegersFails(page, 10 , " failed to read from input stream" )
401410 }
402411 }
403412
404413 test(" SPARK-59832: cut-off level length is rejected" ) {
405414 streams(Array [Byte ](3 , 0 )).foreach { in =>
406415 val reader = new VectorizedRleValuesReader (1 )
407- val e = intercept[ParquetDecodingException ](reader.initFromPage(8 , in()))
408- assert(e.getMessage.contains(
409- " Corrupted RLE data: the 4-byte length is cut off, only 2 bytes are left in the page" ),
410- e.getMessage)
416+ assertCorrupted(" the 4-byte length is cut off, only 2 bytes are left in the page" ) {
417+ reader.initFromPage(8 , in())
418+ }
411419 }
412420 }
413421
414422 test(" SPARK-59832: invalid dictionary id bit width is rejected" ) {
415423 Seq (33 , 255 ).foreach { bitWidth =>
416424 streams(Array [Byte ](bitWidth.toByte, 0x14 , 0 )).foreach { in =>
417425 val reader = new VectorizedRleValuesReader ()
418- val e = intercept[ParquetDecodingException ](reader.initFromPage(10 , in()))
419- assert(e.getMessage.contains(s " Corrupted RLE data: invalid bit width $bitWidth" ),
420- e.getMessage)
426+ assertCorrupted(s " invalid bit width $bitWidth" )(reader.initFromPage(10 , in()))
421427 }
422428 }
423429 }
0 commit comments