Skip to content

Commit bf06bd8

Browse files
viiryaIsaac
andcommitted
Keep the exception type when adding the page and column
- Wrap a `ParquetDecodingException` thrown while initializing a page or decoding the dictionary page in a `ParquetDecodingException` that adds the page or dictionary, the column and the original message, instead of an `IOException`. This keeps the type that callers and the SPARK-59831 test expect, and drops the extra `RuntimeException` layer from `readPage()`. - Test a corrupted dictionary page in `ParquetVectorizedSuite`. - In `VectorizedRleValuesReaderSuite`, add `assertCorrupted` and `assertReadIntegersFails`, stop running reads after the first timeout, and wait for the thread pool to terminate. - In `ParquetIOSuite`, use `failAfter` with a `ThreadSignaler` instead of `Await.result`, tolerate causes without a message, and use `BytesUtils.intToBytes`. Co-authored-by: Isaac <[email protected]>
1 parent 517133c commit bf06bd8

4 files changed

Lines changed: 96 additions & 55 deletions

File tree

‎sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -152,8 +152,11 @@ public VectorizedColumnReader(
152152
try {
153153
this.dictionary = dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage);
154154
this.isCurrentPageDictionaryEncoded = true;
155-
} catch (IOException | ParquetDecodingException e) {
155+
} catch (IOException e) {
156156
throw new IOException("could not decode the dictionary for " + descriptor, e);
157+
} catch (ParquetDecodingException e) {
158+
throw new ParquetDecodingException(
159+
"could not decode the dictionary for " + descriptor + ": " + e.getMessage(), e);
157160
}
158161
} else {
159162
this.dictionary = null;
@@ -454,8 +457,11 @@ private int readPageV1(DataPageV1 page) throws IOException {
454457
defColumn.initFromPage(pageValueCount, in);
455458
initDataReader(pageValueCount, page.getValueEncoding(), in);
456459
return pageValueCount;
457-
} catch (IOException | ParquetDecodingException e) {
460+
} catch (IOException e) {
458461
throw new IOException("could not read page " + page + " in col " + descriptor, e);
462+
} catch (ParquetDecodingException e) {
463+
throw new ParquetDecodingException(
464+
"could not read page " + page + " in col " + descriptor + ": " + e.getMessage(), e);
459465
}
460466
}
461467

@@ -474,8 +480,11 @@ private int readPageV2(DataPageV2 page) throws IOException {
474480
try {
475481
initDataReader(pageValueCount, page.getDataEncoding(), page.getData().toInputStream());
476482
return pageValueCount;
477-
} catch (IOException | ParquetDecodingException e) {
483+
} catch (IOException e) {
478484
throw new IOException("could not read page " + page + " in col " + descriptor, e);
485+
} catch (ParquetDecodingException e) {
486+
throw new ParquetDecodingException(
487+
"could not read page " + page + " in col " + descriptor + ": " + e.getMessage(), e);
479488
}
480489
}
481490
}

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,14 @@ import java.time.{LocalDateTime, LocalTime}
2323
import java.util.Locale
2424

2525
import scala.collection.mutable
26-
import scala.concurrent.{Await, ExecutionContext, Future}
27-
import scala.concurrent.duration._
2826
import scala.jdk.CollectionConverters._
2927
import scala.reflect.ClassTag
3028
import scala.reflect.runtime.universe.TypeTag
3129

3230
import com.google.common.primitives.UnsignedLong
3331
import org.apache.hadoop.fs.{FileSystem, Path}
3432
import org.apache.hadoop.mapreduce.{JobContext, TaskAttemptContext}
33+
import org.apache.parquet.bytes.BytesUtils
3534
import org.apache.parquet.column.{Encoding, ParquetProperties}
3635
import org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0
3736
import org.apache.parquet.example.data.Group
@@ -41,6 +40,8 @@ import org.apache.parquet.hadoop.example.ExampleParquetWriter
4140
import org.apache.parquet.io.ParquetDecodingException
4241
import org.apache.parquet.io.api.Binary
4342
import org.apache.parquet.schema.{MessageType, MessageTypeParser}
43+
import org.scalatest.concurrent.{Signaler, ThreadSignaler}
44+
import org.scalatest.time.{Seconds, Span}
4445

4546
import org.apache.spark.{SPARK_VERSION_SHORT, SparkException, TestUtils}
4647
import org.apache.spark.rdd.RDD
@@ -2517,13 +2518,12 @@ class ParquetIOSuite extends ParquetTest with SharedSparkSession {
25172518
dir.listFiles().filter(_.getName.endsWith(".crc")).foreach(_.delete())
25182519

25192520
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true") {
2520-
// Before SPARK-59832 a truncated read never returned, so run it with a timeout.
2521-
// scalastyle:off awaitresult
2521+
// Before SPARK-59832 a truncated read never returned, so run it with a timeout. The test
2522+
// thread waits interruptibly for the job, so it can be signaled directly.
2523+
implicit val signaler: Signaler = ThreadSignaler
25222524
error = intercept[SparkException] {
2523-
Await.result(
2524-
Future(spark.read.parquet(path).collect())(ExecutionContext.global), 1.minute)
2525+
failAfter(Span(60, Seconds))(spark.read.parquet(path).collect())
25252526
}
2526-
// scalastyle:on awaitresult
25272527
}
25282528
}
25292529
error
@@ -2534,16 +2534,15 @@ class ParquetIOSuite extends ParquetTest with SharedSparkSession {
25342534

25352535
/** Sets the 4-byte length at `offset` to Int.MaxValue, far more than the rest of the page. */
25362536
private def setMaxLength(bytes: Array[Byte], offset: Int): Unit =
2537-
Array(0xFF, 0xFF, 0xFF, 0x7F).zipWithIndex.foreach { case (b, i) =>
2538-
bytes(offset + i) = b.toByte
2539-
}
2537+
BytesUtils.intToBytes(Int.MaxValue).copyToArray(bytes, offset)
25402538

25412539
/**
25422540
* Checks that the page error is reported once with the column, directly on top of the
25432541
* `ParquetDecodingException` about the invalid length.
25442542
*/
25452543
private def assertInvalidLengthReportsColumn(e: Throwable, column: String): Unit = {
2546-
val pageErrors = causes(e).filter(_.getMessage.startsWith("could not read page"))
2544+
val pageErrors =
2545+
causes(e).filter(c => Option(c.getMessage).exists(_.startsWith("could not read page")))
25472546
assert(pageErrors.size == 1, e)
25482547
assert(pageErrors.head.getMessage.contains(s"in col [$column]"), e)
25492548
assert(pageErrors.head.getCause.isInstanceOf[ParquetDecodingException], e)

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -524,6 +524,33 @@ class ParquetVectorizedSuite extends ParquetTest with SharedSparkSession {
524524
}
525525
}
526526

527+
test("SPARK-59832: a corrupted dictionary page reports the column") {
528+
// A PLAIN INT32 dictionary page that declares 3 entries but holds only 2.
529+
val parquetSchema = MessageTypeParser.parseMessageType("message root { optional int32 a; }")
530+
val ty = parquetSchema.asGroupType().getType("a").asPrimitiveType()
531+
val cd = new ColumnDescriptor(Array("a"), ty, 0, 1)
532+
val memPageStore = new MemPageStore(1)
533+
val pageWriter = memPageStore.getPageWriter(cd)
534+
pageWriter.writeDictionaryPage(new DictionaryPage(
535+
BytesInput.from(Array[Byte](1, 0, 0, 0, 2, 0, 0, 0)), 3, Encoding.PLAIN_DICTIONARY))
536+
// One non-null value: the definition levels (length 2, an RLE run of 1 x 1), then the
537+
// dictionary ids (bit width 2, an RLE run of 1 x 0).
538+
pageWriter.writePage(BytesInput.from(Array[Byte](2, 0, 0, 0, 2, 1, 2, 2, 0)), 1, 1,
539+
Statistics.createStats(ty), Encoding.RLE, Encoding.RLE, Encoding.RLE_DICTIONARY)
540+
val recordReader = new VectorizedParquetRecordReader(
541+
DateTimeUtils.getZoneId("EST"), "CORRECTED", "UTC", "CORRECTED", "UTC", true, 1)
542+
try {
543+
recordReader.initialize(parquetSchema, parquetSchema,
544+
TestParquetRowGroupReader(Seq(TestPageReadStore(memPageStore, Seq(0L)))), 1)
545+
val e = intercept[ParquetDecodingException](recordReader.nextKeyValue())
546+
assert(e.getMessage.contains("could not decode the dictionary for [a]"), e.getMessage)
547+
assert(e.getMessage.contains("could not read int"), e.getMessage)
548+
assert(e.getCause.isInstanceOf[ParquetDecodingException], e)
549+
} finally {
550+
recordReader.close()
551+
}
552+
}
553+
527554
truncateTypeTest("primitive type", IntegerType, LongType, IntegerType)
528555

529556
truncateTypeTest("basic struct",

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReaderSuite.scala‎

Lines changed: 47 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.datasources.parquet
1919

2020
import java.nio.ByteBuffer
2121
import java.util.PrimitiveIterator
22-
import java.util.concurrent.{Callable, ExecutionException, TimeUnit}
22+
import java.util.concurrent.{Callable, ExecutionException, TimeoutException, TimeUnit}
2323

2424
import scala.jdk.CollectionConverters._
2525

@@ -50,34 +50,60 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
5050

5151
import VectorizedRleValuesReaderSuite._
5252

53-
// Runs the reads that may never return before SPARK-59832. A thread per read, so that a read
54-
// stuck in a loop does not block the following ones.
53+
// Runs the reads that may never return before SPARK-59832, a thread per read, so that a read
54+
// stuck in a loop does not block the following ones. The decode loops do not check for
55+
// interruption, so a stuck read keeps its thread busy until the JVM exits. After the first
56+
// timeout, later reads fail without running, so that a regression leaves one busy thread.
5557
private lazy val executor = ThreadUtils.newDaemonCachedThreadPool("rle-reader-suite")
58+
@volatile private var timedOut = false
5659

5760
override def afterAll(): Unit = {
5861
try {
5962
executor.shutdownNow()
63+
executor.awaitTermination(10, TimeUnit.SECONDS)
6064
} finally {
6165
super.afterAll()
6266
}
6367
}
6468

6569
/**
66-
* Runs `f` in another thread, so that a reader that never returns fails the test, and checks
67-
* that it fails with a `ParquetDecodingException` whose message contains `expected`.
70+
* Checks that `f` fails with a `ParquetDecodingException` whose message contains
71+
* "Corrupted RLE data: " followed by `expected`.
72+
*/
73+
private def assertCorrupted(expected: String)(f: => Any): Unit = {
74+
val e = intercept[ParquetDecodingException](f)
75+
assert(e.getMessage.contains("Corrupted RLE data: " + expected), e.getMessage)
76+
}
77+
78+
/**
79+
* Like `assertCorrupted`, but runs `f` on another thread with a timeout, so that a read that
80+
* never returns fails the test.
6881
*/
6982
private def interceptCorrupted(expected: String)(f: => Any): Unit = {
83+
assert(!timedOut, "Not run, since an earlier read did not return")
7084
val future = executor.submit(new Callable[Any] {
7185
override def call(): Any = f
7286
})
73-
val e = intercept[ParquetDecodingException] {
87+
assertCorrupted(expected) {
7488
try {
75-
future.get(30, TimeUnit.SECONDS)
89+
future.get(10, TimeUnit.SECONDS)
7690
} catch {
7791
case e: ExecutionException => throw e.getCause
92+
case e: TimeoutException =>
93+
timedOut = true
94+
throw e
7895
}
7996
}
80-
assert(e.getMessage.contains("Corrupted RLE data: " + expected), e.getMessage)
97+
}
98+
99+
/** Checks that reading `n` dictionary ids from `page` fails, on every stream of `page`. */
100+
private def assertReadIntegersFails(page: Array[Byte], n: Int, expected: String): Unit = {
101+
streams(page).foreach { in =>
102+
val reader = new VectorizedRleValuesReader()
103+
reader.initFromPage(n, in())
104+
val c = new OnHeapColumnVector(n, IntegerType)
105+
interceptCorrupted(expected)(reader.readIntegers(n, c, 0))
106+
}
81107
}
82108

83109
test("PACKED: alternating null/non-null (many single-element runs)") {
@@ -240,18 +266,14 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
240266

241267
test("SPARK-59832: truncated dictionary ids fail instead of being partially read") {
242268
// A dictionary id page with 10 ids, read as if it had 20.
243-
streams(dictIdPage(Array.fill(10)(3), bitWidth = 4)).foreach { in =>
244-
val reader = new VectorizedRleValuesReader()
245-
reader.initFromPage(20, in())
246-
val c = new OnHeapColumnVector(20, IntegerType)
247-
interceptCorrupted(PastEnd)(reader.readIntegers(20, c, 0))
248-
}
249-
streams(dictIdPage(Array.fill(10)(3), bitWidth = 4)).foreach { in =>
269+
val page = dictIdPage(Array.fill(10)(3), bitWidth = 4)
270+
assertReadIntegersFails(page, 20, PastEnd)
271+
streams(page).foreach { in =>
250272
val reader = new VectorizedRleValuesReader()
251273
reader.initFromPage(20, in())
252274
interceptCorrupted(PastEnd)(reader.skipIntegers(20))
253275
}
254-
streams(dictIdPage(Array.fill(10)(3), bitWidth = 4)).foreach { in =>
276+
streams(page).foreach { in =>
255277
val reader = new VectorizedRleValuesReader()
256278
reader.initFromPage(20, in())
257279
(0 until 10).foreach(_ => assert(reader.readInteger() == 3))
@@ -355,23 +377,15 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
355377
Seq(-1, -4, -5, payload.length + 1, Int.MaxValue).foreach { length =>
356378
streams(BytesUtils.intToBytes(length) ++ payload).foreach { in =>
357379
val reader = new VectorizedRleValuesReader(1)
358-
val e = intercept[ParquetDecodingException](reader.initFromPage(8, in()))
359-
assert(e.getMessage.contains(s"Corrupted RLE data: invalid length $length"),
360-
e.getMessage)
380+
assertCorrupted(s"invalid length $length")(reader.initFromPage(8, in()))
361381
}
362382
}
363383
}
364384

365385
test("SPARK-59832: invalid bit-packed run is rejected before allocating its buffer") {
366-
def check(bitWidth: Int, numGroups: Long, data: Array[Byte])(expected: String): Unit = {
367-
val page = Array(bitWidth.toByte) ++ varint((numGroups << 1) | 1) ++ data
368-
streams(page).foreach { in =>
369-
val reader = new VectorizedRleValuesReader()
370-
reader.initFromPage(10, in())
371-
val c = new OnHeapColumnVector(10, IntegerType)
372-
interceptCorrupted(expected)(reader.readIntegers(10, c, 0))
373-
}
374-
}
386+
def check(bitWidth: Int, numGroups: Long, data: Array[Byte])(expected: String): Unit =
387+
assertReadIntegersFails(
388+
Array(bitWidth.toByte) ++ varint((numGroups << 1) | 1) ++ data, 10, expected)
375389
// Truncated last group. This was already rejected before (by `in.slice`), even when the
376390
// bytes cover every value read. parquet-java and Arrow accept it, for compatibility with
377391
// writers that do not pad the last group.
@@ -392,32 +406,24 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
392406
// The continuation byte of the run header is missing, and an RLE run of 10 without its
393407
// value.
394408
Seq(Array[Byte](4, 0x80.toByte), Array[Byte](4, 0x14)).foreach { page =>
395-
streams(page).foreach { in =>
396-
val reader = new VectorizedRleValuesReader()
397-
reader.initFromPage(10, in())
398-
val c = new OnHeapColumnVector(10, IntegerType)
399-
interceptCorrupted("failed to read from input stream")(reader.readIntegers(10, c, 0))
400-
}
409+
assertReadIntegersFails(page, 10, "failed to read from input stream")
401410
}
402411
}
403412

404413
test("SPARK-59832: cut-off level length is rejected") {
405414
streams(Array[Byte](3, 0)).foreach { in =>
406415
val reader = new VectorizedRleValuesReader(1)
407-
val e = intercept[ParquetDecodingException](reader.initFromPage(8, in()))
408-
assert(e.getMessage.contains(
409-
"Corrupted RLE data: the 4-byte length is cut off, only 2 bytes are left in the page"),
410-
e.getMessage)
416+
assertCorrupted("the 4-byte length is cut off, only 2 bytes are left in the page") {
417+
reader.initFromPage(8, in())
418+
}
411419
}
412420
}
413421

414422
test("SPARK-59832: invalid dictionary id bit width is rejected") {
415423
Seq(33, 255).foreach { bitWidth =>
416424
streams(Array[Byte](bitWidth.toByte, 0x14, 0)).foreach { in =>
417425
val reader = new VectorizedRleValuesReader()
418-
val e = intercept[ParquetDecodingException](reader.initFromPage(10, in()))
419-
assert(e.getMessage.contains(s"Corrupted RLE data: invalid bit width $bitWidth"),
420-
e.getMessage)
426+
assertCorrupted(s"invalid bit width $bitWidth")(reader.initFromPage(10, in()))
421427
}
422428
}
423429
}

0 commit comments

Comments
 (0)