Skip to content

Commit 1a2868d

Browse files
committed
Address the second round of review comments
- Reject a 4-byte level length that is cut off, and a dictionary id bit width above 32, with "Corrupted RLE data". - Wrap a `ParquetDecodingException` thrown while initializing a V2 page in `readPageV2` instead of `initDataReader`, so the page and column are added once, and also wrap it when decoding the dictionary page. - Check the expected message in `interceptCorrupted`, run it on a dedicated thread pool, and test data ending inside a run header or an RLE value, a cut-off length, an invalid bit width and an invalid RLE boolean length in a V2 page. - Use `BytesUtils.intToBytes` and `BytesInput.fromUnsignedVarLong`, and update stale comments.
1 parent 4121cb9 commit 1a2868d

4 files changed

Lines changed: 150 additions & 70 deletions

File tree

‎sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ public VectorizedColumnReader(
131131
try {
132132
this.dictionary = dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage);
133133
this.isCurrentPageDictionaryEncoded = true;
134-
} catch (IOException e) {
134+
} catch (IOException | ParquetDecodingException e) {
135135
throw new IOException("could not decode the dictionary for " + descriptor, e);
136136
}
137137
} else {
@@ -363,7 +363,7 @@ private void initDataReader(
363363

364364
try {
365365
dataColumn.initFromPage(pageValueCount, in);
366-
} catch (IOException | ParquetDecodingException e) {
366+
} catch (IOException e) {
367367
throw new IOException("could not read page in col " + descriptor, e);
368368
}
369369
// for PARQUET-246 (See VectorizedDeltaByteArrayReader.setPreviousValues)
@@ -452,7 +452,7 @@ private int readPageV2(DataPageV2 page) throws IOException {
452452
try {
453453
initDataReader(pageValueCount, page.getDataEncoding(), page.getData().toInputStream());
454454
return pageValueCount;
455-
} catch (IOException e) {
455+
} catch (IOException | ParquetDecodingException e) {
456456
throw new IOException("could not read page " + page + " in col " + descriptor, e);
457457
}
458458
}

‎sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,10 @@ public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOExce
9999
if (fixedWidth) {
100100
// Initialize for repetition and definition levels
101101
if (readLength) {
102+
if (in.available() < 4) {
103+
throw new ParquetDecodingException("Corrupted RLE data: the 4-byte length is cut off, " +
104+
"only " + in.available() + " bytes are left in the page");
105+
}
102106
int length = readIntLittleEndian();
103107
if (length < 0 || length > in.available()) {
104108
throw new ParquetDecodingException("Corrupted RLE data: invalid length " + length +
@@ -109,7 +113,11 @@ public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOExce
109113
} else {
110114
// Initialize for values
111115
if (in.available() > 0) {
112-
init(in.read());
116+
int width = in.read();
117+
if (width > 32) {
118+
throw new ParquetDecodingException("Corrupted RLE data: invalid bit width " + width);
119+
}
120+
init(width);
113121
} else {
114122
// No encoded values (e.g. an all-null page). Do not treat this as a bit width of 0,
115123
// which would decode every value as 0; any read fails in `readNextGroup` instead.

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala‎

Lines changed: 42 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2486,26 +2486,30 @@ class ParquetIOSuite extends ParquetTest with SharedSparkSession {
24862486
}
24872487

24882488
/**
2489-
* Writes 100 non-null values of a nullable INT32 column to a single uncompressed PLAIN page,
2490-
* lets `corrupt` modify the bytes of its definition levels, and returns the error of reading
2491-
* the file with the vectorized reader.
2489+
* Writes 100 rows of `column` to a single uncompressed data page, lets `corrupt` modify the
2490+
* RLE section that starts with `03 00 00 00 C8 01 01` (the 4-byte length 3, then one RLE run
2491+
* with header 100 << 1 and value 1), and returns the error of reading the file with the
2492+
* vectorized reader. For a nullable INT32 column that is the definition levels; for a
2493+
* BOOLEAN column written as a V2 page it is the RLE-encoded values.
24922494
*/
2493-
private def readCorruptedDefinitionLevels(corrupt: (Array[Byte], Int) => Unit): Throwable = {
2495+
private def readCorruptedRleSection(
2496+
column: String,
2497+
writerVersion: String = "PARQUET_1_0")(
2498+
corrupt: (Array[Byte], Int) => Unit): Throwable = {
24942499
var error: Throwable = null
24952500
withTempPath { dir =>
24962501
val path = dir.getCanonicalPath
2497-
spark.range(100).selectExpr("if(id < 0, null, cast(id as int)) as a").coalesce(1)
2502+
spark.range(100).selectExpr(column).coalesce(1)
24982503
.write
24992504
.option(ParquetOutputFormat.ENABLE_DICTIONARY, "false")
2505+
.option(ParquetOutputFormat.WRITER_VERSION, writerVersion)
25002506
.option("compression", "uncompressed")
25012507
.parquet(path)
25022508
val file = dir.listFiles().filter(_.getName.endsWith(".parquet")).head
25032509
val bytes = Files.readAllBytes(file.toPath)
2504-
// The definition levels of the only data page: the 4-byte length 3, then one RLE run
2505-
// with header 100 << 1 (varint 0xC8 0x01) and value 1.
2506-
val levels = Array[Byte](3, 0, 0, 0, 0xC8.toByte, 0x01, 1)
2510+
val section = Array[Byte](3, 0, 0, 0, 0xC8.toByte, 0x01, 1)
25072511
val offsets = bytes.indices.filter { i =>
2508-
bytes.slice(i, i + levels.length).sameElements(levels)
2512+
bytes.slice(i, i + section.length).sameElements(section)
25092513
}
25102514
assert(offsets.size == 1)
25112515
corrupt(bytes, offsets.head)
@@ -2528,28 +2532,44 @@ class ParquetIOSuite extends ParquetTest with SharedSparkSession {
25282532
private def causes(e: Throwable): Seq[Throwable] =
25292533
Iterator.iterate(e)(_.getCause).takeWhile(_ != null).toSeq
25302534

2535+
/** Sets the 4-byte length at `offset` to Int.MaxValue, far more than the rest of the page. */
2536+
private def setMaxLength(bytes: Array[Byte], offset: Int): Unit =
2537+
Array(0xFF, 0xFF, 0xFF, 0x7F).zipWithIndex.foreach { case (b, i) =>
2538+
bytes(offset + i) = b.toByte
2539+
}
2540+
2541+
/**
2542+
* Checks that the page error is reported once with the column, directly on top of the
2543+
* `ParquetDecodingException` about the invalid length.
2544+
*/
2545+
private def assertInvalidLengthReportsColumn(e: Throwable, column: String): Unit = {
2546+
val pageErrors = causes(e).filter(_.getMessage.startsWith("could not read page"))
2547+
assert(pageErrors.size == 1, e)
2548+
assert(pageErrors.head.getMessage.contains(s"in col [$column]"), e)
2549+
assert(pageErrors.head.getCause.isInstanceOf[ParquetDecodingException], e)
2550+
assert(pageErrors.head.getCause.getMessage.contains(
2551+
s"Corrupted RLE data: invalid length ${Int.MaxValue}"), e)
2552+
}
2553+
25312554
test("SPARK-59832: vectorized reader fails on truncated definition levels") {
25322555
// Shorten the run to 64 values (header 64 << 1 = varint 0x80 0x01).
2533-
val e = readCorruptedDefinitionLevels((bytes, offset) => bytes(offset + 4) = 0x80.toByte)
2556+
val e = readCorruptedRleSection("if(id < 0, null, cast(id as int)) as a") {
2557+
(bytes, offset) => bytes(offset + 4) = 0x80.toByte
2558+
}
25342559
assert(causes(e).exists {
25352560
case c: ParquetDecodingException => c.getMessage.contains("Corrupted RLE data")
25362561
case _ => false
25372562
}, e)
25382563
}
25392564

25402565
test("SPARK-59832: invalid definition level length reports the column") {
2541-
val e = readCorruptedDefinitionLevels { (bytes, offset) =>
2542-
// Length Int.MaxValue, far more than the rest of the page.
2543-
Array(0xFF, 0xFF, 0xFF, 0x7F).zipWithIndex.foreach { case (b, i) =>
2544-
bytes(offset + i) = b.toByte
2545-
}
2546-
}
2547-
val pageError = causes(e).collectFirst {
2548-
case c: java.io.IOException if c.getMessage.startsWith("could not read page") => c
2549-
}
2550-
assert(pageError.exists(_.getMessage.contains("[a]")), e)
2551-
assert(pageError.exists(_.getCause.getMessage.contains(
2552-
s"Corrupted RLE data: invalid length ${Int.MaxValue}")), e)
2566+
val e = readCorruptedRleSection("if(id < 0, null, cast(id as int)) as a")(setMaxLength)
2567+
assertInvalidLengthReportsColumn(e, "a")
2568+
}
2569+
2570+
test("SPARK-59832: invalid length of RLE booleans in a V2 page reports the column") {
2571+
val e = readCorruptedRleSection("id >= 0 as b", writerVersion = "PARQUET_2_0")(setMaxLength)
2572+
assertInvalidLengthReportsColumn(e, "b")
25532573
}
25542574
}
25552575

0 commit comments

Comments
 (0)