Skip to content

Commit 7cdea62

Browse files
viiryaIsaac
andcommitted
Report an empty dictionary id section explicitly, and clean up tests
- An empty dictionary id section now fails with "the page has no data, not even the bit width" instead of the generic "reading past the end of the encoded values". - Run the test for runs of length 0 on multiple buffers too. - In `ParquetIOSuite`, find the RLE section with `indexOfSlice` and list causes with `ExceptionUtils.getThrowableList`. Co-authored-by: Isaac <[email protected]>
1 parent bf06bd8 commit 7cdea62

3 files changed

Lines changed: 29 additions & 15 deletions

File tree

‎sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ private enum MODE {
6262
private int bytesWidth;
6363
private BytePacker packer;
6464

65+
// True if the page has no data at all, not even the bit width of its dictionary ids.
66+
private boolean emptyPage;
67+
6568
// Current decoding mode and values
6669
private MODE mode;
6770
private int currentCount;
@@ -96,6 +99,7 @@ public VectorizedRleValuesReader(int bitWidth, boolean readLength) {
9699
@Override
97100
public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOException {
98101
this.in = in;
102+
this.emptyPage = false;
99103
if (fixedWidth) {
100104
// Initialize for repetition and definition levels
101105
if (readLength) {
@@ -121,6 +125,7 @@ public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOExce
121125
} else {
122126
// No encoded values (e.g. an all-null page). Do not treat this as a bit width of 0,
123127
// which would decode every value as 0; any read fails in `readNextGroup` instead.
128+
this.emptyPage = true;
124129
this.currentCount = 0;
125130
return;
126131
}
@@ -1000,6 +1005,10 @@ private int readIntLittleEndianPaddedOnBitWidth() throws IOException {
10001005
*/
10011006
private void readNextGroup() {
10021007
do {
1008+
if (emptyPage) {
1009+
throw new ParquetDecodingException(
1010+
"Corrupted RLE data: the page has no data, not even the bit width");
1011+
}
10031012
if (bitWidth == 0 || in.available() <= 0) {
10041013
throw new ParquetDecodingException(
10051014
"Corrupted RLE data: reading past the end of the encoded values");

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import scala.reflect.ClassTag
2828
import scala.reflect.runtime.universe.TypeTag
2929

3030
import com.google.common.primitives.UnsignedLong
31+
import org.apache.commons.lang3.exception.ExceptionUtils
3132
import org.apache.hadoop.fs.{FileSystem, Path}
3233
import org.apache.hadoop.mapreduce.{JobContext, TaskAttemptContext}
3334
import org.apache.parquet.bytes.BytesUtils
@@ -2509,11 +2510,9 @@ class ParquetIOSuite extends ParquetTest with SharedSparkSession {
25092510
val file = dir.listFiles().filter(_.getName.endsWith(".parquet")).head
25102511
val bytes = Files.readAllBytes(file.toPath)
25112512
val section = Array[Byte](3, 0, 0, 0, 0xC8.toByte, 0x01, 1)
2512-
val offsets = bytes.indices.filter { i =>
2513-
bytes.slice(i, i + section.length).sameElements(section)
2514-
}
2515-
assert(offsets.size == 1)
2516-
corrupt(bytes, offsets.head)
2513+
val offset = bytes.indexOfSlice(section)
2514+
assert(offset >= 0 && bytes.indexOfSlice(section, offset + 1) < 0)
2515+
corrupt(bytes, offset)
25172516
Files.write(file.toPath, bytes)
25182517
dir.listFiles().filter(_.getName.endsWith(".crc")).foreach(_.delete())
25192518

@@ -2530,7 +2529,7 @@ class ParquetIOSuite extends ParquetTest with SharedSparkSession {
25302529
}
25312530

25322531
private def causes(e: Throwable): Seq[Throwable] =
2533-
Iterator.iterate(e)(_.getCause).takeWhile(_ != null).toSeq
2532+
ExceptionUtils.getThrowableList(e).asScala.toSeq
25342533

25352534
/** Sets the 4-byte length at `offset` to Int.MaxValue, far more than the rest of the page. */
25362535
private def setMaxLength(bytes: Array[Byte], offset: Int): Unit =

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReaderSuite.scala‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -435,20 +435,26 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
435435
val c = new OnHeapColumnVector(5, IntegerType)
436436
reader.readIntegers(0, c, 0)
437437
reader.skipIntegers(0)
438-
interceptCorrupted(PastEnd)(reader.readIntegers(5, c, 0))
438+
interceptCorrupted("the page has no data, not even the bit width") {
439+
reader.readIntegers(5, c, 0)
440+
}
439441
}
440442

441443
test("SPARK-59832: runs of length 0 are skipped") {
442444
// [bit width 4][RLE run of 0 x 7][bit-packed run of 0 groups][RLE run of 5 x 3]
443445
val page = Array[Byte](4, 0, 7, 1, 10, 3)
444-
val r1 = new VectorizedRleValuesReader()
445-
r1.initFromPage(5, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)))
446-
assert((0 until 5).map(_ => r1.readInteger()) == Seq.fill(5)(3))
447-
val r2 = new VectorizedRleValuesReader()
448-
r2.initFromPage(5, ByteBufferInputStream.wrap(ByteBuffer.wrap(page)))
449-
val c = new OnHeapColumnVector(5, IntegerType)
450-
r2.readIntegers(5, c, 0)
451-
assert((0 until 5).map(c.getInt) == Seq.fill(5)(3))
446+
streams(page).foreach { in =>
447+
val reader = new VectorizedRleValuesReader()
448+
reader.initFromPage(5, in())
449+
assert((0 until 5).map(_ => reader.readInteger()) == Seq.fill(5)(3))
450+
}
451+
streams(page).foreach { in =>
452+
val reader = new VectorizedRleValuesReader()
453+
reader.initFromPage(5, in())
454+
val c = new OnHeapColumnVector(5, IntegerType)
455+
reader.readIntegers(5, c, 0)
456+
assert((0 until 5).map(c.getInt) == Seq.fill(5)(3))
457+
}
452458
}
453459

454460
test("SPARK-59832: reads that end exactly at the end of the encoded values succeed") {

0 commit comments

Comments
 (0)