Skip to content

Commit 3398c18

Browse files
viiryaIsaac
andcommitted
Say that the dictionary id section is empty, and add the page and column once
- An empty dictionary id section now fails with "the dictionary id section is empty, not even the bit width", checked once before reading groups. - Remove the inner `IOException` wrapper in `initDataReader`, since `readPageV1` and `readPageV2` already add the page and column, so both exception types get one layer. - Check that the dictionary page error includes the original message instead of parquet-java's wording, and use a plain `val` for the test thread pool. Co-authored-by: Isaac <[email protected]>
1 parent 7cdea62 commit 3398c18

4 files changed

Lines changed: 13 additions & 16 deletions

File tree

‎sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -385,11 +385,8 @@ private void initDataReader(
385385
this.isCurrentPageDictionaryEncoded = false;
386386
}
387387

388-
try {
389-
dataColumn.initFromPage(pageValueCount, in);
390-
} catch (IOException e) {
391-
throw new IOException("could not read page in col " + descriptor, e);
392-
}
388+
// readPageV1 and readPageV2 add the page and the column to any error thrown here.
389+
dataColumn.initFromPage(pageValueCount, in);
393390
// for PARQUET-246 (See VectorizedDeltaByteArrayReader.setPreviousValues)
394391
if (CorruptDeltaByteArrays.requiresSequentialReads(writerVersion, dataEncoding) &&
395392
previousReader instanceof RequiresPreviousReader) {

‎sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,8 @@ private enum MODE {
6262
private int bytesWidth;
6363
private BytePacker packer;
6464

65-
// True if the page has no data at all, not even the bit width of its dictionary ids.
66-
private boolean emptyPage;
65+
// True if the dictionary id section of the page is empty, not even holding the bit width.
66+
private boolean emptyIdSection;
6767

6868
// Current decoding mode and values
6969
private MODE mode;
@@ -99,7 +99,7 @@ public VectorizedRleValuesReader(int bitWidth, boolean readLength) {
9999
@Override
100100
public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOException {
101101
this.in = in;
102-
this.emptyPage = false;
102+
this.emptyIdSection = false;
103103
if (fixedWidth) {
104104
// Initialize for repetition and definition levels
105105
if (readLength) {
@@ -125,7 +125,7 @@ public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOExce
125125
} else {
126126
// No encoded values (e.g. an all-null page). Do not treat this as a bit width of 0,
127127
// which would decode every value as 0; any read fails in `readNextGroup` instead.
128-
this.emptyPage = true;
128+
this.emptyIdSection = true;
129129
this.currentCount = 0;
130130
return;
131131
}
@@ -1004,11 +1004,11 @@ private int readIntLittleEndianPaddedOnBitWidth() throws IOException {
10041004
* values of the page, so reaching here also means reading past the end.
10051005
*/
10061006
private void readNextGroup() {
1007+
if (emptyIdSection) {
1008+
throw new ParquetDecodingException(
1009+
"Corrupted RLE data: the dictionary id section is empty, not even the bit width");
1010+
}
10071011
do {
1008-
if (emptyPage) {
1009-
throw new ParquetDecodingException(
1010-
"Corrupted RLE data: the page has no data, not even the bit width");
1011-
}
10121012
if (bitWidth == 0 || in.available() <= 0) {
10131013
throw new ParquetDecodingException(
10141014
"Corrupted RLE data: reading past the end of the encoded values");

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -544,7 +544,7 @@ class ParquetVectorizedSuite extends ParquetTest with SharedSparkSession {
544544
TestParquetRowGroupReader(Seq(TestPageReadStore(memPageStore, Seq(0L)))), 1)
545545
val e = intercept[ParquetDecodingException](recordReader.nextKeyValue())
546546
assert(e.getMessage.contains("could not decode the dictionary for [a]"), e.getMessage)
547-
assert(e.getMessage.contains("could not read int"), e.getMessage)
547+
assert(e.getMessage.contains(e.getCause.getMessage), e.getMessage)
548548
assert(e.getCause.isInstanceOf[ParquetDecodingException], e)
549549
} finally {
550550
recordReader.close()

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReaderSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
5454
// stuck in a loop does not block the following ones. The decode loops do not check for
5555
// interruption, so a stuck read keeps its thread busy until the JVM exits. After the first
5656
// timeout, later reads fail without running, so that a regression leaves one busy thread.
57-
private lazy val executor = ThreadUtils.newDaemonCachedThreadPool("rle-reader-suite")
57+
private val executor = ThreadUtils.newDaemonCachedThreadPool("rle-reader-suite")
5858
@volatile private var timedOut = false
5959

6060
override def afterAll(): Unit = {
@@ -435,7 +435,7 @@ class VectorizedRleValuesReaderSuite extends SparkFunSuite {
435435
val c = new OnHeapColumnVector(5, IntegerType)
436436
reader.readIntegers(0, c, 0)
437437
reader.skipIntegers(0)
438-
interceptCorrupted("the page has no data, not even the bit width") {
438+
interceptCorrupted("the dictionary id section is empty, not even the bit width") {
439439
reader.readIntegers(5, c, 0)
440440
}
441441
}

0 commit comments

Comments
 (0)