Skip to content

Commit 3568ed4

Browse files
viiryaIsaac
andcommitted
Address the second round of review comments
- Keep only the description of the current page, so that the page and its decompressor stream are not kept alive. - Read the dictionary page in the same try as initDictionary, and narrow its comment. - Reject a DELTA_BINARY_PACKED mini block bit width above 64, and drop a redundant check. - Test the page in the batch error message, the mini block size checks and the bit width check; share the test helpers and simplify the ZSTD test. Co-authored-by: Isaac <[email protected]>
1 parent 862d115 commit 3568ed4

5 files changed

Lines changed: 77 additions & 59 deletions

File tree

‎sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java‎

Lines changed: 24 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,10 @@ public class VectorizedColumnReader {
9999
private long pageFirstRowIndex;
100100

101101
/**
102-
* The data page the values are being decoded from, for error messages.
102+
* The data page the values are being decoded from, for error messages. Only its description is
103+
* kept: for codecs that decompress lazily, the page holds the decompressor stream.
103104
*/
104-
private DataPage currentPage;
105+
private String currentPage;
105106

106107
private final PageReader pageReader;
107108
private final ColumnDescriptor descriptor;
@@ -151,38 +152,31 @@ public VectorizedColumnReader(
151152
int96RebaseMode,
152153
int96RebaseTz);
153154

154-
DictionaryPage dictionaryPage;
155+
Dictionary dict = null;
155156
try {
156157
// This decompresses the page eagerly for some codecs, e.g. ZSTD.
157-
dictionaryPage = pageReader.readDictionaryPage();
158+
DictionaryPage dictionaryPage = pageReader.readDictionaryPage();
159+
if (dictionaryPage != null) {
160+
dict = dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage);
161+
}
162+
} catch (IOException e) {
163+
throw new IOException("could not decode the dictionary for " + descriptor, e);
158164
} catch (ParquetDecodingException e) {
159165
throw new ParquetDecodingException(
160-
"could not read the dictionary page for " + descriptor + ": " + e.getMessage(), e);
161-
}
162-
if (dictionaryPage != null) {
163-
try {
164-
this.dictionary = dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage);
165-
this.isCurrentPageDictionaryEncoded = true;
166-
} catch (IOException e) {
167-
throw new IOException("could not decode the dictionary for " + descriptor, e);
168-
} catch (ParquetDecodingException e) {
169-
throw new ParquetDecodingException(
170-
"could not decode the dictionary for " + descriptor + ": " + e.getMessage(), e);
171-
} catch (IndexOutOfBoundsException | NegativeArraySizeException |
172-
UnsupportedOperationException e) {
173-
// initDictionary reads the bytes of the page, which also decompresses them for codecs
174-
// that decompress lazily, e.g. SNAPPY, and parses them. A corrupted page or page header
175-
// makes it throw these:
176-
// an IndexOutOfBoundsException for a truncated BINARY dictionary, a
177-
// NegativeArraySizeException for a negative dictionary size, and an
178-
// UnsupportedOperationException for an encoding that has no dictionary.
179-
throw new ParquetDecodingException(
180-
"could not decode the dictionary for " + descriptor + ": " + e, e);
181-
}
182-
} else {
183-
this.dictionary = null;
184-
this.isCurrentPageDictionaryEncoded = false;
166+
"could not decode the dictionary for " + descriptor + ": " + e.getMessage(), e);
167+
} catch (IndexOutOfBoundsException | NegativeArraySizeException |
168+
UnsupportedOperationException e) {
169+
// Reading the dictionary page decompresses it, for some codecs only when initDictionary
170+
// reads its bytes, and initDictionary parses it. A corrupted page or page header makes
171+
// them throw these: an IndexOutOfBoundsException when a BINARY dictionary ends within the
172+
// length of an entry, a NegativeArraySizeException for a negative size, and an
173+
// UnsupportedOperationException for an encoding that has no dictionary. An entry that
174+
// claims more bytes than the page has is not detected here (SPARK-60316).
175+
throw new ParquetDecodingException(
176+
"could not decode the dictionary for " + descriptor + ": " + e, e);
185177
}
178+
this.dictionary = dict;
179+
this.isCurrentPageDictionaryEncoded = dict != null;
186180
if (pageReader.getTotalValueCount() == 0) {
187181
throw new IOException("totalValueCount == 0");
188182
}
@@ -380,10 +374,10 @@ private int readPage() {
380374
throw new ParquetDecodingException(
381375
"could not read the next page in col " + descriptor + ": " + e.getMessage(), e);
382376
}
383-
this.currentPage = page;
384377
if (page == null) {
385378
return -1;
386379
}
380+
this.currentPage = page.toString();
387381
this.pageFirstRowIndex = page.getFirstRowIndex().orElse(0L);
388382

389383
return page.accept(new DataPage.Visitor<Integer>() {

‎sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedDeltaBinaryPackedReader.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOExce
101101
throw new ParquetDecodingException("Invalid DELTA_BINARY_PACKED block size in values: "
102102
+ blockSizeInValues + " (must be in [1, " + MAX_BLOCK_SIZE_IN_VALUES + "])");
103103
}
104-
if (miniBlockNumInABlock <= 0 || miniBlockNumInABlock > blockSizeInValues) {
104+
if (miniBlockNumInABlock <= 0) {
105105
throw new ParquetDecodingException("Invalid DELTA_BINARY_PACKED mini block count: "
106106
+ miniBlockNumInABlock + " (block size in values: " + blockSizeInValues + ")");
107107
}
@@ -476,9 +476,15 @@ private void readBlockHeader() {
476476
* see org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesReader#unpackMiniBlock
477477
*/
478478
private void unpackMiniBlock() throws IOException {
479+
int bitWidth = bitWidths[currentMiniBlock];
480+
// The bit widths of the unused mini blocks of the last block can be anything, so a width is
481+
// checked only when its mini block is unpacked.
482+
if (bitWidth > 64) {
483+
throw new ParquetDecodingException("Invalid DELTA_BINARY_PACKED mini block bit width: "
484+
+ bitWidth + " (must be at most 64)");
485+
}
479486
Arrays.fill(this.unpackedValuesBuffer, 0);
480-
BytePackerForLong packer = Packer.LITTLE_ENDIAN.newBytePackerForLong(
481-
bitWidths[currentMiniBlock]);
487+
BytePackerForLong packer = Packer.LITTLE_ENDIAN.newBytePackerForLong(bitWidth);
482488
for (int j = 0; j < miniBlockSizeInValues; j += 8) {
483489
ByteBuffer buffer = in.slice(packer.getBitWidth());
484490
if (buffer.hasArray()) {

‎sql/core/src/test/java/test/org/apache/parquet/column/page/TestDataPage.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,9 @@ public Optional<Integer> getIndexRowCount() {
4141
public <T> T accept(Visitor<T> visitor) {
4242
return wrapped.accept(visitor);
4343
}
44+
45+
@Override
46+
public String toString() {
47+
return wrapped.toString();
48+
}
4449
}

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetDeltaEncodingSuite.scala‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,25 @@ abstract class ParquetDeltaEncodingSuite[T] extends ParquetCompatibilityTest
204204
reader.initFromPage(100, craftHeader(128, 4, -1))
205205
}
206206
assert(e3.getMessage.contains("total value count"))
207+
208+
// The mini blocks must have a multiple of 8 values: 128 values do not split into 3 mini
209+
// blocks, and split into 32 mini blocks of 4 values.
210+
Seq(3, 32).foreach { miniBlockNum =>
211+
reader = new VectorizedDeltaBinaryPackedReader
212+
val e4 = intercept[ParquetDecodingException] {
213+
reader.initFromPage(100, craftHeader(128, miniBlockNum, 1))
214+
}
215+
assert(e4.getMessage == s"Invalid DELTA_BINARY_PACKED mini block count: $miniBlockNum " +
216+
"(block size in values: 128, the mini block size must be a multiple of 8)")
217+
}
218+
219+
// A page must have at least one value.
220+
reader = new VectorizedDeltaBinaryPackedReader
221+
val e5 = intercept[ParquetDecodingException] {
222+
reader.initFromPage(0, craftHeader(128, 4, 1))
223+
}
224+
assert(e5.getMessage ==
225+
"Invalid DELTA_BINARY_PACKED page: it must have at least one value, but it has 0")
207226
}
208227

209228
test("skip()") {

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala‎

Lines changed: 20 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,9 @@ import org.apache.parquet.column.{Encoding, ParquetProperties}
3737
import org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0
3838
import org.apache.parquet.example.data.Group
3939
import org.apache.parquet.example.data.simple.{SimpleGroup, SimpleGroupFactory}
40-
import org.apache.parquet.format.Util
41-
import org.apache.parquet.format.converter.ParquetMetadataConverter
40+
import org.apache.parquet.format.{PageType, Util}
4241
import org.apache.parquet.hadoop._
4342
import org.apache.parquet.hadoop.example.ExampleParquetWriter
44-
import org.apache.parquet.hadoop.util.HadoopInputFile
4543
import org.apache.parquet.io.ParquetDecodingException
4644
import org.apache.parquet.io.api.Binary
4745
import org.apache.parquet.schema.{MessageType, MessageTypeParser}
@@ -2496,14 +2494,14 @@ class ParquetIOSuite extends ParquetTest with SharedSparkSession {
24962494
* the written file, and returns the error of reading the file with the vectorized reader.
24972495
*/
24982496
private def readCorruptedFile(column: String, options: Map[String, String])(
2499-
corrupt: (Path, Array[Byte]) => Unit): Throwable = {
2497+
corrupt: Array[Byte] => Unit): Throwable = {
25002498
var error: Throwable = null
25012499
withTempPath { dir =>
25022500
val path = dir.getCanonicalPath
25032501
spark.range(100).selectExpr(column).coalesce(1).write.options(options).parquet(path)
25042502
val file = dir.listFiles().filter(_.getName.endsWith(".parquet")).head
25052503
val bytes = Files.readAllBytes(file.toPath)
2506-
corrupt(new Path(file.getCanonicalPath), bytes)
2504+
corrupt(bytes)
25072505
Files.write(file.toPath, bytes)
25082506
dir.listFiles().filter(_.getName.endsWith(".crc")).foreach(_.delete())
25092507

@@ -2534,7 +2532,7 @@ class ParquetIOSuite extends ParquetTest with SharedSparkSession {
25342532
ParquetOutputFormat.ENABLE_DICTIONARY -> "false",
25352533
ParquetOutputFormat.WRITER_VERSION -> writerVersion,
25362534
"compression" -> "uncompressed")
2537-
readCorruptedFile(column, options) { (_, bytes) =>
2535+
readCorruptedFile(column, options) { bytes =>
25382536
val section = Array[Byte](3, 0, 0, 0, 0xC8.toByte, 0x01, 1)
25392537
val offset = bytes.indexOfSlice(section)
25402538
assert(offset >= 0 && bytes.indexOfSlice(section, offset + 1) < 0)
@@ -2585,28 +2583,24 @@ class ParquetIOSuite extends ParquetTest with SharedSparkSession {
25852583
}
25862584

25872585
test("SPARK-59854: a ZSTD page that cannot be decompressed reports the column") {
2588-
Seq(false, true).foreach { dictionary =>
2589-
val options = Map(
2590-
ParquetOutputFormat.ENABLE_DICTIONARY -> dictionary.toString,
2591-
"compression" -> "zstd")
2592-
val e = readCorruptedFile("cast(id % 10 as int) as a", options) { (file, bytes) =>
2593-
val footer = ParquetFooterReader.readFooter(
2594-
HadoopInputFile.fromPath(file, spark.sessionState.newHadoopConf()),
2595-
ParquetMetadataConverter.NO_FILTER)
2596-
val chunk = footer.getBlocks.get(0).getColumns.get(0)
2597-
assert(chunk.hasDictionaryPage == dictionary)
2598-
// Invert the first bytes of the first page, the ZSTD frame magic number.
2599-
val in = new ByteArrayInputStream(bytes, chunk.getStartingPos.toInt, bytes.length)
2600-
Util.readPageHeader(in)
2601-
val pageStart = bytes.length - in.available()
2602-
(pageStart until pageStart + 4).foreach(i => bytes(i) = (~bytes(i)).toByte)
2603-
}
2604-
val columnErrors =
2605-
causes(e).filter(c => Option(c.getMessage).exists(_.contains("[a] optional int32 a")))
2606-
assert(columnErrors.size == 1, e)
2607-
assert(columnErrors.head.isInstanceOf[ParquetDecodingException], e)
2586+
Seq(PageType.DATA_PAGE -> false, PageType.DICTIONARY_PAGE -> true).foreach {
2587+
case (pageType, dictionary) =>
2588+
val options = Map(
2589+
ParquetOutputFormat.ENABLE_DICTIONARY -> dictionary.toString,
2590+
"compression" -> "zstd")
2591+
val e = readCorruptedFile("cast(id % 10 as int) as a", options) { bytes =>
2592+
// The first page of the only column chunk starts right after the magic number.
2593+
val in = new ByteArrayInputStream(bytes, ParquetFileWriter.MAGIC.length, bytes.length)
2594+
assert(Util.readPageHeader(in).getType == pageType)
2595+
// Invert the first bytes of the page, the ZSTD frame magic number.
2596+
val pageStart = bytes.length - in.available()
2597+
(pageStart until pageStart + 4).foreach(i => bytes(i) = (~bytes(i)).toByte)
2598+
}
2599+
assert(causes(e).exists(c =>
2600+
Option(c.getMessage).exists(_.contains("[a] optional int32 a"))), e)
26082601
}
26092602
}
2603+
26102604
}
26112605

26122606
class JobCommitFailureParquetOutputCommitter(outputPath: Path, context: TaskAttemptContext)

0 commit comments

Comments
 (0)