Skip to content

Commit 20ce444

Browse files
committed
[SPARK-59854][SQL] Report the column in decoding errors of the vectorized Parquet reader
Add the column to decoding errors of the vectorized Parquet reader that did not report it: - ParquetDecodingExceptions thrown while decoding a batch in VectorizedColumnReader.readBatch. - ParquetDecodingExceptions thrown by PageReader.readPage and readDictionaryPage, which decompress ZSTD pages eagerly. - RuntimeExceptions thrown by Encoding.initDictionary, e.g. for a truncated BINARY dictionary. - Invalid DELTA_BINARY_PACKED headers, which now fail with a ParquetDecodingException instead of an IllegalArgumentException.
1 parent f39493b commit 20ce444

4 files changed

Lines changed: 236 additions & 70 deletions

File tree

‎sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java‎

Lines changed: 98 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,11 @@ public class VectorizedColumnReader {
9898
*/
9999
private long pageFirstRowIndex;
100100

101+
/**
102+
* The data page the values are being decoded from, for error messages.
103+
*/
104+
private DataPage currentPage;
105+
101106
private final PageReader pageReader;
102107
private final ColumnDescriptor descriptor;
103108
private final LogicalTypeAnnotation logicalTypeAnnotation;
@@ -147,7 +152,14 @@ public VectorizedColumnReader(
147152
int96RebaseMode,
148153
int96RebaseTz);
149154

150-
DictionaryPage dictionaryPage = pageReader.readDictionaryPage();
155+
DictionaryPage dictionaryPage;
156+
try {
157+
// This decompresses the page eagerly for some codecs, e.g. ZSTD.
158+
dictionaryPage = pageReader.readDictionaryPage();
159+
} catch (ParquetDecodingException e) {
160+
throw new ParquetDecodingException(
161+
"could not read the dictionary page for " + descriptor + ": " + e.getMessage(), e);
162+
}
151163
if (dictionaryPage != null) {
152164
try {
153165
this.dictionary = dictionaryPage.getEncoding().initDictionary(descriptor, dictionaryPage);
@@ -157,6 +169,11 @@ public VectorizedColumnReader(
157169
} catch (ParquetDecodingException e) {
158170
throw new ParquetDecodingException(
159171
"could not decode the dictionary for " + descriptor + ": " + e.getMessage(), e);
172+
} catch (RuntimeException e) {
173+
// initDictionary only parses the bytes of the page, so a failure here comes from the page
174+
// or its metadata, e.g. an IndexOutOfBoundsException for a truncated BINARY dictionary.
175+
throw new ParquetDecodingException(
176+
"could not decode the dictionary for " + descriptor + ": " + e, e);
160177
}
161178
} else {
162179
this.dictionary = null;
@@ -264,79 +281,96 @@ void readBatch(
264281
}
265282
readState.resetForNewPage(pageValueCount, pageFirstRowIndex);
266283
}
267-
PrimitiveType.PrimitiveTypeName typeName =
268-
descriptor.getPrimitiveType().getPrimitiveTypeName();
269-
if (isCurrentPageDictionaryEncoded) {
270-
// Save starting offset in case we need to decode dictionary IDs.
271-
int startOffset = readState.valueOffset;
272-
// Save starting row index so we can check if we need to eagerly decode dict ids later
273-
long startRowId = readState.rowId;
274-
275-
// Read and decode dictionary ids.
276-
if (readState.maxRepetitionLevel == 0) {
277-
defColumn.readIntegers(readState, dictionaryIds, column, definitionLevels,
278-
(VectorizedValuesReader) dataColumn);
279-
} else {
280-
repColumn.readIntegersRepeated(readState, repetitionLevels, defColumn, definitionLevels,
281-
dictionaryIds, column, (VectorizedValuesReader) dataColumn);
282-
}
283-
284-
// TIMESTAMP_MILLIS encoded as INT64 can't be lazily decoded as we need to post process
285-
// the values to add microseconds precision.
286-
if (column.hasDictionary() || (startRowId == pageFirstRowIndex &&
287-
isLazyDecodingSupported(typeName, column.dataType()))) {
288-
// Column vector supports lazy decoding of dictionary values so just set the dictionary.
289-
// We can't do this if startRowId is not the first row index in the page AND the column
290-
// doesn't have a dictionary (i.e. some non-dictionary encoded values have already been
291-
// added).
292-
PrimitiveType primitiveType = descriptor.getPrimitiveType();
293-
294-
// We need to make sure that we initialize the right type for the dictionary otherwise
295-
// WritableColumnVector will throw an exception when trying to decode to an Int when the
296-
// dictionary is in fact initialized as Long
297-
LogicalTypeAnnotation typeAnnotation = primitiveType.getLogicalTypeAnnotation();
298-
boolean castLongToInt =
299-
typeAnnotation instanceof DecimalLogicalTypeAnnotation annotation &&
300-
annotation.getPrecision() <= Decimal.MAX_INT_DIGITS() &&
301-
primitiveType.getPrimitiveTypeName() == INT64;
302-
303-
// We require a long value, but we need to use dictionary to decode the original
304-
// signed int first
305-
boolean isUnsignedInt32 = updaterFactory.isUnsignedIntTypeMatched(32);
306-
307-
// We require a decimal value, but we need to use dictionary to decode the original
308-
// signed long first
309-
boolean isUnsignedInt64 = updaterFactory.isUnsignedIntTypeMatched(64);
310-
311-
boolean needTransform = castLongToInt || isUnsignedInt32 || isUnsignedInt64;
312-
column.setDictionary(new ParquetDictionary(dictionary, needTransform));
313-
} else {
314-
updater.decodeDictionaryIds(readState.valueOffset - startOffset, startOffset, column,
315-
dictionaryIds, dictionary);
316-
}
317-
} else {
318-
if (column.hasDictionary() && readState.valueOffset != 0) {
319-
// This batch already has dictionary encoded values but this new page is not. The batch
320-
// does not support a mix of dictionary and not so we will decode the dictionary.
321-
updater.decodeDictionaryIds(readState.valueOffset, 0, column, dictionaryIds, dictionary);
322-
}
323-
column.setDictionary(null);
324-
VectorizedValuesReader valuesReader = (VectorizedValuesReader) dataColumn;
325-
if (readState.maxRepetitionLevel == 0) {
326-
defColumn.readBatch(readState, column, definitionLevels, valuesReader, updater);
284+
// A corrupted page makes the readers below throw a ParquetDecodingException without the
285+
// column. Only that is wrapped, so that other exceptions, e.g. SparkUpgradeException,
286+
// keep their types.
287+
try {
288+
PrimitiveType.PrimitiveTypeName typeName =
289+
descriptor.getPrimitiveType().getPrimitiveTypeName();
290+
if (isCurrentPageDictionaryEncoded) {
291+
// Save starting offset in case we need to decode dictionary IDs.
292+
int startOffset = readState.valueOffset;
293+
// Save starting row index so we can check if we need to eagerly decode dict ids later
294+
long startRowId = readState.rowId;
295+
296+
// Read and decode dictionary ids.
297+
if (readState.maxRepetitionLevel == 0) {
298+
defColumn.readIntegers(readState, dictionaryIds, column, definitionLevels,
299+
(VectorizedValuesReader) dataColumn);
300+
} else {
301+
repColumn.readIntegersRepeated(readState, repetitionLevels, defColumn, definitionLevels,
302+
dictionaryIds, column, (VectorizedValuesReader) dataColumn);
303+
}
304+
305+
// TIMESTAMP_MILLIS encoded as INT64 can't be lazily decoded as we need to post process
306+
// the values to add microseconds precision.
307+
if (column.hasDictionary() || (startRowId == pageFirstRowIndex &&
308+
isLazyDecodingSupported(typeName, column.dataType()))) {
309+
// Column vector supports lazy decoding of dictionary values so just set the dictionary.
310+
// We can't do this if startRowId is not the first row index in the page AND the column
311+
// doesn't have a dictionary (i.e. some non-dictionary encoded values have already been
312+
// added).
313+
PrimitiveType primitiveType = descriptor.getPrimitiveType();
314+
315+
// We need to make sure that we initialize the right type for the dictionary otherwise
316+
// WritableColumnVector will throw an exception when trying to decode to an Int when the
317+
// dictionary is in fact initialized as Long
318+
LogicalTypeAnnotation typeAnnotation = primitiveType.getLogicalTypeAnnotation();
319+
boolean castLongToInt =
320+
typeAnnotation instanceof DecimalLogicalTypeAnnotation annotation &&
321+
annotation.getPrecision() <= Decimal.MAX_INT_DIGITS() &&
322+
primitiveType.getPrimitiveTypeName() == INT64;
323+
324+
// We require a long value, but we need to use dictionary to decode the original
325+
// signed int first
326+
boolean isUnsignedInt32 = updaterFactory.isUnsignedIntTypeMatched(32);
327+
328+
// We require a decimal value, but we need to use dictionary to decode the original
329+
// signed long first
330+
boolean isUnsignedInt64 = updaterFactory.isUnsignedIntTypeMatched(64);
331+
332+
boolean needTransform = castLongToInt || isUnsignedInt32 || isUnsignedInt64;
333+
column.setDictionary(new ParquetDictionary(dictionary, needTransform));
334+
} else {
335+
updater.decodeDictionaryIds(readState.valueOffset - startOffset, startOffset, column,
336+
dictionaryIds, dictionary);
337+
}
327338
} else {
328-
repColumn.readBatchRepeated(readState, repetitionLevels, defColumn, definitionLevels,
329-
column, valuesReader, updater);
339+
if (column.hasDictionary() && readState.valueOffset != 0) {
340+
// This batch already has dictionary encoded values but this new page is not. The batch
341+
// does not support a mix of dictionary and not so we will decode the dictionary.
342+
updater.decodeDictionaryIds(readState.valueOffset, 0, column, dictionaryIds,
343+
dictionary);
344+
}
345+
column.setDictionary(null);
346+
VectorizedValuesReader valuesReader = (VectorizedValuesReader) dataColumn;
347+
if (readState.maxRepetitionLevel == 0) {
348+
defColumn.readBatch(readState, column, definitionLevels, valuesReader, updater);
349+
} else {
350+
repColumn.readBatchRepeated(readState, repetitionLevels, defColumn, definitionLevels,
351+
column, valuesReader, updater);
352+
}
330353
}
354+
} catch (ParquetDecodingException e) {
355+
throw new ParquetDecodingException("could not decode page " + currentPage + " in col " +
356+
descriptor + ": " + e.getMessage(), e);
331357
}
332358
}
333359
}
334360

335361
private int readPage() {
336-
DataPage page = pageReader.readPage();
362+
DataPage page;
363+
try {
364+
// This decompresses the page eagerly for some codecs, e.g. ZSTD.
365+
page = pageReader.readPage();
366+
} catch (ParquetDecodingException e) {
367+
throw new ParquetDecodingException(
368+
"could not read the next page in col " + descriptor + ": " + e.getMessage(), e);
369+
}
337370
if (page == null) {
338371
return -1;
339372
}
373+
this.currentPage = page;
340374
this.pageFirstRowIndex = page.getFirstRowIndex().orElse(0L);
341375

342376
return page.accept(new DataPage.Visitor<Integer>() {

‎sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedDeltaBinaryPackedReader.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import org.apache.parquet.column.values.bitpacking.Packer;
2727
import org.apache.parquet.io.ParquetDecodingException;
2828

29-
import org.apache.spark.network.util.JavaUtils;
3029
import org.apache.spark.sql.catalyst.util.RebaseDateTime;
3130
import org.apache.spark.sql.execution.datasources.DataSourceUtils;
3231
import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
@@ -89,8 +88,10 @@ public class VectorizedDeltaBinaryPackedReader extends VectorizedReaderBase {
8988

9089
@Override
9190
public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOException {
92-
JavaUtils.checkArgument(valueCount >= 1,
93-
"Page must have at least one value, but it has " + valueCount);
91+
if (valueCount < 1) {
92+
throw new ParquetDecodingException("Invalid DELTA_BINARY_PACKED page: it must have at "
93+
+ "least one value, but it has " + valueCount);
94+
}
9495
this.in = in;
9596
// Read the header
9697
this.blockSizeInValues = BytesUtils.readUnsignedVarInt(in);
@@ -105,8 +106,10 @@ public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOExce
105106
+ miniBlockNumInABlock + " (block size in values: " + blockSizeInValues + ")");
106107
}
107108
double miniSize = (double) blockSizeInValues / miniBlockNumInABlock;
108-
JavaUtils.checkArgument(miniSize % 8 == 0,
109-
"miniBlockSize must be multiple of 8, but it's " + miniSize);
109+
if (miniSize % 8 != 0) {
110+
throw new ParquetDecodingException("Invalid DELTA_BINARY_PACKED mini block size: "
111+
+ miniSize + " (must be a multiple of 8)");
112+
}
110113
this.miniBlockSizeInValues = (int) miniSize;
111114
// True value count. May be less than valueCount because of nulls
112115
this.totalValueCount = BytesUtils.readUnsignedVarInt(in);

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.spark.sql.execution.datasources.parquet
1919

20+
import java.io.ByteArrayInputStream
2021
import java.math.{BigDecimal => JBigDecimal}
2122
import java.nio.file.Files
2223
import java.time.{LocalDateTime, LocalTime}
@@ -36,6 +37,7 @@ import org.apache.parquet.column.{Encoding, ParquetProperties}
3637
import org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0
3738
import org.apache.parquet.example.data.Group
3839
import org.apache.parquet.example.data.simple.{SimpleGroup, SimpleGroupFactory}
40+
import org.apache.parquet.format.Util
3941
import org.apache.parquet.hadoop._
4042
import org.apache.parquet.hadoop.example.ExampleParquetWriter
4143
import org.apache.parquet.io.ParquetDecodingException
@@ -2569,6 +2571,47 @@ class ParquetIOSuite extends ParquetTest with SharedSparkSession {
25692571
val e = readCorruptedRleSection("id >= 0 as b", writerVersion = "PARQUET_2_0")(setMaxLength)
25702572
assertInvalidLengthReportsColumn(e, "b")
25712573
}
2574+
2575+
test("SPARK-59854: a ZSTD page that cannot be decompressed reports the column") {
2576+
// ZSTD pages are decompressed when they are read from the column chunk, not when decoded.
2577+
Seq(
2578+
(false, "could not read the next page in col", "could not decompress page"),
2579+
(true, "could not read the dictionary page for", "Could not decompress dictionary page")
2580+
).foreach { case (dictionary, error, cause) =>
2581+
withTempPath { dir =>
2582+
val path = dir.getCanonicalPath
2583+
spark.range(1000).selectExpr("cast(id % 100 as int) as a").coalesce(1)
2584+
.write
2585+
.option(ParquetOutputFormat.ENABLE_DICTIONARY, dictionary.toString)
2586+
.option("compression", "zstd")
2587+
.parquet(path)
2588+
val file = dir.listFiles().filter(_.getName.endsWith(".parquet")).head
2589+
val footer = ParquetFileReader.readFooter(
2590+
spark.sessionState.newHadoopConf(), new Path(file.getCanonicalPath))
2591+
val chunk = footer.getBlocks.get(0).getColumns.get(0)
2592+
assert(chunk.hasDictionaryPage == dictionary)
2593+
// Invert the first bytes of the first page, the ZSTD frame magic number.
2594+
val bytes = Files.readAllBytes(file.toPath)
2595+
val in = new ByteArrayInputStream(bytes, chunk.getStartingPos.toInt, bytes.length)
2596+
Util.readPageHeader(in)
2597+
val pageStart = bytes.length - in.available()
2598+
(pageStart until pageStart + 4).foreach(i => bytes(i) = (~bytes(i)).toByte)
2599+
Files.write(file.toPath, bytes)
2600+
dir.listFiles().filter(_.getName.endsWith(".crc")).foreach(_.delete())
2601+
2602+
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true") {
2603+
val e = intercept[SparkException](spark.read.parquet(path).collect())
2604+
val columnErrors =
2605+
causes(e).filter(c => Option(c.getMessage).exists(_.contains("optional int32 a")))
2606+
assert(columnErrors.size == 1, e)
2607+
assert(columnErrors.head.getMessage ==
2608+
s"$error [a] optional int32 a: $cause", e)
2609+
assert(columnErrors.head.isInstanceOf[ParquetDecodingException], e)
2610+
assert(columnErrors.head.getCause.getMessage == cause, e)
2611+
}
2612+
}
2613+
}
2614+
}
25722615
}
25732616

25742617
class JobCommitFailureParquetOutputCommitter(outputPath: Path, context: TaskAttemptContext)

0 commit comments

Comments
 (0)