@@ -98,6 +98,11 @@ public class VectorizedColumnReader {
9898 */
9999 private long pageFirstRowIndex ;
100100
101+ /**
102+ * The data page the values are being decoded from, for error messages.
103+ */
104+ private DataPage currentPage ;
105+
101106 private final PageReader pageReader ;
102107 private final ColumnDescriptor descriptor ;
103108 private final LogicalTypeAnnotation logicalTypeAnnotation ;
@@ -147,7 +152,14 @@ public VectorizedColumnReader(
147152 int96RebaseMode ,
148153 int96RebaseTz );
149154
150- DictionaryPage dictionaryPage = pageReader .readDictionaryPage ();
155+ DictionaryPage dictionaryPage ;
156+ try {
157+ // This decompresses the page eagerly for some codecs, e.g. ZSTD.
158+ dictionaryPage = pageReader .readDictionaryPage ();
159+ } catch (ParquetDecodingException e ) {
160+ throw new ParquetDecodingException (
161+ "could not read the dictionary page for " + descriptor + ": " + e .getMessage (), e );
162+ }
151163 if (dictionaryPage != null ) {
152164 try {
153165 this .dictionary = dictionaryPage .getEncoding ().initDictionary (descriptor , dictionaryPage );
@@ -157,6 +169,11 @@ public VectorizedColumnReader(
157169 } catch (ParquetDecodingException e ) {
158170 throw new ParquetDecodingException (
159171 "could not decode the dictionary for " + descriptor + ": " + e .getMessage (), e );
172+ } catch (RuntimeException e ) {
173+ // initDictionary only parses the bytes of the page, so a failure here comes from the page
174+ // or its metadata, e.g. an IndexOutOfBoundsException for a truncated BINARY dictionary.
175+ throw new ParquetDecodingException (
176+ "could not decode the dictionary for " + descriptor + ": " + e , e );
160177 }
161178 } else {
162179 this .dictionary = null ;
@@ -264,79 +281,96 @@ void readBatch(
264281 }
265282 readState .resetForNewPage (pageValueCount , pageFirstRowIndex );
266283 }
267- PrimitiveType .PrimitiveTypeName typeName =
268- descriptor .getPrimitiveType ().getPrimitiveTypeName ();
269- if (isCurrentPageDictionaryEncoded ) {
270- // Save starting offset in case we need to decode dictionary IDs.
271- int startOffset = readState .valueOffset ;
272- // Save starting row index so we can check if we need to eagerly decode dict ids later
273- long startRowId = readState .rowId ;
274-
275- // Read and decode dictionary ids.
276- if (readState .maxRepetitionLevel == 0 ) {
277- defColumn .readIntegers (readState , dictionaryIds , column , definitionLevels ,
278- (VectorizedValuesReader ) dataColumn );
279- } else {
280- repColumn .readIntegersRepeated (readState , repetitionLevels , defColumn , definitionLevels ,
281- dictionaryIds , column , (VectorizedValuesReader ) dataColumn );
282- }
283-
284- // TIMESTAMP_MILLIS encoded as INT64 can't be lazily decoded as we need to post process
285- // the values to add microseconds precision.
286- if (column .hasDictionary () || (startRowId == pageFirstRowIndex &&
287- isLazyDecodingSupported (typeName , column .dataType ()))) {
288- // Column vector supports lazy decoding of dictionary values so just set the dictionary.
289- // We can't do this if startRowId is not the first row index in the page AND the column
290- // doesn't have a dictionary (i.e. some non-dictionary encoded values have already been
291- // added).
292- PrimitiveType primitiveType = descriptor .getPrimitiveType ();
293-
294- // We need to make sure that we initialize the right type for the dictionary otherwise
295- // WritableColumnVector will throw an exception when trying to decode to an Int when the
296- // dictionary is in fact initialized as Long
297- LogicalTypeAnnotation typeAnnotation = primitiveType .getLogicalTypeAnnotation ();
298- boolean castLongToInt =
299- typeAnnotation instanceof DecimalLogicalTypeAnnotation annotation &&
300- annotation .getPrecision () <= Decimal .MAX_INT_DIGITS () &&
301- primitiveType .getPrimitiveTypeName () == INT64 ;
302-
303- // We require a long value, but we need to use dictionary to decode the original
304- // signed int first
305- boolean isUnsignedInt32 = updaterFactory .isUnsignedIntTypeMatched (32 );
306-
307- // We require a decimal value, but we need to use dictionary to decode the original
308- // signed long first
309- boolean isUnsignedInt64 = updaterFactory .isUnsignedIntTypeMatched (64 );
310-
311- boolean needTransform = castLongToInt || isUnsignedInt32 || isUnsignedInt64 ;
312- column .setDictionary (new ParquetDictionary (dictionary , needTransform ));
313- } else {
314- updater .decodeDictionaryIds (readState .valueOffset - startOffset , startOffset , column ,
315- dictionaryIds , dictionary );
316- }
317- } else {
318- if (column .hasDictionary () && readState .valueOffset != 0 ) {
319- // This batch already has dictionary encoded values but this new page is not. The batch
320- // does not support a mix of dictionary and not so we will decode the dictionary.
321- updater .decodeDictionaryIds (readState .valueOffset , 0 , column , dictionaryIds , dictionary );
322- }
323- column .setDictionary (null );
324- VectorizedValuesReader valuesReader = (VectorizedValuesReader ) dataColumn ;
325- if (readState .maxRepetitionLevel == 0 ) {
326- defColumn .readBatch (readState , column , definitionLevels , valuesReader , updater );
284+ // A corrupted page makes the readers below throw a ParquetDecodingException without the
285+ // column. Only that is wrapped, so that other exceptions, e.g. SparkUpgradeException,
286+ // keep their types.
287+ try {
288+ PrimitiveType .PrimitiveTypeName typeName =
289+ descriptor .getPrimitiveType ().getPrimitiveTypeName ();
290+ if (isCurrentPageDictionaryEncoded ) {
291+ // Save starting offset in case we need to decode dictionary IDs.
292+ int startOffset = readState .valueOffset ;
293+ // Save starting row index so we can check if we need to eagerly decode dict ids later
294+ long startRowId = readState .rowId ;
295+
296+ // Read and decode dictionary ids.
297+ if (readState .maxRepetitionLevel == 0 ) {
298+ defColumn .readIntegers (readState , dictionaryIds , column , definitionLevels ,
299+ (VectorizedValuesReader ) dataColumn );
300+ } else {
301+ repColumn .readIntegersRepeated (readState , repetitionLevels , defColumn , definitionLevels ,
302+ dictionaryIds , column , (VectorizedValuesReader ) dataColumn );
303+ }
304+
305+ // TIMESTAMP_MILLIS encoded as INT64 can't be lazily decoded as we need to post process
306+ // the values to add microseconds precision.
307+ if (column .hasDictionary () || (startRowId == pageFirstRowIndex &&
308+ isLazyDecodingSupported (typeName , column .dataType ()))) {
309+ // Column vector supports lazy decoding of dictionary values so just set the dictionary.
310+ // We can't do this if startRowId is not the first row index in the page AND the column
311+ // doesn't have a dictionary (i.e. some non-dictionary encoded values have already been
312+ // added).
313+ PrimitiveType primitiveType = descriptor .getPrimitiveType ();
314+
315+ // We need to make sure that we initialize the right type for the dictionary otherwise
316+ // WritableColumnVector will throw an exception when trying to decode to an Int when the
317+ // dictionary is in fact initialized as Long
318+ LogicalTypeAnnotation typeAnnotation = primitiveType .getLogicalTypeAnnotation ();
319+ boolean castLongToInt =
320+ typeAnnotation instanceof DecimalLogicalTypeAnnotation annotation &&
321+ annotation .getPrecision () <= Decimal .MAX_INT_DIGITS () &&
322+ primitiveType .getPrimitiveTypeName () == INT64 ;
323+
324+ // We require a long value, but we need to use dictionary to decode the original
325+ // signed int first
326+ boolean isUnsignedInt32 = updaterFactory .isUnsignedIntTypeMatched (32 );
327+
328+ // We require a decimal value, but we need to use dictionary to decode the original
329+ // signed long first
330+ boolean isUnsignedInt64 = updaterFactory .isUnsignedIntTypeMatched (64 );
331+
332+ boolean needTransform = castLongToInt || isUnsignedInt32 || isUnsignedInt64 ;
333+ column .setDictionary (new ParquetDictionary (dictionary , needTransform ));
334+ } else {
335+ updater .decodeDictionaryIds (readState .valueOffset - startOffset , startOffset , column ,
336+ dictionaryIds , dictionary );
337+ }
327338 } else {
328- repColumn .readBatchRepeated (readState , repetitionLevels , defColumn , definitionLevels ,
329- column , valuesReader , updater );
339+ if (column .hasDictionary () && readState .valueOffset != 0 ) {
340+ // This batch already has dictionary encoded values but this new page is not. The batch
341+ // does not support a mix of dictionary and not so we will decode the dictionary.
342+ updater .decodeDictionaryIds (readState .valueOffset , 0 , column , dictionaryIds ,
343+ dictionary );
344+ }
345+ column .setDictionary (null );
346+ VectorizedValuesReader valuesReader = (VectorizedValuesReader ) dataColumn ;
347+ if (readState .maxRepetitionLevel == 0 ) {
348+ defColumn .readBatch (readState , column , definitionLevels , valuesReader , updater );
349+ } else {
350+ repColumn .readBatchRepeated (readState , repetitionLevels , defColumn , definitionLevels ,
351+ column , valuesReader , updater );
352+ }
330353 }
354+ } catch (ParquetDecodingException e ) {
355+ throw new ParquetDecodingException ("could not decode page " + currentPage + " in col " +
356+ descriptor + ": " + e .getMessage (), e );
331357 }
332358 }
333359 }
334360
335361 private int readPage () {
336- DataPage page = pageReader .readPage ();
362+ DataPage page ;
363+ try {
364+ // This decompresses the page eagerly for some codecs, e.g. ZSTD.
365+ page = pageReader .readPage ();
366+ } catch (ParquetDecodingException e ) {
367+ throw new ParquetDecodingException (
368+ "could not read the next page in col " + descriptor + ": " + e .getMessage (), e );
369+ }
337370 if (page == null ) {
338371 return -1 ;
339372 }
373+ this .currentPage = page ;
340374 this .pageFirstRowIndex = page .getFirstRowIndex ().orElse (0L );
341375
342376 return page .accept (new DataPage .Visitor <Integer >() {
0 commit comments