2929import java .nio .ByteBuffer ;
3030import java .nio .ByteOrder ;
3131import org .apache .parquet .bytes .ByteBufferInputStream ;
32+ import org .apache .parquet .bytes .BytesUtils ;
3233import org .apache .parquet .column .values .ValuesReader ;
3334import org .apache .parquet .io .ParquetDecodingException ;
3435
@@ -54,8 +55,12 @@ abstract class AlpValuesReader extends ValuesReader {
5455 protected int pageValueIndex ;
5556 protected int currentVectorIndex ;
5657
58+ // One entry per vector plus a trailing entry for the end of the vector data, so the entry after
59+ // any vector's offset is always where that vector ends.
5760 protected int [] vectorOffsets ;
5861 protected ByteBuffer vectorsData ;
62+ // Captured once when the page is sliced, so nothing later depends on the buffer's position/limit.
63+ protected int vectorDataSize ;
5964 protected int offsetArraySize ;
6065
6166 // Scratch buffer for exception positions within a vector; shared by both readers (int[] in each).
@@ -86,7 +91,11 @@ public void initFromPage(int valuesCount, ByteBufferInputStream stream)
8691 int minVectorBytes = ALP_INFO_SIZE + forInfoSize ();
8792 readOffsetArray (stream , minVectorBytes );
8893 sliceVectorData (stream );
89- validateVectorOffsets (minVectorBytes );
94+ // The trailing entry is filled here rather than in readOffsetArray because the vector data
95+ // size is only known once the data has been sliced.
96+ vectorOffsets [numVectors ] = offsetArraySize + vectorDataSize ;
97+ validateVectorOffsets (
98+ vectorOffsets , minVectorBytes , maxVectorBytes (valuesPerVector , minVectorBytes , exceptionValueSize ()));
9099
91100 allocateDecodedBuffer (valuesPerVector );
92101 this .excPositionsBuffer = new int [valuesPerVector ];
@@ -134,7 +143,10 @@ private void readPageHeader(ByteBufferInputStream stream, int valuesCount) throw
134143 this .currentVectorIndex = -1 ;
135144 }
136145
137- /** Reads the offset array, one entry per vector, after bounding the vector count. */
146+ /**
147+ * Reads the offset array, one entry per vector, after bounding the vector count and checking the
148+ * first offset. Leaves room for the trailing end-of-data entry, which the caller fills in.
149+ */
138150 private void readOffsetArray (ByteBufferInputStream stream , int minVectorBytes ) throws IOException {
139151 // Bound the vector count against the bytes actually present before allocating anything.
140152 // numElements is only capped by valuesCount, which a forged page can set as high as
@@ -154,8 +166,20 @@ private void readOffsetArray(ByteBufferInputStream stream, int minVectorBytes) t
154166
155167 this .offsetArraySize = numVectors * Integer .BYTES ;
156168 ByteBuffer offsetBuf = stream .slice (offsetArraySize ).order (ByteOrder .LITTLE_ENDIAN );
157- this .vectorOffsets = new int [numVectors ];
158- for (int v = 0 ; v < numVectors ; v ++) {
169+ if (numVectors == 0 ) {
170+ this .vectorOffsets = new int [1 ];
171+ return ;
172+ }
173+ // Offsets are measured from the start of the offset array, so the first vector must begin
174+ // exactly where the array ends. Check that before allocating and decoding the rest.
175+ int firstOffset = offsetBuf .getInt ();
176+ if (firstOffset != offsetArraySize ) {
177+ throw new ParquetDecodingException (
178+ "ALP first vector offset " + firstOffset + " must equal the offset array size " + offsetArraySize );
179+ }
180+ this .vectorOffsets = new int [numVectors + 1 ];
181+ vectorOffsets [0 ] = firstOffset ;
182+ for (int v = 1 ; v < numVectors ; v ++) {
159183 vectorOffsets [v ] = offsetBuf .getInt ();
160184 }
161185 }
@@ -165,48 +189,56 @@ private void readOffsetArray(ByteBufferInputStream stream, int minVectorBytes) t
165189 * absolute {@code get(pos)} reads.
166190 */
167191 private void sliceVectorData (ByteBufferInputStream stream ) throws IOException {
168- int remainingBytes = (int ) stream .available ();
169- ByteBuffer rawSlice = stream .slice (remainingBytes );
192+ this . vectorDataSize = (int ) stream .available ();
193+ ByteBuffer rawSlice = stream .slice (vectorDataSize );
170194 this .vectorsData = rawSlice .slice ().order (ByteOrder .LITTLE_ENDIAN );
171195 }
172196
173197 /**
174- * Validates the offset array against the page body before any vector is decoded. Offsets are
175- * relative to the start of the compression body (the offset array itself), so the first vector
176- * must begin exactly where the offset array ends, offsets must increase by at least one vector's
177- * fixed headers, and every vector must start early enough to fit those headers. Without this a
178- * forged offset can point into another vector and silently decode the wrong values.
198+ * Validates the offset array before any vector is decoded. {@code vectorOffsets} holds one entry
199+ * per vector followed by the end of the vector data, all measured from the start of the offset
200+ * array. The first entry is checked when the array is read, so this starts at entry 1: each
201+ * vector, including the last, must span at least its fixed headers and no more than
202+ * {@code maxVectorBytes}. Without this a forged offset can point into another vector and
203+ * silently decode the wrong values.
179204 */
180- private void validateVectorOffsets (int minVectorBytes ) {
181- int bodyLimit = offsetArraySize + vectorsData .limit ();
182- for (int v = 0 ; v < numVectors ; v ++) {
183- int offset = vectorOffsets [v ];
184- if (v == 0 ) {
185- if (offset != offsetArraySize ) {
186- throw new ParquetDecodingException ("ALP first vector offset " + offset
187- + " must equal the offset array size " + offsetArraySize );
188- }
189- } else if ((long ) offset - vectorOffsets [v - 1 ] < minVectorBytes ) {
190- throw new ParquetDecodingException ("ALP vector offsets must increase by at least "
191- + minVectorBytes + " bytes, but vector " + v + " starts at " + offset + " after vector "
192- + (v - 1 ) + " at " + vectorOffsets [v - 1 ]);
205+ static void validateVectorOffsets (int [] vectorOffsets , int minVectorBytes , int maxVectorBytes ) {
206+ int numVectors = vectorOffsets .length - 1 ;
207+ for (int v = 1 ; v <= numVectors ; v ++) {
208+ long span = (long ) vectorOffsets [v ] - vectorOffsets [v - 1 ];
209+ if (span < minVectorBytes ) {
210+ throw new ParquetDecodingException ("ALP vector " + (v - 1 ) + " at offset " + vectorOffsets [v - 1 ]
211+ + " spans " + span + " bytes, fewer than the " + minVectorBytes + " its headers need" );
193212 }
194- if (offset > bodyLimit - minVectorBytes ) {
195- throw new ParquetDecodingException ("ALP vector " + v + " offset " + offset
196- + " leaves no room for its headers before the end of the page body at " + bodyLimit );
213+ if (span > maxVectorBytes ) {
214+ throw new ParquetDecodingException ("ALP vector " + ( v - 1 ) + " at offset " + vectorOffsets [ v - 1 ]
215+ + " spans " + span + " bytes, more than the " + maxVectorBytes + " any vector can occupy" );
197216 }
198217 }
199218 }
200219
201220 /**
202- * The position just past the last byte vector {@code vectorIndex} may read, derived from the
203- * next vector's offset (or the end of the body for the final vector).
221+ * The most bytes one vector can occupy: its fixed headers, every value bit-packed at the full
222+ * width of the type, and every value also stored as an exception. Vector sizes are multiples of
223+ * the 8-value packing group, so the packed section needs no padding allowance.
224+ */
225+ static int maxVectorBytes (int valuesPerVector , int minVectorBytes , int valueSize ) {
226+ int packed = valuesPerVector * valueSize ;
227+ int exceptions = valuesPerVector * (Short .BYTES + valueSize );
228+ return minVectorBytes + packed + exceptions ;
229+ }
230+
231+ /**
232+ * The position in the vector data just past the last byte vector {@code vectorIndex} may read,
233+ * taken from the next entry of the offset array; the trailing entry makes this valid for the last
234+ * vector too.
204235 */
205- private int getVectorEndPosition (int vectorIndex ) {
206- if (!isLast (vectorIndex , numVectors )) {
207- return vectorOffsets [vectorIndex + 1 ] - offsetArraySize ;
236+ int getVectorEndPosition (int vectorIndex ) {
237+ if (vectorIndex < 0 || vectorIndex >= numVectors ) {
238+ throw new IllegalArgumentException (
239+ "ALP vector index " + vectorIndex + " is out of range for " + numVectors + " vectors" );
208240 }
209- return vectorsData . limit () ;
241+ return vectorOffsets [ vectorIndex + 1 ] - offsetArraySize ;
210242 }
211243
212244 // Offsets in the page are relative to the compression body (after header),
@@ -278,14 +310,15 @@ protected final void decodeVector(int vectorIndex) {
278310 // Vector header, part two: ForInfo, whose width depends on the value type.
279311 pos = readForInfo (pos , vectorIndex , forInfo );
280312
281- // Data section, part one: PackedValues.
282- pos = decodePackedValues ( pos , vectorSize , exponent , factor , forInfo );
283- // The packed values' width comes from this vector's own ForInfo, so confirm they did not run
284- // past where the next vector begins; otherwise this vector would decode another one's bytes.
285- if (pos > vectorEnd ) {
286- throw new ParquetDecodingException ("ALP vector " + vectorIndex + " packed values end at " + pos
287- + ", past the start of the next vector at " + vectorEnd );
313+ // Data section, part one: PackedValues. Their width comes from this vector's own ForInfo, so
314+ // confirm they fit before unpacking; otherwise this vector would decode the next one's bytes,
315+ // or for the last vector read past the end of the page.
316+ int packedEnd = pos + BytesUtils . paddedByteCountFromBits ( vectorSize * forInfo . bitWidth );
317+ if (packedEnd > vectorEnd ) {
318+ throw new ParquetDecodingException ("ALP vector " + vectorIndex + " packed values end at " + packedEnd
319+ + ", past the end of the vector at " + vectorEnd );
288320 }
321+ pos = decodePackedValues (pos , vectorSize , exponent , factor , forInfo );
289322
290323 // Data section, parts two and three: ExceptionPositions then ExceptionValues.
291324 if (numExceptions > 0 ) {
@@ -362,9 +395,4 @@ static int readExceptionPositions(ByteBuffer data, int pos, int numExceptions, i
362395 * decoded values.
363396 */
364397 protected abstract void applyExceptions (int pos , int numExceptions , int vectorSize );
365-
366- /**
367- * Reads {@code numExceptions} original values starting at {@code pos} and writes them into the
368- * decoded buffer at the positions previously read into {@link #excPositionsBuffer}.
369- */
370398}
0 commit comments