Skip to content

Commit ce40d68

Browse files
author
Steve Loughran
committed
apacheGH-3561 Harden variant decoding
- reject oversized metadata/value declarations - range checking - depth check of 1000 elements - if metadata length == 0, fail with specific error. With tests - reject version != 1 - skip unknown primitive types.
1 parent ea349cd commit ce40d68

11 files changed

Lines changed: 1034 additions & 84 deletions

File tree

‎parquet-variant/pom.xml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,17 @@
6969
<version>${slf4j.version}</version>
7070
<scope>test</scope>
7171
</dependency>
72+
<dependency>
73+
<groupId>org.apache.parquet</groupId>
74+
<artifactId>parquet-hadoop</artifactId>
75+
<version>${project.version}</version>
76+
<scope>test</scope>
77+
</dependency>
78+
<dependency>
79+
<groupId>org.apache.hadoop</groupId>
80+
<artifactId>hadoop-client</artifactId>
81+
<scope>test</scope>
82+
</dependency>
7283
</dependencies>
7384

7485
<build>

‎parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java‎

Lines changed: 60 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ public final class Variant {
6161
* Lazy cache for the parsed array header.
6262
*/
6363
private VariantUtil.ArrayInfo cachedArrayInfo;
64+
/** Nesting depth of this Variant relative to the top-level value (0 = top-level). */
65+
private final int depth;
6466

6567
/**
6668
* The threshold to switch from linear search to binary search when looking up a field by key in
@@ -69,10 +71,31 @@ public final class Variant {
6971
*/
7072
static final int BINARY_SEARCH_THRESHOLD = 32;
7173

74+
/**
75+
* Create a Variant from a tuple of (value, metadata) byte arrays.
76+
* Includes validation of the version and the top-level structure.
77+
* @param value value buffer
78+
* @param metadata metadata buffer
79+
* @throws UnsupportedOperationException if there is a version mismatch
80+
* @throws IllegalArgumentException for any validation failure.
81+
*/
7282
public Variant(byte[] value, byte[] metadata) {
7383
this(value, 0, value.length, metadata, 0, metadata.length);
7484
}
7585

86+
/**
87+
* Create a Variant from a subset of the (value, metadata) buffers
88+
* supplied.
89+
* Includes validation of the version and the toplevel structure.
90+
* @param value value buffer
91+
* @param valuePos offset where the value data begins
92+
* @param valueLength length of value data
93+
* @param metadata metadata buffer
94+
* @param metadataPos offset where the metadata begins.
95+
* @param metadataLength length of the metadata.
96+
* @throws UnsupportedOperationException if there is a version mismatch
97+
* @throws IllegalArgumentException for any validation failure.
98+
*/
7699
public Variant(byte[] value, int valuePos, int valueLength, byte[] metadata, int metadataPos, int metadataLength) {
77100
this(ByteBuffer.wrap(value, valuePos, valueLength), ByteBuffer.wrap(metadata, metadataPos, metadataLength));
78101
}
@@ -81,10 +104,20 @@ public Variant(byte[] value, int valuePos, int valueLength, byte[] metadata, int
81104
this(value, metadata.getEncodedBuffer());
82105
}
83106

107+
/**
108+
* Create a Variant from a tuple of (value, metadata) buffers.
109+
* Includes validation of the version and the toplevel structure.
110+
* @param value value buffer
111+
* @param metadata metadata buffer
112+
* @throws UnsupportedOperationException if there is a version mismatch
113+
* @throws IllegalArgumentException for any validation failure.
114+
*/
84115
public Variant(ByteBuffer value, ByteBuffer metadata) {
85116
this.value = value.asReadOnlyBuffer().order(ByteOrder.LITTLE_ENDIAN);
86117
this.metadata = metadata.asReadOnlyBuffer().order(ByteOrder.LITTLE_ENDIAN);
87-
118+
this.depth = 0;
119+
// A metadata buffer must contain at least the version byte
120+
Preconditions.checkArgument(this.metadata.remaining() >= 1, "variant metadata is empty");
88121
// There is currently only one allowed version.
89122
if ((metadata.get(metadata.position()) & VariantUtil.VERSION_MASK) != VariantUtil.VERSION) {
90123
throw new UnsupportedOperationException(String.format(
@@ -95,29 +128,39 @@ public Variant(ByteBuffer value, ByteBuffer metadata) {
95128
// Pre-compute dictionary size for lazy metadata cache allocation.
96129
int pos = this.metadata.position();
97130
int metaOffsetSize = ((this.metadata.get(pos) >> 6) & 0x3) + 1;
98-
if (this.metadata.remaining() > 1) {
99-
Preconditions.checkArgument(
100-
this.metadata.remaining() >= 1 + metaOffsetSize,
101-
"variant metadata truncated: offsetSize=" + metaOffsetSize);
102-
this.dictSize = VariantUtil.readUnsignedLittleEndian(this.metadata, pos + 1, metaOffsetSize);
103-
long dictTableEnd = 1L + metaOffsetSize + ((long) this.dictSize + 1) * metaOffsetSize;
104-
Preconditions.checkArgument(
105-
dictTableEnd <= this.metadata.remaining(),
106-
"variant metadata dictionary extends past buffer: dictSize=" + this.dictSize);
107-
} else {
108-
this.dictSize = 0;
109-
}
131+
Preconditions.checkArgument(
132+
this.metadata.remaining() >= 1 + metaOffsetSize,
133+
"variant metadata truncated: offsetSize=" + metaOffsetSize);
134+
this.dictSize = VariantUtil.readUnsignedLittleEndian(this.metadata, pos + 1, metaOffsetSize);
135+
long dictTableEnd = 1L + metaOffsetSize + ((long) this.dictSize + 1) * metaOffsetSize;
136+
Preconditions.checkArgument(
137+
dictTableEnd <= this.metadata.remaining(),
138+
"variant metadata dictionary extends past buffer: dictSize=" + this.dictSize);
110139
this.metadataCache = null;
140+
VariantUtil.validateValueShallow(this.value, dictSize);
111141
}
112142

113143
/**
114-
* Package-private constructor that shares pre-parsed metadata state from a parent Variant.
144+
* Package-private constructor that shares pre-parsed metadata state from a parent Variant, performing shallow validation of the structure.
145+
* @param value value buffer
146+
* @param metadata metadata buffer
147+
* @param metadataCache shared metadata cache.
148+
* @param dictSize shared dictionary size.
149+
* @param depth depth of this variant in a recursive structure.
150+
* @throws IllegalArgumentException if the depth of variants is too high or the structure
151+
* invalid in some other form.
115152
*/
116-
Variant(ByteBuffer value, ByteBuffer metadata, String[] metadataCache, int dictSize) {
153+
Variant(ByteBuffer value, ByteBuffer metadata, String[] metadataCache, int dictSize, int depth) {
154+
Preconditions.checkArgument(
155+
depth <= VariantUtil.MAX_VARIANT_DEPTH,
156+
"variant nesting depth exceeds maximum %s",
157+
VariantUtil.MAX_VARIANT_DEPTH);
117158
this.value = value.asReadOnlyBuffer().order(ByteOrder.LITTLE_ENDIAN);
118159
this.metadata = metadata.asReadOnlyBuffer().order(ByteOrder.LITTLE_ENDIAN);
119160
this.metadataCache = metadataCache;
120161
this.dictSize = dictSize;
162+
this.depth = depth;
163+
VariantUtil.validateValueShallow(this.value, dictSize);
121164
}
122165

123166
public ByteBuffer getValueBuffer() {
@@ -359,9 +402,10 @@ public Variant getElementAtIndex(int index) {
359402

360403
/**
361404
* Creates a child Variant that shares this instance's metadata cache.
405+
* @throws IllegalArgumentException validation error during construction.
362406
*/
363407
private Variant childVariant(ByteBuffer childValue) {
364-
return new Variant(childValue, metadata, metadataCache, dictSize);
408+
return new Variant(childValue, metadata, metadataCache, dictSize, depth + 1);
365409
}
366410

367411
/**

‎parquet-variant/src/main/java/org/apache/parquet/variant/VariantJsonParser.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
*/
1717
package org.apache.parquet.variant;
1818

19+
import static org.apache.parquet.variant.VariantUtil.MAX_VARIANT_DEPTH;
20+
1921
import com.fasterxml.jackson.core.JsonFactory;
2022
import com.fasterxml.jackson.core.JsonParseException;
2123
import com.fasterxml.jackson.core.JsonParser;
@@ -37,7 +39,7 @@ public final class VariantJsonParser {
3739

3840
private static final JsonFactory JSON_FACTORY = JsonFactory.builder()
3941
.streamReadConstraints(StreamReadConstraints.builder()
40-
.maxNestingDepth(500)
42+
.maxNestingDepth(MAX_VARIANT_DEPTH)
4143
.maxStringLength(10_000_000)
4244
.maxDocumentLength(50_000_000L)
4345
.build())

‎parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,12 @@ class VariantUtil {
190190
// The size (in bytes) of a UUID.
191191
static final int UUID_SIZE = 16;
192192

193+
/**
194+
* Maximum permitted nesting depth of a Variant value.
195+
* same limit as in VariantJsonParser.
196+
*/
197+
static final int MAX_VARIANT_DEPTH = 1000;
198+
193199
// header bytes
194200
static final byte HEADER_NULL = primitiveHeader(NULL);
195201
static final byte HEADER_LONG_STRING = primitiveHeader(LONG_STR);
@@ -880,6 +886,158 @@ static HashMap<String, Integer> getMetadataMap(ByteBuffer metadata) {
880886
return result;
881887
}
882888

889+
/**
890+
* Bounds-checks a single Variant value node against its buffer slot. Performs no recursion
891+
* into nested children: child nodes are checked on demand when callers descend into them.
892+
*
893+
* <p>Cost: O(1) for primitives and short strings, O(numElements) for objects and arrays.
894+
* Validation of nested structures is deferred so that opening a large well-formed Variant
895+
* is not penalized by sub-trees the caller never inspects.
896+
*
897+
* @param valueBuffer the variant value buffer (position/limit define the extent of this node's slot)
898+
* @param dictSize the metadata dictionary size, used to bound object field ids
899+
* @throws IllegalArgumentException if the value header or container table does not fit within
900+
* the buffer slot, or if any object field id is out of range
901+
*/
902+
static void validateValueShallow(final ByteBuffer valueBuffer, final int dictSize) {
903+
int pos = valueBuffer.position();
904+
Preconditions.checkArgument(pos >= 0 && pos < valueBuffer.limit(), "variant value is empty");
905+
long slot = (long) valueBuffer.limit() - pos;
906+
int header = valueBuffer.get(pos) & 0xFF;
907+
int basicType = header & BASIC_TYPE_MASK;
908+
int typeInfo = (header >> BASIC_TYPE_BITS) & PRIMITIVE_TYPE_MASK;
909+
switch (basicType) {
910+
case SHORT_STR:
911+
Preconditions.checkArgument(1L + typeInfo <= slot, "variant short string extends past buffer");
912+
return;
913+
case OBJECT:
914+
validateContainerShallow(valueBuffer, typeInfo, pos, slot, dictSize, true);
915+
return;
916+
case ARRAY:
917+
validateContainerShallow(valueBuffer, typeInfo, pos, slot, dictSize, false);
918+
return;
919+
default:
920+
validatePrimitiveShallow(valueBuffer, typeInfo, pos, slot);
921+
}
922+
}
923+
924+
/**
925+
* Shallow validation of a container.
926+
*
927+
* @param valueBuffer buffer with the variant data
928+
* @param typeInfo type information from the metadata
929+
* @param pos buffer read position
930+
* @param length length of slot for this primitive.
931+
* @param dictSize dictionary size
932+
* @param isObject is this an object?
933+
*/
934+
private static void validateContainerShallow(
935+
final ByteBuffer valueBuffer,
936+
final int typeInfo,
937+
final int pos,
938+
final long length,
939+
final int dictSize,
940+
final boolean isObject) {
941+
boolean largeSize;
942+
int idSize;
943+
if (isObject) {
944+
largeSize = ((typeInfo >> 4) & 0x1) != 0;
945+
idSize = ((typeInfo >> 2) & 0x3) + 1;
946+
} else {
947+
largeSize = ((typeInfo >> 2) & 0x1) != 0;
948+
idSize = 0;
949+
}
950+
int offsetSize = (typeInfo & 0x3) + 1;
951+
int sizeBytes = largeSize ? U32_SIZE : 1;
952+
Preconditions.checkArgument(1L + sizeBytes <= length, "variant container header truncated");
953+
int numElements = readUnsignedLittleEndian(valueBuffer, pos + 1, sizeBytes);
954+
long idStart = 1L + sizeBytes;
955+
long idBytes = isObject ? (long) numElements * idSize : 0L;
956+
long offsetStart = idStart + idBytes;
957+
long offsetBytes = (long) (numElements + 1) * offsetSize;
958+
long dataStart = offsetStart + offsetBytes;
959+
Preconditions.checkArgument(
960+
dataStart <= length, "variant container offset table extends past buffer: numElements=%s", numElements);
961+
long dataLen = length - dataStart;
962+
if (isObject) {
963+
for (int i = 0; i < numElements; i++) {
964+
int id = readUnsignedLittleEndian(valueBuffer, pos + (int) idStart + i * idSize, idSize);
965+
Preconditions.checkArgument(
966+
id < dictSize, "variant object key id %s out of range (dictSize=%s)", id, dictSize);
967+
}
968+
}
969+
// Each child offset must lie within the data region. Children may overlap or leave gaps;
970+
// the trailing terminator offset is range-checked for the same reason.
971+
for (int i = 0; i <= numElements; i++) {
972+
// O(elements)
973+
int off = readUnsignedLittleEndian(valueBuffer, pos + (int) offsetStart + i * offsetSize, offsetSize);
974+
Preconditions.checkArgument(
975+
off <= dataLen, "variant child offset out of range: %s (data length %s)", off, dataLen);
976+
}
977+
}
978+
979+
/**
980+
* Validate the primitive type. The buffer must have enough capacity
981+
* for the given type (and of course, the type must be recognized).
982+
*
983+
* @param valueBuffer buffer with the variant data
984+
* @param typeInfo type information from the metadata
985+
* @param pos buffer read position
986+
* @param length length of slot for this primitive.
987+
*/
988+
private static void validatePrimitiveShallow(
989+
final ByteBuffer valueBuffer, final int typeInfo, final int pos, final long length) {
990+
long size;
991+
switch (typeInfo) {
992+
case NULL:
993+
case TRUE:
994+
case FALSE:
995+
size = 1;
996+
break;
997+
case INT8:
998+
size = 2;
999+
break;
1000+
case INT16:
1001+
size = 3;
1002+
break;
1003+
case INT32:
1004+
case DATE:
1005+
case FLOAT:
1006+
size = 5;
1007+
break;
1008+
case INT64:
1009+
case DOUBLE:
1010+
case TIMESTAMP_TZ:
1011+
case TIMESTAMP_NTZ:
1012+
case TIME:
1013+
case TIMESTAMP_NANOS_TZ:
1014+
case TIMESTAMP_NANOS_NTZ:
1015+
size = 9;
1016+
break;
1017+
case DECIMAL4:
1018+
size = 6;
1019+
break;
1020+
case DECIMAL8:
1021+
size = 10;
1022+
break;
1023+
case DECIMAL16:
1024+
size = 18;
1025+
break;
1026+
case BINARY:
1027+
case LONG_STR: {
1028+
Preconditions.checkArgument(1L + U32_SIZE <= length, "variant string/binary length field truncated");
1029+
size = 1L + U32_SIZE + readUnsigned(valueBuffer, pos + 1, U32_SIZE);
1030+
break;
1031+
}
1032+
case UUID:
1033+
size = 1L + UUID_SIZE;
1034+
break;
1035+
default:
1036+
throw new IllegalArgumentException(String.format("Unknown primitive type in variant: %d", typeInfo));
1037+
}
1038+
Preconditions.checkArgument(size <= length, "variant value extends past buffer");
1039+
}
1040+
8831041
/**
8841042
* Computes the actual size (in bytes) of the Variant value.
8851043
* @param value The Variant value binary

0 commit comments

Comments
 (0)