Skip to content

Commit b9c11af

Browse files
gaurav7261gaurav7261
authored
GH-3414: Add parseJson to VariantBuilder for JSON-to-Variant conversion (#3415)
 Move parseJson and helpers from VariantBuilder into dedicated VariantJsonParser class to isolate Jackson dependency. Add StreamReadConstraints to JsonFactory for safety against malicious input. Revert NOTICE (cross-ASF credit not needed). Add edge-case tests: empty input, non-JSON, incomplete array, large object. Ported from Apache Spark's VariantBuilder.parseJson.
1 parent 28593d5 commit b9c11af

4 files changed

Lines changed: 561 additions & 0 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,5 @@ target/
2020
mvn_install.log
2121
.vscode/*
2222
.DS_Store
23+
.memsearch/
2324

‎parquet-variant/pom.xml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,17 @@
4646
<artifactId>parquet-column</artifactId>
4747
<version>${project.version}</version>
4848
</dependency>
49+
<dependency>
50+
<groupId>${jackson.groupId}</groupId>
51+
<artifactId>jackson-core</artifactId>
52+
<version>${jackson.version}</version>
53+
</dependency>
54+
<dependency>
55+
<groupId>org.apache.parquet</groupId>
56+
<artifactId>parquet-jackson</artifactId>
57+
<version>${project.version}</version>
58+
<scope>runtime</scope>
59+
</dependency>
4960
<dependency>
5061
<groupId>com.google.guava</groupId>
5162
<artifactId>guava</artifactId>

‎parquet-variant/src/main/java/org/apache/parquet/variant/VariantJsonParser.java‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.parquet.variant;
18+
19+
import com.fasterxml.jackson.core.JsonFactory;
20+
import com.fasterxml.jackson.core.JsonParseException;
21+
import com.fasterxml.jackson.core.JsonParser;
22+
import com.fasterxml.jackson.core.JsonToken;
23+
import com.fasterxml.jackson.core.StreamReadConstraints;
24+
import com.fasterxml.jackson.core.exc.InputCoercionException;
25+
import java.io.IOException;
26+
import java.math.BigDecimal;
27+
28+
/**
29+
* Parses JSON into {@link Variant} values using Jackson streaming.
30+
*
31+
* <p>This class isolates the Jackson dependency from {@link VariantBuilder},
32+
* so that core variant construction does not require Jackson on the classpath.
33+
*
34+
* <p>Ported from Apache Spark's {@code VariantBuilder.parseJson}.
35+
*/
36+
public final class VariantJsonParser {
37+
38+
private static final JsonFactory JSON_FACTORY = JsonFactory.builder()
39+
.streamReadConstraints(StreamReadConstraints.builder()
40+
.maxNestingDepth(500)
41+
.maxStringLength(10_000_000)
42+
.maxDocumentLength(50_000_000L)
43+
.build())
44+
.build();
45+
46+
private VariantJsonParser() {}
47+
48+
/**
49+
* Parses a JSON string and returns the corresponding {@link Variant}.
50+
*
51+
* <p>Uses Jackson streaming parser for single-pass conversion
52+
* with no intermediate tree. Number handling preserves precision:
53+
* integers use the smallest fitting type, floating-point numbers
54+
* prefer decimal encoding (no scientific notation) and fall back
55+
* to double.
56+
*
57+
* @param json the JSON string to parse
58+
* @return the parsed Variant
59+
* @throws IOException if the JSON is malformed or an I/O error occurs
60+
*/
61+
public static Variant parseJson(String json) throws IOException {
62+
try (JsonParser parser = JSON_FACTORY.createParser(json)) {
63+
parser.nextToken();
64+
return parseJson(parser);
65+
}
66+
}
67+
68+
/**
69+
* Parses a JSON value from an already-positioned {@link JsonParser}
70+
* and returns the corresponding {@link Variant}. The parser must
71+
* have its current token set (i.e., {@code parser.nextToken()}
72+
* or equivalent must have been called).
73+
*
74+
* @param parser a positioned Jackson JsonParser
75+
* @return the parsed Variant
76+
* @throws IOException if the JSON is malformed or an I/O error occurs
77+
*/
78+
public static Variant parseJson(JsonParser parser) throws IOException {
79+
VariantBuilder builder = new VariantBuilder();
80+
buildJson(builder, parser);
81+
return builder.build();
82+
}
83+
84+
/**
85+
* Recursively builds a Variant value from the current position of a
86+
* Jackson streaming parser. Handles objects, arrays, strings, numbers
87+
* (int/long/decimal/double), booleans, and null.
88+
*/
89+
private static void buildJson(VariantBuilder builder, JsonParser parser) throws IOException {
90+
JsonToken token = parser.currentToken();
91+
if (token == null) {
92+
throw new JsonParseException(parser, "Unexpected null token");
93+
}
94+
switch (token) {
95+
case START_OBJECT:
96+
buildJsonObject(builder, parser);
97+
break;
98+
case START_ARRAY:
99+
buildJsonArray(builder, parser);
100+
break;
101+
case VALUE_STRING:
102+
builder.appendString(parser.getText());
103+
break;
104+
case VALUE_NUMBER_INT:
105+
buildJsonInteger(builder, parser);
106+
break;
107+
case VALUE_NUMBER_FLOAT:
108+
buildJsonFloat(builder, parser);
109+
break;
110+
case VALUE_TRUE:
111+
builder.appendBoolean(true);
112+
break;
113+
case VALUE_FALSE:
114+
builder.appendBoolean(false);
115+
break;
116+
case VALUE_NULL:
117+
builder.appendNull();
118+
break;
119+
default:
120+
throw new JsonParseException(parser, "Unexpected token " + token);
121+
}
122+
}
123+
124+
/**
125+
* Builds a Variant object from the current JSON object token.
126+
*
127+
* <p>Iterates over each key-value pair in the JSON object. For each value,
128+
* this method co-recurses into {@link #buildJson(VariantBuilder, JsonParser)}
129+
* to handle arbitrarily nested structures (objects within objects, arrays
130+
* within objects, etc.).
131+
*/
132+
private static void buildJsonObject(VariantBuilder builder, JsonParser parser) throws IOException {
133+
VariantObjectBuilder obj = builder.startObject();
134+
while (parser.nextToken() != JsonToken.END_OBJECT) {
135+
obj.appendKey(parser.currentName());
136+
parser.nextToken();
137+
buildJson(obj, parser);
138+
}
139+
builder.endObject();
140+
}
141+
142+
/**
143+
* Builds a Variant array from the current JSON array token.
144+
*
145+
* <p>Iterates over each element in the JSON array. For each element,
146+
* this method co-recurses into {@link #buildJson(VariantBuilder, JsonParser)}
147+
* to handle arbitrarily nested structures (objects within arrays, arrays
148+
* within arrays, etc.).
149+
*/
150+
private static void buildJsonArray(VariantBuilder builder, JsonParser parser) throws IOException {
151+
VariantArrayBuilder arr = builder.startArray();
152+
while (parser.nextToken() != JsonToken.END_ARRAY) {
153+
buildJson(arr, parser);
154+
}
155+
builder.endArray();
156+
}
157+
158+
private static void buildJsonInteger(VariantBuilder builder, JsonParser parser) throws IOException {
159+
try {
160+
appendSmallestLong(builder, parser.getLongValue());
161+
} catch (InputCoercionException ignored) {
162+
buildJsonFloat(builder, parser);
163+
}
164+
}
165+
166+
private static void buildJsonFloat(VariantBuilder builder, JsonParser parser) throws IOException {
167+
if (!tryAppendDecimal(builder, parser.getText())) {
168+
builder.appendDouble(parser.getDoubleValue());
169+
}
170+
}
171+
172+
/**
173+
* Appends a long value using the smallest integer type that fits.
174+
*/
175+
private static void appendSmallestLong(VariantBuilder builder, long l) {
176+
if (l == (byte) l) {
177+
builder.appendByte((byte) l);
178+
} else if (l == (short) l) {
179+
builder.appendShort((short) l);
180+
} else if (l == (int) l) {
181+
builder.appendInt((int) l);
182+
} else {
183+
builder.appendLong(l);
184+
}
185+
}
186+
187+
/**
188+
* Tries to parse a number string as a decimal. Only accepts plain
189+
* decimal format (digits, minus, dot -- no scientific notation).
190+
* Returns true if the number was successfully appended as a decimal.
191+
*/
192+
private static boolean tryAppendDecimal(VariantBuilder builder, String input) {
193+
for (int i = 0; i < input.length(); i++) {
194+
char ch = input.charAt(i);
195+
if (ch != '-' && ch != '.' && !(ch >= '0' && ch <= '9')) {
196+
return false;
197+
}
198+
}
199+
BigDecimal d = new BigDecimal(input);
200+
if (d.scale() <= VariantUtil.MAX_DECIMAL16_PRECISION && d.precision() <= VariantUtil.MAX_DECIMAL16_PRECISION) {
201+
builder.appendDecimal(d);
202+
return true;
203+
}
204+
return false;
205+
}
206+
}

0 commit comments

Comments
 (0)