Skip to content

Commit c44eca6

Browse files
authored
GH-50779: [C++][Parquet] Replace remaining RapidJSON usage with simdjson (#50781)
### Rationale for this change This PR continues the simdjson migration by replacing the remaining RapidJSON usage under `cpp/src/parquet` with simdjson and `JsonWriter`. It also updates the Meson build to support simdjson and removes unnecessary RapidJSON dependencies from the Parquet build configuration. ### What changes are included in this PR? - Replace the remaining RapidJSON parsing logic in `reader_test.cc` with simdjson. - Replace RapidJSON parsing and serialization in `geospatial/util_json_internal.cc` with simdjson and `JsonWriter`. - Replace RapidJSON string escaping in `types.cc` with `JsonWriter`. - Add Meson support for simdjson, including a fallback CMake subproject. - Link simdjson in the relevant Meson targets. - Remove unnecessary RapidJSON dependencies from the Parquet Meson and CMake build configuration. * GitHub Issue: #50779 Authored-by: Aaditya Srinivasan <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
1 parent 1b004f2 commit c44eca6

14 files changed

Lines changed: 401 additions & 135 deletions

‎cpp/src/arrow/json/from_string.cc‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ class ConcreteConverter : public JSONConverter {
106106
int32_t num_elements = 0;
107107
for (auto element : json_array) {
108108
ARROW_ASSIGN_OR_RAISE(auto value,
109-
internal::GetSimdjsonResult<sj::value>(
109+
internal::ResolveSimdjsonResult<sj::value>(
110110
element, "Could not iterate elements of JSON array: "));
111111
RETURN_NOT_OK(self->AppendValue(value));
112112
num_elements++;
@@ -287,7 +287,7 @@ Status ProcessJsonArrayElements(
287287
}
288288

289289
ARROW_ASSIGN_OR_RAISE(sj::value element,
290-
internal::GetSimdjsonResult<sj::value>(
290+
internal::ResolveSimdjsonResult<sj::value>(
291291
*it, "Could not iterate elements of JSON array: "));
292292
RETURN_NOT_OK(handler(element));
293293
++it;
@@ -652,7 +652,7 @@ class MapConverter final : public ConcreteConverter<MapConverter> {
652652
for (auto json_pair_result : array) {
653653
ARROW_ASSIGN_OR_RAISE(
654654
auto json_pair,
655-
internal::GetSimdjsonResult<sj::value>(
655+
internal::ResolveSimdjsonResult<sj::value>(
656656
json_pair_result, "Could not iterate elements of JSON array: "));
657657
ARROW_ASSIGN_OR_RAISE(auto json_pair_array,
658658
internal::GetJsonAs<sj::array>(json_pair));
@@ -763,7 +763,7 @@ class StructConverter final : public ConcreteConverter<StructConverter> {
763763
size_t i = 0;
764764
for (auto child : array) {
765765
ARROW_ASSIGN_OR_RAISE(auto child_value,
766-
internal::GetSimdjsonResult<sj::value>(
766+
internal::ResolveSimdjsonResult<sj::value>(
767767
child, "Could not iterate elements of JSON array: "));
768768
RETURN_NOT_OK(child_converters_[i]->AppendValue(child_value));
769769
++i;
@@ -779,7 +779,7 @@ class StructConverter final : public ConcreteConverter<StructConverter> {
779779
std::vector<bool> field_seen(num_fields, false);
780780
for (auto field_result : object) {
781781
ARROW_ASSIGN_OR_RAISE(auto field,
782-
internal::GetSimdjsonResult<sj::field>(
782+
internal::ResolveSimdjsonResult<sj::field>(
783783
field_result, "Error getting field of object: "));
784784
std::string_view key;
785785
if (field.unescaped_key(/*allow_replacement=*/false).get(key) !=

‎cpp/src/arrow/json/json_writer_internal.cc‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -108,14 +108,14 @@ Status JsonWriter::WriteValue(sj::value value) {
108108

109109
for (auto field : object) {
110110
ARROW_ASSIGN_OR_RAISE(
111-
auto key, internal::GetSimdjsonResult(field.unescaped_key(),
112-
"Failed to get object key: "));
111+
auto key, internal::ResolveSimdjsonResult(field.unescaped_key(),
112+
"Failed to get object key"));
113113

114114
Key(key);
115115

116-
ARROW_ASSIGN_OR_RAISE(
117-
auto field_value,
118-
internal::GetSimdjsonResult(field.value(), "Failed to get object value: "));
116+
ARROW_ASSIGN_OR_RAISE(auto field_value,
117+
internal::ResolveSimdjsonResult(
118+
field.value(), "Failed to get object value"));
119119

120120
RETURN_NOT_OK(WriteValue(field_value));
121121
}
@@ -130,7 +130,7 @@ Status JsonWriter::WriteValue(sj::value value) {
130130
for (auto element : array) {
131131
ARROW_ASSIGN_OR_RAISE(
132132
auto element_value,
133-
internal::GetSimdjsonResult(element, "Failed to iterate JSON array: "));
133+
internal::ResolveSimdjsonResult(element, "Failed to iterate JSON array"));
134134

135135
RETURN_NOT_OK(WriteValue(element_value));
136136
}
@@ -170,9 +170,9 @@ Status JsonWriter::WriteValue(sj::value value) {
170170
},
171171

172172
[&](sj::value value) -> Status {
173-
ARROW_ASSIGN_OR_RAISE(auto raw_json,
174-
internal::GetSimdjsonResult(simdjson::to_json_string(value),
175-
"Failed to get raw JSON: "));
173+
ARROW_ASSIGN_OR_RAISE(auto raw_json, internal::ResolveSimdjsonResult(
174+
simdjson::to_json_string(value),
175+
"Failed to get raw JSON"));
176176
RawValue(raw_json);
177177
return Status::OK();
178178
});

‎cpp/src/arrow/json/meson.build‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ exc = executable(
2626
'parser_test.cc',
2727
'reader_test.cc',
2828
],
29-
dependencies: [arrow_test_dep, rapidjson_dep],
29+
dependencies: [arrow_test_dep, rapidjson_dep, simdjson_dep],
3030
)
3131
test('arrow-json-test', exc)
3232

‎cpp/src/arrow/meson.build‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,22 @@ else
318318
rapidjson_dep = disabler()
319319
endif
320320

321+
if needs_json or needs_integration
322+
simdjson_dep = dependency('simdjson', allow_fallback: false, required: false)
323+
324+
if not simdjson_dep.found()
325+
cmake = import('cmake')
326+
simdjson_opts = cmake.subproject_options()
327+
328+
simdjson_opts.add_cmake_defines({'SIMDJSON_EXCEPTIONS': 'OFF'})
329+
330+
simdjson_proj = cmake.subproject('simdjson', options: simdjson_opts)
331+
simdjson_dep = simdjson_proj.dependency('simdjson')
332+
endif
333+
else
334+
simdjson_dep = disabler()
335+
endif
336+
321337
azure_dep = disabler()
322338
gcs_dep = disabler()
323339
s3_dep = disabler()
@@ -520,7 +536,7 @@ if needs_json
520536
'json/parser.cc',
521537
'json/reader.cc',
522538
],
523-
'dependencies': [rapidjson_dep],
539+
'dependencies': [rapidjson_dep, simdjson_dep],
524540
},
525541
}
526542
endif

‎cpp/src/arrow/util/simdjson_internal.h‎

Lines changed: 104 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -82,10 +82,11 @@ constexpr const char* JsonTypeName() {
8282
}
8383

8484
template <typename T>
85-
Result<T> GetSimdjsonResult(simdjson::simdjson_result<T> result, std::string_view error) {
85+
Result<T> ResolveSimdjsonResult(simdjson::simdjson_result<T> result,
86+
std::string_view error) {
8687
T value;
8788
if (auto error_code = std::move(result).get(value); error_code != simdjson::SUCCESS) {
88-
return Status::Invalid(error, simdjson::error_message(error_code));
89+
return Status::Invalid(error, ": ", simdjson::error_message(error_code));
8990
}
9091
return value;
9192
}
@@ -98,63 +99,65 @@ Status VisitJsonValue(simdjson::ondemand::value value, ObjectFn&& object_fn,
9899
NullFn&& null_fn, Int64Fn&& int64_fn, Uint64Fn&& uint64_fn,
99100
DoubleFn&& double_fn, BigIntegerFn&& big_integer_fn) {
100101
ARROW_ASSIGN_OR_RAISE(
101-
auto type, GetSimdjsonResult(value.type(), "Failed to determine JSON type: "));
102+
auto type, ResolveSimdjsonResult(value.type(), "Failed to determine JSON type"));
102103

103104
switch (type) {
104105
case simdjson::ondemand::json_type::object: {
105106
ARROW_ASSIGN_OR_RAISE(
106107
auto object,
107-
GetSimdjsonResult(value.get_object(), "Failed to get JSON object: "));
108+
ResolveSimdjsonResult(value.get_object(), "Failed to get JSON object"));
108109
return object_fn(object);
109110
}
110111

111112
case simdjson::ondemand::json_type::array: {
112113
ARROW_ASSIGN_OR_RAISE(
113-
auto array, GetSimdjsonResult(value.get_array(), "Failed to get JSON array: "));
114+
auto array,
115+
ResolveSimdjsonResult(value.get_array(), "Failed to get JSON array"));
114116
return array_fn(array);
115117
}
116118

117119
case simdjson::ondemand::json_type::string: {
118120
ARROW_ASSIGN_OR_RAISE(
119121
auto string,
120-
GetSimdjsonResult(value.get_string(), "Failed to get JSON string: "));
122+
ResolveSimdjsonResult(value.get_string(), "Failed to get JSON string"));
121123
return string_fn(string);
122124
}
123125

124126
case simdjson::ondemand::json_type::boolean: {
125127
ARROW_ASSIGN_OR_RAISE(
126128
auto boolean,
127-
GetSimdjsonResult(value.get_bool(), "Failed to get JSON boolean: "));
129+
ResolveSimdjsonResult(value.get_bool(), "Failed to get JSON boolean"));
128130
return bool_fn(boolean);
129131
}
130132

131133
case simdjson::ondemand::json_type::null:
132134
return null_fn();
133135

134136
case simdjson::ondemand::json_type::number: {
135-
ARROW_ASSIGN_OR_RAISE(auto number_type,
136-
GetSimdjsonResult(value.get_number_type(),
137-
"Failed to determine JSON number type: "));
137+
ARROW_ASSIGN_OR_RAISE(
138+
auto number_type,
139+
ResolveSimdjsonResult(value.get_number_type(),
140+
"Failed to determine JSON number type"));
138141

139142
switch (number_type) {
140143
case simdjson::ondemand::number_type::signed_integer: {
141144
ARROW_ASSIGN_OR_RAISE(
142145
auto number,
143-
GetSimdjsonResult(value.get_int64(), "Failed to get signed integer: "));
146+
ResolveSimdjsonResult(value.get_int64(), "Failed to get signed integer"));
144147
return int64_fn(number);
145148
}
146149

147150
case simdjson::ondemand::number_type::unsigned_integer: {
148-
ARROW_ASSIGN_OR_RAISE(
149-
auto number,
150-
GetSimdjsonResult(value.get_uint64(), "Failed to get unsigned integer: "));
151+
ARROW_ASSIGN_OR_RAISE(auto number,
152+
ResolveSimdjsonResult(value.get_uint64(),
153+
"Failed to get unsigned integer"));
151154
return uint64_fn(number);
152155
}
153156

154157
case simdjson::ondemand::number_type::floating_point_number: {
155158
ARROW_ASSIGN_OR_RAISE(
156-
auto number, GetSimdjsonResult(value.get_double(),
157-
"Failed to get floating-point number: "));
159+
auto number, ResolveSimdjsonResult(value.get_double(),
160+
"Failed to get floating-point number"));
158161
return double_fn(number);
159162
}
160163

@@ -240,5 +243,90 @@ Result<SimdjsonValueType> GetJsonAs(simdjson::ondemand::value& value) {
240243
return typed_value;
241244
}
242245

246+
template <typename T>
247+
Result<T> GetJsonField(simdjson::ondemand::object& object, std::string_view key) {
248+
for (auto field_result : object) {
249+
ARROW_ASSIGN_OR_RAISE(
250+
auto field, ResolveSimdjsonResult(field_result, "Failed to iterate JSON object"));
251+
252+
ARROW_ASSIGN_OR_RAISE(
253+
auto field_key,
254+
ResolveSimdjsonResult(field.unescaped_key(), "Failed to get JSON object key"));
255+
256+
if (field_key == key) {
257+
auto value = field.value();
258+
259+
if constexpr (std::is_same_v<T, simdjson::ondemand::value>) {
260+
return value;
261+
} else {
262+
return GetJsonAs<T>(value);
263+
}
264+
}
265+
}
266+
267+
return Status::KeyError("Missing JSON field: ", key);
268+
}
269+
270+
inline Result<std::string> MinifyJson(std::string_view json) {
271+
std::string minified(json.size(), '\0');
272+
size_t minified_len = 0;
273+
274+
if (auto error =
275+
simdjson::minify(json.data(), json.size(), minified.data(), minified_len);
276+
error != simdjson::SUCCESS) {
277+
return Status::Invalid("Failed to minify JSON: ", simdjson::error_message(error));
278+
}
279+
280+
minified.resize(minified_len);
281+
return minified;
282+
}
283+
284+
inline Status ValidateJsonObject(simdjson::ondemand::object object);
285+
286+
inline Status ValidateJsonArray(simdjson::ondemand::array array);
287+
288+
inline Status ConsumeJsonValue(simdjson::ondemand::value value) {
289+
return VisitJsonValue(
290+
value, ValidateJsonObject, ValidateJsonArray,
291+
[](std::string_view) { return Status::OK(); }, [](bool) { return Status::OK(); },
292+
[]() { return Status::OK(); }, [](int64_t) { return Status::OK(); },
293+
[](uint64_t) { return Status::OK(); }, [](double) { return Status::OK(); },
294+
[](simdjson::ondemand::value) { return Status::OK(); });
295+
}
296+
297+
inline Status ValidateJsonObject(simdjson::ondemand::object object) {
298+
for (auto field_result : object) {
299+
ARROW_ASSIGN_OR_RAISE(
300+
auto field, ResolveSimdjsonResult(field_result, "Failed to iterate JSON object"));
301+
302+
RETURN_NOT_OK(ConsumeJsonValue(field.value()));
303+
}
304+
305+
return Status::OK();
306+
}
307+
308+
inline Status ValidateJsonArray(simdjson::ondemand::array array) {
309+
for (auto element_result : array) {
310+
ARROW_ASSIGN_OR_RAISE(
311+
auto value,
312+
ResolveSimdjsonResult(element_result, "Failed to iterate JSON array"));
313+
314+
RETURN_NOT_OK(ConsumeJsonValue(value));
315+
}
316+
317+
return Status::OK();
318+
}
319+
320+
inline Status ValidateJsonDocument(simdjson::ondemand::parser& parser,
321+
simdjson::padded_string& json) {
322+
ARROW_ASSIGN_OR_RAISE(
323+
auto document, ResolveSimdjsonResult(parser.iterate(json), "Failed to parse JSON"));
324+
325+
ARROW_ASSIGN_OR_RAISE(auto value, ResolveSimdjsonResult(document.get_value(),
326+
"Failed to get JSON value"));
327+
328+
return ConsumeJsonValue(value);
329+
}
330+
243331
} // namespace internal
244332
} // namespace arrow

‎cpp/src/parquet/CMakeLists.txt‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -263,9 +263,9 @@ endif()
263263

264264
list(APPEND PARQUET_SHARED_LINK_LIBS arrow_shared)
265265

266-
# Add RapidJSON & simdjson libraries
267-
list(APPEND PARQUET_SHARED_PRIVATE_LINK_LIBS RapidJSON simdjson::simdjson)
268-
list(APPEND PARQUET_STATIC_LINK_LIBS RapidJSON simdjson::simdjson)
266+
# Add simdjson libraries
267+
list(APPEND PARQUET_SHARED_PRIVATE_LINK_LIBS simdjson::simdjson)
268+
list(APPEND PARQUET_STATIC_LINK_LIBS simdjson::simdjson)
269269

270270
# These are libraries that we will link privately with parquet_shared (as they
271271
# do not need to be linked transitively by other linkers)
@@ -323,7 +323,7 @@ if(ARROW_TESTING)
323323
# "link" our dependencies so that include paths are configured
324324
# correctly
325325
target_link_libraries(parquet_testing PUBLIC ${ARROW_GTEST_GMOCK})
326-
list(APPEND PARQUET_TEST_LINK_LIBS parquet_testing RapidJSON)
326+
list(APPEND PARQUET_TEST_LINK_LIBS parquet_testing simdjson::simdjson)
327327
endif()
328328

329329
if(NOT ARROW_BUILD_SHARED)
@@ -379,6 +379,7 @@ add_parquet_test(internals-test
379379
bloom_filter_test.cc
380380
geospatial/statistics_test.cc
381381
geospatial/util_internal_test.cc
382+
geospatial/util_json_internal_test.cc
382383
metadata_test.cc
383384
page_index_test.cc
384385
properties_test.cc

0 commit comments

Comments
 (0)