Skip to content

Commit ba6c5fe

Browse files
authored
GH-51638: [CI][C++][Parquet] Update expected JSON output in parquet-reader-test for simdjson 5.0 (#51661)
### Rationale for this change See #51638 - `fractured_json` formatting changes with the new simdjson version 5.0 (whitespace, line breaks, key order in tables), cause `parquet-reader-test` to fail on four of its tests `TestJSONWithLocalFile.JSONOutput...` ### What changes are included in this PR? Check `simdjson::SIMDJSON_VERSION_MAJOR`, and for `>= 5` use the new expected formatting (JSON content hasn't changed). Keep existing expected output formatting for simdjson 4.x ### Are these changes tested? Yes, CI runs pass. ### Are there any user-facing changes? No, only test changes. ### Was AI used for this PR? **PR code and description written by:** - [x] Human - [x] AI **Reviewed before submission by:** - [x] Human - [ ] AI - [ ] Not reviewed * GitHub Issue: #51638 Authored-by: Tadeja Kadunc <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
1 parent ccfcdd2 commit ba6c5fe

1 file changed

Lines changed: 196 additions & 32 deletions

File tree

‎cpp/src/parquet/reader_test.cc‎

Lines changed: 196 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1124,6 +1124,11 @@ class TestJSONWithLocalFile : public ::testing::Test {
11241124
};
11251125

11261126
TEST_F(TestJSONWithLocalFile, JSONOutputWithStatistics) {
1127+
// simdjson 5.0 changed the format of fractured_json
1128+
if constexpr (simdjson::SIMDJSON_VERSION_MAJOR < 5) {
1129+
GTEST_SKIP() << "Test requires simdjson >= 5";
1130+
}
1131+
11271132
std::string json_output = R"###({
11281133
"FileName": "nested_lists.snappy.parquet",
11291134
"Version": "1.0",
@@ -1138,15 +1143,9 @@ TEST_F(TestJSONWithLocalFile, JSONOutputWithStatistics) {
11381143
"Name": "a.list.element.list.element.list.element",
11391144
"PhysicalType": "BYTE_ARRAY",
11401145
"ConvertedType": "UTF8",
1141-
"LogicalType": { "Type": "String" }
1146+
"LogicalType": {"Type": "String"}
11421147
},
1143-
{
1144-
"Id": "1",
1145-
"Name": "b",
1146-
"PhysicalType": "INT32",
1147-
"ConvertedType": "NONE",
1148-
"LogicalType": { "Type": "None" }
1149-
}
1148+
{ "Id": "1", "Name": "b", "PhysicalType": "INT32", "ConvertedType": "NONE", "LogicalType": {"Type": "None"} }
11501149
],
11511150
"RowGroups": [
11521151
{
@@ -1191,6 +1190,11 @@ TEST_F(TestJSONWithLocalFile, JSONOutputWithStatistics) {
11911190
}
11921191

11931192
TEST_F(TestJSONWithLocalFile, JSONOutput) {
1193+
// simdjson 5.0 changed the format of fractured_json
1194+
if constexpr (simdjson::SIMDJSON_VERSION_MAJOR < 5) {
1195+
GTEST_SKIP() << "Test requires simdjson >= 5";
1196+
}
1197+
11941198
std::string json_output = R"###({
11951199
"FileName": "alltypes_plain.parquet",
11961200
"Version": "1.0",
@@ -1200,17 +1204,77 @@ TEST_F(TestJSONWithLocalFile, JSONOutput) {
12001204
"NumberOfRealColumns": "11",
12011205
"NumberOfColumns": "11",
12021206
"Columns": [
1203-
{ "ConvertedType": "NONE", "Id": "0" , "LogicalType": { "Type": "None" }, "Name": "id" , "PhysicalType": "INT32" },
1204-
{ "ConvertedType": "NONE", "Id": "1" , "LogicalType": { "Type": "None" }, "Name": "bool_col" , "PhysicalType": "BOOLEAN" },
1205-
{ "ConvertedType": "NONE", "Id": "2" , "LogicalType": { "Type": "None" }, "Name": "tinyint_col" , "PhysicalType": "INT32" },
1206-
{ "ConvertedType": "NONE", "Id": "3" , "LogicalType": { "Type": "None" }, "Name": "smallint_col" , "PhysicalType": "INT32" },
1207-
{ "ConvertedType": "NONE", "Id": "4" , "LogicalType": { "Type": "None" }, "Name": "int_col" , "PhysicalType": "INT32" },
1208-
{ "ConvertedType": "NONE", "Id": "5" , "LogicalType": { "Type": "None" }, "Name": "bigint_col" , "PhysicalType": "INT64" },
1209-
{ "ConvertedType": "NONE", "Id": "6" , "LogicalType": { "Type": "None" }, "Name": "float_col" , "PhysicalType": "FLOAT" },
1210-
{ "ConvertedType": "NONE", "Id": "7" , "LogicalType": { "Type": "None" }, "Name": "double_col" , "PhysicalType": "DOUBLE" },
1211-
{ "ConvertedType": "NONE", "Id": "8" , "LogicalType": { "Type": "None" }, "Name": "date_string_col", "PhysicalType": "BYTE_ARRAY" },
1212-
{ "ConvertedType": "NONE", "Id": "9" , "LogicalType": { "Type": "None" }, "Name": "string_col" , "PhysicalType": "BYTE_ARRAY" },
1213-
{ "ConvertedType": "NONE", "Id": "10", "LogicalType": { "Type": "None" }, "Name": "timestamp_col" , "PhysicalType": "INT96" }
1207+
{ "Id": "0", "Name": "id", "PhysicalType": "INT32", "ConvertedType": "NONE", "LogicalType": {"Type": "None"} },
1208+
{
1209+
"Id": "1",
1210+
"Name": "bool_col",
1211+
"PhysicalType": "BOOLEAN",
1212+
"ConvertedType": "NONE",
1213+
"LogicalType": {"Type": "None"}
1214+
},
1215+
{
1216+
"Id": "2",
1217+
"Name": "tinyint_col",
1218+
"PhysicalType": "INT32",
1219+
"ConvertedType": "NONE",
1220+
"LogicalType": {"Type": "None"}
1221+
},
1222+
{
1223+
"Id": "3",
1224+
"Name": "smallint_col",
1225+
"PhysicalType": "INT32",
1226+
"ConvertedType": "NONE",
1227+
"LogicalType": {"Type": "None"}
1228+
},
1229+
{
1230+
"Id": "4",
1231+
"Name": "int_col",
1232+
"PhysicalType": "INT32",
1233+
"ConvertedType": "NONE",
1234+
"LogicalType": {"Type": "None"}
1235+
},
1236+
{
1237+
"Id": "5",
1238+
"Name": "bigint_col",
1239+
"PhysicalType": "INT64",
1240+
"ConvertedType": "NONE",
1241+
"LogicalType": {"Type": "None"}
1242+
},
1243+
{
1244+
"Id": "6",
1245+
"Name": "float_col",
1246+
"PhysicalType": "FLOAT",
1247+
"ConvertedType": "NONE",
1248+
"LogicalType": {"Type": "None"}
1249+
},
1250+
{
1251+
"Id": "7",
1252+
"Name": "double_col",
1253+
"PhysicalType": "DOUBLE",
1254+
"ConvertedType": "NONE",
1255+
"LogicalType": {"Type": "None"}
1256+
},
1257+
{
1258+
"Id": "8",
1259+
"Name": "date_string_col",
1260+
"PhysicalType": "BYTE_ARRAY",
1261+
"ConvertedType": "NONE",
1262+
"LogicalType": {"Type": "None"}
1263+
},
1264+
{
1265+
"Id": "9",
1266+
"Name": "string_col",
1267+
"PhysicalType": "BYTE_ARRAY",
1268+
"ConvertedType": "NONE",
1269+
"LogicalType": {"Type": "None"}
1270+
},
1271+
{
1272+
"Id": "10",
1273+
"Name": "timestamp_col",
1274+
"PhysicalType": "INT96",
1275+
"ConvertedType": "NONE",
1276+
"LogicalType": {"Type": "None"}
1277+
}
12141278
],
12151279
"RowGroups": [
12161280
{
@@ -1219,17 +1283,105 @@ TEST_F(TestJSONWithLocalFile, JSONOutput) {
12191283
"TotalCompressedBytes": "0",
12201284
"Rows": "8",
12211285
"ColumnChunks": [
1222-
{ "CompressedSize": "73" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "0" , "StatsSet": "False", "UncompressedSize": "73" , "Values": "8" },
1223-
{ "CompressedSize": "24" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "1" , "StatsSet": "False", "UncompressedSize": "24" , "Values": "8" },
1224-
{ "CompressedSize": "47" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "2" , "StatsSet": "False", "UncompressedSize": "47" , "Values": "8" },
1225-
{ "CompressedSize": "47" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "3" , "StatsSet": "False", "UncompressedSize": "47" , "Values": "8" },
1226-
{ "CompressedSize": "47" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "4" , "StatsSet": "False", "UncompressedSize": "47" , "Values": "8" },
1227-
{ "CompressedSize": "55" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "5" , "StatsSet": "False", "UncompressedSize": "55" , "Values": "8" },
1228-
{ "CompressedSize": "47" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "6" , "StatsSet": "False", "UncompressedSize": "47" , "Values": "8" },
1229-
{ "CompressedSize": "55" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "7" , "StatsSet": "False", "UncompressedSize": "55" , "Values": "8" },
1230-
{ "CompressedSize": "88" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "8" , "StatsSet": "False", "UncompressedSize": "88" , "Values": "8" },
1231-
{ "CompressedSize": "49" , "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "9" , "StatsSet": "False", "UncompressedSize": "49" , "Values": "8" },
1232-
{ "CompressedSize": "139", "Compression": "UNCOMPRESSED", "Encodings": "RLE PLAIN_DICTIONARY PLAIN ", "Id": "10", "StatsSet": "False", "UncompressedSize": "139", "Values": "8" }
1286+
{
1287+
"Id": "0",
1288+
"Values": "8",
1289+
"StatsSet": "False",
1290+
"Compression": "UNCOMPRESSED",
1291+
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
1292+
"UncompressedSize": "73",
1293+
"CompressedSize": "73"
1294+
},
1295+
{
1296+
"Id": "1",
1297+
"Values": "8",
1298+
"StatsSet": "False",
1299+
"Compression": "UNCOMPRESSED",
1300+
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
1301+
"UncompressedSize": "24",
1302+
"CompressedSize": "24"
1303+
},
1304+
{
1305+
"Id": "2",
1306+
"Values": "8",
1307+
"StatsSet": "False",
1308+
"Compression": "UNCOMPRESSED",
1309+
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
1310+
"UncompressedSize": "47",
1311+
"CompressedSize": "47"
1312+
},
1313+
{
1314+
"Id": "3",
1315+
"Values": "8",
1316+
"StatsSet": "False",
1317+
"Compression": "UNCOMPRESSED",
1318+
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
1319+
"UncompressedSize": "47",
1320+
"CompressedSize": "47"
1321+
},
1322+
{
1323+
"Id": "4",
1324+
"Values": "8",
1325+
"StatsSet": "False",
1326+
"Compression": "UNCOMPRESSED",
1327+
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
1328+
"UncompressedSize": "47",
1329+
"CompressedSize": "47"
1330+
},
1331+
{
1332+
"Id": "5",
1333+
"Values": "8",
1334+
"StatsSet": "False",
1335+
"Compression": "UNCOMPRESSED",
1336+
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
1337+
"UncompressedSize": "55",
1338+
"CompressedSize": "55"
1339+
},
1340+
{
1341+
"Id": "6",
1342+
"Values": "8",
1343+
"StatsSet": "False",
1344+
"Compression": "UNCOMPRESSED",
1345+
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
1346+
"UncompressedSize": "47",
1347+
"CompressedSize": "47"
1348+
},
1349+
{
1350+
"Id": "7",
1351+
"Values": "8",
1352+
"StatsSet": "False",
1353+
"Compression": "UNCOMPRESSED",
1354+
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
1355+
"UncompressedSize": "55",
1356+
"CompressedSize": "55"
1357+
},
1358+
{
1359+
"Id": "8",
1360+
"Values": "8",
1361+
"StatsSet": "False",
1362+
"Compression": "UNCOMPRESSED",
1363+
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
1364+
"UncompressedSize": "88",
1365+
"CompressedSize": "88"
1366+
},
1367+
{
1368+
"Id": "9",
1369+
"Values": "8",
1370+
"StatsSet": "False",
1371+
"Compression": "UNCOMPRESSED",
1372+
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
1373+
"UncompressedSize": "49",
1374+
"CompressedSize": "49"
1375+
},
1376+
{
1377+
"Id": "10",
1378+
"Values": "8",
1379+
"StatsSet": "False",
1380+
"Compression": "UNCOMPRESSED",
1381+
"Encodings": "RLE PLAIN_DICTIONARY PLAIN ",
1382+
"UncompressedSize": "139",
1383+
"CompressedSize": "139"
1384+
}
12331385
]
12341386
}
12351387
]
@@ -1243,6 +1395,11 @@ TEST_F(TestJSONWithLocalFile, JSONOutput) {
12431395
TEST_F(TestJSONWithLocalFile, JSONOutputFLBA) {
12441396
// min-max stats for FLBA contains non-utf8 output, so we don't check
12451397
// the whole json output.
1398+
// simdjson 5.0 changed the format of fractured_json
1399+
if constexpr (simdjson::SIMDJSON_VERSION_MAJOR < 5) {
1400+
GTEST_SKIP() << "Test requires simdjson >= 5";
1401+
}
1402+
12461403
std::string json_content = ReadFromLocalFile("fixed_length_byte_array.parquet");
12471404

12481405
std::string json_contains = R"###({
@@ -1259,19 +1416,26 @@ TEST_F(TestJSONWithLocalFile, JSONOutputFLBA) {
12591416
"Name": "flba_field",
12601417
"PhysicalType": "FIXED_LEN_BYTE_ARRAY(4)",
12611418
"ConvertedType": "NONE",
1262-
"LogicalType": { "Type": "None" }
1419+
"LogicalType": {"Type": "None"}
12631420
}
12641421
],)###";
12651422

12661423
EXPECT_THAT(json_content, testing::HasSubstr(json_contains));
12671424
}
12681425

12691426
TEST_F(TestJSONWithLocalFile, JSONOutputSortColumns) {
1427+
// simdjson 5.0 changed the format of fractured_json
1428+
if constexpr (simdjson::SIMDJSON_VERSION_MAJOR < 5) {
1429+
GTEST_SKIP() << "Test requires simdjson >= 5";
1430+
}
1431+
12701432
std::string json_content = ReadFromLocalFile("sort_columns.parquet");
12711433

12721434
std::string json_contains = R"###("SortColumns": [
1273-
{ "column_idx": 0, "descending": 1, "nulls_first": 1 }, { "column_idx": 1, "descending": 0, "nulls_first": 0 }
1435+
{"column_idx": 0, "descending": 1, "nulls_first": 1},
1436+
{"column_idx": 1, "descending": 0, "nulls_first": 0}
12741437
],)###";
1438+
12751439
EXPECT_THAT(json_content, testing::HasSubstr(json_contains));
12761440
}
12771441

0 commit comments

Comments
 (0)