Skip to content

Commit 41a58af

Browse files
authored
[fix](variant) Fix Variant test fixtures broken by #68447 (#69261)
Issue Number: DORIS-29778 Problem Summary: Test fixtures still assume the behavior that #68447 removed. 1. `regression-test/suites/variant_p0/test_variant_json_functions.groovy` builds its Variant string row with `parse_to_variant('abc')`, which relied on the old permissive fallback that stored text that is not a JSON document as a Variant string. Since #68447, `parse_to_variant` rejects such input, so the INSERT fails and the suite stops before any assertion runs: ``` errCode = 2, detailMessage = [INVALID_ARGUMENT]Parse json document failed at row 0, error: [INVALID_ARGUMENT]Failed to parse JSON as Variant ``` Parse the equivalent JSON string document `"abc"` instead: it produces the same Variant string value, so the remaining assertions still pass and the expected result file is unchanged. 2. BE unit tests still pass the removed `JsonToVariantOptions::check_duplicate_json_path` field, so the BE UT build fails to compile: ``` core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp:115:32: error: no matching constructor for initialization of 'JsonStringToVariantEncoder' format/transformer/vparquet_writer_test.cpp:198:32: error: no matching constructor for initialization of 'JsonStringToVariantEncoder' ``` Remove the field from the six affected initializers in `data_type_serde_arrow_test.cpp`, `data_type_variant_v2_serde_jsonb_test.cpp`, `variant_v2_execution_test.cpp` and `format/transformer/vparquet_writer_test.cpp` (the last two were added by #69194) (DORIS-29778). 3. `external_table_p0/paimon/write/test_paimon_write_variant_errors` expects `try_parse_to_variant('not-json')` to be stored as the Variant string `not-json`, while it now returns NULL, so the row 21 check fails: ``` ExpectRow: [21, false, null, not-json] RealRow : [21, true, null, null] ```
1 parent 61179b6 commit 41a58af

7 files changed

Lines changed: 16 additions & 22 deletions

File tree

‎be/test/core/data_type_serde/data_type_serde_arrow_test.cpp‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -957,9 +957,8 @@ TEST(DataTypeSerDeArrowTest, PlainArrowConverterDoesNotInferIcebergUuid) {
957957

958958
TEST(DataTypeSerDeArrowTest, IcebergVariantExtensionAndParquetSchema) {
959959
auto make_variant_column = []() {
960-
JsonStringToVariantEncoder encoder({.max_json_key_length = 1024,
961-
.throw_on_invalid_json = true,
962-
.check_duplicate_json_path = false});
960+
JsonStringToVariantEncoder encoder(
961+
{.max_json_key_length = 1024, .throw_on_invalid_json = true});
963962
for (std::string_view json :
964963
{std::string_view {R"({"name":"doris","n":1})"}, std::string_view {"null"},
965964
std::string_view {R"([1,true,"x"])"}}) {
@@ -1054,9 +1053,8 @@ TEST(DataTypeSerDeArrowTest, IcebergVariantExtensionAndParquetSchema) {
10541053

10551054
TEST(DataTypeSerDeArrowTest, NestedIcebergVariantExtensionsAndParquetSchema) {
10561055
auto make_variant_column = [](std::initializer_list<std::string_view> json_values) {
1057-
JsonStringToVariantEncoder encoder({.max_json_key_length = 1024,
1058-
.throw_on_invalid_json = true,
1059-
.check_duplicate_json_path = false});
1056+
JsonStringToVariantEncoder encoder(
1057+
{.max_json_key_length = 1024, .throw_on_invalid_json = true});
10601058
for (std::string_view json : json_values) {
10611059
encoder.add_json({json.data(), json.size()});
10621060
}

‎be/test/core/data_type_serde/data_type_variant_v2_serde_jsonb_test.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -112,9 +112,8 @@ std::string json_at(const DataTypeVariantV2SerDe& serde, const IColumn& column,
112112
}
113113

114114
ColumnVariantV2::MutablePtr encoded_json(std::initializer_list<std::string_view> rows) {
115-
JsonStringToVariantEncoder encoder({.max_json_key_length = 1024,
116-
.throw_on_invalid_json = true,
117-
.check_duplicate_json_path = false});
115+
JsonStringToVariantEncoder encoder(
116+
{.max_json_key_length = 1024, .throw_on_invalid_json = true});
118117
for (std::string_view row : rows) {
119118
encoder.add_json({row.data(), row.size()});
120119
}

‎be/test/exprs/variant_v2_execution_test.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,8 @@ TEST(VariantV2ExecutionTest, TypePropertiesDoNotAffectEquality) {
4949

5050
TEST(VariantV2ExecutionTest, CanonicalComparison) {
5151
auto parse = [](std::initializer_list<std::string_view> rows) {
52-
JsonStringToVariantEncoder encoder({.max_json_key_length = 255,
53-
.throw_on_invalid_json = true,
54-
.check_duplicate_json_path = false});
52+
JsonStringToVariantEncoder encoder(
53+
{.max_json_key_length = 255, .throw_on_invalid_json = true});
5554
for (auto json : rows) {
5655
encoder.add_json({json.data(), json.size()});
5756
}

‎be/test/format/transformer/vparquet_writer_test.cpp‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -195,9 +195,8 @@ TEST_F(VParquetWriterTest, WritesIcebergVariantAndCollectsLogicalMetrics) {
195195
options, &schema_json, *schema);
196196
ASSERT_TRUE(transformer.open().ok());
197197

198-
JsonStringToVariantEncoder encoder({.max_json_key_length = 1024,
199-
.throw_on_invalid_json = true,
200-
.check_duplicate_json_path = false});
198+
JsonStringToVariantEncoder encoder(
199+
{.max_json_key_length = 1024, .throw_on_invalid_json = true});
201200
for (std::string_view json : {std::string_view {R"({"name":"doris","n":1})"},
202201
std::string_view {"null"}, std::string_view {R"([1,2,3])"}}) {
203202
encoder.add_json({json.data(), json.size()});
@@ -282,9 +281,8 @@ TEST_F(VParquetWriterTest, WritesNestedIcebergVariant) {
282281
options, &schema_json, *schema);
283282
ASSERT_TRUE(transformer.open().ok());
284283

285-
JsonStringToVariantEncoder encoder({.max_json_key_length = 1024,
286-
.throw_on_invalid_json = true,
287-
.check_duplicate_json_path = false});
284+
JsonStringToVariantEncoder encoder(
285+
{.max_json_key_length = 1024, .throw_on_invalid_json = true});
288286
for (std::string_view json :
289287
{std::string_view {R"({"kind":"nested"})"}, std::string_view {"null"},
290288
std::string_view {"{}"}, std::string_view {"7"}}) {

‎regression-test/data/external_table_p0/paimon/write/test_paimon_write_variant_errors.out‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
-- This file is automatically generated. You should know what you did if you want to edit this
22
-- !variant_after_errors --
33
20 false true {"recovered":true}
4-
21 false \N not-json
4+
21 true \N \N
55
22 false \N {"typed":"string"}
66
23 false \N 7
77
24 false \N values-string

‎regression-test/suites/external_table_p0/paimon/write/test_paimon_write_variant_errors.groovy‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -202,8 +202,7 @@ suite("test_paimon_write_variant_errors", "p0,external,paimon,nonConcurrent") {
202202
row.collect { value -> value == null ? null : value.toString() }
203203
})
204204

205-
// Invalid JSON is preserved as a Variant string unless the global
206-
// throw-on-invalid-JSON option is enabled.
205+
// try_parse_to_variant returns NULL for text that is not a JSON document.
207206
order_qt_variant_after_errors """
208207
SELECT id, payload IS NULL,
209208
CAST(payload['recovered'] AS BOOLEAN),

‎regression-test/suites/variant_p0/test_variant_json_functions.groovy‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,15 @@ suite("test_variant_json_functions", "p0") {
2323
DUPLICATE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1
2424
PROPERTIES ("replication_num" = "1")
2525
"""
26+
// parse_to_variant only accepts JSON documents, so the Variant string row is a JSON string.
2627
sql """
2728
INSERT INTO test_variant_json_functions
2829
SELECT k, parse_to_variant(doc) FROM (
2930
SELECT 1 AS k, '{"a":1,"s":"x","b":{"c":[1,2]}}' AS doc
3031
UNION ALL SELECT 2, '{"a":2,"s":"yz","b":{"c":[3]},"n":{"m":true}}'
3132
UNION ALL SELECT 3, '[1,"two",{"three":3}]'
3233
UNION ALL SELECT 4, '42'
33-
UNION ALL SELECT 5, 'abc'
34+
UNION ALL SELECT 5, '"abc"'
3435
UNION ALL SELECT 6, NULL) t
3536
"""
3637

0 commit comments

Comments
 (0)