Skip to content

Commit a73b750

Browse files
committed
[fix](load) Account for rejected JSON array roots in V2
1 parent 2f46771 commit a73b750

4 files changed

Lines changed: 113 additions & 3 deletions

File tree

‎be/src/format_v2/json/json_reader.cpp‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -754,27 +754,36 @@ Status JsonReader::_append_flat_array_jsonpath_rows(Block* block, bool* is_empty
754754
_array = _json_value.get_array();
755755
_array_iter = _array.begin();
756756
while (_array_iter != _array.end()) {
757+
const size_t row_start = block->rows();
757758
simdjson::ondemand::object object_value = (*_array_iter).get_object();
758759
if (!_parsed_from_json_root && !_parsed_json_root.empty()) {
759-
// For strip_outer_array, json_root is evaluated against each element. Elements without
760-
// the requested root do not produce rows, matching the load reader behavior.
760+
// Reject roots per element for loads, preserving earlier siblings and their presence
761+
// masks while counting each invalid element against the load quality gate.
761762
simdjson::ondemand::value rooted_value;
762763
Status st = JsonFunctions::extract_from_object(object_value, _parsed_json_root,
763764
&rooted_value);
764765
if (!st.ok()) {
765766
if (st.is<ErrorCode::NOT_FOUND>()) {
767+
if (_request->reject_invalid_rows) {
768+
RETURN_IF_ERROR(
769+
_handle_json_error(Status::DataQualityError("{}", st.to_string()),
770+
block, row_start, is_empty_row));
771+
}
766772
++_array_iter;
767773
continue;
768774
}
769775
return st;
770776
}
771777
if (rooted_value.type().value() != simdjson::ondemand::json_type::object) {
778+
if (_request->reject_invalid_rows) {
779+
RETURN_IF_ERROR(_handle_json_error(Status::DataQualityError("Not object item"),
780+
block, row_start, is_empty_row));
781+
}
772782
++_array_iter;
773783
continue;
774784
}
775785
object_value = rooted_value.get_object();
776786
}
777-
const size_t row_start = block->rows();
778787
const auto status = _write_columns_by_jsonpath(&object_value, block, &valid);
779788
if (!status.ok()) {
780789
if (!_request->reject_invalid_rows) {

‎be/test/format_v2/table/load_reader_test.cpp‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -581,6 +581,71 @@ TEST_F(LoadJsonPipeTest, EmptyOuterArrayIsRejectedBeforeFollowingMessage) {
581581
ASSERT_TRUE(reader.close().ok());
582582
}
583583

584+
TEST_F(LoadJsonPipeTest, ConfiguredRootCountsMissingAndNullElements) {
585+
params.file_attributes.__set_strip_outer_array(true);
586+
params.file_attributes.__set_json_root("$.payload");
587+
params.file_attributes.__set_jsonpaths(R"(["$.k"])");
588+
append(R"([{"other":1},{"payload":null},{"payload":{"k":2}}])");
589+
ASSERT_TRUE(_pipe->finish().ok());
590+
LoadTableReader reader(source.get(), destination.get());
591+
const auto status = prepare(reader);
592+
ASSERT_TRUE(status.ok()) << status;
593+
Block output(destination->slots(), 0);
594+
bool eos = false;
595+
ASSERT_TRUE(reader.get_block(&output, &eos).ok());
596+
ASSERT_EQ(output.rows(), 1);
597+
EXPECT_EQ(output.get_by_position(0).column->get_data_at(0).to_string(), "2");
598+
EXPECT_EQ(io->load_filtered_rows, 2);
599+
ASSERT_TRUE(reader.get_block(&output, &eos).ok());
600+
EXPECT_TRUE(eos);
601+
EXPECT_EQ(io->load_filtered_rows, 2);
602+
ASSERT_TRUE(reader.close().ok());
603+
}
604+
605+
TEST_F(LoadJsonPipeTest, RootRejectionPreservesEarlierAndLaterSiblings) {
606+
params.file_attributes.__set_strip_outer_array(true);
607+
params.file_attributes.__set_json_root("$.payload");
608+
params.file_attributes.__set_jsonpaths(R"(["$.k"])");
609+
append(R"([{"payload":{"k":1}},{"other":1},{"payload":null},
610+
{"payload":3},{"payload":[]},{"payload":{"k":2}}])");
611+
ASSERT_TRUE(_pipe->finish().ok());
612+
LoadTableReader reader(source.get(), destination.get());
613+
const auto status = prepare(reader);
614+
ASSERT_TRUE(status.ok()) << status;
615+
Block output(destination->slots(), 0);
616+
bool eos = false;
617+
ASSERT_TRUE(reader.get_block(&output, &eos).ok());
618+
ASSERT_EQ(output.rows(), 2);
619+
EXPECT_EQ(output.get_by_position(0).column->get_data_at(0).to_string(), "1");
620+
EXPECT_EQ(output.get_by_position(0).column->get_data_at(1).to_string(), "2");
621+
EXPECT_EQ(io->load_filtered_rows, 4);
622+
ASSERT_TRUE(reader.get_block(&output, &eos).ok());
623+
EXPECT_TRUE(eos);
624+
ASSERT_TRUE(reader.close().ok());
625+
}
626+
627+
TEST_F(LoadJsonPipeTest, RejectedRootsDoNotHideFollowingMessage) {
628+
params.file_attributes.__set_strip_outer_array(true);
629+
params.file_attributes.__set_json_root("$.payload");
630+
params.file_attributes.__set_jsonpaths(R"(["$.k"])");
631+
append(R"([{"other":1},{"payload":null}])");
632+
append(R"([{"payload":{"k":2}}])");
633+
ASSERT_TRUE(_pipe->finish().ok());
634+
LoadTableReader reader(source.get(), destination.get());
635+
const auto status = prepare(reader);
636+
ASSERT_TRUE(status.ok()) << status;
637+
Block output(destination->slots(), 0);
638+
bool eos = false;
639+
ASSERT_TRUE(reader.get_block(&output, &eos).ok());
640+
ASSERT_EQ(output.rows(), 1);
641+
EXPECT_FALSE(eos);
642+
EXPECT_EQ(output.get_by_position(0).column->get_data_at(0).to_string(), "2");
643+
EXPECT_EQ(io->load_filtered_rows, 2);
644+
ASSERT_TRUE(reader.get_block(&output, &eos).ok());
645+
EXPECT_TRUE(eos);
646+
ASSERT_TRUE(reader.close().ok());
647+
}
648+
584649
TEST_F(LoadReaderTest, SourceSlotsAreConvertedBeforeDestinationFiltering) {
585650
auto child = std::make_unique<SourceReader>();
586651
auto* input = child.get();
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[{"other":1},{"payload":null},{"payload":{"k":2}}]

‎regression-test/suites/load_p0/stream_load/test_file_scanner_v2_load.groovy‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,41 @@ suite("test_file_scanner_v2_load", "p0") {
9696
sql("SELECT k, v FROM ${tableName} ORDER BY k"))
9797
}
9898

99+
["0", "1"].each { ratio ->
100+
sql "TRUNCATE TABLE ${tableName}"
101+
streamLoad {
102+
table tableName
103+
set 'format', 'json'
104+
set 'read_json_by_line', 'true'
105+
set 'strip_outer_array', 'true'
106+
set 'json_root', '$.payload'
107+
set 'jsonpaths', '["$.k"]'
108+
set 'columns', 'k'
109+
set 'max_filter_ratio', ratio
110+
file 'file_scanner_v2_load_invalid_roots.json'
111+
check { result, exception, startTime, endTime ->
112+
if (exception != null) {
113+
throw exception
114+
}
115+
def response = parseJson(result)
116+
assertEquals(3, response.NumberTotalRows as int)
117+
assertEquals(2, response.NumberFilteredRows as int)
118+
assertEquals(0, response.NumberUnselectedRows as int)
119+
if (ratio == "0") {
120+
// Missing and non-object roots must each count against the quality gate.
121+
assertEquals("Fail", response.Status)
122+
assertEquals(0, response.NumberLoadedRows as int)
123+
assertTrue(response.Message.contains("too many filtered rows"))
124+
} else {
125+
checkRows(result, exception, 1, 2, 0)
126+
}
127+
}
128+
}
129+
sql "sync"
130+
assertEquals(ratio == "0" ? [] : [[2, 7]],
131+
sql("SELECT k, v FROM ${tableName} ORDER BY k"))
132+
}
133+
99134
sql "DROP TABLE ${tableName}"
100135
sql """CREATE TABLE ${tableName} (k INT NOT NULL, v INT NULL)
101136
UNIQUE KEY(k) DISTRIBUTED BY HASH(k) BUCKETS 1

0 commit comments

Comments
 (0)