@@ -310,6 +310,7 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) {
310310 const auto max_block_bytes = _runtime_state->preferred_block_size_bytes ();
311311 *rows = 0 ;
312312 *eof = false ;
313+ _missing_fields.clear ();
313314
314315 while (file_block->rows () < batch_size && !_reader_eof &&
315316 file_block->bytes () < max_block_bytes) {
@@ -355,6 +356,9 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) {
355356 }
356357
357358 *rows = file_block->rows ();
359+ for (auto & [_, mask] : _missing_fields) {
360+ mask.resize (*rows, 0 );
361+ }
358362 _record_scan_rows (cast_set<int64_t >(*rows));
359363 {
360364 SCOPED_TIMER (_filter_time);
@@ -702,7 +706,15 @@ Status JsonReader::_append_simple_json_rows(Block* block, bool* is_empty_row, bo
702706 _array_iter = _array.begin ();
703707 while (_array_iter != _array.end ()) {
704708 simdjson::ondemand::object object_value = (*_array_iter).get_object ();
705- RETURN_IF_ERROR (_set_column_values_from_object (&object_value, block, &valid));
709+ const size_t row_start = block->rows ();
710+ const auto status = _set_column_values_from_object (&object_value, block, &valid);
711+ if (!status.ok ()) {
712+ if (!_request->reject_invalid_rows ) {
713+ return status;
714+ }
715+ RETURN_IF_ERROR (_handle_json_error (status, block, row_start, is_empty_row));
716+ valid = true ;
717+ }
706718 ++_array_iter;
707719 if (!valid) {
708720 *is_empty_row = true ;
@@ -750,7 +762,14 @@ Status JsonReader::_append_flat_array_jsonpath_rows(Block* block, bool* is_empty
750762 }
751763 object_value = rooted_value.get_object ();
752764 }
753- RETURN_IF_ERROR (_write_columns_by_jsonpath (&object_value, block, &valid));
765+ const size_t row_start = block->rows ();
766+ const auto status = _write_columns_by_jsonpath (&object_value, block, &valid);
767+ if (!status.ok ()) {
768+ if (!_request->reject_invalid_rows ) {
769+ return status;
770+ }
771+ RETURN_IF_ERROR (_handle_json_error (status, block, row_start, is_empty_row));
772+ }
754773 ++_array_iter;
755774 }
756775 *is_empty_row = block->rows () == original_rows;
@@ -787,6 +806,10 @@ Status JsonReader::_set_column_values_from_object(simdjson::ondemand::object* ob
787806 if (column_index == static_cast <size_t >(-1 )) {
788807 continue ;
789808 }
809+ if (_request->reject_invalid_rows &&
810+ _requested_columns[column_index].slot_desc ->is_skip_bitmap_col ()) {
811+ continue ;
812+ }
790813 if (seen_columns[column_index]) {
791814 if (_is_hive_table) {
792815 // Hive JSON keeps the last duplicate key ignoring case. The earlier value has
@@ -809,6 +832,9 @@ Status JsonReader::_set_column_values_from_object(simdjson::ondemand::object* ob
809832 has_valid_value = true ;
810833 }
811834
835+ if (!has_valid_value && _request->reject_invalid_rows ) {
836+ return Status::DataQualityError (" No source column matches this JSON object" );
837+ }
812838 for (size_t i = 0 ; i < _requested_columns.size (); ++i) {
813839 if (seen_columns[i]) {
814840 continue ;
@@ -881,6 +907,9 @@ Status JsonReader::_write_columns_by_jsonpath(simdjson::ondemand::object* object
881907 }
882908 }
883909
910+ if (!has_valid_value && _request->reject_invalid_rows ) {
911+ return Status::DataQualityError (" No source column matches the JSON paths" );
912+ }
884913 if (!has_valid_value) {
885914 // jsonpaths can legally match nothing. Roll the row back so an all-missing path set does
886915 // not create a synthetic row of nulls.
@@ -1063,6 +1092,11 @@ Status JsonReader::_write_data_to_column(simdjson::ondemand::value& value,
10631092
10641093Status JsonReader::_fill_missing_column (const RequestedColumn& column, IColumn* column_ptr,
10651094 bool * valid) {
1095+ if (_request->reject_invalid_rows ) {
1096+ auto & missing = _missing_fields[column.file_column_id ];
1097+ missing.resize (column_ptr->size () + 1 , 0 );
1098+ missing.back () = 1 ;
1099+ }
10661100 if (column.slot_desc ->is_nullable ()) {
10671101 auto * nullable_column = assert_cast<ColumnNullable*>(column_ptr);
10681102 nullable_column->insert_default ();
@@ -1097,6 +1131,20 @@ Status JsonReader::_handle_json_error(const Status& status, Block* block, size_t
10971131 // block to the row count before this document before either surfacing the error or appending
10981132 // the ignore-malformed null row.
10991133 _truncate_block_to_rows (block, original_rows);
1134+ for (auto & [_, mask] : _missing_fields) {
1135+ mask.resize (original_rows, 0 );
1136+ }
1137+ if (_request->reject_invalid_rows &&
1138+ (status.is <ErrorCode::DATA_QUALITY_ERROR >() || status.is <ErrorCode::INVALID_ARGUMENT >())) {
1139+ RETURN_IF_ERROR (_runtime_state->append_error_msg_to_file (
1140+ [&] { return std::string (_padding_buffer.data (), _original_doc_size); },
1141+ [&] { return status.to_string (); }));
1142+ if (_io_ctx != nullptr ) {
1143+ ++_io_ctx->load_filtered_rows ;
1144+ }
1145+ *is_empty_row = true ;
1146+ return Status::OK ();
1147+ }
11001148 if (_openx_json_ignore_malformed && status.is <ErrorCode::DATA_QUALITY_ERROR >()) {
11011149 RETURN_IF_ERROR (_append_null_for_malformed_json (block));
11021150 *is_empty_row = false ;
0 commit comments