Skip to content

Commit abdba83

Browse files
committed
[Enhancement](load) Adapt file loads to FileScannerV2 with LoadTableReader
1 parent 62cb1df commit abdba83

22 files changed

Lines changed: 1444 additions & 11 deletions

‎be/src/exec/operator/file_scan_operator.cpp‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,29 @@ bool FileScanLocalState::TEST_should_use_file_scanner_v2(const TQueryOptions& qu
145145
bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_options,
146146
bool is_load,
147147
const TFileScanRangeParams& scan_params) {
148+
if (is_load) {
149+
// Load planners omit query session options. Thrift's default makes an omitted V2
150+
// option indistinguishable from false, so ingestion selects by format instead.
151+
switch (scan_params.format_type) {
152+
case TFileFormatType::FORMAT_PARQUET:
153+
case TFileFormatType::FORMAT_ORC:
154+
case TFileFormatType::FORMAT_JSON:
155+
case TFileFormatType::FORMAT_CSV_PLAIN:
156+
case TFileFormatType::FORMAT_CSV_GZ:
157+
case TFileFormatType::FORMAT_CSV_BZ2:
158+
case TFileFormatType::FORMAT_CSV_LZ4FRAME:
159+
case TFileFormatType::FORMAT_CSV_LZ4BLOCK:
160+
case TFileFormatType::FORMAT_CSV_LZOP:
161+
case TFileFormatType::FORMAT_CSV_DEFLATE:
162+
case TFileFormatType::FORMAT_CSV_SNAPPYBLOCK:
163+
case TFileFormatType::FORMAT_PROTO:
164+
case TFileFormatType::FORMAT_TEXT:
165+
case TFileFormatType::FORMAT_WAL:
166+
return true;
167+
default:
168+
return false;
169+
}
170+
}
148171
const bool is_transactional_hive =
149172
scan_params.__isset.table_format_params &&
150173
scan_params.table_format_params.table_format_type == "transactional_hive";
@@ -174,7 +197,7 @@ bool FileScanLocalState::_can_generate_physical_splits(const TQueryOptions& quer
174197
bool is_load,
175198
const TFileScanRangeParams& scan_params,
176199
const TFileRangeDesc& range) {
177-
if (!should_use_file_scanner_v2(query_options, is_load, scan_params)) {
200+
if (is_load || !should_use_file_scanner_v2(query_options, is_load, scan_params)) {
178201
return false;
179202
}
180203
const auto format = range.__isset.format_type ? range.format_type : scan_params.format_type;

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
#include "format_v2/table/iceberg_position_delete_sys_table_reader.h"
6464
#include "format_v2/table/iceberg_reader.h"
6565
#include "format_v2/table/lance_reader.h"
66+
#include "format_v2/table/load_reader.h"
6667
#include "format_v2/table/paimon_reader.h"
6768
#include "format_v2/table/remote_doris_reader.h"
6869
#include "format_v2/table_reader.h"
@@ -422,6 +423,7 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat
422423

423424
Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) {
424425
RETURN_IF_ERROR(Scanner::init(state, conjuncts));
426+
_is_load = state->desc_tbl().get_tuple_descriptor(_params->src_tuple_id) != nullptr;
425427
auto* profile = _local_state->scanner_profile();
426428
const auto hierarchy = file_scan_profile::ensure_hierarchy(profile);
427429
_scanner_total_timer = hierarchy.scanner;
@@ -487,6 +489,12 @@ Status FileScannerV2::_open_impl(RuntimeState* state) {
487489
if (_first_scan_range) {
488490
RETURN_IF_ERROR(_create_table_reader_for_format(_current_range, &_table_reader));
489491
DORIS_CHECK(_table_reader != nullptr);
492+
if (_is_load) {
493+
_table_reader = std::make_unique<format::LoadTableReader>(
494+
std::move(_table_reader),
495+
state->desc_tbl().get_tuple_descriptor(_params->src_tuple_id),
496+
const_cast<TupleDescriptor*>(_output_tuple_desc));
497+
}
490498
RETURN_IF_ERROR(_init_expr_ctxes());
491499
RETURN_IF_ERROR(_init_table_reader(_current_range));
492500
// Refine the first source split before yielding the scanner worker. Other scanners may be
@@ -1069,7 +1077,18 @@ Status FileScannerV2::_init_expr_ctxes() {
10691077
_slot_id_to_desc.emplace(slot_desc->id(), slot_desc);
10701078
}
10711079
DORIS_CHECK(_table_reader != nullptr);
1072-
RETURN_IF_ERROR(_build_projected_columns(*_table_reader));
1080+
if (_is_load) {
1081+
// Load required_slots describe the source tuple, not this destination tuple.
1082+
// The wrapper owns source projection, defaults and partition materialization.
1083+
_projected_columns.clear();
1084+
for (size_t i = 0; i < _output_tuple_desc->slots().size(); ++i) {
1085+
const auto* slot = _output_tuple_desc->slots()[i];
1086+
_projected_columns.push_back(_build_table_column(slot));
1087+
_slot_id_to_global_index.emplace(slot->id(), format::GlobalIndex(i));
1088+
}
1089+
} else {
1090+
RETURN_IF_ERROR(_build_projected_columns(*_table_reader));
1091+
}
10731092
return Status::OK();
10741093
}
10751094

@@ -1156,6 +1175,9 @@ format::ColumnDefinition FileScannerV2::_build_table_column(const SlotDescriptor
11561175
Status FileScannerV2::_build_table_conjuncts(VExprContextSPtrs* conjuncts) const {
11571176
DORIS_CHECK(conjuncts != nullptr);
11581177
conjuncts->clear();
1178+
if (_is_load) {
1179+
return Status::OK();
1180+
}
11591181
conjuncts->reserve(_conjuncts.size());
11601182
for (const auto& conjunct : _conjuncts) {
11611183
VExprSPtr root;
@@ -1522,6 +1544,13 @@ bool FileScannerV2::_should_update_load_counters() const {
15221544
}
15231545

15241546
void FileScannerV2::_report_file_reader_predicate_filtered_rows() {
1547+
if (_io_ctx != nullptr && _is_load) {
1548+
_counter.num_rows_filtered += _io_ctx->load_filtered_rows - _reported_load_filtered_rows;
1549+
_counter.num_rows_unselected +=
1550+
_io_ctx->load_unselected_rows - _reported_load_unselected_rows;
1551+
_reported_load_filtered_rows = _io_ctx->load_filtered_rows;
1552+
_reported_load_unselected_rows = _io_ctx->load_unselected_rows;
1553+
}
15251554
const int64_t filtered_rows = _io_ctx != nullptr ? _io_ctx->predicate_filtered_rows : 0;
15261555
const int64_t filtered_delta = filtered_rows - _reported_predicate_filtered_rows;
15271556
if (filtered_delta > 0) {

‎be/src/exec/scan/file_scanner_v2.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,8 @@ class FileScannerV2 final : public Scanner {
281281
RuntimeProfile::Counter* _adaptive_batch_probe_count_counter = nullptr;
282282
std::unique_ptr<AdaptiveBlockSizePredictor> _block_size_predictor;
283283
int64_t _reported_predicate_filtered_rows = 0;
284+
int64_t _reported_load_filtered_rows = 0;
285+
int64_t _reported_load_unselected_rows = 0;
284286
int64_t _reported_condition_cache_hit_count = 0;
285287
int64_t _reported_condition_cache_filtered_rows = 0;
286288
int64_t _last_read_bytes = 0;

‎be/src/format_v2/delimited_text/csv_reader.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,9 @@ Status CsvReader::_create_line_reader() {
159159
Status CsvReader::_validate_line(const Slice& line) {
160160
if (_file_format_type != TFileFormatType::FORMAT_PROTO && _enable_text_validate_utf8 &&
161161
!validate_utf8(line.data, line.size)) {
162+
if (_request != nullptr && _request->reject_invalid_rows) {
163+
return Status::DataQualityError("Only support csv data in utf8 codec");
164+
}
162165
return Status::InternalError<false>("Only support csv data in utf8 codec");
163166
}
164167
return Status::OK();

‎be/src/format_v2/delimited_text/delimited_text_reader.cpp‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,30 @@ Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool* eof
344344
if (line_eof) {
345345
break;
346346
}
347-
RETURN_IF_ERROR(_fill_columns_from_line(line, &columns, rows));
347+
const auto status = _fill_columns_from_line(line, &columns, rows);
348+
if (!status.ok()) {
349+
if (!_request->reject_invalid_rows ||
350+
(!status.is<ErrorCode::DATA_QUALITY_ERROR>() &&
351+
!status.is<ErrorCode::INVALID_ARGUMENT>())) {
352+
return status;
353+
}
354+
// A bad cell may follow already appended cells. Roll back the entire record.
355+
for (auto& column : columns) {
356+
if (column->size() > *rows) {
357+
column->pop_back(column->size() - *rows);
358+
}
359+
}
360+
RETURN_IF_ERROR(_runtime_state->append_error_msg_to_file(
361+
[&] {
362+
return _scan_params->format_type == TFileFormatType::FORMAT_PROTO
363+
? std::string("binary record")
364+
: line.to_string();
365+
},
366+
[&] { return status.to_string(); }));
367+
if (_io_ctx != nullptr) {
368+
++_io_ctx->load_filtered_rows;
369+
}
370+
}
348371
}
349372
}
350373

@@ -623,6 +646,15 @@ Status DelimitedTextReader::_fill_columns_from_line(const Slice& line,
623646
_split_line(line);
624647
}
625648
SCOPED_TIMER(_text_profile.deserialize_time);
649+
if (_request->reject_invalid_rows) {
650+
const auto expected = cast_set<size_t>(_scan_params->num_of_columns_from_file);
651+
const bool ignore_extra = _scan_params->file_attributes.ignore_csv_redundant_col;
652+
if (_split_values.size() < expected ||
653+
(!ignore_extra && _split_values.size() != expected)) {
654+
return Status::DataQualityError("Expected {} fields, found {}", expected,
655+
_split_values.size());
656+
}
657+
}
626658
for (const auto& column : _requested_columns) {
627659
auto* output = (*columns)[column.block_position.value()].get();
628660
const int32_t field_index = column.file_column_id.value();

‎be/src/format_v2/file_reader.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,9 @@ struct FileScanRequest {
165165
}
166166
return count;
167167
}
168+
169+
// Ingestion rejects malformed records instead of aborting a whole file.
170+
bool reject_invalid_rows = false;
168171
};
169172

170173
// Helper for constructing the scan-column layout in FileScanRequest.
@@ -374,6 +377,12 @@ class FileReader {
374377
_profile(profile) {}
375378
virtual ~FileReader() = default;
376379

380+
// Per-batch presence metadata distinguishes absent fields from explicit NULLs. These are
381+
// physical field identities; TableColumnMapper owns their translation to table positions.
382+
const std::map<LocalColumnId, std::vector<uint8_t>>& missing_fields() const {
383+
return _missing_fields;
384+
}
385+
377386
// Initialize file reader and parse file metadata.
378387
virtual Status init(RuntimeState* state);
379388

@@ -506,6 +515,7 @@ class FileReader {
506515
std::unique_ptr<io::FileDescription> _file_description;
507516
std::shared_ptr<io::IOContext> _io_ctx;
508517
RuntimeProfile* _profile = nullptr;
518+
std::map<LocalColumnId, std::vector<uint8_t>> _missing_fields;
509519
};
510520

511521
} // namespace doris::format

‎be/src/format_v2/json/json_reader.cpp‎

Lines changed: 50 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,7 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) {
310310
const auto max_block_bytes = _runtime_state->preferred_block_size_bytes();
311311
*rows = 0;
312312
*eof = false;
313+
_missing_fields.clear();
313314

314315
while (file_block->rows() < batch_size && !_reader_eof &&
315316
file_block->bytes() < max_block_bytes) {
@@ -355,6 +356,9 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) {
355356
}
356357

357358
*rows = file_block->rows();
359+
for (auto& [_, mask] : _missing_fields) {
360+
mask.resize(*rows, 0);
361+
}
358362
_record_scan_rows(cast_set<int64_t>(*rows));
359363
{
360364
SCOPED_TIMER(_filter_time);
@@ -702,7 +706,15 @@ Status JsonReader::_append_simple_json_rows(Block* block, bool* is_empty_row, bo
702706
_array_iter = _array.begin();
703707
while (_array_iter != _array.end()) {
704708
simdjson::ondemand::object object_value = (*_array_iter).get_object();
705-
RETURN_IF_ERROR(_set_column_values_from_object(&object_value, block, &valid));
709+
const size_t row_start = block->rows();
710+
const auto status = _set_column_values_from_object(&object_value, block, &valid);
711+
if (!status.ok()) {
712+
if (!_request->reject_invalid_rows) {
713+
return status;
714+
}
715+
RETURN_IF_ERROR(_handle_json_error(status, block, row_start, is_empty_row));
716+
valid = true;
717+
}
706718
++_array_iter;
707719
if (!valid) {
708720
*is_empty_row = true;
@@ -750,7 +762,14 @@ Status JsonReader::_append_flat_array_jsonpath_rows(Block* block, bool* is_empty
750762
}
751763
object_value = rooted_value.get_object();
752764
}
753-
RETURN_IF_ERROR(_write_columns_by_jsonpath(&object_value, block, &valid));
765+
const size_t row_start = block->rows();
766+
const auto status = _write_columns_by_jsonpath(&object_value, block, &valid);
767+
if (!status.ok()) {
768+
if (!_request->reject_invalid_rows) {
769+
return status;
770+
}
771+
RETURN_IF_ERROR(_handle_json_error(status, block, row_start, is_empty_row));
772+
}
754773
++_array_iter;
755774
}
756775
*is_empty_row = block->rows() == original_rows;
@@ -787,6 +806,10 @@ Status JsonReader::_set_column_values_from_object(simdjson::ondemand::object* ob
787806
if (column_index == static_cast<size_t>(-1)) {
788807
continue;
789808
}
809+
if (_request->reject_invalid_rows &&
810+
_requested_columns[column_index].slot_desc->is_skip_bitmap_col()) {
811+
continue;
812+
}
790813
if (seen_columns[column_index]) {
791814
if (_is_hive_table) {
792815
// Hive JSON keeps the last duplicate key ignoring case. The earlier value has
@@ -809,6 +832,9 @@ Status JsonReader::_set_column_values_from_object(simdjson::ondemand::object* ob
809832
has_valid_value = true;
810833
}
811834

835+
if (!has_valid_value && _request->reject_invalid_rows) {
836+
return Status::DataQualityError("No source column matches this JSON object");
837+
}
812838
for (size_t i = 0; i < _requested_columns.size(); ++i) {
813839
if (seen_columns[i]) {
814840
continue;
@@ -881,6 +907,9 @@ Status JsonReader::_write_columns_by_jsonpath(simdjson::ondemand::object* object
881907
}
882908
}
883909

910+
if (!has_valid_value && _request->reject_invalid_rows) {
911+
return Status::DataQualityError("No source column matches the JSON paths");
912+
}
884913
if (!has_valid_value) {
885914
// jsonpaths can legally match nothing. Roll the row back so an all-missing path set does
886915
// not create a synthetic row of nulls.
@@ -1063,6 +1092,11 @@ Status JsonReader::_write_data_to_column(simdjson::ondemand::value& value,
10631092

10641093
Status JsonReader::_fill_missing_column(const RequestedColumn& column, IColumn* column_ptr,
10651094
bool* valid) {
1095+
if (_request->reject_invalid_rows) {
1096+
auto& missing = _missing_fields[column.file_column_id];
1097+
missing.resize(column_ptr->size() + 1, 0);
1098+
missing.back() = 1;
1099+
}
10661100
if (column.slot_desc->is_nullable()) {
10671101
auto* nullable_column = assert_cast<ColumnNullable*>(column_ptr);
10681102
nullable_column->insert_default();
@@ -1097,6 +1131,20 @@ Status JsonReader::_handle_json_error(const Status& status, Block* block, size_t
10971131
// block to the row count before this document before either surfacing the error or appending
10981132
// the ignore-malformed null row.
10991133
_truncate_block_to_rows(block, original_rows);
1134+
for (auto& [_, mask] : _missing_fields) {
1135+
mask.resize(original_rows, 0);
1136+
}
1137+
if (_request->reject_invalid_rows &&
1138+
(status.is<ErrorCode::DATA_QUALITY_ERROR>() || status.is<ErrorCode::INVALID_ARGUMENT>())) {
1139+
RETURN_IF_ERROR(_runtime_state->append_error_msg_to_file(
1140+
[&] { return std::string(_padding_buffer.data(), _original_doc_size); },
1141+
[&] { return status.to_string(); }));
1142+
if (_io_ctx != nullptr) {
1143+
++_io_ctx->load_filtered_rows;
1144+
}
1145+
*is_empty_row = true;
1146+
return Status::OK();
1147+
}
11001148
if (_openx_json_ignore_malformed && status.is<ErrorCode::DATA_QUALITY_ERROR>()) {
11011149
RETURN_IF_ERROR(_append_null_for_malformed_json(block));
11021150
*is_empty_row = false;

0 commit comments

Comments
 (0)