Skip to content

Commit 9af1840

Browse files
authored
[improvement](load) Add LoadTableReader for FileScannerV2 ingestion (apache#69333)
### What problem does this PR solve? Add `LoadTableReader` so supported traditional file loads use FileScannerV2 with the existing FE/Thrift source and destination tuples. Source defaults, pre-filters, destination conversion, strict/nullability checks, and flexible partial-update bitmaps run in the table-reader finalization hook. CSV/JSON malformed records are rejected with rollback and load counters. JSON pipes consume every message until EOF; empty arrays and invalid configured roots contribute to the load quality gate without losing valid siblings. Master-specific adaptations: - Preserve master's ADBC/Fluss V2 query selection and Lance exclusion. - Match master's single-argument `TableReader::refresh_conjuncts()` and `RowDescriptor` constructors, and use `SlotDescriptor::type()`. - Omit the physical-split override and shared split-cache field, which are absent on master. Loads retain the existing FE split boundaries. Avro and Arrow IPC loads keep their existing path. No FE or Thrift changes. This port is independent of apache#69312. ### Validation - Passed the repository clang-format 16 check, covering all 17 changed C++ source/header files. - Passed `build-support/check-build-hygiene.sh` and `git diff --cached --check`. - Passed focused Clang syntax checks for all 11 changed C++ translation units and `table_reader.cpp`, using this master's generated headers and UT build configuration. The standalone scanner check suppresses two pre-existing unused-constant warnings from master. - Carried over the source PR's BE unit tests, JSON pipe tests, and Stream Load regression fixtures. - `run-be-ut.sh` completed configuration and started the full build. The fresh build requires about 14,750 targets, so it was stopped and replaced with the focused syntax checks above. No master runtime unit-test or cluster-regression pass is claimed; full build and runtime validation remain for CI. ### Release note Route supported traditional file loads through FileScannerV2 with a table reader for load conversion and validation.
1 parent 211965d commit 9af1840

27 files changed

Lines changed: 1985 additions & 35 deletions

‎be/src/exec/operator/file_scan_operator.cpp‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,36 @@ bool FileScanLocalState::TEST_should_use_file_scanner_v2(const TQueryOptions& qu
155155
bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_options,
156156
bool is_load,
157157
const TFileScanRangeParams& scan_params) {
158+
if (is_load) {
159+
// ADBC and Fluss are query-only connectors. Their default Parquet format must not
160+
// admit them to the ordinary file-load conversion path.
161+
if (scan_params.__isset.table_format_params &&
162+
(scan_params.table_format_params.table_format_type == "adbc" ||
163+
scan_params.table_format_params.table_format_type == "fluss")) {
164+
return false;
165+
}
166+
// Load planners omit query session options. Thrift's default makes an omitted V2
167+
// option indistinguishable from false, so ingestion selects by format instead.
168+
switch (scan_params.format_type) {
169+
case TFileFormatType::FORMAT_PARQUET:
170+
case TFileFormatType::FORMAT_ORC:
171+
case TFileFormatType::FORMAT_JSON:
172+
case TFileFormatType::FORMAT_CSV_PLAIN:
173+
case TFileFormatType::FORMAT_CSV_GZ:
174+
case TFileFormatType::FORMAT_CSV_BZ2:
175+
case TFileFormatType::FORMAT_CSV_LZ4FRAME:
176+
case TFileFormatType::FORMAT_CSV_LZ4BLOCK:
177+
case TFileFormatType::FORMAT_CSV_LZOP:
178+
case TFileFormatType::FORMAT_CSV_DEFLATE:
179+
case TFileFormatType::FORMAT_CSV_SNAPPYBLOCK:
180+
case TFileFormatType::FORMAT_PROTO:
181+
case TFileFormatType::FORMAT_TEXT:
182+
case TFileFormatType::FORMAT_WAL:
183+
return true;
184+
default:
185+
return false;
186+
}
187+
}
158188
// ADBC only has a FileScannerV2 reader, and enable_file_scanner_v2 is a session variable marked
159189
// fuzzy=true, so the regression harness flips it to false at random. Without letting adbc
160190
// through unconditionally, those queries land in v1, which has no "adbc" branch, and come back

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
#include "format_v2/table/hudi_reader.h"
6666
#include "format_v2/table/iceberg_position_delete_sys_table_reader.h"
6767
#include "format_v2/table/iceberg_reader.h"
68+
#include "format_v2/table/load_reader.h"
6869
#include "format_v2/table/paimon_reader.h"
6970
#include "format_v2/table/remote_doris_reader.h"
7071
#include "format_v2/table_reader.h"
@@ -414,6 +415,7 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat
414415

415416
Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) {
416417
RETURN_IF_ERROR(Scanner::init(state, conjuncts));
418+
_is_load = state->desc_tbl().get_tuple_descriptor(_params->src_tuple_id) != nullptr;
417419
auto* profile = _local_state->scanner_profile();
418420
const auto hierarchy = file_scan_profile::ensure_hierarchy(profile);
419421
_scanner_total_timer = hierarchy.scanner;
@@ -466,7 +468,13 @@ Status FileScannerV2::_open_impl(RuntimeState* state) {
466468
RETURN_IF_ERROR(Scanner::_open_impl(state));
467469
RETURN_IF_ERROR(_get_next_scan_range(&_first_scan_range));
468470
if (_first_scan_range) {
469-
RETURN_IF_ERROR(_create_table_reader_for_format(_current_range, &_table_reader));
471+
if (_is_load) {
472+
_table_reader = std::make_unique<format::LoadTableReader>(
473+
state->desc_tbl().get_tuple_descriptor(_params->src_tuple_id),
474+
_output_tuple_desc);
475+
} else {
476+
RETURN_IF_ERROR(_create_table_reader_for_format(_current_range, &_table_reader));
477+
}
470478
DORIS_CHECK(_table_reader != nullptr);
471479
RETURN_IF_ERROR(_init_expr_ctxes());
472480
RETURN_IF_ERROR(_init_table_reader(_current_range));
@@ -927,7 +935,18 @@ Status FileScannerV2::_init_expr_ctxes() {
927935
_slot_id_to_desc.emplace(slot_desc->id(), slot_desc);
928936
}
929937
DORIS_CHECK(_table_reader != nullptr);
930-
RETURN_IF_ERROR(_build_projected_columns(*_table_reader));
938+
if (_is_load) {
939+
// Load required_slots describe the source tuple, not this destination tuple.
940+
// LoadTableReader owns source projection, defaults and partition materialization.
941+
_projected_columns.clear();
942+
for (size_t i = 0; i < _output_tuple_desc->slots().size(); ++i) {
943+
const auto* slot = _output_tuple_desc->slots()[i];
944+
_projected_columns.push_back(_build_table_column(slot));
945+
_slot_id_to_global_index.emplace(slot->id(), format::GlobalIndex(i));
946+
}
947+
} else {
948+
RETURN_IF_ERROR(_build_projected_columns(*_table_reader));
949+
}
931950
return Status::OK();
932951
}
933952

@@ -1023,6 +1042,9 @@ format::ColumnDefinition FileScannerV2::_build_table_column(const SlotDescriptor
10231042
Status FileScannerV2::_build_table_conjuncts(VExprContextSPtrs* conjuncts) const {
10241043
DORIS_CHECK(conjuncts != nullptr);
10251044
conjuncts->clear();
1045+
if (_is_load) {
1046+
return Status::OK();
1047+
}
10261048
conjuncts->reserve(_conjuncts.size());
10271049
for (const auto& conjunct : _conjuncts) {
10281050
VExprSPtr root;
@@ -1364,6 +1386,13 @@ bool FileScannerV2::_should_update_load_counters() const {
13641386
}
13651387

13661388
void FileScannerV2::_report_file_reader_predicate_filtered_rows() {
1389+
if (_io_ctx != nullptr && _is_load) {
1390+
_counter.num_rows_filtered += _io_ctx->load_filtered_rows - _reported_load_filtered_rows;
1391+
_counter.num_rows_unselected +=
1392+
_io_ctx->load_unselected_rows - _reported_load_unselected_rows;
1393+
_reported_load_filtered_rows = _io_ctx->load_filtered_rows;
1394+
_reported_load_unselected_rows = _io_ctx->load_unselected_rows;
1395+
}
13671396
const int64_t filtered_rows = _io_ctx != nullptr ? _io_ctx->predicate_filtered_rows : 0;
13681397
const int64_t filtered_delta = filtered_rows - _reported_predicate_filtered_rows;
13691398
if (filtered_delta > 0) {

‎be/src/exec/scan/file_scanner_v2.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,8 @@ class FileScannerV2 final : public Scanner {
243243
RuntimeProfile::Counter* _adaptive_batch_probe_count_counter = nullptr;
244244
std::unique_ptr<AdaptiveBlockSizePredictor> _block_size_predictor;
245245
int64_t _reported_predicate_filtered_rows = 0;
246+
int64_t _reported_load_filtered_rows = 0;
247+
int64_t _reported_load_unselected_rows = 0;
246248
int64_t _reported_condition_cache_hit_count = 0;
247249
int64_t _reported_condition_cache_filtered_rows = 0;
248250
int64_t _last_read_bytes = 0;

‎be/src/format_v2/delimited_text/csv_reader.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,9 @@ Status CsvReader::_create_line_reader() {
174174
Status CsvReader::_validate_line(const Slice& line) {
175175
if (_file_format_type != TFileFormatType::FORMAT_PROTO && _enable_text_validate_utf8 &&
176176
!validate_utf8(line.data, line.size)) {
177+
if (_request != nullptr && _request->reject_invalid_rows) {
178+
return Status::DataQualityError("Only support csv data in utf8 codec");
179+
}
177180
return Status::InternalError<false>("Only support csv data in utf8 codec");
178181
}
179182
return Status::OK();

‎be/src/format_v2/delimited_text/delimited_text_reader.cpp‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,30 @@ Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool* eof
344344
if (line_eof) {
345345
break;
346346
}
347-
RETURN_IF_ERROR(_fill_columns_from_line(line, &columns, rows));
347+
const auto status = _fill_columns_from_line(line, &columns, rows);
348+
if (!status.ok()) {
349+
if (!_request->reject_invalid_rows ||
350+
(!status.is<ErrorCode::DATA_QUALITY_ERROR>() &&
351+
!status.is<ErrorCode::INVALID_ARGUMENT>())) {
352+
return status;
353+
}
354+
// A bad cell may follow already appended cells. Roll back the entire record.
355+
for (auto& column : columns) {
356+
if (column->size() > *rows) {
357+
column->pop_back(column->size() - *rows);
358+
}
359+
}
360+
RETURN_IF_ERROR(_runtime_state->append_error_msg_to_file(
361+
[&] {
362+
return _scan_params->format_type == TFileFormatType::FORMAT_PROTO
363+
? std::string("binary record")
364+
: line.to_string();
365+
},
366+
[&] { return status.to_string(); }));
367+
if (_io_ctx != nullptr) {
368+
++_io_ctx->load_filtered_rows;
369+
}
370+
}
348371
}
349372
}
350373

@@ -623,6 +646,15 @@ Status DelimitedTextReader::_fill_columns_from_line(const Slice& line,
623646
_split_line(line);
624647
}
625648
SCOPED_TIMER(_text_profile.deserialize_time);
649+
if (_request->reject_invalid_rows) {
650+
const auto expected = cast_set<size_t>(_scan_params->num_of_columns_from_file);
651+
const bool ignore_extra = _scan_params->file_attributes.ignore_csv_redundant_col;
652+
if (_split_values.size() < expected ||
653+
(!ignore_extra && _split_values.size() != expected)) {
654+
return Status::DataQualityError("Expected {} fields, found {}", expected,
655+
_split_values.size());
656+
}
657+
}
626658
for (const auto& column : _requested_columns) {
627659
auto* output = (*columns)[column.block_position.value()].get();
628660
const int32_t field_index = column.file_column_id.value();

‎be/src/format_v2/file_reader.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,9 @@ struct FileScanRequest {
141141
}
142142
return count;
143143
}
144+
145+
// Ingestion rejects malformed records instead of aborting a whole file.
146+
bool reject_invalid_rows = false;
144147
};
145148

146149
// Helper for constructing the scan-column layout in FileScanRequest.
@@ -350,6 +353,12 @@ class FileReader {
350353
_profile(profile) {}
351354
virtual ~FileReader() = default;
352355

356+
// Per-batch presence metadata distinguishes absent fields from explicit NULLs. These are
357+
// physical field identities; TableColumnMapper owns their translation to table positions.
358+
const std::map<LocalColumnId, std::vector<uint8_t>>& missing_fields() const {
359+
return _missing_fields;
360+
}
361+
353362
// Initialize file reader and parse file metadata.
354363
virtual Status init(RuntimeState* state);
355364

@@ -466,6 +475,7 @@ class FileReader {
466475
std::unique_ptr<io::FileDescription> _file_description;
467476
std::shared_ptr<io::IOContext> _io_ctx;
468477
RuntimeProfile* _profile = nullptr;
478+
std::map<LocalColumnId, std::vector<uint8_t>> _missing_fields;
469479
};
470480

471481
} // namespace doris::format

0 commit comments

Comments
 (0)