Skip to content

Commit 69e66d9

Browse files
dengfangyuanclaude
authored andcommitted
[refactor][paimon] remove paimon-rust reader from FileScanner V1
The paimon-rust reader now lives under FileScannerV2 (PaimonRustTableReader inside PaimonHybridReader). Remove the obsolete V1 implementation: - Delete be/src/format/table/paimon_rust_reader.{h,cpp} and its V1 unit test. - FileScanner::_get_next_reader: drop the PAIMON_RUST branch and the enable_paimon_rust_reader session-variable fallback; a PAIMON_RUST split reaching V1 now returns NotSupported pointing at enable_file_scanner_v2. - PaimonRustPredicateConverter: drop the V1 slot-descriptor constructor and _resolve_field desc-table path (V2 resolves by projected column name); remove the now-unused _state / _file_columns members. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1 parent 223e395 commit 69e66d9

7 files changed

Lines changed: 30 additions & 974 deletions

File tree

‎be/src/exec/scan/file_scanner.cpp‎

Lines changed: 7 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@
7575
#include "format/table/max_compute_jni_reader.h"
7676
#include "format/table/paimon_jni_reader.h"
7777
#include "format/table/paimon_reader.h"
78-
#include "format/table/paimon_rust_reader.h"
7978
#include "format/table/partition_column_filler.h"
8079
#include "format/table/remote_doris_reader.h"
8180
#include "format/table/transactional_hive_reader.h"
@@ -1111,14 +1110,14 @@ Status FileScanner::_get_next_reader() {
11111110
} else if (range.__isset.table_format_params &&
11121111
range.table_format_params.table_format_type == "paimon") {
11131112
const auto& paimon_params = range.table_format_params.paimon_params;
1114-
bool use_paimon_rust_reader = false;
11151113
if (paimon_params.__isset.reader_type) {
11161114
switch (paimon_params.reader_type) {
1117-
case TPaimonReaderType::PAIMON_RUST:
1118-
use_paimon_rust_reader = true;
1119-
break;
11201115
case TPaimonReaderType::PAIMON_JNI:
11211116
break;
1117+
case TPaimonReaderType::PAIMON_RUST:
1118+
return Status::NotSupported(
1119+
"paimon rust reader requires enable_file_scanner_v2=true; the V1 "
1120+
"FileScanner no longer supports PAIMON_RUST splits");
11221121
case TPaimonReaderType::PAIMON_NATIVE:
11231122
return Status::InternalError(
11241123
"invalid PAIMON_NATIVE reader_type for paimon FORMAT_JNI split, "
@@ -1128,38 +1127,10 @@ Status FileScanner::_get_next_reader() {
11281127
"unknown paimon reader_type for paimon FORMAT_JNI split, possibly "
11291128
"caused by FE/BE protocol mismatch");
11301129
}
1131-
} else {
1132-
// TODO: Remove this fallback after all FE versions set TPaimonReaderType.
1133-
// Rust has higher priority than cpp.
1134-
use_paimon_rust_reader =
1135-
_state->query_options().__isset.enable_paimon_rust_reader &&
1136-
_state->query_options().enable_paimon_rust_reader;
1137-
}
1138-
// Native paimon readers (rust/cpp) push conjuncts down, but they live
1139-
// under FORMAT_JNI where push_down_predicates is false, so the generic
1140-
// parquet/orc path never ran _process_late_arrival_conjuncts() and
1141-
// _push_down_conjuncts is still empty here. Populate it now (same call
1142-
// parquet/orc make) so the predicate converters see the conjuncts.
1143-
if (!_is_load && use_paimon_rust_reader) {
1144-
RETURN_IF_ERROR(_process_late_arrival_conjuncts());
1145-
}
1146-
if (use_paimon_rust_reader) {
1147-
auto rust_reader = PaimonRustReader::create_unique(_file_slot_descs, _state,
1148-
_profile, range, _params);
1149-
rust_reader->set_push_down_agg_type(_get_push_down_agg_type());
1150-
if (!_is_load && !_push_down_conjuncts.empty()) {
1151-
// paimon-rust needs the live table handle to build predicates,
1152-
// so the converter runs inside init_reader. Just hand over the
1153-
// conjuncts here.
1154-
rust_reader->set_push_down_conjuncts(_push_down_conjuncts);
1155-
}
1156-
init_status = rust_reader->init_reader();
1157-
_cur_reader = std::move(rust_reader);
1158-
} else {
1159-
_cur_reader = PaimonJniReader::create_unique(_file_slot_descs, _state, _profile,
1160-
range, _params);
1161-
init_status = ((PaimonJniReader*)(_cur_reader.get()))->init_reader();
11621130
}
1131+
_cur_reader = PaimonJniReader::create_unique(_file_slot_descs, _state, _profile,
1132+
range, _params);
1133+
init_status = ((PaimonJniReader*)(_cur_reader.get()))->init_reader();
11631134
} else if (range.__isset.table_format_params &&
11641135
range.table_format_params.table_format_type == "hudi") {
11651136
_cur_reader = HudiJniReader::create_unique(*_params,

‎be/src/format/table/paimon_rust_predicate_converter.cpp‎

Lines changed: 8 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,6 @@
3939
#include "exprs/vin_predicate.h"
4040
#include "exprs/vliteral.h"
4141
#include "exprs/vslot_ref.h"
42-
#include "runtime/descriptors.h"
43-
#include "runtime/runtime_state.h"
4442
#include "util/timezone_utils.h"
4543

4644
namespace doris {
@@ -98,19 +96,6 @@ std::string consume_predicate_error(paimon_error* err) {
9896
}
9997
} // namespace
10098

101-
PaimonRustPredicateConverter::PaimonRustPredicateConverter(
102-
const std::vector<SlotDescriptor*>& file_slot_descs, RuntimeState* state,
103-
const paimon_table* table)
104-
: _state(state), _table(table) {
105-
_file_columns.reserve(file_slot_descs.size());
106-
for (const auto& slot : file_slot_descs) {
107-
_file_columns.insert(_normalize_name(slot->col_name()));
108-
}
109-
if (!TimezoneUtils::find_cctz_time_zone("GMT", _gmt_tz)) {
110-
TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, _gmt_tz);
111-
}
112-
}
113-
11499
PaimonRustPredicateConverter::PaimonRustPredicateConverter(
115100
const std::vector<std::string>& column_names, const std::vector<DataTypePtr>& column_types,
116101
const paimon_table* table)
@@ -386,38 +371,18 @@ std::optional<PaimonRustPredicateConverter::FieldMeta> PaimonRustPredicateConver
386371
if (!slot_ref) {
387372
return std::nullopt;
388373
}
389-
390-
if (!_columns_by_name.empty()) {
391-
// V2 mode: the conjunct VSlotRefs were rewritten to table global indices, so
392-
// slot_id is a position, not a slot id; resolve by the carried column name
393-
// against the projected-column registry instead of the desc table.
394-
auto it = _columns_by_name.find(_normalize_name(slot_ref->column_name()));
395-
if (it == _columns_by_name.end()) {
396-
return std::nullopt;
397-
}
398-
const auto& [column, type] = it->second;
399-
if (!_is_supported_slot_type(type->get_primitive_type(), type->get_precision())) {
400-
return std::nullopt;
401-
}
402-
return FieldMeta {column, type};
403-
}
404-
405-
// V1 mode: resolve through the desc table.
406-
if (!_state) {
407-
return std::nullopt;
408-
}
409-
auto* slot_desc = _state->desc_tbl().get_slot_descriptor(slot_ref->slot_id());
410-
if (!slot_desc) {
411-
return std::nullopt;
412-
}
413-
if (_file_columns.find(_normalize_name(slot_desc->col_name())) == _file_columns.end()) {
374+
// FileScannerV2 rewrites conjunct VSlotRefs to table global indices, so slot_id
375+
// is a position, not a slot id; resolve by the carried column name against the
376+
// projected-column registry instead of the desc table.
377+
auto it = _columns_by_name.find(_normalize_name(slot_ref->column_name()));
378+
if (it == _columns_by_name.end()) {
414379
return std::nullopt;
415380
}
416-
auto slot_type = slot_desc->type();
417-
if (!_is_supported_slot_type(slot_type->get_primitive_type(), slot_type->get_precision())) {
381+
const auto& [column, type] = it->second;
382+
if (!_is_supported_slot_type(type->get_primitive_type(), type->get_precision())) {
418383
return std::nullopt;
419384
}
420-
return FieldMeta {slot_desc->col_name(), slot_type};
385+
return FieldMeta {column, type};
421386
}
422387

423388
std::optional<PaimonRustPredicateConverter::DatumHolder>

‎be/src/format/table/paimon_rust_predicate_converter.h‎

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
#include <optional>
2222
#include <string>
2323
#include <unordered_map>
24-
#include <unordered_set>
2524
#include <utility>
2625
#include <vector>
2726

@@ -39,8 +38,6 @@ extern "C" {
3938
}
4039

4140
namespace doris {
42-
class RuntimeState;
43-
class SlotDescriptor;
4441
} // namespace doris
4542

4643
namespace doris {
@@ -67,16 +64,12 @@ namespace doris {
6764
// a partial filter only ever prunes a superset and never changes results.
6865
class PaimonRustPredicateConverter {
6966
public:
70-
// V1 mode: resolve fields via slot descriptors from RuntimeState's desc table.
71-
// The conjuncts are the scanner's original slot-based expressions.
72-
PaimonRustPredicateConverter(const std::vector<SlotDescriptor*>& file_slot_descs,
73-
RuntimeState* state, const paimon_table* table);
74-
// V2 mode: resolve fields by column name from the supplied registry. FileScannerV2
75-
// rewrites conjunct VSlotRefs to table global indices (positions, not slot ids),
76-
// so the v1 desc-table lookup must not run; the rewritten refs carry the original
77-
// column name and data type instead. `column_names` should list only the columns
78-
// the reader actually reads (e.g. exclude partition keys); the caller guarantees
79-
// names and types are parallel.
67+
// Resolve fields by column name from the supplied registry. FileScannerV2
68+
// rewrites conjunct VSlotRefs to table global indices (positions, not slot
69+
// ids), so a desc-table lookup must not run; the rewritten refs carry the
70+
// original column name and data type instead. `column_names` should list only
71+
// the columns the reader actually reads (e.g. exclude partition keys); the
72+
// caller guarantees names and types are parallel.
8073
PaimonRustPredicateConverter(const std::vector<std::string>& column_names,
8174
const std::vector<DataTypePtr>& column_types,
8275
const paimon_table* table);
@@ -90,7 +83,7 @@ class PaimonRustPredicateConverter {
9083
// Original file column name passed verbatim to the paimon_predicate_*
9184
// functions (rust resolves the field index/type from the schema by name).
9285
std::string column;
93-
// Column type used for literal conversion (replaces the v1 SlotDescriptor).
86+
// Column type used for literal conversion.
9487
DataTypePtr type;
9588
};
9689

@@ -137,11 +130,7 @@ class PaimonRustPredicateConverter {
137130
// the type coverage of paimon-cpp's _to_paimon_field_type).
138131
static bool _is_supported_slot_type(PrimitiveType type, uint32_t precision);
139132

140-
std::unordered_set<std::string> _file_columns; // normalized file column names
141-
// V2 mode registry: normalized column name -> original name and type. Empty in
142-
// v1 mode, which resolves through the desc table instead.
143133
std::unordered_map<std::string, std::pair<std::string, DataTypePtr>> _columns_by_name;
144-
RuntimeState* _state = nullptr;
145134
const paimon_table* _table = nullptr;
146135
cctz::time_zone _gmt_tz;
147136
};

0 commit comments

Comments
 (0)