Skip to content

Commit 518be1a

Browse files
committed
[refactor](external) Remove Hive ACID, Avro and LakeSoul readers
1 parent 9b38487 commit 518be1a

143 files changed

Lines changed: 249 additions & 9844 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/exec/operator/file_scan_operator.cpp‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
#include "core/data_type/data_type_struct.h"
3030
#include "exec/operator/olap_scan_operator.h"
3131
#include "exec/operator/scan_operator.h"
32+
#include "exec/scan/file_scan_range_utils.h"
3233
#include "exec/scan/file_scanner.h"
3334
#include "exec/scan/file_scanner_v2.h"
3435
#include "exec/scan/scanner_context.h"
@@ -145,9 +146,6 @@ bool FileScanLocalState::TEST_should_use_file_scanner_v2(const TQueryOptions& qu
145146
bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_options,
146147
bool is_load,
147148
const TFileScanRangeParams& scan_params) {
148-
const bool is_transactional_hive =
149-
scan_params.__isset.table_format_params &&
150-
scan_params.table_format_params.table_format_type == "transactional_hive";
151149
// Hybrid scans advertise native Parquet at scan level because ranges can arrive after the
152150
// scanner is selected; FORMAT_JNI alone does not imply that every range uses JNI.
153151
const bool is_hybrid_native_parquet = scan_params.format_type == TFileFormatType::FORMAT_JNI &&
@@ -167,7 +165,7 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o
167165
// after scanner construction. Keep versioned Iceberg scans on V2 without eagerly listing
168166
// every file.
169167
return scanner_v2_requested && !is_load &&
170-
scan_params.format_type != TFileFormatType::FORMAT_ES_HTTP && !is_transactional_hive;
168+
scan_params.format_type != TFileFormatType::FORMAT_ES_HTTP;
171169
}
172170

173171
bool FileScanLocalState::_can_generate_physical_splits(const TQueryOptions& query_options,
@@ -224,6 +222,7 @@ Status FileScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) {
224222
} else {
225223
scan_params = _split_source->get_params();
226224
}
225+
RETURN_IF_ERROR(validate_file_scan_support(*scan_params));
227226
const bool is_load =
228227
state()->desc_tbl().get_tuple_descriptor(scan_params->src_tuple_id) != nullptr;
229228
// TODO: Use scanner v2 for all queries.

‎be/src/exec/rowid_fetcher.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -803,7 +803,6 @@ TFileRangeDesc RowIdStorageReader::build_external_fetch_range(const TFileRangeDe
803803
// needed by virtual columns, and do not mutate the FileMapping shared by other fetches.
804804
auto range = source_range;
805805
range.table_format_params.iceberg_params.__set_delete_files({});
806-
range.table_format_params.transactional_hive_params = TTransactionalHiveDesc {};
807806
return range;
808807
}
809808

‎be/src/exec/scan/file_scan_range_utils.h‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,38 @@
1717

1818
#pragma once
1919

20+
#include "common/status.h"
2021
#include "gen_cpp/PlanNodes_types.h"
2122

2223
namespace doris {
2324

25+
inline Status validate_file_scan_support(const TFileScanRangeParams& params,
26+
const TFileRangeDesc* range = nullptr) {
27+
// Deprecated table markers must never fall through to plain file reads: that would lose
28+
// Hive transaction visibility and LakeSoul merge semantics, including in metadata counts.
29+
auto validate_table = [](const TTableFormatFileDesc& table) -> Status {
30+
if (table.table_format_type == "transactional_hive") {
31+
return Status::NotSupported("Hive transactional table reads are no longer supported");
32+
}
33+
if (table.table_format_type == "lakesoul") {
34+
return Status::NotSupported("LakeSoul reads are no longer supported");
35+
}
36+
return Status::OK();
37+
};
38+
if (params.__isset.table_format_params) {
39+
RETURN_IF_ERROR(validate_table(params.table_format_params));
40+
}
41+
if (range != nullptr && range->__isset.table_format_params) {
42+
RETURN_IF_ERROR(validate_table(range->table_format_params));
43+
}
44+
const auto format = range != nullptr && range->__isset.format_type ? range->format_type
45+
: params.format_type;
46+
if (format == TFileFormatType::FORMAT_AVRO) {
47+
return Status::NotSupported("Avro file reads are no longer supported");
48+
}
49+
return Status::OK();
50+
}
51+
2452
inline bool can_ignore_not_found_file(const TFileRangeDesc& range, bool ignore_not_found) {
2553
// Iceberg ranges belong to a committed snapshot, not a stale directory listing. Skipping a
2654
// missing data or delete file would silently return an incomplete or incorrect snapshot.

‎be/src/exec/scan/file_scanner.cpp‎

Lines changed: 8 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@
6060
#include "exprs/vexpr_fwd.h"
6161
#include "exprs/vslot_ref.h"
6262
#include "format/arrow/arrow_stream_reader.h"
63-
#include "format/avro/avro_jni_reader.h"
6463
#include "format/csv/csv_reader.h"
6564
#include "format/json/new_json_reader.h"
6665
#include "format/orc/vorc_reader.h"
@@ -71,13 +70,11 @@
7170
#include "format/table/iceberg_position_delete_sys_table_reader.h"
7271
#include "format/table/iceberg_reader.h"
7372
#include "format/table/iceberg_sys_table_jni_reader.h"
74-
#include "format/table/lakesoul_jni_reader.h"
7573
#include "format/table/max_compute_jni_reader.h"
7674
#include "format/table/paimon_jni_reader.h"
7775
#include "format/table/paimon_reader.h"
7876
#include "format/table/partition_column_filler.h"
7977
#include "format/table/remote_doris_reader.h"
80-
#include "format/table/transactional_hive_reader.h"
8178
#include "format/table/trino_connector_jni_reader.h"
8279
#include "format/text/text_reader.h"
8380
#include "io/cache/block_file_cache_profile.h"
@@ -145,6 +142,7 @@ FileScanner::FileScanner(RuntimeState* state, FileScanLocalState* local_state, i
145142
}
146143

147144
Status FileScanner::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) {
145+
RETURN_IF_ERROR(validate_file_scan_support(*_params));
148146
RETURN_IF_ERROR(Scanner::init(state, conjuncts));
149147
_get_block_timer =
150148
ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileScannerGetBlockTime", 1);
@@ -1053,6 +1051,7 @@ Status FileScanner::_get_next_reader() {
10531051
}
10541052

10551053
const TFileRangeDesc& range = _current_range;
1054+
RETURN_IF_ERROR(validate_file_scan_support(*_params, &range));
10561055
_current_range_path = range.path;
10571056

10581057
if (!_partition_slot_descs.empty()) {
@@ -1145,12 +1144,6 @@ Status FileScanner::_get_next_reader() {
11451144
range.table_format_params.hudi_params,
11461145
_file_slot_descs, _state, _profile);
11471146
init_status = ((HudiJniReader*)_cur_reader.get())->init_reader();
1148-
} else if (range.__isset.table_format_params &&
1149-
range.table_format_params.table_format_type == "lakesoul") {
1150-
_cur_reader =
1151-
LakeSoulJniReader::create_unique(range.table_format_params.lakesoul_params,
1152-
_file_slot_descs, _state, _profile);
1153-
init_status = ((LakeSoulJniReader*)_cur_reader.get())->init_reader();
11541147
} else if (range.__isset.table_format_params &&
11551148
range.table_format_params.table_format_type == "trino_connector") {
11561149
_cur_reader = TrinoConnectorJniReader::create_unique(_file_slot_descs, _state,
@@ -1267,17 +1260,6 @@ Status FileScanner::_get_next_reader() {
12671260
->init_reader(_col_default_value_ctx, _is_load);
12681261
break;
12691262
}
1270-
case TFileFormatType::FORMAT_AVRO: {
1271-
_cur_reader = AvroJNIReader::create_unique(_state, _profile, *_params, _file_slot_descs,
1272-
range);
1273-
init_status = ((AvroJNIReader*)(_cur_reader.get()))->init_reader();
1274-
// Set col_name_to_block_idx for JNI readers to avoid repeated map creation
1275-
if (_cur_reader) {
1276-
static_cast<JniReader*>(_cur_reader.get())
1277-
->set_col_name_to_block_idx(&_src_block_name_to_idx);
1278-
}
1279-
break;
1280-
}
12811263
case TFileFormatType::FORMAT_WAL: {
12821264
_cur_reader = WalReader::create_unique(_state);
12831265
init_status = ((WalReader*)(_cur_reader.get()))->init_reader(_output_tuple_desc);
@@ -1482,19 +1464,7 @@ Status FileScanner::_init_orc_reader(std::unique_ptr<OrcReader>&& orc_reader,
14821464
Status init_status = Status::OK();
14831465

14841466
if (range.__isset.table_format_params &&
1485-
range.table_format_params.table_format_type == "transactional_hive") {
1486-
std::unique_ptr<TransactionalHiveReader> tran_orc_reader =
1487-
TransactionalHiveReader::create_unique(std::move(orc_reader), _profile, _state,
1488-
*_params, range, _io_ctx.get(),
1489-
file_meta_cache_ptr);
1490-
init_status = tran_orc_reader->init_reader(
1491-
_file_col_names, &_src_block_name_to_idx, _push_down_conjuncts, _real_tuple_desc,
1492-
_default_val_row_desc.get(), &_not_single_slot_filter_conjuncts,
1493-
&_slot_id_to_filter_conjuncts);
1494-
RETURN_IF_ERROR(tran_orc_reader->init_row_filters());
1495-
_cur_reader = std::move(tran_orc_reader);
1496-
} else if (range.__isset.table_format_params &&
1497-
range.table_format_params.table_format_type == "iceberg") {
1467+
range.table_format_params.table_format_type == "iceberg") {
14981468
std::unique_ptr<IcebergOrcReader> iceberg_reader = IcebergOrcReader::create_unique(
14991469
std::move(orc_reader), _profile, _state, *_params, range, _kv_cache, _io_ctx.get(),
15001470
file_meta_cache_ptr);
@@ -1558,8 +1528,8 @@ Status FileScanner::_init_orc_reader(std::unique_ptr<OrcReader>&& orc_reader,
15581528
RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil::by_orc_name(
15591529
_real_tuple_desc, orc_type_ptr, tvf_info_node));
15601530
init_status = orc_reader->init_reader(
1561-
&_file_col_names, &_src_block_name_to_idx, _push_down_conjuncts, false,
1562-
_real_tuple_desc, _default_val_row_desc.get(), &_not_single_slot_filter_conjuncts,
1531+
&_file_col_names, &_src_block_name_to_idx, _push_down_conjuncts, _real_tuple_desc,
1532+
_default_val_row_desc.get(), &_not_single_slot_filter_conjuncts,
15631533
&_slot_id_to_filter_conjuncts, tvf_info_node);
15641534
_cur_reader = std::move(orc_reader);
15651535
} else if (_is_load) {
@@ -1583,8 +1553,8 @@ Status FileScanner::_init_orc_reader(std::unique_ptr<OrcReader>&& orc_reader,
15831553
}
15841554
}
15851555
init_status = orc_reader->init_reader(
1586-
&_file_col_names, &_src_block_name_to_idx, _push_down_conjuncts, false,
1587-
_real_tuple_desc, _default_val_row_desc.get(), &_not_single_slot_filter_conjuncts,
1556+
&_file_col_names, &_src_block_name_to_idx, _push_down_conjuncts, _real_tuple_desc,
1557+
_default_val_row_desc.get(), &_not_single_slot_filter_conjuncts,
15881558
&_slot_id_to_filter_conjuncts, load_info_node);
15891559
_cur_reader = std::move(orc_reader);
15901560
}
@@ -1673,6 +1643,7 @@ Status FileScanner::_generate_truncate_columns(bool need_to_get_parsed_schema) {
16731643
}
16741644

16751645
Status FileScanner::prepare_for_read_lines(const TFileRangeDesc& range) {
1646+
RETURN_IF_ERROR(validate_file_scan_support(*_params, &range));
16761647
_current_range = range;
16771648

16781649
_file_cache_statistics.reset(new io::FileCacheStatistics());

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -395,6 +395,7 @@ bool FileScannerV2::is_supported(const TFileScanRangeParams& params, const TFile
395395

396396
Status FileScannerV2::_validate_scan_range(const TFileScanRangeParams& params,
397397
const TFileRangeDesc& range) {
398+
RETURN_IF_ERROR(validate_file_scan_support(params, &range));
398399
if (!is_supported(params, range)) {
399400
return Status::NotSupported(
400401
"FileScannerV2 does not support table format {} with file format {}",

‎be/src/format/avro/avro_jni_reader.cpp‎

Lines changed: 0 additions & 181 deletions
This file was deleted.

0 commit comments

Comments
 (0)