Skip to content

Commit 457bc77

Browse files
committed
[refactor](external) Remove Elasticsearch HTTP scans
### What problem does this PR solve? Related PR: #69601 Remove the Elasticsearch HTTP scan implementation and its FE connector, ES-only functions, HTTP proxy endpoints, test services, and regression fixtures. Reject retired formats before reader creation or metadata-only counts. Keep the old catalog metadata readable during image loading and journal replay, but reject catalog creation and access even when an obsolete plugin remains. Preserve all Thrift definitions and numeric identities. Accept the retired ES scroll session variable as a compatibility no-op for old clients. ### Release note Elasticsearch HTTP scans and the esquery function are no longer supported. Existing Elasticsearch metadata can still load during an upgrade. ### Check List (For Author) - Test: Focused FE unit and upgrade tests, FE Checkstyle, clang-format 16, BE syntax-only compilation, build hygiene and shell syntax checks. - Behavior changed: Yes, remove Elasticsearch reads. - Does this need documentation: No; removal is described in the release note.
1 parent e0ab4f9 commit 457bc77

154 files changed

Lines changed: 182 additions & 14773 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/common/config.cpp‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -849,12 +849,6 @@ DEFINE_mInt32(max_memory_sink_batch_count, "20");
849849
// note: unit is minute, default is 5min
850850
DEFINE_mInt32(scan_context_gc_interval_min, "5");
851851

852-
// es scroll keep-alive
853-
DEFINE_String(es_scroll_keepalive, "5m");
854-
855-
// HTTP connection timeout for es
856-
DEFINE_mInt32(es_http_timeout_ms, "5000");
857-
858852
// the max client cache number per each host
859853
// There are variety of client cache in BE, but currently we use the
860854
// same cache size configuration.

‎be/src/common/config.h‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -916,12 +916,6 @@ DECLARE_mInt32(max_memory_sink_batch_count);
916916
// note: unit is minute, default is 5min
917917
DECLARE_mInt32(scan_context_gc_interval_min);
918918

919-
// es scroll keep-alive
920-
DECLARE_String(es_scroll_keepalive);
921-
922-
// HTTP connection timeout for es
923-
DECLARE_mInt32(es_http_timeout_ms);
924-
925919
// the max client cache number per each host
926920
// There are variety of client cache in BE, but currently we use the
927921
// same cache size configuration.

‎be/src/exec/operator/file_scan_operator.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,6 @@ bool FileScanLocalState::should_use_file_scanner_v2(const TQueryOptions& query_o
222222
// after scanner construction. Keep versioned Iceberg scans on V2 without eagerly listing
223223
// every file.
224224
return scanner_v2_requested && !is_load &&
225-
scan_params.format_type != TFileFormatType::FORMAT_ES_HTTP &&
226225
scan_params.format_type != TFileFormatType::FORMAT_LANCE;
227226
}
228227

‎be/src/exec/scan/file_scan_range_utils.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ inline Status validate_file_scan_support(const TFileScanRangeParams& params,
3333
if (table.table_format_type == "lakesoul") {
3434
return Status::NotSupported("LakeSoul reads are no longer supported");
3535
}
36+
if (table.table_format_type == "es") {
37+
return Status::NotSupported("Elasticsearch reads are no longer supported");
38+
}
3639
return Status::OK();
3740
};
3841
if (params.__isset.table_format_params) {
@@ -41,6 +44,12 @@ inline Status validate_file_scan_support(const TFileScanRangeParams& params,
4144
if (range != nullptr && range->__isset.table_format_params) {
4245
RETURN_IF_ERROR(validate_table(range->table_format_params));
4346
}
47+
// Old FE plans may still carry ES at either level; reject them before metadata-only counts.
48+
if (params.format_type == TFileFormatType::FORMAT_ES_HTTP ||
49+
(range != nullptr && range->__isset.format_type &&
50+
range->format_type == TFileFormatType::FORMAT_ES_HTTP)) {
51+
return Status::NotSupported("Elasticsearch reads are no longer supported");
52+
}
4453
const auto format = range != nullptr && range->__isset.format_type ? range->format_type
4554
: params.format_type;
4655
if (format == TFileFormatType::FORMAT_AVRO) {

‎be/src/exec/scan/file_scanner.cpp‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@
6666
#include "format/json/new_json_reader.h"
6767
#include "format/orc/vorc_reader.h"
6868
#include "format/parquet/vparquet_reader.h"
69-
#include "format/table/es/es_http_reader.h"
7069
#include "format/table/hive_reader.h"
7170
#include "format/table/hudi_jni_reader.h"
7271
#include "format/table/hudi_reader.h"
@@ -1281,12 +1280,6 @@ Status FileScanner::_get_next_reader() {
12811280
}
12821281
break;
12831282
}
1284-
case TFileFormatType::FORMAT_ES_HTTP: {
1285-
_cur_reader = EsHttpReader::create_unique(_file_slot_descs, _state, _profile, range,
1286-
*_params, _real_tuple_desc);
1287-
init_status = static_cast<EsHttpReader*>(_cur_reader.get())->init_reader();
1288-
break;
1289-
}
12901283
default:
12911284
return Status::NotSupported("Not supported create reader for file format: {}.",
12921285
to_string(_params->format_type));

‎be/src/exprs/function/function_fake.cpp‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -173,14 +173,6 @@ struct FunctionJsonEachText {
173173
static std::string get_error_msg() { return "Fake function do not support execute"; }
174174
};
175175

176-
struct FunctionEsquery {
177-
static DataTypePtr get_return_type_impl(const DataTypes& arguments) {
178-
return FunctionFakeBaseImpl<DataTypeUInt8>::get_return_type_impl(arguments);
179-
}
180-
static DataTypes get_variadic_argument_types() { return {}; }
181-
static std::string get_error_msg() { return "esquery only supported on es table"; }
182-
};
183-
184176
class FunctionStack : public FunctionFake<UDTFImpl> {
185177
public:
186178
static FunctionPtr create() { return std::make_shared<FunctionStack>(); }
@@ -262,7 +254,6 @@ void register_table_function_with_impl(SimpleFunctionFactory& factory, const std
262254
};
263255

264256
void register_function_fake(SimpleFunctionFactory& factory) {
265-
register_function<FunctionEsquery>(factory, "esquery");
266257
factory.register_function<FunctionStack>("stack");
267258

268259
register_table_function_expand_outer<FunctionExplodeV2>(factory, "explode");

‎be/src/format/table/es/es_http_reader.cpp‎

Lines changed: 0 additions & 256 deletions
This file was deleted.

0 commit comments

Comments
 (0)