Skip to content

Commit ca4e916

Browse files
committed
[chore](fe) Merge master and advance the spatial connector API
### What problem does this PR solve? Issue Number: N/A Related PR: apache#69032, apache#68791 Problem Summary: Apache master added ConnectorScanRange split identity and advanced the connector plugin API to 14.0 while this spatial branch independently used 14.0 for ConnectorType metadata. Merge upstream/master at d3772cb, preserve both frozen interface surfaces, and resolve the version-test conflict by advancing the spatial API to 15.0. No spatial behavior or unrelated implementation is changed by the manual resolution. ### Release note Use connector plugin API 15.0 for the development spatial metadata extension on top of master API 14.0. ### Check List (For Author) - Test: Standard FE build and integrated Checkstyle; full connector SPI suite (34 classes) plus 3 spatial DDL/write classes, 185/185 tests with no failures/errors/skips; build hygiene; filtered SPI resource and Iceberg plugin manifest both report API 15.0 - Behavior changed: Yes; connector API major advances from 14.0 to 15.0 to distinguish the combined interface from upstream's existing API 14.0 - Does this need documentation: No; this resolves the existing development feature's interface version and merge conflict; feature documentation remains apache/doris-website#4239
2 parents c844f46 + d3772cb commit ca4e916

278 files changed

Lines changed: 15743 additions & 1217 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build-thirdparty.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,9 @@ jobs:
201201
cd "${GITHUB_WORKSPACE}"
202202
thirdparty/test/libunwind-page-size-test.sh
203203
204+
# Check descriptor ownership when resuming and rotating existing logs.
205+
thirdparty/test/glog-logfile-resume-test.sh
206+
204207
build_macos:
205208
name: Build Third Party Libraries (macOS)
206209
needs: changes

‎be/src/cloud/cloud_delta_writer.cpp‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ Status CloudDeltaWriter::write(const Block* block, const TabletAddRowsPayload& r
9393
ExecEnv::GetInstance()->memtable_memory_limiter()->handle_table_memtable_backpressure(
9494
[this]() {
9595
std::lock_guard lock(_mtx);
96-
return _is_cancelled;
96+
return _is_cancelled || !_get_load_cancel_status().ok();
9797
},
9898
table_id());
9999
}
@@ -115,6 +115,7 @@ Status CloudDeltaWriter::write(const Block* block, const TabletAddRowsPayload& r
115115
return _memtable_writer->flush_running_count() >= effective_flush_running_count_limit;
116116
};
117117
while (need_backpressure()) {
118+
RETURN_IF_ERROR(_get_load_cancel_status());
118119
std::this_thread::sleep_for(std::chrono::milliseconds(10));
119120
}
120121
}
@@ -154,6 +155,7 @@ void CloudDeltaWriter::update_tablet_stats() {
154155
Status CloudDeltaWriter::commit_rowset() {
155156
g_cloud_commit_rowset_count << 1;
156157
std::lock_guard<bthread::Mutex> lock(_mtx);
158+
RETURN_IF_ERROR(_get_load_cancel_status());
157159

158160
// Handle empty rowset (no data written)
159161
if (!_is_init) {
@@ -173,6 +175,7 @@ Status CloudDeltaWriter::_commit_empty_rowset() {
173175

174176
RETURN_IF_ERROR(_rowset_builder->init());
175177
RETURN_IF_ERROR(_rowset_builder->build_rowset());
178+
RETURN_IF_ERROR(_get_load_cancel_status());
176179

177180
// If skip writing empty rowset metadata is enabled, we do not commit rowset to meta service.
178181
if (config::skip_writing_empty_rowset_metadata) {
@@ -183,6 +186,7 @@ Status CloudDeltaWriter::_commit_empty_rowset() {
183186
}
184187

185188
Status CloudDeltaWriter::set_txn_related_info() {
189+
RETURN_IF_ERROR(_get_load_cancel_status());
186190
return rowset_builder()->set_txn_related_info();
187191
}
188192

‎be/src/cloud/cloud_rowset_builder.cpp‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "cloud/cloud_storage_engine.h"
2424
#include "cloud/cloud_tablet.h"
2525
#include "cloud/cloud_tablet_mgr.h"
26+
#include "cpp/sync_point.h"
2627
#include "io/fs/file_system.h"
2728
#include "storage/rowset/group_rowset_writer.h"
2829
#include "storage/rowset/rowset_factory.h"
@@ -53,6 +54,8 @@ CloudGroupRowsetBuilder::CloudGroupRowsetBuilder(CloudStorageEngine& engine,
5354
CloudRowsetBuilder::~CloudRowsetBuilder() {
5455
// Clear file cache immediately when load fails
5556
if (_is_init && _rowset != nullptr && _rowset->rowset_meta()->rowset_state() == PREPARED) {
57+
TEST_SYNC_POINT_CALLBACK("CloudRowsetBuilder::~CloudRowsetBuilder:before_clear_cache",
58+
this);
5659
_rowset->clear_cache();
5760
}
5861
}
@@ -89,6 +92,7 @@ Status CloudRowsetBuilder::init() {
8992
context.txn_id = _req.txn_id;
9093
context.txn_expiration = _req.txn_expiration;
9194
context.load_id = _req.load_id;
95+
context.load_cancel_status = _req.load_cancel_status;
9296
context.db_id = _req.table_schema_param->db_id();
9397
context.table_id = _req.table_schema_param->table_id();
9498
context.rowset_state = PREPARED;
@@ -117,7 +121,8 @@ Status CloudRowsetBuilder::init() {
117121
_rowset_writer = DORIS_TRY(_tablet->create_rowset_writer(context, false));
118122
_rowset_id = context.rowset_id;
119123

120-
_calc_delete_bitmap_token = _engine.calc_delete_bitmap_executor()->create_token();
124+
_calc_delete_bitmap_token =
125+
_engine.calc_delete_bitmap_executor()->create_token(_req.load_cancel_status);
121126

122127
if (!_skip_writing_rowset_metadata) {
123128
RETURN_IF_ERROR(_engine.meta_mgr().prepare_rowset(*_rowset_writer->rowset_meta(), "",
@@ -172,6 +177,12 @@ Status CloudGroupRowsetBuilder::wait_calc_delete_bitmap() {
172177
return _data_builder->wait_calc_delete_bitmap();
173178
}
174179

180+
Status CloudGroupRowsetBuilder::cancel(const Status& st) {
181+
RETURN_IF_ERROR(_data_builder->cancel(st));
182+
RETURN_IF_ERROR(_row_binlog_builder->cancel(st));
183+
return BaseRowsetBuilder::cancel(st);
184+
}
185+
175186
void CloudGroupRowsetBuilder::update_tablet_stats() {
176187
_data_builder->update_tablet_stats();
177188
_row_binlog_builder->update_tablet_stats();

‎be/src/cloud/cloud_rowset_builder.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@ class CloudGroupRowsetBuilder final : public CloudRowsetBuilder {
7474

7575
Status wait_calc_delete_bitmap() override;
7676

77+
Status cancel(const Status& st) override;
78+
7779
void update_tablet_stats() override;
7880

7981
Status commit_rowset(const std::string& job_id, int64_t table_id) override;

‎be/src/cloud/cloud_rowset_writer.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,8 @@ Status CloudRowsetWriter::init(const RowsetWriterContext& rowset_writer_context)
8989
_context.segment_collector = std::make_shared<SegmentCollectorT<BaseBetaRowsetWriter>>(this);
9090
_context.file_writer_creator = std::make_shared<FileWriterCreatorT<BaseBetaRowsetWriter>>(this);
9191
if (_context.mow_context != nullptr) {
92-
_calc_delete_bitmap_token = _engine.calc_delete_bitmap_executor_for_load()->create_token();
92+
_calc_delete_bitmap_token = _engine.calc_delete_bitmap_executor_for_load()->create_token(
93+
_context.load_cancel_status);
9394
}
9495
return Status::OK();
9596
}

‎be/src/cloud/cloud_tablets_channel.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,7 @@ Status CloudTabletsChannel::_close_writers(PTabletWriterAddBlockResult* res) {
205205
bool success = true;
206206

207207
for (auto&& [tablet_id, base_writer] : _tablet_writers) {
208+
RETURN_IF_ERROR(_check_cancelled());
208209
auto* writer = static_cast<CloudDeltaWriter*>(base_writer.get());
209210
// ATTN: the strict mode means strict filtering of column type conversions during import.
210211
// Sometimes all inputs are filtered, but the partition ID is still set, and the writer is

‎be/src/exec/operator/aggregation_sink_operator.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -163,14 +163,15 @@ Status AggSinkLocalState::open(RuntimeState* state) {
163163
// instead of storing the full aggregate state, saving memory and computation overhead.
164164
// Requirements:
165165
// 0. The aggregation has a GROUP BY clause.
166-
// 1. There is exactly one count aggregate function.
166+
// 1. There is exactly one count aggregate function, and its arguments need no evaluation
167+
// (slots or literals only).
167168
// 2. No limit optimization is applied.
168169
// 3. Spill is not enabled (the spill path accesses aggregate_data_container, which is empty in inline count mode).
169170
// Supports update / merge / finalize / serialize phases, since count's serialization format is UInt64 itself.
170171

171172
if (!Base::_shared_state->probe_expr_ctxs.empty() /* has GROUP BY */
172173
&& (p._aggregate_evaluators.size() == 1 &&
173-
p._aggregate_evaluators[0]->is_simple_count()) /* only one count(*) */
174+
p._aggregate_evaluators[0]->is_simple_count()) /* only one count(*) / count(slot) */
174175
&& !_should_limit_output /* no limit optimization */ &&
175176
!Base::_shared_state->enable_spill /* spill not enabled */) {
176177
_shared_state->use_simple_count = true;

‎be/src/exec/operator/bucketed_aggregation_sink_operator.cpp‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
#include "exec/operator/bucketed_aggregation_sink_operator.h"
1919

20+
#include <algorithm>
2021
#include <memory>
2122
#include <string>
2223

@@ -87,7 +88,7 @@ Status BucketedAggSinkLocalState::open(RuntimeState* state) {
8788
shared_state.aggregate_evaluators.push_back(evaluator->clone(state, p._pool));
8889
}
8990

90-
// Detect simple_count: exactly one COUNT(*) with no args, with GROUP BY present.
91+
// Detect simple_count: exactly one COUNT(*) / COUNT(slot), with GROUP BY present.
9192
// Bucketed agg always has GROUP BY (without-key not supported).
9293
if (p._aggregate_evaluators.size() == 1 && p._aggregate_evaluators[0]->is_simple_count()) {
9394
shared_state.use_simple_count = true;
@@ -386,6 +387,10 @@ Status BucketedAggSinkLocalState::close(RuntimeState* state, Status exec_status)
386387
return Base::close(state, exec_status);
387388
}
388389

390+
bool BucketedAggSinkLocalState::is_blockable() const {
391+
return Base::_parent->template cast<BucketedAggSinkOperatorX>().has_blockable_aggregate();
392+
}
393+
389394
// ============ BucketedAggSinkOperatorX ============
390395

391396
BucketedAggSinkOperatorX::BucketedAggSinkOperatorX(ObjectPool* pool, int operator_id, int dest_id,
@@ -395,6 +400,11 @@ BucketedAggSinkOperatorX::BucketedAggSinkOperatorX(ObjectPool* pool, int operato
395400
_tuple_id(tnode.bucketed_agg_node.tuple_id),
396401
_pool(pool) {}
397402

403+
bool BucketedAggSinkOperatorX::has_blockable_aggregate() const {
404+
return std::any_of(_aggregate_evaluators.begin(), _aggregate_evaluators.end(),
405+
[](const AggFnEvaluator* evaluator) { return evaluator->is_blockable(); });
406+
}
407+
398408
Status BucketedAggSinkOperatorX::init(const TPlanNode& tnode, RuntimeState* state) {
399409
RETURN_IF_ERROR(DataSinkOperatorX<BucketedAggSinkLocalState>::init(tnode, state));
400410

‎be/src/exec/operator/bucketed_aggregation_sink_operator.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ class BucketedAggSinkLocalState : public PipelineXSinkLocalState<BucketedAggShar
4444

4545
size_t get_reserve_mem_size(RuntimeState* state, bool eos) const;
4646

47+
bool is_blockable() const override;
48+
4749
private:
4850
friend class BucketedAggSinkOperatorX;
4951

@@ -120,6 +122,13 @@ class BucketedAggSinkOperatorX final : public DataSinkOperatorX<BucketedAggSinkL
120122

121123
size_t get_reserve_mem_size(RuntimeState* state, bool eos) override;
122124

125+
// Whether an aggregate function (or one of its arguments) may block, e.g. AI_AGG sends a
126+
// synchronous HTTP request. Both the sink (update) and the source (merge / finalize) run
127+
// these functions, so both tasks must go to the blocking scheduler.
128+
// Only valid after prepare(). Reads the operator's own evaluators, which never change
129+
// after prepare(), so it is safe to call before the local states are opened.
130+
bool has_blockable_aggregate() const;
131+
123132
using DataSinkOperatorX<BucketedAggSinkLocalState>::node_id;
124133
using DataSinkOperatorX<BucketedAggSinkLocalState>::operator_id;
125134
using DataSinkOperatorX<BucketedAggSinkLocalState>::get_local_state;

‎be/src/exec/operator/bucketed_aggregation_source_operator.cpp‎

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,12 @@
2525
#include "exec/common/agg_utils.h"
2626
#include "exec/common/hash_table/hash.h"
2727
#include "exec/common/util.hpp"
28+
#include "exec/operator/bucketed_aggregation_sink_operator.h"
2829
#include "exec/operator/operator.h"
2930
#include "exprs/vectorized_agg_fn.h"
3031
#include "runtime/runtime_profile.h"
3132
#include "runtime/thread_context.h"
33+
#include "util/debug_points.h"
3234

3335
namespace doris {
3436

@@ -158,10 +160,21 @@ Status BucketedAggLocalState::init(RuntimeState* state, LocalStateInfo& info) {
158160
_insert_keys_to_column_timer = ADD_TIMER(custom_profile(), "InsertKeysToColumnTime");
159161
_insert_values_to_column_timer = ADD_TIMER(custom_profile(), "InsertValuesToColumnTime");
160162
_merge_timer = ADD_TIMER(custom_profile(), "MergeTime");
163+
_memory_usage_merge_arena =
164+
ADD_COUNTER(custom_profile(), "MemoryUsageMergeArena", TUnit::BYTES);
165+
_memory_usage_merged_hash_tables =
166+
ADD_COUNTER(custom_profile(), "MemoryUsageMergedHashTables", TUnit::BYTES);
161167

162168
return Status::OK();
163169
}
164170

171+
void BucketedAggLocalState::_update_memusage(Arena& merge_arena) {
172+
int64_t arena_memory_usage = merge_arena.size();
173+
COUNTER_SET(_memory_usage_merge_arena, arena_memory_usage);
174+
COUNTER_SET(_memory_usage_merged_hash_tables, _hash_table_merge_growth);
175+
COUNTER_SET(_memory_used_counter, arena_memory_usage + _hash_table_merge_growth);
176+
}
177+
165178
Status BucketedAggLocalState::close(RuntimeState* state) {
166179
SCOPED_TIMER(exec_time_counter());
167180
SCOPED_TIMER(_close_timer);
@@ -222,6 +235,7 @@ int BucketedAggLocalState::_merge_bucket(int bucket, int merge_target) {
222235
[&](auto& dst_method) -> void {
223236
using AggMethodType = std::decay_t<decltype(dst_method)>;
224237
auto& dst_data = *dst_method.hash_table;
238+
const int64_t dst_bytes_before = dst_data.get_buffer_size_in_bytes();
225239

226240
// Merge all finished sink instances (except merge_target itself)
227241
// into the merge target's bucket.
@@ -264,15 +278,17 @@ int BucketedAggLocalState::_merge_bucket(int bucket, int merge_target) {
264278
// allocating intermediate vectors (keys,
265279
// mappeds, hashes) and eliminates the separate
266280
// null-out traversal.
281+
// The source slot is cleared only after the
282+
// state is owned by dst: emplace and merge can
283+
// throw, and then shared-state cleanup must
284+
// still find and destroy the source state.
267285
const bool use_simple_count =
268286
shared_state.use_simple_count;
269287
src_data.for_each([&](const auto& key,
270288
auto& mapped) {
271289
if (!mapped) {
272290
return;
273291
}
274-
auto src_mapped = mapped;
275-
mapped = nullptr;
276292

277293
typename std::remove_reference_t<
278294
decltype(dst_data)>::LookupResult
@@ -283,20 +299,31 @@ int BucketedAggLocalState::_merge_bucket(int bucket, int merge_target) {
283299

284300
if (inserted) {
285301
*::lookup_result_get_mapped(dst_it) =
286-
src_mapped;
302+
mapped;
287303
} else {
288304
auto& dst_mapped =
289305
*::lookup_result_get_mapped(
290306
dst_it);
307+
DBUG_EXECUTE_IF(
308+
"BucketedAggLocalState._merge_"
309+
"bucket.fail_merge",
310+
{
311+
throw Exception(
312+
ErrorCode::
313+
INTERNAL_ERROR,
314+
"injected merge "
315+
"failure");
316+
});
291317
merge_agg_states(
292-
dst_mapped, src_mapped,
318+
dst_mapped, mapped,
293319
use_simple_count,
294320
shared_state
295321
.aggregate_evaluators,
296322
shared_state
297323
.offsets_of_aggregate_states,
298324
merge_arena);
299325
}
326+
mapped = nullptr;
300327
});
301328

302329
merge_null_key(
@@ -312,9 +339,16 @@ int BucketedAggLocalState::_merge_bucket(int bucket, int merge_target) {
312339
}},
313340
src_agg_data.method_variant);
314341
}
342+
// Emplacing the source entries can rehash the destination bucket.
343+
_hash_table_merge_growth +=
344+
static_cast<int64_t>(dst_data.get_buffer_size_in_bytes()) -
345+
dst_bytes_before;
315346
}},
316347
dst_agg_data.method_variant);
317348

349+
if (merged_count > 0) {
350+
_update_memusage(merge_arena);
351+
}
318352
return merged_count;
319353
}
320354

@@ -733,9 +767,18 @@ BucketedAggSourceOperatorX::BucketedAggSourceOperatorX(ObjectPool* pool, const T
733767
: Base(pool, tnode, operator_id, descs),
734768
_needs_finalize(tnode.bucketed_agg_node.need_finalize) {}
735769

770+
bool BucketedAggSourceOperatorX::is_blockable(RuntimeState* state) const {
771+
return Base::is_blockable(state) ||
772+
(_sink_operator != nullptr && _sink_operator->has_blockable_aggregate());
773+
}
774+
736775
Status BucketedAggSourceOperatorX::get_block_impl(RuntimeState* state, Block* block, bool* eos) {
737776
auto& local_state = get_local_state(state);
738777
SCOPED_TIMER(local_state.exec_time_counter());
778+
// Merging finished sinks (hash table rehash, aggregate merge() allocations) happens
779+
// inside _get_results, so let the pipeline task reserve what the last round needed
780+
// instead of only the minimum operator memory (see OperatorX::get_reserve_mem_size).
781+
SCOPED_PEAK_MEM(&local_state.estimate_memory_usage());
739782

740783
RETURN_IF_ERROR(local_state._get_results(state, block, eos));
741784

0 commit comments

Comments
 (0)