Skip to content

Commit fc38b7b

Browse files
committed
[Improvement](agg) Add a knob to control local exchange
1 parent bcfcef3 commit fc38b7b

7 files changed

Lines changed: 20 additions & 8 deletions

File tree

‎be/src/exec/operator/aggregation_sink_operator.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,9 @@ class AggSinkOperatorX MOCK_REMOVE(final) : public DataSinkOperatorX<AggSinkLoca
163163
: DataSinkOperatorX<AggSinkLocalState>::required_data_distribution(
164164
state);
165165
}
166+
if (!_needs_finalize && !state->enable_local_exchange_before_agg()) {
167+
return DataSinkOperatorX<AggSinkLocalState>::required_data_distribution(state);
168+
}
166169
return _is_colocate && _require_bucket_distribution
167170
? DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE, _partition_exprs)
168171
: DataDistribution(ExchangeType::HASH_SHUFFLE, _partition_exprs);

‎be/src/exec/operator/distinct_streaming_aggregation_operator.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,10 @@ class DistinctStreamingAggOperatorX final
118118
if (_needs_finalize && _probe_expr_ctxs.empty()) {
119119
return {ExchangeType::NOOP};
120120
}
121+
if (!_needs_finalize && !state->enable_local_exchange_before_agg()) {
122+
return StatefulOperatorX<DistinctStreamingAggLocalState>::required_data_distribution(
123+
state);
124+
}
121125
if (_needs_finalize || (!_probe_expr_ctxs.empty() && !_is_streaming_preagg)) {
122126
return _is_colocate && _require_bucket_distribution
123127
? DataDistribution(ExchangeType::BUCKET_HASH_SHUFFLE, _partition_exprs)

‎be/src/exec/operator/streaming_aggregation_operator.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -226,8 +226,7 @@ class StreamingAggOperatorX MOCK_REMOVE(final) : public StatefulOperatorX<Stream
226226
state->enable_streaming_agg_hash_join_force_passthrough()) {
227227
return DataDistribution(ExchangeType::PASSTHROUGH);
228228
}
229-
if (!state->get_query_ctx()->should_be_shuffled_agg(
230-
StatefulOperatorX<StreamingAggLocalState>::node_id())) {
229+
if (!_needs_finalize && !state->enable_local_exchange_before_agg()) {
231230
return StatefulOperatorX<StreamingAggLocalState>::required_data_distribution(state);
232231
}
233232
if (_partition_exprs.empty()) {

‎be/src/runtime/query_context.h‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -190,12 +190,6 @@ class QueryContext : public std::enable_shared_from_this<QueryContext> {
190190
return _query_options.__isset.enable_force_spill && _query_options.enable_force_spill;
191191
}
192192
const TQueryOptions& query_options() const { return _query_options; }
193-
bool should_be_shuffled_agg(int node_id) const {
194-
return _query_options.__isset.shuffled_agg_ids &&
195-
std::any_of(_query_options.shuffled_agg_ids.begin(),
196-
_query_options.shuffled_agg_ids.end(),
197-
[&](const int id) -> bool { return id == node_id; });
198-
}
199193

200194
// global runtime filter mgr, the runtime filter have remote target or
201195
// need local merge should regist here. before publish() or push_to_remote()

‎be/src/runtime/runtime_state.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -558,6 +558,11 @@ class RuntimeState {
558558
_query_options.enable_streaming_agg_hash_join_force_passthrough;
559559
}
560560

561+
bool enable_local_exchange_before_agg() const {
562+
return _query_options.__isset.enable_local_exchange_before_agg &&
563+
_query_options.enable_local_exchange_before_agg;
564+
}
565+
561566
bool enable_distinct_streaming_agg_force_passthrough() const {
562567
return _query_options.__isset.enable_distinct_streaming_agg_force_passthrough &&
563568
_query_options.enable_distinct_streaming_agg_force_passthrough;

‎fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ public class SessionVariable implements Serializable, Writable {
171171
public static final String ENABLE_DISTINCT_STREAMING_AGG_FORCE_PASSTHROUGH =
172172
"enable_distinct_streaming_agg_force_passthrough";
173173
public static final String ENABLE_BROADCAST_JOIN_FORCE_PASSTHROUGH = "enable_broadcast_join_force_passthrough";
174+
public static final String ENABLE_LOCAL_EXCHANGE_BEFORE_AGG = "enable_local_exchange_before_agg";
174175
public static final String DISABLE_COLOCATE_PLAN = "disable_colocate_plan";
175176
public static final String COLOCATE_MAX_PARALLEL_NUM = "colocate_max_parallel_num";
176177
public static final String ENABLE_BUCKET_SHUFFLE_JOIN = "enable_bucket_shuffle_join";
@@ -1300,6 +1301,9 @@ public void checkQuerySlotCount(String slotCnt) {
13001301
@VarAttrDef.VarAttr(name = ENABLE_STREAMING_AGG_HASH_JOIN_FORCE_PASSTHROUGH, fuzzy = true)
13011302
public boolean enableStreamingAggHashJoinForcePassthrough = true;
13021303

1304+
@VarAttrDef.VarAttr(name = ENABLE_LOCAL_EXCHANGE_BEFORE_AGG, fuzzy = true)
1305+
public boolean enableLocalExchangeBeforeAgg = true;
1306+
13031307
@VarAttrDef.VarAttr(name = ENABLE_DISTINCT_STREAMING_AGG_FORCE_PASSTHROUGH, fuzzy = true)
13041308
public boolean enableDistinctStreamingAggForcePassthrough = true;
13051309

@@ -3615,6 +3619,7 @@ public void initFuzzyModeVariables() {
36153619
this.enableCommonExpPushDownForInvertedIndex = random.nextBoolean();
36163620
this.disableStreamPreaggregations = random.nextBoolean();
36173621
this.enableStreamingAggHashJoinForcePassthrough = random.nextBoolean();
3622+
this.enableLocalExchangeBeforeAgg = random.nextBoolean();
36183623
this.enableDistinctStreamingAggForcePassthrough = random.nextBoolean();
36193624
this.enableBroadcastJoinForcePassthrough = random.nextBoolean();
36203625
this.enableShareHashTableForBroadcastJoin = random.nextBoolean();
@@ -5350,6 +5355,7 @@ public TQueryOptions toThrift() {
53505355
tResult.setDisableStreamPreaggregations(disableStreamPreaggregations);
53515356
tResult.setEnableDistinctStreamingAggregation(enableDistinctStreamingAggregation);
53525357
tResult.setEnableStreamingAggHashJoinForcePassthrough(enableStreamingAggHashJoinForcePassthrough);
5358+
tResult.setEnableLocalExchangeBeforeAgg(enableLocalExchangeBeforeAgg);
53535359
tResult.setEnableDistinctStreamingAggForcePassthrough(enableDistinctStreamingAggForcePassthrough);
53545360
tResult.setEnableBroadcastJoinForcePassthrough(enableBroadcastJoinForcePassthrough);
53555361
tResult.setPartitionTopnMaxPartitions(partitionTopNMaxPartitions);

‎gensrc/thrift/PaloInternalService.thrift‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,7 @@ struct TQueryOptions {
473473

474474
210: optional double max_scan_mem_ratio = 0.3;
475475
211: optional bool enable_adaptive_scan = false;
476+
212: optional bool enable_local_exchange_before_agg = true;
476477
// For cloud, to control if the content would be written into file cache
477478
// In write path, to control if the content would be written into file cache.
478479
// In read path, read from file cache or remote storage when execute query.

0 commit comments

Comments
 (0)