Skip to content

Commit 735b4e9

Browse files
kaka11chenHappenLee
authored andcommitted
[Enhancement](hive-writer) Adjust table sink exchange rebalancer params. (apache#33397)
Issue Number: apache#31442 Change table sink exchange rebalancer params to node level and adjust these params to improve write performance by better balance. rebalancer params: ``` DEFINE_mInt64(table_sink_partition_write_min_data_processed_rebalance_threshold, "26214400"); // 25MB // Minimum partition data processed to rebalance writers in exchange when partition writing DEFINE_mInt64(table_sink_partition_write_min_partition_data_processed_rebalance_threshold, "15728640"); // 15MB ```
1 parent a1739a1 commit 735b4e9

5 files changed

Lines changed: 30 additions & 17 deletions

File tree

‎be/src/common/config.cpp‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1181,14 +1181,15 @@ DEFINE_mString(ca_cert_file_paths,
11811181
"/etc/ssl/ca-bundle.pem");
11821182

11831183
/** Table sink configurations(currently contains only external table types) **/
1184-
// Minimum data processed to scale writers when non partition writing
1184+
// Minimum data processed to scale writers in exchange when non partition writing
11851185
DEFINE_mInt64(table_sink_non_partition_write_scaling_data_processed_threshold,
1186-
"125829120"); // 120MB
1187-
// Minimum data processed to start rebalancing in exchange when partition writing
1188-
DEFINE_mInt64(table_sink_partition_write_data_processed_threshold, "209715200"); // 200MB
1186+
"26214400"); // 25MB
11891187
// Minimum data processed to trigger skewed partition rebalancing in exchange when partition writing
1190-
DEFINE_mInt64(table_sink_partition_write_skewed_data_processed_rebalance_threshold,
1191-
"209715200"); // 200MB
1188+
DEFINE_mInt64(table_sink_partition_write_min_data_processed_rebalance_threshold,
1189+
"26214400"); // 25MB
1190+
// Minimum partition data processed to rebalance writers in exchange when partition writing
1191+
DEFINE_mInt64(table_sink_partition_write_min_partition_data_processed_rebalance_threshold,
1192+
"15728640"); // 15MB
11921193
// Maximum processed partition nums of per writer when partition writing
11931194
DEFINE_mInt32(table_sink_partition_write_max_partition_nums_per_writer, "128");
11941195

‎be/src/common/config.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1265,12 +1265,12 @@ DECLARE_String(trino_connector_plugin_dir);
12651265
DECLARE_mString(ca_cert_file_paths);
12661266

12671267
/** Table sink configurations(currently contains only external table types) **/
1268-
// Minimum data processed to scale writers when non partition writing
1268+
// Minimum data processed to scale writers in exchange when non partition writing
12691269
DECLARE_mInt64(table_sink_non_partition_write_scaling_data_processed_threshold);
1270-
// Minimum data processed to start rebalancing in exchange when partition writing
1271-
DECLARE_mInt64(table_sink_partition_write_data_processed_threshold);
12721270
// Minimum data processed to trigger skewed partition rebalancing in exchange when partition writing
1273-
DECLARE_mInt64(table_sink_partition_write_skewed_data_processed_rebalance_threshold);
1271+
DECLARE_mInt64(table_sink_partition_write_min_data_processed_rebalance_threshold);
1272+
// Minimum partition data processed to rebalance writers in exchange when partition writing
1273+
DECLARE_mInt64(table_sink_partition_write_min_partition_data_processed_rebalance_threshold);
12741274
// Maximum processed partition nums of per writer when partition writing
12751275
DECLARE_mInt32(table_sink_partition_write_max_partition_nums_per_writer);
12761276

‎be/src/pipeline/exec/exchange_sink_operator.cpp‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -257,17 +257,23 @@ Status ExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& inf
257257
_partitioner.reset(
258258
new vectorized::Crc32HashPartitioner<LocalExchangeChannelIds>(_partition_count));
259259
_partition_function.reset(new HashPartitionFunction(_partitioner.get()));
260-
// const long MEGABYTE = 1024 * 1024;
261-
// const long MIN_PARTITION_DATA_PROCESSED_REBALANCE_THRESHOLD = 10000 * MEGABYTE; // 1MB
262-
// const long MIN_DATA_PROCESSED_REBALANCE_THRESHOLD = 50000 * MEGABYTE; // 50MB
263260

264-
// const long MIN_PARTITION_DATA_PROCESSED_REBALANCE_THRESHOLD = 1; // 1MB
265-
// const long MIN_DATA_PROCESSED_REBALANCE_THRESHOLD = 1; // 50MB
266261
scale_writer_partitioning_exchanger.reset(new vectorized::ScaleWriterPartitioningExchanger<
267262
HashPartitionFunction>(
268263
channels.size(), *_partition_function, _partition_count, channels.size(), 1,
269-
config::table_sink_partition_write_data_processed_threshold,
270-
config::table_sink_partition_write_skewed_data_processed_rebalance_threshold));
264+
config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold /
265+
state->task_num() ==
266+
0
267+
? config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold
268+
: config::table_sink_partition_write_min_partition_data_processed_rebalance_threshold /
269+
state->task_num(),
270+
config::table_sink_partition_write_min_data_processed_rebalance_threshold /
271+
state->task_num() ==
272+
0
273+
? config::table_sink_partition_write_min_data_processed_rebalance_threshold
274+
: config::table_sink_partition_write_min_data_processed_rebalance_threshold /
275+
state->task_num()));
276+
271277
RETURN_IF_ERROR(_partitioner->init(p._texprs));
272278
RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc));
273279
_profile->add_info_string("Partitioner",

‎be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -593,6 +593,7 @@ Status PipelineXFragmentContext::_build_pipeline_tasks(
593593
init_runtime_state(task_runtime_state);
594594
auto cur_task_id = _total_tasks++;
595595
task_runtime_state->set_task_id(cur_task_id);
596+
task_runtime_state->set_task_num(pipeline->num_tasks());
596597
auto task = std::make_unique<PipelineXTask>(
597598
pipeline, cur_task_id, get_task_runtime_state(cur_task_id), this,
598599
pipeline_id_to_profile[pip_idx].get(), get_local_exchange_state(pipeline),

‎be/src/runtime/runtime_state.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -619,6 +619,10 @@ class RuntimeState {
619619

620620
int task_id() const { return _task_id; }
621621

622+
void set_task_num(int task_num) { _task_num = task_num; }
623+
624+
int task_num() const { return _task_num; }
625+
622626
private:
623627
Status create_error_log_file();
624628

@@ -729,6 +733,7 @@ class RuntimeState {
729733
std::vector<TErrorTabletInfo> _error_tablet_infos;
730734
int _max_operator_id = 0;
731735
int _task_id = -1;
736+
int _task_num = 0;
732737

733738
std::vector<THivePartitionUpdate> _hive_partition_updates;
734739

0 commit comments

Comments
 (0)