Skip to content

Commit 5e3277e

Browse files
authored
[improvement](routine-load) add routine load rows check (apache#25818)
1 parent 552091f commit 5e3277e

3 files changed

Lines changed: 17 additions & 1 deletion

File tree

‎be/src/runtime/routine_load/data_consumer_group.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,9 +125,10 @@ Status KafkaDataConsumerGroup::start_all(std::shared_ptr<StreamLoadContext> ctx,
125125
bool eos = false;
126126
while (true) {
127127
if (eos || left_time <= 0 || left_rows <= 0 || left_bytes <= 0) {
128+
_rows = ctx->max_batch_rows - left_rows;
128129
LOG(INFO) << "consumer group done: " << _grp_id
129130
<< ". consume time(ms)=" << ctx->max_interval_s * 1000 - left_time
130-
<< ", received rows=" << ctx->max_batch_rows - left_rows
131+
<< ", received rows=" << _rows
131132
<< ", received bytes=" << ctx->max_batch_size - left_bytes << ", eos: " << eos
132133
<< ", left_time: " << left_time << ", left_rows: " << left_rows
133134
<< ", left_bytes: " << left_bytes

‎be/src/runtime/routine_load/data_consumer_group.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ class DataConsumerGroup {
6060
++_counter;
6161
}
6262

63+
int64_t get_consumer_rows() const { return _rows; }
64+
65+
void set_consumer_rows(int64_t rows) { _rows = rows; }
66+
6367
// start all consumers
6468
virtual Status start_all(std::shared_ptr<StreamLoadContext> ctx,
6569
std::shared_ptr<io::KafkaConsumerPipe> kafka_pipe) {
@@ -77,6 +81,8 @@ class DataConsumerGroup {
7781
// when the counter becomes zero, shutdown the queue to finish
7882
std::mutex _mutex;
7983
int _counter;
84+
// received total rows
85+
int64_t _rows {0};
8086
};
8187

8288
// for kafka

‎be/src/runtime/routine_load/routine_load_task_executor.cpp‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -355,6 +355,15 @@ void RoutineLoadTaskExecutor::exec_task(std::shared_ptr<StreamLoadContext> ctx,
355355
// wait for all consumers finished
356356
HANDLE_ERROR(ctx->future.get(), "consume failed");
357357

358+
// check received and load rows
359+
LOG(INFO) << "routine load task received rows: " << consumer_grp.get()->get_consumer_rows()
360+
<< " load total rows: " << ctx.get()->number_total_rows
361+
<< " loaded rows: " << ctx.get()->number_loaded_rows
362+
<< " filtered rows: " << ctx.get()->number_filtered_rows
363+
<< " unselected rows: " << ctx.get()->number_unselected_rows;
364+
DCHECK(consumer_grp.get()->get_consumer_rows() == ctx.get()->number_total_rows);
365+
consumer_grp.get()->set_consumer_rows(0);
366+
358367
ctx->load_cost_millis = UnixMillis() - ctx->start_millis;
359368

360369
// return the consumer back to pool

0 commit comments

Comments
 (0)