Skip to content

Commit 1004182

Browse files
kaijchenpull[bot]
authored andcommitted
[fix](move-memtable) init auto partition context in VRowDistribution::open (#26911)
1 parent 1ae3588 commit 1004182

5 files changed

Lines changed: 32 additions & 19 deletions

File tree

‎be/src/vec/sink/vrow_distribution.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,22 @@ class VRowDistribution {
8888
_schema = ctx->schema;
8989
}
9090

91+
Status open(RowDescriptor* output_row_desc) {
92+
if (_vpartition->is_auto_partition()) {
93+
auto [part_ctx, part_func] = _get_partition_function();
94+
RETURN_IF_ERROR(part_ctx->prepare(_state, *output_row_desc));
95+
RETURN_IF_ERROR(part_ctx->open(_state));
96+
}
97+
for (auto& index : _schema->indexes()) {
98+
auto& where_clause = index->where_clause;
99+
if (where_clause != nullptr) {
100+
RETURN_IF_ERROR(where_clause->prepare(_state, *output_row_desc));
101+
RETURN_IF_ERROR(where_clause->open(_state));
102+
}
103+
}
104+
return Status::OK();
105+
}
106+
91107
// auto partition
92108
// mv where clause
93109
// v1 needs index->node->row_ids - tabletids

‎be/src/vec/sink/vtablet_sink_v2.cpp‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ static Status on_partitions_created(void* writer, TCreatePartitionResult* result
9191
return static_cast<VOlapTableSinkV2*>(writer)->on_partitions_created(result);
9292
}
9393

94-
void VOlapTableSinkV2::_init_row_distribution() {
94+
Status VOlapTableSinkV2::_init_row_distribution() {
9595
VRowDistributionContext ctx;
9696

9797
ctx.state = _state;
@@ -108,6 +108,10 @@ void VOlapTableSinkV2::_init_row_distribution() {
108108
ctx.schema = _schema;
109109

110110
_row_distribution.init(&ctx);
111+
112+
RETURN_IF_ERROR(_row_distribution.open(_output_row_desc));
113+
114+
return Status::OK();
111115
}
112116

113117
Status VOlapTableSinkV2::init(const TDataSink& t_sink) {
@@ -174,6 +178,8 @@ Status VOlapTableSinkV2::prepare(RuntimeState* state) {
174178
_block_convertor = std::make_unique<OlapTableBlockConvertor>(_output_tuple_desc);
175179
_block_convertor->init_autoinc_info(_schema->db_id(), _schema->table_id(),
176180
_state->batch_size());
181+
_output_row_desc = _pool->add(new RowDescriptor(_output_tuple_desc, false));
182+
177183
// add all counter
178184
_input_rows_counter = ADD_COUNTER(_profile, "RowsRead", TUnit::UNIT);
179185
_output_rows_counter = ADD_COUNTER(_profile, "RowsReturned", TUnit::UNIT);
@@ -209,7 +215,7 @@ Status VOlapTableSinkV2::open(RuntimeState* state) {
209215

210216
_build_tablet_node_mapping();
211217
RETURN_IF_ERROR(_open_streams(state->backend_id()));
212-
_init_row_distribution();
218+
RETURN_IF_ERROR(_init_row_distribution());
213219

214220
return Status::OK();
215221
}

‎be/src/vec/sink/vtablet_sink_v2.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ class VOlapTableSinkV2 final : public DataSink {
119119
Status on_partitions_created(TCreatePartitionResult* result);
120120

121121
private:
122-
void _init_row_distribution();
122+
Status _init_row_distribution();
123123

124124
Status _open_streams(int64_t src_id);
125125

@@ -149,6 +149,7 @@ class VOlapTableSinkV2 final : public DataSink {
149149

150150
// this is tuple descriptor of destination OLAP table
151151
TupleDescriptor* _output_tuple_desc = nullptr;
152+
RowDescriptor* _output_row_desc = nullptr;
152153

153154
// number of senders used to insert into OlapTable, if we only support single node insert,
154155
// all data from select should collectted and then send to OlapTable.

‎be/src/vec/sink/writer/vtablet_writer.cpp‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1063,7 +1063,7 @@ static Status on_partitions_created(void* writer, TCreatePartitionResult* result
10631063
return static_cast<VTabletWriter*>(writer)->on_partitions_created(result);
10641064
}
10651065

1066-
void VTabletWriter::_init_row_distribution() {
1066+
Status VTabletWriter::_init_row_distribution() {
10671067
VRowDistributionContext ctx;
10681068

10691069
ctx.state = _state;
@@ -1080,6 +1080,9 @@ void VTabletWriter::_init_row_distribution() {
10801080
ctx.schema = _schema;
10811081

10821082
_row_distribution.init(&ctx);
1083+
1084+
RETURN_IF_ERROR(_row_distribution.open(_output_row_desc));
1085+
return Status::OK();
10831086
}
10841087

10851088
Status VTabletWriter::_init(RuntimeState* state, RuntimeProfile* profile) {
@@ -1218,19 +1221,13 @@ Status VTabletWriter::_init(RuntimeState* state, RuntimeProfile* profile) {
12181221
RETURN_IF_ERROR(_channels.back()->init(state, tablets));
12191222
}
12201223

1221-
// prepare for auto partition functions
1222-
if (_vpartition->is_auto_partition()) {
1223-
auto [part_ctx, part_func] = _get_partition_function();
1224-
RETURN_IF_ERROR(part_ctx->prepare(_state, *_output_row_desc));
1225-
RETURN_IF_ERROR(part_ctx->open(_state));
1226-
}
12271224
if (_group_commit) {
12281225
RETURN_IF_ERROR(_state->exec_env()->wal_mgr()->add_wal_path(_db_id, _tb_id, _wal_id,
12291226
_state->import_label()));
12301227
RETURN_IF_ERROR(_state->exec_env()->wal_mgr()->create_wal_writer(_wal_id, _wal_writer));
12311228
}
12321229

1233-
_init_row_distribution();
1230+
RETURN_IF_ERROR(_init_row_distribution());
12341231

12351232
_inited = true;
12361233
return Status::OK();
@@ -1287,11 +1284,6 @@ Status VTabletWriter::_incremental_open_node_channel(
12871284
return Status::OK();
12881285
}
12891286

1290-
std::pair<vectorized::VExprContextSPtr, vectorized::VExprSPtr>
1291-
VTabletWriter::_get_partition_function() {
1292-
return {_vpartition->get_part_func_ctx(), _vpartition->get_partition_function()};
1293-
}
1294-
12951287
Status VTabletWriter::_cancel_channel_and_check_intolerable_failure(
12961288
Status status, const std::string& err_msg, const std::shared_ptr<IndexChannel> ich,
12971289
const std::shared_ptr<VNodeChannel> nch) {

‎be/src/vec/sink/writer/vtablet_writer.h‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -549,7 +549,7 @@ class VTabletWriter final : public AsyncResultWriter {
549549
using ChannelDistributionPayload = std::unordered_map<VNodeChannel*, Payload>;
550550
using ChannelDistributionPayloadVec = std::vector<std::unordered_map<VNodeChannel*, Payload>>;
551551

552-
void _init_row_distribution();
552+
Status _init_row_distribution();
553553

554554
Status _init(RuntimeState* state, RuntimeProfile* profile);
555555

@@ -564,8 +564,6 @@ class VTabletWriter final : public AsyncResultWriter {
564564
const std::shared_ptr<IndexChannel> ich,
565565
const std::shared_ptr<VNodeChannel> nch);
566566

567-
std::pair<vectorized::VExprContextSPtr, vectorized::VExprSPtr> _get_partition_function();
568-
569567
void _cancel_all_channel(Status status);
570568

571569
void _save_missing_values(vectorized::ColumnPtr col, vectorized::DataTypePtr value_type,

0 commit comments

Comments
 (0)