Skip to content

Commit ff73491

Browse files
committed
modify PushTask process (apache#217)
1 parent 0e62965 commit ff73491

11 files changed

Lines changed: 79 additions & 50 deletions

File tree

‎be/src/olap/delta_writer.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ Status DeltaWriter::write(const vectorized::Block* block, const std::vector<int>
195195
if (_is_cancelled) {
196196
return Status::OLAPInternalError(OLAP_ERR_ALREADY_CANCELLED);
197197
}
198-
198+
LOG(INFO) << "3 block columns: " << block->columns();
199199
_mem_table->insert(block, row_idxs);
200200

201201
if (_mem_table->need_to_agg()) {

‎be/src/olap/push_handler.cpp‎

Lines changed: 37 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -143,14 +143,24 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR
143143
return Status::OLAPInternalError(OLAP_ERR_TOO_MANY_VERSION);
144144
}
145145

146-
// write
146+
auto tablet_schema = tablet_vars->at(0).tablet->tablet_schema();
147+
if (!request.columns_desc.empty() && request.columns_desc[0].col_unique_id >= 0) {
148+
tablet_schema.clear_columns();
149+
for (const auto& column_desc : request.columns_desc) {
150+
tablet_schema.append_column(TabletColumn(column_desc));
151+
}
152+
}
153+
154+
// writes
147155
if (push_type == PUSH_NORMAL_V2) {
148156
res = _convert_v2(tablet_vars->at(0).tablet, tablet_vars->at(1).tablet,
149-
&(tablet_vars->at(0).rowset_to_add), &(tablet_vars->at(1).rowset_to_add));
157+
&(tablet_vars->at(0).rowset_to_add), &(tablet_vars->at(1).rowset_to_add),
158+
&tablet_schema);
150159

151160
} else {
152161
res = _convert(tablet_vars->at(0).tablet, tablet_vars->at(1).tablet,
153-
&(tablet_vars->at(0).rowset_to_add), &(tablet_vars->at(1).rowset_to_add));
162+
&(tablet_vars->at(0).rowset_to_add), &(tablet_vars->at(1).rowset_to_add),
163+
&tablet_schema);
154164
}
155165
if (!res.ok()) {
156166
LOG(WARNING) << "fail to convert tmp file when realtime push. res=" << res
@@ -210,7 +220,8 @@ void PushHandler::_get_tablet_infos(const std::vector<TabletVars>& tablet_vars,
210220
}
211221

212222
Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_tablet,
213-
RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset) {
223+
RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset,
224+
const TabletSchema* tablet_schema) {
214225
Status res = Status::OK();
215226
uint32_t num_rows = 0;
216227
PUniqueId load_id;
@@ -222,14 +233,13 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_
222233

223234
// 1. init RowsetBuilder of cur_tablet for current push
224235
VLOG_NOTICE << "init rowset builder. tablet=" << cur_tablet->full_name()
225-
<< ", block_row_size=" << cur_tablet->num_rows_per_row_block();
236+
<< ", block_row_size=" << tablet_schema->num_rows_per_row_block();
226237
// although the spark load output files are fully sorted,
227238
// but it depends on thirparty implementation, so we conservatively
228239
// set this value to OVERLAP_UNKNOWN
229240
std::unique_ptr<RowsetWriter> rowset_writer;
230241
res = cur_tablet->create_rowset_writer(_request.transaction_id, load_id, PREPARED,
231-
OVERLAP_UNKNOWN, &cur_tablet->tablet_schema(),
232-
&rowset_writer);
242+
OVERLAP_UNKNOWN, tablet_schema, &rowset_writer);
233243
if (!res.ok()) {
234244
LOG(WARNING) << "failed to init rowset writer, tablet=" << cur_tablet->full_name()
235245
<< ", txn_id=" << _request.transaction_id << ", res=" << res;
@@ -251,7 +261,7 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_
251261
}
252262

253263
// init schema
254-
std::unique_ptr<Schema> schema(new (std::nothrow) Schema(cur_tablet->tablet_schema()));
264+
std::unique_ptr<Schema> schema(new (std::nothrow) Schema(*tablet_schema));
255265
if (schema == nullptr) {
256266
LOG(WARNING) << "fail to create schema. tablet=" << cur_tablet->full_name();
257267
res = Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR);
@@ -319,8 +329,8 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_
319329
// 5. Convert data for schema change tables
320330
VLOG_TRACE << "load to related tables of schema_change if possible.";
321331
if (new_tablet != nullptr) {
322-
res = SchemaChangeHandler::schema_version_convert(cur_tablet, new_tablet, cur_rowset,
323-
new_rowset, *_desc_tbl);
332+
res = SchemaChangeHandler::schema_version_convert(
333+
cur_tablet, new_tablet, cur_rowset, new_rowset, *_desc_tbl, tablet_schema);
324334
if (!res.ok()) {
325335
LOG(WARNING) << "failed to change schema version for delta."
326336
<< "[res=" << res << " new_tablet='" << new_tablet->full_name()
@@ -335,7 +345,8 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_
335345
}
336346

337347
Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tablet,
338-
RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset) {
348+
RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset,
349+
const TabletSchema* tablet_schema) {
339350
Status res = Status::OK();
340351
RowCursor row;
341352
BinaryFile raw_file;
@@ -382,7 +393,7 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tab
382393
}
383394

384395
// init BinaryReader
385-
if (!(res = reader->init(cur_tablet, &raw_file))) {
396+
if (!(res = reader->init(tablet_schema, &raw_file))) {
386397
LOG(WARNING) << "fail to init reader. res=" << res
387398
<< ", tablet=" << cur_tablet->full_name()
388399
<< ", file=" << _request.http_file_path;
@@ -394,8 +405,7 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tab
394405
// 2. init RowsetBuilder of cur_tablet for current push
395406
std::unique_ptr<RowsetWriter> rowset_writer;
396407
res = cur_tablet->create_rowset_writer(_request.transaction_id, load_id, PREPARED,
397-
OVERLAP_UNKNOWN, &cur_tablet->tablet_schema(),
398-
&rowset_writer);
408+
OVERLAP_UNKNOWN, tablet_schema, &rowset_writer);
399409
if (!res.ok()) {
400410
LOG(WARNING) << "failed to init rowset writer, tablet=" << cur_tablet->full_name()
401411
<< ", txn_id=" << _request.transaction_id << ", res=" << res;
@@ -407,7 +417,7 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tab
407417
<< ", block_row_size=" << cur_tablet->num_rows_per_row_block();
408418

409419
// 4. Init RowCursor
410-
if (!(res = row.init(cur_tablet->tablet_schema()))) {
420+
if (!(res = row.init(*tablet_schema))) {
411421
LOG(WARNING) << "fail to init rowcursor. res=" << res;
412422
break;
413423
}
@@ -460,8 +470,8 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tab
460470
// 7. Convert data for schema change tables
461471
VLOG_TRACE << "load to related tables of schema_change if possible.";
462472
if (new_tablet != nullptr) {
463-
res = SchemaChangeHandler::schema_version_convert(cur_tablet, new_tablet, cur_rowset,
464-
new_rowset, *_desc_tbl);
473+
res = SchemaChangeHandler::schema_version_convert(
474+
cur_tablet, new_tablet, cur_rowset, new_rowset, *_desc_tbl, tablet_schema);
465475
if (!res.ok()) {
466476
LOG(WARNING) << "failed to change schema version for delta."
467477
<< "[res=" << res << " new_tablet='" << new_tablet->full_name()
@@ -507,13 +517,13 @@ IBinaryReader* IBinaryReader::create(bool need_decompress) {
507517

508518
BinaryReader::BinaryReader() : _row_buf(nullptr), _row_buf_size(0) {}
509519

510-
Status BinaryReader::init(TabletSharedPtr tablet, BinaryFile* file) {
520+
Status BinaryReader::init(const TabletSchema* tablet_schema, BinaryFile* file) {
511521
Status res = Status::OK();
512522

513523
do {
514524
_file = file;
515525
_content_len = _file->file_length() - _file->header_size();
516-
_row_buf_size = tablet->row_size();
526+
_row_buf_size = _tablet_schema->row_size();
517527

518528
_row_buf = new (std::nothrow) char[_row_buf_size];
519529
if (_row_buf == nullptr) {
@@ -528,7 +538,7 @@ Status BinaryReader::init(TabletSharedPtr tablet, BinaryFile* file) {
528538
break;
529539
}
530540

531-
_tablet = tablet;
541+
_tablet_schema = tablet_schema;
532542
_ready = true;
533543
} while (false);
534544

@@ -552,10 +562,10 @@ Status BinaryReader::next(RowCursor* row) {
552562
return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR);
553563
}
554564

555-
const TabletSchema& schema = _tablet->tablet_schema();
565+
const TabletSchema& schema = *_tablet_schema;
556566
size_t offset = 0;
557567
size_t field_size = 0;
558-
size_t num_null_bytes = (_tablet->num_null_columns() + 7) / 8;
568+
size_t num_null_bytes = (schema.num_null_columns() + 7) / 8;
559569

560570
if (!(res = _file->read(_row_buf + offset, num_null_bytes))) {
561571
LOG(WARNING) << "read file for one row fail. res=" << res;
@@ -649,7 +659,7 @@ LzoBinaryReader::LzoBinaryReader()
649659
_row_num(0),
650660
_next_row_start(0) {}
651661

652-
Status LzoBinaryReader::init(TabletSharedPtr tablet, BinaryFile* file) {
662+
Status LzoBinaryReader::init(const TabletSchema* tablet_schema, BinaryFile* file) {
653663
Status res = Status::OK();
654664

655665
do {
@@ -670,7 +680,7 @@ Status LzoBinaryReader::init(TabletSharedPtr tablet, BinaryFile* file) {
670680
break;
671681
}
672682

673-
_tablet = tablet;
683+
_tablet_schema = tablet_schema;
674684
_ready = true;
675685
} while (false);
676686

@@ -703,10 +713,10 @@ Status LzoBinaryReader::next(RowCursor* row) {
703713
}
704714
}
705715

706-
const TabletSchema& schema = _tablet->tablet_schema();
716+
const TabletSchema& schema = *_tablet_schema;
707717
size_t offset = 0;
708718
size_t field_size = 0;
709-
size_t num_null_bytes = (_tablet->num_null_columns() + 7) / 8;
719+
size_t num_null_bytes = (schema.num_null_columns() + 7) / 8;
710720

711721
size_t p = 0;
712722
for (size_t i = 0; i < schema.num_columns(); ++i) {
@@ -794,7 +804,7 @@ Status LzoBinaryReader::_next_block() {
794804
SAFE_DELETE_ARRAY(_row_buf);
795805

796806
_max_row_num = _row_num;
797-
_max_row_buf_size = _max_row_num * _tablet->row_size();
807+
_max_row_buf_size = _max_row_num * _tablet_schema->row_size();
798808
_row_buf = new (std::nothrow) char[_max_row_buf_size];
799809
if (_row_buf == nullptr) {
800810
LOG(WARNING) << "fail to malloc rows buf. size=" << _max_row_buf_size;

‎be/src/olap/push_handler.h‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,13 @@ class PushHandler {
6060

6161
private:
6262
Status _convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_tablet_vec,
63-
RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset);
63+
RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset,
64+
const TabletSchema* tablet_schema);
6465
// Convert local data file to internal formatted delta,
6566
// return new delta's SegmentGroup
6667
Status _convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tablet_vec,
67-
RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset);
68+
RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset,
69+
const TabletSchema* tablet_schema);
6870

6971
// Only for debug
7072
std::string _debug_version_list(const Versions& versions) const;
@@ -112,7 +114,7 @@ class IBinaryReader {
112114
static IBinaryReader* create(bool need_decompress);
113115
virtual ~IBinaryReader() = default;
114116

115-
virtual Status init(TabletSharedPtr tablet, BinaryFile* file) = 0;
117+
virtual Status init(const TabletSchema* tablet_schema, BinaryFile* file) = 0;
116118
virtual Status finalize() = 0;
117119

118120
virtual Status next(RowCursor* row) = 0;
@@ -131,7 +133,7 @@ class IBinaryReader {
131133
_ready(false) {}
132134

133135
BinaryFile* _file;
134-
TabletSharedPtr _tablet;
136+
const TabletSchema* _tablet_schema;
135137
size_t _content_len;
136138
size_t _curr;
137139
uint32_t _adler_checksum;
@@ -144,7 +146,7 @@ class BinaryReader : public IBinaryReader {
144146
explicit BinaryReader();
145147
~BinaryReader() override { finalize(); }
146148

147-
Status init(TabletSharedPtr tablet, BinaryFile* file) override;
149+
Status init(const TabletSchema* tablet_schema, BinaryFile* file) override;
148150
Status finalize() override;
149151

150152
Status next(RowCursor* row) override;
@@ -161,7 +163,7 @@ class LzoBinaryReader : public IBinaryReader {
161163
explicit LzoBinaryReader();
162164
~LzoBinaryReader() override { finalize(); }
163165

164-
Status init(TabletSharedPtr tablet, BinaryFile* file) override;
166+
Status init(const TabletSchema* tablet_schema, BinaryFile* file) override;
165167
Status finalize() override;
166168

167169
Status next(RowCursor* row) override;

‎be/src/olap/rowset/segment_v2/segment_writer.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ Status SegmentWriter::append_block(const vectorized::Block* block, size_t row_po
116116
size_t num_rows) {
117117
assert(block && num_rows > 0 && row_pos + num_rows <= block->rows() &&
118118
block->columns() == _column_writers.size());
119+
LOG(INFO) << "block columns: " << block->columns();
119120
_olap_data_convertor.set_source_content(block, row_pos, num_rows);
120121

121122
// find all row pos for short key indexes

‎be/src/olap/schema_change.cpp‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1595,7 +1595,8 @@ Status SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tablet,
15951595
TabletSharedPtr new_tablet,
15961596
RowsetSharedPtr* base_rowset,
15971597
RowsetSharedPtr* new_rowset,
1598-
DescriptorTbl desc_tbl) {
1598+
DescriptorTbl desc_tbl,
1599+
const TabletSchema* base_schema_change) {
15991600
Status res = Status::OK();
16001601
LOG(INFO) << "begin to convert delta version for schema changing. "
16011602
<< "base_tablet=" << base_tablet->full_name()
@@ -1609,7 +1610,7 @@ Status SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tablet,
16091610

16101611
const std::unordered_map<std::string, AlterMaterializedViewParam> materialized_function_map;
16111612
if (res = _parse_request(base_tablet, new_tablet, &rb_changer, &sc_sorting, &sc_directly,
1612-
materialized_function_map, desc_tbl, nullptr);
1613+
materialized_function_map, desc_tbl, base_schema_change);
16131614
!res) {
16141615
LOG(WARNING) << "failed to parse the request. res=" << res;
16151616
return res;
@@ -1623,15 +1624,15 @@ Status SchemaChangeHandler::schema_version_convert(TabletSharedPtr base_tablet,
16231624
// c. Convert data
16241625
DeleteHandler delete_handler;
16251626
std::vector<ColumnId> return_columns;
1626-
size_t num_cols = base_tablet->tablet_schema().num_columns();
1627+
size_t num_cols = base_schema_change->num_columns();
16271628
return_columns.resize(num_cols);
16281629
for (int i = 0; i < num_cols; ++i) {
16291630
return_columns[i] = i;
16301631
}
16311632

16321633
RowsetReaderContext reader_context;
16331634
reader_context.reader_type = READER_ALTER_TABLE;
1634-
reader_context.tablet_schema = &base_tablet->tablet_schema();
1635+
reader_context.tablet_schema = base_schema_change;
16351636
reader_context.need_ordered_result = true;
16361637
reader_context.delete_handler = &delete_handler;
16371638
reader_context.return_columns = &return_columns;

‎be/src/olap/schema_change.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,8 @@ class SchemaChangeHandler {
210210
public:
211211
static Status schema_version_convert(TabletSharedPtr base_tablet, TabletSharedPtr new_tablet,
212212
RowsetSharedPtr* base_rowset, RowsetSharedPtr* new_rowset,
213-
DescriptorTbl desc_tbl);
213+
DescriptorTbl desc_tbl,
214+
const TabletSchema* base_schema_change);
214215

215216
// schema change v2, it will not set alter task in base tablet
216217
static Status process_alter_tablet_v2(const TAlterTabletReqV2& request);

‎be/src/runtime/tablets_channel.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,7 @@ Status TabletsChannel::add_batch(const TabletWriterAddRequest& request,
205205
if constexpr (std::is_same_v<TabletWriterAddRequest, PTabletWriterAddBatchRequest>) {
206206
return RowBatch(*_row_desc, request.row_batch());
207207
} else {
208+
LOG(INFO) << "2 block columns: " << vectorized::Block(request.block()).columns();
208209
return vectorized::Block(request.block());
209210
}
210211
};

‎fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -217,12 +217,6 @@ public void process(DeleteStmt stmt) throws DdlException, QueryStateException {
217217
// to make sure that the delete transaction can be done successfully.
218218
txnState.addTableIndexes(olapTable);
219219

220-
//for light schema change
221-
List<TColumn> columnsDesc = new ArrayList<TColumn>();
222-
for (Column column : olapTable.getFullSchema()) {
223-
columnsDesc.add(column.toThrift());
224-
}
225-
226220
// task sent to be
227221
AgentBatchTask batchTask = new AgentBatchTask();
228222
// count total replica num
@@ -242,6 +236,11 @@ public void process(DeleteStmt stmt) throws DdlException, QueryStateException {
242236
long indexId = index.getId();
243237
int schemaHash = olapTable.getSchemaHashByIndexId(indexId);
244238

239+
List<TColumn> columnsDesc = new ArrayList<TColumn>();
240+
for (Column column : olapTable.getSchemaByIndexId(indexId)) {
241+
columnsDesc.add(column.toThrift());
242+
}
243+
245244
for (Tablet tablet : index.getTablets()) {
246245
long tabletId = tablet.getId();
247246

‎fe/fe-core/src/main/java/org/apache/doris/load/LoadChecker.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.doris.load;
1919

2020
import org.apache.doris.catalog.Catalog;
21+
import org.apache.doris.catalog.Column;
2122
import org.apache.doris.catalog.Database;
2223
import org.apache.doris.catalog.MaterializedIndex;
2324
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
@@ -41,6 +42,7 @@
4142
import org.apache.doris.task.MasterTask;
4243
import org.apache.doris.task.MasterTaskExecutor;
4344
import org.apache.doris.task.PushTask;
45+
import org.apache.doris.thrift.TColumn;
4446
import org.apache.doris.thrift.TPriority;
4547
import org.apache.doris.thrift.TPushType;
4648
import org.apache.doris.thrift.TTaskType;
@@ -425,6 +427,10 @@ private Set<Long> submitPushTasks(LoadJob job, Database db) {
425427
}
426428

427429
int schemaHash = tableLoadInfo.getIndexSchemaHash(indexId);
430+
List<TColumn> columnsDesc = new ArrayList<TColumn>();
431+
for (Column column : table.getSchemaByIndexId(indexId)) {
432+
columnsDesc.add(column.toThrift());
433+
}
428434
short quorumNum = (short) (replicationNum / 2 + 1);
429435
for (Tablet tablet : index.getTablets()) {
430436
long tabletId = tablet.getId();
@@ -465,7 +471,8 @@ private Set<Long> submitPushTasks(LoadJob job, Database db) {
465471
needDecompress, job.getPriority(),
466472
TTaskType.REALTIME_PUSH,
467473
job.getTransactionId(),
468-
Catalog.getCurrentGlobalTransactionMgr().getTransactionIDGenerator().getNextTransactionId(), null);
474+
Catalog.getCurrentGlobalTransactionMgr().getTransactionIDGenerator().getNextTransactionId(),
475+
columnsDesc);
469476
pushTask.setIsSchemaChanging(autoLoadToTwoTablet);
470477
if (AgentTaskQueue.addTask(pushTask)) {
471478
batchTask.addTask(pushTask);

0 commit comments

Comments
 (0)