@@ -143,14 +143,24 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR
143143 return Status::OLAPInternalError (OLAP_ERR_TOO_MANY_VERSION );
144144 }
145145
146- // write
146+ auto tablet_schema = tablet_vars->at (0 ).tablet ->tablet_schema ();
147+ if (!request.columns_desc .empty () && request.columns_desc [0 ].col_unique_id >= 0 ) {
148+ tablet_schema.clear_columns ();
149+ for (const auto & column_desc : request.columns_desc ) {
150+ tablet_schema.append_column (TabletColumn (column_desc));
151+ }
152+ }
153+
154+ // writes
147155 if (push_type == PUSH_NORMAL_V2 ) {
148156 res = _convert_v2 (tablet_vars->at (0 ).tablet , tablet_vars->at (1 ).tablet ,
149- &(tablet_vars->at (0 ).rowset_to_add ), &(tablet_vars->at (1 ).rowset_to_add ));
157+ &(tablet_vars->at (0 ).rowset_to_add ), &(tablet_vars->at (1 ).rowset_to_add ),
158+ &tablet_schema);
150159
151160 } else {
152161 res = _convert (tablet_vars->at (0 ).tablet , tablet_vars->at (1 ).tablet ,
153- &(tablet_vars->at (0 ).rowset_to_add ), &(tablet_vars->at (1 ).rowset_to_add ));
162+ &(tablet_vars->at (0 ).rowset_to_add ), &(tablet_vars->at (1 ).rowset_to_add ),
163+ &tablet_schema);
154164 }
155165 if (!res.ok ()) {
156166 LOG (WARNING ) << " fail to convert tmp file when realtime push. res=" << res
@@ -210,7 +220,8 @@ void PushHandler::_get_tablet_infos(const std::vector<TabletVars>& tablet_vars,
210220}
211221
212222Status PushHandler::_convert_v2 (TabletSharedPtr cur_tablet, TabletSharedPtr new_tablet,
213- RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset) {
223+ RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset,
224+ const TabletSchema* tablet_schema) {
214225 Status res = Status::OK ();
215226 uint32_t num_rows = 0 ;
216227 PUniqueId load_id;
@@ -222,14 +233,13 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_
222233
223234 // 1. init RowsetBuilder of cur_tablet for current push
224235 VLOG_NOTICE << " init rowset builder. tablet=" << cur_tablet->full_name ()
225- << " , block_row_size=" << cur_tablet ->num_rows_per_row_block ();
236+ << " , block_row_size=" << tablet_schema ->num_rows_per_row_block ();
226237 // although the spark load output files are fully sorted,
227238 // but it depends on thirparty implementation, so we conservatively
228239 // set this value to OVERLAP_UNKNOWN
229240 std::unique_ptr<RowsetWriter> rowset_writer;
230241 res = cur_tablet->create_rowset_writer (_request.transaction_id , load_id, PREPARED ,
231- OVERLAP_UNKNOWN , &cur_tablet->tablet_schema (),
232- &rowset_writer);
242+ OVERLAP_UNKNOWN , tablet_schema, &rowset_writer);
233243 if (!res.ok ()) {
234244 LOG (WARNING ) << " failed to init rowset writer, tablet=" << cur_tablet->full_name ()
235245 << " , txn_id=" << _request.transaction_id << " , res=" << res;
@@ -251,7 +261,7 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_
251261 }
252262
253263 // init schema
254- std::unique_ptr<Schema> schema (new (std::nothrow) Schema (cur_tablet-> tablet_schema () ));
264+ std::unique_ptr<Schema> schema (new (std::nothrow) Schema (* tablet_schema));
255265 if (schema == nullptr ) {
256266 LOG (WARNING ) << " fail to create schema. tablet=" << cur_tablet->full_name ();
257267 res = Status::OLAPInternalError (OLAP_ERR_MALLOC_ERROR );
@@ -319,8 +329,8 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_
319329 // 5. Convert data for schema change tables
320330 VLOG_TRACE << " load to related tables of schema_change if possible." ;
321331 if (new_tablet != nullptr ) {
322- res = SchemaChangeHandler::schema_version_convert (cur_tablet, new_tablet, cur_rowset,
323- new_rowset, *_desc_tbl);
332+ res = SchemaChangeHandler::schema_version_convert (
333+ cur_tablet, new_tablet, cur_rowset, new_rowset, *_desc_tbl, tablet_schema );
324334 if (!res.ok ()) {
325335 LOG (WARNING ) << " failed to change schema version for delta."
326336 << " [res=" << res << " new_tablet='" << new_tablet->full_name ()
@@ -335,7 +345,8 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, TabletSharedPtr new_
335345}
336346
337347Status PushHandler::_convert (TabletSharedPtr cur_tablet, TabletSharedPtr new_tablet,
338- RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset) {
348+ RowsetSharedPtr* cur_rowset, RowsetSharedPtr* new_rowset,
349+ const TabletSchema* tablet_schema) {
339350 Status res = Status::OK ();
340351 RowCursor row;
341352 BinaryFile raw_file;
@@ -382,7 +393,7 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tab
382393 }
383394
384395 // init BinaryReader
385- if (!(res = reader->init (cur_tablet , &raw_file))) {
396+ if (!(res = reader->init (tablet_schema , &raw_file))) {
386397 LOG (WARNING ) << " fail to init reader. res=" << res
387398 << " , tablet=" << cur_tablet->full_name ()
388399 << " , file=" << _request.http_file_path ;
@@ -394,8 +405,7 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tab
394405 // 2. init RowsetBuilder of cur_tablet for current push
395406 std::unique_ptr<RowsetWriter> rowset_writer;
396407 res = cur_tablet->create_rowset_writer (_request.transaction_id , load_id, PREPARED ,
397- OVERLAP_UNKNOWN , &cur_tablet->tablet_schema (),
398- &rowset_writer);
408+ OVERLAP_UNKNOWN , tablet_schema, &rowset_writer);
399409 if (!res.ok ()) {
400410 LOG (WARNING ) << " failed to init rowset writer, tablet=" << cur_tablet->full_name ()
401411 << " , txn_id=" << _request.transaction_id << " , res=" << res;
@@ -407,7 +417,7 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tab
407417 << " , block_row_size=" << cur_tablet->num_rows_per_row_block ();
408418
409419 // 4. Init RowCursor
410- if (!(res = row.init (cur_tablet-> tablet_schema () ))) {
420+ if (!(res = row.init (* tablet_schema))) {
411421 LOG (WARNING ) << " fail to init rowcursor. res=" << res;
412422 break ;
413423 }
@@ -460,8 +470,8 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, TabletSharedPtr new_tab
460470 // 7. Convert data for schema change tables
461471 VLOG_TRACE << " load to related tables of schema_change if possible." ;
462472 if (new_tablet != nullptr ) {
463- res = SchemaChangeHandler::schema_version_convert (cur_tablet, new_tablet, cur_rowset,
464- new_rowset, *_desc_tbl);
473+ res = SchemaChangeHandler::schema_version_convert (
474+ cur_tablet, new_tablet, cur_rowset, new_rowset, *_desc_tbl, tablet_schema );
465475 if (!res.ok ()) {
466476 LOG (WARNING ) << " failed to change schema version for delta."
467477 << " [res=" << res << " new_tablet='" << new_tablet->full_name ()
@@ -507,13 +517,13 @@ IBinaryReader* IBinaryReader::create(bool need_decompress) {
507517
508518BinaryReader::BinaryReader () : _row_buf(nullptr ), _row_buf_size(0 ) {}
509519
510- Status BinaryReader::init (TabletSharedPtr tablet , BinaryFile* file) {
520+ Status BinaryReader::init (const TabletSchema* tablet_schema , BinaryFile* file) {
511521 Status res = Status::OK ();
512522
513523 do {
514524 _file = file;
515525 _content_len = _file->file_length () - _file->header_size ();
516- _row_buf_size = tablet ->row_size ();
526+ _row_buf_size = _tablet_schema ->row_size ();
517527
518528 _row_buf = new (std::nothrow) char [_row_buf_size];
519529 if (_row_buf == nullptr ) {
@@ -528,7 +538,7 @@ Status BinaryReader::init(TabletSharedPtr tablet, BinaryFile* file) {
528538 break ;
529539 }
530540
531- _tablet = tablet ;
541+ _tablet_schema = tablet_schema ;
532542 _ready = true ;
533543 } while (false );
534544
@@ -552,10 +562,10 @@ Status BinaryReader::next(RowCursor* row) {
552562 return Status::OLAPInternalError (OLAP_ERR_INPUT_PARAMETER_ERROR );
553563 }
554564
555- const TabletSchema& schema = _tablet-> tablet_schema () ;
565+ const TabletSchema& schema = *_tablet_schema ;
556566 size_t offset = 0 ;
557567 size_t field_size = 0 ;
558- size_t num_null_bytes = (_tablet-> num_null_columns () + 7 ) / 8 ;
568+ size_t num_null_bytes = (schema. num_null_columns () + 7 ) / 8 ;
559569
560570 if (!(res = _file->read (_row_buf + offset, num_null_bytes))) {
561571 LOG (WARNING ) << " read file for one row fail. res=" << res;
@@ -649,7 +659,7 @@ LzoBinaryReader::LzoBinaryReader()
649659 _row_num(0 ),
650660 _next_row_start(0 ) {}
651661
652- Status LzoBinaryReader::init (TabletSharedPtr tablet , BinaryFile* file) {
662+ Status LzoBinaryReader::init (const TabletSchema* tablet_schema , BinaryFile* file) {
653663 Status res = Status::OK ();
654664
655665 do {
@@ -670,7 +680,7 @@ Status LzoBinaryReader::init(TabletSharedPtr tablet, BinaryFile* file) {
670680 break ;
671681 }
672682
673- _tablet = tablet ;
683+ _tablet_schema = tablet_schema ;
674684 _ready = true ;
675685 } while (false );
676686
@@ -703,10 +713,10 @@ Status LzoBinaryReader::next(RowCursor* row) {
703713 }
704714 }
705715
706- const TabletSchema& schema = _tablet-> tablet_schema () ;
716+ const TabletSchema& schema = *_tablet_schema ;
707717 size_t offset = 0 ;
708718 size_t field_size = 0 ;
709- size_t num_null_bytes = (_tablet-> num_null_columns () + 7 ) / 8 ;
719+ size_t num_null_bytes = (schema. num_null_columns () + 7 ) / 8 ;
710720
711721 size_t p = 0 ;
712722 for (size_t i = 0 ; i < schema.num_columns (); ++i) {
@@ -794,7 +804,7 @@ Status LzoBinaryReader::_next_block() {
794804 SAFE_DELETE_ARRAY (_row_buf);
795805
796806 _max_row_num = _row_num;
797- _max_row_buf_size = _max_row_num * _tablet ->row_size ();
807+ _max_row_buf_size = _max_row_num * _tablet_schema ->row_size ();
798808 _row_buf = new (std::nothrow) char [_max_row_buf_size];
799809 if (_row_buf == nullptr ) {
800810 LOG (WARNING ) << " fail to malloc rows buf. size=" << _max_row_buf_size;
0 commit comments