@@ -53,6 +53,7 @@ MemTable::MemTable(int64_t tablet_id, Schema* schema, const TabletSchema* tablet
5353 // TODO: Support ZOrderComparator in the future
5454 _vec_skip_list = std::make_unique<VecTable>(
5555 _vec_row_comparator.get (), _table_mem_pool.get (), _keys_type == KeysType::DUP_KEYS );
56+ _init_profile ();
5657 } else {
5758 _vec_skip_list = nullptr ;
5859 if (_keys_type == KeysType::DUP_KEYS ) {
@@ -98,6 +99,7 @@ void MemTable::_init_agg_functions(const vectorized::Block* block) {
9899MemTable::~MemTable () {
99100 std::for_each (_row_in_blocks.begin (), _row_in_blocks.end (), std::default_delete<RowInBlock>());
100101 _mem_tracker->release (_mem_usage);
102+ print_profile ();
101103}
102104
103105MemTable::RowCursorComparator::RowCursorComparator (const Schema* schema) : _schema(schema) {}
@@ -115,55 +117,61 @@ int MemTable::RowInBlockComparator::operator()(const RowInBlock* left,
115117}
116118
117119void MemTable::insert (const vectorized::Block* block, size_t row_pos, size_t num_rows) {
118- if (_is_first_insertion) {
119- _is_first_insertion = false ;
120- auto cloneBlock = block->clone_without_columns ();
121- _input_mutable_block = vectorized::MutableBlock::build_mutable_block (&cloneBlock);
122- _vec_row_comparator->set_block (&_input_mutable_block);
123- _output_mutable_block = vectorized::MutableBlock::build_mutable_block (&cloneBlock);
124- if (_keys_type != KeysType::DUP_KEYS ) {
125- _init_agg_functions (block);
120+ {
121+ SCOPED_TIMER (_insert_time);
122+ if (_is_first_insertion) {
123+ _is_first_insertion = false ;
124+ auto cloneBlock = block->clone_without_columns ();
125+ _input_mutable_block = vectorized::MutableBlock::build_mutable_block (&cloneBlock);
126+ _vec_row_comparator->set_block (&_input_mutable_block);
127+ _output_mutable_block = vectorized::MutableBlock::build_mutable_block (&cloneBlock);
128+ if (_keys_type != KeysType::DUP_KEYS ) {
129+ _init_agg_functions (block);
130+ }
131+ }
132+ size_t cursor_in_mutableblock = _input_mutable_block.rows ();
133+ size_t oldsize = _input_mutable_block.allocated_bytes ();
134+ _input_mutable_block.add_rows (block, row_pos, num_rows);
135+ size_t newsize = _input_mutable_block.allocated_bytes ();
136+ _mem_usage += newsize - oldsize;
137+ _mem_tracker->consume (newsize - oldsize);
138+ // when new data inserted, the mem_usage of memtable should be re-shrunk again.
139+ _is_shrunk_by_agg = false ;
140+
141+ for (int i = 0 ; i < num_rows; i++) {
142+ _row_in_blocks.emplace_back (new RowInBlock {cursor_in_mutableblock + i});
143+ _insert_one_row_from_block (_row_in_blocks.back ());
126144 }
127- }
128- size_t cursor_in_mutableblock = _input_mutable_block.rows ();
129- size_t oldsize = _input_mutable_block.allocated_bytes ();
130- _input_mutable_block.add_rows (block, row_pos, num_rows);
131- size_t newsize = _input_mutable_block.allocated_bytes ();
132- _mem_usage += newsize - oldsize;
133- _mem_tracker->consume (newsize - oldsize);
134- // when new data inserted, the mem_usage of memtable should be re-shrunk again.
135- _is_shrunk_by_agg = false ;
136-
137- for (int i = 0 ; i < num_rows; i++) {
138- _row_in_blocks.emplace_back (new RowInBlock {cursor_in_mutableblock + i});
139- _insert_one_row_from_block (_row_in_blocks.back ());
140145 }
141146}
142147
143148void MemTable::_insert_one_row_from_block (RowInBlock* row_in_block) {
144- _rows++;
145- bool overwritten = false ;
146- if (_keys_type == KeysType::DUP_KEYS ) {
147- // TODO: dup keys only need sort opertaion. Rethink skiplist is the beat way to sort columns?
148- _vec_skip_list->Insert (row_in_block, &overwritten);
149- DCHECK (!overwritten) << " Duplicate key model meet overwrite in SkipList" ;
150- return ;
151- }
152-
153- bool is_exist = _vec_skip_list->Find (row_in_block, &_vec_hint);
154- if (is_exist) {
155- _aggregate_two_row_in_block (row_in_block, _vec_hint.curr ->key );
156- } else {
157- row_in_block->init_agg_places (_agg_functions, _schema->num_key_columns ());
158- for (auto cid = _schema->num_key_columns (); cid < _schema->num_columns (); cid++) {
159- auto col_ptr = _input_mutable_block.mutable_columns ()[cid].get ();
160- auto place = row_in_block->_agg_places [cid];
161- _agg_functions[cid]->add (place,
162- const_cast <const doris::vectorized::IColumn**>(&col_ptr),
163- row_in_block->_row_pos , nullptr );
149+ {
150+ SCOPED_TIMER (_sort_agg_time);
151+ _rows++;
152+ bool overwritten = false ;
153+ if (_keys_type == KeysType::DUP_KEYS ) {
154+ // TODO: dup keys only need sort opertaion. Rethink skiplist is the beat way to sort columns?
155+ _vec_skip_list->Insert (row_in_block, &overwritten);
156+ DCHECK (!overwritten) << " Duplicate key model meet overwrite in SkipList" ;
157+ return ;
164158 }
165159
166- _vec_skip_list->InsertWithHint (row_in_block, is_exist, &_vec_hint);
160+ bool is_exist = _vec_skip_list->Find (row_in_block, &_vec_hint);
161+ if (is_exist) {
162+ _aggregate_two_row_in_block (row_in_block, _vec_hint.curr ->key );
163+ } else {
164+ row_in_block->init_agg_places (_agg_functions, _schema->num_key_columns ());
165+ for (auto cid = _schema->num_key_columns (); cid < _schema->num_columns (); cid++) {
166+ auto col_ptr = _input_mutable_block.mutable_columns ()[cid].get ();
167+ auto place = row_in_block->_agg_places [cid];
168+ _agg_functions[cid]->add (place,
169+ const_cast <const doris::vectorized::IColumn**>(&col_ptr),
170+ row_in_block->_row_pos , nullptr );
171+ }
172+
173+ _vec_skip_list->InsertWithHint (row_in_block, is_exist, &_vec_hint);
174+ }
167175 }
168176}
169177
@@ -284,31 +292,38 @@ void MemTable::_collect_vskiplist_to_output(bool final) {
284292}
285293
286294void MemTable::shrink_memtable_by_agg () {
287- if (_is_shrunk_by_agg) {
288- return ;
295+ {
296+ SCOPED_TIMER (_shrunk_agg_time);
297+ if (_is_shrunk_by_agg) {
298+ return ;
299+ }
300+ size_t old_size = _input_mutable_block.allocated_bytes ();
301+ _collect_vskiplist_to_output (false );
302+ size_t new_size = _input_mutable_block.allocated_bytes ();
303+ // shrink mem usage of memetable after agged.
304+ _mem_usage += new_size - old_size;
305+ _mem_tracker->consume (new_size - old_size);
306+ _is_shrunk_by_agg = true ;
289307 }
290- size_t old_size = _input_mutable_block.allocated_bytes ();
291- _collect_vskiplist_to_output (false );
292- size_t new_size = _input_mutable_block.allocated_bytes ();
293- // shrink mem usage of memetable after agged.
294- _mem_usage += new_size - old_size;
295- _mem_tracker->consume (new_size - old_size);
296- _is_shrunk_by_agg = true ;
297308}
298309
299310bool MemTable::is_full () {
300311 return memory_usage () >= config::write_buffer_size;
301312}
302313
303314Status MemTable::flush () {
304- VLOG_CRITICAL << " begin to flush memtable for tablet: " << _tablet_id
315+ clock_t now = clock ();
316+ LOG (INFO ) << " begin to flush memtable for tablet: " << _tablet_id
305317 << " , memsize: " << memory_usage () << " , rows: " << _rows;
306318 int64_t duration_ns = 0 ;
307319 RETURN_NOT_OK (_do_flush (duration_ns));
308320 DorisMetrics::instance ()->memtable_flush_total ->increment (1 );
309321 DorisMetrics::instance ()->memtable_flush_duration_us ->increment (duration_ns / 1000 );
310- VLOG_CRITICAL << " after flush memtable for tablet: " << _tablet_id
311- << " , flushsize: " << _flush_size;
322+ LOG (INFO ) << " after flush memtable for tablet: " << _tablet_id
323+ << " , flushsize: " << _flush_size
324+ << " , duration(ms): " << duration_ns / (1000 *1000 );
325+ double flush_waiting_time = (now - _flush_submit_time) / (CLOCKS_PER_SEC /1000 );
326+ LOG (INFO ) << " flush waiting time(ms): " << flush_waiting_time;
312327 return Status::OK ();
313328}
314329
0 commit comments