Skip to content

Commit be03bac

Browse files
committed
V2 segment file should contain no more than INT32_MAX rows apache#1647
1 parent 24afed7 commit be03bac

3 files changed

Lines changed: 10 additions & 0 deletions

File tree

‎be/src/olap/rowset/beta_rowset_writer.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,9 @@ OLAPStatus BetaRowsetWriter::_add_row(const RowType& row) {
8484
if (PREDICT_FALSE(_segment_writer == nullptr)) {
8585
RETURN_NOT_OK(_create_segment_writer());
8686
}
87+
if (PREDICT_FALSE(_segment_writer->reach_capacity())) {
88+
RETURN_NOT_OK(_flush_segment_writer());
89+
}
8790
// TODO update rowset's zonemap
8891
auto s = _segment_writer->append_row(row);
8992
if (PREDICT_FALSE(!s.ok())) {

‎be/src/olap/rowset/segment_v2/segment_writer.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,10 @@ uint64_t SegmentWriter::estimate_segment_size() {
101101
return 0;
102102
}
103103

104+
bool SegmentWriter::reach_capacity() {
105+
return _row_count >= MAX_ROW_COUNT;
106+
}
107+
104108
Status SegmentWriter::finalize(uint32_t* segment_file_size) {
105109
for (auto column_writer : _column_writers) {
106110
RETURN_IF_ERROR(column_writer->finish());

‎be/src/olap/rowset/segment_v2/segment_writer.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ class SegmentWriter {
6060

6161
uint64_t estimate_segment_size();
6262

63+
bool reach_capacity();
64+
6365
Status finalize(uint32_t* segment_file_size);
6466

6567
private:
@@ -82,6 +84,7 @@ class SegmentWriter {
8284
std::unique_ptr<WritableFile> _output_file;
8385
std::vector<ColumnWriter*> _column_writers;
8486
uint64_t _row_count = 0;
87+
const int32_t MAX_ROW_COUNT = INT_MAX;
8588
uint32_t _block_count = 0;
8689
};
8790

0 commit comments

Comments
 (0)