Skip to content

Commit 211a79a

Browse files
authored
[fix](be) Count variant data in information_schema.column_data_sizes (#68444)
1 parent fb53e87 commit 211a79a

40 files changed

Lines changed: 287 additions & 37 deletions

File tree

‎be/src/information_schema/schema_column_data_sizes_scanner.cpp‎

Lines changed: 35 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -111,40 +111,57 @@ void SchemaColumnDataSizesScanner::_collect_column_data_sizes_from_rowsets(
111111

112112
// Get column data page stats from each segment footer and aggregate by column_unique_id
113113
for (const auto& segment : segments) {
114-
auto collector = [&](const segment_v2::ColumnMetaPB& column_meta) {
114+
auto collector = [&](const segment_v2::ColumnMetaPB& column_meta,
115+
uint32_t column_unique_id) {
115116
if (column_meta.has_compressed_data_bytes() &&
116117
column_meta.has_uncompressed_data_bytes()) {
117-
auto cid = schema->field_index(column_meta.unique_id());
118+
auto cid = schema->field_index(column_unique_id);
118119
if (cid == -1) {
119120
return;
120121
}
121-
// Aggregate stats by column_unique_id
122-
if (aggregated_stats.contains(column_meta.unique_id())) {
123-
auto& existing_stats = aggregated_stats[column_meta.unique_id()];
124-
existing_stats.compressed_data_bytes += column_meta.compressed_data_bytes();
125-
existing_stats.uncompressed_data_bytes +=
126-
column_meta.uncompressed_data_bytes();
127-
existing_stats.raw_data_bytes += column_meta.raw_data_bytes();
128-
} else {
129-
aggregated_stats[column_meta.unique_id()] = ColumnDataSizeInfo {
122+
auto [it, inserted] = aggregated_stats.try_emplace(column_unique_id);
123+
auto& stats = it->second;
124+
if (inserted) {
125+
stats = ColumnDataSizeInfo {
130126
.backend_id = backend_id_,
131127
.table_id = table_id,
132128
.index_id = index_id,
133129
.partition_id = partition_id,
134130
.tablet_id = tablet_id,
135131
.rowset_id = rowset_id,
136-
.column_unique_id = column_meta.unique_id(),
132+
.column_unique_id = column_unique_id,
137133
.column_name = schema->column(cid).name(),
138134
.column_type = TabletColumn::get_string_by_field_type(
139-
static_cast<FieldType>(column_meta.type())),
140-
.compressed_data_bytes = column_meta.compressed_data_bytes(),
141-
.uncompressed_data_bytes = column_meta.uncompressed_data_bytes(),
142-
.raw_data_bytes = column_meta.raw_data_bytes(),
135+
schema->column(cid).type()),
136+
.compressed_data_bytes = 0,
137+
.uncompressed_data_bytes = 0,
138+
.raw_data_bytes = 0,
143139
};
144140
}
141+
stats.compressed_data_bytes += column_meta.compressed_data_bytes();
142+
stats.uncompressed_data_bytes += column_meta.uncompressed_data_bytes();
143+
stats.raw_data_bytes += column_meta.raw_data_bytes();
145144
}
146145
};
147-
st = segment->traverse_column_meta_pbs(collector);
146+
auto footer_collector = [&](const segment_v2::ColumnMetaPB& column_meta) {
147+
// A variant meta counts the data of all its inner writers, whose own metas carry no
148+
// data bytes. Compaction writes variant subcolumns as columns of their own, with
149+
// data bytes but no unique id; count them to their root variant column.
150+
uint32_t column_unique_id = column_meta.unique_id();
151+
if (column_meta.unique_id() == -1 && column_meta.has_compressed_data_bytes()) {
152+
DORIS_CHECK(column_meta.has_column_path_info());
153+
column_unique_id = column_meta.column_path_info().parrent_column_unique_id();
154+
}
155+
collector(column_meta, column_unique_id);
156+
if (column_meta.type() == static_cast<int>(FieldType::OLAP_FIELD_TYPE_VARIANT)) {
157+
// External column meta (V3) embeds the sparse and doc value columns of a
158+
// variant into its root column meta.
159+
for (const auto& child : column_meta.children_columns()) {
160+
collector(child, column_unique_id);
161+
}
162+
}
163+
};
164+
st = segment->traverse_column_meta_pbs(footer_collector);
148165
if (!st.ok()) {
149166
continue;
150167
}
@@ -354,4 +371,4 @@ Status SchemaColumnDataSizesScanner::_fill_block_impl(Block* block) {
354371
return Status::OK();
355372
}
356373

357-
} // namespace doris
374+
} // namespace doris

‎be/src/storage/merger.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -706,8 +706,9 @@ Status Merger::vertical_merge_rowsets(
706706
const auto& col = tablet_schema.column(col_ordinal);
707707
int32_t uid = col.unique_id();
708708

709-
// Variant columns (root or subcolumn): raw_data_bytes is 0 (TODO in writer),
710-
// cannot estimate from footer, fallback to default for the entire group.
709+
// Variant columns (root or subcolumn): subcolumns written as columns of their own
710+
// (e.g. by compaction) are not counted in the root's raw_data_bytes, cannot estimate
711+
// from footer, fallback to default for the entire group.
711712
if (uid < 0 || col.is_variant_type()) {
712713
need_fallback = true;
713714
break;

‎be/src/storage/segment/column_writer.cpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1342,6 +1342,19 @@ uint64_t VariantColumnWriter::estimate_buffer_size() {
13421342
return _impl->estimate_buffer_size();
13431343
}
13441344

1345+
uint64_t VariantColumnWriter::get_raw_data_bytes() const {
1346+
return _impl->get_total_data_pages_bytes(&ColumnWriter::get_raw_data_bytes);
1347+
}
1348+
1349+
uint64_t VariantColumnWriter::get_total_uncompressed_data_pages_bytes() const {
1350+
return _impl->get_total_data_pages_bytes(
1351+
&ColumnWriter::get_total_uncompressed_data_pages_bytes);
1352+
}
1353+
1354+
uint64_t VariantColumnWriter::get_total_compressed_data_pages_bytes() const {
1355+
return _impl->get_total_data_pages_bytes(&ColumnWriter::get_total_compressed_data_pages_bytes);
1356+
}
1357+
13451358
Status VariantColumnWriter::finish() {
13461359
return _impl->finish();
13471360
}

‎be/src/storage/segment/column_writer.h‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,8 @@ class ColumnWriter {
210210
virtual uint64_t get_raw_data_bytes() const = 0;
211211
virtual uint64_t get_total_uncompressed_data_pages_bytes() const = 0;
212212
virtual uint64_t get_total_compressed_data_pages_bytes() const = 0;
213+
// Points to one of the three data bytes getters above.
214+
using DataBytesGetter = uint64_t (ColumnWriter::*)() const;
213215

214216
// used for append not null data.
215217
virtual Status append_data(const uint8_t** ptr, size_t num_rows) = 0;
@@ -620,16 +622,14 @@ class VariantSubcolumnWriter : public ColumnWriter {
620622
Status write_bloom_filter_index() override;
621623
ordinal_t get_next_rowid() const override { return _next_rowid; }
622624

623-
uint64_t get_raw_data_bytes() const override {
624-
return 0; // TODO
625-
}
625+
uint64_t get_raw_data_bytes() const override { return _writer->get_raw_data_bytes(); }
626626

627627
uint64_t get_total_uncompressed_data_pages_bytes() const override {
628-
return 0; // TODO
628+
return _writer->get_total_uncompressed_data_pages_bytes();
629629
}
630630

631631
uint64_t get_total_compressed_data_pages_bytes() const override {
632-
return 0; // TODO
632+
return _writer->get_total_compressed_data_pages_bytes();
633633
}
634634

635635
Status append_nulls(size_t num_rows) override {
@@ -688,17 +688,11 @@ class VariantColumnWriter : public ColumnWriter {
688688
Status write_bloom_filter_index() override;
689689
ordinal_t get_next_rowid() const override { return _next_rowid; }
690690

691-
uint64_t get_raw_data_bytes() const override {
692-
return 0; // TODO
693-
}
691+
uint64_t get_raw_data_bytes() const override;
694692

695-
uint64_t get_total_uncompressed_data_pages_bytes() const override {
696-
return 0; // TODO
697-
}
693+
uint64_t get_total_uncompressed_data_pages_bytes() const override;
698694

699-
uint64_t get_total_compressed_data_pages_bytes() const override {
700-
return 0; // TODO
701-
}
695+
uint64_t get_total_compressed_data_pages_bytes() const override;
702696

703697
Status append_nulls(size_t num_rows) override {
704698
return Status::NotSupported("variant writer can not append_nulls");

‎be/src/storage/segment/variant/v2/variant_column_writer.cpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,4 +359,17 @@ uint64_t VariantV2ColumnWriter::estimate_buffer_size() {
359359
return size;
360360
}
361361

362+
uint64_t VariantV2ColumnWriter::get_total_data_pages_bytes(
363+
ColumnWriter::DataBytesGetter getter) const {
364+
DORIS_CHECK(_is_finalized);
365+
uint64_t size = std::invoke(getter, _root_writer.get());
366+
for (const auto& writer : _subcolumn_writers) {
367+
size += std::invoke(getter, writer.get());
368+
}
369+
if (_binary_writer != nullptr) {
370+
size += _binary_writer->get_total_data_pages_bytes(getter);
371+
}
372+
return size;
373+
}
374+
362375
} // namespace doris::segment_v2

‎be/src/storage/segment/variant/v2/variant_column_writer.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ class VariantV2ColumnWriter {
5959
Status write_inverted_index();
6060
Status write_bloom_filter_index();
6161
uint64_t estimate_buffer_size();
62+
// Sums `getter`, one of the ColumnWriter data bytes getters, over every column writer of this
63+
// variant: the root, the materialized subcolumns and the sparse or doc value columns.
64+
uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter) const;
6265

6366
private:
6467
Status _write_root(const IColumn* root_jsonb, int& column_id);

‎be/src/storage/segment/variant/variant_column_writer_impl.cpp‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,18 @@ uint64_t UnifiedSparseColumnWriter::estimate_buffer_size() const {
211211
return size;
212212
}
213213

214+
uint64_t UnifiedSparseColumnWriter::get_total_data_pages_bytes(
215+
ColumnWriter::DataBytesGetter getter) const {
216+
uint64_t size = 0;
217+
if (_single_writer) {
218+
size += std::invoke(getter, _single_writer.get());
219+
}
220+
for (const auto& w : _bucket_writers) {
221+
size += std::invoke(getter, w.get());
222+
}
223+
return size;
224+
}
225+
214226
Status UnifiedSparseColumnWriter::finish() {
215227
if (_single_writer) {
216228
RETURN_IF_ERROR(_single_writer->finish());
@@ -350,6 +362,17 @@ uint64_t VariantDocWriter::estimate_buffer_size() const {
350362
return size;
351363
}
352364

365+
uint64_t VariantDocWriter::get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter) const {
366+
uint64_t size = 0;
367+
for (const auto& writer : _subcolumn_writers) {
368+
size += std::invoke(getter, writer.get());
369+
}
370+
for (const auto& writer : _doc_value_column_writers) {
371+
size += std::invoke(getter, writer.get());
372+
}
373+
return size;
374+
}
375+
353376
Status VariantDocWriter::finish() {
354377
for (auto& writer : _subcolumn_writers) {
355378
RETURN_IF_ERROR(writer->finish());
@@ -520,6 +543,11 @@ uint64_t VariantColumnWriterImpl::estimate_buffer_size() {
520543
return _v2_writer ? _v2_writer->estimate_buffer_size() : 0;
521544
}
522545

546+
uint64_t VariantColumnWriterImpl::get_total_data_pages_bytes(
547+
ColumnWriter::DataBytesGetter getter) const {
548+
return _v2_writer->get_total_data_pages_bytes(getter);
549+
}
550+
523551
VariantSubcolumnWriter::VariantSubcolumnWriter(const ColumnWriterOptions& opts,
524552
TabletColumnPtr column)
525553
: ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta) {

‎be/src/storage/segment/variant/variant_column_writer_impl.h‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ class VariantBinaryWriter {
6262
virtual Status write_inverted_index() = 0;
6363
virtual Status write_bloom_filter_index() = 0;
6464
virtual uint64_t estimate_buffer_size() const = 0;
65+
// Sums `getter`, one of the ColumnWriter data bytes getters, over the owned column writers.
66+
virtual uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter) const = 0;
6567
virtual void merge_stats_to(VariantStatistics* stats) const = 0;
6668
};
6769

@@ -80,6 +82,7 @@ class VariantDocWriter : public VariantBinaryWriter {
8082
Status write_inverted_index() override;
8183
Status write_bloom_filter_index() override;
8284
uint64_t estimate_buffer_size() const override;
85+
uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter) const override;
8386
void merge_stats_to(VariantStatistics* stats) const override;
8487

8588
private:
@@ -127,6 +130,7 @@ class UnifiedSparseColumnWriter : public VariantBinaryWriter {
127130
Status write_zone_map() override;
128131
Status write_inverted_index() override;
129132
Status write_bloom_filter_index() override;
133+
uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter) const override;
130134
void merge_stats_to(VariantStatistics* stats) const override;
131135

132136
private:
@@ -171,6 +175,7 @@ class VariantColumnWriterImpl {
171175
Status write_inverted_index();
172176
Status write_bloom_filter_index();
173177
uint64_t estimate_buffer_size();
178+
uint64_t get_total_data_pages_bytes(ColumnWriter::DataBytesGetter getter) const;
174179

175180
private:
176181
Status _ensure_writer();
@@ -205,15 +210,15 @@ class VariantDocCompactWriter : public ColumnWriter {
205210
ordinal_t get_next_rowid() const override { return _next_rowid; }
206211

207212
uint64_t get_raw_data_bytes() const override {
208-
return 0; // TODO
213+
return _get_total_data_pages_bytes(&ColumnWriter::get_raw_data_bytes);
209214
}
210215

211216
uint64_t get_total_uncompressed_data_pages_bytes() const override {
212-
return 0; // TODO
217+
return _get_total_data_pages_bytes(&ColumnWriter::get_total_uncompressed_data_pages_bytes);
213218
}
214219

215220
uint64_t get_total_compressed_data_pages_bytes() const override {
216-
return 0; // TODO
221+
return _get_total_data_pages_bytes(&ColumnWriter::get_total_compressed_data_pages_bytes);
217222
}
218223

219224
Status append_nulls(size_t num_rows) override {
@@ -228,6 +233,15 @@ class VariantDocCompactWriter : public ColumnWriter {
228233
Status finalize();
229234

230235
private:
236+
template <typename Func>
237+
uint64_t _get_total_data_pages_bytes(Func func) const {
238+
uint64_t size = std::invoke(func, _doc_value_column_writer.get());
239+
for (const auto& writer : _subcolumn_writers) {
240+
size += std::invoke(func, writer.get());
241+
}
242+
return size;
243+
}
244+
231245
Status _append(const uint8_t* null_map, const uint8_t** ptr, size_t num_rows);
232246
Status _ensure_input_format(const VariantColumnData& column);
233247
Status _initialize_v2_shredder();

0 commit comments

Comments
 (0)