@@ -111,40 +111,57 @@ void SchemaColumnDataSizesScanner::_collect_column_data_sizes_from_rowsets(
111111
112112 // Get column data page stats from each segment footer and aggregate by column_unique_id
113113 for (const auto & segment : segments) {
114- auto collector = [&](const segment_v2::ColumnMetaPB& column_meta) {
114+ auto collector = [&](const segment_v2::ColumnMetaPB& column_meta,
115+ uint32_t column_unique_id) {
115116 if (column_meta.has_compressed_data_bytes () &&
116117 column_meta.has_uncompressed_data_bytes ()) {
117- auto cid = schema->field_index (column_meta. unique_id () );
118+ auto cid = schema->field_index (column_unique_id );
118119 if (cid == -1 ) {
119120 return ;
120121 }
121- // Aggregate stats by column_unique_id
122- if (aggregated_stats.contains (column_meta.unique_id ())) {
123- auto & existing_stats = aggregated_stats[column_meta.unique_id ()];
124- existing_stats.compressed_data_bytes += column_meta.compressed_data_bytes ();
125- existing_stats.uncompressed_data_bytes +=
126- column_meta.uncompressed_data_bytes ();
127- existing_stats.raw_data_bytes += column_meta.raw_data_bytes ();
128- } else {
129- aggregated_stats[column_meta.unique_id ()] = ColumnDataSizeInfo {
122+ auto [it, inserted] = aggregated_stats.try_emplace (column_unique_id);
123+ auto & stats = it->second ;
124+ if (inserted) {
125+ stats = ColumnDataSizeInfo {
130126 .backend_id = backend_id_,
131127 .table_id = table_id,
132128 .index_id = index_id,
133129 .partition_id = partition_id,
134130 .tablet_id = tablet_id,
135131 .rowset_id = rowset_id,
136- .column_unique_id = column_meta. unique_id () ,
132+ .column_unique_id = column_unique_id ,
137133 .column_name = schema->column (cid).name (),
138134 .column_type = TabletColumn::get_string_by_field_type (
139- static_cast <FieldType>(column_meta .type () )),
140- .compressed_data_bytes = column_meta. compressed_data_bytes () ,
141- .uncompressed_data_bytes = column_meta. uncompressed_data_bytes () ,
142- .raw_data_bytes = column_meta. raw_data_bytes () ,
135+ schema-> column (cid) .type ()),
136+ .compressed_data_bytes = 0 ,
137+ .uncompressed_data_bytes = 0 ,
138+ .raw_data_bytes = 0 ,
143139 };
144140 }
141+ stats.compressed_data_bytes += column_meta.compressed_data_bytes ();
142+ stats.uncompressed_data_bytes += column_meta.uncompressed_data_bytes ();
143+ stats.raw_data_bytes += column_meta.raw_data_bytes ();
145144 }
146145 };
147- st = segment->traverse_column_meta_pbs (collector);
146+ auto footer_collector = [&](const segment_v2::ColumnMetaPB& column_meta) {
147+ // A variant meta counts the data of all its inner writers, whose own metas carry no
148+ // data bytes. Compaction writes variant subcolumns as columns of their own, with
149+ // data bytes but no unique id; count them to their root variant column.
150+ uint32_t column_unique_id = column_meta.unique_id ();
151+ if (column_meta.unique_id () == -1 && column_meta.has_compressed_data_bytes ()) {
152+ DORIS_CHECK (column_meta.has_column_path_info ());
153+ column_unique_id = column_meta.column_path_info ().parrent_column_unique_id ();
154+ }
155+ collector (column_meta, column_unique_id);
156+ if (column_meta.type () == static_cast <int >(FieldType::OLAP_FIELD_TYPE_VARIANT )) {
157+ // External column meta (V3) embeds the sparse and doc value columns of a
158+ // variant into its root column meta.
159+ for (const auto & child : column_meta.children_columns ()) {
160+ collector (child, column_unique_id);
161+ }
162+ }
163+ };
164+ st = segment->traverse_column_meta_pbs (footer_collector);
148165 if (!st.ok ()) {
149166 continue ;
150167 }
@@ -354,4 +371,4 @@ Status SchemaColumnDataSizesScanner::_fill_block_impl(Block* block) {
354371 return Status::OK ();
355372}
356373
357- } // namespace doris
374+ } // namespace doris
0 commit comments