Skip to content

Commit e440788

Browse files
committed
branch-4.2: [feature](inverted index) Add index_disk_usage table function #67977
### What problem does this PR solve? Issue Number: N/A Related PR: #67977 Problem Summary: Backport the index_disk_usage table function so branch-4.2 can report per-index physical bytes and their components at tablet, rowset, or segment level. Preserve the source feature, its limits, replica routing, cloud snapshots, query IO accounting, and original regression and unit tests. Use branch-4.2's GenericReader::get_next_block, direct init_reader entry, SlotDescriptor data type accessor, and IndexDef.IndexType. Traverse rowset segment IDs through existing paths and metadata, retaining direct footer reads when per-segment row counts are absent. Declare the two forwarded limits with VariableMgr.VarAttr and supply English descriptions for both language slots. Use the target schema create_block API for the unchanged integer rowset test fixture. Load the same VARIANT JSON fixture with the parser matching enable_variant_v2 on this branch. Keep existing TVFs, scanner settings, and session-clone tests. The master-only storage unity skip does not apply because this branch compiles Storage sources separately. Source PR: #67977 ### Release note Add index_disk_usage to report the physical size and components of each index. ### Check List (For Author) - Test: Preserve the source BE/FE unit tests and regression suite. Backport execution results are recorded in the pull request. - Behavior changed: Yes; add the index_disk_usage table function to branch-4.2. - Does this need documentation: Yes; tracked by the source feature PR. Conflicts: be/src/storage/CMakeLists.txt be/src/storage/segment/page_decoder.h fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java (cherry picked from commit 3c6af92)
1 parent 151738a commit e440788

30 files changed

Lines changed: 5379 additions & 14 deletions

‎be/src/exec/scan/meta_scanner.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include "core/column/column_vector.h"
3838
#include "core/data_type/define_primitive_type.h"
3939
#include "core/types.h"
40+
#include "format/table/index_disk_usage_reader.h"
4041
#include "format/table/parquet_metadata_reader.h"
4142
#include "runtime/descriptors.h"
4243
#include "runtime/exec_env.h"
@@ -69,6 +70,11 @@ Status MetaScanner::_open_impl(RuntimeState* state) {
6970
_scan_range.meta_scan_range);
7071
RETURN_IF_ERROR(reader->init_reader());
7172
_reader = std::move(reader);
73+
} else if (_scan_range.meta_scan_range.metadata_type == TMetadataType::INDEX_DISK_USAGE) {
74+
auto reader = IndexDiskUsageReader::create_unique(_tuple_desc->slots(), state, _profile,
75+
_scan_range.meta_scan_range);
76+
RETURN_IF_ERROR(reader->init_reader());
77+
_reader = std::move(reader);
7278
} else {
7379
RETURN_IF_ERROR(_fetch_metadata(_scan_range.meta_scan_range));
7480
}

‎be/src/format/table/index_disk_usage_reader.cpp‎

Lines changed: 436 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include <gen_cpp/PlanNodes_types.h>
21+
22+
#include <cstdint>
23+
#include <vector>
24+
25+
#include "common/factory_creator.h"
26+
#include "common/status.h"
27+
#include "format/generic_reader.h"
28+
#include "io/io_common.h"
29+
#include "runtime/descriptors.h"
30+
#include "storage/index/index_disk_usage.h"
31+
#include "storage/tablet/tablet_fwd.h"
32+
33+
namespace doris {
34+
class Block;
35+
class RuntimeProfile;
36+
class RuntimeState;
37+
38+
// Produces the rows of the index_disk_usage table function for the tablets of one scan range.
39+
// Each block holds the rows of one tablet, so memory stays bounded by the largest tablet.
40+
class IndexDiskUsageReader : public GenericReader {
41+
ENABLE_FACTORY_CREATOR(IndexDiskUsageReader);
42+
43+
public:
44+
// Output columns, in the order the FE schema declares them.
45+
enum class Column : uint8_t {
46+
kPartitionName,
47+
kMaterializedIndexName,
48+
kTabletId,
49+
kBackendId,
50+
kRowsetId,
51+
kSegmentId,
52+
kIndexId,
53+
kIndexName,
54+
kIndexType,
55+
kColumnName,
56+
kIndexSuffix,
57+
kStructure,
58+
kStorageFormat,
59+
kSegmentCount,
60+
kRowCount,
61+
kTotalBytes,
62+
kDictBytes,
63+
kPostingBytes,
64+
kPositionBytes,
65+
kStatsBytes,
66+
kOtherBytes,
67+
kStatsSource,
68+
};
69+
70+
IndexDiskUsageReader(std::vector<SlotDescriptor*> slots, RuntimeState* state,
71+
RuntimeProfile* profile, TMetaScanRange scan_range);
72+
~IndexDiskUsageReader() override = default;
73+
74+
Status init_reader();
75+
Status get_next_block(Block* block, size_t* read_rows, bool* eof) override;
76+
Status close() override { return Status::OK(); }
77+
const segment_v2::IndexDiskUsageOptions& options() const { return _options; }
78+
79+
// The schema that labels the rows of a tablet: the newest schema among its rowsets, because a
80+
// cached cloud tablet may not know an index that later rowsets already carry.
81+
static TabletSchemaSPtr label_schema(const TabletSchemaSPtr& tablet_schema,
82+
const std::vector<RowsetSharedPtr>& rowsets);
83+
// The IO context of one tablet: the query context plus the tablet TTL that classifies its
84+
// reads in the file cache.
85+
static io::IOContext tablet_io_context(const io::IOContext& query_io_ctx, int64_t ttl_seconds);
86+
// A tablet and its rowsets through the scan version.
87+
struct TabletRowsets {
88+
BaseTabletSPtr tablet;
89+
std::vector<RowsetSharedPtr> rowsets;
90+
};
91+
// Looks up a tablet and captures its rowsets through `version`. A cloud tablet is brought up
92+
// to date with meta-service, so rowsets that a compaction on another backend replaced are not
93+
// reported, and a tablet that was just loaded is only synchronized if it lacks `version`.
94+
static Result<TabletRowsets> capture_tablet_rowsets(int64_t tablet_id, int64_t version);
95+
96+
private:
97+
Status _collect_tablet(const TIndexDiskUsageTablet& target,
98+
std::vector<segment_v2::IndexDiskUsageRow>* rows,
99+
TabletSchemaSPtr* current_schema) const;
100+
Status _fill_block(Block* block, const TIndexDiskUsageTablet& target,
101+
const TabletSchema& schema,
102+
const std::vector<segment_v2::IndexDiskUsageRow>& rows) const;
103+
void _append_rows(std::vector<MutableColumnPtr>& columns, const TIndexDiskUsageTablet& target,
104+
const TabletSchema& schema,
105+
const std::vector<segment_v2::IndexDiskUsageRow>& rows) const;
106+
107+
RuntimeState* _state = nullptr;
108+
std::vector<SlotDescriptor*> _slots;
109+
TMetaScanRange _scan_range;
110+
std::vector<Column> _slot_columns;
111+
segment_v2::IndexDiskUsageLevel _level = segment_v2::IndexDiskUsageLevel::kTablet;
112+
// Referenced by _options, so the reader must not move after init_reader().
113+
io::IOContext _io_ctx;
114+
segment_v2::IndexDiskUsageOptions _options;
115+
size_t _next_tablet = 0;
116+
};
117+
118+
} // namespace doris

0 commit comments

Comments
 (0)