Skip to content

Commit df9ec8f

Browse files
committed
[opt](build) Prepare cutting the RLE/FMT_COMPILE instantiation amplifiers
Pure-additive / behavior-preserving preparation so that two kinds of template-instantiation edges can be cut next: both parquet decoder.h headers' util/rle_encoding.h include (their inline BaseDictDecoder bodies force ~530 TUs to instantiate the RleBatchDecoder<uint32_t> -> GetLiteralValues -> UnpackBatch -> UnpackValues chain at ~0.43 CPU s per TU, 231 CPU s total) and the FMT_COMPILE formatter instantiations that uint24.h / large_int_value.h push into ~1150 TUs (53.5 CPU s). No include edge is removed here: - format/parquet/decoder.h: move set_data, skip_values and the defaulted ~BaseDictDecoder out of line into decoder.cpp; everything moved is a per-page/per-batch virtual call that is dispatched through the vtable at every call site already, so outlining changes no generated call. With the dtor out of line the unique_ptr<RleBatchDecoder<uint32_t>> member no longer needs the complete type in the header; forward-declare it - format_v2/parquet/reader/native/decoder.h: same treatment for set_data, decode_dictionary_indices, decode_selected_dictionary_indices, _decode_fragmented_selection, _decode_and_validate_skipped, _decode_dictionary_values and the dtor; the pure orchestrator bodies (skip_values, decode_dictionary_values, decode_selected_dictionary_values) stay inline as they only call the now-declared members - decoder.cpp (both trees): include util/rle_encoding.h and common/cast_set.h directly (cast_set previously rode in through rle_encoding.h) - core/uint24.h: move to_string (the FMT_COMPILE("{:04d}-{:02d}-{:02d}") date formatter, the single biggest fmt instantiation in the codebase) into a new core/uint24.cpp; a std::string-returning formatter is allocation-dominated, so the outlined call is noise - core/value/large_int_value.h: move to_buffer/to_string x2 (the FMT_COMPILE int128 formatters) into large_int_value.cpp, which now includes fmt/compile.h + fmt/format.h directly instead of riding its own header The RleBatchDecoder methods stay fully inline in rle_encoding.h for the real decode TUs; the per-value-hot LevelDecoder::get_next path is deliberately untouched.
1 parent 13fca3d commit df9ec8f

8 files changed

Lines changed: 320 additions & 235 deletions

File tree

‎be/src/core/uint24.cpp‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "core/uint24.h"
19+
20+
#include <fmt/compile.h>
21+
#include <fmt/format.h>
22+
23+
namespace doris {
24+
25+
std::string uint24_t::to_string() const {
26+
int value = *reinterpret_cast<const uint24_t*>(data);
27+
int mday = value & 31;
28+
int mon = value >> 5 & 15;
29+
int year = value >> 9;
30+
31+
return fmt::format(FMT_COMPILE("{:04d}-{:02d}-{:02d}"), year, mon, mday);
32+
}
33+
34+
} // namespace doris

‎be/src/core/uint24.h‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -128,14 +128,9 @@ struct uint24_t {
128128
return 0;
129129
}
130130

131-
std::string to_string() const {
132-
int value = *reinterpret_cast<const uint24_t*>(data);
133-
int mday = value & 31;
134-
int mon = value >> 5 & 15;
135-
int year = value >> 9;
136-
137-
return fmt::format(FMT_COMPILE("{:04d}-{:02d}-{:02d}"), year, mon, mday);
138-
}
131+
// Defined in uint24.cpp: the FMT_COMPILE format machinery is expensive to
132+
// instantiate and this header is included nearly everywhere.
133+
std::string to_string() const;
139134

140135
const uint8_t* get_data() const { return data; }
141136

‎be/src/core/value/large_int_value.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,27 @@
1717

1818
#include "core/value/large_int_value.h"
1919

20+
#include <fmt/compile.h>
21+
#include <fmt/format.h>
22+
2023
#include <string>
2124

2225
#include "util/string_parser.hpp"
2326

2427
namespace doris {
2528

29+
int64_t LargeIntValue::to_buffer(__int128 value, char* buffer) {
30+
return fmt::format_to(buffer, FMT_COMPILE("{}"), value) - buffer;
31+
}
32+
33+
std::string LargeIntValue::to_string(__int128 value) {
34+
return fmt::format(FMT_COMPILE("{}"), value);
35+
}
36+
37+
std::string LargeIntValue::to_string(__uint128_t value) {
38+
return fmt::format(FMT_COMPILE("{}"), value);
39+
}
40+
2641
std::ostream& operator<<(std::ostream& os, __int128 const& value) {
2742
std::ostream::sentry s(os);
2843
if (s) {

‎be/src/core/value/large_int_value.h‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,12 @@ inline const __int128 MIN_INT128 = ((__int128)0x01 << 127);
3131

3232
class LargeIntValue {
3333
public:
34-
static int64_t to_buffer(__int128 value, char* buffer) {
35-
return fmt::format_to(buffer, FMT_COMPILE("{}"), value) - buffer;
36-
}
37-
38-
static std::string to_string(__int128 value) { return fmt::format(FMT_COMPILE("{}"), value); }
39-
static std::string to_string(__uint128_t value) {
40-
return fmt::format(FMT_COMPILE("{}"), value);
41-
}
34+
// Defined in large_int_value.cpp: instantiating the FMT_COMPILE int128
35+
// formatter in every includer of this header is expensive.
36+
static int64_t to_buffer(__int128 value, char* buffer);
37+
38+
static std::string to_string(__int128 value);
39+
static std::string to_string(__uint128_t value);
4240
};
4341

4442
std::ostream& operator<<(std::ostream& os, __int128 const& value);

‎be/src/format/parquet/decoder.cpp‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <cctz/time_zone.h>
2121
#include <gen_cpp/parquet_types.h>
2222

23+
#include "common/cast_set.h"
2324
#include "format/parquet/bool_plain_decoder.h"
2425
#include "format/parquet/bool_rle_decoder.h"
2526
#include "format/parquet/byte_array_dict_decoder.h"
@@ -28,8 +29,28 @@
2829
#include "format/parquet/delta_bit_pack_decoder.h"
2930
#include "format/parquet/fix_length_dict_decoder.hpp"
3031
#include "format/parquet/fix_length_plain_decoder.h"
32+
#include "util/rle_encoding.h"
3133

3234
namespace doris {
35+
36+
BaseDictDecoder::~BaseDictDecoder() = default;
37+
38+
Status BaseDictDecoder::set_data(Slice* data) {
39+
_data = data;
40+
_offset = 0;
41+
uint8_t bit_width = *data->data;
42+
_index_batch_decoder = std::make_unique<RleBatchDecoder<uint32_t>>(
43+
reinterpret_cast<uint8_t*>(data->data) + 1, static_cast<int>(data->size) - 1,
44+
bit_width);
45+
return Status::OK();
46+
}
47+
48+
Status BaseDictDecoder::skip_values(size_t num_values) {
49+
_indexes.resize(num_values);
50+
_index_batch_decoder->GetBatch(_indexes.data(), cast_set<uint32_t>(num_values));
51+
return Status::OK();
52+
}
53+
3354
Status Decoder::get_decoder(tparquet::Type::type type, tparquet::Encoding::type encoding,
3455
std::unique_ptr<Decoder>& decoder) {
3556
switch (encoding) {

‎be/src/format/parquet/decoder.h‎

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ namespace doris {
4545
template <typename T>
4646
class ColumnStr;
4747
using ColumnString = ColumnStr<UInt32>;
48+
template <typename T>
49+
class RleBatchDecoder;
4850

4951
class Decoder {
5052
public:
@@ -94,18 +96,13 @@ class Decoder {
9496
class BaseDictDecoder : public Decoder {
9597
public:
9698
BaseDictDecoder() = default;
97-
~BaseDictDecoder() override = default;
99+
// Out-of-line: member unique_ptr<RleBatchDecoder<uint32_t>> only needs the
100+
// complete type where the dtor/set_data/skip_values are defined (decoder.cpp),
101+
// keeping the costly RLE template machinery out of every includer of this header.
102+
~BaseDictDecoder() override;
98103

99104
// Set the data to be decoded
100-
Status set_data(Slice* data) override {
101-
_data = data;
102-
_offset = 0;
103-
uint8_t bit_width = *data->data;
104-
_index_batch_decoder = std::make_unique<RleBatchDecoder<uint32_t>>(
105-
reinterpret_cast<uint8_t*>(data->data) + 1, static_cast<int>(data->size) - 1,
106-
bit_width);
107-
return Status::OK();
108-
}
105+
Status set_data(Slice* data) override;
109106

110107
protected:
111108
/**
@@ -146,11 +143,7 @@ class BaseDictDecoder : public Decoder {
146143
return Status::OK();
147144
}
148145

149-
Status skip_values(size_t num_values) override {
150-
_indexes.resize(num_values);
151-
_index_batch_decoder->GetBatch(_indexes.data(), cast_set<uint32_t>(num_values));
152-
return Status::OK();
153-
}
146+
Status skip_values(size_t num_values) override;
154147

155148
// For dictionary encoding
156149
DorisUniqueBufferPtr<uint8_t> _dict;

0 commit comments

Comments
 (0)