Skip to content

Commit d0536c0

Browse files
authored
[refine](column) use strong-typed ColumnUInt8 for null_map in ColumnNullable (apache#63491)
### What problem does this PR solve? The null_map column in `ColumnNullable` was stored as a generic `IColumn::WrappedPtr`, requiring `assert_cast<ColumnUInt8>` at every call site that accessed it. This had two problems: 1. **Redundant runtime casts**: Every access to `get_null_map_column()` or `get_null_map_column_ptr()` performed an `assert_cast`, adding unnecessary overhead on hot paths (filter evaluation, null propagation, etc.). 2. **Type safety**: The generic `WrappedPtr` type provided no compile-time guarantee that the null map is always a `ColumnUInt8`, making it possible to accidentally store a wrong column type. Root cause: The `_null_map` member was typed as a generic `IColumn::WrappedPtr` instead of the concrete `ColumnUInt8::WrappedPtr`. Fix: Change `_null_map` to `ColumnUInt8::WrappedPtr` so the type is fixed at the member level. The constructor now validates and converts to `ColumnUInt8::MutablePtr` on entry via a helper `assert_mutable_null_map()`. All call sites that previously did `assert_cast<ColumnUInt8>` now get the correct type directly. ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [ ] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent 99cb71a commit d0536c0

31 files changed

Lines changed: 178 additions & 164 deletions

‎be/src/core/column/column_array.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -266,8 +266,8 @@ class ColumnArray final : public COWHelper<IColumn, ColumnArray> {
266266
private:
267267
// [2,1,5,9,1]\n[1,2,4] --> data column [2,1,5,9,1,1,2,4], offset[-1] = 0, offset[0] = 5, offset[1] = 8
268268
// [[2,1,5],[9,1]]\n[[1,2]] --> data column [3 column array], offset[-1] = 0, offset[0] = 2, offset[1] = 3
269-
WrappedPtr data;
270-
WrappedPtr offsets;
269+
IColumn::WrappedPtr data;
270+
IColumn::WrappedPtr offsets;
271271
};
272272

273273
} // namespace doris

‎be/src/core/column/column_const.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ class ColumnConst final : public COWHelper<IColumn, ColumnConst> {
103103
private:
104104
friend class COWHelper<IColumn, ColumnConst>;
105105
using Self = ColumnConst;
106-
WrappedPtr data;
106+
IColumn::WrappedPtr data;
107107
size_t s;
108108

109109
ColumnConst(const ColumnPtr& data, size_t s_, bool create_with_empty = false,

‎be/src/core/column/column_map.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -240,9 +240,9 @@ class ColumnMap final : public COWHelper<IColumn, ColumnMap> {
240240
private:
241241
friend class COWHelper<IColumn, ColumnMap>;
242242

243-
WrappedPtr keys_column; // nullable
244-
WrappedPtr values_column; // nullable
245-
WrappedPtr offsets_column; // offset
243+
IColumn::WrappedPtr keys_column; // nullable
244+
IColumn::WrappedPtr values_column; // nullable
245+
IColumn::WrappedPtr offsets_column; // offset
246246

247247
ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values, MutableColumnPtr&& offsets);
248248
ColumnMap(SharedTag, ColumnPtr keys, ColumnPtr values, ColumnPtr offsets);

‎be/src/core/column/column_nullable.cpp‎

Lines changed: 41 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,10 @@ const ColumnUInt8& check_nullable_null_map_column(const IColumn& null_map) {
4141
return *concrete;
4242
}
4343

44+
ColumnUInt8::Ptr check_nullable_null_map_column_ptr(const ColumnPtr& null_map) {
45+
return ColumnUInt8::cast_to_column_ptr(&check_nullable_null_map_column(*null_map));
46+
}
47+
4448
void check_nullable_sizes(const IColumn& nested_column, const IColumn& null_map) {
4549
const auto& null_map_concrete = check_nullable_null_map_column(null_map);
4650
if (nested_column.size() != null_map_concrete.size()) {
@@ -53,55 +57,68 @@ void check_nullable_sizes(const IColumn& nested_column, const IColumn& null_map)
5357

5458
} // namespace
5559

60+
namespace {
61+
ColumnUInt8::MutablePtr assert_mutable_null_map(MutableColumnPtr&& null_map) {
62+
if (is_column_const(*null_map)) [[unlikely]] {
63+
throw doris::Exception(ErrorCode::INTERNAL_ERROR,
64+
"ColumnNullable cannot have constant null map");
65+
}
66+
auto mutable_null_map = ColumnUInt8::cast_to_column_mutptr(
67+
assert_cast<ColumnUInt8*, TypeCheckOnRelease::DISABLE>(null_map.get()));
68+
null_map = nullptr;
69+
return mutable_null_map;
70+
}
71+
} // namespace
72+
5673
ColumnNullable::ColumnNullable(MutableColumnPtr&& nested_column_, MutableColumnPtr&& null_map_)
74+
: ColumnNullable(std::move(nested_column_), assert_mutable_null_map(std::move(null_map_))) {
75+
}
76+
77+
ColumnNullable::ColumnNullable(MutableColumnPtr&& nested_column_,
78+
ColumnUInt8::MutablePtr&& null_map_)
5779
: _nested_column(std::move(nested_column_)), _null_map(std::move(null_map_)) {
5880
check_const_only_in_top_level();
5981
// after convert const column to full column, it may be a nullable column
6082
if (_nested_column->is_nullable()) {
61-
assert_cast<ColumnNullable&>(*_nested_column)
62-
.apply_null_map(static_cast<const ColumnUInt8&>(get_null_map_column()));
63-
_null_map = assert_cast<ColumnNullable&>(*_nested_column).get_null_map_column_ptr();
64-
_nested_column = assert_cast<ColumnNullable&>(*_nested_column).get_nested_column_ptr();
65-
}
66-
67-
if (is_column_const(get_null_map_column())) [[unlikely]] {
68-
throw doris::Exception(ErrorCode::INTERNAL_ERROR,
69-
"ColumnNullable cannot have constant null map");
70-
__builtin_unreachable();
83+
auto& nested_nullable = assert_cast<ColumnNullable&>(*_nested_column);
84+
nested_nullable.apply_null_map(get_null_map_column());
85+
_null_map = nested_nullable._null_map;
86+
_nested_column = nested_nullable.get_nested_column_ptr();
7187
}
7288
check_nullable_sizes(*static_cast<const IColumn::Ptr&>(_nested_column),
73-
*static_cast<const IColumn::Ptr&>(_null_map));
89+
*static_cast<const ColumnUInt8::Ptr&>(_null_map));
7490
}
7591

7692
ColumnNullable::ColumnNullable(SharedTag, ColumnPtr nested_column_, ColumnPtr null_map_) {
7793
check_nullable_sizes(*nested_column_, *null_map_);
7894

7995
if (const auto* nullable_nested = check_and_get_column<ColumnNullable>(nested_column_.get())) {
8096
auto merged_null_map = null_map_->clone_empty();
81-
merged_null_map->insert_range_from(*null_map_, 0, null_map_->size());
82-
auto& merged_null_map_data = assert_cast<ColumnUInt8&>(*merged_null_map).get_data();
97+
auto merged_null_map_ptr = assert_mutable_null_map(std::move(merged_null_map));
98+
merged_null_map_ptr->insert_range_from(*null_map_, 0, null_map_->size());
99+
auto& merged_null_map_data = merged_null_map_ptr->get_data();
83100
const auto& nested_null_map_data = nullable_nested->get_null_map_data();
84101
DCHECK_EQ(merged_null_map_data.size(), nested_null_map_data.size());
85102
for (size_t i = 0; i != merged_null_map_data.size(); ++i) {
86103
merged_null_map_data[i] |= nested_null_map_data[i];
87104
}
88105

89106
static_cast<IColumn::Ptr&>(_nested_column) = nullable_nested->get_nested_column_ptr();
90-
static_cast<IColumn::Ptr&>(_null_map) = std::move(merged_null_map);
107+
static_cast<ColumnUInt8::Ptr&>(_null_map) = std::move(merged_null_map_ptr);
91108
} else {
92109
static_cast<IColumn::Ptr&>(_nested_column) = std::move(nested_column_);
93-
static_cast<IColumn::Ptr&>(_null_map) = std::move(null_map_);
110+
static_cast<ColumnUInt8::Ptr&>(_null_map) = check_nullable_null_map_column_ptr(null_map_);
94111
}
95112

96113
check_const_only_in_top_level();
97114
check_nullable_sizes(*static_cast<const IColumn::Ptr&>(_nested_column),
98-
*static_cast<const IColumn::Ptr&>(_null_map));
115+
*static_cast<const ColumnUInt8::Ptr&>(_null_map));
99116
}
100117

101118
void ColumnNullable::replace_columns(ColumnPtr nested_column, ColumnPtr null_map) {
102119
check_nullable_sizes(*nested_column, *null_map);
103120
static_cast<IColumn::Ptr&>(_nested_column) = std::move(nested_column);
104-
static_cast<IColumn::Ptr&>(_null_map) = std::move(null_map);
121+
static_cast<ColumnUInt8::Ptr&>(_null_map) = check_nullable_null_map_column_ptr(null_map);
105122
check_const_only_in_top_level();
106123
}
107124

@@ -178,7 +195,8 @@ void ColumnNullable::update_crc32c_batch(uint32_t* __restrict hashes,
178195
// before this normalized nested column is written back.
179196
auto nested_mut = std::move(*static_cast<const IColumn::Ptr&>(_nested_column)).mutate();
180197
nested_mut->replace_column_null_data(real_null_data);
181-
static_cast<IColumn::Ptr&>(const_cast<WrappedPtr&>(_nested_column)) = std::move(nested_mut);
198+
static_cast<IColumn::Ptr&>(const_cast<IColumn::WrappedPtr&>(_nested_column)) =
199+
std::move(nested_mut);
182200
_nested_column->update_crc32c_batch(hashes, nullptr);
183201
} else {
184202
auto s = size();
@@ -412,10 +430,10 @@ void ColumnNullable::append_data_by_selector(IColumn::MutablePtr& res,
412430
const IColumn::Selector& selector, size_t begin,
413431
size_t end) const {
414432
auto& res_column = assert_cast<ColumnNullable&>(*res);
415-
auto res_nested_column = res_column.get_nested_column_ptr();
433+
MutableColumnPtr res_nested_column = res_column.get_nested_column_ptr();
416434
get_nested_column().append_data_by_selector(res_nested_column, selector, begin, end);
417-
auto res_null_map = res_column.get_null_map_column_ptr();
418-
get_null_map_column().append_data_by_selector(res_null_map, selector, begin, end);
435+
IColumn::MutablePtr res_null_map_column = res_column.get_null_map_column_ptr();
436+
get_null_map_column().append_data_by_selector(res_null_map_column, selector, begin, end);
419437
}
420438

421439
void ColumnNullable::pop_back(size_t n) {
@@ -440,8 +458,8 @@ Status ColumnNullable::filter_by_selector(const uint16_t* sel, size_t sel_size,
440458
auto* nullable_col_ptr = assert_cast<ColumnNullable*>(col_ptr);
441459
// Access the nested column via const path to avoid assert_mutable_ref (which requires
442460
// exclusive ownership). The output col_ptr was just created, so its nested column is exclusive.
443-
IColumn* nest_col_raw = const_cast<IColumn*>(
444-
static_cast<const WrappedPtr&>(nullable_col_ptr->_nested_column).get());
461+
auto nest_col_raw = const_cast<IColumn*>(
462+
static_cast<const IColumn::WrappedPtr&>(nullable_col_ptr->_nested_column).get());
445463

446464
/// `get_null_map_data` will set `_need_update_has_null` to true
447465
auto& res_nullmap = nullable_col_ptr->get_null_map_data();

‎be/src/core/column/column_nullable.h‎

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,13 @@
2626
#include "core/column/column_vector.h"
2727
#include "core/cow.h"
2828
#include "core/data_type/define_primitive_type.h"
29+
#include "core/data_type/primitive_type.h"
2930
#include "core/field.h"
3031
#include "core/string_ref.h"
3132
#include "core/typeid_cast.h"
3233
#include "core/types.h"
3334
#include "storage/olap_common.h"
35+
#include "util/defer_op.h"
3436

3537
class SipHash;
3638

@@ -57,6 +59,7 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable> {
5759
ColumnNullable(MutableColumnPtr&& nested_column_, MutableColumnPtr&& null_map_);
5860
struct SharedTag {};
5961
ColumnNullable(SharedTag, ColumnPtr nested_column_, ColumnPtr null_map_);
62+
ColumnNullable(MutableColumnPtr&& nested_column_, ColumnUInt8::MutablePtr&& null_map_);
6063
ColumnNullable(const ColumnNullable&) = default;
6164

6265
public:
@@ -68,6 +71,11 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable> {
6871
return Base::create(SharedTag {}, nested_column_, null_map_);
6972
}
7073

74+
static MutablePtr create(MutableColumnPtr&& nested_column_,
75+
ColumnUInt8::MutablePtr&& null_map_) {
76+
return Base::create(std::move(nested_column_), std::move(null_map_));
77+
}
78+
7179
template <typename... Args>
7280
static MutablePtr create(Args&&... args)
7381
requires IsMutableColumns<Args...>::value
@@ -246,7 +254,12 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable> {
246254

247255
void for_each_subcolumn(ColumnCallback callback) override {
248256
callback(_nested_column);
249-
callback(_null_map);
257+
258+
IColumn::WrappedPtr null_map(std::move(static_cast<ColumnUInt8::Ptr&>(_null_map)));
259+
Defer defer([&] {
260+
_null_map = cast_to_column<ColumnUInt8>(static_cast<const IColumn::Ptr&>(null_map));
261+
});
262+
callback(null_map);
250263
}
251264

252265
bool structure_equals(const IColumn& rhs) const override {
@@ -380,16 +393,16 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable> {
380393
}
381394

382395
// return the column that represents the byte map. if want use null_map, just call this.
383-
const ColumnPtr& get_null_map_column_ptr() const { return _null_map; }
384-
const ColumnUInt8& get_null_map_column() const {
385-
return assert_cast<const ColumnUInt8&, TypeCheckOnRelease::DISABLE>(*_null_map);
386-
}
396+
const ColumnUInt8::Ptr& get_null_map_column_ptr() const { return _null_map; }
397+
const ColumnUInt8& get_null_map_column() const { return *_null_map; }
387398
const NullMap& get_null_map_data() const { return get_null_map_column().get_data(); }
388399

389-
MutableColumnPtr get_null_map_column_ptr() { return _null_map->assert_mutable(); }
390-
ColumnUInt8& get_null_map_column() {
391-
return assert_cast<ColumnUInt8&, TypeCheckOnRelease::DISABLE>(*_null_map);
400+
ColumnUInt8::MutablePtr get_null_map_column_ptr() {
401+
auto null_map = _null_map->assert_mutable();
402+
return ColumnUInt8::cast_to_column_mutptr(
403+
assert_cast<ColumnUInt8*, TypeCheckOnRelease::DISABLE>(null_map.get()));
392404
}
405+
ColumnUInt8& get_null_map_column() { return *_null_map; }
393406
NullMap& get_null_map_data() { return get_null_map_column().get_data(); }
394407

395408
// push not null value wouldn't change the nullity. no need to update _has_null
@@ -403,8 +416,8 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable> {
403416
template <bool negative>
404417
void apply_null_map_impl(const ColumnUInt8& map);
405418

406-
WrappedPtr _nested_column;
407-
WrappedPtr _null_map;
419+
IColumn::WrappedPtr _nested_column;
420+
ColumnUInt8::WrappedPtr _null_map;
408421
};
409422

410423
ColumnPtr make_nullable(const ColumnPtr& column, bool is_nullable = false);

‎be/src/core/column/column_struct.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ class ColumnStruct final : public COWHelper<IColumn, ColumnStruct> {
5151
private:
5252
friend class COWHelper<IColumn, ColumnStruct>;
5353

54-
using TupleColumns = std::vector<WrappedPtr>;
54+
using TupleColumns = std::vector<IColumn::WrappedPtr>;
5555
TupleColumns columns;
5656

5757
template <bool positive>

‎be/src/core/column/column_variant.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2118,7 +2118,7 @@ void ColumnVariant::clear_sparse_column() {
21182118
}
21192119

21202120
void ColumnVariant::ensure_binary_columns_rows() {
2121-
auto resize_if_empty = [this](WrappedPtr& column) {
2121+
auto resize_if_empty = [this](IColumn::WrappedPtr& column) {
21222122
const auto& const_column = static_cast<const IColumn::Ptr&>(column);
21232123
if (const_column->size() == num_rows) {
21242124
return;

‎be/src/core/column/column_variant.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,7 @@ class ColumnVariant final : public COWHelper<IColumn, ColumnVariant> {
247247
/// Parts of column. Parts should be in increasing order in terms of subtypes/supertypes.
248248
/// That means that the least common type for i-th prefix is the type of i-th part
249249
/// and it's the supertype for all type of column from 0 to i-1.
250-
std::vector<WrappedPtr> data;
250+
std::vector<IColumn::WrappedPtr> data;
251251
std::vector<DataTypePtr> data_types;
252252
std::vector<DataTypeSerDeSPtr> data_serdes;
253253
/// Until we insert any non-default field we don't know further
@@ -275,10 +275,10 @@ class ColumnVariant final : public COWHelper<IColumn, ColumnVariant> {
275275

276276
// It's filled when the number of subcolumns reaches the limit.
277277
// It has type Map(String, String) and stores a map (path, binary serialized subcolumn value) for each row.
278-
WrappedPtr serialized_sparse_column = ColumnMap::create(
278+
IColumn::WrappedPtr serialized_sparse_column = ColumnMap::create(
279279
ColumnString::create(), ColumnString::create(), ColumnArray::ColumnOffsets::create());
280280

281-
WrappedPtr serialized_doc_value_column = ColumnMap::create(
281+
IColumn::WrappedPtr serialized_doc_value_column = ColumnMap::create(
282282
ColumnString::create(), ColumnString::create(), ColumnArray::ColumnOffsets::create());
283283

284284
// if `_max_subcolumns_count == 0`, all subcolumns are materialized.

‎be/src/core/cow.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -351,13 +351,13 @@ class COW {
351351
: value(std::forward<std::initializer_list<U>>(arg)) {}
352352

353353
const T* get() const { return value.get(); }
354-
T* get() { return &value->assert_mutable_ref(); }
354+
T* get() { return &static_cast<T&>(value->assert_mutable_ref()); }
355355

356356
const T* operator->() const { return get(); }
357357
T* operator->() { return get(); }
358358

359359
const T& operator*() const { return *value; }
360-
T& operator*() { return value->assert_mutable_ref(); }
360+
T& operator*() { return static_cast<T&>(value->assert_mutable_ref()); }
361361

362362
operator const immutable_ptr<T>&() const { return value; }
363363
operator immutable_ptr<T>&() { return value; }
@@ -419,6 +419,7 @@ class COWHelper : public Base {
419419
static_assert(std::is_base_of_v<doris::IColumn, Base>, "COWHelper only use in IColumn");
420420
using Ptr = typename Base::template immutable_ptr<Derived>;
421421
using MutablePtr = typename Base::template mutable_ptr<Derived>;
422+
using WrappedPtr = typename Base::template chameleon_ptr<Derived>;
422423

423424
#include "common/compile_check_avoid_begin.h"
424425

‎be/src/exec/operator/table_function_operator.cpp‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -282,8 +282,7 @@ Status TableFunctionLocalState::_get_expanded_block_block_fast_path(
282282
if (out_col->is_nullable()) {
283283
auto* nullable = assert_cast<ColumnNullable*>(out_col.get());
284284
struct_col_ptr = assert_cast<ColumnStruct*>(nullable->get_nested_column_ptr().get());
285-
outer_struct_nullmap_ptr =
286-
assert_cast<ColumnUInt8*>(nullable->get_null_map_column_ptr().get());
285+
outer_struct_nullmap_ptr = nullable->get_null_map_column_ptr().get();
287286
} else {
288287
struct_col_ptr = assert_cast<ColumnStruct*>(out_col.get());
289288
}
@@ -343,8 +342,7 @@ Status TableFunctionLocalState::_get_expanded_block_block_fast_path(
343342
val_nullable->get_nested_column_ptr()->insert_range_from(
344343
*_block_fast_path_ctx.nested_col, segment_ctx.seg_nested_start,
345344
segment_ctx.seg_nested_count);
346-
auto* val_nullmap =
347-
assert_cast<ColumnUInt8*>(val_nullable->get_null_map_column_ptr().get());
345+
auto* val_nullmap = val_nullable->get_null_map_column_ptr().get();
348346
auto& val_nullmap_data = val_nullmap->get_data();
349347
const size_t old_size = val_nullmap_data.size();
350348
val_nullmap_data.resize(old_size + segment_ctx.seg_nested_count);
@@ -365,8 +363,7 @@ Status TableFunctionLocalState::_get_expanded_block_block_fast_path(
365363
out_nullable->get_nested_column_ptr()->insert_range_from(
366364
*_block_fast_path_ctx.nested_col, segment_ctx.seg_nested_start,
367365
segment_ctx.seg_nested_count);
368-
auto* nullmap_column =
369-
assert_cast<ColumnUInt8*>(out_nullable->get_null_map_column_ptr().get());
366+
auto* nullmap_column = out_nullable->get_null_map_column_ptr().get();
370367
auto& nullmap_data = nullmap_column->get_data();
371368
const size_t old_size = nullmap_data.size();
372369
nullmap_data.resize(old_size + segment_ctx.seg_nested_count);

0 commit comments

Comments
 (0)