Skip to content

Commit 2ad56a8

Browse files
authored
[refine](column) strong type array and map offsets (apache#63678)
### What problem does this PR solve? Array and map offset subcolumns were still stored and exposed through generic IColumn pointers, so callers had to repeatedly downcast them back to their concrete offset column types. Root cause: the ownership, COW mutation, deserialization, and segment reader write-back paths still treated these offsets as generic subcolumns, which left the strong-typing change incomplete and kept redundant same-type assert_cast usage in affected callers and tests. This change promotes ColumnArray offsets and ColumnMap offsets_column to typed wrapped pointers, updates the typed accessors and generic write-back paths to preserve the concrete offset column type, and removes the now-redundant casts in array functions, lambda functions, and unit tests. It also cleans up one redundant nullable null-map cast uncovered during the BE UT compile. ### Release note None
1 parent a9a87f8 commit 2ad56a8

15 files changed

Lines changed: 145 additions & 91 deletions

‎be/src/core/column/column_array.cpp‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,21 @@ const ColumnArray::ColumnOffsets& check_array_offsets_column(const IColumn& offs
5858
return *offsets_concrete;
5959
}
6060

61+
ColumnArray::ColumnOffsets::Ptr check_array_offsets_column_ptr(const ColumnPtr& offsets_column) {
62+
return ColumnArray::ColumnOffsets::cast_to_column_ptr(
63+
&check_array_offsets_column(*offsets_column));
64+
}
65+
66+
ColumnArray::ColumnOffsets::MutablePtr assert_mutable_array_offsets(
67+
MutableColumnPtr&& offsets_column) {
68+
check_array_offsets_column(*offsets_column);
69+
auto mutable_offsets = ColumnArray::ColumnOffsets::cast_to_column_mutptr(
70+
assert_cast<ColumnArray::ColumnOffsets*, TypeCheckOnRelease::DISABLE>(
71+
offsets_column.get()));
72+
offsets_column = nullptr;
73+
return mutable_offsets;
74+
}
75+
6176
void validate_array_offsets(const IColumn& nested_column, const IColumn& offsets_column) {
6277
const auto& offsets_concrete = check_array_offsets_column(offsets_column);
6378
if (!offsets_concrete.empty()) {
@@ -84,7 +99,9 @@ void check_empty_array_data_without_offsets(const IColumn& nested_column) {
8499
} // namespace
85100

86101
ColumnArray::ColumnArray(MutableColumnPtr&& nested_column, MutableColumnPtr&& offsets_column)
87-
: data(std::move(nested_column)), offsets(std::move(offsets_column)) {
102+
: data(std::move(nested_column)) {
103+
static_cast<ColumnOffsets::Ptr&>(offsets) =
104+
assert_mutable_array_offsets(std::move(offsets_column));
88105
// TODO(lihangyu) : we need to check the nullable attribute of array's data column.
89106
// but currently ColumnMap<ColumnString, ColumnString> is used to store sparse data of variant type,
90107
// so I temporarily disable this check.
@@ -99,7 +116,7 @@ ColumnArray::ColumnArray(MutableColumnPtr&& nested_column, MutableColumnPtr&& of
99116
// #endif
100117
check_const_only_in_top_level();
101118
validate_array_offsets(*static_cast<const IColumn::Ptr&>(data),
102-
*static_cast<const IColumn::Ptr&>(offsets));
119+
*static_cast<const ColumnOffsets::Ptr&>(offsets));
103120

104121
/** NOTE
105122
* Arrays with constant value are possible and used in implementation of higher order functions (see FunctionReplicate).
@@ -110,15 +127,15 @@ ColumnArray::ColumnArray(MutableColumnPtr&& nested_column, MutableColumnPtr&& of
110127
ColumnArray::ColumnArray(MutableColumnPtr&& nested_column) : data(std::move(nested_column)) {
111128
data = data->convert_to_full_column_if_const();
112129
check_empty_array_data_without_offsets(*data);
113-
offsets = ColumnOffsets::create();
130+
static_cast<ColumnOffsets::Ptr&>(offsets) = ColumnOffsets::create();
114131
}
115132

116133
ColumnArray::ColumnArray(SharedTag, ColumnPtr nested_column, ColumnPtr offsets_column) {
117134
static_cast<IColumn::Ptr&>(data) = std::move(nested_column);
118-
static_cast<IColumn::Ptr&>(offsets) = std::move(offsets_column);
135+
static_cast<ColumnOffsets::Ptr&>(offsets) = check_array_offsets_column_ptr(offsets_column);
119136
check_const_only_in_top_level();
120137
validate_array_offsets(*static_cast<const IColumn::Ptr&>(data),
121-
*static_cast<const IColumn::Ptr&>(offsets));
138+
*static_cast<const ColumnOffsets::Ptr&>(offsets));
122139
}
123140

124141
std::string ColumnArray::get_name() const {
@@ -880,7 +897,7 @@ ColumnPtr ColumnArray::filter(const Filter& filt, ssize_t result_size_hint) cons
880897
res_null_map->get_data(), filt, result_size_hint);
881898

882899
auto src_data = nullable_data_column->get_nested_column_ptr();
883-
const auto* src_offsets = assert_cast<const ColumnOffsets*>(offsets.get());
900+
const auto* src_offsets = offsets.get();
884901
auto array_of_nested =
885902
filter_return_new_dispatch(filt, result_size_hint, src_data, src_offsets);
886903

@@ -889,7 +906,7 @@ ColumnPtr ColumnArray::filter(const Filter& filt, ssize_t result_size_hint) cons
889906
array_of_nested.offsets);
890907
} else {
891908
// filter offsets
892-
const auto* src_offsets = assert_cast<const ColumnOffsets*>(offsets.get());
909+
const auto* src_offsets = offsets.get();
893910
auto array_of_nested =
894911
filter_return_new_dispatch(filt, result_size_hint, data, src_offsets);
895912
return ColumnArray::create(array_of_nested.data, std::move(array_of_nested.offsets));
@@ -941,12 +958,12 @@ size_t ColumnArray::filter(const Filter& filter) {
941958
nullable_data_column->get_null_map_data(), get_offsets(), filter);
942959

943960
auto& src_data = nullable_data_column->get_nested_column();
944-
auto& src_offsets = assert_cast<ColumnOffsets&>(*offsets);
961+
auto& src_offsets = *offsets;
945962
filter_inplace_dispatch(filter, src_data, src_offsets);
946963
return result_size;
947964
} else {
948965
auto& src_data = get_data();
949-
auto& src_offsets = assert_cast<ColumnOffsets&>(*offsets);
966+
auto& src_offsets = *offsets;
950967

951968
return filter_inplace_dispatch(filter, src_data, src_offsets);
952969
}

‎be/src/core/column/column_array.h‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include "core/field.h"
3838
#include "core/string_ref.h"
3939
#include "core/types.h"
40+
#include "util/defer_op.h"
4041

4142
class SipHash;
4243

@@ -193,32 +194,35 @@ class ColumnArray final : public COWHelper<IColumn, ColumnArray> {
193194
IColumn& get_data() { return *data; }
194195
const IColumn& get_data() const { return *data; }
195196

196-
IColumn& get_offsets_column() { return *offsets; }
197-
const IColumn& get_offsets_column() const { return *offsets; }
197+
ColumnOffsets& get_offsets_column() { return *offsets; }
198+
const ColumnOffsets& get_offsets_column() const { return *offsets; }
198199

199-
Offsets64& ALWAYS_INLINE get_offsets() {
200-
return assert_cast<ColumnOffsets&, TypeCheckOnRelease::DISABLE>(*offsets).get_data();
201-
}
200+
Offsets64& ALWAYS_INLINE get_offsets() { return offsets->get_data(); }
202201

203-
const Offsets64& ALWAYS_INLINE get_offsets() const {
204-
return assert_cast<const ColumnOffsets&, TypeCheckOnRelease::DISABLE>(*offsets).get_data();
205-
}
202+
const Offsets64& ALWAYS_INLINE get_offsets() const { return offsets->get_data(); }
206203

207204
bool has_equal_offsets(const ColumnArray& other) const;
208205

209206
const ColumnPtr& get_data_ptr() const { return data; }
210207
ColumnPtr& get_data_ptr() { return data; }
211208

212-
const ColumnPtr& get_offsets_ptr() const { return offsets; }
213-
ColumnPtr& get_offsets_ptr() { return offsets; }
209+
const ColumnOffsets::Ptr& get_offsets_ptr() const {
210+
return static_cast<const ColumnOffsets::Ptr&>(offsets);
211+
}
212+
ColumnOffsets::Ptr& get_offsets_ptr() { return static_cast<ColumnOffsets::Ptr&>(offsets); }
214213

215214
size_t ALWAYS_INLINE offset_at(ssize_t i) const { return get_offsets()[i - 1]; }
216215
size_t ALWAYS_INLINE size_at(ssize_t i) const {
217216
return get_offsets()[i] - get_offsets()[i - 1];
218217
}
219218

220219
void for_each_subcolumn(ColumnCallback callback) override {
221-
callback(offsets);
220+
IColumn::WrappedPtr offsets_column(std::move(static_cast<ColumnOffsets::Ptr&>(offsets)));
221+
Defer defer([&] {
222+
static_cast<ColumnOffsets::Ptr&>(offsets) =
223+
cast_to_column<ColumnOffsets>(static_cast<const IColumn::Ptr&>(offsets_column));
224+
});
225+
callback(offsets_column);
222226
callback(data);
223227
}
224228

@@ -265,7 +269,7 @@ class ColumnArray final : public COWHelper<IColumn, ColumnArray> {
265269
// [2,1,5,9,1]\n[1,2,4] --> data column [2,1,5,9,1,1,2,4], offset[-1] = 0, offset[0] = 5, offset[1] = 8
266270
// [[2,1,5],[9,1]]\n[[1,2]] --> data column [3 column array], offset[-1] = 0, offset[0] = 2, offset[1] = 3
267271
IColumn::WrappedPtr data;
268-
IColumn::WrappedPtr offsets;
272+
ColumnOffsets::WrappedPtr offsets;
269273
};
270274

271275
} // namespace doris

‎be/src/core/column/column_map.cpp‎

Lines changed: 27 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,18 @@ const ColumnMap::COffsets& check_map_offsets_column(const IColumn& offsets_colum
5252
return *offsets_concrete;
5353
}
5454

55+
ColumnMap::COffsets::Ptr check_map_offsets_column_ptr(const ColumnPtr& offsets_column) {
56+
return ColumnMap::COffsets::cast_to_column_ptr(&check_map_offsets_column(*offsets_column));
57+
}
58+
59+
ColumnMap::COffsets::MutablePtr assert_mutable_map_offsets(MutableColumnPtr&& offsets_column) {
60+
check_map_offsets_column(*offsets_column);
61+
auto mutable_offsets = ColumnMap::COffsets::cast_to_column_mutptr(
62+
assert_cast<ColumnMap::COffsets*, TypeCheckOnRelease::DISABLE>(offsets_column.get()));
63+
offsets_column = nullptr;
64+
return mutable_offsets;
65+
}
66+
5567
void validate_map_columns(const IColumn& keys, const IColumn& values, const IColumn& offsets) {
5668
const auto& offsets_concrete = check_map_offsets_column(offsets);
5769

@@ -86,21 +98,21 @@ std::string ColumnMap::get_name() const {
8698
ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values, MutableColumnPtr&& offsets)
8799
: keys_column(std::move(keys)),
88100
values_column(std::move(values)),
89-
offsets_column(std::move(offsets)) {
101+
offsets_column(assert_mutable_map_offsets(std::move(offsets))) {
90102
check_const_only_in_top_level();
91103
validate_map_columns(*static_cast<const IColumn::Ptr&>(keys_column),
92104
*static_cast<const IColumn::Ptr&>(values_column),
93-
*static_cast<const IColumn::Ptr&>(offsets_column));
105+
*static_cast<const COffsets::Ptr&>(offsets_column));
94106
}
95107

96108
ColumnMap::ColumnMap(SharedTag, ColumnPtr keys, ColumnPtr values, ColumnPtr offsets) {
97109
static_cast<IColumn::Ptr&>(keys_column) = std::move(keys);
98110
static_cast<IColumn::Ptr&>(values_column) = std::move(values);
99-
static_cast<IColumn::Ptr&>(offsets_column) = std::move(offsets);
111+
static_cast<COffsets::Ptr&>(offsets_column) = check_map_offsets_column_ptr(offsets);
100112
check_const_only_in_top_level();
101113
validate_map_columns(*static_cast<const IColumn::Ptr&>(keys_column),
102114
*static_cast<const IColumn::Ptr&>(values_column),
103-
*static_cast<const IColumn::Ptr&>(offsets_column));
115+
*static_cast<const COffsets::Ptr&>(offsets_column));
104116
}
105117

106118
// todo. here to resize every row map
@@ -547,10 +559,10 @@ void ColumnMap::insert_range_from_ignore_overflow(const IColumn& src, size_t sta
547559

548560
ColumnPtr ColumnMap::filter(const Filter& filt, ssize_t result_size_hint) const {
549561
auto k_arr = ColumnArray::create(static_cast<const IColumn::Ptr&>(keys_column),
550-
static_cast<const IColumn::Ptr&>(offsets_column))
562+
static_cast<const IColumn::Ptr&>(get_offsets_ptr()))
551563
->filter(filt, result_size_hint);
552564
auto v_arr = ColumnArray::create(static_cast<const IColumn::Ptr&>(values_column),
553-
static_cast<const IColumn::Ptr&>(offsets_column))
565+
static_cast<const IColumn::Ptr&>(get_offsets_ptr()))
554566
->filter(filt, result_size_hint);
555567
return ColumnMap::create(assert_cast<const ColumnArray&>(*k_arr).get_data_ptr(),
556568
assert_cast<const ColumnArray&>(*v_arr).get_data_ptr(),
@@ -560,7 +572,7 @@ ColumnPtr ColumnMap::filter(const Filter& filt, ssize_t result_size_hint) const
560572
size_t ColumnMap::filter(const Filter& filter) {
561573
// Move subcolumns out of this ColumnMap to get exclusive ownership, then write back.
562574
auto keys_mut = IColumn::mutate(std::move(static_cast<IColumn::Ptr&>(keys_column)));
563-
auto offsets_mut = IColumn::mutate(std::move(static_cast<IColumn::Ptr&>(offsets_column)));
575+
auto offsets_mut = IColumn::mutate(std::move(get_offsets_ptr()));
564576
auto values_mut = IColumn::mutate(std::move(static_cast<IColumn::Ptr&>(values_column)));
565577
// Clone offsets for values (both keys and values share the same offsets structure)
566578
MutableColumnPtr copied_off = offsets_mut->clone_empty();
@@ -571,19 +583,19 @@ size_t ColumnMap::filter(const Filter& filter) {
571583
v_arr->filter(filter);
572584
// Put filtered subcolumns back
573585
static_cast<IColumn::Ptr&>(keys_column) = k_arr->get_data_ptr();
574-
static_cast<IColumn::Ptr&>(offsets_column) = k_arr->get_offsets_ptr();
586+
offsets_column = k_arr->get_offsets_ptr();
575587
static_cast<IColumn::Ptr&>(values_column) = v_arr->get_data_ptr();
576-
// Use const access to avoid assert_mutable_ref() on the just-written-back offsets_column
577-
// (k_arr still holds a ref, so use_count > 1 until k_arr goes out of scope)
578-
return static_cast<const IColumn::Ptr&>(offsets_column)->size();
588+
// Read through a const view because k_arr still shares the just-written-back offsets
589+
// until it goes out of scope, so mutable WrappedPtr access would hit assert_mutable_ref().
590+
return static_cast<const COffsets::Ptr&>(offsets_column)->size();
579591
}
580592

581593
MutableColumnPtr ColumnMap::permute(const Permutation& perm, size_t limit) const {
582594
auto k_arr = ColumnArray::create(static_cast<const IColumn::Ptr&>(keys_column),
583-
static_cast<const IColumn::Ptr&>(offsets_column))
595+
static_cast<const IColumn::Ptr&>(get_offsets_ptr()))
584596
->permute(perm, limit);
585597
auto v_arr = ColumnArray::create(static_cast<const IColumn::Ptr&>(values_column),
586-
static_cast<const IColumn::Ptr&>(offsets_column))
598+
static_cast<const IColumn::Ptr&>(get_offsets_ptr()))
587599
->permute(perm, limit);
588600

589601
return ColumnMap::create(assert_cast<const ColumnArray&>(*k_arr).get_data_ptr(),
@@ -593,9 +605,9 @@ MutableColumnPtr ColumnMap::permute(const Permutation& perm, size_t limit) const
593605

594606
Status ColumnMap::deduplicate_keys(bool recursive) {
595607
const IColumn& ck = *static_cast<const IColumn::Ptr&>(keys_column);
596-
const IColumn& co = *static_cast<const IColumn::Ptr&>(offsets_column);
608+
const auto& offsets = static_cast<const ColumnMap*>(this)->get_offsets();
597609
const auto inner_rows = ck.size();
598-
const auto rows = co.size();
610+
const auto rows = offsets.size();
599611

600612
if (recursive) {
601613
const auto& values_ptr = static_cast<const IColumn::Ptr&>(values_column);
@@ -651,8 +663,6 @@ Status ColumnMap::deduplicate_keys(bool recursive) {
651663
auto& new_offsets_data = new_offsets->get_data();
652664

653665
IColumn::Filter filter(inner_rows, 1);
654-
const auto& offsets = static_cast<const ColumnMap*>(this)->get_offsets();
655-
656666
Offset64 offset = 0;
657667
bool has_duplicated_key = false;
658668

‎be/src/core/column/column_map.h‎

Lines changed: 30 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
#include "core/string_ref.h"
4444
#include "core/types.h"
4545
#include "exec/common/sip_hash.h"
46+
#include "util/defer_op.h"
4647

4748
class SipHash;
4849

@@ -74,9 +75,14 @@ class ColumnMap final : public COWHelper<IColumn, ColumnMap> {
7475
std::string get_name() const override;
7576

7677
void for_each_subcolumn(ColumnCallback callback) override {
78+
IColumn::WrappedPtr offsets(std::move(static_cast<COffsets::Ptr&>(offsets_column)));
79+
Defer defer([&] {
80+
static_cast<COffsets::Ptr&>(offsets_column) =
81+
cast_to_column<COffsets>(static_cast<const IColumn::Ptr&>(offsets));
82+
});
7783
callback(keys_column);
7884
callback(values_column);
79-
callback(offsets_column);
85+
callback(offsets);
8086
}
8187

8288
void sanity_check() const override {
@@ -131,18 +137,17 @@ class ColumnMap final : public COWHelper<IColumn, ColumnMap> {
131137
"Method replace_column_data is not supported for " + get_name());
132138
}
133139

134-
ColumnArray::Offsets64& ALWAYS_INLINE get_offsets() {
135-
return assert_cast<COffsets&, TypeCheckOnRelease::DISABLE>(*offsets_column).get_data();
136-
}
140+
ColumnArray::Offsets64& ALWAYS_INLINE get_offsets() { return offsets_column->get_data(); }
137141
const ColumnArray::Offsets64& ALWAYS_INLINE get_offsets() const {
138-
return assert_cast<const COffsets&, TypeCheckOnRelease::DISABLE>(*offsets_column)
139-
.get_data();
142+
return offsets_column->get_data();
140143
}
141-
IColumn& get_offsets_column() { return *offsets_column; }
142-
const IColumn& get_offsets_column() const { return *offsets_column; }
144+
COffsets& get_offsets_column() { return *offsets_column; }
145+
const COffsets& get_offsets_column() const { return *offsets_column; }
143146

144-
const ColumnPtr& get_offsets_ptr() const { return offsets_column; }
145-
ColumnPtr& get_offsets_ptr() { return offsets_column; }
147+
const COffsets::Ptr& get_offsets_ptr() const {
148+
return static_cast<const COffsets::Ptr&>(offsets_column);
149+
}
150+
COffsets::Ptr& get_offsets_ptr() { return static_cast<COffsets::Ptr&>(offsets_column); }
146151

147152
size_t ALWAYS_INLINE offset_at(ssize_t i) const { return get_offsets()[i - 1]; }
148153

@@ -179,9 +184,13 @@ class ColumnMap final : public COWHelper<IColumn, ColumnMap> {
179184
IColumn& get_keys() { return *keys_column; }
180185

181186
const ColumnPtr get_keys_array_ptr() const {
182-
return ColumnArray::create(keys_column, offsets_column);
187+
return ColumnArray::create(keys_column,
188+
static_cast<const IColumn::Ptr&>(get_offsets_ptr()));
189+
}
190+
ColumnPtr get_keys_array_ptr() {
191+
return ColumnArray::create(keys_column,
192+
static_cast<const IColumn::Ptr&>(get_offsets_ptr()));
183193
}
184-
ColumnPtr get_keys_array_ptr() { return ColumnArray::create(keys_column, offsets_column); }
185194

186195
const ColumnPtr& get_values_ptr() const { return values_column; }
187196
ColumnPtr& get_values_ptr() { return values_column; }
@@ -190,9 +199,13 @@ class ColumnMap final : public COWHelper<IColumn, ColumnMap> {
190199
IColumn& get_values() { return *values_column; }
191200

192201
const ColumnPtr get_values_array_ptr() const {
193-
return ColumnArray::create(values_column, offsets_column);
202+
return ColumnArray::create(values_column,
203+
static_cast<const IColumn::Ptr&>(get_offsets_ptr()));
204+
}
205+
ColumnPtr get_values_array_ptr() {
206+
return ColumnArray::create(values_column,
207+
static_cast<const IColumn::Ptr&>(get_offsets_ptr()));
194208
}
195-
ColumnPtr get_values_array_ptr() { return ColumnArray::create(values_column, offsets_column); }
196209

197210
size_t ALWAYS_INLINE size_at(ssize_t i) const {
198211
return get_offsets()[i] - get_offsets()[i - 1];
@@ -239,9 +252,9 @@ class ColumnMap final : public COWHelper<IColumn, ColumnMap> {
239252
private:
240253
friend class COWHelper<IColumn, ColumnMap>;
241254

242-
IColumn::WrappedPtr keys_column; // nullable
243-
IColumn::WrappedPtr values_column; // nullable
244-
IColumn::WrappedPtr offsets_column; // offset
255+
IColumn::WrappedPtr keys_column; // nullable
256+
IColumn::WrappedPtr values_column; // nullable
257+
COffsets::WrappedPtr offsets_column; // offset
245258

246259
ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values, MutableColumnPtr&& offsets);
247260
ColumnMap(SharedTag, ColumnPtr keys, ColumnPtr values, ColumnPtr offsets);

‎be/src/core/data_type/data_type_array.cpp‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,11 @@ const char* DataTypeArray::deserialize(const char* buf, MutableColumnPtr* column
130130
// children
131131
auto nested_column = std::move(*data_column->get_data_ptr()).mutate();
132132
buf = get_nested_type()->deserialize(buf, &nested_column, be_exec_version);
133-
data_column->get_offsets_ptr() = std::move(offsets_column);
133+
auto typed_offsets_column = ColumnArray::ColumnOffsets::cast_to_column_mutptr(
134+
assert_cast<ColumnArray::ColumnOffsets*, TypeCheckOnRelease::DISABLE>(
135+
offsets_column.get()));
136+
offsets_column = nullptr;
137+
data_column->get_offsets_ptr() = std::move(typed_offsets_column);
134138
data_column->get_data_ptr() = std::move(nested_column);
135139
return buf;
136140
}

0 commit comments

Comments
 (0)