Skip to content

Commit 627b590

Browse files
committed
f
1 parent 81556a1 commit 627b590

100 files changed

Lines changed: 5207 additions & 1617 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/AGENTS.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,20 @@ past a gate is never to bypass it, but one of:
2929

3030
## Rules while writing BE code
3131

32+
- **Make non-obvious comments concrete.** Comments should explain why the
33+
code works this way, including the underlying invariant, algorithm, or
34+
trade-off. When that explanation is abstract or requires careful reasoning,
35+
also include a small representative example (such as sample inputs, a state
36+
transition, or an edge case) that shows the principle in action. The example
37+
should clarify the reasoning rather than merely restate the code, and it must
38+
be updated when the behavior changes.
39+
- **Document conditional branches introduced by bug fixes.** Whenever a bug
40+
fix adds or changes an `if`/`else` condition, add a nearby comment that
41+
describes the concrete failure scenario and explains why the condition is
42+
required. If the boundary is not immediately obvious, include a concise
43+
example showing which path the affected input or state must take. The
44+
comment does not replace the requirement that every error check have a known,
45+
inevitable failure path.
3246
- **Adding an `#include` to a widely-included (hub) header is a design
3347
decision, not a convenience.** Everything a hub includes is reparsed by
3448
every TU behind it (~1000 TUs for `exec_env.h`, `thread_context.h`,
@@ -56,6 +70,11 @@ past a gate is never to bypass it, but one of:
5670

5771
## Review checkpoints (AI review and self-review)
5872

73+
- [ ] Non-obvious comments explain the reasoning, and abstract principles are
74+
backed by a concise, representative example that still matches the code.
75+
- [ ] An `if`/`else` condition added or changed by a bug fix has a nearby
76+
comment describing the concrete failure scenario and why the branch is
77+
required.
5978
- [ ] New includes in hub headers: could a forward declaration or `*_fwd.h`
6079
carry this instead? Does the PR pay a closure/reach budget bump — and
6180
if so, does the commit message justify it?

‎be/src/core/data_type/data_type_factory.cpp‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
#include "core/data_type/data_type_time.h"
6262
#include "core/data_type/data_type_timestamp_ns.h"
6363
#include "core/data_type/data_type_timestamptz.h"
64+
#include "core/data_type/data_type_uint.h"
6465
#include "core/data_type/data_type_uuid.h"
6566
#include "core/data_type/data_type_varbinary.h"
6667
#include "core/data_type/data_type_variant.h"
@@ -130,12 +131,18 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type,
130131
case FieldType::OLAP_FIELD_TYPE_INT:
131132
result = std::make_shared<DataTypeInt32>();
132133
break;
134+
case FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT:
135+
result = std::make_shared<DataTypeUInt32>();
136+
break;
133137
case FieldType::OLAP_FIELD_TYPE_FLOAT:
134138
result = std::make_shared<DataTypeFloat32>();
135139
break;
136140
case FieldType::OLAP_FIELD_TYPE_BIGINT:
137141
result = std::make_shared<DataTypeInt64>();
138142
break;
143+
case FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT:
144+
result = std::make_shared<DataTypeUInt64>();
145+
break;
139146
case FieldType::OLAP_FIELD_TYPE_LARGEINT:
140147
result = std::make_shared<DataTypeInt128>();
141148
break;
@@ -157,6 +164,9 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type,
157164
case FieldType::OLAP_FIELD_TYPE_DATETIMEV2:
158165
result = create_datetimev2(scale);
159166
break;
167+
case FieldType::OLAP_FIELD_TYPE_TIMEV2:
168+
result = std::make_shared<DataTypeTimeV2>(scale);
169+
break;
160170
case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS:
161171
result = std::make_shared<DataTypeTimeStampNs>();
162172
break;
@@ -203,8 +213,8 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type,
203213
result = create_decimal(precision, scale, false);
204214
break;
205215
default:
206-
result = nullptr;
207-
break;
216+
throw doris::Exception(ErrorCode::INTERNAL_ERROR, "Unsupported FieldType: {}",
217+
static_cast<int>(type));
208218
}
209219
return result;
210220
}
@@ -365,12 +375,13 @@ DataTypePtr DataTypeFactory::create_data_type(const segment_v2::ColumnMetaPB& pc
365375
DataTypePtr nested = nullptr;
366376
if (pcolumn.type() == static_cast<int>(FieldType::OLAP_FIELD_TYPE_AGG_STATE)) {
367377
DataTypes data_types;
368-
for (auto child : pcolumn.children_columns()) {
369-
auto type = DataTypeFactory::instance().create_data_type(child);
370-
// may have length column with OLAP_FIELD_TYPE_UNSIGNED_BIGINT, then type will be nullptr
371-
if (type) {
372-
data_types.push_back(type);
378+
for (const auto& child : pcolumn.children_columns()) {
379+
// ARRAY/MAP serialized aggregate states append their physical offset stream after the
380+
// logical argument columns. It is storage metadata, not an aggregate argument type.
381+
if (child.type() == static_cast<int>(FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT)) {
382+
continue;
373383
}
384+
data_types.push_back(DataTypeFactory::instance().create_data_type(child));
374385
}
375386
nested = std::make_shared<DataTypeAggState>(data_types, pcolumn.result_is_nullable(),
376387
pcolumn.function_name(),
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "core/data_type/data_type_uint.h"
19+
20+
#include <typeinfo>
21+
22+
#include "common/exception.h"
23+
#include "core/column/column_vector.h"
24+
25+
namespace doris {
26+
27+
template <PrimitiveType T>
28+
const std::string DataTypeUInt<T>::get_family_name() const {
29+
return type_to_string(T);
30+
}
31+
32+
template <PrimitiveType T>
33+
Field DataTypeUInt<T>::get_field(const TExprNode&) const {
34+
throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
35+
"SQL literals are not supported for storage-only type {}", get_name());
36+
}
37+
38+
template <PrimitiveType T>
39+
bool DataTypeUInt<T>::equals(const IDataType& rhs) const {
40+
return typeid(rhs) == typeid(*this);
41+
}
42+
43+
template <PrimitiveType T>
44+
int64_t DataTypeUInt<T>::get_uncompressed_serialized_bytes(const IColumn&, int) const {
45+
throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
46+
"Block serialization is not supported for storage-only type {}", get_name());
47+
}
48+
49+
template <PrimitiveType T>
50+
char* DataTypeUInt<T>::serialize(const IColumn&, char*, int) const {
51+
throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
52+
"Block serialization is not supported for storage-only type {}", get_name());
53+
}
54+
55+
template <PrimitiveType T>
56+
const char* DataTypeUInt<T>::deserialize(const char*, MutableColumnPtr*, int) const {
57+
throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
58+
"Block deserialization is not supported for storage-only type {}", get_name());
59+
}
60+
61+
template <PrimitiveType T>
62+
MutableColumnPtr DataTypeUInt<T>::create_column() const {
63+
return ColumnType::create();
64+
}
65+
66+
template <PrimitiveType T>
67+
Status DataTypeUInt<T>::check_column(const IColumn& column) const {
68+
return check_column_non_nested_type<ColumnType>(column);
69+
}
70+
71+
template <PrimitiveType T>
72+
DataTypeSerDeSPtr DataTypeUInt<T>::get_serde(int) const {
73+
throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
74+
"SerDe is not supported for storage-only type {}", get_name());
75+
}
76+
77+
template class DataTypeUInt<TYPE_UINT32>;
78+
template class DataTypeUInt<TYPE_UINT64>;
79+
80+
} // namespace doris
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include "core/data_type/data_type.h"
21+
22+
namespace doris {
23+
24+
// UINT32/UINT64 are physical storage types for internal columns such as ARRAY/MAP offsets. They
25+
// need an IDataType so every physical ColumnReader has a valid type, but they are not SQL numeric
26+
// types. For example, an ARRAY offset reader can create and validate ColumnOffset64, while attempts
27+
// to serialize that physical column through a logical SerDe must fail explicitly.
28+
template <PrimitiveType T>
29+
class DataTypeUInt final : public IDataType {
30+
static_assert(T == TYPE_UINT32 || T == TYPE_UINT64);
31+
32+
public:
33+
static constexpr PrimitiveType PType = T;
34+
using ColumnType = typename PrimitiveTypeTraits<T>::ColumnType;
35+
36+
const std::string get_family_name() const override;
37+
PrimitiveType get_primitive_type() const override { return PType; }
38+
39+
[[noreturn]] Field get_field(const TExprNode& node) const override;
40+
bool equals(const IDataType& rhs) const override;
41+
42+
bool have_maximum_size_of_value() const override { return true; }
43+
size_t get_size_of_value_in_memory() const override {
44+
return sizeof(typename PrimitiveTypeTraits<T>::CppType);
45+
}
46+
47+
[[noreturn]] int64_t get_uncompressed_serialized_bytes(const IColumn& column,
48+
int be_exec_version) const override;
49+
[[noreturn]] char* serialize(const IColumn& column, char* buf,
50+
int be_exec_version) const override;
51+
[[noreturn]] const char* deserialize(const char* buf, MutableColumnPtr* column,
52+
int be_exec_version) const override;
53+
54+
MutableColumnPtr create_column() const override;
55+
Status check_column(const IColumn& column) const override;
56+
57+
[[noreturn]] DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override;
58+
};
59+
60+
extern template class DataTypeUInt<TYPE_UINT32>;
61+
extern template class DataTypeUInt<TYPE_UINT64>;
62+
63+
} // namespace doris

‎be/src/core/data_type/primitive_type.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,12 @@ std::string type_to_string(PrimitiveType t) {
391391

392392
case TYPE_TIMESTAMPTZ:
393393
return "TIMESTAMPTZ";
394+
395+
case TYPE_UINT32:
396+
return "UINT32";
397+
398+
case TYPE_UINT64:
399+
return "UINT64";
394400
default:
395401
return "";
396402
};

‎be/src/core/data_type/primitive_type.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ using ColumnHLL = ColumnComplexType<TYPE_HLL>;
6060
using ColumnQuantileState = ColumnComplexType<TYPE_QUANTILE_STATE>;
6161
template <PrimitiveType T>
6262
class DataTypeNumber;
63+
template <PrimitiveType T>
64+
class DataTypeUInt;
6365
using DataTypeInt8 = DataTypeNumber<TYPE_TINYINT>;
6466
using DataTypeInt16 = DataTypeNumber<TYPE_SMALLINT>;
6567
using DataTypeInt32 = DataTypeNumber<TYPE_INT>;
@@ -69,6 +71,8 @@ using DataTypeFloat32 = DataTypeNumber<TYPE_FLOAT>;
6971
using DataTypeFloat64 = DataTypeNumber<TYPE_DOUBLE>;
7072
using DataTypeUInt8 = DataTypeNumber<TYPE_BOOLEAN>;
7173
using DataTypeBool = DataTypeNumber<TYPE_BOOLEAN>;
74+
using DataTypeUInt32 = DataTypeUInt<TYPE_UINT32>;
75+
using DataTypeUInt64 = DataTypeUInt<TYPE_UINT64>;
7276

7377
class DataTypeNothing;
7478
class DataTypeTimeV2;
@@ -544,14 +548,14 @@ template <>
544548
struct PrimitiveTypeTraits<TYPE_UINT32> {
545549
using CppType = UInt32;
546550
using StorageFieldType = CppType;
547-
using DataType = DataTypeNothing;
551+
using DataType = DataTypeUInt32;
548552
using ColumnType = ColumnOffset32;
549553
};
550554
template <>
551555
struct PrimitiveTypeTraits<TYPE_UINT64> {
552556
using CppType = UInt64;
553557
using StorageFieldType = CppType;
554-
using DataType = DataTypeNothing;
558+
using DataType = DataTypeUInt64;
555559
using ColumnType = ColumnOffset64;
556560
};
557561

‎be/src/core/data_type_serde/data_type_serde.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -356,7 +356,7 @@ class DataTypeSerDe {
356356

357357
/// Parse a human-readable string from FE (delete conditions, default values,
358358
/// schema change defaults) into a Field. Uses standard decimal parsing with full
359-
/// scale. Callers: delete_handler.cpp, column_reader.cpp (DefaultValueColumnIterator),
359+
/// scale. Callers: delete_handler.cpp, segment.cpp (Segment::get_default_value_field),
360360
/// schema_change.cpp.
361361
virtual Status from_fe_string(const std::string& str, Field& field) const {
362362
FormatOptions options;

‎be/src/core/wide_integer_to_string.h‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,7 @@
2323

2424
#include <string>
2525

26-
namespace wide {
27-
template <size_t Bits, typename Signed>
28-
class integer;
29-
}
26+
#include "core/wide_integer.h"
3027

3128
using Int128 = wide::integer<128, signed>;
3229
using UInt128 = wide::integer<128, unsigned>;

0 commit comments

Comments
 (0)