Skip to content

Commit 1d03523

Browse files
authored
[feature](ivm) Track MIN/MAX extreme counts for incremental refresh (apache#69286)
Incremental refresh of a MIN/MAX materialized view currently falls back to a complete refresh whenever a deleted row could be the current extreme. Deleting one of several rows that share the minimum value is a common case, yet the MV does not remember how many rows hold that extreme, so the refresh cannot tell that the extreme survives. This PR adds `COUNT_AT_MIN` / `COUNT_AT_MAX` and stores that count as a hidden IVM state column. The delta plan now emits the count at the insert-side and delete-side extreme. Apply keeps the old extreme while its surviving count is positive; only deleting the last row at the extreme still falls back to a complete refresh. Trace issue: apache#65418 ### What changed? - Add public `COUNT_AT_MIN(expr)` / `COUNT_AT_MAX(expr)` aggregate functions. They return the number of input rows equal to the group's minimum/maximum, ignore NULLs, return 0 for empty or all-NULL input, and reject DISTINCT. They are equivalent to Oracle's `COUNT(*) KEEP (DENSE_RANK FIRST/LAST ORDER BY expr)`. - Add `MIN_COUNT` / `MAX_COUNT` IVM hidden state keys and hidden columns named in the existing `__DORIS_IVM_AGG_{ordinal}_{STATE}_COL__` format. - Emit insert/delete extreme counts in the delta aggregate. - Use the surviving extreme count in the apply guard and update the hidden count state. - Add BE/FE unit coverage and a regression suite.
1 parent 9e98d36 commit 1d03523

30 files changed

Lines changed: 1564 additions & 85 deletions
Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "exprs/aggregate/aggregate_function_count_at_min_max.h"
19+
20+
#include "common/exception.h"
21+
#include "core/data_type/data_type_nullable.h"
22+
#include "core/data_type/define_primitive_type.h"
23+
#include "core/data_type/primitive_type.h"
24+
#include "exprs/aggregate/aggregate_function_simple_factory.h"
25+
#include "exprs/aggregate/factory_helpers.h"
26+
#include "exprs/aggregate/helpers.h"
27+
28+
namespace doris {
29+
30+
namespace {
31+
32+
template <template <typename> class ExtremeData>
33+
AggregateFunctionPtr create_aggregate_function_count_at_extreme(const String& name,
34+
const DataTypes& argument_types,
35+
const DataTypePtr& result_type,
36+
const bool result_is_nullable,
37+
const AggregateFunctionAttr& attr) {
38+
assert_arity_range(name, argument_types, 1, 1);
39+
switch (argument_types[0]->get_primitive_type()) {
40+
case PrimitiveType::TYPE_STRING:
41+
case PrimitiveType::TYPE_CHAR:
42+
case PrimitiveType::TYPE_VARCHAR:
43+
case PrimitiveType::TYPE_JSONB:
44+
return creator_without_type::create_unary_arguments_return_not_nullable<
45+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataString>>>(
46+
argument_types, result_is_nullable, attr);
47+
case PrimitiveType::TYPE_DATE:
48+
return creator_without_type::create_unary_arguments_return_not_nullable<
49+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_DATE>>>>(
50+
argument_types, result_is_nullable, attr);
51+
case PrimitiveType::TYPE_DATETIME:
52+
return creator_without_type::create_unary_arguments_return_not_nullable<
53+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_DATETIME>>>>(
54+
argument_types, result_is_nullable, attr);
55+
case PrimitiveType::TYPE_DATEV2:
56+
return creator_without_type::create_unary_arguments_return_not_nullable<
57+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_DATEV2>>>>(
58+
argument_types, result_is_nullable, attr);
59+
case PrimitiveType::TYPE_DATETIMEV2:
60+
return creator_without_type::create_unary_arguments_return_not_nullable<
61+
AggregateFunctionsCountAtExtreme<
62+
ExtremeData<SingleValueDataFixed<TYPE_DATETIMEV2>>>>(
63+
argument_types, result_is_nullable, attr);
64+
case PrimitiveType::TYPE_TIMESTAMP_NS:
65+
return creator_without_type::create_unary_arguments_return_not_nullable<
66+
AggregateFunctionsCountAtExtreme<
67+
ExtremeData<SingleValueDataFixed<TYPE_TIMESTAMP_NS>>>>(
68+
argument_types, result_is_nullable, attr);
69+
case PrimitiveType::TYPE_TIMESTAMPTZ:
70+
return creator_without_type::create_unary_arguments_return_not_nullable<
71+
AggregateFunctionsCountAtExtreme<
72+
ExtremeData<SingleValueDataFixed<TYPE_TIMESTAMPTZ>>>>(
73+
argument_types, result_is_nullable, attr);
74+
case PrimitiveType::TYPE_TIMEV2:
75+
return creator_without_type::create_unary_arguments_return_not_nullable<
76+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_TIMEV2>>>>(
77+
argument_types, result_is_nullable, attr);
78+
case PrimitiveType::TYPE_IPV4:
79+
return creator_without_type::create_unary_arguments_return_not_nullable<
80+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_IPV4>>>>(
81+
argument_types, result_is_nullable, attr);
82+
case PrimitiveType::TYPE_IPV6:
83+
return creator_without_type::create_unary_arguments_return_not_nullable<
84+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_IPV6>>>>(
85+
argument_types, result_is_nullable, attr);
86+
case PrimitiveType::TYPE_UUID:
87+
return creator_without_type::create_unary_arguments_return_not_nullable<
88+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_UUID>>>>(
89+
argument_types, result_is_nullable, attr);
90+
case PrimitiveType::TYPE_BOOLEAN:
91+
return creator_without_type::create_unary_arguments_return_not_nullable<
92+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_BOOLEAN>>>>(
93+
argument_types, result_is_nullable, attr);
94+
case PrimitiveType::TYPE_TINYINT:
95+
return creator_without_type::create_unary_arguments_return_not_nullable<
96+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_TINYINT>>>>(
97+
argument_types, result_is_nullable, attr);
98+
case PrimitiveType::TYPE_SMALLINT:
99+
return creator_without_type::create_unary_arguments_return_not_nullable<
100+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_SMALLINT>>>>(
101+
argument_types, result_is_nullable, attr);
102+
case PrimitiveType::TYPE_INT:
103+
return creator_without_type::create_unary_arguments_return_not_nullable<
104+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_INT>>>>(
105+
argument_types, result_is_nullable, attr);
106+
case PrimitiveType::TYPE_BIGINT:
107+
return creator_without_type::create_unary_arguments_return_not_nullable<
108+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_BIGINT>>>>(
109+
argument_types, result_is_nullable, attr);
110+
case PrimitiveType::TYPE_LARGEINT:
111+
return creator_without_type::create_unary_arguments_return_not_nullable<
112+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_LARGEINT>>>>(
113+
argument_types, result_is_nullable, attr);
114+
case PrimitiveType::TYPE_FLOAT:
115+
return creator_without_type::create_unary_arguments_return_not_nullable<
116+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_FLOAT>>>>(
117+
argument_types, result_is_nullable, attr);
118+
case PrimitiveType::TYPE_DOUBLE:
119+
return creator_without_type::create_unary_arguments_return_not_nullable<
120+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataFixed<TYPE_DOUBLE>>>>(
121+
argument_types, result_is_nullable, attr);
122+
case PrimitiveType::TYPE_DECIMAL32:
123+
return creator_without_type::create_unary_arguments_return_not_nullable<
124+
AggregateFunctionsCountAtExtreme<
125+
ExtremeData<SingleValueDataFixed<TYPE_DECIMAL32>>>>(
126+
argument_types, result_is_nullable, attr);
127+
case PrimitiveType::TYPE_DECIMAL64:
128+
return creator_without_type::create_unary_arguments_return_not_nullable<
129+
AggregateFunctionsCountAtExtreme<
130+
ExtremeData<SingleValueDataFixed<TYPE_DECIMAL64>>>>(
131+
argument_types, result_is_nullable, attr);
132+
case PrimitiveType::TYPE_DECIMALV2:
133+
return creator_without_type::create_unary_arguments_return_not_nullable<
134+
AggregateFunctionsCountAtExtreme<
135+
ExtremeData<SingleValueDataFixed<TYPE_DECIMALV2>>>>(
136+
argument_types, result_is_nullable, attr);
137+
case PrimitiveType::TYPE_DECIMAL128I:
138+
return creator_without_type::create_unary_arguments_return_not_nullable<
139+
AggregateFunctionsCountAtExtreme<
140+
ExtremeData<SingleValueDataFixed<TYPE_DECIMAL128I>>>>(
141+
argument_types, result_is_nullable, attr);
142+
case PrimitiveType::TYPE_DECIMAL256:
143+
return creator_without_type::create_unary_arguments_return_not_nullable<
144+
AggregateFunctionsCountAtExtreme<
145+
ExtremeData<SingleValueDataFixed<TYPE_DECIMAL256>>>>(
146+
argument_types, result_is_nullable, attr);
147+
case PrimitiveType::TYPE_ARRAY:
148+
case PrimitiveType::TYPE_MAP:
149+
case PrimitiveType::TYPE_STRUCT:
150+
case PrimitiveType::TYPE_AGG_STATE:
151+
case PrimitiveType::TYPE_BITMAP:
152+
case PrimitiveType::TYPE_HLL:
153+
case PrimitiveType::TYPE_QUANTILE_STATE:
154+
return creator_without_type::create_unary_arguments_return_not_nullable<
155+
AggregateFunctionsCountAtExtreme<ExtremeData<SingleValueDataColumn>>>(
156+
argument_types, result_is_nullable, attr);
157+
case PrimitiveType::TYPE_VARBINARY:
158+
throw Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
159+
"Aggregate function {} does not support "
160+
"VARBINARY arguments",
161+
name);
162+
default:
163+
return nullptr;
164+
}
165+
}
166+
167+
} // namespace
168+
169+
void register_aggregate_function_count_at_min_max(AggregateFunctionSimpleFactory& factory) {
170+
factory.register_function_both(
171+
"count_at_min",
172+
create_aggregate_function_count_at_extreme<AggregateFunctionCountAtMinData>);
173+
factory.register_function_both(
174+
"count_at_max",
175+
create_aggregate_function_count_at_extreme<AggregateFunctionCountAtMaxData>);
176+
}
177+
178+
} // namespace doris
Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include <cstdint>
21+
#include <memory>
22+
#include <string>
23+
24+
#include "core/column/column_vector.h"
25+
#include "core/data_type/data_type.h"
26+
#include "core/data_type/data_type_number.h"
27+
#include "core/types.h"
28+
#include "exprs/aggregate/aggregate_function.h"
29+
#include "exprs/aggregate/single_value_data.h"
30+
31+
namespace doris {
32+
33+
/**
34+
* State shared by COUNT_AT_MIN and COUNT_AT_MAX: the current extreme value and the
35+
* number of input rows that are equal to it. The wrapped Data type owns the value and
36+
* decides whether a new value is better (smaller for MIN, greater for MAX).
37+
*/
38+
template <typename Data>
39+
struct AggregateFunctionCountAtMinData : Data {
40+
using Self = AggregateFunctionCountAtMinData;
41+
int64_t count = 0;
42+
43+
AggregateFunctionCountAtMinData() { reset(); }
44+
45+
void change_if_better(const IColumn& column, size_t row_num, Arena& arena) {
46+
if (!this->has()) {
47+
this->set(column, row_num, arena);
48+
count = 1;
49+
} else if (this->is_equal_to(column, row_num)) {
50+
++count;
51+
} else if (this->set_if_smaller(column, row_num, arena)) {
52+
count = 1;
53+
}
54+
}
55+
56+
void change_if_better(const Self& to, Arena& arena) {
57+
if (!to.has()) {
58+
return;
59+
}
60+
if (!this->has()) {
61+
this->set(to, arena);
62+
count = to.count;
63+
} else if (this->is_equal_to(to)) {
64+
count += to.count;
65+
} else if (this->set_if_smaller(to, arena)) {
66+
count = to.count;
67+
}
68+
}
69+
70+
void reset() {
71+
Data::reset();
72+
count = 0;
73+
}
74+
75+
static const char* name() { return "count_at_min"; }
76+
};
77+
78+
template <typename Data>
79+
struct AggregateFunctionCountAtMaxData : Data {
80+
using Self = AggregateFunctionCountAtMaxData;
81+
int64_t count = 0;
82+
83+
AggregateFunctionCountAtMaxData() { reset(); }
84+
85+
void change_if_better(const IColumn& column, size_t row_num, Arena& arena) {
86+
if (!this->has()) {
87+
this->set(column, row_num, arena);
88+
count = 1;
89+
} else if (this->is_equal_to(column, row_num)) {
90+
++count;
91+
} else if (this->set_if_greater(column, row_num, arena)) {
92+
count = 1;
93+
}
94+
}
95+
96+
void change_if_better(const Self& to, Arena& arena) {
97+
if (!to.has()) {
98+
return;
99+
}
100+
if (!this->has()) {
101+
this->set(to, arena);
102+
count = to.count;
103+
} else if (this->is_equal_to(to)) {
104+
count += to.count;
105+
} else if (this->set_if_greater(to, arena)) {
106+
count = to.count;
107+
}
108+
}
109+
110+
void reset() {
111+
Data::reset();
112+
count = 0;
113+
}
114+
115+
static const char* name() { return "count_at_max"; }
116+
};
117+
118+
/**
119+
* Aggregate function returning the number of rows equal to the minimum (or maximum)
120+
* value in the group. NULL inputs are ignored. An empty group, or a group whose inputs
121+
* are all NULL, returns 0.
122+
*/
123+
template <typename Data>
124+
class AggregateFunctionsCountAtExtreme final
125+
: public IAggregateFunctionDataHelper<Data, AggregateFunctionsCountAtExtreme<Data>> {
126+
private:
127+
using Base = IAggregateFunctionDataHelper<Data, AggregateFunctionsCountAtExtreme<Data>>;
128+
129+
public:
130+
explicit AggregateFunctionsCountAtExtreme(const DataTypes& arguments) : Base(arguments) {}
131+
132+
String get_name() const override { return Data::name(); }
133+
134+
DataTypePtr get_return_type() const override { return std::make_shared<DataTypeInt64>(); }
135+
136+
void create(AggregateDataPtr __restrict place) const override { new (place) Data; }
137+
138+
void add(AggregateDataPtr __restrict place, const IColumn** columns, ssize_t row_num,
139+
Arena& arena) const override {
140+
this->data(place).change_if_better(*columns[0], row_num, arena);
141+
}
142+
143+
void reset(AggregateDataPtr place) const override { this->data(place).reset(); }
144+
145+
void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs,
146+
Arena& arena) const override {
147+
this->data(place).change_if_better(this->data(rhs), arena);
148+
}
149+
150+
void serialize(ConstAggregateDataPtr __restrict place, BufferWritable& buf) const override {
151+
this->data(place).write(buf, this->argument_types[0], IAggregateFunction::version);
152+
buf.write_binary(this->data(place).count);
153+
}
154+
155+
void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf,
156+
Arena& arena) const override {
157+
this->data(place).read(buf, this->argument_types[0], IAggregateFunction::version, arena);
158+
buf.read_binary(this->data(place).count);
159+
}
160+
161+
void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn& to) const override {
162+
assert_cast<ColumnInt64&, TypeCheckOnRelease::DISABLE>(to).insert_value(
163+
this->data(place).count);
164+
}
165+
};
166+
167+
} // namespace doris

‎be/src/exprs/aggregate/aggregate_function_simple_factory.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ void register_aggregate_function_combinator_foreachv2(AggregateFunctionSimpleFac
3232

3333
void register_aggregate_function_sum(AggregateFunctionSimpleFactory& factory);
3434
void register_aggregate_function_minmax(AggregateFunctionSimpleFactory& factory);
35+
void register_aggregate_function_count_at_min_max(AggregateFunctionSimpleFactory& factory);
3536
void register_aggregate_function_max_min_by(AggregateFunctionSimpleFactory& factory);
3637
void register_aggregate_function_avg(AggregateFunctionSimpleFactory& factory);
3738
void register_aggregate_function_count(AggregateFunctionSimpleFactory& factory);
@@ -89,6 +90,7 @@ AggregateFunctionSimpleFactory& AggregateFunctionSimpleFactory::instance() {
8990
std::call_once(oc, [&]() {
9091
register_aggregate_function_sum(instance);
9192
register_aggregate_function_minmax(instance);
93+
register_aggregate_function_count_at_min_max(instance);
9294
register_aggregate_function_max_min_by(instance);
9395
register_aggregate_function_avg(instance);
9496
register_aggregate_function_count(instance);

‎be/src/exprs/aggregate/single_value_data.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,10 @@ struct SingleValueDataFixed {
137137
return Compare::equal(_value_at(column, row_num), value);
138138
}
139139

140+
bool is_equal_to(const Self& to) const {
141+
return has() && to.has() && Compare::equal(value, to.value);
142+
}
143+
140144
bool set_if_greater(const Self& to, Arena& arena) {
141145
if (to.has() && (!has() || Compare::greater(to.value, value))) {
142146
set(to, arena);
@@ -301,6 +305,8 @@ struct SingleValueDataString {
301305
}
302306
return _value_at(column, row_num) == _value();
303307
}
308+
309+
bool is_equal_to(const Self& to) const { return has() && to.has() && _value() == to._value(); }
304310
};
305311

306312
static_assert(sizeof(SingleValueDataString) == SingleValueDataString::AUTOMATIC_STORAGE_SIZE);
@@ -412,6 +418,10 @@ class SingleValueDataColumn {
412418
bool is_equal_to(const IColumn& column, size_t row_num) const {
413419
return has() && column.compare_at(row_num, 0, *_value, 1) == 0;
414420
}
421+
422+
bool is_equal_to(const Self& to) const {
423+
return has() && to.has() && _value->compare_at(0, 0, *to._value, 1) == 0;
424+
}
415425
};
416426

417427
} // namespace doris

0 commit comments

Comments
 (0)