Skip to content

Commit 822f212

Browse files
committed
[fix](be) Ignore parameters of non-contributing aggregate states
### What problem does this PR solve? Issue Number: N/A Related PR: #67805 Problem Summary: Parameter checks rejected merges of aggregate states without retained counters, elements, samples or events even though these payloads do not contribute to the result. An empty destination could also retain parameters that conflict with subsequent contributing states. Skip non-contributing states before compatibility checks and adopt contributing parameters into empty destinations across TopN, limited collect, percentile, reservoir, sequence and Window Funnel V2 implementations. Remove the V2 initialization flag and sorted-field tag. Retain checks for states that still contribute, including V1 fixed-mode rows, bitmap filter keys, group_concat empty strings and EMA value/time pairs. Preserve empty percentile-array result shapes and negative collection-limit behavior. Historical trial AggState encodings remain outside the compatibility scope. ### Release note Non-contributing aggregate states ignore parameter differences during merging. Empty destinations adopt the contributing state's parameters. Nonempty states continue to reject incompatible parameters. ### Check List (For Author) - Test: Unit Test / Regression test / Manual test - BE ASAN and FE builds passed. - 72 BE ASAN tests passed across 10 suites. - All 7 SQL regression suites passed with zero failures or skips. Generated the 132 new outputs with the repository runner and verified them in a normal run; merge/union and both operand orders agree. - clang-format 16.0.6, header hygiene and git diff --check passed. - clang-tidy reports no diagnostics on changed lines after test-helper refactoring. The full check still reports existing core/types.h NOLINTEND and other pre-existing diagnostics. - Behavior changed: Yes, empty payload parameters no longer reject merges. - Does this need documentation: Yes, updated docs/aggregate-state-parameters.md.
1 parent 1ba9b39 commit 822f212

12 files changed

Lines changed: 645 additions & 174 deletions

‎be/src/exprs/aggregate/aggregate_function_collect.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -465,10 +465,12 @@ class AggregateFunctionCollect final
465465
auto& data = this->data(place);
466466
const auto& rhs_data = this->data(rhs);
467467
if constexpr (HasLimit) {
468-
if (!rhs_data.is_initialized()) {
468+
if (rhs_data.size() == 0) {
469469
return;
470470
}
471-
if (data.is_initialized()) {
471+
if (data.size() == 0) {
472+
data.max_size = rhs_data.max_size;
473+
} else {
472474
if (UNLIKELY(data.max_size != rhs_data.max_size)) {
473475
throw Exception(ErrorCode::INVALID_ARGUMENT,
474476
"{} aggregate states have incompatible limits: {} vs {}",

‎be/src/exprs/aggregate/aggregate_function_percentile.h‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -137,10 +137,10 @@ struct PercentileApproxState {
137137
}
138138

139139
void merge(const PercentileApproxState& rhs) {
140-
if (!rhs.init_flag) {
140+
if (!rhs.init_flag || rhs.digest->total_size() == 0) {
141141
return;
142142
}
143-
if (!init_flag) {
143+
if (!init_flag || digest->total_size() == 0) {
144144
target_quantile = rhs.target_quantile;
145145
compressions = rhs.compressions;
146146
digest = TDigest::create_unique(compressions);
@@ -408,13 +408,17 @@ struct PercentileApproxArrayState {
408408
return;
409409
}
410410

411-
if (!init_flag) {
412-
levels.merge(rhs.levels);
411+
// Preserve the result shape when every state is empty, but let contributing
412+
// samples replace parameters recorded by an empty destination.
413+
if (!init_flag || levels.empty() || digest->total_size() == 0) {
414+
levels = rhs.levels;
413415
compressions = rhs.compressions;
414416
if (!levels.empty()) {
415417
digest = TDigest::create_unique(compressions);
416418
}
417419
init_flag = true;
420+
} else if (rhs.levels.empty() || rhs.digest->total_size() == 0) {
421+
return;
418422
} else {
419423
if (UNLIKELY(compressions != rhs.compressions ||
420424
levels.quantiles != rhs.levels.quantiles)) {
@@ -639,11 +643,11 @@ struct PercentileState {
639643
}
640644

641645
void merge(const PercentileState& rhs) {
642-
if (!rhs.inited_flag) {
646+
if (rhs.vec_counts.empty()) {
643647
return;
644648
}
645649
int size_num = cast_set<int>(rhs.vec_quantile.size());
646-
if (!inited_flag) {
650+
if (vec_counts.empty()) {
647651
vec_counts.resize(size_num);
648652
vec_quantile = rhs.vec_quantile;
649653
inited_flag = true;
@@ -736,9 +740,11 @@ struct PercentileExactState {
736740
return;
737741
}
738742

739-
if (!inited_flag) {
743+
if (values.empty()) {
740744
levels = rhs.levels;
741745
inited_flag = true;
746+
} else if (rhs.values.empty()) {
747+
return;
742748
} else if (UNLIKELY(levels.quantiles != rhs.levels.quantiles)) {
743749
throw Exception(ErrorCode::INVALID_ARGUMENT,
744750
"percentile aggregate states have incompatible quantiles");

‎be/src/exprs/aggregate/aggregate_function_percentile_reservoir.h‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,10 @@ struct QuantileReservoirSampler {
4545
}
4646

4747
void merge(const QuantileReservoirSampler& rhs) {
48-
// NaN samples do not erase the quantile established by a non-null input row.
49-
if (rhs.level == INIT_QUANTILE) {
48+
if (rhs.data.empty()) {
5049
return;
5150
}
52-
if (level == INIT_QUANTILE) {
51+
if (data.empty()) {
5352
level = rhs.level;
5453
} else if (UNLIKELY(level != rhs.level)) {
5554
throw Exception(ErrorCode::INVALID_ARGUMENT,
@@ -59,7 +58,7 @@ struct QuantileReservoirSampler {
5958
}
6059

6160
void reset() {
62-
level = INIT_QUANTILE;
61+
level = 0.0;
6362
data.clear();
6463
}
6564

@@ -80,9 +79,7 @@ struct QuantileReservoirSampler {
8079
}
8180

8281
private:
83-
// Valid quantiles include zero, so use an out-of-range value for fresh/reset states.
84-
static constexpr double INIT_QUANTILE = -1.0;
85-
double level = INIT_QUANTILE;
82+
double level = 0.0;
8683
ReservoirSampler data;
8784
};
8885

‎be/src/exprs/aggregate/aggregate_function_sequence_match.h‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -118,20 +118,17 @@ struct AggregateFunctionSequenceMatchData final {
118118
}
119119

120120
void merge(const AggregateFunctionSequenceMatchData& other) {
121-
// All-false event rows still establish a pattern that must match during merge.
122-
if (!other.init_flag) {
121+
if (other.events_list.empty()) {
123122
return;
124123
}
125124

126-
if (!init_flag) {
125+
if (events_list.empty()) {
126+
reset();
127127
init(other.pattern, other.arg_count);
128128
} else if (UNLIKELY(pattern != other.pattern || arg_count != other.arg_count)) {
129129
throw Exception(ErrorCode::INVALID_ARGUMENT,
130130
"sequence aggregate states have incompatible patterns or event counts");
131131
}
132-
if (other.events_list.empty()) {
133-
return;
134-
}
135132

136133
events_list.insert(std::end(events_list), std::begin(other.events_list),
137134
std::end(other.events_list));

‎be/src/exprs/aggregate/aggregate_function_topn.h‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -82,30 +82,21 @@ struct AggregateFunctionTopNData {
8282
}
8383

8484
void merge(const AggregateFunctionTopNData& rhs) {
85-
if (!rhs.top_num) {
85+
if (!rhs.top_num || rhs.counter_map.empty()) {
8686
return;
8787
}
8888

89-
if (!top_num) {
90-
top_num = rhs.top_num;
91-
capacity = rhs.capacity;
92-
} else if (UNLIKELY(top_num != rhs.top_num || capacity != rhs.capacity)) {
89+
if (counter_map.empty()) {
90+
*this = rhs;
91+
return;
92+
}
93+
if (UNLIKELY(top_num != rhs.top_num || capacity != rhs.capacity)) {
9394
throw Exception(ErrorCode::INVALID_ARGUMENT,
9495
"topn aggregate states have incompatible parameters: "
9596
"({}, {}) vs ({}, {}) (N, capacity)",
9697
top_num, capacity, rhs.top_num, rhs.capacity);
9798
}
9899

99-
// Empty payloads still carry configuration. Check it above, then avoid treating
100-
// an empty zero-capacity map as full and adding UINT64_MAX to real counters.
101-
if (rhs.counter_map.empty()) {
102-
return;
103-
}
104-
if (counter_map.empty()) {
105-
counter_map = rhs.counter_map;
106-
return;
107-
}
108-
109100
bool lhs_full = (counter_map.size() >= capacity);
110101
bool rhs_full = (rhs.counter_map.size() >= capacity);
111102

‎be/src/exprs/aggregate/aggregate_function_window_funnel_v2.h‎

Lines changed: 16 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,6 @@ struct WindowFunnelStateV2 {
124124
int event_count = 0;
125125
int64_t window = WINDOW_UNSET;
126126
WindowFunnelMode window_funnel_mode = WindowFunnelMode::INVALID;
127-
bool initialized = false;
128127
bool sorted = true;
129128
std::vector<TimestampEvent> events_list;
130129

@@ -134,15 +133,13 @@ struct WindowFunnelStateV2 {
134133
void reset() {
135134
window = WINDOW_UNSET;
136135
window_funnel_mode = WindowFunnelMode::INVALID;
137-
initialized = false;
138136
events_list.clear();
139137
sorted = true;
140138
}
141139

142140
void add(const IColumn** arg_columns, ssize_t row_num, int64_t win, WindowFunnelMode mode) {
143141
window = win;
144142
window_funnel_mode = mode;
145-
initialized = true;
146143

147144
auto timestamp = assert_cast<const typename PrimitiveTypeTraits<T>::ColumnType&,
148145
TypeCheckOnRelease::DISABLE>(*arg_columns[2])
@@ -184,49 +181,35 @@ struct WindowFunnelStateV2 {
184181
}
185182

186183
void merge(const WindowFunnelStateV2<T>& other) {
187-
if (!other.initialized) {
184+
if (other.events_list.empty()) {
185+
return;
186+
}
187+
if (events_list.empty()) {
188+
*this = other;
188189
return;
189190
}
190-
// All-false input establishes configuration without retaining any events.
191-
if (!initialized) {
192-
window = other.window;
193-
window_funnel_mode = other.window_funnel_mode;
194-
event_count = other.event_count;
195-
initialized = true;
196-
} else if (UNLIKELY(window != other.window ||
197-
window_funnel_mode != other.window_funnel_mode)) {
191+
if (UNLIKELY(window != other.window || window_funnel_mode != other.window_funnel_mode)) {
198192
throw Exception(ErrorCode::INVALID_ARGUMENT,
199193
"window_funnel aggregate states have incompatible window or mode");
200194
}
201195

202-
if (other.events_list.empty()) {
203-
return;
204-
}
205-
if (events_list.empty()) {
206-
events_list = other.events_list;
207-
sorted = other.sorted;
208-
} else {
209-
const auto prefix_size = events_list.size();
210-
events_list.insert(std::end(events_list), std::begin(other.events_list),
211-
std::end(other.events_list));
212-
// Both stable_sort and inplace_merge preserve relative order of equal elements.
213-
// Since same-row events have the same timestamp (and thus compare equal in
214-
// the primary sort key), they remain consecutive after merge — preserving
215-
// the validity of continuation flags.
216-
merge_events_list(events_list, prefix_size, sorted, other.sorted);
217-
sorted = true;
218-
}
196+
const auto prefix_size = events_list.size();
197+
events_list.insert(std::end(events_list), std::begin(other.events_list),
198+
std::end(other.events_list));
199+
// Both stable_sort and inplace_merge preserve relative order of equal elements.
200+
// Since same-row events have the same timestamp (and thus compare equal in
201+
// the primary sort key), they remain consecutive after merge — preserving
202+
// the validity of continuation flags.
203+
merge_events_list(events_list, prefix_size, sorted, other.sorted);
204+
sorted = true;
219205
}
220206

221207
void write(BufferWritable& out) const {
222208
write_var_int(event_count, out);
223209
write_var_int(window, out);
224210
write_var_int(static_cast<std::underlying_type_t<WindowFunnelMode>>(window_funnel_mode),
225211
out);
226-
// Tag configured eventless states in the existing sorted field. Legacy readers
227-
// interpret every nonzero value as sorted, so the layout remains compatible.
228-
const auto sorted_flag = static_cast<Int64>(sorted);
229-
write_var_int(initialized && events_list.empty() ? 2 : sorted_flag, out);
212+
write_var_int(sorted ? 1 : 0, out);
230213
write_var_int(cast_set<Int64>(events_list.size()), out);
231214
for (const auto& evt : events_list) {
232215
// Use fixed-size binary write for timestamp (8 bytes) and event_idx (1 byte).
@@ -248,13 +231,9 @@ struct WindowFunnelStateV2 {
248231

249232
read_var_int(tmp, in);
250233
sorted = (tmp != 0);
251-
// Legacy states use 0/1 and retain their configuration even without events.
252-
initialized = tmp == 2 || window != WINDOW_UNSET ||
253-
window_funnel_mode != WindowFunnelMode::INVALID;
254234

255235
Int64 size = 0;
256236
read_var_int(size, in);
257-
initialized |= size != 0;
258237
events_list.clear();
259238
events_list.resize(size);
260239
for (Int64 i = 0; i < size; ++i) {

‎be/src/util/reservoir_sampler.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,8 @@ class ReservoirSampler {
329329
rng.seed(123456);
330330
}
331331

332+
bool empty() const { return total_values == 0; }
333+
332334
double quantileInterpolated(double level) {
333335
if (samples.empty()) {
334336
return std::numeric_limits<double>::quiet_NaN();

0 commit comments

Comments
 (0)