Skip to content

Commit e7a3dbf

Browse files
committed
[fix](function) Reject NULL counters and overflow-free post counts in top-K functions
topk_merge_min/max read n, dn and k with DORIS_CHECK, which terminates a debugging BE when the caller passes NULL, and computed S + I - D in int64, so a cancel pair (S = INT64_MAX, I = D = 1) overflowed even though the final count fits. The state insert path also rebuilt the whole value column for every appended entry. Key changes: - Return Status from read_int64 and reject NULL n/dn/k with InvalidArgument - Evaluate S + I - D in __int128 and range-check the result before narrowing - Append to the top-K state in place when the new value sorts last - Cover NULL counters, the INT64_MAX cancel pair and different-key delete/insert rows in the regression suite Unit Test: - be/test/exprs/aggregate/agg_topk_count_test.cpp: TopKCountTest.* (10 cases) - regression-test/suites/query_p0/aggregate/topk_min_max_count.groovy
1 parent e4acad3 commit e7a3dbf

4 files changed

Lines changed: 136 additions & 19 deletions

File tree

‎be/src/exprs/aggregate/aggregate_function_topk_count.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,13 @@ class AggregateFunctionTopKCount final
285285
/// Inserts (value, count) at `pos`, keeping the value column sorted by value.
286286
static void insert_entry(AggregateTopKCountData& state, size_t pos, const IColumn& column,
287287
size_t row_num, int64_t count) {
288+
if (pos == state.value_column->size()) {
289+
// Appending needs no rebuild: the state column is mutable, so ascending MIN input
290+
// (descending MAX input) fills the list without copying the retained prefix.
291+
state.value_column->insert_from(column, row_num);
292+
state.counts.push_back(count);
293+
return;
294+
}
288295
auto new_values = state.value_column->clone_empty();
289296
new_values->reserve(state.value_column->size() + 1);
290297
for (size_t i = 0; i < pos; ++i) {

‎be/src/exprs/function/function_topk_merge.cpp‎

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include <glog/logging.h>
1919

2020
#include <cstddef>
21+
#include <limits>
2122
#include <memory>
2223
#include <string>
2324
#include <utility>
@@ -134,14 +135,19 @@ TopKBoundView make_bound_view(const IColumn& column, size_t row) {
134135
return view;
135136
}
136137

137-
int64_t read_int64(const IColumn& column, size_t row, const char* argument_name) {
138+
Status read_int64(const IColumn& column, size_t row, const char* function_name,
139+
const char* argument_name, int64_t* out) {
138140
const IColumn* nested = &column;
139141
if (const auto* nullable = check_and_get_column<ColumnNullable>(&column)) {
140-
// The contract declares these arguments as non-NULL BIGINT.
141-
DORIS_CHECK(!nullable->is_null_at(row)) << argument_name << " must not be NULL";
142+
// The signature accepts a nullable BIGINT, so a NULL here is invalid SQL input and must
143+
// be a query error; a checked invariant would terminate a debugging BE.
144+
if (UNLIKELY(nullable->is_null_at(row))) {
145+
return Status::InvalidArgument("{}: {} must not be NULL", function_name, argument_name);
146+
}
142147
nested = &nullable->get_nested_column();
143148
}
144-
return assert_cast<const ColumnInt64&, TypeCheckOnRelease::DISABLE>(*nested).get_element(row);
149+
*out = assert_cast<const ColumnInt64&, TypeCheckOnRelease::DISABLE>(*nested).get_element(row);
150+
return Status::OK();
145151
}
146152

147153
int64_t checked_add(int64_t lhs, int64_t rhs, const char* function_name) {
@@ -152,14 +158,6 @@ int64_t checked_add(int64_t lhs, int64_t rhs, const char* function_name) {
152158
return result;
153159
}
154160

155-
int64_t checked_sub(int64_t lhs, int64_t rhs, const char* function_name) {
156-
int64_t result = 0;
157-
if (UNLIKELY(__builtin_sub_overflow(lhs, rhs, &result))) {
158-
throw Exception(ErrorCode::INVALID_ARGUMENT, "{}: value count overflow", function_name);
159-
}
160-
return result;
161-
}
162-
163161
/// FLOAT/DOUBLE use CompareFloat so NaN sorts greatest and is distinct from every number; other
164162
/// types use the column comparison, i.e. exactly the MIN/MAX comparison.
165163
int compare_values(PrimitiveType value_type, const IColumn& lhs, size_t lhs_row, const IColumn& rhs,
@@ -281,13 +279,16 @@ class FunctionTopKMerge : public IFunction {
281279
auto& out_bound_column = assert_cast<ColumnNullable&>(result_struct.get_column(1));
282280

283281
for (size_t row = 0; row != input_rows_count; ++row) {
284-
const int64_t k = read_int64(*columns[8], row, "k");
282+
int64_t k = 0;
283+
RETURN_IF_ERROR(read_int64(*columns[8], row, name, "k", &k));
285284
if (UNLIKELY(k < 1 || k > MAX_TOPK_K)) {
286285
return Status::InvalidArgument("{} requires 1 <= k <= {}, but got {}", name,
287286
MAX_TOPK_K, k);
288287
}
289-
const int64_t n = read_int64(*columns[6], row, "n");
290-
const int64_t dn = read_int64(*columns[7], row, "dn");
288+
int64_t n = 0;
289+
RETURN_IF_ERROR(read_int64(*columns[6], row, name, "n", &n));
290+
int64_t dn = 0;
291+
RETURN_IF_ERROR(read_int64(*columns[7], row, name, "dn", &dn));
291292
const int64_t new_n = checked_add(n, dn, name);
292293

293294
const TopKArrayView s_view = make_array_view(*columns[0], row);
@@ -446,13 +447,20 @@ class FunctionTopKMerge : public IFunction {
446447
RETURN_IF_ERROR(consume_equal(i_view, i_index, *head, head_row, type, &count_i));
447448
RETURN_IF_ERROR(consume_equal(d_view, d_index, *head, head_row, type, &count_d));
448449

449-
int64_t post = checked_add(count_s, count_i, name);
450-
post = checked_sub(post, count_d, name);
451-
if (UNLIKELY(post < 0)) {
450+
// A cancel pair can make an intermediate exceed int64 even though the final count
451+
// fits (for example S = INT64_MAX and I = D = 1), so evaluate the whole expression
452+
// in a wider type and range-check the result.
453+
const __int128 post_wide = static_cast<__int128>(count_s) + count_i - count_d;
454+
if (UNLIKELY(post_wide < 0)) {
452455
// Deleting a row that the pre-image did not have breaks the top-K invariant.
453456
return Status::InternalError(
454-
"{}: negative post count {} for a value in the trusted range", name, post);
457+
"{}: negative post count {} for a value in the trusted range", name,
458+
static_cast<int64_t>(post_wide));
459+
}
460+
if (UNLIKELY(post_wide > std::numeric_limits<int64_t>::max())) {
461+
return Status::InvalidArgument("{}: value count overflow", name);
455462
}
463+
const int64_t post = static_cast<int64_t>(post_wide);
456464
if (post > 0) {
457465
++*positive;
458466
if (std::cmp_less(kept, k)) {

‎regression-test/data/query_p0/aggregate/topk_min_max_count.out‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,3 +151,14 @@ e 0 0 {"values":[{"value":0, "count":5}], "bound":null} {"values":[{"value":0, "
151151
-- !merge_decimal_null_bounds --
152152
{"values":[{"value":1.23, "count":1}], "bound":null}
153153

154+
-- !merge_min_cancel_at_int64_max --
155+
{"values":[{"value":1, "count":9223372036854775807}], "bound":null}
156+
157+
-- !merge_min_different_keys_one_block --
158+
{"values":[], "bound":1}
159+
{"values":[{"value":10, "count":1}], "bound":10}
160+
161+
-- !merge_max_different_keys_one_block --
162+
{"values":[], "bound":3}
163+
{"values":[{"value":3, "count":1}], "bound":3}
164+

‎regression-test/suites/query_p0/aggregate/topk_min_max_count.groovy‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -477,4 +477,95 @@ suite("topk_min_max_count") {
477477
GROUP BY grp"""
478478
exception "must be a non-null constant"
479479
}
480+
481+
// A NULL counter is invalid SQL input and must be a query error; a checked invariant would
482+
// terminate a debugging BE.
483+
test {
484+
sql """SELECT topk_merge_min(
485+
CAST(array() AS ARRAY<STRUCT<value:INT, count:BIGINT>>),
486+
CAST(NULL AS INT),
487+
CAST(array() AS ARRAY<STRUCT<value:INT, count:BIGINT>>),
488+
CAST(NULL AS INT),
489+
CAST(array() AS ARRAY<STRUCT<value:INT, count:BIGINT>>),
490+
CAST(NULL AS INT),
491+
CAST(NULL AS BIGINT), 0, 2)"""
492+
exception "n must not be NULL"
493+
}
494+
495+
test {
496+
sql """SELECT topk_merge_max(
497+
CAST(array() AS ARRAY<STRUCT<value:INT, count:BIGINT>>),
498+
CAST(NULL AS INT),
499+
CAST(array() AS ARRAY<STRUCT<value:INT, count:BIGINT>>),
500+
CAST(NULL AS INT),
501+
CAST(array() AS ARRAY<STRUCT<value:INT, count:BIGINT>>),
502+
CAST(NULL AS INT),
503+
0, CAST(NULL AS BIGINT), 2)"""
504+
exception "dn must not be NULL"
505+
}
506+
507+
test {
508+
sql """SELECT topk_merge_min(
509+
CAST(array() AS ARRAY<STRUCT<value:INT, count:BIGINT>>),
510+
CAST(NULL AS INT),
511+
CAST(array() AS ARRAY<STRUCT<value:INT, count:BIGINT>>),
512+
CAST(NULL AS INT),
513+
CAST(array() AS ARRAY<STRUCT<value:INT, count:BIGINT>>),
514+
CAST(NULL AS INT),
515+
0, 0, CAST(NULL AS BIGINT))"""
516+
exception "k must not be NULL"
517+
}
518+
519+
// A cancel pair can exceed int64 in an intermediate while the final count still fits.
520+
order_qt_merge_min_cancel_at_int64_max """
521+
SELECT topk_merge_min(
522+
CAST(array(named_struct('value', 1, 'count', 9223372036854775807))
523+
AS ARRAY<STRUCT<value:INT, count:BIGINT>>),
524+
CAST(NULL AS INT),
525+
CAST(array(named_struct('value', 1, 'count', 1))
526+
AS ARRAY<STRUCT<value:INT, count:BIGINT>>),
527+
CAST(NULL AS INT),
528+
CAST(array(named_struct('value', 1, 'count', 1))
529+
AS ARRAY<STRUCT<value:INT, count:BIGINT>>),
530+
CAST(NULL AS INT),
531+
CAST(9223372036854775807 AS BIGINT), CAST(0 AS BIGINT), CAST(1 AS BIGINT))
532+
"""
533+
534+
// Two rows in one block whose insert and delete sides name different keys. Row 1 deletes the
535+
// only trusted value and inserts a worse one, so the candidate list must be empty with the
536+
// bound kept: a non-empty list would hide the fallback that finds the real new extreme.
537+
sql """DROP TABLE IF EXISTS topk_merge_block_t"""
538+
sql """
539+
CREATE TABLE topk_merge_block_t (
540+
row_id INT,
541+
s ARRAY<STRUCT<value:INT, count:BIGINT>>,
542+
b INT,
543+
i ARRAY<STRUCT<value:INT, count:BIGINT>>,
544+
bi INT,
545+
d ARRAY<STRUCT<value:INT, count:BIGINT>>,
546+
bd INT,
547+
n BIGINT,
548+
dn BIGINT,
549+
k BIGINT
550+
)
551+
DISTRIBUTED BY HASH(row_id) BUCKETS 1
552+
PROPERTIES ("replication_num" = "1")
553+
"""
554+
sql """INSERT INTO topk_merge_block_t VALUES
555+
(0, array(named_struct('value', 10, 'count', 1)), 10, array(), NULL, array(), NULL, 1, 0, 1),
556+
(1, array(named_struct('value', 1, 'count', 1)), 1,
557+
array(named_struct('value', 5, 'count', 1)), NULL,
558+
array(named_struct('value', 1, 'count', 1)), NULL, 2, 0, 1),
559+
(2, array(named_struct('value', 3, 'count', 1)), 3, array(), NULL, array(), NULL, 1, 0, 1),
560+
(3, array(named_struct('value', 3, 'count', 1)), 3,
561+
array(named_struct('value', 0, 'count', 1)), NULL,
562+
array(named_struct('value', 3, 'count', 1)), NULL, 2, 0, 1)"""
563+
order_qt_merge_min_different_keys_one_block """
564+
SELECT topk_merge_min(s, b, i, bi, d, bd, n, dn, k)
565+
FROM topk_merge_block_t WHERE row_id < 2 ORDER BY row_id
566+
"""
567+
order_qt_merge_max_different_keys_one_block """
568+
SELECT topk_merge_max(s, b, i, bi, d, bd, n, dn, k)
569+
FROM topk_merge_block_t WHERE row_id >= 2 ORDER BY row_id
570+
"""
480571
}

0 commit comments

Comments
 (0)