Skip to content

Commit 42efc96

Browse files
committed
[fix](function) Bind captured inputs and wrap nonnullable results in array_sorted_merge
The merge lambda may reference enclosing columns, but the function executed the body against a block that held only the two candidates: a captured column resolved against the wrong column when its index collided with a candidate slot, and reported a missing column when it was beyond the two candidates. A non-nullable body (COALESCE/IFNULL) also reached ColumnNullable::insert_from and failed the assert_cast. Key changes: - Materialize captured input columns at their original positions in the one-row lambda block for the current row, and reject a nested lambda that captures a merge argument - Normalize a non-nullable lambda result into the nullable array element representation - Cover a table-column capture at index 0 and beyond the candidates, a COALESCE body, and a rejected nested capture in the regression suite Unit Test: - regression-test/suites/query_p0/sql_functions/array_functions/test_array_sorted_merge.groovy
1 parent a742080 commit 42efc96

3 files changed

Lines changed: 177 additions & 12 deletions

File tree

‎be/src/exprs/lambda_function/varray_sorted_merge_function.cpp‎

Lines changed: 117 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,13 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
#include <algorithm>
19+
#include <map>
1820
#include <memory>
21+
#include <set>
1922
#include <string>
2023
#include <utility>
24+
#include <vector>
2125

2226
#include "common/status.h"
2327
#include "core/assert_cast.h"
@@ -30,6 +34,7 @@
3034
#include "core/data_type/data_type.h"
3135
#include "core/data_type/data_type_array.h"
3236
#include "core/data_type/data_type_nullable.h"
37+
#include "core/data_type/data_type_number.h"
3338
#include "exprs/lambda_function/lambda_execution_context.h"
3439
#include "exprs/lambda_function/lambda_function.h"
3540
#include "exprs/lambda_function/lambda_function_factory.h"
@@ -60,6 +65,58 @@ class ArraySortedMergeFunction : public LambdaFunction {
6065

6166
std::string get_name() const override { return name; }
6267

68+
/// Rejects a nested lambda that captures one of this lambda's arguments: the position-based
69+
/// frame below does not publish them into nested lambdas, so such a capture would resolve
70+
/// against the wrong column. Nested lambdas may still use their own arguments (shadowing).
71+
static Status validate_nested_captures(const VExprSPtr& expr,
72+
const std::vector<std::string>& argument_names,
73+
bool inside_nested_lambda,
74+
std::vector<std::string> shadowed_names) {
75+
if (expr->is_column_ref()) {
76+
const std::string& column_name = expr->expr_name();
77+
if (std::ranges::find(shadowed_names, column_name) != shadowed_names.end()) {
78+
return Status::OK();
79+
}
80+
if (inside_nested_lambda &&
81+
std::ranges::find(argument_names, column_name) != argument_names.end()) {
82+
return Status::NotSupported(
83+
"array_sorted_merge merge lambda argument '{}' cannot be captured by a "
84+
"nested lambda",
85+
column_name);
86+
}
87+
return Status::OK();
88+
}
89+
if (expr->node_type() == TExprNodeType::LAMBDA_FUNCTION_EXPR) {
90+
const auto* lambda = assert_cast<const VLambdaFunctionExpr*>(expr.get());
91+
if (lambda->has_argument_names()) {
92+
for (const auto& argument_name : lambda->argument_names()) {
93+
shadowed_names.emplace_back(argument_name);
94+
}
95+
}
96+
return validate_nested_captures(lambda->get_child(0), argument_names, true,
97+
std::move(shadowed_names));
98+
}
99+
for (const auto& child : expr->children()) {
100+
RETURN_IF_ERROR(validate_nested_captures(child, argument_names, inside_nested_lambda,
101+
shadowed_names));
102+
}
103+
return Status::OK();
104+
}
105+
106+
Status prepare(RuntimeState* state, const VExprSPtrs& children) override {
107+
RETURN_IF_ERROR(LambdaFunction::prepare(state, children));
108+
DCHECK_EQ(children.size(), 4);
109+
const auto* lambda_expr = assert_cast<const VLambdaFunctionExpr*>(children[0].get());
110+
if (!lambda_expr->has_argument_names()) {
111+
return Status::OK();
112+
}
113+
// A nested lambda that captures one of this lambda's arguments would need a binding
114+
// that does not exist in the position-based frame below; reject it during prepare so
115+
// the query fails instead of resolving the argument against the wrong column.
116+
return validate_nested_captures(children[0]->get_child(0), lambda_expr->argument_names(),
117+
false, {});
118+
}
119+
63120
Status execute(VExprContext* context, const Block* block, const Selector* expr_selector,
64121
size_t count, ColumnPtr& result_column, const DataTypePtr& result_type,
65122
const VExprSPtrs& children) const override {
@@ -106,13 +163,41 @@ class ArraySortedMergeFunction : public LambdaFunction {
106163
auto result_nullmap = ColumnUInt8::create(rows, 0);
107164
auto& result_nullmap_data = result_nullmap->get_data();
108165

109-
// Two one-row columns hold the lambda arguments of the current merge step. They carry
110-
// the lambda argument names because the column refs resolve by name when no
111-
// position-based gap is set.
166+
// The lambda may capture enclosing columns (for example a table column compared against
167+
// the candidates). Materialize every captured input at its original position in the
168+
// one-row lambda block, so slot refs read the current row; the two merge candidates
169+
// take the positions after the last captured column and are bound by name.
170+
std::set<int> captured_ids;
171+
children[0]->collect_slot_column_ids(captured_ids);
172+
if (!captured_ids.empty() && block == nullptr) {
173+
return Status::InternalError(
174+
"array_sorted_merge lambda references outer columns but has no input block");
175+
}
176+
std::map<int, ColumnPtr> captured_columns;
177+
for (int id : captured_ids) {
178+
if (id < 0 || static_cast<size_t>(id) >= block->columns()) {
179+
return Status::InternalError(
180+
"array_sorted_merge lambda input column id {} is outside input block", id);
181+
}
182+
captured_columns[id] =
183+
block->get_by_position(id).column->convert_to_full_column_if_const();
184+
}
185+
const int lambda_argument_base = captured_ids.empty() ? 0 : *captured_ids.rbegin() + 1;
112186
const auto* lambda_expr = assert_cast<const VLambdaFunctionExpr*>(children[0].get());
113187
const std::vector<std::string>* argument_names =
114188
lambda_expr->has_argument_names() ? &lambda_expr->argument_names() : nullptr;
115189
Block lambda_block;
190+
for (int i = 0; i < lambda_argument_base; ++i) {
191+
auto captured = captured_columns.find(i);
192+
if (captured != captured_columns.end()) {
193+
lambda_block.insert(ColumnWithTypeAndName(captured->second->clone_empty(),
194+
block->get_by_position(i).type,
195+
block->get_by_position(i).name));
196+
} else {
197+
lambda_block.insert(ColumnWithTypeAndName(
198+
ColumnUInt8::create(1, 0), std::make_shared<DataTypeUInt8>(), "temp"));
199+
}
200+
}
116201
for (int i = 0; i < 2; ++i) {
117202
const std::string argument_name =
118203
argument_names != nullptr && i < argument_names->size()
@@ -121,11 +206,13 @@ class ArraySortedMergeFunction : public LambdaFunction {
121206
lambda_block.insert(ColumnWithTypeAndName(nested_types[i]->create_column(),
122207
nested_types[i], argument_name));
123208
}
124-
// The lambda arguments are bound by position (column ids 0 and 1), like array_sort's
125-
// comparator, so the body reads these two one-row columns.
209+
// The lambda arguments are bound by name (the column refs carry the argument names),
210+
// so the body reads these two one-row columns.
126211
IColumn* argument_columns[2] = {
127-
lambda_block.get_by_position(0).column->assert_mutable().get(),
128-
lambda_block.get_by_position(1).column->assert_mutable().get()};
212+
lambda_block.get_by_position(lambda_argument_base).column->assert_mutable().get(),
213+
lambda_block.get_by_position(lambda_argument_base + 1)
214+
.column->assert_mutable()
215+
.get()};
129216

130217
LambdaExecutionContext::Frame lambda_frame;
131218
lambda_frame.bind_by_name = false;
@@ -137,6 +224,21 @@ class ArraySortedMergeFunction : public LambdaFunction {
137224
const auto& offsets1 = arrays[1]->get_offsets();
138225
const int lambda_result_base = static_cast<int>(lambda_block.columns());
139226

227+
// The merged element may come from a non-nullable lambda body (for example COALESCE),
228+
// while array elements are always nullable; normalize before inserting.
229+
auto append_element = [&](const IColumn& source, size_t source_row) {
230+
if (auto* nullable_target = check_and_get_column<ColumnNullable>(result_nested.get())) {
231+
if (const auto* nullable_source = check_and_get_column<ColumnNullable>(&source)) {
232+
nullable_target->insert_from(*nullable_source, source_row);
233+
} else {
234+
nullable_target->get_nested_column().insert_from(source, source_row);
235+
nullable_target->get_null_map_data().push_back(0);
236+
}
237+
} else {
238+
result_nested->insert_from(source, source_row);
239+
}
240+
};
241+
140242
for (size_t row = 0; row < rows; ++row) {
141243
const bool row_is_null = (nullmaps[0] != nullptr && (*nullmaps[0])[row] != 0) ||
142244
(nullmaps[1] != nullptr && (*nullmaps[1])[row] != 0);
@@ -150,6 +252,9 @@ class ArraySortedMergeFunction : public LambdaFunction {
150252
"array_sorted_merge requires a non-null ascending flag");
151253
}
152254
const bool ascending = flags[row] != 0;
255+
for (const auto& [id, column] : captured_columns) {
256+
lambda_block.get_by_position(id).column = column->cut(row, 1);
257+
}
153258

154259
size_t left = row == 0 ? 0 : offsets0[row - 1];
155260
const size_t left_end = offsets0[row];
@@ -158,11 +263,11 @@ class ArraySortedMergeFunction : public LambdaFunction {
158263

159264
while (left < left_end || right < right_end) {
160265
if (left == left_end) {
161-
result_nested->insert_from(*data[1], right++);
266+
append_element(*data[1], right++);
162267
continue;
163268
}
164269
if (right == right_end) {
165-
result_nested->insert_from(*data[0], left++);
270+
append_element(*data[0], left++);
166271
continue;
167272
}
168273

@@ -181,15 +286,15 @@ class ArraySortedMergeFunction : public LambdaFunction {
181286
merged_is_null = nullable->is_null_at(0);
182287
}
183288
if (!merged_is_null) {
184-
result_nested->insert_from(*merged_column, 0);
289+
append_element(*merged_column, 0);
185290
++left;
186291
++right;
187292
} else {
188293
const int cmp = data[0]->compare_at(left, right, *data[1], 1);
189294
if (ascending ? cmp <= 0 : cmp >= 0) {
190-
result_nested->insert_from(*data[0], left++);
295+
append_element(*data[0], left++);
191296
} else {
192-
result_nested->insert_from(*data[1], right++);
297+
append_element(*data[1], right++);
193298
}
194299
}
195300
lambda_block.erase_tail(lambda_result_base);

‎regression-test/data/query_p0/sql_functions/array_functions/test_array_sorted_merge.out‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,12 @@
1414
-- !empty_and_null --
1515
[1] \N []
1616

17+
-- !capture_column_zero --
18+
[1]
19+
20+
-- !capture_column_two --
21+
[1]
22+
23+
-- !nonnullable_body --
24+
[1, 2]
25+

‎regression-test/suites/query_p0/sql_functions/array_functions/test_array_sorted_merge.groovy‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,57 @@ suite("test_array_sorted_merge") {
7272
CAST([] AS ARRAY<INT>), CAST([] AS ARRAY<INT>), true)
7373
"""
7474

75+
// A captured input column must be read at the current row: with the capture at column
76+
// index 0 the predicate compares against a candidate unless it is rebound, and with the
77+
// capture at index 2 an unbound read reports a missing column.
78+
sql """DROP TABLE IF EXISTS array_sorted_merge_capture_0"""
79+
sql """
80+
CREATE TABLE array_sorted_merge_capture_0 (
81+
lim INT,
82+
l ARRAY<INT>,
83+
r ARRAY<INT>
84+
)
85+
DISTRIBUTED BY HASH(lim) BUCKETS 1
86+
PROPERTIES ("replication_num" = "1")
87+
"""
88+
sql """INSERT INTO array_sorted_merge_capture_0 VALUES (2, [1], [1])"""
89+
order_qt_capture_column_zero """
90+
SELECT array_sorted_merge((a, b) -> IF(a = b AND a < lim, a, NULL), l, r, true)
91+
FROM array_sorted_merge_capture_0
92+
"""
93+
94+
sql """DROP TABLE IF EXISTS array_sorted_merge_capture_2"""
95+
sql """
96+
CREATE TABLE array_sorted_merge_capture_2 (
97+
id INT,
98+
l ARRAY<INT>,
99+
r ARRAY<INT>,
100+
lim INT
101+
)
102+
DISTRIBUTED BY HASH(id) BUCKETS 1
103+
PROPERTIES ("replication_num" = "1")
104+
"""
105+
sql """INSERT INTO array_sorted_merge_capture_2 VALUES (0, [1], [1], 2)"""
106+
order_qt_capture_column_two """
107+
SELECT array_sorted_merge((a, b) -> IF(a = b AND a < lim, a, NULL), l, r, true)
108+
FROM array_sorted_merge_capture_2
109+
"""
110+
111+
// A non-nullable lambda body (COALESCE) must be wrapped into the nullable element
112+
// representation instead of being inserted directly.
113+
order_qt_nonnullable_body """
114+
SELECT array_sorted_merge((a, b) -> COALESCE(a, b, CAST(0 AS INT)),
115+
CAST([1, 2] AS ARRAY<INT>),
116+
CAST([1, 2] AS ARRAY<INT>), true)
117+
"""
118+
119+
test {
120+
sql """SELECT array_sorted_merge(
121+
(a, b) -> CAST(a + element_at(array_map(x -> x + a, [1]), 1) AS INT),
122+
CAST([1] AS ARRAY<INT>), CAST([2] AS ARRAY<INT>), true)"""
123+
exception "cannot be captured by a nested lambda"
124+
}
125+
75126
test {
76127
sql """SELECT array_sorted_merge((a, b) -> IF(a = b, a, NULL), [1], [2], 'asc')"""
77128
exception "ascending flag"

0 commit comments

Comments
 (0)