Skip to content

Commit 2d17333

Browse files
committed
address pr review
1 parent 960bf69 commit 2d17333

2 files changed

Lines changed: 67 additions & 7 deletions

File tree

‎datafusion/expr/src/higher_order_function.rs‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -266,11 +266,39 @@ impl LambdaArgument {
266266
/// so they align with the evaluation batch. Captures are snapshotted from the
267267
/// outer batch at construction time, giving one value per outer row, but the
268268
/// function may evaluate the lambda body over a batch with a different number
269-
/// of rows. It is the function responsibility to provide the appropriate `spread_captures`
270-
/// closure to expand (or otherwise reshape) the captures to match.
271-
/// Function working on lists, for example `array_transform(arr, v -> v + 1)`
272-
/// flattens all list elements into a single batch, duplicating captured
273-
/// values for rows with multiple elements and dropping them for empty lists.
269+
/// of rows. It is the function's responsibility to provide the appropriate
270+
/// `spread_captures` closure to expand (or otherwise reshape) the captures
271+
/// to match.
272+
///
273+
/// Taking as an example the following table:
274+
///
275+
/// ```sql
276+
/// CREATE TABLE t (arr INT[], a INT) AS VALUES
277+
/// ([1, 2, 3], 10),
278+
/// ([], 20),
279+
/// ([4], 30);
280+
/// ```
281+
///
282+
/// `SELECT array_transform(arr, v -> v + a) from t` would execute over three outer rows:
283+
///
284+
/// ```text
285+
/// arr (ListArray): [[1, 2, 3], [], [4]] -- 3 outer rows, 4 total elements
286+
/// a (captured): [10, 20, 30] -- one value per outer row
287+
/// ```
288+
///
289+
/// `array_transform` flattens the list elements into a single batch of 4 rows,
290+
/// so `spread_captures` must repeat/drop captured values to match:
291+
///
292+
/// ```text
293+
/// v (flattened args): [1, 2, 3, 4]
294+
/// a (spread): [10, 10, 10, 30] -- 10 repeated for 3 elements in row 0,
295+
/// -- 20 dropped for the empty sublist in row 1,
296+
/// -- 30 once for the single element in row 2
297+
/// ```
298+
///
299+
/// The lambda body `v + a` then evaluates element-wise over these 4-row arrays,
300+
/// producing `[11, 12, 13, 34]`, which `array_transform` reassembles into `[[11, 12, 13], [], [34]]`.
301+
///
274302
/// If the lambda has no captures, `spread_captures` is never called.
275303
pub fn evaluate(
276304
&self,

‎datafusion/sqllogictest/test_files/array/array_transform.slt‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ select array_transform([1, 5, 9], v -> case when v = 1 then 2 when v = 5 then 6
170170
----
171171
[2, 6, 8]
172172

173-
#case with inner higher order function
173+
#case with inner higher order function (case EvalMethod::ExpressionOrExpression)
174174
query I??
175175
select
176176
t.number,
@@ -186,7 +186,7 @@ order by t.number;
186186
40 [4, 50] [5, 51]
187187
60 [7, 50] [8, 51]
188188

189-
# case with inner nested higher order function
189+
# case with inner nested higher order function (case EvalMethod::ExpressionOrExpression)
190190
query T?I?
191191
select
192192
t.text,
@@ -222,6 +222,7 @@ b [4, 50] 40 [[[48, 94]]]
222222
c [7, 50] 60 [[[74, 117]]]
223223

224224
# case with inner nested higher order function where the condition is a lambda function
225+
# (case EvalMethod::ExpressionOrExpression)
225226
query T?I?
226227
select
227228
t.text,
@@ -256,6 +257,37 @@ a [1, 50] 10 [[[11, 11]]]
256257
b [4, 50] 40 [[[44, 44]]]
257258
c [7, 50] 60 [[[74, 117]]]
258259

260+
# case with condition (case EvalMethod::WithExpression)
261+
query ?IB
262+
select
263+
t.list,
264+
t.number,
265+
case array_transform(t.list, v -> v + t.number)[1]
266+
when array_transform(t.list, v -> v + 10)[1] then array_any_match(t.list, v -> v == 1)
267+
when array_transform(t.list, v -> v + 40)[1] then array_any_match(t.list, v -> v == 4)
268+
when array_transform(t.list, v -> v + 60)[1] then array_any_match(t.list, v -> v == 7)
269+
end
270+
from t;
271+
----
272+
[1, 50] 10 true
273+
[4, 50] 40 true
274+
[7, 50] 60 true
275+
276+
# case without condition and multiple when/then pairs (case EvalMethod::NoExpression)
277+
query ??
278+
select
279+
t.list,
280+
case
281+
when array_any_match(t.list, v -> v == 1) then array_transform(t.list, v -> v + 1)
282+
when array_any_match(t.list, v -> v == 4) then array_transform(t.list, v -> v + 2)
283+
when array_any_match(t.list, v -> v == 7) then array_transform(t.list, v -> v + 3)
284+
end
285+
from t;
286+
----
287+
[1, 50] [2, 51]
288+
[4, 50] [6, 52]
289+
[7, 50] [10, 53]
290+
259291
# array_transform coercion rules
260292
query TT
261293
explain select array_transform(arrow_cast(t.list, 'FixedSizeList(2, Int32)'), a -> a+1) from t;

0 commit comments

Comments
 (0)