Skip to content

Commit c9e0d62

Browse files
committed
resolve lambda variables of nested higher order functions
1 parent 5b927b9 commit c9e0d62

3 files changed

Lines changed: 252 additions & 16 deletions

File tree

‎datafusion/expr/src/higher_order_function.rs‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1219,24 +1219,25 @@ fn resolve_higher_order_function(
12191219
// a map of lambda variable name => a never empty stack of fields [ [..shadowed], in_scope ]
12201220
vars: &mut HashMap<String, Vec<FieldRef>>,
12211221
) -> Result<Transformed<Expr>> {
1222-
let args = if !vars.is_empty() {
1223-
/* if this is a nested lambda, we must resolve non-lambda args before invoking
1224-
lambda_parameters because it will invoke ExprSchemable::to_field for every
1225-
non-lambda parameter, and if one them contains a lambda variable, it will fail
1226-
due to it being unresolved. Example query:
1222+
/* non-lambda args are resolved before invoking lambda_parameters because it will
1223+
invoke ExprSchemable::to_field for every non-lambda parameter, and if one of them
1224+
contains a lambda variable, it will fail due to it being unresolved. Example query:
12271225
1228-
array_transform([[1, 2]], a -> array_transform(a, b -> b+1))
1226+
array_transform([[1, 2]], a -> array_transform(a, b -> b+1))
12291227
1230-
the nested array_transform's lambda_parameters will call Lambdavariable::to_field
1231-
on it's first argument, the variable `a`, which must be resolved
1232-
*/
1233-
args.map_elements(|arg| match arg {
1234-
Expr::Lambda(_) => Ok(Transformed::no(arg)),
1235-
_ => resolve_lambda_variables(arg, schema, vars),
1236-
})?
1237-
} else {
1238-
Transformed::no(args)
1239-
};
1228+
the nested array_transform's lambda_parameters will call Lambdavariable::to_field
1229+
on it's first argument, the variable `a`, which must be resolved
1230+
1231+
They are also resolved when no variable is in scope yet, as an argument can be a
1232+
higher order function of its own, whose variables this is the only chance to
1233+
resolve: the traversal jumps over the arguments once this returns. Example query:
1234+
1235+
array_any_match(array_transform(tags, x -> x), y -> y = 'c')
1236+
*/
1237+
let args = args.map_elements(|arg| match arg {
1238+
Expr::Lambda(_) => Ok(Transformed::no(arg)),
1239+
_ => resolve_lambda_variables(arg, schema, vars),
1240+
})?;
12401241

12411242
let transformed = args.transformed;
12421243
let mut args = args.data;

‎datafusion/substrait/tests/cases/logical_plans.rs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -406,4 +406,31 @@ mod tests {
406406
);
407407
Ok(())
408408
}
409+
410+
// A higher order function can be the value argument of another one, where its own
411+
// lambda parameter has to be resolved too.
412+
#[tokio::test]
413+
async fn nested_higher_order_function_with_dictionary_encoded_lambda_parameter()
414+
-> Result<()> {
415+
let proto_plan = read_json(
416+
"tests/testdata/test_plans/nested_any_match_dictionary_list_element.substrait.json",
417+
);
418+
let ctx = add_plan_schemas_to_ctx(SessionContext::new(), &proto_plan)?;
419+
let plan = from_substrait_plan(&ctx.state(), &proto_plan).await?;
420+
421+
let df = DataFrame::new(ctx.state(), plan);
422+
423+
// Without resolution of the inner function this fails in physical planning.
424+
df.clone().show().await?;
425+
426+
// Both parameters are resolved to the encoding the list actually carries.
427+
assert_snapshot!(
428+
df.into_optimized_plan()?,
429+
@r#"
430+
Projection: array_any_match(array_transform(t.tags, (p0) -> p0), (p1) -> p1 = Dictionary(UInt32, Utf8("c"))) AS matched
431+
TableScan: t projection=[tags]
432+
"#
433+
);
434+
Ok(())
435+
}
409436
}
Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
{
2+
"version": {
3+
"minorNumber": 85,
4+
"producer": "datafusion"
5+
},
6+
"extensions": [
7+
{
8+
"extensionFunction": {
9+
"extensionUrnReference": 4294967295,
10+
"functionAnchor": 1,
11+
"name": "equal"
12+
}
13+
},
14+
{
15+
"extensionFunction": {
16+
"extensionUrnReference": 4294967295,
17+
"functionAnchor": 2,
18+
"name": "array_any_match"
19+
}
20+
},
21+
{
22+
"extensionFunction": {
23+
"extensionUrnReference": 4294967295,
24+
"name": "array_transform"
25+
}
26+
}
27+
],
28+
"relations": [
29+
{
30+
"root": {
31+
"input": {
32+
"project": {
33+
"common": {
34+
"emit": {
35+
"outputMapping": [
36+
2
37+
]
38+
}
39+
},
40+
"input": {
41+
"read": {
42+
"baseSchema": {
43+
"names": [
44+
"id",
45+
"tags"
46+
],
47+
"struct": {
48+
"types": [
49+
{
50+
"i32": {
51+
"nullability": "NULLABILITY_NULLABLE"
52+
}
53+
},
54+
{
55+
"list": {
56+
"type": {
57+
"map": {
58+
"key": {
59+
"i32": {
60+
"typeVariationReference": 1,
61+
"nullability": "NULLABILITY_REQUIRED"
62+
}
63+
},
64+
"value": {
65+
"string": {
66+
"nullability": "NULLABILITY_NULLABLE"
67+
}
68+
},
69+
"typeVariationReference": 1,
70+
"nullability": "NULLABILITY_NULLABLE"
71+
}
72+
},
73+
"nullability": "NULLABILITY_NULLABLE"
74+
}
75+
}
76+
],
77+
"nullability": "NULLABILITY_REQUIRED"
78+
}
79+
},
80+
"namedTable": {
81+
"names": [
82+
"t"
83+
]
84+
}
85+
}
86+
},
87+
"expressions": [
88+
{
89+
"scalarFunction": {
90+
"functionReference": 2,
91+
"arguments": [
92+
{
93+
"value": {
94+
"scalarFunction": {
95+
"arguments": [
96+
{
97+
"value": {
98+
"selection": {
99+
"directReference": {
100+
"structField": {
101+
"field": 1
102+
}
103+
},
104+
"rootReference": {}
105+
}
106+
}
107+
},
108+
{
109+
"value": {
110+
"lambda": {
111+
"parameters": {
112+
"types": [
113+
{
114+
"string": {
115+
"nullability": "NULLABILITY_NULLABLE"
116+
}
117+
}
118+
],
119+
"nullability": "NULLABILITY_REQUIRED"
120+
},
121+
"body": {
122+
"selection": {
123+
"directReference": {
124+
"structField": {}
125+
},
126+
"lambdaParameterReference": {}
127+
}
128+
}
129+
}
130+
}
131+
}
132+
],
133+
"outputType": {
134+
"list": {
135+
"type": {
136+
"string": {
137+
"nullability": "NULLABILITY_NULLABLE"
138+
}
139+
},
140+
"nullability": "NULLABILITY_NULLABLE"
141+
}
142+
}
143+
}
144+
}
145+
},
146+
{
147+
"value": {
148+
"lambda": {
149+
"parameters": {
150+
"types": [
151+
{
152+
"string": {
153+
"nullability": "NULLABILITY_NULLABLE"
154+
}
155+
}
156+
],
157+
"nullability": "NULLABILITY_REQUIRED"
158+
},
159+
"body": {
160+
"scalarFunction": {
161+
"functionReference": 1,
162+
"arguments": [
163+
{
164+
"value": {
165+
"selection": {
166+
"directReference": {
167+
"structField": {}
168+
},
169+
"lambdaParameterReference": {}
170+
}
171+
}
172+
},
173+
{
174+
"value": {
175+
"literal": {
176+
"string": "c"
177+
}
178+
}
179+
}
180+
],
181+
"outputType": {
182+
"bool": {
183+
"nullability": "NULLABILITY_NULLABLE"
184+
}
185+
}
186+
}
187+
}
188+
}
189+
}
190+
}
191+
],
192+
"outputType": {
193+
"bool": {
194+
"nullability": "NULLABILITY_NULLABLE"
195+
}
196+
}
197+
}
198+
}
199+
]
200+
}
201+
},
202+
"names": [
203+
"matched"
204+
]
205+
}
206+
}
207+
]
208+
}

0 commit comments

Comments
 (0)