Skip to content

Commit ae427aa

Browse files
authored
fix: cast lambda variables to their declared type instead of erroring (apache#25418) (#186)
## Which issue does this PR close? - Closes apache#25411 ## Rationale for this change `create_physical_expr` required an `Expr::LambdaVariable`'s field to equal the planning schema's field and returned a plan error otherwise. The two can legitimately differ when a plan producer records a type that Substrait cannot express, such as dictionary encoding or a string view (Utf8view), since the recorded field and the field derived by `lambda_parameters` come from independent sources. ## What changes are included in this PR? Cast the variable to the recorded type instead, so the declared type is enforced rather than asserted. This is the same approach already used for `ScalarFunction` arguments, which the `TypeCoercion` analyzer casts to their coerced types instead of requiring them to match. Incompatible types still fail, as a cast error. ## What is the testing strategy for this PR? I added - Unit tests covering dictionary-encoded and `Utf8View` list elements with the lambda variable declared as plain `Utf8`, plus plan-shape tests pinning that the cast is inserted on a mismatch and omitted when the types already agree. - A Substrait integration test with a plan whose read schema declares a dictionary-encoded list element while the lambda carries its own plain `Utf8` parameter type. ## Are there any user-facing changes? No, this is a bug fix (cherry picked from commit 5da95ac)
1 parent 7269807 commit ae427aa

7 files changed

Lines changed: 709 additions & 19 deletions

File tree

‎datafusion/expr/src/higher_order_function.rs‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1219,24 +1219,25 @@ fn resolve_higher_order_function(
12191219
// a map of lambda variable name => a never empty stack of fields [ [..shadowed], in_scope ]
12201220
vars: &mut HashMap<String, Vec<FieldRef>>,
12211221
) -> Result<Transformed<Expr>> {
1222-
let args = if !vars.is_empty() {
1223-
/* if this is a nested lambda, we must resolve non-lambda args before invoking
1224-
lambda_parameters because it will invoke ExprSchemable::to_field for every
1225-
non-lambda parameter, and if one them contains a lambda variable, it will fail
1226-
due to it being unresolved. Example query:
1222+
/* non-lambda args are resolved before invoking lambda_parameters because it will
1223+
invoke ExprSchemable::to_field for every non-lambda parameter, and if one of them
1224+
contains a lambda variable, it will fail due to it being unresolved. Example query:
12271225
1228-
array_transform([[1, 2]], a -> array_transform(a, b -> b+1))
1226+
array_transform([[1, 2]], a -> array_transform(a, b -> b+1))
12291227
1230-
the nested array_transform's lambda_parameters will call Lambdavariable::to_field
1231-
on it's first argument, the variable `a`, which must be resolved
1232-
*/
1233-
args.map_elements(|arg| match arg {
1234-
Expr::Lambda(_) => Ok(Transformed::no(arg)),
1235-
_ => resolve_lambda_variables(arg, schema, vars),
1236-
})?
1237-
} else {
1238-
Transformed::no(args)
1239-
};
1228+
the nested array_transform's lambda_parameters will call Lambdavariable::to_field
1229+
on it's first argument, the variable `a`, which must be resolved
1230+
1231+
They are also resolved when no variable is in scope yet, as an argument can be a
1232+
higher order function of its own, whose variables this is the only chance to
1233+
resolve: the traversal jumps over the arguments once this returns. Example query:
1234+
1235+
array_any_match(array_transform(tags, x -> x), y -> y = 'c')
1236+
*/
1237+
let args = args.map_elements(|arg| match arg {
1238+
Expr::Lambda(_) => Ok(Transformed::no(arg)),
1239+
_ => resolve_lambda_variables(arg, schema, vars),
1240+
})?;
12401241

12411242
let transformed = args.transformed;
12421243
let mut args = args.data;

‎datafusion/functions-nested/src/array_any_match.rs‎

Lines changed: 76 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -191,11 +191,14 @@ mod tests {
191191
use std::{collections::HashMap, sync::Arc};
192192

193193
use arrow::{
194-
array::{ArrayRef, BooleanArray, Int32Array, ListArray, RecordBatch},
194+
array::{
195+
ArrayRef, BooleanArray, DictionaryArray, Int32Array, ListArray, RecordBatch,
196+
StringViewArray,
197+
},
195198
buffer::{NullBuffer, OffsetBuffer},
196-
datatypes::{DataType, Field},
199+
datatypes::{DataType, Field, UInt32Type},
197200
};
198-
use datafusion_common::{DFSchema, Result};
201+
use datafusion_common::{DFSchema, Result, ScalarValue};
199202
use datafusion_expr::{
200203
Expr, HigherOrderReturnFieldArgs, HigherOrderUDFImpl, ValueOrLambda, col,
201204
execution_props::ExecutionProps,
@@ -490,4 +493,74 @@ mod tests {
490493
);
491494
Ok(())
492495
}
496+
497+
/// Evaluates `any_match(list, x -> x = needle)` over two rows, `[a, b]` and
498+
/// `[c, a]`, built from `values`, with the lambda variable declared as the element
499+
/// type. `needle` carries its own type, as these expressions are planned directly
500+
/// and so are never type coerced.
501+
fn run_any_match_eq(values: ArrayRef, needle: Expr) -> Result<ArrayRef> {
502+
let element_type = values.data_type().clone();
503+
let element_field = Arc::new(Field::new_list_field(element_type.clone(), true));
504+
let list = ListArray::new(
505+
Arc::clone(&element_field),
506+
OffsetBuffer::<i32>::from_lengths(vec![2, 2]),
507+
values,
508+
None,
509+
);
510+
511+
let schema = DFSchema::from_unqualified_fields(
512+
vec![Field::new("list", DataType::List(element_field), true)].into(),
513+
HashMap::new(),
514+
)?;
515+
516+
create_physical_expr(
517+
&Expr::HigherOrderFunction(HigherOrderFunction::new(
518+
array_any_match_higher_order_function(),
519+
vec![
520+
col("list"),
521+
lambda(
522+
["x"],
523+
Expr::LambdaVariable(LambdaVariable::new(
524+
"x".to_string(),
525+
Some(Arc::new(Field::new("x", element_type, true))),
526+
))
527+
.eq(needle),
528+
),
529+
],
530+
)),
531+
&schema,
532+
&ExecutionProps::new(),
533+
&PhysicalPlanningContext::default(),
534+
)?
535+
.evaluate(&RecordBatch::try_new(
536+
Arc::clone(schema.inner()),
537+
vec![Arc::new(list)],
538+
)?)?
539+
.into_array(2)
540+
}
541+
542+
#[test]
543+
fn test_any_match_on_dictionary_encoded_elements() -> Result<()> {
544+
let values: ArrayRef = Arc::new(DictionaryArray::<UInt32Type>::from_iter([
545+
"a", "b", "c", "a",
546+
]));
547+
let result = run_any_match_eq(values, lit("c"))?;
548+
assert_eq!(
549+
result.as_any().downcast_ref::<BooleanArray>().unwrap(),
550+
&BooleanArray::from(vec![Some(false), Some(true)])
551+
);
552+
Ok(())
553+
}
554+
555+
#[test]
556+
fn test_any_match_on_string_view_elements() -> Result<()> {
557+
let values: ArrayRef = Arc::new(StringViewArray::from(vec!["a", "b", "c", "a"]));
558+
let needle = lit(ScalarValue::Utf8View(Some("c".to_string())));
559+
let result = run_any_match_eq(values, needle)?;
560+
assert_eq!(
561+
result.as_any().downcast_ref::<BooleanArray>().unwrap(),
562+
&BooleanArray::from(vec![Some(false), Some(true)])
563+
);
564+
Ok(())
565+
}
493566
}

‎datafusion/optimizer/src/analyzer/type_coercion.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,16 @@ fn analyze_internal(
163163
// apply coercion rewrite all expressions in the plan individually
164164
plan.map_expressions(|expr| {
165165
let original_name = name_preserver.save(&expr);
166+
167+
// A lambda variable carries the field recorded when the plan was built, which
168+
// need not be the one its function derives from the arguments. Resolve them
169+
// before coercing, so lambda bodies are coerced against the types they receive.
170+
let expr = if expr.exists(|e| Ok(matches!(e, Expr::HigherOrderFunction(_))))? {
171+
expr.resolve_lambda_variables(&schema)?.data
172+
} else {
173+
expr
174+
};
175+
166176
expr.rewrite(&mut expr_rewrite)
167177
.map(|transformed| transformed.update_data(|e| original_name.restore(e)))
168178
})?

‎datafusion/substrait/tests/cases/logical_plans.rs‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,4 +340,97 @@ mod tests {
340340
DataFrame::new(ctx.state(), plan).show().await?;
341341
Ok(())
342342
}
343+
344+
// The read schema declares the list element as dictionary encoded while the lambda
345+
// carries its own plain parameter type, as a producer that cannot express dictionary
346+
// encoding in Substrait emits. Physical planning reconciles the two with a cast.
347+
#[tokio::test]
348+
async fn higher_order_function_with_dictionary_encoded_lambda_parameter() -> Result<()>
349+
{
350+
let proto_plan = read_json(
351+
"tests/testdata/test_plans/any_match_dictionary_list_element.substrait.json",
352+
);
353+
let ctx = add_plan_schemas_to_ctx(SessionContext::new(), &proto_plan)?;
354+
let plan = from_substrait_plan(&ctx.state(), &proto_plan).await?;
355+
356+
assert_snapshot!(
357+
plan,
358+
@r#"
359+
Projection: array_any_match(t.tags, (p0) -> p0 = Utf8("c")) AS matched
360+
TableScan: t
361+
"#
362+
);
363+
364+
let df = DataFrame::new(ctx.state(), plan);
365+
366+
// Without resolution this fails in physical planning, where the field recorded
367+
// on the parameter does not match the one the schema carries.
368+
df.clone().show().await?;
369+
370+
// The lambda parameter is resolved to the encoding the list actually carries, so
371+
// the comparison coerces the literal rather than decoding every element.
372+
assert_snapshot!(
373+
df.into_optimized_plan()?,
374+
@r#"
375+
Projection: array_any_match(t.tags, (p0) -> p0 = Dictionary(UInt32, Utf8("c"))) AS matched
376+
TableScan: t projection=[tags]
377+
"#
378+
);
379+
Ok(())
380+
}
381+
382+
// Substrait has no string view type, so a producer records a plain parameter for a
383+
// lambda over a column the plan itself declares as a view.
384+
#[tokio::test]
385+
async fn higher_order_function_with_string_view_lambda_parameter() -> Result<()> {
386+
let proto_plan = read_json(
387+
"tests/testdata/test_plans/any_match_string_view_list_element.substrait.json",
388+
);
389+
let ctx = add_plan_schemas_to_ctx(SessionContext::new(), &proto_plan)?;
390+
let plan = from_substrait_plan(&ctx.state(), &proto_plan).await?;
391+
392+
let df = DataFrame::new(ctx.state(), plan);
393+
394+
// Without resolution this fails in physical planning, where the field recorded
395+
// on the parameter does not match the one the schema carries.
396+
df.clone().show().await?;
397+
398+
// The lambda parameter is resolved to the view the list actually carries, so the
399+
// comparison coerces the literal rather than materializing every element.
400+
assert_snapshot!(
401+
df.into_optimized_plan()?,
402+
@r#"
403+
Projection: array_any_match(t.tags, (p0) -> p0 = Utf8View("c")) AS matched
404+
TableScan: t projection=[tags]
405+
"#
406+
);
407+
Ok(())
408+
}
409+
410+
// A higher order function can be the value argument of another one, where its own
411+
// lambda parameter has to be resolved too.
412+
#[tokio::test]
413+
async fn nested_higher_order_function_with_dictionary_encoded_lambda_parameter()
414+
-> Result<()> {
415+
let proto_plan = read_json(
416+
"tests/testdata/test_plans/nested_any_match_dictionary_list_element.substrait.json",
417+
);
418+
let ctx = add_plan_schemas_to_ctx(SessionContext::new(), &proto_plan)?;
419+
let plan = from_substrait_plan(&ctx.state(), &proto_plan).await?;
420+
421+
let df = DataFrame::new(ctx.state(), plan);
422+
423+
// Without resolution of the inner function this fails in physical planning.
424+
df.clone().show().await?;
425+
426+
// Both parameters are resolved to the encoding the list actually carries.
427+
assert_snapshot!(
428+
df.into_optimized_plan()?,
429+
@r#"
430+
Projection: array_any_match(array_transform(t.tags, (p0) -> p0), (p1) -> p1 = Dictionary(UInt32, Utf8("c"))) AS matched
431+
TableScan: t projection=[tags]
432+
"#
433+
);
434+
Ok(())
435+
}
343436
}

0 commit comments

Comments
 (0)