Skip to content

Commit 3a8ad01

Browse files
committed
add fields to lambda expr
1 parent 1ce3957 commit 3a8ad01

7 files changed

Lines changed: 135 additions & 177 deletions

File tree

‎datafusion/expr/src/expr.rs‎

Lines changed: 38 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,8 @@ use std::sync::Arc;
2727
use crate::expr_fn::binary_expr;
2828
use crate::function::WindowFunctionSimplification;
2929
use crate::logical_plan::Subquery;
30-
use crate::type_coercion::functions::value_fields_with_higher_order_udf;
3130
use crate::udhof::HigherOrderUDF;
32-
use crate::{AggregateUDF, LambdaParametersProgress, ValueOrLambda, Volatility};
31+
use crate::{AggregateUDF, Volatility};
3332
use crate::{ExprSchemable, Operator, Signature, WindowFrame, WindowUDF};
3433

3534
use arrow::datatypes::{DataType, Field, FieldRef};
@@ -40,7 +39,7 @@ use datafusion_common::tree_node::{
4039
Transformed, TransformedResult, TreeNode, TreeNodeContainer, TreeNodeRecursion,
4140
};
4241
use datafusion_common::{
43-
Column, DFSchema, ExprSchema, HashMap, Result, ScalarValue, Spans, TableReference,
42+
Column, DFSchema, HashMap, Result, ScalarValue, Spans, TableReference,
4443
};
4544
use datafusion_expr_common::placement::ExpressionPlacement;
4645
use datafusion_functions_window_common::field::WindowUDFFieldArgs;
@@ -444,32 +443,6 @@ impl HigherOrderFunction {
444443
pub fn name(&self) -> &str {
445444
self.func.name()
446445
}
447-
448-
/// Invokes the inner function [`HigherOrderUDF::lambda_parameters`]
449-
/// using the arguments of this invocation
450-
pub fn lambda_parameters(
451-
&self,
452-
schema: &dyn ExprSchema,
453-
) -> Result<Vec<Vec<FieldRef>>> {
454-
let args = self
455-
.args
456-
.iter()
457-
.map(|e| match e {
458-
Expr::Lambda(lambda) => {
459-
Ok(ValueOrLambda::Lambda(Some(lambda.body.to_field(schema)?.1)))
460-
}
461-
_ => Ok(ValueOrLambda::Value(e.to_field(schema)?.1)),
462-
})
463-
.collect::<Result<Vec<_>>>()?;
464-
465-
let coerced_fields =
466-
value_fields_with_higher_order_udf(&args, self.func.as_ref())?;
467-
468-
match self.func.lambda_parameters(0, &coerced_fields)? {
469-
LambdaParametersProgress::Partial(_) => todo!(),
470-
LambdaParametersProgress::Complete(items) => Ok(items),
471-
}
472-
}
473446
}
474447

475448
impl Hash for HigherOrderFunction {
@@ -1395,16 +1368,38 @@ impl GroupingSet {
13951368
#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
13961369
pub struct Lambda {
13971370
/// The parameters names
1398-
pub params: Vec<String>,
1371+
pub params: LambdaParams,
13991372
/// The body expression
14001373
pub body: Box<Expr>,
14011374
}
14021375

1376+
/// The params of a [Lambda]
1377+
#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
1378+
pub enum LambdaParams {
1379+
Bound(Vec<FieldRef>),
1380+
Unbound(Vec<String>),
1381+
}
1382+
1383+
impl LambdaParams {
1384+
pub fn names(&self) -> impl Iterator<Item = &str> {
1385+
let (b, u) = match self {
1386+
LambdaParams::Bound(fields) => {
1387+
(Some(fields.iter().map(|f| f.name().as_str())), None)
1388+
}
1389+
LambdaParams::Unbound(items) => {
1390+
(None, Some(items.iter().map(|n| n.as_str())))
1391+
}
1392+
};
1393+
1394+
b.into_iter().flatten().chain(u.into_iter().flatten())
1395+
}
1396+
}
1397+
14031398
impl Lambda {
14041399
/// Create a new lambda expression
14051400
pub fn new(params: Vec<String>, body: Expr) -> Self {
14061401
Self {
1407-
params,
1402+
params: LambdaParams::Unbound(params),
14081403
body: Box::new(body),
14091404
}
14101405
}
@@ -3250,7 +3245,7 @@ impl Display for SchemaDisplay<'_> {
32503245
write!(
32513246
f,
32523247
"({}) -> {}",
3253-
display_comma_separated(params),
3248+
params.names().collect::<Vec<_>>().join(", "),
32543249
SchemaDisplay(body)
32553250
)
32563251
}
@@ -3436,7 +3431,12 @@ impl Display for SqlDisplay<'_> {
34363431
}
34373432
}
34383433
Expr::Lambda(Lambda { params, body }) => {
3439-
write!(f, "({}) -> {}", params.join(", "), SchemaDisplay(body))
3434+
write!(
3435+
f,
3436+
"({}) -> {}",
3437+
params.names().collect::<Vec<_>>().join(", "),
3438+
SchemaDisplay(body)
3439+
)
34403440
}
34413441
_ => write!(f, "{}", self.0),
34423442
}
@@ -3759,7 +3759,11 @@ impl Display for Expr {
37593759
fmt_function(f, fun.name(), false, &fun.args, true)
37603760
}
37613761
Expr::Lambda(Lambda { params, body }) => {
3762-
write!(f, "({}) -> {body}", params.join(", "))
3762+
write!(
3763+
f,
3764+
"({}) -> {body}",
3765+
params.names().collect::<Vec<_>>().join(", ")
3766+
)
37633767
}
37643768
Expr::LambdaVariable(c) => f.write_str(&c.name),
37653769
}

‎datafusion/expr/src/udhof.rs‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -328,12 +328,6 @@ pub trait HigherOrderUDF: Debug + DynEq + DynHash + Send + Sync + Any {
328328
/// lambdas are supported and some are optional, all should be returned,
329329
/// regardless of whether they are used on a particular invocation
330330
///
331-
/// Tip: If you have a [`HigherOrderFunction`] invocation, you can call the helper
332-
/// [`HigherOrderFunction::lambda_parameters`] instead of this method directly
333-
///
334-
/// [`HigherOrderFunction`]: crate::expr::HigherOrderFunction
335-
/// [`HigherOrderFunction::lambda_parameters`]: crate::expr::HigherOrderFunction::lambda_parameters
336-
///
337331
/// Example for array_transform:
338332
///
339333
/// `array_transform([2.0, 8.0], v -> v > 4.0)`

‎datafusion/physical-expr/src/expressions/lambda.rs‎

Lines changed: 47 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ use std::sync::Arc;
2222

2323
use crate::physical_expr::PhysicalExpr;
2424
use arrow::{
25-
datatypes::{DataType, Schema},
25+
datatypes::{DataType, FieldRef, Schema},
2626
record_batch::RecordBatch,
2727
};
2828
use datafusion_common::plan_err;
@@ -32,7 +32,7 @@ use datafusion_expr::ColumnarValue;
3232
/// Represents a lambda with the given parameters names and body
3333
#[derive(Debug, Eq, Clone)]
3434
pub struct LambdaExpr {
35-
params: Vec<String>,
35+
params: Vec<FieldRef>,
3636
body: Arc<dyn PhysicalExpr>,
3737
}
3838

@@ -52,23 +52,35 @@ impl Hash for LambdaExpr {
5252

5353
impl LambdaExpr {
5454
/// Create a new lambda expression with the given parameters and body
55-
pub fn try_new(params: Vec<String>, body: Arc<dyn PhysicalExpr>) -> Result<Self> {
55+
pub fn try_new(params: Vec<FieldRef>, body: Arc<dyn PhysicalExpr>) -> Result<Self> {
5656
if all_unique(&params) {
5757
Ok(Self::new(params, body))
5858
} else {
59-
plan_err!("lambda params must be unique, got ({})", params.join(", "))
59+
plan_err!(
60+
"lambda params must be unique, got ({})",
61+
params
62+
.iter()
63+
.map(|f| f.name().as_str())
64+
.collect::<Vec<_>>()
65+
.join(", ")
66+
)
6067
}
6168
}
6269

63-
fn new(params: Vec<String>, body: Arc<dyn PhysicalExpr>) -> Self {
70+
fn new(params: Vec<FieldRef>, body: Arc<dyn PhysicalExpr>) -> Self {
6471
Self { params, body }
6572
}
6673

67-
/// Get the lambda's params names
68-
pub fn params(&self) -> &[String] {
74+
/// Get the lambda's params
75+
pub fn params(&self) -> &[FieldRef] {
6976
&self.params
7077
}
7178

79+
/// Get the lambda's params
80+
pub fn param_names(&self) -> impl Iterator<Item = &str> {
81+
self.params.iter().map(|f| f.name().as_str())
82+
}
83+
7284
/// Get the lambda's body
7385
pub fn body(&self) -> &Arc<dyn PhysicalExpr> {
7486
&self.body
@@ -77,7 +89,12 @@ impl LambdaExpr {
7789

7890
impl std::fmt::Display for LambdaExpr {
7991
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
80-
write!(f, "({}) -> {}", self.params.join(", "), self.body)
92+
write!(
93+
f,
94+
"({}) -> {}",
95+
self.param_names().collect::<Vec<_>>().join(", "),
96+
self.body
97+
)
8198
}
8299
}
83100

@@ -113,48 +130,61 @@ impl PhysicalExpr for LambdaExpr {
113130
}
114131

115132
fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
116-
write!(f, "({}) -> {}", self.params.join(", "), self.body)
133+
write!(
134+
f,
135+
"({}) -> {}",
136+
self.param_names().collect::<Vec<_>>().join(", "),
137+
self.body
138+
)
117139
}
118140
}
119141

120142
/// Create a lambda expression
121143
pub fn lambda(
122-
params: impl IntoIterator<Item = impl Into<String>>,
144+
params: impl IntoIterator<Item = FieldRef>,
123145
body: Arc<dyn PhysicalExpr>,
124146
) -> Result<Arc<dyn PhysicalExpr>> {
125147
Ok(Arc::new(LambdaExpr::try_new(
126-
params.into_iter().map(Into::into).collect(),
148+
params.into_iter().collect(),
127149
body,
128150
)?))
129151
}
130152

131-
fn all_unique(params: &[String]) -> bool {
153+
// Checks wheter params names are all unique
154+
fn all_unique(params: &[FieldRef]) -> bool {
132155
match params.len() {
133156
0 | 1 => true,
134-
2 => params[0] != params[1],
157+
2 => params[0].name() != params[1].name(),
135158
_ => {
136159
let mut set = HashSet::with_capacity(params.len());
137160

138-
params.iter().all(|p| set.insert(p.as_str()))
161+
params.iter().all(|p| set.insert(p.name()))
139162
}
140163
}
141164
}
142165

143166
#[cfg(test)]
144167
mod tests {
145168
use crate::expressions::{NoOp, lambda::lambda};
146-
use arrow::{array::RecordBatch, datatypes::Schema};
169+
use arrow::{
170+
array::RecordBatch,
171+
datatypes::{DataType, Field, FieldRef, Schema},
172+
};
147173
use std::sync::Arc;
148174

175+
fn field_a() -> FieldRef {
176+
Arc::new(Field::new("a", DataType::Null, true))
177+
}
178+
149179
#[test]
150180
fn test_lambda_evaluate() {
151-
let lambda = lambda(["a"], Arc::new(NoOp::new())).unwrap();
181+
let lambda = lambda([field_a()], Arc::new(NoOp::new())).unwrap();
152182
let batch = RecordBatch::new_empty(Arc::new(Schema::empty()));
153183
assert!(lambda.evaluate(&batch).is_err());
154184
}
155185

156186
#[test]
157187
fn test_lambda_duplicate_name() {
158-
assert!(lambda(["a", "a"], Arc::new(NoOp::new())).is_err());
188+
assert!(lambda([field_a(), field_a()], Arc::new(NoOp::new())).is_err());
159189
}
160190
}

0 commit comments

Comments
 (0)