Skip to content

Commit 7ac784f

Browse files
authored
Add protobuf support for lambdas (apache#22362)
## Which issue does this PR close? Part of apache#21172 ## Rationale for this change Protobuf support wasn't implemented in main lambda PR to not make it even bigger ## What changes are included in this PR? Protobuf encoding and decoding (~1000 LOC in generated files, ~210 impl, ~400 tests) ## Are these changes tested? Unit tests, similar to the existing ones for scalar functions ## Are there any user-facing changes? Proto `ExprType` has new variants
1 parent b79d396 commit 7ac784f

14 files changed

Lines changed: 1691 additions & 189 deletions

File tree

‎datafusion/physical-expr/src/expressions/lambda.rs‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,27 @@ impl LambdaExpr {
142142
&self.body
143143
}
144144

145+
#[cfg(feature = "proto")]
146+
/// Reconstruct a [`LambdaExpr`] from a proto node.
147+
pub fn try_from_proto(
148+
node: &datafusion_proto_models::protobuf::PhysicalExprNode,
149+
ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,
150+
) -> Result<Arc<dyn PhysicalExpr>> {
151+
use datafusion_physical_expr_common::expect_expr_variant;
152+
use datafusion_proto_models::protobuf;
153+
154+
let lambda = expect_expr_variant!(
155+
node,
156+
protobuf::physical_expr_node::ExprType::Lambda,
157+
"LambdaExpr",
158+
);
159+
160+
Ok(Arc::new(LambdaExpr::try_new(
161+
lambda.params.clone(),
162+
ctx.decode_required_expression(lambda.body.as_deref(), "LambdaExpr", "body")?,
163+
)?))
164+
}
165+
145166
pub(crate) fn projection(&self) -> &[usize] {
146167
&self.projection
147168
}
@@ -193,6 +214,24 @@ impl PhysicalExpr for LambdaExpr {
193214
fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
194215
write!(f, "({}) -> {}", self.params.join(", "), self.body)
195216
}
217+
218+
#[cfg(feature = "proto")]
219+
fn try_to_proto(
220+
&self,
221+
ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,
222+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {
223+
use datafusion_proto_models::protobuf;
224+
225+
Ok(Some(protobuf::PhysicalExprNode {
226+
expr_id: None,
227+
expr_type: Some(protobuf::physical_expr_node::ExprType::Lambda(Box::new(
228+
protobuf::PhysicalLambdaExprNode {
229+
params: self.params().to_vec(),
230+
body: Some(Box::new(ctx.encode_child(self.body())?)),
231+
},
232+
))),
233+
}))
234+
}
196235
}
197236

198237
/// Create a lambda expression

‎datafusion/physical-expr/src/expressions/lambda_variable.rs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,32 @@ impl LambdaVariable {
7272
pub fn field(&self) -> &FieldRef {
7373
&self.field
7474
}
75+
76+
#[cfg(feature = "proto")]
77+
/// Reconstruct a [`LambdaVariable`] from a proto node.
78+
pub fn try_from_proto(
79+
node: &datafusion_proto_models::protobuf::PhysicalExprNode,
80+
_ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,
81+
) -> Result<Arc<dyn PhysicalExpr>> {
82+
use datafusion_physical_expr_common::{
83+
expect_expr_variant, physical_expr::proto_decode::require_proto_field,
84+
};
85+
use datafusion_proto_models::protobuf;
86+
87+
let var = expect_expr_variant!(
88+
node,
89+
protobuf::physical_expr_node::ExprType::LambdaVariable,
90+
"LambdaVariable",
91+
);
92+
93+
Ok(Arc::new(LambdaVariable::new(
94+
var.index as usize,
95+
Arc::new(
96+
require_proto_field(var.field.as_ref(), "LambdaVariable", "field")?
97+
.try_into()?,
98+
),
99+
)))
100+
}
75101
}
76102

77103
impl std::fmt::Display for LambdaVariable {
@@ -135,6 +161,24 @@ impl PhysicalExpr for LambdaVariable {
135161
fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
136162
write!(f, "{}@{}", self.name(), self.index)
137163
}
164+
165+
#[cfg(feature = "proto")]
166+
fn try_to_proto(
167+
&self,
168+
_ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,
169+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {
170+
use datafusion_proto_models::protobuf;
171+
172+
Ok(Some(protobuf::PhysicalExprNode {
173+
expr_id: None,
174+
expr_type: Some(protobuf::physical_expr_node::ExprType::LambdaVariable(
175+
protobuf::PhysicalLambdaVariableExprNode {
176+
index: self.index() as u32,
177+
field: Some(self.field().as_ref().try_into()?),
178+
},
179+
)),
180+
}))
181+
}
138182
}
139183

140184
/// Create a lambda variable expression

‎datafusion/proto-models/proto/datafusion.proto‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -501,6 +501,10 @@ message LogicalExprNode {
501501

502502
// Subquery expressions
503503
ScalarSubqueryExprNode scalar_subquery_expr = 36;
504+
505+
HigherOrderUDFExprNode higher_order_udf_expr = 37;
506+
Lambda lambda = 38;
507+
LambdaVariable lambda_variable = 39;
504508
}
505509
}
506510

@@ -638,6 +642,22 @@ message ScalarUDFExprNode {
638642
optional bytes fun_definition = 3;
639643
}
640644

645+
message HigherOrderUDFExprNode {
646+
string fun_name = 1;
647+
repeated LogicalExprNode args = 2;
648+
optional bytes fun_definition = 3;
649+
}
650+
651+
message Lambda {
652+
repeated string params = 1;
653+
LogicalExprNode body = 2;
654+
}
655+
656+
message LambdaVariable {
657+
string name = 1;
658+
datafusion_common.Field field = 2;
659+
}
660+
641661
message WindowExprNode {
642662
oneof window_function {
643663
// BuiltInWindowFunction built_in_function = 2;
@@ -1007,6 +1027,10 @@ message PhysicalExprNode {
10071027
PhysicalScalarSubqueryExprNode scalar_subquery = 22;
10081028

10091029
PhysicalDynamicFilterNode dynamic_filter = 23;
1030+
1031+
PhysicalHigherOrderUdfNode higher_order_udf = 24;
1032+
PhysicalLambdaExprNode lambda = 25;
1033+
PhysicalLambdaVariableExprNode lambda_variable = 26;
10101034
}
10111035
}
10121036

@@ -1027,6 +1051,22 @@ message PhysicalScalarUdfNode {
10271051
string return_field_name = 6;
10281052
}
10291053

1054+
message PhysicalHigherOrderUdfNode {
1055+
string name = 1;
1056+
repeated PhysicalExprNode args = 2;
1057+
optional bytes fun_definition = 3;
1058+
}
1059+
1060+
message PhysicalLambdaExprNode {
1061+
repeated string params = 1;
1062+
PhysicalExprNode body = 2;
1063+
}
1064+
1065+
message PhysicalLambdaVariableExprNode {
1066+
uint32 index = 1;
1067+
datafusion_common.Field field = 2;
1068+
}
1069+
10301070
message PhysicalAggregateExprNode {
10311071
oneof AggregateFunction {
10321072
string user_defined_aggr_function = 4;

0 commit comments

Comments
 (0)