Skip to content

Commit f96892a

Browse files
authored
refactor(proto): destructure lambda expression serde hooks (apache#24775)
## Which issue does this PR close? - Closes apache#24617. ## Rationale for this change Proto hooks that access fields individually can silently omit newly added state. Exhaustive destructuring makes such omissions compile errors. ## What changes are included in this PR? - Exhaustively destructure `LambdaExpr` and `LambdaVariable` in their encoders. - Exhaustively destructure their protobuf nodes in the decoders. - Document constructor-derived state and add focused proto hook tests. The behavior and wire format are unchanged. ## Are these changes tested? Covered by the added lambda proto hook tests: ```bash cargo test -p datafusion-physical-expr --features proto lambda::proto_tests cargo test -p datafusion-physical-expr --features proto lambda_variable::proto_tests ``` ## Are there any user-facing changes? No.
1 parent 6fc7baa commit f96892a

3 files changed

Lines changed: 290 additions & 9 deletions

File tree

‎datafusion/physical-expr/src/expressions/lambda.rs‎

Lines changed: 162 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -168,10 +168,11 @@ impl LambdaExpr {
168168
protobuf::physical_expr_node::ExprType::Lambda,
169169
"LambdaExpr",
170170
);
171+
let protobuf::PhysicalLambdaExprNode { params, body } = &**lambda;
171172

172173
Ok(Arc::new(LambdaExpr::try_new(
173-
lambda.params.clone(),
174-
ctx.decode_required_expression(lambda.body.as_deref(), "LambdaExpr", "body")?,
174+
params.clone(),
175+
ctx.decode_required_expression(body.as_deref(), "LambdaExpr", "body")?,
175176
)?))
176177
}
177178

@@ -303,12 +304,24 @@ impl PhysicalExpr for LambdaExpr {
303304
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {
304305
use datafusion_proto_models::protobuf;
305306

307+
// Exhaustive destructure: adding a field to `LambdaExpr` without
308+
// deciding how it is serialized is a compile error, not a silent
309+
// round-trip gap.
310+
let Self {
311+
params,
312+
body,
313+
// Derived from `params` and `body` by `try_new` on decode.
314+
projected_body: _,
315+
projection: _,
316+
used_param_indices: _,
317+
} = self;
318+
306319
Ok(Some(protobuf::PhysicalExprNode {
307320
expr_id: None,
308321
expr_type: Some(protobuf::physical_expr_node::ExprType::Lambda(Box::new(
309322
protobuf::PhysicalLambdaExprNode {
310-
params: self.params().to_vec(),
311-
body: Some(Box::new(ctx.encode_child(self.body())?)),
323+
params: params.clone(),
324+
body: Some(Box::new(ctx.encode_child(body)?)),
312325
},
313326
))),
314327
}))
@@ -496,3 +509,148 @@ mod tests {
496509
);
497510
}
498511
}
512+
513+
/// Tests for the `try_to_proto` / `try_from_proto` hooks.
514+
#[cfg(all(test, feature = "proto"))]
515+
mod proto_tests {
516+
use super::*;
517+
use crate::proto_test_util::{
518+
StubDecoder, StubEncoder, UnreachableDecoder, column_node,
519+
};
520+
use arrow::datatypes::Field;
521+
use datafusion_common::DataFusionError;
522+
use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
523+
use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
524+
use datafusion_proto_models::protobuf::{self, PhysicalExprNode};
525+
526+
/// A one-parameter lambda whose body references that parameter.
527+
fn proto_lambda_fixture() -> LambdaExpr {
528+
let v_field = Arc::new(Field::new("v", DataType::Int32, true));
529+
LambdaExpr::try_new(
530+
vec!["v".to_string()],
531+
Arc::new(LambdaVariable::new(0, v_field)),
532+
)
533+
.unwrap()
534+
}
535+
536+
fn proto_lambda_node(
537+
params: Vec<String>,
538+
body: Option<Box<PhysicalExprNode>>,
539+
) -> PhysicalExprNode {
540+
PhysicalExprNode {
541+
expr_id: None,
542+
expr_type: Some(protobuf::physical_expr_node::ExprType::Lambda(Box::new(
543+
protobuf::PhysicalLambdaExprNode { params, body },
544+
))),
545+
}
546+
}
547+
548+
#[test]
549+
fn try_to_proto_encodes_lambda() {
550+
let lambda = proto_lambda_fixture();
551+
let encoder = StubEncoder::ok();
552+
let ctx = PhysicalExprEncodeCtx::new(&encoder);
553+
554+
let node = lambda
555+
.try_to_proto(&ctx)
556+
.unwrap()
557+
.expect("LambdaExpr should encode to Some(node)");
558+
559+
assert!(node.expr_id.is_none());
560+
let lambda_node = match node.expr_type {
561+
Some(protobuf::physical_expr_node::ExprType::Lambda(boxed)) => *boxed,
562+
other => panic!("expected a LambdaExpr node, got {other:?}"),
563+
};
564+
assert_eq!(lambda_node.params, vec!["v".to_string()]);
565+
assert!(lambda_node.body.is_some());
566+
}
567+
568+
#[test]
569+
fn try_to_proto_propagates_body_encode_error() {
570+
let lambda = proto_lambda_fixture();
571+
let encoder = StubEncoder::failing_on(1);
572+
let ctx = PhysicalExprEncodeCtx::new(&encoder);
573+
574+
let err = lambda.try_to_proto(&ctx).unwrap_err();
575+
assert!(matches!(err, DataFusionError::Internal(msg) if msg.contains("call 1")));
576+
}
577+
578+
/// Decoding goes through `try_new`, which recomputes `projection`,
579+
/// `projected_body`, and `used_param_indices` from the decoded body, so
580+
/// they need no wire representation.
581+
#[test]
582+
fn try_from_proto_recomputes_derived_state() {
583+
let node =
584+
proto_lambda_node(vec!["v".to_string()], Some(Box::new(column_node("body"))));
585+
let schema = Schema::empty();
586+
let decoder = StubDecoder::ok();
587+
let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
588+
589+
let decoded = LambdaExpr::try_from_proto(&node, &ctx).unwrap();
590+
let lambda = decoded
591+
.downcast_ref::<LambdaExpr>()
592+
.expect("decoded expr should be a LambdaExpr");
593+
594+
assert_eq!(lambda.params(), &["v".to_string()]);
595+
// The stub decodes the body to a `Column` at index 0: the projection
596+
// covers exactly that index, no declared param is referenced, and an
597+
// identity projection leaves the body untouched.
598+
assert_eq!(lambda.projection(), &[0]);
599+
assert!(lambda.used_param_indices().is_empty());
600+
assert!(lambda.projected_body().eq(lambda.body()));
601+
}
602+
603+
#[test]
604+
fn try_from_proto_rejects_non_lambda_node() {
605+
let node = column_node("a");
606+
let schema = Schema::empty();
607+
let decoder = UnreachableDecoder;
608+
let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
609+
610+
let err = LambdaExpr::try_from_proto(&node, &ctx).unwrap_err();
611+
assert!(
612+
matches!(err, DataFusionError::Internal(msg) if msg.contains("PhysicalExprNode is not a LambdaExpr"))
613+
);
614+
}
615+
616+
#[test]
617+
fn try_from_proto_rejects_missing_body() {
618+
let node = proto_lambda_node(vec!["v".to_string()], None);
619+
let schema = Schema::empty();
620+
let decoder = UnreachableDecoder;
621+
let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
622+
623+
let err = LambdaExpr::try_from_proto(&node, &ctx).unwrap_err();
624+
assert!(
625+
matches!(err, DataFusionError::Internal(msg) if msg.contains("LambdaExpr is missing required field 'body'"))
626+
);
627+
}
628+
629+
#[test]
630+
fn try_from_proto_rejects_duplicate_params() {
631+
let node = proto_lambda_node(
632+
vec!["v".to_string(), "v".to_string()],
633+
Some(Box::new(column_node("body"))),
634+
);
635+
let schema = Schema::empty();
636+
let decoder = StubDecoder::ok();
637+
let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
638+
639+
let err = LambdaExpr::try_from_proto(&node, &ctx).unwrap_err();
640+
assert!(
641+
matches!(err, DataFusionError::Plan(msg) if msg.contains("lambda params must be unique"))
642+
);
643+
}
644+
645+
#[test]
646+
fn try_from_proto_propagates_body_decode_error() {
647+
let node =
648+
proto_lambda_node(vec!["v".to_string()], Some(Box::new(column_node("body"))));
649+
let schema = Schema::empty();
650+
let decoder = StubDecoder::failing_on(1);
651+
let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
652+
653+
let err = LambdaExpr::try_from_proto(&node, &ctx).unwrap_err();
654+
assert!(matches!(err, DataFusionError::Internal(msg) if msg.contains("call 1")));
655+
}
656+
}

‎datafusion/physical-expr/src/expressions/lambda_variable.rs‎

Lines changed: 109 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -89,11 +89,12 @@ impl LambdaVariable {
8989
protobuf::physical_expr_node::ExprType::LambdaVariable,
9090
"LambdaVariable",
9191
);
92+
let protobuf::PhysicalLambdaVariableExprNode { index, field } = var;
9293

9394
Ok(Arc::new(LambdaVariable::new(
94-
var.index as usize,
95+
*index as usize,
9596
Arc::new(
96-
require_proto_field(var.field.as_ref(), "LambdaVariable", "field")?
97+
require_proto_field(field.as_ref(), "LambdaVariable", "field")?
9798
.try_into()?,
9899
),
99100
)))
@@ -169,12 +170,17 @@ impl PhysicalExpr for LambdaVariable {
169170
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {
170171
use datafusion_proto_models::protobuf;
171172

173+
// Exhaustive destructure: adding a field to `LambdaVariable` without
174+
// deciding how it is serialized is a compile error, not a silent
175+
// round-trip gap.
176+
let Self { index, field } = self;
177+
172178
Ok(Some(protobuf::PhysicalExprNode {
173179
expr_id: None,
174180
expr_type: Some(protobuf::physical_expr_node::ExprType::LambdaVariable(
175181
protobuf::PhysicalLambdaVariableExprNode {
176-
index: self.index() as u32,
177-
field: Some(self.field().as_ref().try_into()?),
182+
index: *index as u32,
183+
field: Some(field.as_ref().try_into()?),
178184
},
179185
)),
180186
}))
@@ -188,3 +194,102 @@ pub fn lambda_variable(name: &str, schema: &Schema) -> Result<Arc<dyn PhysicalEx
188194

189195
Ok(Arc::new(LambdaVariable::new(index, field)))
190196
}
197+
198+
/// Tests for the `try_to_proto` / `try_from_proto` hooks.
199+
#[cfg(all(test, feature = "proto"))]
200+
mod proto_tests {
201+
use super::*;
202+
use crate::proto_test_util::{StubEncoder, UnreachableDecoder, column_node};
203+
use arrow::datatypes::Field;
204+
use datafusion_common::DataFusionError;
205+
use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
206+
use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
207+
use datafusion_proto_models::protobuf::{self, physical_expr_node};
208+
209+
fn v_field() -> FieldRef {
210+
Arc::new(Field::new("v", DataType::Int32, true))
211+
}
212+
213+
#[test]
214+
fn try_to_proto_encodes_lambda_variable() {
215+
let var = LambdaVariable::new(3, v_field());
216+
// LambdaVariable has no child exprs so the encoder is never called.
217+
let encoder = StubEncoder::ok();
218+
let ctx = PhysicalExprEncodeCtx::new(&encoder);
219+
220+
let node = var
221+
.try_to_proto(&ctx)
222+
.unwrap()
223+
.expect("LambdaVariable should encode to Some(node)");
224+
225+
assert!(node.expr_id.is_none());
226+
let protobuf::PhysicalLambdaVariableExprNode { index, field } =
227+
match node.expr_type {
228+
Some(physical_expr_node::ExprType::LambdaVariable(v)) => v,
229+
other => panic!("expected a LambdaVariable node, got {other:?}"),
230+
};
231+
assert_eq!(index, 3);
232+
let field = field.expect("field should be encoded");
233+
assert_eq!(field.name, "v");
234+
assert!(field.nullable);
235+
}
236+
237+
#[test]
238+
fn try_from_proto_rejects_non_lambda_variable_node() {
239+
let node = column_node("a");
240+
let schema = Schema::empty();
241+
let decoder = UnreachableDecoder;
242+
let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
243+
244+
let err = LambdaVariable::try_from_proto(&node, &ctx).unwrap_err();
245+
assert!(
246+
matches!(err, DataFusionError::Internal(msg) if msg.contains("PhysicalExprNode is not a LambdaVariable"))
247+
);
248+
}
249+
250+
#[test]
251+
fn try_from_proto_rejects_missing_field() {
252+
let node = protobuf::PhysicalExprNode {
253+
expr_id: None,
254+
expr_type: Some(physical_expr_node::ExprType::LambdaVariable(
255+
protobuf::PhysicalLambdaVariableExprNode {
256+
index: 0,
257+
field: None,
258+
},
259+
)),
260+
};
261+
let schema = Schema::empty();
262+
let decoder = UnreachableDecoder;
263+
let ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
264+
265+
let err = LambdaVariable::try_from_proto(&node, &ctx).unwrap_err();
266+
assert!(
267+
matches!(err, DataFusionError::Internal(msg) if msg.contains("LambdaVariable is missing required field 'field'"))
268+
);
269+
}
270+
271+
/// `PartialEq` covers every field of `LambdaVariable`, so equality after
272+
/// the roundtrip proves the encoding is lossless.
273+
#[test]
274+
fn lambda_variable_proto_roundtrip() {
275+
let var = LambdaVariable::new(5, v_field());
276+
let encoder = StubEncoder::ok();
277+
let enc_ctx = PhysicalExprEncodeCtx::new(&encoder);
278+
279+
let node = var
280+
.try_to_proto(&enc_ctx)
281+
.unwrap()
282+
.expect("LambdaVariable should encode to Some(node)");
283+
284+
let schema = Schema::empty();
285+
let decoder = UnreachableDecoder;
286+
let dec_ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
287+
288+
let decoded = LambdaVariable::try_from_proto(&node, &dec_ctx).unwrap();
289+
let decoded = decoded
290+
.downcast_ref::<LambdaVariable>()
291+
.expect("decoded expr should be a LambdaVariable");
292+
293+
assert_eq!(decoded, &var);
294+
}
295+
}

‎datafusion/proto/tests/cases/plans/exprs.rs‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use arrow::datatypes::Fields;
2323
use datafusion::arrow::compute::SortOptions;
2424
use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema};
2525
use datafusion::logical_expr::Operator;
26-
use datafusion::physical_expr::expressions::Literal;
26+
use datafusion::physical_expr::expressions::{LambdaVariable, Literal, lambda};
2727
use datafusion::physical_plan::empty::EmptyExec;
2828
use datafusion::physical_plan::expressions::{
2929
BinaryExpr, Column, PhysicalSortExpr, SqlSimilarToPattern, binary, col, like, lit,
@@ -248,6 +248,24 @@ fn roundtrip_sql_similar_to_pattern() -> Result<()> {
248248
Ok(())
249249
}
250250

251+
#[test]
252+
fn roundtrip_lambda_with_variable_dispatch() -> Result<()> {
253+
let field = Arc::new(Field::new("v", DataType::Int32, true));
254+
let expr = lambda(["v"], Arc::new(LambdaVariable::new(0, field)))?;
255+
256+
let codec = DefaultPhysicalExtensionCodec {};
257+
let converter = DefaultPhysicalProtoConverter {};
258+
let proto = converter.physical_expr_to_proto(&expr, &codec)?;
259+
let ctx = SessionContext::new();
260+
let task_ctx = ctx.task_ctx();
261+
let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
262+
let decoded =
263+
converter.proto_to_physical_expr(&proto, &Schema::empty(), &decode_ctx)?;
264+
265+
assert!(decoded.as_ref().eq(expr.as_ref()));
266+
Ok(())
267+
}
268+
251269
#[test]
252270
fn roundtrip_call_null_scalar_struct_dict() -> Result<()> {
253271
let data_type = DataType::Struct(Fields::from(vec![Field::new(

0 commit comments

Comments
 (0)