Skip to content

Commit aacf025

Browse files
peterxcliOmega359
authored andcommitted
refactor: exhaustively destructure dynamic filter and scalar subquery proto hooks (apache#25178)
## Which issue does this PR close? Closes apache#24618. ## Rationale for this change This change makes the compiler catch fields that serialization does not handle. When a field is added to a dynamic filter, scalar subquery expression, or its protobuf payload, the corresponding hook must account for it. This helps prevent new fields from being silently dropped when expressions are encoded and decoded. ## What changes are included in this PR? All four encoding and decoding hooks for `DynamicFilterPhysicalExpr` and `ScalarSubqueryExpr` explicitly list every field without a rest pattern. Dynamic filter encoding also destructures the snapshot of its shared inner state, and decoding handles the expression ID carried by the enclosing message. Comments explain why caches, watch channels, cached type and nullability checks, and subquery results are omitted, including how they are rebuilt or supplied. The PR also replaces one use of unstable `std::assert_matches` in an existing execution test with `assert!(…is_ok())`, allowing the required checks to run on the pinned stable compiler. ## What is the testing strategy for this PR? Existing tests cover filter identity, shared updates, remapped children, subquery metadata, nested result scopes, and execution after serialization. The physical expression suite passed 1,739 tests with 2 ignored, and all 13 relevant protobuf integration tests passed. The full extended workspace run passed 11,591 Rust tests, with 8 ignored, and completed all 516 SQLLogicTest files. Formatting, Clippy with all targets and features and warnings denied, and the complete `./dev/rust_lint.sh` suite also passed. ## Are there any user-facing changes? No.
1 parent 469ee29 commit aacf025

3 files changed

Lines changed: 64 additions & 33 deletions

File tree

‎datafusion/execution/src/async_stream.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,6 @@ mod test {
302302
use crate::{async_stream, async_try_stream};
303303
use futures::stream::FusedStream;
304304
use futures::{Stream, StreamExt, pin_mut};
305-
use std::assert_matches;
306305
use std::pin::Pin;
307306
use std::sync::Arc;
308307
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -464,7 +463,7 @@ mod test {
464463
pin_mut!(s);
465464

466465
for i in 0..3 {
467-
assert_matches!(tx.send(i).await, Ok(_));
466+
assert!(tx.send(i).await.is_ok());
468467
assert_eq!(Some(i), s.next().await);
469468
}
470469

‎datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs‎

Lines changed: 39 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -606,32 +606,46 @@ impl PhysicalExpr for DynamicFilterPhysicalExpr {
606606
use datafusion_proto_models::protobuf;
607607
use datafusion_proto_models::protobuf::physical_expr_node::ExprType;
608608

609-
let children = self
610-
.children
609+
let Self {
610+
children,
611+
remapped_children,
612+
current_cache: _, // Runtime cache, repopulated by current().
613+
inner,
614+
state_watch: _, // Runtime channel, recreated from inner state by from_parts().
615+
data_type: _, // Cached test invariant, recomputed from the expression.
616+
nullable: _, // Cached test invariant, recomputed from the expression.
617+
} = self;
618+
619+
let children = children
611620
.iter()
612621
.map(|c| ctx.encode_child(c))
613622
.collect::<Result<Vec<_>>>()?;
614623

615-
let remapped_children = match &self.remapped_children {
624+
let remapped_children = match remapped_children {
616625
Some(remapped) => remapped
617626
.iter()
618627
.map(|c| ctx.encode_child(c))
619628
.collect::<Result<Vec<_>>>()?,
620629
None => vec![],
621630
};
622631

623-
let inner = self.inner.read().clone();
624-
let inner_expr = Box::new(ctx.encode_child(&inner.expr)?);
632+
let Inner {
633+
expression_id,
634+
generation,
635+
expr,
636+
is_complete,
637+
} = inner.read().clone();
638+
let inner_expr = Box::new(ctx.encode_child(&expr)?);
625639

626640
Ok(Some(protobuf::PhysicalExprNode {
627-
expr_id: Some(inner.expression_id),
641+
expr_id: Some(expression_id),
628642
expr_type: Some(ExprType::DynamicFilter(Box::new(
629643
protobuf::PhysicalDynamicFilterNode {
630644
children,
631645
remapped_children,
632-
generation: inner.generation,
646+
generation,
633647
inner_expr: Some(inner_expr),
634-
is_complete: inner.is_complete,
648+
is_complete,
635649
},
636650
))),
637651
}))
@@ -649,52 +663,60 @@ impl DynamicFilterPhysicalExpr {
649663
proto: &datafusion_proto_models::protobuf::PhysicalExprNode,
650664
ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,
651665
) -> Result<Arc<dyn PhysicalExpr>> {
666+
use datafusion_proto_models::protobuf;
652667
use datafusion_proto_models::protobuf::physical_expr_node::ExprType;
653668

654-
let ExprType::DynamicFilter(df) = proto.expr_type.as_ref().ok_or_else(|| {
669+
let protobuf::PhysicalExprNode { expr_id, expr_type } = proto;
670+
let ExprType::DynamicFilter(df) = expr_type.as_ref().ok_or_else(|| {
655671
internal_datafusion_err!("Missing expr_type in PhysicalExprNode")
656672
})?
657673
else {
658674
return Err(internal_datafusion_err!("Expected DynamicFilter expr_type"));
659675
};
676+
let protobuf::PhysicalDynamicFilterNode {
677+
children,
678+
remapped_children,
679+
generation,
680+
inner_expr,
681+
is_complete,
682+
} = df.as_ref();
660683

661684
// Decode original children
662-
let children = df
663-
.children
685+
let children = children
664686
.iter()
665687
.map(|c| ctx.decode(c))
666688
.collect::<Result<Vec<_>>>()?;
667689

668690
// Decode remapped children (empty vec means None)
669-
let remapped_children = if df.remapped_children.is_empty() {
691+
let remapped_children = if remapped_children.is_empty() {
670692
None
671693
} else {
672694
Some(
673-
df.remapped_children
695+
remapped_children
674696
.iter()
675697
.map(|c| ctx.decode(c))
676698
.collect::<Result<Vec<_>>>()?,
677699
)
678700
};
679701

680702
// Decode the inner expression
681-
let inner_expr_proto = df.inner_expr.as_ref().ok_or_else(|| {
703+
let inner_expr_proto = inner_expr.as_ref().ok_or_else(|| {
682704
internal_datafusion_err!("Missing inner_expr in PhysicalDynamicFilterNode")
683705
})?;
684706
let inner_expr = ctx.decode(inner_expr_proto)?;
685707

686708
// Restore the expression_id from the outer PhysicalExprNode
687-
let expression_id = proto.expr_id.ok_or_else(|| {
709+
let expression_id = expr_id.ok_or_else(|| {
688710
internal_datafusion_err!(
689711
"Missing expr_id in PhysicalExprNode for DynamicFilter"
690712
)
691713
})?;
692714

693715
let inner = Inner {
694716
expression_id,
695-
generation: df.generation,
717+
generation: *generation,
696718
expr: inner_expr,
697-
is_complete: df.is_complete,
719+
is_complete: *is_complete,
698720
};
699721

700722
Ok(Arc::new(Self::from_parts(

‎datafusion/physical-expr/src/scalar_subquery.rs‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -171,18 +171,25 @@ impl PhysicalExpr for ScalarSubqueryExpr {
171171
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {
172172
use datafusion_common::utils::usize_to_wire;
173173
use datafusion_proto_models::protobuf;
174+
175+
let Self {
176+
field,
177+
index,
178+
results: _, // Runtime state, supplied by ScalarSubqueryExec on decode.
179+
} = self;
180+
174181
Ok(Some(protobuf::PhysicalExprNode {
175182
expr_id: None,
176183
expr_type: Some(protobuf::physical_expr_node::ExprType::ScalarSubquery(
177184
protobuf::PhysicalScalarSubqueryExprNode {
178-
data_type: Some(self.field.data_type().try_into()?),
179-
nullable: self.field.is_nullable(),
185+
data_type: Some(field.data_type().try_into()?),
186+
nullable: field.is_nullable(),
180187
index: usize_to_wire(
181-
self.index.as_usize(),
188+
index.as_usize(),
182189
"ScalarSubqueryExpr",
183190
"index",
184191
)?,
185-
metadata: self.field.metadata().clone(),
192+
metadata: field.metadata().clone(),
186193
},
187194
)),
188195
}))
@@ -214,22 +221,25 @@ impl ScalarSubqueryExpr {
214221
protobuf::physical_expr_node::ExprType::ScalarSubquery,
215222
"ScalarSubqueryExpr",
216223
);
217-
let data_type = require_proto_field(
218-
sq.data_type.as_ref(),
219-
"ScalarSubqueryExpr",
220-
"data_type",
221-
)?
222-
.try_into()?;
223-
let metadata = if sq.metadata.is_empty() {
224+
let protobuf::PhysicalScalarSubqueryExprNode {
225+
data_type,
226+
nullable,
227+
index,
228+
metadata,
229+
} = sq;
230+
let data_type =
231+
require_proto_field(data_type.as_ref(), "ScalarSubqueryExpr", "data_type")?
232+
.try_into()?;
233+
let metadata = if metadata.is_empty() {
224234
None
225235
} else {
226-
Some(FieldMetadata::from(sq.metadata.clone()))
236+
Some(FieldMetadata::from(metadata.clone()))
227237
};
228238
Ok(Arc::new(ScalarSubqueryExpr::new_with_metadata(
229239
data_type,
230-
sq.nullable,
240+
*nullable,
231241
metadata,
232-
SubqueryIndex::new(sq.index as usize),
242+
SubqueryIndex::new(*index as usize),
233243
results.clone(),
234244
)))
235245
}

0 commit comments

Comments
 (0)