Skip to content

Commit 1ea3955

Browse files
committed
[X-2935] PhysicalExtensionCodec: thread proto_converter through methods
Modifies the PhysicalExtensionCodec trait so that try_encode, try_decode, try_encode_expr, and try_decode_expr receive the active PhysicalProtoConverterExtension. Codecs that embed nested PhysicalExprNode fields inside their custom proto (e.g. atlas's ReverseParquetNode.predicate) must now route those through proto_converter.proto_to_physical_expr / proto_converter.physical_expr_to_proto so the dedup cache extends across the extension boundary. Without this plumbing, an outer DeduplicatingSerializer can't reach into extension nodes -- the dedup cache stops at the extension boundary and Arc<Inner> sharing breaks for any expression carried inside an extension codec, even though the wire format already supports it via the expr_id field. With this change, atlas can finally drop the share_dynamic_filters walker once its codecs override the new methods. This is the breaking-change pattern matching the apache/datafusion upstream design (issue apache#22920) instead of additive _with_converter variants -- branch-53 is internal-only so callers can be updated in lockstep.
1 parent 0e6d988 commit 1ea3955

5 files changed

Lines changed: 82 additions & 25 deletions

File tree

‎datafusion/physical-expr/src/expressions/mod.rs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,7 @@ pub use cast::{CastExpr, cast};
4545
pub use cast_column::CastColumnExpr;
4646
pub use column::{Column, col, with_new_schema};
4747
pub use datafusion_expr::utils::format_state_name;
48-
pub use dynamic_filters::{
49-
DynamicFilterPhysicalExpr, Inner as DynamicFilterInner,
50-
};
48+
pub use dynamic_filters::{DynamicFilterPhysicalExpr, Inner as DynamicFilterInner};
5149
pub use in_list::{InListExpr, in_list};
5250
pub use is_not_null::{IsNotNullExpr, is_not_null};
5351
pub use is_null::{IsNullExpr, is_null};

‎datafusion/proto/src/physical_plan/from_proto.rs‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -506,7 +506,12 @@ pub fn parse_physical_expr_with_converter(
506506
proto_converter.proto_to_physical_expr(e, ctx, input_schema, codec)
507507
})
508508
.collect::<Result<_>>()?;
509-
codec.try_decode_expr(extension.expr.as_slice(), &inputs)? as _
509+
// Use the converter-aware decode entry point so extension codecs
510+
// that embed nested `PhysicalExprNode` fields can thread the
511+
// active `proto_converter` into their own parsing of those
512+
// nested expressions, sharing the dedup cache.
513+
codec.try_decode_expr(extension.expr.as_slice(), &inputs, proto_converter)?
514+
as _
510515
}
511516
ExprType::DynamicFilter(df_proto) => {
512517
// Reconstruct the DynamicFilterPhysicalExpr wrapper. When this
@@ -518,8 +523,9 @@ pub fn parse_physical_expr_with_converter(
518523
//
519524
// Ported from upstream apache/datafusion#21807 (minimal subset).
520525
let inner_expr = match df_proto.inner_expr.as_deref() {
521-
Some(p) => proto_converter
522-
.proto_to_physical_expr(p, ctx, input_schema, codec)?,
526+
Some(p) => {
527+
proto_converter.proto_to_physical_expr(p, ctx, input_schema, codec)?
528+
}
523529
None => {
524530
return Err(proto_error(
525531
"PhysicalDynamicFilterNode missing inner_expr",
@@ -530,8 +536,7 @@ pub fn parse_physical_expr_with_converter(
530536
.children
531537
.iter()
532538
.map(|c| {
533-
proto_converter
534-
.proto_to_physical_expr(c, ctx, input_schema, codec)
539+
proto_converter.proto_to_physical_expr(c, ctx, input_schema, codec)
535540
})
536541
.collect::<Result<Vec<_>>>()?;
537542
let remapped_children = if df_proto.remapped_children.is_empty() {

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 46 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -572,7 +572,7 @@ impl protobuf::PhysicalPlanNode {
572572
}
573573

574574
let mut buf: Vec<u8> = vec![];
575-
match codec.try_encode(Arc::clone(&plan_clone), &mut buf) {
575+
match codec.try_encode(Arc::clone(&plan_clone), &mut buf, proto_converter) {
576576
Ok(_) => {
577577
let inputs: Vec<protobuf::PhysicalPlanNode> = plan_clone
578578
.children()
@@ -1787,7 +1787,8 @@ impl protobuf::PhysicalPlanNode {
17871787
.map(|i| proto_converter.proto_to_execution_plan(ctx, codec, i))
17881788
.collect::<Result<_>>()?;
17891789

1790-
let extension_node = codec.try_decode(extension.node.as_slice(), &inputs, ctx)?;
1790+
let extension_node =
1791+
codec.try_decode(extension.node.as_slice(), &inputs, ctx, proto_converter)?;
17911792

17921793
Ok(extension_node)
17931794
}
@@ -3717,14 +3718,33 @@ pub trait AsExecutionPlan: Debug + Send + Sync + Clone {
37173718
}
37183719

37193720
pub trait PhysicalExtensionCodec: Debug + Send + Sync {
3721+
/// Decode an extension execution plan.
3722+
///
3723+
/// `proto_converter` is the active conversion strategy (e.g.
3724+
/// `DeduplicatingDeserializer`). Codecs whose custom proto embeds
3725+
/// nested `PhysicalExprNode` fields (e.g. a predicate on a custom
3726+
/// file source) should route those through
3727+
/// `proto_converter.proto_to_physical_expr` so the dedup cache
3728+
/// extends across the extension boundary.
3729+
///
3730+
/// Mirrors the upstream apache/datafusion pattern (#22920).
37203731
fn try_decode(
37213732
&self,
37223733
buf: &[u8],
37233734
inputs: &[Arc<dyn ExecutionPlan>],
37243735
ctx: &TaskContext,
3736+
proto_converter: &dyn PhysicalProtoConverterExtension,
37253737
) -> Result<Arc<dyn ExecutionPlan>>;
37263738

3727-
fn try_encode(&self, node: Arc<dyn ExecutionPlan>, buf: &mut Vec<u8>) -> Result<()>;
3739+
/// Encode an extension execution plan.
3740+
///
3741+
/// See [`Self::try_decode`] for the role of `proto_converter`.
3742+
fn try_encode(
3743+
&self,
3744+
node: Arc<dyn ExecutionPlan>,
3745+
buf: &mut Vec<u8>,
3746+
proto_converter: &dyn PhysicalProtoConverterExtension,
3747+
) -> Result<()>;
37283748

37293749
fn try_decode_udf(&self, name: &str, _buf: &[u8]) -> Result<Arc<ScalarUDF>> {
37303750
not_impl_err!("PhysicalExtensionCodec is not provided for scalar function {name}")
@@ -3734,18 +3754,26 @@ pub trait PhysicalExtensionCodec: Debug + Send + Sync {
37343754
Ok(())
37353755
}
37363756

3757+
/// Decode an extension expression.
3758+
///
3759+
/// See [`Self::try_decode`] for the role of `proto_converter`.
37373760
fn try_decode_expr(
37383761
&self,
37393762
_buf: &[u8],
37403763
_inputs: &[Arc<dyn PhysicalExpr>],
3764+
_proto_converter: &dyn PhysicalProtoConverterExtension,
37413765
) -> Result<Arc<dyn PhysicalExpr>> {
37423766
not_impl_err!("PhysicalExtensionCodec is not provided")
37433767
}
37443768

3769+
/// Encode an extension expression.
3770+
///
3771+
/// See [`Self::try_decode`] for the role of `proto_converter`.
37453772
fn try_encode_expr(
37463773
&self,
37473774
_node: &Arc<dyn PhysicalExpr>,
37483775
_buf: &mut Vec<u8>,
3776+
_proto_converter: &dyn PhysicalProtoConverterExtension,
37493777
) -> Result<()> {
37503778
not_impl_err!("PhysicalExtensionCodec is not provided")
37513779
}
@@ -3778,6 +3806,7 @@ impl PhysicalExtensionCodec for DefaultPhysicalExtensionCodec {
37783806
_buf: &[u8],
37793807
_inputs: &[Arc<dyn ExecutionPlan>],
37803808
_ctx: &TaskContext,
3809+
_proto_converter: &dyn PhysicalProtoConverterExtension,
37813810
) -> Result<Arc<dyn ExecutionPlan>> {
37823811
not_impl_err!("PhysicalExtensionCodec is not provided")
37833812
}
@@ -3786,6 +3815,7 @@ impl PhysicalExtensionCodec for DefaultPhysicalExtensionCodec {
37863815
&self,
37873816
_node: Arc<dyn ExecutionPlan>,
37883817
_buf: &mut Vec<u8>,
3818+
_proto_converter: &dyn PhysicalProtoConverterExtension,
37893819
) -> Result<()> {
37903820
not_impl_err!("PhysicalExtensionCodec is not provided")
37913821
}
@@ -4213,12 +4243,22 @@ impl PhysicalExtensionCodec for ComposedPhysicalExtensionCodec {
42134243
buf: &[u8],
42144244
inputs: &[Arc<dyn ExecutionPlan>],
42154245
ctx: &TaskContext,
4246+
proto_converter: &dyn PhysicalProtoConverterExtension,
42164247
) -> Result<Arc<dyn ExecutionPlan>> {
4217-
self.decode_protobuf(buf, |codec, data| codec.try_decode(data, inputs, ctx))
4248+
self.decode_protobuf(buf, |codec, data| {
4249+
codec.try_decode(data, inputs, ctx, proto_converter)
4250+
})
42184251
}
42194252

4220-
fn try_encode(&self, node: Arc<dyn ExecutionPlan>, buf: &mut Vec<u8>) -> Result<()> {
4221-
self.encode_protobuf(buf, |codec, data| codec.try_encode(Arc::clone(&node), data))
4253+
fn try_encode(
4254+
&self,
4255+
node: Arc<dyn ExecutionPlan>,
4256+
buf: &mut Vec<u8>,
4257+
proto_converter: &dyn PhysicalProtoConverterExtension,
4258+
) -> Result<()> {
4259+
self.encode_protobuf(buf, |codec, data| {
4260+
codec.try_encode(Arc::clone(&node), data, proto_converter)
4261+
})
42224262
}
42234263

42244264
fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result<Arc<ScalarUDF>> {

‎datafusion/proto/src/physical_plan/to_proto.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -576,7 +576,12 @@ pub fn serialize_physical_expr_with_converter(
576576
})
577577
} else {
578578
let mut buf: Vec<u8> = vec![];
579-
match codec.try_encode_expr(&value, &mut buf) {
579+
// Use the converter-aware encode entry point so extension codecs
580+
// that embed nested `PhysicalExprNode` fields can thread the
581+
// active `proto_converter` (e.g. `DeduplicatingSerializer`) into
582+
// their own serialization of those nested expressions, preserving
583+
// Arc-identity dedup end-to-end.
584+
match codec.try_encode_expr(&value, &mut buf, proto_converter) {
580585
Ok(_) => {
581586
let inputs: Vec<protobuf::PhysicalExprNode> = value
582587
.children()

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3186,15 +3186,18 @@ fn dynamic_filter_dedup_with_deduplicating_codec() -> Result<()> {
31863186
let schema = Schema::new(vec![Field::new("a", DataType::Int64, false)]);
31873187
let initial: Arc<dyn PhysicalExpr> = lit(true);
31883188
let children: Vec<Arc<dyn PhysicalExpr>> = vec![Arc::new(Column::new("a", 0))];
3189-
let df = Arc::new(DynamicFilterPhysicalExpr::new(children, Arc::clone(&initial)));
3189+
let df = Arc::new(DynamicFilterPhysicalExpr::new(
3190+
children,
3191+
Arc::clone(&initial),
3192+
));
31903193
let id_before = df.inner().expression_id;
31913194

31923195
let codec = DefaultPhysicalExtensionCodec {};
31933196
let serializer = DeduplicatingSerializer::new();
3194-
let proto1 =
3195-
serializer.physical_expr_to_proto(&(df.clone() as Arc<dyn PhysicalExpr>), &codec)?;
3196-
let proto2 =
3197-
serializer.physical_expr_to_proto(&(df.clone() as Arc<dyn PhysicalExpr>), &codec)?;
3197+
let proto1 = serializer
3198+
.physical_expr_to_proto(&(df.clone() as Arc<dyn PhysicalExpr>), &codec)?;
3199+
let proto2 = serializer
3200+
.physical_expr_to_proto(&(df.clone() as Arc<dyn PhysicalExpr>), &codec)?;
31983201

31993202
assert_eq!(
32003203
proto1.expr_id, proto2.expr_id,
@@ -3214,7 +3217,9 @@ fn dynamic_filter_dedup_with_deduplicating_codec() -> Result<()> {
32143217
let d1_df = d1
32153218
.as_any()
32163219
.downcast_ref::<DynamicFilterPhysicalExpr>()
3217-
.expect("decoded expr must be DynamicFilterPhysicalExpr; snapshot path is bypassed");
3220+
.expect(
3221+
"decoded expr must be DynamicFilterPhysicalExpr; snapshot path is bypassed",
3222+
);
32183223
assert_eq!(
32193224
d1_df.inner().expression_id,
32203225
id_before,
@@ -3244,9 +3249,10 @@ fn dynamic_filter_dedup_distinct_outer_arcs_same_inner() -> Result<()> {
32443249
let schema = Schema::new(vec![Field::new("a", DataType::Int64, false)]);
32453250
let initial: Arc<dyn PhysicalExpr> = lit(true);
32463251
let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
3247-
let df_arc1: Arc<dyn PhysicalExpr> = Arc::new(
3248-
DynamicFilterPhysicalExpr::new(vec![Arc::clone(&col_a)], Arc::clone(&initial)),
3249-
);
3252+
let df_arc1: Arc<dyn PhysicalExpr> = Arc::new(DynamicFilterPhysicalExpr::new(
3253+
vec![Arc::clone(&col_a)],
3254+
Arc::clone(&initial),
3255+
));
32503256
let expected_id = df_arc1
32513257
.as_any()
32523258
.downcast_ref::<DynamicFilterPhysicalExpr>()
@@ -3310,7 +3316,10 @@ fn dynamic_filter_roundtrip_without_dedup() -> Result<()> {
33103316
let schema = Schema::new(vec![Field::new("a", DataType::Int64, false)]);
33113317
let initial: Arc<dyn PhysicalExpr> = lit(true);
33123318
let children: Vec<Arc<dyn PhysicalExpr>> = vec![Arc::new(Column::new("a", 0))];
3313-
let df = Arc::new(DynamicFilterPhysicalExpr::new(children, Arc::clone(&initial)));
3319+
let df = Arc::new(DynamicFilterPhysicalExpr::new(
3320+
children,
3321+
Arc::clone(&initial),
3322+
));
33143323
let id_before = df.inner().expression_id;
33153324

33163325
let codec = DefaultPhysicalExtensionCodec {};

0 commit comments

Comments
 (0)