Skip to content

Commit fcab4f2

Browse files
committed
[X-2935] PhysicalExtensionCodec: thread proto_converter through methods
Modifies the PhysicalExtensionCodec trait so that try_encode, try_decode, try_encode_expr, and try_decode_expr receive the active PhysicalProtoConverterExtension. Codecs that embed nested PhysicalExprNode fields inside their custom proto (e.g. atlas's ReverseParquetNode.predicate) must now route those through proto_converter.proto_to_physical_expr / proto_converter.physical_expr_to_proto so the dedup cache extends across the extension boundary. Without this plumbing, an outer DeduplicatingSerializer can't reach into extension nodes -- the dedup cache stops at the extension boundary and Arc<Inner> sharing breaks for any expression carried inside an extension codec, even though the wire format already supports it via the expr_id field. With this change, atlas can finally drop the share_dynamic_filters walker once its codecs override the new methods. This is the breaking-change pattern matching the apache/datafusion upstream design (issue apache#22920) instead of additive _with_converter variants -- branch-53 is internal-only so callers can be updated in lockstep.
1 parent 0e6d988 commit fcab4f2

3 files changed

Lines changed: 69 additions & 8 deletions

File tree

‎datafusion/proto/src/physical_plan/from_proto.rs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -506,7 +506,15 @@ pub fn parse_physical_expr_with_converter(
506506
proto_converter.proto_to_physical_expr(e, ctx, input_schema, codec)
507507
})
508508
.collect::<Result<_>>()?;
509-
codec.try_decode_expr(extension.expr.as_slice(), &inputs)? as _
509+
// Use the converter-aware decode entry point so extension codecs
510+
// that embed nested `PhysicalExprNode` fields can thread the
511+
// active `proto_converter` into their own parsing of those
512+
// nested expressions, sharing the dedup cache.
513+
codec.try_decode_expr(
514+
extension.expr.as_slice(),
515+
&inputs,
516+
proto_converter,
517+
)? as _
510518
}
511519
ExprType::DynamicFilter(df_proto) => {
512520
// Reconstruct the DynamicFilterPhysicalExpr wrapper. When this

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -572,7 +572,11 @@ impl protobuf::PhysicalPlanNode {
572572
}
573573

574574
let mut buf: Vec<u8> = vec![];
575-
match codec.try_encode(Arc::clone(&plan_clone), &mut buf) {
575+
match codec.try_encode(
576+
Arc::clone(&plan_clone),
577+
&mut buf,
578+
proto_converter,
579+
) {
576580
Ok(_) => {
577581
let inputs: Vec<protobuf::PhysicalPlanNode> = plan_clone
578582
.children()
@@ -1787,7 +1791,12 @@ impl protobuf::PhysicalPlanNode {
17871791
.map(|i| proto_converter.proto_to_execution_plan(ctx, codec, i))
17881792
.collect::<Result<_>>()?;
17891793

1790-
let extension_node = codec.try_decode(extension.node.as_slice(), &inputs, ctx)?;
1794+
let extension_node = codec.try_decode(
1795+
extension.node.as_slice(),
1796+
&inputs,
1797+
ctx,
1798+
proto_converter,
1799+
)?;
17911800

17921801
Ok(extension_node)
17931802
}
@@ -3717,14 +3726,33 @@ pub trait AsExecutionPlan: Debug + Send + Sync + Clone {
37173726
}
37183727

37193728
pub trait PhysicalExtensionCodec: Debug + Send + Sync {
3729+
/// Decode an extension execution plan.
3730+
///
3731+
/// `proto_converter` is the active conversion strategy (e.g.
3732+
/// `DeduplicatingDeserializer`). Codecs whose custom proto embeds
3733+
/// nested `PhysicalExprNode` fields (e.g. a predicate on a custom
3734+
/// file source) should route those through
3735+
/// `proto_converter.proto_to_physical_expr` so the dedup cache
3736+
/// extends across the extension boundary.
3737+
///
3738+
/// Mirrors the upstream apache/datafusion pattern (#22920).
37203739
fn try_decode(
37213740
&self,
37223741
buf: &[u8],
37233742
inputs: &[Arc<dyn ExecutionPlan>],
37243743
ctx: &TaskContext,
3744+
proto_converter: &dyn PhysicalProtoConverterExtension,
37253745
) -> Result<Arc<dyn ExecutionPlan>>;
37263746

3727-
fn try_encode(&self, node: Arc<dyn ExecutionPlan>, buf: &mut Vec<u8>) -> Result<()>;
3747+
/// Encode an extension execution plan.
3748+
///
3749+
/// See [`Self::try_decode`] for the role of `proto_converter`.
3750+
fn try_encode(
3751+
&self,
3752+
node: Arc<dyn ExecutionPlan>,
3753+
buf: &mut Vec<u8>,
3754+
proto_converter: &dyn PhysicalProtoConverterExtension,
3755+
) -> Result<()>;
37283756

37293757
fn try_decode_udf(&self, name: &str, _buf: &[u8]) -> Result<Arc<ScalarUDF>> {
37303758
not_impl_err!("PhysicalExtensionCodec is not provided for scalar function {name}")
@@ -3734,18 +3762,26 @@ pub trait PhysicalExtensionCodec: Debug + Send + Sync {
37343762
Ok(())
37353763
}
37363764

3765+
/// Decode an extension expression.
3766+
///
3767+
/// See [`Self::try_decode`] for the role of `proto_converter`.
37373768
fn try_decode_expr(
37383769
&self,
37393770
_buf: &[u8],
37403771
_inputs: &[Arc<dyn PhysicalExpr>],
3772+
_proto_converter: &dyn PhysicalProtoConverterExtension,
37413773
) -> Result<Arc<dyn PhysicalExpr>> {
37423774
not_impl_err!("PhysicalExtensionCodec is not provided")
37433775
}
37443776

3777+
/// Encode an extension expression.
3778+
///
3779+
/// See [`Self::try_decode`] for the role of `proto_converter`.
37453780
fn try_encode_expr(
37463781
&self,
37473782
_node: &Arc<dyn PhysicalExpr>,
37483783
_buf: &mut Vec<u8>,
3784+
_proto_converter: &dyn PhysicalProtoConverterExtension,
37493785
) -> Result<()> {
37503786
not_impl_err!("PhysicalExtensionCodec is not provided")
37513787
}
@@ -3778,6 +3814,7 @@ impl PhysicalExtensionCodec for DefaultPhysicalExtensionCodec {
37783814
_buf: &[u8],
37793815
_inputs: &[Arc<dyn ExecutionPlan>],
37803816
_ctx: &TaskContext,
3817+
_proto_converter: &dyn PhysicalProtoConverterExtension,
37813818
) -> Result<Arc<dyn ExecutionPlan>> {
37823819
not_impl_err!("PhysicalExtensionCodec is not provided")
37833820
}
@@ -3786,6 +3823,7 @@ impl PhysicalExtensionCodec for DefaultPhysicalExtensionCodec {
37863823
&self,
37873824
_node: Arc<dyn ExecutionPlan>,
37883825
_buf: &mut Vec<u8>,
3826+
_proto_converter: &dyn PhysicalProtoConverterExtension,
37893827
) -> Result<()> {
37903828
not_impl_err!("PhysicalExtensionCodec is not provided")
37913829
}
@@ -4213,12 +4251,22 @@ impl PhysicalExtensionCodec for ComposedPhysicalExtensionCodec {
42134251
buf: &[u8],
42144252
inputs: &[Arc<dyn ExecutionPlan>],
42154253
ctx: &TaskContext,
4254+
proto_converter: &dyn PhysicalProtoConverterExtension,
42164255
) -> Result<Arc<dyn ExecutionPlan>> {
4217-
self.decode_protobuf(buf, |codec, data| codec.try_decode(data, inputs, ctx))
4256+
self.decode_protobuf(buf, |codec, data| {
4257+
codec.try_decode(data, inputs, ctx, proto_converter)
4258+
})
42184259
}
42194260

4220-
fn try_encode(&self, node: Arc<dyn ExecutionPlan>, buf: &mut Vec<u8>) -> Result<()> {
4221-
self.encode_protobuf(buf, |codec, data| codec.try_encode(Arc::clone(&node), data))
4261+
fn try_encode(
4262+
&self,
4263+
node: Arc<dyn ExecutionPlan>,
4264+
buf: &mut Vec<u8>,
4265+
proto_converter: &dyn PhysicalProtoConverterExtension,
4266+
) -> Result<()> {
4267+
self.encode_protobuf(buf, |codec, data| {
4268+
codec.try_encode(Arc::clone(&node), data, proto_converter)
4269+
})
42224270
}
42234271

42244272
fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result<Arc<ScalarUDF>> {

‎datafusion/proto/src/physical_plan/to_proto.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -576,7 +576,12 @@ pub fn serialize_physical_expr_with_converter(
576576
})
577577
} else {
578578
let mut buf: Vec<u8> = vec![];
579-
match codec.try_encode_expr(&value, &mut buf) {
579+
// Use the converter-aware encode entry point so extension codecs
580+
// that embed nested `PhysicalExprNode` fields can thread the
581+
// active `proto_converter` (e.g. `DeduplicatingSerializer`) into
582+
// their own serialization of those nested expressions, preserving
583+
// Arc-identity dedup end-to-end.
584+
match codec.try_encode_expr(&value, &mut buf, proto_converter) {
580585
Ok(_) => {
581586
let inputs: Vec<protobuf::PhysicalExprNode> = value
582587
.children()

0 commit comments

Comments
 (0)