Skip to content

Commit 5dc260a

Browse files
adriangbclaude
andcommitted
Stamp expr_id for hook-encoded exprs; add identity and same-name roundtrip tests
- serialize_physical_expr_with_converter now sets the node's expr_id from PhysicalExpr::expression_id when try_to_proto left it unset, so extension expressions built with extension_expr_node keep dedup identity. - Add a dedup regression test and a plan+expr same-wire-name roundtrip test. - Fix stale PhysicalExprRegistry reference in datafusion.proto / prost.rs. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent f456bdc commit 5dc260a

4 files changed

Lines changed: 291 additions & 8 deletions

File tree

‎datafusion/proto-models/proto/datafusion.proto‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1304,7 +1304,7 @@ message PhysicalExtensionExprNode {
13041304
repeated PhysicalExprNode inputs = 2;
13051305
// Optional type discriminator naming the extension expression, e.g.
13061306
// "my_crate.MyExpr". When present and registered in the session's
1307-
// PhysicalExprRegistry, decoding is routed to that registered
1307+
// ProtoDecoderRegistry, decoding is routed to that registered
13081308
// decoder; otherwise it falls back to PhysicalExtensionCodec::try_decode_expr.
13091309
optional string expr_name = 3;
13101310
}

‎datafusion/proto-models/src/generated/prost.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2006,7 +2006,7 @@ pub struct PhysicalExtensionExprNode {
20062006
pub inputs: ::prost::alloc::vec::Vec<PhysicalExprNode>,
20072007
/// Optional type discriminator naming the extension expression, e.g.
20082008
/// "my_crate.MyExpr". When present and registered in the session's
2009-
/// PhysicalExprRegistry, decoding is routed to that registered
2009+
/// ProtoDecoderRegistry, decoding is routed to that registered
20102010
/// decoder; otherwise it falls back to PhysicalExtensionCodec::try_decode_expr.
20112011
#[prost(string, optional, tag = "3")]
20122012
pub expr_name: ::core::option::Option<::prost::alloc::string::String>,

‎datafusion/proto/src/physical_plan/to_proto.rs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,15 @@ pub fn serialize_physical_expr_with_converter(
284284
proto_converter,
285285
};
286286
let ctx = datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx::new(&encoder);
287-
if let Some(node) = expr.try_to_proto(&ctx)? {
287+
if let Some(mut node) = expr.try_to_proto(&ctx)? {
288+
// Stamp the expression's identity here, at the common boundary, so
289+
// a hook that builds its node with a helper such as
290+
// `extension_expr_node` (which cannot see the id) keeps the same
291+
// dedup behaviour as the downcast path below. A hook that set its
292+
// own `expr_id` keeps it.
293+
if node.expr_id.is_none() {
294+
node.expr_id = expr_id;
295+
}
288296
return Ok(node);
289297
}
290298

‎datafusion/proto/tests/cases/plans/expr_registry.rs‎

Lines changed: 280 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,15 @@ use datafusion::physical_plan::filter::FilterExec;
3232
use datafusion::physical_plan::proto::PhysicalExprDecodeCtx;
3333
use datafusion::physical_plan::proto::PhysicalExprEncodeCtx;
3434
use datafusion::physical_plan::proto::{
35-
ExtensionExprFromProto, ProtoDecoderRegistry, extension_expr_node,
36-
physical_expr_names, register_physical_expr,
35+
ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx, ExtensionExprFromProto,
36+
ExtensionPlanFromProto, ProtoDecoderRegistry, extension_expr_node,
37+
physical_expr_names, register_execution_plan, register_physical_expr,
38+
};
39+
use datafusion::physical_plan::{
40+
DisplayAs, DisplayFormatType, PlanProperties, SendableRecordBatchStream,
3741
};
3842
use datafusion::prelude::{SessionConfig, SessionContext};
43+
use datafusion_common::tree_node::TreeNodeRecursion;
3944
use datafusion_common::{
4045
DataFusionError, Result, internal_datafusion_err, internal_err, resources_err,
4146
};
@@ -50,10 +55,13 @@ use datafusion_proto::bytes::{
5055
physical_plan_to_bytes_with_extension_codec,
5156
};
5257
use datafusion_proto::physical_plan::{
53-
DefaultPhysicalProtoConverter, PhysicalExtensionCodec, PhysicalPlanDecodeContext,
54-
PhysicalProtoConverterExtension,
58+
DeduplicatingProtoConverter, DefaultPhysicalProtoConverter, PhysicalExtensionCodec,
59+
PhysicalPlanDecodeContext, PhysicalProtoConverterExtension,
60+
};
61+
use datafusion_proto::protobuf::physical_plan_node::PhysicalPlanType;
62+
use datafusion_proto::protobuf::{
63+
PhysicalExprNode, PhysicalPlanNode, physical_expr_node,
5564
};
56-
use datafusion_proto::protobuf::{PhysicalExprNode, physical_expr_node};
5765
use prost::Message;
5866

5967
/// The wire name of a [`TaggedExpr`], and whether its decoder fails on
@@ -705,3 +713,270 @@ fn registry_expr_survives_a_whole_plan_roundtrip() -> Result<()> {
705713
assert_eq!(tag.decoded_batch_size, Some(4096));
706714
Ok(())
707715
}
716+
717+
/// An extension expression with an identity: two references with the same
718+
/// `expression_id` share `state`, as two pushed-down copies of one dynamic
719+
/// filter would. `with_new_children` keeps `state`, so a deduplicating decoder
720+
/// that re-shares the first decode keeps the two copies linked.
721+
#[derive(Debug, Clone)]
722+
struct IdentityExpr {
723+
id: u64,
724+
state: Arc<String>,
725+
child: Arc<dyn PhysicalExpr>,
726+
}
727+
728+
impl Display for IdentityExpr {
729+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
730+
write!(f, "IdentityExpr({}, {})", self.id, self.child)
731+
}
732+
}
733+
734+
impl PartialEq for IdentityExpr {
735+
fn eq(&self, other: &Self) -> bool {
736+
self.id == other.id && self.child.eq(&other.child)
737+
}
738+
}
739+
740+
impl Eq for IdentityExpr {}
741+
742+
impl std::hash::Hash for IdentityExpr {
743+
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
744+
self.id.hash(state);
745+
self.child.hash(state);
746+
}
747+
}
748+
749+
impl PhysicalExpr for IdentityExpr {
750+
fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
751+
self.child.data_type(input_schema)
752+
}
753+
754+
fn nullable(&self, input_schema: &Schema) -> Result<bool> {
755+
self.child.nullable(input_schema)
756+
}
757+
758+
fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
759+
self.child.evaluate(batch)
760+
}
761+
762+
fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
763+
vec![&self.child]
764+
}
765+
766+
fn with_new_children(
767+
self: Arc<Self>,
768+
children: Vec<Arc<dyn PhysicalExpr>>,
769+
) -> Result<Arc<dyn PhysicalExpr>> {
770+
Ok(Arc::new(Self {
771+
child: Arc::clone(&children[0]),
772+
..(*self).clone()
773+
}))
774+
}
775+
776+
fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
777+
Display::fmt(self, f)
778+
}
779+
780+
fn expression_id(&self) -> Option<u64> {
781+
Some(self.id)
782+
}
783+
784+
fn try_to_proto(
785+
&self,
786+
ctx: &PhysicalExprEncodeCtx<'_>,
787+
) -> Result<Option<PhysicalExprNode>> {
788+
// The helper cannot see `expression_id`; the serializer stamps it.
789+
Ok(Some(extension_expr_node::<Self>(
790+
ctx,
791+
name_payload(&self.state),
792+
self.children(),
793+
)?))
794+
}
795+
}
796+
797+
impl ExtensionExprFromProto for IdentityExpr {
798+
const NAME: &'static str = "datafusion-test.IdentityExpr";
799+
800+
fn try_from_proto(
801+
node: &PhysicalExprNode,
802+
ctx: &PhysicalExprDecodeCtx<'_>,
803+
) -> Result<Arc<dyn PhysicalExpr>> {
804+
let extension = expect_expr_variant!(
805+
node,
806+
physical_expr_node::ExprType::Extension,
807+
"Extension"
808+
);
809+
let Some(id) = node.expr_id else {
810+
return internal_err!("IdentityExpr needs an expr_id");
811+
};
812+
let mut children = ctx.decode_children_expressions(&extension.inputs)?;
813+
Ok(Arc::new(Self {
814+
id,
815+
state: Arc::new(decode_name_payload(&extension.expr)?),
816+
child: children.remove(0),
817+
}))
818+
}
819+
}
820+
821+
/// An extension expression built with `extension_expr_node` keeps its
822+
/// `expression_id` on the wire. Two references to it, with different
823+
/// children, decode through the deduplicating converter to two expressions
824+
/// that share state, each with its own child.
825+
#[test]
826+
fn registry_expr_keeps_its_expression_id_for_deduplication() -> Result<()> {
827+
let schema = Arc::new(Schema::new(vec![
828+
Field::new("a", DataType::Int64, false),
829+
Field::new("b", DataType::Int64, false),
830+
]));
831+
let state = Arc::new("shared".to_string());
832+
let left: Arc<dyn PhysicalExpr> = Arc::new(IdentityExpr {
833+
id: 42,
834+
state: Arc::clone(&state),
835+
child: col("a", &schema)?,
836+
});
837+
let right: Arc<dyn PhysicalExpr> = Arc::new(IdentityExpr {
838+
id: 42,
839+
state,
840+
child: col("b", &schema)?,
841+
});
842+
let pair: Arc<dyn PhysicalExpr> =
843+
Arc::new(BinaryExpr::new(Arc::clone(&left), Operator::And, right));
844+
845+
let codec = TestCodec::Refusing;
846+
let converter = DeduplicatingProtoConverter {};
847+
let node = converter.physical_expr_to_proto(&pair, &codec)?;
848+
assert_eq!(
849+
to_wire(&left, &codec)?.expr_id,
850+
Some(42),
851+
"the serializer must stamp the expression id"
852+
);
853+
854+
let session =
855+
session_with(SessionConfig::new(), register_physical_expr::<IdentityExpr>)?;
856+
let task_ctx = session.task_ctx();
857+
let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
858+
let decoded = converter.proto_to_physical_expr(&node, &schema, &decode_ctx)?;
859+
let binary = decoded.downcast_ref::<BinaryExpr>().unwrap();
860+
let left = binary.left().downcast_ref::<IdentityExpr>().unwrap();
861+
let right = binary.right().downcast_ref::<IdentityExpr>().unwrap();
862+
assert!(
863+
Arc::ptr_eq(&left.state, &right.state),
864+
"state must be shared"
865+
);
866+
assert_eq!(left.child.to_string(), "a@0");
867+
assert_eq!(right.child.to_string(), "b@1");
868+
Ok(())
869+
}
870+
871+
/// An extension plan whose wire name equals [`TagExpr::NAME`]. The registry
872+
/// scopes names per kind, so the plan and the expression do not collide.
873+
#[derive(Debug)]
874+
struct SameNameExec {
875+
child: Arc<dyn ExecutionPlan>,
876+
}
877+
878+
impl DisplayAs for SameNameExec {
879+
fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
880+
write!(f, "SameNameExec")
881+
}
882+
}
883+
884+
impl ExecutionPlan for SameNameExec {
885+
fn name(&self) -> &str {
886+
"SameNameExec"
887+
}
888+
889+
fn properties(&self) -> &Arc<PlanProperties> {
890+
self.child.properties()
891+
}
892+
893+
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
894+
vec![&self.child]
895+
}
896+
897+
fn apply_expressions(
898+
&self,
899+
_f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
900+
) -> Result<TreeNodeRecursion> {
901+
Ok(TreeNodeRecursion::Continue)
902+
}
903+
904+
fn with_new_children(
905+
self: Arc<Self>,
906+
mut children: Vec<Arc<dyn ExecutionPlan>>,
907+
) -> Result<Arc<dyn ExecutionPlan>> {
908+
Ok(Arc::new(Self {
909+
child: children.remove(0),
910+
}))
911+
}
912+
913+
fn execute(
914+
&self,
915+
_partition: usize,
916+
_context: Arc<TaskContext>,
917+
) -> Result<SendableRecordBatchStream> {
918+
internal_err!("SameNameExec is a serde-only test plan")
919+
}
920+
921+
fn try_to_proto(
922+
&self,
923+
ctx: &ExecutionPlanEncodeCtx<'_>,
924+
) -> Result<Option<PhysicalPlanNode>> {
925+
Ok(Some(ctx.extension_node::<Self>(vec![], self.children())?))
926+
}
927+
}
928+
929+
impl ExtensionPlanFromProto for SameNameExec {
930+
const NAME: &'static str = Tag::NAME;
931+
932+
fn try_from_proto(
933+
node: &PhysicalPlanNode,
934+
ctx: &ExecutionPlanDecodeCtx<'_>,
935+
) -> Result<Arc<dyn ExecutionPlan>> {
936+
let Some(PhysicalPlanType::Extension(extension)) = &node.physical_plan_type
937+
else {
938+
return internal_err!("expected an extension node");
939+
};
940+
let mut children = ctx.decode_children(&extension.inputs)?;
941+
Ok(Arc::new(Self {
942+
child: children.remove(0),
943+
}))
944+
}
945+
}
946+
947+
/// A plan that holds a registered extension plan and a registered extension
948+
/// expression with the same wire name. Both decode through the registry, each
949+
/// to its own type.
950+
#[test]
951+
fn registry_plan_and_expr_with_the_same_name_roundtrip_together() -> Result<()> {
952+
let schema = test_schema();
953+
let extension_plan: Arc<dyn ExecutionPlan> = Arc::new(SameNameExec {
954+
child: Arc::new(EmptyExec::new(Arc::clone(&schema))),
955+
});
956+
let predicate: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
957+
Arc::new(TagExpr::new("same_name", col("a", &schema)?)),
958+
Operator::Eq,
959+
lit(1i64),
960+
));
961+
let plan: Arc<dyn ExecutionPlan> =
962+
Arc::new(FilterExec::try_new(predicate, extension_plan)?);
963+
let bytes = physical_plan_to_bytes_with_extension_codec(plan, &TestCodec::Refusing)?;
964+
965+
let session = session_with(SessionConfig::new().with_batch_size(2048), |registry| {
966+
register_execution_plan::<SameNameExec>(registry)?;
967+
register_physical_expr::<TagExpr>(registry)
968+
})?;
969+
let decoded = physical_plan_from_bytes_with_extension_codec(
970+
&bytes,
971+
session.task_ctx().as_ref(),
972+
&TestCodec::Refusing,
973+
)?;
974+
let filter = decoded.downcast_ref::<FilterExec>().unwrap();
975+
let binary = filter.predicate().downcast_ref::<BinaryExpr>().unwrap();
976+
let tag = binary.left().downcast_ref::<TagExpr>().unwrap();
977+
assert_eq!(tag.tag, "same_name");
978+
assert_eq!(tag.decoded_batch_size, Some(2048));
979+
let extension = filter.input().downcast_ref::<SameNameExec>().unwrap();
980+
assert_eq!(extension.child.name(), "EmptyExec");
981+
Ok(())
982+
}

0 commit comments

Comments
 (0)