@@ -32,10 +32,15 @@ use datafusion::physical_plan::filter::FilterExec;
3232use datafusion:: physical_plan:: proto:: PhysicalExprDecodeCtx ;
3333use datafusion:: physical_plan:: proto:: PhysicalExprEncodeCtx ;
3434use datafusion:: physical_plan:: proto:: {
35- ExtensionExprFromProto , ProtoDecoderRegistry , extension_expr_node,
36- physical_expr_names, register_physical_expr,
35+ ExecutionPlanDecodeCtx , ExecutionPlanEncodeCtx , ExtensionExprFromProto ,
36+ ExtensionPlanFromProto , ProtoDecoderRegistry , extension_expr_node,
37+ physical_expr_names, register_execution_plan, register_physical_expr,
38+ } ;
39+ use datafusion:: physical_plan:: {
40+ DisplayAs , DisplayFormatType , PlanProperties , SendableRecordBatchStream ,
3741} ;
3842use datafusion:: prelude:: { SessionConfig , SessionContext } ;
43+ use datafusion_common:: tree_node:: TreeNodeRecursion ;
3944use datafusion_common:: {
4045 DataFusionError , Result , internal_datafusion_err, internal_err, resources_err,
4146} ;
@@ -50,10 +55,13 @@ use datafusion_proto::bytes::{
5055 physical_plan_to_bytes_with_extension_codec,
5156} ;
5257use datafusion_proto:: physical_plan:: {
53- DefaultPhysicalProtoConverter , PhysicalExtensionCodec , PhysicalPlanDecodeContext ,
54- PhysicalProtoConverterExtension ,
58+ DeduplicatingProtoConverter , DefaultPhysicalProtoConverter , PhysicalExtensionCodec ,
59+ PhysicalPlanDecodeContext , PhysicalProtoConverterExtension ,
60+ } ;
61+ use datafusion_proto:: protobuf:: physical_plan_node:: PhysicalPlanType ;
62+ use datafusion_proto:: protobuf:: {
63+ PhysicalExprNode , PhysicalPlanNode , physical_expr_node,
5564} ;
56- use datafusion_proto:: protobuf:: { PhysicalExprNode , physical_expr_node} ;
5765use prost:: Message ;
5866
5967/// The wire name of a [`TaggedExpr`], and whether its decoder fails on
@@ -705,3 +713,270 @@ fn registry_expr_survives_a_whole_plan_roundtrip() -> Result<()> {
705713 assert_eq ! ( tag. decoded_batch_size, Some ( 4096 ) ) ;
706714 Ok ( ( ) )
707715}
716+
717+ /// An extension expression with an identity: two references with the same
718+ /// `expression_id` share `state`, as two pushed-down copies of one dynamic
719+ /// filter would. `with_new_children` keeps `state`, so a deduplicating decoder
720+ /// that re-shares the first decode keeps the two copies linked.
721+ #[ derive( Debug , Clone ) ]
722+ struct IdentityExpr {
723+ id : u64 ,
724+ state : Arc < String > ,
725+ child : Arc < dyn PhysicalExpr > ,
726+ }
727+
728+ impl Display for IdentityExpr {
729+ fn fmt ( & self , f : & mut Formatter < ' _ > ) -> std:: fmt:: Result {
730+ write ! ( f, "IdentityExpr({}, {})" , self . id, self . child)
731+ }
732+ }
733+
734+ impl PartialEq for IdentityExpr {
735+ fn eq ( & self , other : & Self ) -> bool {
736+ self . id == other. id && self . child . eq ( & other. child )
737+ }
738+ }
739+
740+ impl Eq for IdentityExpr { }
741+
742+ impl std:: hash:: Hash for IdentityExpr {
743+ fn hash < H : std:: hash:: Hasher > ( & self , state : & mut H ) {
744+ self . id . hash ( state) ;
745+ self . child . hash ( state) ;
746+ }
747+ }
748+
749+ impl PhysicalExpr for IdentityExpr {
750+ fn data_type ( & self , input_schema : & Schema ) -> Result < DataType > {
751+ self . child . data_type ( input_schema)
752+ }
753+
754+ fn nullable ( & self , input_schema : & Schema ) -> Result < bool > {
755+ self . child . nullable ( input_schema)
756+ }
757+
758+ fn evaluate ( & self , batch : & RecordBatch ) -> Result < ColumnarValue > {
759+ self . child . evaluate ( batch)
760+ }
761+
762+ fn children ( & self ) -> Vec < & Arc < dyn PhysicalExpr > > {
763+ vec ! [ & self . child]
764+ }
765+
766+ fn with_new_children (
767+ self : Arc < Self > ,
768+ children : Vec < Arc < dyn PhysicalExpr > > ,
769+ ) -> Result < Arc < dyn PhysicalExpr > > {
770+ Ok ( Arc :: new ( Self {
771+ child : Arc :: clone ( & children[ 0 ] ) ,
772+ ..( * self ) . clone ( )
773+ } ) )
774+ }
775+
776+ fn fmt_sql ( & self , f : & mut Formatter < ' _ > ) -> std:: fmt:: Result {
777+ Display :: fmt ( self , f)
778+ }
779+
780+ fn expression_id ( & self ) -> Option < u64 > {
781+ Some ( self . id )
782+ }
783+
784+ fn try_to_proto (
785+ & self ,
786+ ctx : & PhysicalExprEncodeCtx < ' _ > ,
787+ ) -> Result < Option < PhysicalExprNode > > {
788+ // The helper cannot see `expression_id`; the serializer stamps it.
789+ Ok ( Some ( extension_expr_node :: < Self > (
790+ ctx,
791+ name_payload ( & self . state ) ,
792+ self . children ( ) ,
793+ ) ?) )
794+ }
795+ }
796+
797+ impl ExtensionExprFromProto for IdentityExpr {
798+ const NAME : & ' static str = "datafusion-test.IdentityExpr" ;
799+
800+ fn try_from_proto (
801+ node : & PhysicalExprNode ,
802+ ctx : & PhysicalExprDecodeCtx < ' _ > ,
803+ ) -> Result < Arc < dyn PhysicalExpr > > {
804+ let extension = expect_expr_variant ! (
805+ node,
806+ physical_expr_node:: ExprType :: Extension ,
807+ "Extension"
808+ ) ;
809+ let Some ( id) = node. expr_id else {
810+ return internal_err ! ( "IdentityExpr needs an expr_id" ) ;
811+ } ;
812+ let mut children = ctx. decode_children_expressions ( & extension. inputs ) ?;
813+ Ok ( Arc :: new ( Self {
814+ id,
815+ state : Arc :: new ( decode_name_payload ( & extension. expr ) ?) ,
816+ child : children. remove ( 0 ) ,
817+ } ) )
818+ }
819+ }
820+
821+ /// An extension expression built with `extension_expr_node` keeps its
822+ /// `expression_id` on the wire. Two references to it, with different
823+ /// children, decode through the deduplicating converter to two expressions
824+ /// that share state, each with its own child.
825+ #[ test]
826+ fn registry_expr_keeps_its_expression_id_for_deduplication ( ) -> Result < ( ) > {
827+ let schema = Arc :: new ( Schema :: new ( vec ! [
828+ Field :: new( "a" , DataType :: Int64 , false ) ,
829+ Field :: new( "b" , DataType :: Int64 , false ) ,
830+ ] ) ) ;
831+ let state = Arc :: new ( "shared" . to_string ( ) ) ;
832+ let left: Arc < dyn PhysicalExpr > = Arc :: new ( IdentityExpr {
833+ id : 42 ,
834+ state : Arc :: clone ( & state) ,
835+ child : col ( "a" , & schema) ?,
836+ } ) ;
837+ let right: Arc < dyn PhysicalExpr > = Arc :: new ( IdentityExpr {
838+ id : 42 ,
839+ state,
840+ child : col ( "b" , & schema) ?,
841+ } ) ;
842+ let pair: Arc < dyn PhysicalExpr > =
843+ Arc :: new ( BinaryExpr :: new ( Arc :: clone ( & left) , Operator :: And , right) ) ;
844+
845+ let codec = TestCodec :: Refusing ;
846+ let converter = DeduplicatingProtoConverter { } ;
847+ let node = converter. physical_expr_to_proto ( & pair, & codec) ?;
848+ assert_eq ! (
849+ to_wire( & left, & codec) ?. expr_id,
850+ Some ( 42 ) ,
851+ "the serializer must stamp the expression id"
852+ ) ;
853+
854+ let session =
855+ session_with ( SessionConfig :: new ( ) , register_physical_expr :: < IdentityExpr > ) ?;
856+ let task_ctx = session. task_ctx ( ) ;
857+ let decode_ctx = PhysicalPlanDecodeContext :: new ( task_ctx. as_ref ( ) , & codec) ;
858+ let decoded = converter. proto_to_physical_expr ( & node, & schema, & decode_ctx) ?;
859+ let binary = decoded. downcast_ref :: < BinaryExpr > ( ) . unwrap ( ) ;
860+ let left = binary. left ( ) . downcast_ref :: < IdentityExpr > ( ) . unwrap ( ) ;
861+ let right = binary. right ( ) . downcast_ref :: < IdentityExpr > ( ) . unwrap ( ) ;
862+ assert ! (
863+ Arc :: ptr_eq( & left. state, & right. state) ,
864+ "state must be shared"
865+ ) ;
866+ assert_eq ! ( left. child. to_string( ) , "a@0" ) ;
867+ assert_eq ! ( right. child. to_string( ) , "b@1" ) ;
868+ Ok ( ( ) )
869+ }
870+
871+ /// An extension plan whose wire name equals [`TagExpr::NAME`]. The registry
872+ /// scopes names per kind, so the plan and the expression do not collide.
873+ #[ derive( Debug ) ]
874+ struct SameNameExec {
875+ child : Arc < dyn ExecutionPlan > ,
876+ }
877+
878+ impl DisplayAs for SameNameExec {
879+ fn fmt_as ( & self , _t : DisplayFormatType , f : & mut Formatter ) -> std:: fmt:: Result {
880+ write ! ( f, "SameNameExec" )
881+ }
882+ }
883+
884+ impl ExecutionPlan for SameNameExec {
885+ fn name ( & self ) -> & str {
886+ "SameNameExec"
887+ }
888+
889+ fn properties ( & self ) -> & Arc < PlanProperties > {
890+ self . child . properties ( )
891+ }
892+
893+ fn children ( & self ) -> Vec < & Arc < dyn ExecutionPlan > > {
894+ vec ! [ & self . child]
895+ }
896+
897+ fn apply_expressions (
898+ & self ,
899+ _f : & mut dyn FnMut ( & Arc < dyn PhysicalExpr > ) -> Result < TreeNodeRecursion > ,
900+ ) -> Result < TreeNodeRecursion > {
901+ Ok ( TreeNodeRecursion :: Continue )
902+ }
903+
904+ fn with_new_children (
905+ self : Arc < Self > ,
906+ mut children : Vec < Arc < dyn ExecutionPlan > > ,
907+ ) -> Result < Arc < dyn ExecutionPlan > > {
908+ Ok ( Arc :: new ( Self {
909+ child : children. remove ( 0 ) ,
910+ } ) )
911+ }
912+
913+ fn execute (
914+ & self ,
915+ _partition : usize ,
916+ _context : Arc < TaskContext > ,
917+ ) -> Result < SendableRecordBatchStream > {
918+ internal_err ! ( "SameNameExec is a serde-only test plan" )
919+ }
920+
921+ fn try_to_proto (
922+ & self ,
923+ ctx : & ExecutionPlanEncodeCtx < ' _ > ,
924+ ) -> Result < Option < PhysicalPlanNode > > {
925+ Ok ( Some ( ctx. extension_node :: < Self > ( vec ! [ ] , self . children ( ) ) ?) )
926+ }
927+ }
928+
929+ impl ExtensionPlanFromProto for SameNameExec {
930+ const NAME : & ' static str = Tag :: NAME ;
931+
932+ fn try_from_proto (
933+ node : & PhysicalPlanNode ,
934+ ctx : & ExecutionPlanDecodeCtx < ' _ > ,
935+ ) -> Result < Arc < dyn ExecutionPlan > > {
936+ let Some ( PhysicalPlanType :: Extension ( extension) ) = & node. physical_plan_type
937+ else {
938+ return internal_err ! ( "expected an extension node" ) ;
939+ } ;
940+ let mut children = ctx. decode_children ( & extension. inputs ) ?;
941+ Ok ( Arc :: new ( Self {
942+ child : children. remove ( 0 ) ,
943+ } ) )
944+ }
945+ }
946+
947+ /// A plan that holds a registered extension plan and a registered extension
948+ /// expression with the same wire name. Both decode through the registry, each
949+ /// to its own type.
950+ #[ test]
951+ fn registry_plan_and_expr_with_the_same_name_roundtrip_together ( ) -> Result < ( ) > {
952+ let schema = test_schema ( ) ;
953+ let extension_plan: Arc < dyn ExecutionPlan > = Arc :: new ( SameNameExec {
954+ child : Arc :: new ( EmptyExec :: new ( Arc :: clone ( & schema) ) ) ,
955+ } ) ;
956+ let predicate: Arc < dyn PhysicalExpr > = Arc :: new ( BinaryExpr :: new (
957+ Arc :: new ( TagExpr :: new ( "same_name" , col ( "a" , & schema) ?) ) ,
958+ Operator :: Eq ,
959+ lit ( 1i64 ) ,
960+ ) ) ;
961+ let plan: Arc < dyn ExecutionPlan > =
962+ Arc :: new ( FilterExec :: try_new ( predicate, extension_plan) ?) ;
963+ let bytes = physical_plan_to_bytes_with_extension_codec ( plan, & TestCodec :: Refusing ) ?;
964+
965+ let session = session_with ( SessionConfig :: new ( ) . with_batch_size ( 2048 ) , |registry| {
966+ register_execution_plan :: < SameNameExec > ( registry) ?;
967+ register_physical_expr :: < TagExpr > ( registry)
968+ } ) ?;
969+ let decoded = physical_plan_from_bytes_with_extension_codec (
970+ & bytes,
971+ session. task_ctx ( ) . as_ref ( ) ,
972+ & TestCodec :: Refusing ,
973+ ) ?;
974+ let filter = decoded. downcast_ref :: < FilterExec > ( ) . unwrap ( ) ;
975+ let binary = filter. predicate ( ) . downcast_ref :: < BinaryExpr > ( ) . unwrap ( ) ;
976+ let tag = binary. left ( ) . downcast_ref :: < TagExpr > ( ) . unwrap ( ) ;
977+ assert_eq ! ( tag. tag, "same_name" ) ;
978+ assert_eq ! ( tag. decoded_batch_size, Some ( 2048 ) ) ;
979+ let extension = filter. input ( ) . downcast_ref :: < SameNameExec > ( ) . unwrap ( ) ;
980+ assert_eq ! ( extension. child. name( ) , "EmptyExec" ) ;
981+ Ok ( ( ) )
982+ }
0 commit comments