@@ -520,7 +520,29 @@ pub trait PhysicalExpr: Any + Send + Sync + Display + Debug + DynEq + DynHash {
520520 /// constructors; both sides of the round-trip are fallible and named
521521 /// consistently.
522522 ///
523+ /// # Why decoding is not a method here
524+ ///
525+ /// The two halves cannot be symmetric on this trait, and the reason is
526+ /// structural rather than a matter of taste:
527+ ///
528+ /// * Encoding has a receiver, and `datafusion-proto` calls it on a
529+ /// `&dyn PhysicalExpr`, so it must be dispatched through the vtable —
530+ /// which means it must live here.
531+ /// * Decoding is a *constructor*. With no `self` there is nothing to
532+ /// dispatch on, so it could only sit here as a `where Self: Sized`
533+ /// method, which is excluded from the vtable and could never be called
534+ /// through `dyn PhysicalExpr` anyway. And the wire name it needs cannot
535+ /// live here at all: an associated constant would make `PhysicalExpr`
536+ /// not dyn-compatible, breaking every `Arc<dyn PhysicalExpr>`.
537+ ///
538+ /// So decoders live off-trait. Built-in expressions write an inherent
539+ /// `try_from_proto` and are dispatched to it by `datafusion-proto`'s
540+ /// central match. A third-party expression implements
541+ /// [`ExtensionPhysicalExpr`], which pairs the constructor with the name it
542+ /// is registered and encoded under.
543+ ///
523544 /// [`PhysicalExprNode`]: datafusion_proto_models::protobuf::PhysicalExprNode
545+ /// [`ExtensionPhysicalExpr`]: proto_decode::ExtensionPhysicalExpr
524546 #[ cfg( feature = "proto" ) ]
525547 fn try_to_proto (
526548 & self ,
@@ -547,9 +569,12 @@ pub mod proto_encode {
547569 use std:: sync:: Arc ;
548570
549571 use datafusion_common:: Result ;
550- use datafusion_proto_models:: protobuf:: PhysicalExprNode ;
572+ use datafusion_proto_models:: protobuf:: {
573+ PhysicalExprNode , PhysicalExtensionExprNode , physical_expr_node,
574+ } ;
551575
552576 use super :: PhysicalExpr ;
577+ use super :: proto_decode:: ExtensionPhysicalExpr ;
553578
554579 /// Encoder context handed to [`super::PhysicalExpr::try_to_proto`].
555580 ///
@@ -593,6 +618,57 @@ pub mod proto_encode {
593618 . map ( |expr| self . encode_child ( expr) )
594619 . collect ( )
595620 }
621+
622+ /// Build the wrapper node for a third-party expression that decodes
623+ /// through a [`PhysicalExprDecoderRegistry`] rather than through a
624+ /// `PhysicalExtensionCodec`.
625+ ///
626+ /// `payload` is the expression's own serialized state; everything else
627+ /// is taken from `expr` so it cannot disagree with the decode side:
628+ ///
629+ /// * the wire name is [`ExtensionPhysicalExpr::EXPR_NAME`], the key the
630+ /// registry is looked up by;
631+ /// * the children are [`PhysicalExpr::children`], encoded through this
632+ /// context so recursion stays dedup-aware;
633+ /// * `expr_id` is [`PhysicalExpr::expression_id`], so an expression
634+ /// that participates in deduplication keeps doing so.
635+ ///
636+ /// ```ignore
637+ /// fn try_to_proto(
638+ /// &self,
639+ /// ctx: &PhysicalExprEncodeCtx<'_>,
640+ /// ) -> Result<Option<PhysicalExprNode>> {
641+ /// let mut payload = Vec::new();
642+ /// MyExprProto { /* ... */ }.encode(&mut payload)?;
643+ /// Ok(Some(ctx.encode_extension(self, payload)?))
644+ /// }
645+ /// ```
646+ ///
647+ /// An expression that serializes its children into `payload` itself
648+ /// should not use this helper, since its children would then be
649+ /// written twice. Build the
650+ /// [`PhysicalExtensionExprNode`] directly in that case — its fields
651+ /// are public, and [`Self::encode_children_expressions`] is still
652+ /// available for whichever children do belong in `inputs`.
653+ ///
654+ /// [`PhysicalExprDecoderRegistry`]: super::proto_decode::PhysicalExprDecoderRegistry
655+ /// [`ExtensionPhysicalExpr::EXPR_NAME`]: super::proto_decode::ExtensionPhysicalExpr::EXPR_NAME
656+ pub fn encode_extension < T : ExtensionPhysicalExpr > (
657+ & self ,
658+ expr : & T ,
659+ payload : Vec < u8 > ,
660+ ) -> Result < PhysicalExprNode > {
661+ Ok ( PhysicalExprNode {
662+ expr_id : expr. expression_id ( ) ,
663+ expr_type : Some ( physical_expr_node:: ExprType :: Extension (
664+ PhysicalExtensionExprNode {
665+ expr : payload,
666+ inputs : self . encode_children_expressions ( expr. children ( ) ) ?,
667+ expr_name : Some ( T :: EXPR_NAME . to_string ( ) ) ,
668+ } ,
669+ ) ) ,
670+ } )
671+ }
596672 }
597673
598674 /// Internal dispatch trait. Implementors live in `datafusion-proto` and
@@ -637,10 +713,12 @@ pub mod proto_encode {
637713#[ cfg( feature = "proto" ) ]
638714pub mod proto_decode {
639715 use std:: any:: { Any , type_name} ;
716+ use std:: collections:: HashMap ;
717+ use std:: collections:: hash_map:: Entry ;
640718 use std:: sync:: Arc ;
641719
642720 use arrow:: datatypes:: Schema ;
643- use datafusion_common:: { Result , internal_datafusion_err} ;
721+ use datafusion_common:: { Result , config_err , internal_datafusion_err} ;
644722 use datafusion_proto_models:: protobuf:: PhysicalExprNode ;
645723
646724 use super :: PhysicalExpr ;
@@ -746,9 +824,10 @@ pub mod proto_decode {
746824 /// Decode an expression node, recursing into child sub-expressions.
747825 ///
748826 /// Routes built-in `ExprType` variants through `datafusion-proto`'s
749- /// central match and forwards extension nodes to the registered codec
750- /// (today via [`PhysicalExtensionCodec::try_decode_expr`]; later via
751- /// a per-type registry — see #21835).
827+ /// central match. An extension node that names itself on the wire and
828+ /// is registered in the session's [`PhysicalExprDecoderRegistry`] goes
829+ /// to that decoder; anything else falls back to
830+ /// [`PhysicalExtensionCodec::try_decode_expr`].
752831 ///
753832 /// [`PhysicalExtensionCodec::try_decode_expr`]: https://docs.rs/datafusion-proto/latest/datafusion_proto/physical_plan/trait.PhysicalExtensionCodec.html#method.try_decode_expr
754833 pub fn decode ( & self , node : & PhysicalExprNode ) -> Result < Arc < dyn PhysicalExpr > > {
@@ -820,6 +899,185 @@ pub mod proto_decode {
820899 } )
821900 }
822901
902+ /// A third-party [`PhysicalExpr`] that can be decoded by name, without a
903+ /// `PhysicalExtensionCodec`.
904+ ///
905+ /// Extension expressions reach the wire as a `PhysicalExtensionExprNode`:
906+ /// an opaque payload plus the encoded children. Historically that node
907+ /// carried no type discriminator, so the codec itself had to serve as one
908+ /// — every composed codec tried each registered codec in turn and read a
909+ /// decode error as "not mine".
910+ ///
911+ /// Implementing this trait names the expression on the wire instead.
912+ /// Register the type in a [`PhysicalExprDecoderRegistry`] on the session
913+ /// and the name routes straight to [`Self::try_from_proto`].
914+ ///
915+ /// The encode half is [`PhysicalExpr::try_to_proto`], via
916+ /// [`PhysicalExprEncodeCtx::encode_extension`]:
917+ ///
918+ /// ```ignore
919+ /// impl ExtensionPhysicalExpr for MyExpr {
920+ /// const EXPR_NAME: &'static str = "my_crate.MyExpr";
921+ ///
922+ /// fn try_from_proto(
923+ /// node: &PhysicalExprNode,
924+ /// ctx: &PhysicalExprDecodeCtx<'_>,
925+ /// ) -> Result<Arc<dyn PhysicalExpr>> {
926+ /// let extension = expect_expr_variant!(
927+ /// node,
928+ /// physical_expr_node::ExprType::Extension,
929+ /// "Extension",
930+ /// );
931+ /// let payload = MyExprProto::decode(extension.expr.as_slice())?;
932+ /// let children = ctx.decode_children_expressions(&extension.inputs)?;
933+ /// Ok(Arc::new(MyExpr::new(payload, children)))
934+ /// }
935+ /// }
936+ /// ```
937+ ///
938+ /// [`PhysicalExpr::try_to_proto`]: super::PhysicalExpr::try_to_proto
939+ /// [`PhysicalExprEncodeCtx::encode_extension`]: super::proto_encode::PhysicalExprEncodeCtx::encode_extension
940+ pub trait ExtensionPhysicalExpr : PhysicalExpr + Sized {
941+ /// The name written to the wire and used as the registry key.
942+ ///
943+ /// Namespace it — `"my_crate.MyExpr"`, not `"MyExpr"` — so that two
944+ /// independent crates registering into the same session collide at
945+ /// registration time instead of silently decoding each other's nodes.
946+ const EXPR_NAME : & ' static str ;
947+
948+ /// Reconstruct the expression from its proto node.
949+ ///
950+ /// Takes the whole [`PhysicalExprNode`] — the exact inverse of what
951+ /// [`PhysicalExpr::try_to_proto`] returns — so the constructor can also
952+ /// see outer-node fields such as `expr_id`. This is the same signature
953+ /// the built-in expressions' inherent `try_from_proto` uses.
954+ ///
955+ /// [`PhysicalExpr::try_to_proto`]: super::PhysicalExpr::try_to_proto
956+ fn try_from_proto (
957+ node : & PhysicalExprNode ,
958+ ctx : & PhysicalExprDecodeCtx < ' _ > ,
959+ ) -> Result < Arc < dyn PhysicalExpr > > ;
960+ }
961+
962+ /// The decoder half of an [`ExtensionPhysicalExpr`], as stored in a
963+ /// [`PhysicalExprDecoderRegistry`].
964+ ///
965+ /// A plain `fn` pointer: [`ExtensionPhysicalExpr::try_from_proto`] is an
966+ /// associated function returning `Arc<dyn PhysicalExpr>`, so nothing about
967+ /// it needs a trait object.
968+ pub type PhysicalExprDecoderFn = fn (
969+ & PhysicalExprNode ,
970+ & PhysicalExprDecodeCtx < ' _ > ,
971+ ) -> Result < Arc < dyn PhysicalExpr > > ;
972+
973+ /// Maps extension-expression names to their decoders.
974+ ///
975+ /// Attach one to a session and extension expressions that name themselves
976+ /// on the wire decode through it instead of through a
977+ /// `PhysicalExtensionCodec`. Register through `datafusion-proto`'s
978+ /// `SessionConfig::with_physical_expr`, which merges into whatever registry
979+ /// the config already carries:
980+ ///
981+ /// ```ignore
982+ /// let config = SessionConfig::new().with_physical_expr::<MyExpr>()?;
983+ /// ```
984+ ///
985+ /// Building a registry by hand and attaching it with
986+ /// `SessionConfig::with_extension` also works, but it replaces any registry
987+ /// already on the config — including one another library attached — so
988+ /// prefer the additive form.
989+ ///
990+ /// Lookup is by name, so resolution does not depend on registration order
991+ /// and a collision between two crates surfaces as an error at registration
992+ /// rather than as a silent wrong decode.
993+ ///
994+ /// A name absent from the registry falls back to
995+ /// `PhysicalExtensionCodec::try_decode_expr`. For nodes written before this
996+ /// mechanism existed that fallback is exactly the old behavior. For an
997+ /// expression that has migrated its encoding to
998+ /// [`PhysicalExprEncodeCtx::encode_extension`] it is not: the payload on
999+ /// the wire is then the expression's own message, and handing it to a codec
1000+ /// that still recognizes the expression can decode it as the old message
1001+ /// rather than failing. Register migrated expressions everywhere their
1002+ /// plans are read.
1003+ ///
1004+ /// [`PhysicalExprEncodeCtx::encode_extension`]: super::proto_encode::PhysicalExprEncodeCtx::encode_extension
1005+ #[ derive( Debug , Clone , Default ) ]
1006+ pub struct PhysicalExprDecoderRegistry {
1007+ decoders : HashMap < String , PhysicalExprDecoderFn > ,
1008+ }
1009+
1010+ impl PhysicalExprDecoderRegistry {
1011+ /// An empty registry.
1012+ pub fn new ( ) -> Self {
1013+ Self :: default ( )
1014+ }
1015+
1016+ /// Register `T`, returning the registry for chaining.
1017+ pub fn with_expr < T : ExtensionPhysicalExpr > ( mut self ) -> Result < Self > {
1018+ self . register :: < T > ( ) ?;
1019+ Ok ( self )
1020+ }
1021+
1022+ /// Register `T` under its [`ExtensionPhysicalExpr::EXPR_NAME`].
1023+ ///
1024+ /// Errors if that name is already registered.
1025+ pub fn register < T : ExtensionPhysicalExpr > ( & mut self ) -> Result < ( ) > {
1026+ self . register_named ( T :: EXPR_NAME , T :: try_from_proto)
1027+ }
1028+
1029+ /// Register a decoder under an explicit name.
1030+ ///
1031+ /// Prefer [`Self::register`], which takes the name from the type and
1032+ /// so cannot disagree with what the encode side writes. This form is
1033+ /// for decoding names produced elsewhere — a renamed type, or a wire
1034+ /// name owned by another crate.
1035+ ///
1036+ /// Errors if the name is empty or already registered.
1037+ pub fn register_named (
1038+ & mut self ,
1039+ name : impl Into < String > ,
1040+ decoder : PhysicalExprDecoderFn ,
1041+ ) -> Result < ( ) > {
1042+ let name = name. into ( ) ;
1043+ if name. is_empty ( ) {
1044+ return config_err ! (
1045+ "extension PhysicalExpr decoders cannot be registered under an empty name"
1046+ ) ;
1047+ }
1048+ match self . decoders . entry ( name) {
1049+ Entry :: Occupied ( entry) => config_err ! (
1050+ "an extension PhysicalExpr decoder is already registered as '{}'" ,
1051+ entry. key( )
1052+ ) ,
1053+ Entry :: Vacant ( entry) => {
1054+ entry. insert ( decoder) ;
1055+ Ok ( ( ) )
1056+ }
1057+ }
1058+ }
1059+
1060+ /// The decoder registered under `name`, if any.
1061+ pub fn get ( & self , name : & str ) -> Option < PhysicalExprDecoderFn > {
1062+ self . decoders . get ( name) . copied ( )
1063+ }
1064+
1065+ /// The registered names, in arbitrary order.
1066+ pub fn names ( & self ) -> impl Iterator < Item = & str > {
1067+ self . decoders . keys ( ) . map ( String :: as_str)
1068+ }
1069+
1070+ /// The number of registered decoders.
1071+ pub fn len ( & self ) -> usize {
1072+ self . decoders . len ( )
1073+ }
1074+
1075+ /// Whether no decoder is registered.
1076+ pub fn is_empty ( & self ) -> bool {
1077+ self . decoders . is_empty ( )
1078+ }
1079+ }
1080+
8231081 /// Internal dispatch trait. Implementors live in `datafusion-proto`.
8241082 /// Expression authors should use [`PhysicalExprDecodeCtx`] instead of
8251083 /// calling this directly.
0 commit comments