5656//! `physical-expr-common`, *below* `datafusion-expr`) cannot do this, which is
5757//! why `ScalarFunctionExpr` remains special-cased there.
5858//!
59+ //! # Extension plans
60+ //!
61+ //! Third-party plans have no `PhysicalPlanType` variant of their own; they all
62+ //! share `PhysicalExtensionNode`, whose payload is opaque bytes. Such a plan
63+ //! rides the same hooks by pairing
64+ //! [`ExecutionPlanEncodeCtx::encode_extension`] with
65+ //! [`ExecutionPlanDecodeCtx::decode_extension`], and declaring a name through
66+ //! [`ExtensionExecutionPlan`] so the decoding session can select its decoder by
67+ //! name rather than by trying every registered `PhysicalExtensionCodec` in
68+ //! turn. See [`registry`].
69+ //!
5970//! [`ExecutionPlan`]: crate::ExecutionPlan
6071
72+ pub mod registry;
73+
6174use std:: sync:: Arc ;
6275
6376use arrow:: datatypes:: Schema ;
64- use datafusion_common:: { Result , internal_datafusion_err} ;
77+ use datafusion_common:: { Result , internal_datafusion_err, internal_err } ;
6578use datafusion_execution:: TaskContext ;
6679use datafusion_expr:: physical_planning_context:: ScalarSubqueryResults ;
6780use datafusion_expr:: { AggregateUDF , ScalarUDF , WindowUDF } ;
@@ -72,10 +85,18 @@ use datafusion_physical_expr_common::physical_expr::proto_decode::{
7285use datafusion_physical_expr_common:: physical_expr:: proto_encode:: {
7386 PhysicalExprEncode , PhysicalExprEncodeCtx ,
7487} ;
75- use datafusion_proto_models:: protobuf:: { PhysicalExprNode , PhysicalPlanNode } ;
88+ use datafusion_proto_models:: protobuf:: {
89+ PhysicalExprNode , PhysicalExtensionNode , PhysicalPlanNode ,
90+ physical_plan_node:: PhysicalPlanType ,
91+ } ;
7692
7793use crate :: ExecutionPlan ;
7894
95+ pub use registry:: {
96+ ExecutionPlanDecoder , ExecutionPlanRegistry , ExecutionPlanRegistryExt ,
97+ ExtensionExecutionPlan ,
98+ } ;
99+
79100/// Internal dispatch trait backing [`ExecutionPlanEncodeCtx`].
80101///
81102/// Implemented by `datafusion-proto`. Plan authors never name this trait; they
@@ -155,6 +176,12 @@ pub trait ExecutionPlanDecode {
155176 fn decode_udwf ( & self , name : & str , payload : Option < & [ u8 ] > ) -> Result < Arc < WindowUDF > > ;
156177}
157178
179+ /// The wire parts of an extension plan: its opaque payload, borrowed from the
180+ /// node it was read from, and its already-decoded children.
181+ ///
182+ /// Returned by [`ExecutionPlanDecodeCtx::decode_extension`].
183+ pub type ExtensionPlanParts < ' n > = ( & ' n [ u8 ] , Vec < Arc < dyn ExecutionPlan > > ) ;
184+
158185/// Context handed to [`ExecutionPlan::try_to_proto`].
159186///
160187///
@@ -217,6 +244,64 @@ impl<'a> ExecutionPlanEncodeCtx<'a> {
217244 self . encoder . encode_udwf ( udwf)
218245 }
219246
247+ /// Serialize an extension plan: an opaque `payload` plus its children,
248+ /// tagged with `T`'s [`PLAN_NAME`](ExtensionExecutionPlan::PLAN_NAME).
249+ ///
250+ /// This is the supported way for a third-party plan to be written by
251+ /// [`ExecutionPlan::try_to_proto`]:
252+ /// `PhysicalPlanType` is a closed `oneof`, so extension plans share the
253+ /// single `PhysicalExtensionNode` variant and carry their own bytes.
254+ ///
255+ /// Taking the name from `T` rather than as a string is what keeps the
256+ /// written name and the registered name from drifting: a mismatch would
257+ /// compile, then silently fall back to the `PhysicalExtensionCodec` on
258+ /// whichever machine does the decoding.
259+ ///
260+ /// The name is the wire discriminator. Register `T` on the decoding session
261+ /// (see [`registry`]) and decode selects the plan's own `try_from_proto` by
262+ /// name; a reader that does not know the name still falls back to its
263+ /// `PhysicalExtensionCodec`, so stamping it is safe against older readers.
264+ ///
265+ /// Recursing into `children` here is what keeps a plan's inputs on the
266+ /// wire — hand-rolling the `PhysicalExtensionNode` is easy to get wrong.
267+ pub fn encode_extension < ' b , T , I > (
268+ & self ,
269+ payload : Vec < u8 > ,
270+ children : I ,
271+ ) -> Result < PhysicalPlanNode >
272+ where
273+ T : ExtensionExecutionPlan ,
274+ I : IntoIterator < Item = & ' b Arc < dyn ExecutionPlan > > ,
275+ {
276+ self . encode_extension_named ( T :: PLAN_NAME , payload, children)
277+ }
278+
279+ /// [`Self::encode_extension`] with the name supplied explicitly.
280+ ///
281+ /// For plans registered through
282+ /// [`ExecutionPlanRegistry::register_decoder`](registry::ExecutionPlanRegistry::register_decoder),
283+ /// whose name is not a compile-time constant of a single type. Prefer
284+ /// [`Self::encode_extension`], which cannot disagree with the registration.
285+ pub fn encode_extension_named < ' b , I > (
286+ & self ,
287+ plan_name : & str ,
288+ payload : Vec < u8 > ,
289+ children : I ,
290+ ) -> Result < PhysicalPlanNode >
291+ where
292+ I : IntoIterator < Item = & ' b Arc < dyn ExecutionPlan > > ,
293+ {
294+ Ok ( PhysicalPlanNode {
295+ physical_plan_type : Some ( PhysicalPlanType :: Extension (
296+ PhysicalExtensionNode {
297+ node : payload,
298+ inputs : self . encode_children ( children) ?,
299+ plan_name : Some ( plan_name. to_string ( ) ) ,
300+ } ,
301+ ) ) ,
302+ } )
303+ }
304+
220305 /// An expression-level encode context backed by this plan context.
221306 ///
222307 /// Lets a plan hand `ctx` to expression-level conversions that own their own
@@ -343,6 +428,38 @@ impl<'a> ExecutionPlanDecodeCtx<'a> {
343428 self . decoder . decode_udwf ( name, payload)
344429 }
345430
431+ /// Deserialize a slice of child plans.
432+ pub fn decode_children (
433+ & self ,
434+ nodes : & [ PhysicalPlanNode ] ,
435+ ) -> Result < Vec < Arc < dyn ExecutionPlan > > > {
436+ nodes. iter ( ) . map ( |node| self . decode_child ( node) ) . collect ( )
437+ }
438+
439+ /// Deserialize an extension plan written by
440+ /// [`ExecutionPlanEncodeCtx::encode_extension`], returning its opaque
441+ /// payload and its decoded children.
442+ ///
443+ /// `plan_name` names the caller in the error raised when `node` is not an
444+ /// extension node at all; it is not re-checked against the wire, because
445+ /// the name is what routed decoding here in the first place.
446+ pub fn decode_extension < ' n > (
447+ & self ,
448+ node : & ' n PhysicalPlanNode ,
449+ plan_name : & str ,
450+ ) -> Result < ExtensionPlanParts < ' n > > {
451+ let Some ( PhysicalPlanType :: Extension ( extension) ) = & node. physical_plan_type
452+ else {
453+ return internal_err ! (
454+ "PhysicalPlanNode is not an extension node ({plan_name})"
455+ ) ;
456+ } ;
457+ Ok ( (
458+ extension. node . as_slice ( ) ,
459+ self . decode_children ( & extension. inputs ) ?,
460+ ) )
461+ }
462+
346463 /// An expression-level decode context backed by this plan context, bound to
347464 /// `input_schema`.
348465 ///
@@ -384,3 +501,115 @@ macro_rules! expect_plan_variant {
384501 }
385502 } } ;
386503}
504+
505+ #[ cfg( test) ]
506+ mod tests {
507+ use super :: * ;
508+ use crate :: proto_test_util:: registry_test_plan:: RegisteredExec ;
509+ use crate :: proto_test_util:: {
510+ StubPlanDecoder , StubPlanEncoder , encoded_child_node, stub_child,
511+ } ;
512+
513+ /// The wire node a `RegisteredExec` with one child encodes to.
514+ fn encode ( plan : & RegisteredExec , encoder : & StubPlanEncoder ) -> PhysicalExtensionNode {
515+ let node = plan
516+ . try_to_proto ( & ExecutionPlanEncodeCtx :: new ( encoder) )
517+ . expect ( "encode must succeed" )
518+ . expect ( "an extension plan must serialize itself" ) ;
519+ match node. physical_plan_type {
520+ Some ( PhysicalPlanType :: Extension ( extension) ) => extension,
521+ other => panic ! ( "expected an extension node, got {other:?}" ) ,
522+ }
523+ }
524+
525+ #[ test]
526+ fn encode_extension_stamps_the_name_and_recurses_into_children ( ) {
527+ let encoder = StubPlanEncoder :: ok ( ) ;
528+ let extension = encode (
529+ & RegisteredExec :: new ( "payload" , vec ! [ stub_child( ) ] ) ,
530+ & encoder,
531+ ) ;
532+
533+ assert_eq ! (
534+ extension. plan_name. as_deref( ) ,
535+ Some ( RegisteredExec :: PLAN_NAME )
536+ ) ;
537+ assert_eq ! ( extension. node, b"payload" ) ;
538+ // The child rode the central serializer rather than being dropped.
539+ assert_eq ! ( encoder. plan_calls( ) , 1 ) ;
540+ assert_eq ! ( extension. inputs, vec![ encoded_child_node( ) ] ) ;
541+ }
542+
543+ #[ test]
544+ fn encode_extension_propagates_a_child_failure ( ) {
545+ let encoder = StubPlanEncoder :: failing_on_plan ( 1 ) ;
546+ let err = RegisteredExec :: new ( "payload" , vec ! [ stub_child( ) ] )
547+ . try_to_proto ( & ExecutionPlanEncodeCtx :: new ( & encoder) )
548+ . expect_err ( "a failing child encode must fail the plan" ) ;
549+ assert ! (
550+ err. to_string( ) . contains( "stub plan encode failure" ) ,
551+ "unexpected error: {err}"
552+ ) ;
553+ }
554+
555+ #[ test]
556+ fn decode_extension_returns_the_payload_and_children ( ) {
557+ let node = PhysicalPlanNode {
558+ physical_plan_type : Some ( PhysicalPlanType :: Extension (
559+ PhysicalExtensionNode {
560+ node : b"payload" . to_vec ( ) ,
561+ inputs : vec ! [ encoded_child_node( ) , encoded_child_node( ) ] ,
562+ plan_name : Some ( RegisteredExec :: PLAN_NAME . to_string ( ) ) ,
563+ } ,
564+ ) ) ,
565+ } ;
566+ let decoder = StubPlanDecoder :: ok ( ) ;
567+ let ctx = ExecutionPlanDecodeCtx :: new ( & decoder) ;
568+
569+ let ( payload, children) = ctx
570+ . decode_extension ( & node, RegisteredExec :: PLAN_NAME )
571+ . expect ( "decode must succeed" ) ;
572+
573+ assert_eq ! ( payload, b"payload" ) ;
574+ assert_eq ! ( children. len( ) , 2 ) ;
575+ assert_eq ! ( decoder. plan_calls( ) , 2 ) ;
576+ }
577+
578+ #[ test]
579+ fn decode_extension_rejects_a_non_extension_node ( ) {
580+ let decoder = StubPlanDecoder :: ok ( ) ;
581+ let ctx = ExecutionPlanDecodeCtx :: new ( & decoder) ;
582+
583+ let err = ctx
584+ . decode_extension ( & encoded_child_node ( ) , RegisteredExec :: PLAN_NAME )
585+ . expect_err ( "a built-in node must not decode as an extension" ) ;
586+ assert ! (
587+ err. to_string( ) . contains( RegisteredExec :: PLAN_NAME ) ,
588+ "unexpected error: {err}"
589+ ) ;
590+ assert_eq ! ( decoder. plan_calls( ) , 0 ) ;
591+ }
592+
593+ #[ test]
594+ fn extension_plan_round_trips_through_the_hooks ( ) {
595+ let encoder = StubPlanEncoder :: ok ( ) ;
596+ let extension = encode (
597+ & RegisteredExec :: new ( "payload" , vec ! [ stub_child( ) ] ) ,
598+ & encoder,
599+ ) ;
600+ let node = PhysicalPlanNode {
601+ physical_plan_type : Some ( PhysicalPlanType :: Extension ( extension) ) ,
602+ } ;
603+
604+ let decoder = StubPlanDecoder :: ok ( ) ;
605+ let decoded =
606+ RegisteredExec :: try_from_proto ( & node, & ExecutionPlanDecodeCtx :: new ( & decoder) )
607+ . expect ( "decode must succeed" ) ;
608+
609+ let decoded = decoded
610+ . downcast_ref :: < RegisteredExec > ( )
611+ . expect ( "decoded plan must be a RegisteredExec" ) ;
612+ assert_eq ! ( decoded. payload, "payload" ) ;
613+ assert_eq ! ( decoded. children. len( ) , 1 ) ;
614+ }
615+ }
0 commit comments