Skip to content

Commit 8444d7f

Browse files
adriangbclaude
andcommitted
feat: add a session-scoped decode registry for extension ExecutionPlans
Adds the pieces an extension plan needs to (de)serialize itself through the `try_to_proto` / `try_from_proto` hooks instead of a `PhysicalExtensionCodec`: * `ExecutionPlanEncodeCtx::encode_extension(name, payload, children)` — builds the `PhysicalExtensionNode`, stamps `plan_name`, and recurses into the children (the step hand-rolled wrappers forget). * `ExecutionPlanDecodeCtx::decode_extension` / `decode_children` — the inverse. * `ExtensionExecutionPlan` — the per-type contract: a namespaced `PLAN_NAME` plus the `try_from_proto` the registry stores. `try_from_proto` is a plain function, so a `fn` pointer is all the registry needs. * `ExecutionPlanRegistry` + `SessionConfig::register_execution_plan::<T>()` — session scoped, matching the `FunctionRegistry` precedent, so it reaches every `TaskContext` derived from the session with no new plumbing. A name collision between two crates is an error at registration instead of a mis-decode. `proto.rs` becomes `proto/mod.rs` so the registry lands in its own module. Decode dispatch is not wired up yet, so nothing reaches the registry. Part of apache#24625. Co-Authored-By: Claude Opus 5 <[email protected]>
1 parent 7d2d5a7 commit 8444d7f

3 files changed

Lines changed: 824 additions & 2 deletions

File tree

‎datafusion/physical-plan/src/proto/mod.rs‎

Lines changed: 231 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,25 @@
5656
//! `physical-expr-common`, *below* `datafusion-expr`) cannot do this, which is
5757
//! why `ScalarFunctionExpr` remains special-cased there.
5858
//!
59+
//! # Extension plans
60+
//!
61+
//! Third-party plans have no `PhysicalPlanType` variant of their own; they all
62+
//! share `PhysicalExtensionNode`, whose payload is opaque bytes. Such a plan
63+
//! rides the same hooks by pairing
64+
//! [`ExecutionPlanEncodeCtx::encode_extension`] with
65+
//! [`ExecutionPlanDecodeCtx::decode_extension`], and declaring a name through
66+
//! [`ExtensionExecutionPlan`] so the decoding session can select its decoder by
67+
//! name rather than by trying every registered `PhysicalExtensionCodec` in
68+
//! turn. See [`registry`].
69+
//!
5970
//! [`ExecutionPlan`]: crate::ExecutionPlan
6071
72+
pub mod registry;
73+
6174
use std::sync::Arc;
6275

6376
use arrow::datatypes::Schema;
64-
use datafusion_common::{Result, internal_datafusion_err};
77+
use datafusion_common::{Result, internal_datafusion_err, internal_err};
6578
use datafusion_execution::TaskContext;
6679
use datafusion_expr::physical_planning_context::ScalarSubqueryResults;
6780
use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
@@ -72,10 +85,18 @@ use datafusion_physical_expr_common::physical_expr::proto_decode::{
7285
use datafusion_physical_expr_common::physical_expr::proto_encode::{
7386
PhysicalExprEncode, PhysicalExprEncodeCtx,
7487
};
75-
use datafusion_proto_models::protobuf::{PhysicalExprNode, PhysicalPlanNode};
88+
use datafusion_proto_models::protobuf::{
89+
PhysicalExprNode, PhysicalExtensionNode, PhysicalPlanNode,
90+
physical_plan_node::PhysicalPlanType,
91+
};
7692

7793
use crate::ExecutionPlan;
7894

95+
pub use registry::{
96+
ExecutionPlanDecoder, ExecutionPlanRegistry, ExecutionPlanRegistryExt,
97+
ExtensionExecutionPlan,
98+
};
99+
79100
/// Internal dispatch trait backing [`ExecutionPlanEncodeCtx`].
80101
///
81102
/// Implemented by `datafusion-proto`. Plan authors never name this trait; they
@@ -155,6 +176,12 @@ pub trait ExecutionPlanDecode {
155176
fn decode_udwf(&self, name: &str, payload: Option<&[u8]>) -> Result<Arc<WindowUDF>>;
156177
}
157178

179+
/// The wire parts of an extension plan: its opaque payload, borrowed from the
180+
/// node it was read from, and its already-decoded children.
181+
///
182+
/// Returned by [`ExecutionPlanDecodeCtx::decode_extension`].
183+
pub type ExtensionPlanParts<'n> = (&'n [u8], Vec<Arc<dyn ExecutionPlan>>);
184+
158185
/// Context handed to [`ExecutionPlan::try_to_proto`].
159186
///
160187
///
@@ -217,6 +244,64 @@ impl<'a> ExecutionPlanEncodeCtx<'a> {
217244
self.encoder.encode_udwf(udwf)
218245
}
219246

247+
/// Serialize an extension plan: an opaque `payload` plus its children,
248+
/// tagged with `T`'s [`PLAN_NAME`](ExtensionExecutionPlan::PLAN_NAME).
249+
///
250+
/// This is the supported way for a third-party plan to be written by
251+
/// [`ExecutionPlan::try_to_proto`]:
252+
/// `PhysicalPlanType` is a closed `oneof`, so extension plans share the
253+
/// single `PhysicalExtensionNode` variant and carry their own bytes.
254+
///
255+
/// Taking the name from `T` rather than as a string is what keeps the
256+
/// written name and the registered name from drifting: a mismatch would
257+
/// compile, then silently fall back to the `PhysicalExtensionCodec` on
258+
/// whichever machine does the decoding.
259+
///
260+
/// The name is the wire discriminator. Register `T` on the decoding session
261+
/// (see [`registry`]) and decode selects the plan's own `try_from_proto` by
262+
/// name; a reader that does not know the name still falls back to its
263+
/// `PhysicalExtensionCodec`, so stamping it is safe against older readers.
264+
///
265+
/// Recursing into `children` here is what keeps a plan's inputs on the
266+
/// wire — hand-rolling the `PhysicalExtensionNode` is easy to get wrong.
267+
pub fn encode_extension<'b, T, I>(
268+
&self,
269+
payload: Vec<u8>,
270+
children: I,
271+
) -> Result<PhysicalPlanNode>
272+
where
273+
T: ExtensionExecutionPlan,
274+
I: IntoIterator<Item = &'b Arc<dyn ExecutionPlan>>,
275+
{
276+
self.encode_extension_named(T::PLAN_NAME, payload, children)
277+
}
278+
279+
/// [`Self::encode_extension`] with the name supplied explicitly.
280+
///
281+
/// For plans registered through
282+
/// [`ExecutionPlanRegistry::register_decoder`](registry::ExecutionPlanRegistry::register_decoder),
283+
/// whose name is not a compile-time constant of a single type. Prefer
284+
/// [`Self::encode_extension`], which cannot disagree with the registration.
285+
pub fn encode_extension_named<'b, I>(
286+
&self,
287+
plan_name: &str,
288+
payload: Vec<u8>,
289+
children: I,
290+
) -> Result<PhysicalPlanNode>
291+
where
292+
I: IntoIterator<Item = &'b Arc<dyn ExecutionPlan>>,
293+
{
294+
Ok(PhysicalPlanNode {
295+
physical_plan_type: Some(PhysicalPlanType::Extension(
296+
PhysicalExtensionNode {
297+
node: payload,
298+
inputs: self.encode_children(children)?,
299+
plan_name: Some(plan_name.to_string()),
300+
},
301+
)),
302+
})
303+
}
304+
220305
/// An expression-level encode context backed by this plan context.
221306
///
222307
/// Lets a plan hand `ctx` to expression-level conversions that own their own
@@ -343,6 +428,38 @@ impl<'a> ExecutionPlanDecodeCtx<'a> {
343428
self.decoder.decode_udwf(name, payload)
344429
}
345430

431+
/// Deserialize a slice of child plans.
432+
pub fn decode_children(
433+
&self,
434+
nodes: &[PhysicalPlanNode],
435+
) -> Result<Vec<Arc<dyn ExecutionPlan>>> {
436+
nodes.iter().map(|node| self.decode_child(node)).collect()
437+
}
438+
439+
/// Deserialize an extension plan written by
440+
/// [`ExecutionPlanEncodeCtx::encode_extension`], returning its opaque
441+
/// payload and its decoded children.
442+
///
443+
/// `plan_name` names the caller in the error raised when `node` is not an
444+
/// extension node at all; it is not re-checked against the wire, because
445+
/// the name is what routed decoding here in the first place.
446+
pub fn decode_extension<'n>(
447+
&self,
448+
node: &'n PhysicalPlanNode,
449+
plan_name: &str,
450+
) -> Result<ExtensionPlanParts<'n>> {
451+
let Some(PhysicalPlanType::Extension(extension)) = &node.physical_plan_type
452+
else {
453+
return internal_err!(
454+
"PhysicalPlanNode is not an extension node ({plan_name})"
455+
);
456+
};
457+
Ok((
458+
extension.node.as_slice(),
459+
self.decode_children(&extension.inputs)?,
460+
))
461+
}
462+
346463
/// An expression-level decode context backed by this plan context, bound to
347464
/// `input_schema`.
348465
///
@@ -384,3 +501,115 @@ macro_rules! expect_plan_variant {
384501
}
385502
}};
386503
}
504+
505+
#[cfg(test)]
506+
mod tests {
507+
use super::*;
508+
use crate::proto_test_util::registry_test_plan::RegisteredExec;
509+
use crate::proto_test_util::{
510+
StubPlanDecoder, StubPlanEncoder, encoded_child_node, stub_child,
511+
};
512+
513+
/// The wire node a `RegisteredExec` with one child encodes to.
514+
fn encode(plan: &RegisteredExec, encoder: &StubPlanEncoder) -> PhysicalExtensionNode {
515+
let node = plan
516+
.try_to_proto(&ExecutionPlanEncodeCtx::new(encoder))
517+
.expect("encode must succeed")
518+
.expect("an extension plan must serialize itself");
519+
match node.physical_plan_type {
520+
Some(PhysicalPlanType::Extension(extension)) => extension,
521+
other => panic!("expected an extension node, got {other:?}"),
522+
}
523+
}
524+
525+
#[test]
526+
fn encode_extension_stamps_the_name_and_recurses_into_children() {
527+
let encoder = StubPlanEncoder::ok();
528+
let extension = encode(
529+
&RegisteredExec::new("payload", vec![stub_child()]),
530+
&encoder,
531+
);
532+
533+
assert_eq!(
534+
extension.plan_name.as_deref(),
535+
Some(RegisteredExec::PLAN_NAME)
536+
);
537+
assert_eq!(extension.node, b"payload");
538+
// The child rode the central serializer rather than being dropped.
539+
assert_eq!(encoder.plan_calls(), 1);
540+
assert_eq!(extension.inputs, vec![encoded_child_node()]);
541+
}
542+
543+
#[test]
544+
fn encode_extension_propagates_a_child_failure() {
545+
let encoder = StubPlanEncoder::failing_on_plan(1);
546+
let err = RegisteredExec::new("payload", vec![stub_child()])
547+
.try_to_proto(&ExecutionPlanEncodeCtx::new(&encoder))
548+
.expect_err("a failing child encode must fail the plan");
549+
assert!(
550+
err.to_string().contains("stub plan encode failure"),
551+
"unexpected error: {err}"
552+
);
553+
}
554+
555+
#[test]
556+
fn decode_extension_returns_the_payload_and_children() {
557+
let node = PhysicalPlanNode {
558+
physical_plan_type: Some(PhysicalPlanType::Extension(
559+
PhysicalExtensionNode {
560+
node: b"payload".to_vec(),
561+
inputs: vec![encoded_child_node(), encoded_child_node()],
562+
plan_name: Some(RegisteredExec::PLAN_NAME.to_string()),
563+
},
564+
)),
565+
};
566+
let decoder = StubPlanDecoder::ok();
567+
let ctx = ExecutionPlanDecodeCtx::new(&decoder);
568+
569+
let (payload, children) = ctx
570+
.decode_extension(&node, RegisteredExec::PLAN_NAME)
571+
.expect("decode must succeed");
572+
573+
assert_eq!(payload, b"payload");
574+
assert_eq!(children.len(), 2);
575+
assert_eq!(decoder.plan_calls(), 2);
576+
}
577+
578+
#[test]
579+
fn decode_extension_rejects_a_non_extension_node() {
580+
let decoder = StubPlanDecoder::ok();
581+
let ctx = ExecutionPlanDecodeCtx::new(&decoder);
582+
583+
let err = ctx
584+
.decode_extension(&encoded_child_node(), RegisteredExec::PLAN_NAME)
585+
.expect_err("a built-in node must not decode as an extension");
586+
assert!(
587+
err.to_string().contains(RegisteredExec::PLAN_NAME),
588+
"unexpected error: {err}"
589+
);
590+
assert_eq!(decoder.plan_calls(), 0);
591+
}
592+
593+
#[test]
594+
fn extension_plan_round_trips_through_the_hooks() {
595+
let encoder = StubPlanEncoder::ok();
596+
let extension = encode(
597+
&RegisteredExec::new("payload", vec![stub_child()]),
598+
&encoder,
599+
);
600+
let node = PhysicalPlanNode {
601+
physical_plan_type: Some(PhysicalPlanType::Extension(extension)),
602+
};
603+
604+
let decoder = StubPlanDecoder::ok();
605+
let decoded =
606+
RegisteredExec::try_from_proto(&node, &ExecutionPlanDecodeCtx::new(&decoder))
607+
.expect("decode must succeed");
608+
609+
let decoded = decoded
610+
.downcast_ref::<RegisteredExec>()
611+
.expect("decoded plan must be a RegisteredExec");
612+
assert_eq!(decoded.payload, "payload");
613+
assert_eq!(decoded.children.len(), 1);
614+
}
615+
}

0 commit comments

Comments
 (0)