|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +//! Serialization hooks for [`ExecutionPlan`], mirroring the |
| 19 | +//! `try_to_proto`/`try_from_proto` pattern used for `PhysicalExpr`. |
| 20 | +//! |
| 21 | +//! # Why the indirection |
| 22 | +//! |
| 23 | +//! An `ExecutionPlan` must be able to (de)serialize its child plans and its |
| 24 | +//! child physical expressions recursively. The concrete recursion lives in |
| 25 | +//! `datafusion-proto` (it owns the extension codec, the session context and the |
| 26 | +//! central converter), but `datafusion-proto` sits *above* `datafusion-physical-plan` |
| 27 | +//! in the crate graph. To let a plan drive that recursion without a dependency |
| 28 | +//! cycle, this module defines: |
| 29 | +//! |
| 30 | +//! * [`ExecutionPlanEncodeCtx`] / [`ExecutionPlanDecodeCtx`] — the stable, |
| 31 | +//! concrete context types a plan author interacts with. New capabilities can |
| 32 | +//! be added here without changing every plan's hook signature. |
| 33 | +//! * [`ExecutionPlanEncode`] / [`ExecutionPlanDecode`] — internal dispatch |
| 34 | +//! traits, *defined* here but *implemented* in `datafusion-proto`, that the |
| 35 | +//! context types delegate to. This is the dependency inversion that keeps the |
| 36 | +//! proto types flowing in one direction only. |
| 37 | +//! |
| 38 | +//! `datafusion-physical-plan` depends on the pure prost types in |
| 39 | +//! `datafusion-proto-models` (feature `proto`), never on `datafusion-proto`. |
| 40 | +//! |
| 41 | +//! # Scope: self-contained plans only |
| 42 | +//! |
| 43 | +//! These hooks are for plans that are *self-contained*: everything they need to |
| 44 | +//! round-trip is reachable from their child plans, their child expressions and |
| 45 | +//! their input schema. Plans that reference session-scoped machinery only |
| 46 | +//! `datafusion-proto` owns — the [`PhysicalExtensionCodec`] (for UDAF/UDWF and |
| 47 | +//! extension nodes) — stay as typed dispatch arms inside `datafusion-proto`, |
| 48 | +//! exactly as `ScalarFunctionExpr` does on the expression side. The decode |
| 49 | +//! context intentionally exposes the [`TaskContext`] (for function-registry and |
| 50 | +//! session-config lookups reachable from `datafusion-execution`) but *not* the |
| 51 | +//! extension codec. |
| 52 | +//! |
| 53 | +//! [`PhysicalExtensionCodec`]: (owned by `datafusion-proto`) |
| 54 | +//! [`ExecutionPlan`]: crate::ExecutionPlan |
| 55 | +
|
| 56 | +use std::sync::Arc; |
| 57 | + |
| 58 | +use arrow::datatypes::Schema; |
| 59 | +use datafusion_common::{Result, internal_datafusion_err}; |
| 60 | +use datafusion_execution::TaskContext; |
| 61 | +use datafusion_physical_expr::PhysicalExpr; |
| 62 | +use datafusion_proto_models::protobuf::{PhysicalExprNode, PhysicalPlanNode}; |
| 63 | + |
| 64 | +use crate::ExecutionPlan; |
| 65 | + |
| 66 | +/// Internal dispatch trait backing [`ExecutionPlanEncodeCtx`]. |
| 67 | +/// |
| 68 | +/// Implemented by `datafusion-proto`. Plan authors never name this trait; they |
| 69 | +/// call methods on [`ExecutionPlanEncodeCtx`] instead. |
| 70 | +pub trait ExecutionPlanEncode { |
| 71 | + /// Serialize a child execution plan (recursing through the central |
| 72 | + /// serializer, so the child's own `try_to_proto` hook is honored). |
| 73 | + fn encode_plan(&self, plan: &Arc<dyn ExecutionPlan>) -> Result<PhysicalPlanNode>; |
| 74 | + |
| 75 | + /// Serialize a physical expression owned by the plan. |
| 76 | + fn encode_expr(&self, expr: &Arc<dyn PhysicalExpr>) -> Result<PhysicalExprNode>; |
| 77 | +} |
| 78 | + |
| 79 | +/// Internal dispatch trait backing [`ExecutionPlanDecodeCtx`]. |
| 80 | +/// |
| 81 | +/// Implemented by `datafusion-proto`. Plan authors never name this trait; they |
| 82 | +/// call methods on [`ExecutionPlanDecodeCtx`] instead. |
| 83 | +pub trait ExecutionPlanDecode { |
| 84 | + /// Deserialize a child execution plan (recursing through the central |
| 85 | + /// deserializer, so the child's own `try_from_proto` is honored). |
| 86 | + fn decode_plan(&self, node: &PhysicalPlanNode) -> Result<Arc<dyn ExecutionPlan>>; |
| 87 | + |
| 88 | + /// Deserialize a physical expression against `input_schema`. |
| 89 | + fn decode_expr( |
| 90 | + &self, |
| 91 | + node: &PhysicalExprNode, |
| 92 | + input_schema: &Schema, |
| 93 | + ) -> Result<Arc<dyn PhysicalExpr>>; |
| 94 | + |
| 95 | + /// The session task context, used by plans that need the function registry |
| 96 | + /// or session configuration. Never exposes the proto extension codec. |
| 97 | + fn task_ctx(&self) -> &TaskContext; |
| 98 | +} |
| 99 | + |
| 100 | +/// Context handed to [`ExecutionPlan::try_to_proto`](crate::ExecutionPlan::try_to_proto). |
| 101 | +/// |
| 102 | +/// Provides the primitives a plan needs to serialize its children and |
| 103 | +/// expressions without naming `datafusion-proto`. |
| 104 | +pub struct ExecutionPlanEncodeCtx<'a> { |
| 105 | + encoder: &'a dyn ExecutionPlanEncode, |
| 106 | +} |
| 107 | + |
| 108 | +impl<'a> ExecutionPlanEncodeCtx<'a> { |
| 109 | + /// Create a new encode context wrapping an [`ExecutionPlanEncode`] |
| 110 | + /// implementation (supplied by `datafusion-proto`). |
| 111 | + pub fn new(encoder: &'a dyn ExecutionPlanEncode) -> Self { |
| 112 | + Self { encoder } |
| 113 | + } |
| 114 | + |
| 115 | + /// Serialize a single child plan. |
| 116 | + pub fn encode_child( |
| 117 | + &self, |
| 118 | + plan: &Arc<dyn ExecutionPlan>, |
| 119 | + ) -> Result<PhysicalPlanNode> { |
| 120 | + self.encoder.encode_plan(plan) |
| 121 | + } |
| 122 | + |
| 123 | + /// Serialize an iterator of child plans. |
| 124 | + pub fn encode_children<'b, I>(&self, plans: I) -> Result<Vec<PhysicalPlanNode>> |
| 125 | + where |
| 126 | + I: IntoIterator<Item = &'b Arc<dyn ExecutionPlan>>, |
| 127 | + { |
| 128 | + plans.into_iter().map(|p| self.encode_child(p)).collect() |
| 129 | + } |
| 130 | + |
| 131 | + /// Serialize a single physical expression. |
| 132 | + pub fn encode_expr(&self, expr: &Arc<dyn PhysicalExpr>) -> Result<PhysicalExprNode> { |
| 133 | + self.encoder.encode_expr(expr) |
| 134 | + } |
| 135 | + |
| 136 | + /// Serialize an iterator of physical expressions. |
| 137 | + pub fn encode_expressions<'b, I>(&self, exprs: I) -> Result<Vec<PhysicalExprNode>> |
| 138 | + where |
| 139 | + I: IntoIterator<Item = &'b Arc<dyn PhysicalExpr>>, |
| 140 | + { |
| 141 | + exprs.into_iter().map(|e| self.encode_expr(e)).collect() |
| 142 | + } |
| 143 | +} |
| 144 | + |
| 145 | +/// Context handed to a plan's `try_from_proto` associated function. |
| 146 | +/// |
| 147 | +/// Provides the primitives a plan needs to deserialize its children and |
| 148 | +/// expressions without naming `datafusion-proto`. |
| 149 | +pub struct ExecutionPlanDecodeCtx<'a> { |
| 150 | + decoder: &'a dyn ExecutionPlanDecode, |
| 151 | +} |
| 152 | + |
| 153 | +impl<'a> ExecutionPlanDecodeCtx<'a> { |
| 154 | + /// Create a new decode context wrapping an [`ExecutionPlanDecode`] |
| 155 | + /// implementation (supplied by `datafusion-proto`). |
| 156 | + pub fn new(decoder: &'a dyn ExecutionPlanDecode) -> Self { |
| 157 | + Self { decoder } |
| 158 | + } |
| 159 | + |
| 160 | + /// Deserialize a single child plan. |
| 161 | + pub fn decode_child( |
| 162 | + &self, |
| 163 | + node: &PhysicalPlanNode, |
| 164 | + ) -> Result<Arc<dyn ExecutionPlan>> { |
| 165 | + self.decoder.decode_plan(node) |
| 166 | + } |
| 167 | + |
| 168 | + /// Deserialize a required child plan, producing a uniform "missing required |
| 169 | + /// field" error when the optional wire field is absent. |
| 170 | + pub fn decode_required_child( |
| 171 | + &self, |
| 172 | + node: Option<&PhysicalPlanNode>, |
| 173 | + plan_name: &str, |
| 174 | + field: &str, |
| 175 | + ) -> Result<Arc<dyn ExecutionPlan>> { |
| 176 | + let node = node.ok_or_else(|| { |
| 177 | + internal_datafusion_err!("{plan_name} is missing required field '{field}'") |
| 178 | + })?; |
| 179 | + self.decode_child(node) |
| 180 | + } |
| 181 | + |
| 182 | + /// Deserialize a physical expression against `input_schema`. |
| 183 | + pub fn decode_expr( |
| 184 | + &self, |
| 185 | + node: &PhysicalExprNode, |
| 186 | + input_schema: &Schema, |
| 187 | + ) -> Result<Arc<dyn PhysicalExpr>> { |
| 188 | + self.decoder.decode_expr(node, input_schema) |
| 189 | + } |
| 190 | + |
| 191 | + /// Deserialize a required physical expression against `input_schema`. |
| 192 | + pub fn decode_required_expr( |
| 193 | + &self, |
| 194 | + node: Option<&PhysicalExprNode>, |
| 195 | + input_schema: &Schema, |
| 196 | + plan_name: &str, |
| 197 | + field: &str, |
| 198 | + ) -> Result<Arc<dyn PhysicalExpr>> { |
| 199 | + let node = node.ok_or_else(|| { |
| 200 | + internal_datafusion_err!("{plan_name} is missing required field '{field}'") |
| 201 | + })?; |
| 202 | + self.decode_expr(node, input_schema) |
| 203 | + } |
| 204 | + |
| 205 | + /// The session task context (function registry + session config). Never |
| 206 | + /// exposes the proto extension codec — session-codec-dependent plans stay as |
| 207 | + /// typed dispatch arms in `datafusion-proto`. |
| 208 | + pub fn task_ctx(&self) -> &TaskContext { |
| 209 | + self.decoder.task_ctx() |
| 210 | + } |
| 211 | +} |
| 212 | + |
| 213 | +/// Assert that a [`PhysicalPlanNode`] carries the expected `PhysicalPlanType` |
| 214 | +/// variant, returning a reference to the inner payload, else an `internal_err!`. |
| 215 | +/// Mirrors `expect_expr_variant!` on the expression side. Field access on the |
| 216 | +/// result auto-derefs through the `Box` that boxed variants use. |
| 217 | +#[macro_export] |
| 218 | +macro_rules! expect_plan_variant { |
| 219 | + ($node:expr, $variant:path, $plan_name:literal $(,)?) => {{ |
| 220 | + match &$node.physical_plan_type { |
| 221 | + Some($variant(inner)) => inner, |
| 222 | + _ => { |
| 223 | + return ::datafusion_common::internal_err!(concat!( |
| 224 | + "PhysicalPlanNode is not a ", |
| 225 | + $plan_name |
| 226 | + )); |
| 227 | + } |
| 228 | + } |
| 229 | + }}; |
| 230 | +} |
0 commit comments