|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +//! The `PhysicalExpr` face of the session-scoped [`ProtoDecoderRegistry`], |
| 19 | +//! which routes extension expressions to their decoders by name: |
| 20 | +//! [`register_physical_expr`] and [`physical_expr_names`], all keyed on `dyn PhysicalExpr`. The store itself |
| 21 | +//! is shared with every other extension kind, so a session carries one |
| 22 | +//! registry. |
| 23 | +
|
| 24 | +use std::sync::Arc; |
| 25 | + |
| 26 | +use datafusion_common::Result; |
| 27 | +use datafusion_physical_expr_common::physical_expr::PhysicalExpr; |
| 28 | +pub use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx; |
| 29 | +pub use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx; |
| 30 | +pub use datafusion_proto_models::ProtoDecoderRegistry; |
| 31 | +use datafusion_proto_models::protobuf::PhysicalExprNode; |
| 32 | +use datafusion_proto_models::protobuf::physical_expr_node::ExprType; |
| 33 | + |
| 34 | +/// The wire name of an extension [`PhysicalExpr`], and the constructor that |
| 35 | +/// rebuilds it. |
| 36 | +/// |
| 37 | +/// Only extension expressions implement this. A built-in expression has an |
| 38 | +/// `ExprType` variant of its own, keeps the inherent `try_from_proto` |
| 39 | +/// dispatched from that variant, and has no wire name to be registered under — |
| 40 | +/// so [`register_physical_expr`] will not accept one. |
| 41 | +/// |
| 42 | +/// One impl block says what the expression is called and how it decodes; its |
| 43 | +/// `try_to_proto` on [`PhysicalExpr`] writes the matching node with |
| 44 | +/// [`extension_expr_node`]: |
| 45 | +/// |
| 46 | +/// ```ignore |
| 47 | +/// impl PhysicalExpr for MyExpr { |
| 48 | +/// // ... |
| 49 | +/// fn try_to_proto( |
| 50 | +/// &self, |
| 51 | +/// ctx: &PhysicalExprEncodeCtx<'_>, |
| 52 | +/// ) -> Result<Option<PhysicalExprNode>> { |
| 53 | +/// // Stamps `Self::NAME`, so the encoded name and the registry key |
| 54 | +/// // cannot drift apart. |
| 55 | +/// Ok(Some(extension_expr_node::<Self>( |
| 56 | +/// ctx, |
| 57 | +/// self.state.to_bytes()?, |
| 58 | +/// self.children(), |
| 59 | +/// )?)) |
| 60 | +/// } |
| 61 | +/// } |
| 62 | +/// |
| 63 | +/// impl ExtensionExprFromProto for MyExpr { |
| 64 | +/// // Namespace the name with the owning crate, so a collision with |
| 65 | +/// // another crate is an error at registration and not a wrong decode. |
| 66 | +/// const NAME: &'static str = "my_crate.MyExpr"; |
| 67 | +/// |
| 68 | +/// fn try_from_proto( |
| 69 | +/// node: &PhysicalExprNode, |
| 70 | +/// ctx: &PhysicalExprDecodeCtx<'_>, |
| 71 | +/// ) -> Result<Arc<dyn PhysicalExpr>> { |
| 72 | +/// let extension = expect_expr_variant!( |
| 73 | +/// node, |
| 74 | +/// physical_expr_node::ExprType::Extension, |
| 75 | +/// "Extension", |
| 76 | +/// ); |
| 77 | +/// let state = MyState::from_bytes(&extension.expr)?; |
| 78 | +/// let children = ctx.decode_children_expressions(&extension.inputs)?; |
| 79 | +/// Ok(Arc::new(MyExpr::new(state, children))) |
| 80 | +/// } |
| 81 | +/// } |
| 82 | +/// |
| 83 | +/// let mut registry = ProtoDecoderRegistry::new(); |
| 84 | +/// register_physical_expr::<MyExpr>(&mut registry)?; |
| 85 | +/// let config = SessionConfig::new().with_extension(Arc::new(registry)); |
| 86 | +/// ``` |
| 87 | +pub trait ExtensionExprFromProto: PhysicalExpr + Sized { |
| 88 | + /// The name this expression type is written and registered under. |
| 89 | + /// |
| 90 | + /// Namespace it with the owning crate — `"my_crate.MyExpr"`, not |
| 91 | + /// `"MyExpr"` — so that two independent crates registering into the same |
| 92 | + /// session collide at registration time instead of silently decoding each |
| 93 | + /// other's nodes. Never write it by hand on the wire: |
| 94 | + /// [`extension_expr_node`] stamps it for you. |
| 95 | + const NAME: &'static str; |
| 96 | + |
| 97 | + /// Rebuild the expression from the `PhysicalExprNode` its |
| 98 | + /// [`PhysicalExpr::try_to_proto`] wrote. |
| 99 | + /// |
| 100 | + /// Takes the whole node — the exact inverse of what `try_to_proto` |
| 101 | + /// returns — so the constructor can also see outer-node fields such as |
| 102 | + /// `expr_id`. `ctx.session::<TaskContext>()` gives the session, for |
| 103 | + /// resolving UDFs by name or reading configuration. |
| 104 | + fn try_from_proto( |
| 105 | + node: &PhysicalExprNode, |
| 106 | + ctx: &PhysicalExprDecodeCtx<'_>, |
| 107 | + ) -> Result<Arc<dyn PhysicalExpr>>; |
| 108 | +} |
| 109 | + |
| 110 | +/// Build the `PhysicalExprNode` for an extension expression `T`: an |
| 111 | +/// `Extension` variant carrying `payload`, the encoded `children`, and `T`'s |
| 112 | +/// [`NAME`](ExtensionExprFromProto::NAME). |
| 113 | +/// |
| 114 | +/// The one supported way for an extension expression to write itself. It is a |
| 115 | +/// free function rather than a method on [`PhysicalExprEncodeCtx`] because the |
| 116 | +/// context lives in `datafusion-physical-expr-common`, below the crate that |
| 117 | +/// can name [`ExtensionExprFromProto`]. |
| 118 | +pub fn extension_expr_node<T: ExtensionExprFromProto>( |
| 119 | + ctx: &PhysicalExprEncodeCtx<'_>, |
| 120 | + payload: Vec<u8>, |
| 121 | + children: Vec<&Arc<dyn PhysicalExpr>>, |
| 122 | +) -> Result<PhysicalExprNode> { |
| 123 | + Ok(PhysicalExprNode { |
| 124 | + expr_type: Some(ExprType::Extension( |
| 125 | + datafusion_proto_models::protobuf::PhysicalExtensionExprNode { |
| 126 | + expr: payload, |
| 127 | + inputs: ctx.encode_children_expressions(children)?, |
| 128 | + expr_name: Some(T::NAME.to_string()), |
| 129 | + }, |
| 130 | + )), |
| 131 | + ..Default::default() |
| 132 | + }) |
| 133 | +} |
| 134 | + |
| 135 | +/// How this facade stores a decoder in the shared registry: a function pointer |
| 136 | +/// to the monomorphized [`ExtensionExprFromProto::try_from_proto`]. |
| 137 | +/// |
| 138 | +/// Deliberately private, and the same type on both the |
| 139 | +/// [`register_physical_expr`] and the [`decode_physical_expr`] side. |
| 140 | +/// [`ExtensionExprFromProto`] is the public contract and |
| 141 | +/// [`decode_physical_expr`] (hidden, internal to DataFusion) is the only way |
| 142 | +/// to invoke one, so the storage |
| 143 | +/// can change later without a breaking change. |
| 144 | +type PhysicalExprDecoder = |
| 145 | + fn(&PhysicalExprNode, &PhysicalExprDecodeCtx<'_>) -> Result<Arc<dyn PhysicalExpr>>; |
| 146 | + |
| 147 | +/// Register `T` in `registry` under its [`ExtensionExprFromProto::NAME`]. |
| 148 | +/// |
| 149 | +/// Registering the same type twice is a no-op. Registering a *different* type |
| 150 | +/// under a name already taken is an error, so collisions surface here rather |
| 151 | +/// than as a wrong decode later. The name is scoped to `dyn PhysicalExpr`, so a |
| 152 | +/// plan or a data source may use the same name in the same registry. |
| 153 | +/// |
| 154 | +/// # Who builds the registry |
| 155 | +/// |
| 156 | +/// The application that owns the session builds it. A session carries at most |
| 157 | +/// one registry, and `SessionConfig::with_extension` replaces what is there, so |
| 158 | +/// a library must never attach a registry of its own: it would silently discard |
| 159 | +/// another library's. A library exposes a function that fills a registry it is |
| 160 | +/// handed, and the application composes them. One registry holds every kind, so |
| 161 | +/// a library registers its plans and its expressions into the same object. |
| 162 | +/// |
| 163 | +/// # A built-in expression cannot be registered |
| 164 | +/// |
| 165 | +/// The bound is [`ExtensionExprFromProto`], which only extension expressions |
| 166 | +/// implement. A built-in keeps an inherent `try_from_proto`, dispatched from |
| 167 | +/// its own `ExprType` variant: |
| 168 | +/// |
| 169 | +/// ``` |
| 170 | +/// use datafusion_physical_expr::expressions::Column; |
| 171 | +/// |
| 172 | +/// let _ = Column::try_from_proto; |
| 173 | +/// ``` |
| 174 | +/// |
| 175 | +/// but it is not an extension, and registering it does not compile: |
| 176 | +/// |
| 177 | +/// ```compile_fail |
| 178 | +/// use datafusion_physical_expr::expressions::Column; |
| 179 | +/// use datafusion_physical_expr::proto::register_physical_expr; |
| 180 | +/// use datafusion_proto_models::ProtoDecoderRegistry; |
| 181 | +/// |
| 182 | +/// let mut registry = ProtoDecoderRegistry::new(); |
| 183 | +/// register_physical_expr::<Column>(&mut registry).unwrap(); |
| 184 | +/// ``` |
| 185 | +pub fn register_physical_expr<T: ExtensionExprFromProto>( |
| 186 | + registry: &mut ProtoDecoderRegistry, |
| 187 | +) -> Result<()> { |
| 188 | + registry.register_decoder::<dyn PhysicalExpr, T, PhysicalExprDecoder>( |
| 189 | + T::NAME, |
| 190 | + T::try_from_proto, |
| 191 | + ) |
| 192 | +} |
| 193 | + |
| 194 | +/// Decode `node` with the extension expression decoder registered under the |
| 195 | +/// name `node` carries. |
| 196 | +/// |
| 197 | +/// The name is read from the node's `Extension` variant rather than passed in, |
| 198 | +/// so a caller cannot pair a node with a name it does not carry. |
| 199 | +/// |
| 200 | +/// `None` means "this node names no extension expression decoder of ours": it |
| 201 | +/// is not an extension node, it carries no name, or no registered name matches. |
| 202 | +/// The caller then falls back to the `PhysicalExtensionCodec` chain. |
| 203 | +/// `Some(Err(..))` means the decoder that *does* own the name failed, which is |
| 204 | +/// fatal: falling back there would let a codec decode the payload wrongly, the |
| 205 | +/// very thing the name exists to prevent. |
| 206 | +/// |
| 207 | +/// Not intended as public API: `datafusion-proto` drives decoding and is the |
| 208 | +/// only intended caller. It is `pub` only because that call crosses a crate |
| 209 | +/// boundary, so it is hidden from the docs and may change without notice. |
| 210 | +#[doc(hidden)] |
| 211 | +pub fn decode_physical_expr( |
| 212 | + registry: &ProtoDecoderRegistry, |
| 213 | + node: &PhysicalExprNode, |
| 214 | + ctx: &PhysicalExprDecodeCtx<'_>, |
| 215 | +) -> Option<Result<Arc<dyn PhysicalExpr>>> { |
| 216 | + let name = expr_name(node)?; |
| 217 | + let decoder = registry.decoder::<dyn PhysicalExpr, PhysicalExprDecoder>(name)?; |
| 218 | + Some(decoder(node, ctx)) |
| 219 | +} |
| 220 | + |
| 221 | +/// Every extension expression name registered in `registry`, in arbitrary |
| 222 | +/// order. |
| 223 | +/// |
| 224 | +/// Names registered for another kind are not included. |
| 225 | +pub fn physical_expr_names( |
| 226 | + registry: &ProtoDecoderRegistry, |
| 227 | +) -> impl Iterator<Item = &str> { |
| 228 | + registry.names::<dyn PhysicalExpr>() |
| 229 | +} |
| 230 | + |
| 231 | +/// The registry name `node` was written with, if it is an extension node that |
| 232 | +/// carries one. |
| 233 | +fn expr_name(node: &PhysicalExprNode) -> Option<&str> { |
| 234 | + match node.expr_type.as_ref()? { |
| 235 | + ExprType::Extension(extension) => extension.expr_name.as_deref(), |
| 236 | + _ => None, |
| 237 | + } |
| 238 | +} |
0 commit comments