Skip to content

Commit 10ef51a

Browse files
adriangbclaude
andcommitted
feat: add typed function-codec and session-config capabilities to proto expr contexts
PhysicalExprEncodeCtx::encode_function<F> and PhysicalExprDecodeCtx::decode_function<F> expose the serialization layer's UDF codec through generic, fully-typed accessors: call sites name the function type (e.g. datafusion_expr::ScalarUDF) and never see erasure or write downcasts. Function types live above this crate in the dependency graph, so the internal dispatch traits carry monomorphic type-erased channels routed by TypeId; the ctx verifies the returned type and turns contract violations into clean internal errors. Which types a layer supports is an explicit part of its contract. PhysicalExprDecodeCtx::config_options provides session configuration with an erroring default: a layer without a session must opt in explicitly rather than silently substituting ConfigOptions::default(). All new dispatch-trait methods are defaulted, so existing implementors keep compiling. Part of apache#22418; groundwork for apache#22430. Co-Authored-By: Claude Fable 5 <[email protected]> Claude-Session: https://claude.ai/code/session_018MmNo1SfPHXZNaMW26KmaG
1 parent cd47b95 commit 10ef51a

1 file changed

Lines changed: 131 additions & 4 deletions

File tree

‎datafusion/physical-expr-common/src/physical_expr.rs‎

Lines changed: 131 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -519,12 +519,18 @@ pub trait PhysicalExpr: Any + Send + Sync + Display + Debug + DynEq + DynHash {
519519
/// `datafusion-proto`, which is what lets `physical-expr-common` stay free
520520
/// of `datafusion-proto` as a dep.
521521
///
522-
/// More specialized helpers (e.g. encoding UDFs/UDAFs/UDWFs through the
523-
/// extension codec) can be added to the context as expressions migrate;
524-
/// today they're not required because the encoder forwards to the existing
525-
/// codec via the proto converter.
522+
/// Beyond child-expression encoding, the context exposes *capabilities* of
523+
/// the serialization layer through generic, fully-typed accessors — see
524+
/// [`proto_encode::PhysicalExprEncodeCtx::encode_function`], which routes
525+
/// function objects (e.g. `datafusion_expr::ScalarUDF`) through the layer's
526+
/// extension codec. Those function types live *above* this crate in the
527+
/// dependency graph, so the accessors are generic over the requested type
528+
/// and the dispatch trait carries a type-erased channel internally; call
529+
/// sites never see the erasure. Which types a layer supports is part of its
530+
/// contract — requesting an unsupported one returns an error naming it.
526531
#[cfg(feature = "proto")]
527532
pub mod proto_encode {
533+
use std::any::Any;
528534
use std::sync::Arc;
529535

530536
use datafusion_common::Result;
@@ -574,6 +580,29 @@ pub mod proto_encode {
574580
.map(|expr| self.encode_child(expr))
575581
.collect()
576582
}
583+
584+
/// Serialize a function object through the serialization layer's
585+
/// extension codec, returning the opaque payload for the node's
586+
/// `fun_definition`-style field — `None` when the function can be
587+
/// resolved by name alone on the decoding side (the common case for
588+
/// built-in and registered functions).
589+
///
590+
/// Which function types the layer supports is part of its contract:
591+
/// `datafusion-proto` supports `datafusion_expr::ScalarUDF` (with
592+
/// aggregate/window UDFs to follow as those expressions migrate).
593+
/// Requesting an unsupported type returns an error naming it. Call
594+
/// sites are fully typed — e.g. `ctx.encode_function(&*self.fun)`
595+
/// with `fun: Arc<ScalarUDF>` — because the generic carries the type
596+
/// across the internal erased channel; function types cannot be
597+
/// named in this crate directly, as they live above it in the
598+
/// dependency graph.
599+
pub fn encode_function<F: Any + Send + Sync>(
600+
&self,
601+
function: &F,
602+
) -> Result<Option<Vec<u8>>> {
603+
self.encoder
604+
.encode_function_erased(function, std::any::type_name::<F>())
605+
}
577606
}
578607

579608
/// Internal dispatch trait. Implementors live in `datafusion-proto` and
@@ -583,6 +612,25 @@ pub mod proto_encode {
583612
pub trait PhysicalExprEncode {
584613
/// Encode an expression to a protobuf node.
585614
fn encode(&self, expr: &Arc<dyn PhysicalExpr>) -> Result<PhysicalExprNode>;
615+
616+
/// Erased channel behind [`PhysicalExprEncodeCtx::encode_function`].
617+
///
618+
/// Implementations downcast `function` to each function type they
619+
/// support (e.g. `datafusion_expr::ScalarUDF` routed through
620+
/// `PhysicalExtensionCodec::try_encode_udf`) and error — naming
621+
/// `type_name` — for the rest. The default supports none, so
622+
/// encoders that never see function-bearing expressions need not
623+
/// implement it.
624+
fn encode_function_erased(
625+
&self,
626+
_function: &(dyn Any + Send + Sync),
627+
type_name: &'static str,
628+
) -> Result<Option<Vec<u8>>> {
629+
datafusion_common::internal_err!(
630+
"this serialization layer does not support encoding function \
631+
objects (requested `{type_name}`)"
632+
)
633+
}
586634
}
587635
}
588636

@@ -611,10 +659,12 @@ pub mod proto_encode {
611659
/// [`PhysicalExprNode`]: datafusion_proto_models::protobuf::PhysicalExprNode
612660
#[cfg(feature = "proto")]
613661
pub mod proto_decode {
662+
use std::any::{Any, TypeId};
614663
use std::sync::Arc;
615664

616665
use arrow::datatypes::Schema;
617666
use datafusion_common::Result;
667+
use datafusion_common::config::ConfigOptions;
618668
use datafusion_proto_models::protobuf::PhysicalExprNode;
619669

620670
use super::PhysicalExpr;
@@ -733,6 +783,49 @@ pub mod proto_decode {
733783
{
734784
nodes.into_iter().map(|node| self.decode(node)).collect()
735785
}
786+
787+
/// Resolve a function object from its name and the payload written
788+
/// by [`encode_function`] on the encoding side (`None` if none was
789+
/// written). The inverse of [`encode_function`]; the same contract
790+
/// applies — which function types the deserialization layer supports
791+
/// is part of its contract, and requesting an unsupported one
792+
/// returns an error naming it.
793+
///
794+
/// `datafusion-proto` resolves `datafusion_expr::ScalarUDF` by
795+
/// preferring the codec for explicit payloads and falling back to
796+
/// the session's function registry (then the codec with an empty
797+
/// payload) for by-name references. Call sites are fully typed:
798+
///
799+
/// ```ignore
800+
/// let udf: Arc<ScalarUDF> = ctx.decode_function(&node.name, node.fun_definition.as_deref())?;
801+
/// ```
802+
///
803+
/// [`encode_function`]: super::proto_encode::PhysicalExprEncodeCtx::encode_function
804+
pub fn decode_function<F: Any + Send + Sync>(
805+
&self,
806+
name: &str,
807+
payload: Option<&[u8]>,
808+
) -> Result<Arc<F>> {
809+
let type_name = std::any::type_name::<F>();
810+
self.decoder
811+
.decode_function_erased(name, payload, TypeId::of::<F>(), type_name)?
812+
.downcast::<F>()
813+
.map_err(|_| {
814+
datafusion_common::internal_datafusion_err!(
815+
"deserialization layer resolved function '{name}' to a \
816+
value that is not a `{type_name}`"
817+
)
818+
})
819+
}
820+
821+
/// Session configuration for expressions that carry it (e.g.
822+
/// `ScalarFunctionExpr`). Errors when the deserialization layer has
823+
/// no session — deliberately not defaulted to
824+
/// [`ConfigOptions::default`], which would silently detach a decoded
825+
/// expression from the session's actual configuration.
826+
pub fn config_options(&self) -> Result<Arc<ConfigOptions>> {
827+
self.decoder.config_options()
828+
}
736829
}
737830

738831
/// Unwrap a required non-expression proto field.
@@ -774,6 +867,40 @@ pub mod proto_decode {
774867
node: &PhysicalExprNode,
775868
schema: &Schema,
776869
) -> Result<Arc<dyn PhysicalExpr>>;
870+
871+
/// Erased channel behind [`PhysicalExprDecodeCtx::decode_function`].
872+
///
873+
/// Implementations route on `requested` (the [`TypeId`] of the
874+
/// function type the caller asked for, e.g.
875+
/// `datafusion_expr::ScalarUDF`) and must return a value of exactly
876+
/// that type — [`PhysicalExprDecodeCtx::decode_function`] verifies
877+
/// the downcast and turns a mismatch into an internal error. Error
878+
/// messages should name `type_name`. The default supports no
879+
/// function types, so decoders that never see function-bearing
880+
/// expressions need not implement it.
881+
fn decode_function_erased(
882+
&self,
883+
name: &str,
884+
_payload: Option<&[u8]>,
885+
_requested: TypeId,
886+
type_name: &'static str,
887+
) -> Result<Arc<dyn Any + Send + Sync>> {
888+
datafusion_common::internal_err!(
889+
"this deserialization layer does not support resolving \
890+
function objects (requested `{type_name}` for '{name}')"
891+
)
892+
}
893+
894+
/// Session configuration handed to decoded expressions that carry
895+
/// it. The default errors: a layer without a session must opt in
896+
/// explicitly rather than silently substituting
897+
/// [`ConfigOptions::default`].
898+
fn config_options(&self) -> Result<Arc<ConfigOptions>> {
899+
datafusion_common::internal_err!(
900+
"this deserialization layer does not provide session \
901+
configuration"
902+
)
903+
}
777904
}
778905
}
779906

0 commit comments

Comments
 (0)