@@ -2175,6 +2175,373 @@ impl ExecutionPlan for AggregateExec {
21752175
21762176 Ok ( result)
21772177 }
2178+
2179+ #[ cfg( feature = "proto" ) ]
2180+ fn try_to_proto (
2181+ & self ,
2182+ ctx : & crate :: proto:: ExecutionPlanEncodeCtx < ' _ > ,
2183+ ) -> Result < Option < datafusion_proto_models:: protobuf:: PhysicalPlanNode > > {
2184+ use datafusion_proto_models:: protobuf;
2185+
2186+ let input = ctx. encode_child ( self . input ( ) ) ?;
2187+ let group_by = self . group_expr ( ) ;
2188+ let group_expr =
2189+ ctx. encode_expressions ( group_by. expr ( ) . iter ( ) . map ( |( expr, _) | expr) ) ?;
2190+ let group_expr_name = group_by
2191+ . expr ( )
2192+ . iter ( )
2193+ . map ( |( _, name) | name. to_owned ( ) )
2194+ . collect ( ) ;
2195+ let null_expr =
2196+ ctx. encode_expressions ( group_by. null_expr ( ) . iter ( ) . map ( |( expr, _) | expr) ) ?;
2197+ let groups = group_by. groups ( ) . iter ( ) . flatten ( ) . copied ( ) . collect ( ) ;
2198+ let aggr_expr = self
2199+ . aggr_expr ( )
2200+ . iter ( )
2201+ . map ( |expr| encode_aggregate_expr ( expr, ctx) )
2202+ . collect :: < Result < Vec < _ > > > ( ) ?;
2203+ let aggr_expr_name = self
2204+ . aggr_expr ( )
2205+ . iter ( )
2206+ . map ( |expr| expr. name ( ) . to_string ( ) )
2207+ . collect ( ) ;
2208+ let filter_expr = self
2209+ . filter_expr ( )
2210+ . iter ( )
2211+ . map ( |filter| {
2212+ Ok ( protobuf:: MaybeFilter {
2213+ expr : filter
2214+ . as_ref ( )
2215+ . map ( |expr| ctx. encode_expr ( expr) )
2216+ . transpose ( ) ?,
2217+ } )
2218+ } )
2219+ . collect :: < Result < Vec < _ > > > ( ) ?;
2220+ let mode = match self . mode ( ) {
2221+ AggregateMode :: Partial => protobuf:: AggregateMode :: Partial ,
2222+ AggregateMode :: Final => protobuf:: AggregateMode :: Final ,
2223+ AggregateMode :: FinalPartitioned => protobuf:: AggregateMode :: FinalPartitioned ,
2224+ AggregateMode :: Single => protobuf:: AggregateMode :: Single ,
2225+ AggregateMode :: SinglePartitioned => {
2226+ protobuf:: AggregateMode :: SinglePartitioned
2227+ }
2228+ AggregateMode :: PartialReduce => protobuf:: AggregateMode :: PartialReduce ,
2229+ } ;
2230+ let limit = self . limit_options ( ) . map ( |options| protobuf:: AggLimit {
2231+ limit : options. limit ( ) as u64 ,
2232+ descending : options. descending ( ) ,
2233+ } ) ;
2234+ let dynamic_filter = match self . dynamic_filter_expr ( ) {
2235+ Some ( filter) => {
2236+ let expr: Arc < dyn PhysicalExpr > =
2237+ Arc :: clone ( filter) as Arc < dyn PhysicalExpr > ;
2238+ Some ( ctx. encode_expr ( & expr) ?)
2239+ }
2240+ None => None ,
2241+ } ;
2242+
2243+ Ok ( Some ( protobuf:: PhysicalPlanNode {
2244+ physical_plan_type : Some (
2245+ protobuf:: physical_plan_node:: PhysicalPlanType :: Aggregate ( Box :: new (
2246+ protobuf:: AggregateExecNode {
2247+ group_expr,
2248+ group_expr_name,
2249+ aggr_expr,
2250+ filter_expr,
2251+ aggr_expr_name,
2252+ mode : mode as i32 ,
2253+ input : Some ( Box :: new ( input) ) ,
2254+ input_schema : Some ( self . input_schema ( ) . as_ref ( ) . try_into ( ) ?) ,
2255+ null_expr,
2256+ groups,
2257+ limit,
2258+ has_grouping_set : group_by. has_grouping_set ( ) ,
2259+ dynamic_filter,
2260+ } ,
2261+ ) ) ,
2262+ ) ,
2263+ } ) )
2264+ }
2265+ }
2266+
2267+ #[ cfg( feature = "proto" ) ]
2268+ const HUMAN_DISPLAY_ALIAS_PREFIX : & str = "\u{1f} datafusion_human_display_alias_v1:" ;
2269+
2270+ #[ cfg( feature = "proto" ) ]
2271+ fn encode_human_display_alias ( human_display : & str , alias : & str ) -> String {
2272+ format ! (
2273+ "{HUMAN_DISPLAY_ALIAS_PREFIX}{}:{alias}{human_display}" ,
2274+ alias. len( )
2275+ )
2276+ }
2277+
2278+ #[ cfg( feature = "proto" ) ]
2279+ fn split_human_display_alias < ' a > (
2280+ human_display : & ' a str ,
2281+ name : & ' a str ,
2282+ ) -> ( & ' a str , Option < & ' a str > ) {
2283+ if let Some ( encoded) = human_display. strip_prefix ( HUMAN_DISPLAY_ALIAS_PREFIX )
2284+ && let Some ( ( alias_len, encoded) ) = encoded. split_once ( ':' )
2285+ && let Ok ( alias_len) = alias_len. parse :: < usize > ( )
2286+ && let Some ( alias) = encoded. get ( ..alias_len)
2287+ && let Some ( human_display) = encoded. get ( alias_len..)
2288+ && alias == name
2289+ && !human_display. is_empty ( )
2290+ {
2291+ return ( human_display, Some ( alias) ) ;
2292+ }
2293+
2294+ ( human_display, None )
2295+ }
2296+
2297+ #[ cfg( feature = "proto" ) ]
2298+ fn encode_aggregate_expr (
2299+ aggr_expr : & Arc < AggregateFunctionExpr > ,
2300+ ctx : & crate :: proto:: ExecutionPlanEncodeCtx < ' _ > ,
2301+ ) -> Result < datafusion_proto_models:: protobuf:: PhysicalExprNode > {
2302+ use datafusion_proto_models:: protobuf;
2303+
2304+ let expressions = aggr_expr. expressions ( ) ;
2305+ let expr = ctx. encode_expressions ( expressions. iter ( ) ) ?;
2306+ let ordering_req = aggr_expr
2307+ . order_bys ( )
2308+ . iter ( )
2309+ . map ( |sort_expr| {
2310+ Ok ( protobuf:: PhysicalSortExprNode {
2311+ expr : Some ( Box :: new ( ctx. encode_expr ( & sort_expr. expr ) ?) ) ,
2312+ asc : !sort_expr. options . descending ,
2313+ nulls_first : sort_expr. options . nulls_first ,
2314+ } )
2315+ } )
2316+ . collect :: < Result < Vec < _ > > > ( ) ?;
2317+ let name = aggr_expr. fun ( ) . name ( ) . to_string ( ) ;
2318+ let fun_definition = ctx. encode_udaf ( aggr_expr. fun ( ) ) ?;
2319+ let human_display = match ( aggr_expr. human_display ( ) , aggr_expr. human_display_alias ( ) )
2320+ {
2321+ ( Some ( display) , Some ( alias) ) => encode_human_display_alias ( display, alias) ,
2322+ ( Some ( display) , None ) => display. to_string ( ) ,
2323+ ( None , _) => String :: new ( ) ,
2324+ } ;
2325+
2326+ Ok ( protobuf:: PhysicalExprNode {
2327+ expr_id : None ,
2328+ expr_type : Some ( protobuf:: physical_expr_node:: ExprType :: AggregateExpr (
2329+ protobuf:: PhysicalAggregateExprNode {
2330+ aggregate_function : Some (
2331+ protobuf:: physical_aggregate_expr_node:: AggregateFunction :: UserDefinedAggrFunction ( name) ,
2332+ ) ,
2333+ expr,
2334+ ordering_req,
2335+ distinct : aggr_expr. is_distinct ( ) ,
2336+ ignore_nulls : aggr_expr. ignore_nulls ( ) ,
2337+ fun_definition,
2338+ human_display,
2339+ } ,
2340+ ) ) ,
2341+ } )
2342+ }
2343+
2344+ #[ cfg( feature = "proto" ) ]
2345+ impl AggregateExec {
2346+ /// Reconstruct an [`AggregateExec`] from its protobuf representation.
2347+ ///
2348+ /// Grouping expressions are decoded against the child schema. Aggregate
2349+ /// arguments, ordering, filters, and the dynamic filter are decoded against
2350+ /// the aggregate input schema carried in the protobuf node.
2351+ pub fn try_from_proto (
2352+ node : & datafusion_proto_models:: protobuf:: PhysicalPlanNode ,
2353+ ctx : & crate :: proto:: ExecutionPlanDecodeCtx < ' _ > ,
2354+ ) -> Result < Arc < dyn ExecutionPlan > > {
2355+ use datafusion_physical_expr:: PhysicalSortExpr ;
2356+ use datafusion_physical_expr:: aggregate:: AggregateExprBuilder ;
2357+ use datafusion_proto_models:: protobuf;
2358+ use protobuf:: physical_aggregate_expr_node:: AggregateFunction ;
2359+ use protobuf:: physical_expr_node:: ExprType ;
2360+
2361+ let hash_agg = crate :: expect_plan_variant!(
2362+ node,
2363+ protobuf:: physical_plan_node:: PhysicalPlanType :: Aggregate ,
2364+ "AggregateExec" ,
2365+ ) ;
2366+ let input = ctx. decode_required_child (
2367+ hash_agg. input . as_deref ( ) ,
2368+ "AggregateExec" ,
2369+ "input" ,
2370+ ) ?;
2371+ let mode = protobuf:: AggregateMode :: try_from ( hash_agg. mode ) . map_err ( |_| {
2372+ datafusion_common:: internal_datafusion_err!(
2373+ "Received an AggregateNode message with unknown AggregateMode {}" ,
2374+ hash_agg. mode
2375+ )
2376+ } ) ?;
2377+ let mode = match mode {
2378+ protobuf:: AggregateMode :: Partial => AggregateMode :: Partial ,
2379+ protobuf:: AggregateMode :: Final => AggregateMode :: Final ,
2380+ protobuf:: AggregateMode :: FinalPartitioned => AggregateMode :: FinalPartitioned ,
2381+ protobuf:: AggregateMode :: Single => AggregateMode :: Single ,
2382+ protobuf:: AggregateMode :: SinglePartitioned => {
2383+ AggregateMode :: SinglePartitioned
2384+ }
2385+ protobuf:: AggregateMode :: PartialReduce => AggregateMode :: PartialReduce ,
2386+ } ;
2387+ let num_expr = hash_agg. group_expr . len ( ) ;
2388+ let child_schema = input. schema ( ) ;
2389+ let group_expr = hash_agg
2390+ . group_expr
2391+ . iter ( )
2392+ . zip ( hash_agg. group_expr_name . iter ( ) )
2393+ . map ( |( expr, name) | {
2394+ Ok ( (
2395+ ctx. decode_expr ( expr, child_schema. as_ref ( ) ) ?,
2396+ name. to_string ( ) ,
2397+ ) )
2398+ } )
2399+ . collect :: < Result < Vec < _ > > > ( ) ?;
2400+ let null_expr = hash_agg
2401+ . null_expr
2402+ . iter ( )
2403+ . zip ( hash_agg. group_expr_name . iter ( ) )
2404+ . map ( |( expr, name) | {
2405+ Ok ( (
2406+ ctx. decode_expr ( expr, child_schema. as_ref ( ) ) ?,
2407+ name. to_string ( ) ,
2408+ ) )
2409+ } )
2410+ . collect :: < Result < Vec < _ > > > ( ) ?;
2411+ let groups = if hash_agg. groups . is_empty ( ) {
2412+ vec ! [ ]
2413+ } else {
2414+ hash_agg
2415+ . groups
2416+ . chunks ( num_expr)
2417+ . map ( |group| group. to_vec ( ) )
2418+ . collect ( )
2419+ } ;
2420+ let input_schema = hash_agg. input_schema . as_ref ( ) . ok_or_else ( || {
2421+ datafusion_common:: internal_datafusion_err!(
2422+ "input_schema in AggregateNode is missing."
2423+ )
2424+ } ) ?;
2425+ let input_schema: SchemaRef = SchemaRef :: new ( input_schema. try_into ( ) ?) ;
2426+ let filter_expr = hash_agg
2427+ . filter_expr
2428+ . iter ( )
2429+ . map ( |filter| {
2430+ filter
2431+ . expr
2432+ . as_ref ( )
2433+ . map ( |expr| ctx. decode_expr ( expr, input_schema. as_ref ( ) ) )
2434+ . transpose ( )
2435+ } )
2436+ . collect :: < Result < Vec < _ > > > ( ) ?;
2437+ let aggr_expr = hash_agg
2438+ . aggr_expr
2439+ . iter ( )
2440+ . zip ( hash_agg. aggr_expr_name . iter ( ) )
2441+ . map ( |( expr, name) | {
2442+ let expr_type = expr. expr_type . as_ref ( ) . ok_or_else ( || {
2443+ datafusion_common:: internal_datafusion_err!(
2444+ "Unexpected empty aggregate physical expression"
2445+ )
2446+ } ) ?;
2447+ let ExprType :: AggregateExpr ( aggregate) = expr_type else {
2448+ return internal_err ! (
2449+ "Invalid aggregate expression for AggregateExec"
2450+ ) ;
2451+ } ;
2452+ let args = aggregate
2453+ . expr
2454+ . iter ( )
2455+ . map ( |expr| ctx. decode_expr ( expr, input_schema. as_ref ( ) ) )
2456+ . collect :: < Result < Vec < _ > > > ( ) ?;
2457+ let order_by = aggregate
2458+ . ordering_req
2459+ . iter ( )
2460+ . map ( |sort_expr| {
2461+ let expr = sort_expr. expr . as_deref ( ) . ok_or_else ( || {
2462+ datafusion_common:: internal_datafusion_err!(
2463+ "AggregateExec ordering expression is missing its inner expr"
2464+ )
2465+ } ) ?;
2466+ Ok ( PhysicalSortExpr {
2467+ expr : ctx. decode_expr ( expr, input_schema. as_ref ( ) ) ?,
2468+ options : arrow:: compute:: SortOptions {
2469+ descending : !sort_expr. asc ,
2470+ nulls_first : sort_expr. nulls_first ,
2471+ } ,
2472+ } )
2473+ } )
2474+ . collect :: < Result < Vec < _ > > > ( ) ?;
2475+ let Some ( AggregateFunction :: UserDefinedAggrFunction ( udaf_name) ) =
2476+ aggregate. aggregate_function . as_ref ( )
2477+ else {
2478+ return internal_err ! (
2479+ "Invalid AggregateExpr, missing aggregate_function"
2480+ ) ;
2481+ } ;
2482+ let udaf = ctx. decode_udaf (
2483+ udaf_name,
2484+ aggregate. fun_definition . as_deref ( ) ,
2485+ ) ?;
2486+ let ( human_display, human_display_alias) =
2487+ split_human_display_alias ( & aggregate. human_display , name) ;
2488+ let builder = AggregateExprBuilder :: new ( udaf, args)
2489+ . schema ( Arc :: clone ( & input_schema) )
2490+ . alias ( name)
2491+ . with_ignore_nulls ( aggregate. ignore_nulls )
2492+ . with_distinct ( aggregate. distinct )
2493+ . order_by ( order_by)
2494+ . human_display ( human_display) ;
2495+ let builder = if let Some ( alias) = human_display_alias {
2496+ builder. human_display_alias ( alias)
2497+ } else {
2498+ builder
2499+ } ;
2500+ builder. build ( ) . map ( Arc :: new)
2501+ } )
2502+ . collect :: < Result < Vec < _ > > > ( ) ?;
2503+ let aggregate = AggregateExec :: try_new (
2504+ mode,
2505+ PhysicalGroupBy :: new (
2506+ group_expr,
2507+ null_expr,
2508+ groups,
2509+ hash_agg. has_grouping_set ,
2510+ ) ,
2511+ aggr_expr,
2512+ filter_expr,
2513+ input,
2514+ Arc :: clone ( & input_schema) ,
2515+ ) ?;
2516+ let aggregate = if let Some ( limit) = & hash_agg. limit {
2517+ let options = match limit. descending {
2518+ Some ( descending) => {
2519+ LimitOptions :: new_with_order ( limit. limit as usize , descending)
2520+ }
2521+ None => LimitOptions :: new ( limit. limit as usize ) ,
2522+ } ;
2523+ aggregate. with_limit_options ( Some ( options) )
2524+ } else {
2525+ aggregate
2526+ } ;
2527+ let aggregate = if let Some ( dynamic_filter) = & hash_agg. dynamic_filter {
2528+ let dynamic_filter =
2529+ ctx. decode_expr ( dynamic_filter, input_schema. as_ref ( ) ) ?;
2530+ let dynamic_filter = ( dynamic_filter
2531+ as Arc < dyn std:: any:: Any + Send + Sync > )
2532+ . downcast :: < DynamicFilterPhysicalExpr > ( )
2533+ . map_err ( |_| {
2534+ datafusion_common:: internal_datafusion_err!(
2535+ "AggregateExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr"
2536+ )
2537+ } ) ?;
2538+ aggregate. with_dynamic_filter_expr ( dynamic_filter) ?
2539+ } else {
2540+ aggregate
2541+ } ;
2542+
2543+ Ok ( Arc :: new ( aggregate) )
2544+ }
21782545}
21792546
21802547/// Creates the output schema for an [`AggregateExec`] containing the group by columns followed
0 commit comments