@@ -2168,6 +2168,373 @@ impl ExecutionPlan for AggregateExec {
21682168
21692169 Ok ( result)
21702170 }
2171+
2172+ #[ cfg( feature = "proto" ) ]
2173+ fn try_to_proto (
2174+ & self ,
2175+ ctx : & crate :: proto:: ExecutionPlanEncodeCtx < ' _ > ,
2176+ ) -> Result < Option < datafusion_proto_models:: protobuf:: PhysicalPlanNode > > {
2177+ use datafusion_proto_models:: protobuf;
2178+
2179+ let input = ctx. encode_child ( self . input ( ) ) ?;
2180+ let group_by = self . group_expr ( ) ;
2181+ let group_expr =
2182+ ctx. encode_expressions ( group_by. expr ( ) . iter ( ) . map ( |( expr, _) | expr) ) ?;
2183+ let group_expr_name = group_by
2184+ . expr ( )
2185+ . iter ( )
2186+ . map ( |( _, name) | name. to_owned ( ) )
2187+ . collect ( ) ;
2188+ let null_expr =
2189+ ctx. encode_expressions ( group_by. null_expr ( ) . iter ( ) . map ( |( expr, _) | expr) ) ?;
2190+ let groups = group_by. groups ( ) . iter ( ) . flatten ( ) . copied ( ) . collect ( ) ;
2191+ let aggr_expr = self
2192+ . aggr_expr ( )
2193+ . iter ( )
2194+ . map ( |expr| encode_aggregate_expr ( expr, ctx) )
2195+ . collect :: < Result < Vec < _ > > > ( ) ?;
2196+ let aggr_expr_name = self
2197+ . aggr_expr ( )
2198+ . iter ( )
2199+ . map ( |expr| expr. name ( ) . to_string ( ) )
2200+ . collect ( ) ;
2201+ let filter_expr = self
2202+ . filter_expr ( )
2203+ . iter ( )
2204+ . map ( |filter| {
2205+ Ok ( protobuf:: MaybeFilter {
2206+ expr : filter
2207+ . as_ref ( )
2208+ . map ( |expr| ctx. encode_expr ( expr) )
2209+ . transpose ( ) ?,
2210+ } )
2211+ } )
2212+ . collect :: < Result < Vec < _ > > > ( ) ?;
2213+ let mode = match self . mode ( ) {
2214+ AggregateMode :: Partial => protobuf:: AggregateMode :: Partial ,
2215+ AggregateMode :: Final => protobuf:: AggregateMode :: Final ,
2216+ AggregateMode :: FinalPartitioned => protobuf:: AggregateMode :: FinalPartitioned ,
2217+ AggregateMode :: Single => protobuf:: AggregateMode :: Single ,
2218+ AggregateMode :: SinglePartitioned => {
2219+ protobuf:: AggregateMode :: SinglePartitioned
2220+ }
2221+ AggregateMode :: PartialReduce => protobuf:: AggregateMode :: PartialReduce ,
2222+ } ;
2223+ let limit = self . limit_options ( ) . map ( |options| protobuf:: AggLimit {
2224+ limit : options. limit ( ) as u64 ,
2225+ descending : options. descending ( ) ,
2226+ } ) ;
2227+ let dynamic_filter = match self . dynamic_filter_expr ( ) {
2228+ Some ( filter) => {
2229+ let expr: Arc < dyn PhysicalExpr > =
2230+ Arc :: clone ( filter) as Arc < dyn PhysicalExpr > ;
2231+ Some ( ctx. encode_expr ( & expr) ?)
2232+ }
2233+ None => None ,
2234+ } ;
2235+
2236+ Ok ( Some ( protobuf:: PhysicalPlanNode {
2237+ physical_plan_type : Some (
2238+ protobuf:: physical_plan_node:: PhysicalPlanType :: Aggregate ( Box :: new (
2239+ protobuf:: AggregateExecNode {
2240+ group_expr,
2241+ group_expr_name,
2242+ aggr_expr,
2243+ filter_expr,
2244+ aggr_expr_name,
2245+ mode : mode as i32 ,
2246+ input : Some ( Box :: new ( input) ) ,
2247+ input_schema : Some ( self . input_schema ( ) . as_ref ( ) . try_into ( ) ?) ,
2248+ null_expr,
2249+ groups,
2250+ limit,
2251+ has_grouping_set : group_by. has_grouping_set ( ) ,
2252+ dynamic_filter,
2253+ } ,
2254+ ) ) ,
2255+ ) ,
2256+ } ) )
2257+ }
2258+ }
2259+
2260+ #[ cfg( feature = "proto" ) ]
2261+ const HUMAN_DISPLAY_ALIAS_PREFIX : & str = "\u{1f} datafusion_human_display_alias_v1:" ;
2262+
2263+ #[ cfg( feature = "proto" ) ]
2264+ fn encode_human_display_alias ( human_display : & str , alias : & str ) -> String {
2265+ format ! (
2266+ "{HUMAN_DISPLAY_ALIAS_PREFIX}{}:{alias}{human_display}" ,
2267+ alias. len( )
2268+ )
2269+ }
2270+
2271+ #[ cfg( feature = "proto" ) ]
2272+ fn split_human_display_alias < ' a > (
2273+ human_display : & ' a str ,
2274+ name : & ' a str ,
2275+ ) -> ( & ' a str , Option < & ' a str > ) {
2276+ if let Some ( encoded) = human_display. strip_prefix ( HUMAN_DISPLAY_ALIAS_PREFIX )
2277+ && let Some ( ( alias_len, encoded) ) = encoded. split_once ( ':' )
2278+ && let Ok ( alias_len) = alias_len. parse :: < usize > ( )
2279+ && let Some ( alias) = encoded. get ( ..alias_len)
2280+ && let Some ( human_display) = encoded. get ( alias_len..)
2281+ && alias == name
2282+ && !human_display. is_empty ( )
2283+ {
2284+ return ( human_display, Some ( alias) ) ;
2285+ }
2286+
2287+ ( human_display, None )
2288+ }
2289+
2290+ #[ cfg( feature = "proto" ) ]
2291+ fn encode_aggregate_expr (
2292+ aggr_expr : & Arc < AggregateFunctionExpr > ,
2293+ ctx : & crate :: proto:: ExecutionPlanEncodeCtx < ' _ > ,
2294+ ) -> Result < datafusion_proto_models:: protobuf:: PhysicalExprNode > {
2295+ use datafusion_proto_models:: protobuf;
2296+
2297+ let expressions = aggr_expr. expressions ( ) ;
2298+ let expr = ctx. encode_expressions ( expressions. iter ( ) ) ?;
2299+ let ordering_req = aggr_expr
2300+ . order_bys ( )
2301+ . iter ( )
2302+ . map ( |sort_expr| {
2303+ Ok ( protobuf:: PhysicalSortExprNode {
2304+ expr : Some ( Box :: new ( ctx. encode_expr ( & sort_expr. expr ) ?) ) ,
2305+ asc : !sort_expr. options . descending ,
2306+ nulls_first : sort_expr. options . nulls_first ,
2307+ } )
2308+ } )
2309+ . collect :: < Result < Vec < _ > > > ( ) ?;
2310+ let name = aggr_expr. fun ( ) . name ( ) . to_string ( ) ;
2311+ let fun_definition = ctx. encode_udaf ( aggr_expr. fun ( ) ) ?;
2312+ let human_display = match ( aggr_expr. human_display ( ) , aggr_expr. human_display_alias ( ) )
2313+ {
2314+ ( Some ( display) , Some ( alias) ) => encode_human_display_alias ( display, alias) ,
2315+ ( Some ( display) , None ) => display. to_string ( ) ,
2316+ ( None , _) => String :: new ( ) ,
2317+ } ;
2318+
2319+ Ok ( protobuf:: PhysicalExprNode {
2320+ expr_id : None ,
2321+ expr_type : Some ( protobuf:: physical_expr_node:: ExprType :: AggregateExpr (
2322+ protobuf:: PhysicalAggregateExprNode {
2323+ aggregate_function : Some (
2324+ protobuf:: physical_aggregate_expr_node:: AggregateFunction :: UserDefinedAggrFunction ( name) ,
2325+ ) ,
2326+ expr,
2327+ ordering_req,
2328+ distinct : aggr_expr. is_distinct ( ) ,
2329+ ignore_nulls : aggr_expr. ignore_nulls ( ) ,
2330+ fun_definition,
2331+ human_display,
2332+ } ,
2333+ ) ) ,
2334+ } )
2335+ }
2336+
2337+ #[ cfg( feature = "proto" ) ]
2338+ impl AggregateExec {
2339+ /// Reconstruct an [`AggregateExec`] from its protobuf representation.
2340+ ///
2341+ /// Grouping expressions are decoded against the child schema. Aggregate
2342+ /// arguments, ordering, filters, and the dynamic filter are decoded against
2343+ /// the aggregate input schema carried in the protobuf node.
2344+ pub fn try_from_proto (
2345+ node : & datafusion_proto_models:: protobuf:: PhysicalPlanNode ,
2346+ ctx : & crate :: proto:: ExecutionPlanDecodeCtx < ' _ > ,
2347+ ) -> Result < Arc < dyn ExecutionPlan > > {
2348+ use datafusion_physical_expr:: PhysicalSortExpr ;
2349+ use datafusion_physical_expr:: aggregate:: AggregateExprBuilder ;
2350+ use datafusion_proto_models:: protobuf;
2351+ use protobuf:: physical_aggregate_expr_node:: AggregateFunction ;
2352+ use protobuf:: physical_expr_node:: ExprType ;
2353+
2354+ let hash_agg = crate :: expect_plan_variant!(
2355+ node,
2356+ protobuf:: physical_plan_node:: PhysicalPlanType :: Aggregate ,
2357+ "AggregateExec" ,
2358+ ) ;
2359+ let input = ctx. decode_required_child (
2360+ hash_agg. input . as_deref ( ) ,
2361+ "AggregateExec" ,
2362+ "input" ,
2363+ ) ?;
2364+ let mode = protobuf:: AggregateMode :: try_from ( hash_agg. mode ) . map_err ( |_| {
2365+ datafusion_common:: internal_datafusion_err!(
2366+ "Received an AggregateNode message with unknown AggregateMode {}" ,
2367+ hash_agg. mode
2368+ )
2369+ } ) ?;
2370+ let mode = match mode {
2371+ protobuf:: AggregateMode :: Partial => AggregateMode :: Partial ,
2372+ protobuf:: AggregateMode :: Final => AggregateMode :: Final ,
2373+ protobuf:: AggregateMode :: FinalPartitioned => AggregateMode :: FinalPartitioned ,
2374+ protobuf:: AggregateMode :: Single => AggregateMode :: Single ,
2375+ protobuf:: AggregateMode :: SinglePartitioned => {
2376+ AggregateMode :: SinglePartitioned
2377+ }
2378+ protobuf:: AggregateMode :: PartialReduce => AggregateMode :: PartialReduce ,
2379+ } ;
2380+ let num_expr = hash_agg. group_expr . len ( ) ;
2381+ let child_schema = input. schema ( ) ;
2382+ let group_expr = hash_agg
2383+ . group_expr
2384+ . iter ( )
2385+ . zip ( hash_agg. group_expr_name . iter ( ) )
2386+ . map ( |( expr, name) | {
2387+ Ok ( (
2388+ ctx. decode_expr ( expr, child_schema. as_ref ( ) ) ?,
2389+ name. to_string ( ) ,
2390+ ) )
2391+ } )
2392+ . collect :: < Result < Vec < _ > > > ( ) ?;
2393+ let null_expr = hash_agg
2394+ . null_expr
2395+ . iter ( )
2396+ . zip ( hash_agg. group_expr_name . iter ( ) )
2397+ . map ( |( expr, name) | {
2398+ Ok ( (
2399+ ctx. decode_expr ( expr, child_schema. as_ref ( ) ) ?,
2400+ name. to_string ( ) ,
2401+ ) )
2402+ } )
2403+ . collect :: < Result < Vec < _ > > > ( ) ?;
2404+ let groups = if hash_agg. groups . is_empty ( ) {
2405+ vec ! [ ]
2406+ } else {
2407+ hash_agg
2408+ . groups
2409+ . chunks ( num_expr)
2410+ . map ( |group| group. to_vec ( ) )
2411+ . collect ( )
2412+ } ;
2413+ let input_schema = hash_agg. input_schema . as_ref ( ) . ok_or_else ( || {
2414+ datafusion_common:: internal_datafusion_err!(
2415+ "input_schema in AggregateNode is missing."
2416+ )
2417+ } ) ?;
2418+ let input_schema: SchemaRef = SchemaRef :: new ( input_schema. try_into ( ) ?) ;
2419+ let filter_expr = hash_agg
2420+ . filter_expr
2421+ . iter ( )
2422+ . map ( |filter| {
2423+ filter
2424+ . expr
2425+ . as_ref ( )
2426+ . map ( |expr| ctx. decode_expr ( expr, input_schema. as_ref ( ) ) )
2427+ . transpose ( )
2428+ } )
2429+ . collect :: < Result < Vec < _ > > > ( ) ?;
2430+ let aggr_expr = hash_agg
2431+ . aggr_expr
2432+ . iter ( )
2433+ . zip ( hash_agg. aggr_expr_name . iter ( ) )
2434+ . map ( |( expr, name) | {
2435+ let expr_type = expr. expr_type . as_ref ( ) . ok_or_else ( || {
2436+ datafusion_common:: internal_datafusion_err!(
2437+ "Unexpected empty aggregate physical expression"
2438+ )
2439+ } ) ?;
2440+ let ExprType :: AggregateExpr ( aggregate) = expr_type else {
2441+ return internal_err ! (
2442+ "Invalid aggregate expression for AggregateExec"
2443+ ) ;
2444+ } ;
2445+ let args = aggregate
2446+ . expr
2447+ . iter ( )
2448+ . map ( |expr| ctx. decode_expr ( expr, input_schema. as_ref ( ) ) )
2449+ . collect :: < Result < Vec < _ > > > ( ) ?;
2450+ let order_by = aggregate
2451+ . ordering_req
2452+ . iter ( )
2453+ . map ( |sort_expr| {
2454+ let expr = sort_expr. expr . as_deref ( ) . ok_or_else ( || {
2455+ datafusion_common:: internal_datafusion_err!(
2456+ "AggregateExec ordering expression is missing its inner expr"
2457+ )
2458+ } ) ?;
2459+ Ok ( PhysicalSortExpr {
2460+ expr : ctx. decode_expr ( expr, input_schema. as_ref ( ) ) ?,
2461+ options : arrow:: compute:: SortOptions {
2462+ descending : !sort_expr. asc ,
2463+ nulls_first : sort_expr. nulls_first ,
2464+ } ,
2465+ } )
2466+ } )
2467+ . collect :: < Result < Vec < _ > > > ( ) ?;
2468+ let Some ( AggregateFunction :: UserDefinedAggrFunction ( udaf_name) ) =
2469+ aggregate. aggregate_function . as_ref ( )
2470+ else {
2471+ return internal_err ! (
2472+ "Invalid AggregateExpr, missing aggregate_function"
2473+ ) ;
2474+ } ;
2475+ let udaf = ctx. decode_udaf (
2476+ udaf_name,
2477+ aggregate. fun_definition . as_deref ( ) ,
2478+ ) ?;
2479+ let ( human_display, human_display_alias) =
2480+ split_human_display_alias ( & aggregate. human_display , name) ;
2481+ let builder = AggregateExprBuilder :: new ( udaf, args)
2482+ . schema ( Arc :: clone ( & input_schema) )
2483+ . alias ( name)
2484+ . with_ignore_nulls ( aggregate. ignore_nulls )
2485+ . with_distinct ( aggregate. distinct )
2486+ . order_by ( order_by)
2487+ . human_display ( human_display) ;
2488+ let builder = if let Some ( alias) = human_display_alias {
2489+ builder. human_display_alias ( alias)
2490+ } else {
2491+ builder
2492+ } ;
2493+ builder. build ( ) . map ( Arc :: new)
2494+ } )
2495+ . collect :: < Result < Vec < _ > > > ( ) ?;
2496+ let aggregate = AggregateExec :: try_new (
2497+ mode,
2498+ PhysicalGroupBy :: new (
2499+ group_expr,
2500+ null_expr,
2501+ groups,
2502+ hash_agg. has_grouping_set ,
2503+ ) ,
2504+ aggr_expr,
2505+ filter_expr,
2506+ input,
2507+ Arc :: clone ( & input_schema) ,
2508+ ) ?;
2509+ let aggregate = if let Some ( limit) = & hash_agg. limit {
2510+ let options = match limit. descending {
2511+ Some ( descending) => {
2512+ LimitOptions :: new_with_order ( limit. limit as usize , descending)
2513+ }
2514+ None => LimitOptions :: new ( limit. limit as usize ) ,
2515+ } ;
2516+ aggregate. with_limit_options ( Some ( options) )
2517+ } else {
2518+ aggregate
2519+ } ;
2520+ let aggregate = if let Some ( dynamic_filter) = & hash_agg. dynamic_filter {
2521+ let dynamic_filter =
2522+ ctx. decode_expr ( dynamic_filter, input_schema. as_ref ( ) ) ?;
2523+ let dynamic_filter = ( dynamic_filter
2524+ as Arc < dyn std:: any:: Any + Send + Sync > )
2525+ . downcast :: < DynamicFilterPhysicalExpr > ( )
2526+ . map_err ( |_| {
2527+ datafusion_common:: internal_datafusion_err!(
2528+ "AggregateExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr"
2529+ )
2530+ } ) ?;
2531+ aggregate. with_dynamic_filter_expr ( dynamic_filter) ?
2532+ } else {
2533+ aggregate
2534+ } ;
2535+
2536+ Ok ( Arc :: new ( aggregate) )
2537+ }
21712538}
21722539
21732540/// Creates the output schema for an [`AggregateExec`] containing the group by columns followed
0 commit comments