Skip to content

Commit 1366315

Browse files
committed
refactor(proto): migrate aggregate exec serde
Aggregate protobuf conversion lived in the central proto crate, which prevented AggregateExec from owning its function-codec serialization. Move encoding and decoding into AggregateExec and keep the deprecated helpers as delegates so existing callers retain wire-compatible behavior. CLOSES apache#23512 Signed-off-by: Jiawei Zhao <[email protected]>
1 parent 1763417 commit 1366315

2 files changed

Lines changed: 398 additions & 352 deletions

File tree

  • datafusion
    • physical-plan/src/aggregates
    • proto/src/physical_plan

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 367 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2175,6 +2175,373 @@ impl ExecutionPlan for AggregateExec {
21752175

21762176
Ok(result)
21772177
}
2178+
2179+
#[cfg(feature = "proto")]
2180+
fn try_to_proto(
2181+
&self,
2182+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
2183+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
2184+
use datafusion_proto_models::protobuf;
2185+
2186+
let input = ctx.encode_child(self.input())?;
2187+
let group_by = self.group_expr();
2188+
let group_expr =
2189+
ctx.encode_expressions(group_by.expr().iter().map(|(expr, _)| expr))?;
2190+
let group_expr_name = group_by
2191+
.expr()
2192+
.iter()
2193+
.map(|(_, name)| name.to_owned())
2194+
.collect();
2195+
let null_expr =
2196+
ctx.encode_expressions(group_by.null_expr().iter().map(|(expr, _)| expr))?;
2197+
let groups = group_by.groups().iter().flatten().copied().collect();
2198+
let aggr_expr = self
2199+
.aggr_expr()
2200+
.iter()
2201+
.map(|expr| encode_aggregate_expr(expr, ctx))
2202+
.collect::<Result<Vec<_>>>()?;
2203+
let aggr_expr_name = self
2204+
.aggr_expr()
2205+
.iter()
2206+
.map(|expr| expr.name().to_string())
2207+
.collect();
2208+
let filter_expr = self
2209+
.filter_expr()
2210+
.iter()
2211+
.map(|filter| {
2212+
Ok(protobuf::MaybeFilter {
2213+
expr: filter
2214+
.as_ref()
2215+
.map(|expr| ctx.encode_expr(expr))
2216+
.transpose()?,
2217+
})
2218+
})
2219+
.collect::<Result<Vec<_>>>()?;
2220+
let mode = match self.mode() {
2221+
AggregateMode::Partial => protobuf::AggregateMode::Partial,
2222+
AggregateMode::Final => protobuf::AggregateMode::Final,
2223+
AggregateMode::FinalPartitioned => protobuf::AggregateMode::FinalPartitioned,
2224+
AggregateMode::Single => protobuf::AggregateMode::Single,
2225+
AggregateMode::SinglePartitioned => {
2226+
protobuf::AggregateMode::SinglePartitioned
2227+
}
2228+
AggregateMode::PartialReduce => protobuf::AggregateMode::PartialReduce,
2229+
};
2230+
let limit = self.limit_options().map(|options| protobuf::AggLimit {
2231+
limit: options.limit() as u64,
2232+
descending: options.descending(),
2233+
});
2234+
let dynamic_filter = match self.dynamic_filter_expr() {
2235+
Some(filter) => {
2236+
let expr: Arc<dyn PhysicalExpr> =
2237+
Arc::clone(filter) as Arc<dyn PhysicalExpr>;
2238+
Some(ctx.encode_expr(&expr)?)
2239+
}
2240+
None => None,
2241+
};
2242+
2243+
Ok(Some(protobuf::PhysicalPlanNode {
2244+
physical_plan_type: Some(
2245+
protobuf::physical_plan_node::PhysicalPlanType::Aggregate(Box::new(
2246+
protobuf::AggregateExecNode {
2247+
group_expr,
2248+
group_expr_name,
2249+
aggr_expr,
2250+
filter_expr,
2251+
aggr_expr_name,
2252+
mode: mode as i32,
2253+
input: Some(Box::new(input)),
2254+
input_schema: Some(self.input_schema().as_ref().try_into()?),
2255+
null_expr,
2256+
groups,
2257+
limit,
2258+
has_grouping_set: group_by.has_grouping_set(),
2259+
dynamic_filter,
2260+
},
2261+
)),
2262+
),
2263+
}))
2264+
}
2265+
}
2266+
2267+
#[cfg(feature = "proto")]
2268+
const HUMAN_DISPLAY_ALIAS_PREFIX: &str = "\u{1f}datafusion_human_display_alias_v1:";
2269+
2270+
#[cfg(feature = "proto")]
2271+
fn encode_human_display_alias(human_display: &str, alias: &str) -> String {
2272+
format!(
2273+
"{HUMAN_DISPLAY_ALIAS_PREFIX}{}:{alias}{human_display}",
2274+
alias.len()
2275+
)
2276+
}
2277+
2278+
#[cfg(feature = "proto")]
2279+
fn split_human_display_alias<'a>(
2280+
human_display: &'a str,
2281+
name: &'a str,
2282+
) -> (&'a str, Option<&'a str>) {
2283+
if let Some(encoded) = human_display.strip_prefix(HUMAN_DISPLAY_ALIAS_PREFIX)
2284+
&& let Some((alias_len, encoded)) = encoded.split_once(':')
2285+
&& let Ok(alias_len) = alias_len.parse::<usize>()
2286+
&& let Some(alias) = encoded.get(..alias_len)
2287+
&& let Some(human_display) = encoded.get(alias_len..)
2288+
&& alias == name
2289+
&& !human_display.is_empty()
2290+
{
2291+
return (human_display, Some(alias));
2292+
}
2293+
2294+
(human_display, None)
2295+
}
2296+
2297+
#[cfg(feature = "proto")]
2298+
fn encode_aggregate_expr(
2299+
aggr_expr: &Arc<AggregateFunctionExpr>,
2300+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
2301+
) -> Result<datafusion_proto_models::protobuf::PhysicalExprNode> {
2302+
use datafusion_proto_models::protobuf;
2303+
2304+
let expressions = aggr_expr.expressions();
2305+
let expr = ctx.encode_expressions(expressions.iter())?;
2306+
let ordering_req = aggr_expr
2307+
.order_bys()
2308+
.iter()
2309+
.map(|sort_expr| {
2310+
Ok(protobuf::PhysicalSortExprNode {
2311+
expr: Some(Box::new(ctx.encode_expr(&sort_expr.expr)?)),
2312+
asc: !sort_expr.options.descending,
2313+
nulls_first: sort_expr.options.nulls_first,
2314+
})
2315+
})
2316+
.collect::<Result<Vec<_>>>()?;
2317+
let name = aggr_expr.fun().name().to_string();
2318+
let fun_definition = ctx.encode_udaf(aggr_expr.fun())?;
2319+
let human_display = match (aggr_expr.human_display(), aggr_expr.human_display_alias())
2320+
{
2321+
(Some(display), Some(alias)) => encode_human_display_alias(display, alias),
2322+
(Some(display), None) => display.to_string(),
2323+
(None, _) => String::new(),
2324+
};
2325+
2326+
Ok(protobuf::PhysicalExprNode {
2327+
expr_id: None,
2328+
expr_type: Some(protobuf::physical_expr_node::ExprType::AggregateExpr(
2329+
protobuf::PhysicalAggregateExprNode {
2330+
aggregate_function: Some(
2331+
protobuf::physical_aggregate_expr_node::AggregateFunction::UserDefinedAggrFunction(name),
2332+
),
2333+
expr,
2334+
ordering_req,
2335+
distinct: aggr_expr.is_distinct(),
2336+
ignore_nulls: aggr_expr.ignore_nulls(),
2337+
fun_definition,
2338+
human_display,
2339+
},
2340+
)),
2341+
})
2342+
}
2343+
2344+
#[cfg(feature = "proto")]
2345+
impl AggregateExec {
2346+
/// Reconstruct an [`AggregateExec`] from its protobuf representation.
2347+
///
2348+
/// Grouping expressions are decoded against the child schema. Aggregate
2349+
/// arguments, ordering, filters, and the dynamic filter are decoded against
2350+
/// the aggregate input schema carried in the protobuf node.
2351+
pub fn try_from_proto(
2352+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
2353+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
2354+
) -> Result<Arc<dyn ExecutionPlan>> {
2355+
use datafusion_physical_expr::PhysicalSortExpr;
2356+
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
2357+
use datafusion_proto_models::protobuf;
2358+
use protobuf::physical_aggregate_expr_node::AggregateFunction;
2359+
use protobuf::physical_expr_node::ExprType;
2360+
2361+
let hash_agg = crate::expect_plan_variant!(
2362+
node,
2363+
protobuf::physical_plan_node::PhysicalPlanType::Aggregate,
2364+
"AggregateExec",
2365+
);
2366+
let input = ctx.decode_required_child(
2367+
hash_agg.input.as_deref(),
2368+
"AggregateExec",
2369+
"input",
2370+
)?;
2371+
let mode = protobuf::AggregateMode::try_from(hash_agg.mode).map_err(|_| {
2372+
datafusion_common::internal_datafusion_err!(
2373+
"Received an AggregateNode message with unknown AggregateMode {}",
2374+
hash_agg.mode
2375+
)
2376+
})?;
2377+
let mode = match mode {
2378+
protobuf::AggregateMode::Partial => AggregateMode::Partial,
2379+
protobuf::AggregateMode::Final => AggregateMode::Final,
2380+
protobuf::AggregateMode::FinalPartitioned => AggregateMode::FinalPartitioned,
2381+
protobuf::AggregateMode::Single => AggregateMode::Single,
2382+
protobuf::AggregateMode::SinglePartitioned => {
2383+
AggregateMode::SinglePartitioned
2384+
}
2385+
protobuf::AggregateMode::PartialReduce => AggregateMode::PartialReduce,
2386+
};
2387+
let num_expr = hash_agg.group_expr.len();
2388+
let child_schema = input.schema();
2389+
let group_expr = hash_agg
2390+
.group_expr
2391+
.iter()
2392+
.zip(hash_agg.group_expr_name.iter())
2393+
.map(|(expr, name)| {
2394+
Ok((
2395+
ctx.decode_expr(expr, child_schema.as_ref())?,
2396+
name.to_string(),
2397+
))
2398+
})
2399+
.collect::<Result<Vec<_>>>()?;
2400+
let null_expr = hash_agg
2401+
.null_expr
2402+
.iter()
2403+
.zip(hash_agg.group_expr_name.iter())
2404+
.map(|(expr, name)| {
2405+
Ok((
2406+
ctx.decode_expr(expr, child_schema.as_ref())?,
2407+
name.to_string(),
2408+
))
2409+
})
2410+
.collect::<Result<Vec<_>>>()?;
2411+
let groups = if hash_agg.groups.is_empty() {
2412+
vec![]
2413+
} else {
2414+
hash_agg
2415+
.groups
2416+
.chunks(num_expr)
2417+
.map(|group| group.to_vec())
2418+
.collect()
2419+
};
2420+
let input_schema = hash_agg.input_schema.as_ref().ok_or_else(|| {
2421+
datafusion_common::internal_datafusion_err!(
2422+
"input_schema in AggregateNode is missing."
2423+
)
2424+
})?;
2425+
let input_schema: SchemaRef = SchemaRef::new(input_schema.try_into()?);
2426+
let filter_expr = hash_agg
2427+
.filter_expr
2428+
.iter()
2429+
.map(|filter| {
2430+
filter
2431+
.expr
2432+
.as_ref()
2433+
.map(|expr| ctx.decode_expr(expr, input_schema.as_ref()))
2434+
.transpose()
2435+
})
2436+
.collect::<Result<Vec<_>>>()?;
2437+
let aggr_expr = hash_agg
2438+
.aggr_expr
2439+
.iter()
2440+
.zip(hash_agg.aggr_expr_name.iter())
2441+
.map(|(expr, name)| {
2442+
let expr_type = expr.expr_type.as_ref().ok_or_else(|| {
2443+
datafusion_common::internal_datafusion_err!(
2444+
"Unexpected empty aggregate physical expression"
2445+
)
2446+
})?;
2447+
let ExprType::AggregateExpr(aggregate) = expr_type else {
2448+
return internal_err!(
2449+
"Invalid aggregate expression for AggregateExec"
2450+
);
2451+
};
2452+
let args = aggregate
2453+
.expr
2454+
.iter()
2455+
.map(|expr| ctx.decode_expr(expr, input_schema.as_ref()))
2456+
.collect::<Result<Vec<_>>>()?;
2457+
let order_by = aggregate
2458+
.ordering_req
2459+
.iter()
2460+
.map(|sort_expr| {
2461+
let expr = sort_expr.expr.as_deref().ok_or_else(|| {
2462+
datafusion_common::internal_datafusion_err!(
2463+
"AggregateExec ordering expression is missing its inner expr"
2464+
)
2465+
})?;
2466+
Ok(PhysicalSortExpr {
2467+
expr: ctx.decode_expr(expr, input_schema.as_ref())?,
2468+
options: arrow::compute::SortOptions {
2469+
descending: !sort_expr.asc,
2470+
nulls_first: sort_expr.nulls_first,
2471+
},
2472+
})
2473+
})
2474+
.collect::<Result<Vec<_>>>()?;
2475+
let Some(AggregateFunction::UserDefinedAggrFunction(udaf_name)) =
2476+
aggregate.aggregate_function.as_ref()
2477+
else {
2478+
return internal_err!(
2479+
"Invalid AggregateExpr, missing aggregate_function"
2480+
);
2481+
};
2482+
let udaf = ctx.decode_udaf(
2483+
udaf_name,
2484+
aggregate.fun_definition.as_deref(),
2485+
)?;
2486+
let (human_display, human_display_alias) =
2487+
split_human_display_alias(&aggregate.human_display, name);
2488+
let builder = AggregateExprBuilder::new(udaf, args)
2489+
.schema(Arc::clone(&input_schema))
2490+
.alias(name)
2491+
.with_ignore_nulls(aggregate.ignore_nulls)
2492+
.with_distinct(aggregate.distinct)
2493+
.order_by(order_by)
2494+
.human_display(human_display);
2495+
let builder = if let Some(alias) = human_display_alias {
2496+
builder.human_display_alias(alias)
2497+
} else {
2498+
builder
2499+
};
2500+
builder.build().map(Arc::new)
2501+
})
2502+
.collect::<Result<Vec<_>>>()?;
2503+
let aggregate = AggregateExec::try_new(
2504+
mode,
2505+
PhysicalGroupBy::new(
2506+
group_expr,
2507+
null_expr,
2508+
groups,
2509+
hash_agg.has_grouping_set,
2510+
),
2511+
aggr_expr,
2512+
filter_expr,
2513+
input,
2514+
Arc::clone(&input_schema),
2515+
)?;
2516+
let aggregate = if let Some(limit) = &hash_agg.limit {
2517+
let options = match limit.descending {
2518+
Some(descending) => {
2519+
LimitOptions::new_with_order(limit.limit as usize, descending)
2520+
}
2521+
None => LimitOptions::new(limit.limit as usize),
2522+
};
2523+
aggregate.with_limit_options(Some(options))
2524+
} else {
2525+
aggregate
2526+
};
2527+
let aggregate = if let Some(dynamic_filter) = &hash_agg.dynamic_filter {
2528+
let dynamic_filter =
2529+
ctx.decode_expr(dynamic_filter, input_schema.as_ref())?;
2530+
let dynamic_filter = (dynamic_filter
2531+
as Arc<dyn std::any::Any + Send + Sync>)
2532+
.downcast::<DynamicFilterPhysicalExpr>()
2533+
.map_err(|_| {
2534+
datafusion_common::internal_datafusion_err!(
2535+
"AggregateExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr"
2536+
)
2537+
})?;
2538+
aggregate.with_dynamic_filter_expr(dynamic_filter)?
2539+
} else {
2540+
aggregate
2541+
};
2542+
2543+
Ok(Arc::new(aggregate))
2544+
}
21782545
}
21792546

21802547
/// Creates the output schema for an [`AggregateExec`] containing the group by columns followed

0 commit comments

Comments
 (0)