Skip to content

Commit 47c1ef0

Browse files
committed
refactor(proto): migrate aggregate exec serde
Aggregate protobuf conversion lived in the central proto crate, which prevented AggregateExec from owning its function-codec serialization. Move encoding and decoding into AggregateExec and keep the deprecated helpers as delegates so existing callers retain wire-compatible behavior. CLOSES apache#23512 Signed-off-by: Jiawei Zhao <[email protected]>
1 parent 16471ee commit 47c1ef0

2 files changed

Lines changed: 398 additions & 354 deletions

File tree

  • datafusion
    • physical-plan/src/aggregates
    • proto/src/physical_plan

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 367 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2168,6 +2168,373 @@ impl ExecutionPlan for AggregateExec {
21682168

21692169
Ok(result)
21702170
}
2171+
2172+
#[cfg(feature = "proto")]
2173+
fn try_to_proto(
2174+
&self,
2175+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
2176+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
2177+
use datafusion_proto_models::protobuf;
2178+
2179+
let input = ctx.encode_child(self.input())?;
2180+
let group_by = self.group_expr();
2181+
let group_expr =
2182+
ctx.encode_expressions(group_by.expr().iter().map(|(expr, _)| expr))?;
2183+
let group_expr_name = group_by
2184+
.expr()
2185+
.iter()
2186+
.map(|(_, name)| name.to_owned())
2187+
.collect();
2188+
let null_expr =
2189+
ctx.encode_expressions(group_by.null_expr().iter().map(|(expr, _)| expr))?;
2190+
let groups = group_by.groups().iter().flatten().copied().collect();
2191+
let aggr_expr = self
2192+
.aggr_expr()
2193+
.iter()
2194+
.map(|expr| encode_aggregate_expr(expr, ctx))
2195+
.collect::<Result<Vec<_>>>()?;
2196+
let aggr_expr_name = self
2197+
.aggr_expr()
2198+
.iter()
2199+
.map(|expr| expr.name().to_string())
2200+
.collect();
2201+
let filter_expr = self
2202+
.filter_expr()
2203+
.iter()
2204+
.map(|filter| {
2205+
Ok(protobuf::MaybeFilter {
2206+
expr: filter
2207+
.as_ref()
2208+
.map(|expr| ctx.encode_expr(expr))
2209+
.transpose()?,
2210+
})
2211+
})
2212+
.collect::<Result<Vec<_>>>()?;
2213+
let mode = match self.mode() {
2214+
AggregateMode::Partial => protobuf::AggregateMode::Partial,
2215+
AggregateMode::Final => protobuf::AggregateMode::Final,
2216+
AggregateMode::FinalPartitioned => protobuf::AggregateMode::FinalPartitioned,
2217+
AggregateMode::Single => protobuf::AggregateMode::Single,
2218+
AggregateMode::SinglePartitioned => {
2219+
protobuf::AggregateMode::SinglePartitioned
2220+
}
2221+
AggregateMode::PartialReduce => protobuf::AggregateMode::PartialReduce,
2222+
};
2223+
let limit = self.limit_options().map(|options| protobuf::AggLimit {
2224+
limit: options.limit() as u64,
2225+
descending: options.descending(),
2226+
});
2227+
let dynamic_filter = match self.dynamic_filter_expr() {
2228+
Some(filter) => {
2229+
let expr: Arc<dyn PhysicalExpr> =
2230+
Arc::clone(filter) as Arc<dyn PhysicalExpr>;
2231+
Some(ctx.encode_expr(&expr)?)
2232+
}
2233+
None => None,
2234+
};
2235+
2236+
Ok(Some(protobuf::PhysicalPlanNode {
2237+
physical_plan_type: Some(
2238+
protobuf::physical_plan_node::PhysicalPlanType::Aggregate(Box::new(
2239+
protobuf::AggregateExecNode {
2240+
group_expr,
2241+
group_expr_name,
2242+
aggr_expr,
2243+
filter_expr,
2244+
aggr_expr_name,
2245+
mode: mode as i32,
2246+
input: Some(Box::new(input)),
2247+
input_schema: Some(self.input_schema().as_ref().try_into()?),
2248+
null_expr,
2249+
groups,
2250+
limit,
2251+
has_grouping_set: group_by.has_grouping_set(),
2252+
dynamic_filter,
2253+
},
2254+
)),
2255+
),
2256+
}))
2257+
}
2258+
}
2259+
2260+
#[cfg(feature = "proto")]
2261+
const HUMAN_DISPLAY_ALIAS_PREFIX: &str = "\u{1f}datafusion_human_display_alias_v1:";
2262+
2263+
#[cfg(feature = "proto")]
2264+
fn encode_human_display_alias(human_display: &str, alias: &str) -> String {
2265+
format!(
2266+
"{HUMAN_DISPLAY_ALIAS_PREFIX}{}:{alias}{human_display}",
2267+
alias.len()
2268+
)
2269+
}
2270+
2271+
#[cfg(feature = "proto")]
2272+
fn split_human_display_alias<'a>(
2273+
human_display: &'a str,
2274+
name: &'a str,
2275+
) -> (&'a str, Option<&'a str>) {
2276+
if let Some(encoded) = human_display.strip_prefix(HUMAN_DISPLAY_ALIAS_PREFIX)
2277+
&& let Some((alias_len, encoded)) = encoded.split_once(':')
2278+
&& let Ok(alias_len) = alias_len.parse::<usize>()
2279+
&& let Some(alias) = encoded.get(..alias_len)
2280+
&& let Some(human_display) = encoded.get(alias_len..)
2281+
&& alias == name
2282+
&& !human_display.is_empty()
2283+
{
2284+
return (human_display, Some(alias));
2285+
}
2286+
2287+
(human_display, None)
2288+
}
2289+
2290+
#[cfg(feature = "proto")]
2291+
fn encode_aggregate_expr(
2292+
aggr_expr: &Arc<AggregateFunctionExpr>,
2293+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
2294+
) -> Result<datafusion_proto_models::protobuf::PhysicalExprNode> {
2295+
use datafusion_proto_models::protobuf;
2296+
2297+
let expressions = aggr_expr.expressions();
2298+
let expr = ctx.encode_expressions(expressions.iter())?;
2299+
let ordering_req = aggr_expr
2300+
.order_bys()
2301+
.iter()
2302+
.map(|sort_expr| {
2303+
Ok(protobuf::PhysicalSortExprNode {
2304+
expr: Some(Box::new(ctx.encode_expr(&sort_expr.expr)?)),
2305+
asc: !sort_expr.options.descending,
2306+
nulls_first: sort_expr.options.nulls_first,
2307+
})
2308+
})
2309+
.collect::<Result<Vec<_>>>()?;
2310+
let name = aggr_expr.fun().name().to_string();
2311+
let fun_definition = ctx.encode_udaf(aggr_expr.fun())?;
2312+
let human_display = match (aggr_expr.human_display(), aggr_expr.human_display_alias())
2313+
{
2314+
(Some(display), Some(alias)) => encode_human_display_alias(display, alias),
2315+
(Some(display), None) => display.to_string(),
2316+
(None, _) => String::new(),
2317+
};
2318+
2319+
Ok(protobuf::PhysicalExprNode {
2320+
expr_id: None,
2321+
expr_type: Some(protobuf::physical_expr_node::ExprType::AggregateExpr(
2322+
protobuf::PhysicalAggregateExprNode {
2323+
aggregate_function: Some(
2324+
protobuf::physical_aggregate_expr_node::AggregateFunction::UserDefinedAggrFunction(name),
2325+
),
2326+
expr,
2327+
ordering_req,
2328+
distinct: aggr_expr.is_distinct(),
2329+
ignore_nulls: aggr_expr.ignore_nulls(),
2330+
fun_definition,
2331+
human_display,
2332+
},
2333+
)),
2334+
})
2335+
}
2336+
2337+
#[cfg(feature = "proto")]
2338+
impl AggregateExec {
2339+
/// Reconstruct an [`AggregateExec`] from its protobuf representation.
2340+
///
2341+
/// Grouping expressions are decoded against the child schema. Aggregate
2342+
/// arguments, ordering, filters, and the dynamic filter are decoded against
2343+
/// the aggregate input schema carried in the protobuf node.
2344+
pub fn try_from_proto(
2345+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
2346+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
2347+
) -> Result<Arc<dyn ExecutionPlan>> {
2348+
use datafusion_physical_expr::PhysicalSortExpr;
2349+
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
2350+
use datafusion_proto_models::protobuf;
2351+
use protobuf::physical_aggregate_expr_node::AggregateFunction;
2352+
use protobuf::physical_expr_node::ExprType;
2353+
2354+
let hash_agg = crate::expect_plan_variant!(
2355+
node,
2356+
protobuf::physical_plan_node::PhysicalPlanType::Aggregate,
2357+
"AggregateExec",
2358+
);
2359+
let input = ctx.decode_required_child(
2360+
hash_agg.input.as_deref(),
2361+
"AggregateExec",
2362+
"input",
2363+
)?;
2364+
let mode = protobuf::AggregateMode::try_from(hash_agg.mode).map_err(|_| {
2365+
datafusion_common::internal_datafusion_err!(
2366+
"Received an AggregateNode message with unknown AggregateMode {}",
2367+
hash_agg.mode
2368+
)
2369+
})?;
2370+
let mode = match mode {
2371+
protobuf::AggregateMode::Partial => AggregateMode::Partial,
2372+
protobuf::AggregateMode::Final => AggregateMode::Final,
2373+
protobuf::AggregateMode::FinalPartitioned => AggregateMode::FinalPartitioned,
2374+
protobuf::AggregateMode::Single => AggregateMode::Single,
2375+
protobuf::AggregateMode::SinglePartitioned => {
2376+
AggregateMode::SinglePartitioned
2377+
}
2378+
protobuf::AggregateMode::PartialReduce => AggregateMode::PartialReduce,
2379+
};
2380+
let num_expr = hash_agg.group_expr.len();
2381+
let child_schema = input.schema();
2382+
let group_expr = hash_agg
2383+
.group_expr
2384+
.iter()
2385+
.zip(hash_agg.group_expr_name.iter())
2386+
.map(|(expr, name)| {
2387+
Ok((
2388+
ctx.decode_expr(expr, child_schema.as_ref())?,
2389+
name.to_string(),
2390+
))
2391+
})
2392+
.collect::<Result<Vec<_>>>()?;
2393+
let null_expr = hash_agg
2394+
.null_expr
2395+
.iter()
2396+
.zip(hash_agg.group_expr_name.iter())
2397+
.map(|(expr, name)| {
2398+
Ok((
2399+
ctx.decode_expr(expr, child_schema.as_ref())?,
2400+
name.to_string(),
2401+
))
2402+
})
2403+
.collect::<Result<Vec<_>>>()?;
2404+
let groups = if hash_agg.groups.is_empty() {
2405+
vec![]
2406+
} else {
2407+
hash_agg
2408+
.groups
2409+
.chunks(num_expr)
2410+
.map(|group| group.to_vec())
2411+
.collect()
2412+
};
2413+
let input_schema = hash_agg.input_schema.as_ref().ok_or_else(|| {
2414+
datafusion_common::internal_datafusion_err!(
2415+
"input_schema in AggregateNode is missing."
2416+
)
2417+
})?;
2418+
let input_schema: SchemaRef = SchemaRef::new(input_schema.try_into()?);
2419+
let filter_expr = hash_agg
2420+
.filter_expr
2421+
.iter()
2422+
.map(|filter| {
2423+
filter
2424+
.expr
2425+
.as_ref()
2426+
.map(|expr| ctx.decode_expr(expr, input_schema.as_ref()))
2427+
.transpose()
2428+
})
2429+
.collect::<Result<Vec<_>>>()?;
2430+
let aggr_expr = hash_agg
2431+
.aggr_expr
2432+
.iter()
2433+
.zip(hash_agg.aggr_expr_name.iter())
2434+
.map(|(expr, name)| {
2435+
let expr_type = expr.expr_type.as_ref().ok_or_else(|| {
2436+
datafusion_common::internal_datafusion_err!(
2437+
"Unexpected empty aggregate physical expression"
2438+
)
2439+
})?;
2440+
let ExprType::AggregateExpr(aggregate) = expr_type else {
2441+
return internal_err!(
2442+
"Invalid aggregate expression for AggregateExec"
2443+
);
2444+
};
2445+
let args = aggregate
2446+
.expr
2447+
.iter()
2448+
.map(|expr| ctx.decode_expr(expr, input_schema.as_ref()))
2449+
.collect::<Result<Vec<_>>>()?;
2450+
let order_by = aggregate
2451+
.ordering_req
2452+
.iter()
2453+
.map(|sort_expr| {
2454+
let expr = sort_expr.expr.as_deref().ok_or_else(|| {
2455+
datafusion_common::internal_datafusion_err!(
2456+
"AggregateExec ordering expression is missing its inner expr"
2457+
)
2458+
})?;
2459+
Ok(PhysicalSortExpr {
2460+
expr: ctx.decode_expr(expr, input_schema.as_ref())?,
2461+
options: arrow::compute::SortOptions {
2462+
descending: !sort_expr.asc,
2463+
nulls_first: sort_expr.nulls_first,
2464+
},
2465+
})
2466+
})
2467+
.collect::<Result<Vec<_>>>()?;
2468+
let Some(AggregateFunction::UserDefinedAggrFunction(udaf_name)) =
2469+
aggregate.aggregate_function.as_ref()
2470+
else {
2471+
return internal_err!(
2472+
"Invalid AggregateExpr, missing aggregate_function"
2473+
);
2474+
};
2475+
let udaf = ctx.decode_udaf(
2476+
udaf_name,
2477+
aggregate.fun_definition.as_deref(),
2478+
)?;
2479+
let (human_display, human_display_alias) =
2480+
split_human_display_alias(&aggregate.human_display, name);
2481+
let builder = AggregateExprBuilder::new(udaf, args)
2482+
.schema(Arc::clone(&input_schema))
2483+
.alias(name)
2484+
.with_ignore_nulls(aggregate.ignore_nulls)
2485+
.with_distinct(aggregate.distinct)
2486+
.order_by(order_by)
2487+
.human_display(human_display);
2488+
let builder = if let Some(alias) = human_display_alias {
2489+
builder.human_display_alias(alias)
2490+
} else {
2491+
builder
2492+
};
2493+
builder.build().map(Arc::new)
2494+
})
2495+
.collect::<Result<Vec<_>>>()?;
2496+
let aggregate = AggregateExec::try_new(
2497+
mode,
2498+
PhysicalGroupBy::new(
2499+
group_expr,
2500+
null_expr,
2501+
groups,
2502+
hash_agg.has_grouping_set,
2503+
),
2504+
aggr_expr,
2505+
filter_expr,
2506+
input,
2507+
Arc::clone(&input_schema),
2508+
)?;
2509+
let aggregate = if let Some(limit) = &hash_agg.limit {
2510+
let options = match limit.descending {
2511+
Some(descending) => {
2512+
LimitOptions::new_with_order(limit.limit as usize, descending)
2513+
}
2514+
None => LimitOptions::new(limit.limit as usize),
2515+
};
2516+
aggregate.with_limit_options(Some(options))
2517+
} else {
2518+
aggregate
2519+
};
2520+
let aggregate = if let Some(dynamic_filter) = &hash_agg.dynamic_filter {
2521+
let dynamic_filter =
2522+
ctx.decode_expr(dynamic_filter, input_schema.as_ref())?;
2523+
let dynamic_filter = (dynamic_filter
2524+
as Arc<dyn std::any::Any + Send + Sync>)
2525+
.downcast::<DynamicFilterPhysicalExpr>()
2526+
.map_err(|_| {
2527+
datafusion_common::internal_datafusion_err!(
2528+
"AggregateExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr"
2529+
)
2530+
})?;
2531+
aggregate.with_dynamic_filter_expr(dynamic_filter)?
2532+
} else {
2533+
aggregate
2534+
};
2535+
2536+
Ok(Arc::new(aggregate))
2537+
}
21712538
}
21722539

21732540
/// Creates the output schema for an [`AggregateExec`] containing the group by columns followed

0 commit comments

Comments
 (0)