Skip to content

Commit 1c766ca

Browse files
committed
refactor(proto): isolate window serde helpers
Window protobuf helpers obscured the execution plan implementations. Keeping them feature-gated together makes the shared serialization invariants easier to maintain without widening their visibility. Refs apache#23513
1 parent 76b1311 commit 1c766ca

4 files changed

Lines changed: 297 additions & 286 deletions

File tree

‎datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -407,7 +407,7 @@ impl ExecutionPlan for BoundedWindowAggExec {
407407
&self,
408408
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
409409
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
410-
use super::window_agg_exec::encode_physical_window_expr;
410+
use super::proto::encode_physical_window_expr;
411411
use datafusion_proto_common::protobuf_common::EmptyMessage;
412412
use datafusion_proto_models::protobuf;
413413
use protobuf::window_agg_exec_node::InputOrderMode as ProtoInputOrderMode;
@@ -423,6 +423,8 @@ impl ExecutionPlan for BoundedWindowAggExec {
423423
.iter()
424424
.map(|expr| ctx.encode_expr(expr))
425425
.collect::<Result<Vec<_>>>()?;
426+
// A `Some(input_order_mode)` is what tells the shared `Window` decode
427+
// arm to rebuild a `BoundedWindowAggExec` rather than a `WindowAggExec`.
426428
let input_order_mode = match &self.input_order_mode {
427429
InputOrderMode::Linear => ProtoInputOrderMode::Linear(EmptyMessage {}),
428430
InputOrderMode::PartiallySorted(columns) => {

‎datafusion/physical-plan/src/windows/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
//! Physical expressions for window functions
1919
2020
mod bounded_window_agg_exec;
21+
#[cfg(feature = "proto")]
22+
mod proto;
2123
mod utils;
2224
mod window_agg_exec;
2325

Lines changed: 288 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,288 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Protobuf conversions shared by window execution plans.
19+
20+
use std::sync::Arc;
21+
22+
use arrow::compute::SortOptions;
23+
use arrow::datatypes::Schema;
24+
use datafusion_common::{
25+
Result, ScalarValue, internal_datafusion_err, internal_err, not_impl_err,
26+
};
27+
use datafusion_expr::{
28+
WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition,
29+
};
30+
use datafusion_physical_expr::window::SlidingAggregateWindowExpr;
31+
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
32+
use datafusion_proto_common::protobuf_common;
33+
use datafusion_proto_models::protobuf::{self, physical_window_expr_node};
34+
35+
use super::{
36+
PlainAggregateWindowExpr, StandardWindowExpr, WindowExpr, WindowUDFExpr,
37+
create_window_expr, schema_add_window_field,
38+
};
39+
40+
pub(super) fn encode_physical_window_expr(
41+
window_expr: &Arc<dyn WindowExpr>,
42+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
43+
) -> Result<protobuf::PhysicalWindowExprNode> {
44+
let expr = window_expr.as_any();
45+
let mut args = window_expr.expressions().to_vec();
46+
let window_frame = window_expr.get_window_frame();
47+
let (window_function, fun_definition, ignore_nulls, distinct) =
48+
if let Some(plain) = expr.downcast_ref::<PlainAggregateWindowExpr>() {
49+
let aggregate_expr = plain.get_aggregate_expr();
50+
(
51+
physical_window_expr_node::WindowFunction::UserDefinedAggrFunction(
52+
aggregate_expr.fun().name().to_string(),
53+
),
54+
ctx.encode_udaf(aggregate_expr.fun())?,
55+
aggregate_expr.ignore_nulls(),
56+
aggregate_expr.is_distinct(),
57+
)
58+
} else if let Some(sliding) = expr.downcast_ref::<SlidingAggregateWindowExpr>() {
59+
let aggregate_expr = sliding.get_aggregate_expr();
60+
(
61+
physical_window_expr_node::WindowFunction::UserDefinedAggrFunction(
62+
aggregate_expr.fun().name().to_string(),
63+
),
64+
ctx.encode_udaf(aggregate_expr.fun())?,
65+
aggregate_expr.ignore_nulls(),
66+
aggregate_expr.is_distinct(),
67+
)
68+
} else if let Some(standard) = expr.downcast_ref::<StandardWindowExpr>() {
69+
if let Some(window_udf) = standard
70+
.get_standard_func_expr()
71+
.as_any()
72+
.downcast_ref::<WindowUDFExpr>()
73+
{
74+
// `WindowUDFExpr::args` returns the full, unfiltered argument list so
75+
// every argument survives the round-trip.
76+
args = window_udf.args().to_vec();
77+
(
78+
physical_window_expr_node::WindowFunction::UserDefinedWindowFunction(
79+
window_udf.fun().name().to_string(),
80+
),
81+
ctx.encode_udwf(window_udf.fun().as_ref())?,
82+
false,
83+
false,
84+
)
85+
} else {
86+
return not_impl_err!(
87+
"User-defined window function not supported: {window_expr:?}"
88+
);
89+
}
90+
} else {
91+
return not_impl_err!("WindowExpr not supported: {window_expr:?}");
92+
};
93+
94+
let args = ctx.encode_expressions(&args)?;
95+
let partition_by = ctx.encode_expressions(window_expr.partition_by())?;
96+
let order_by = window_expr
97+
.order_by()
98+
.iter()
99+
.map(|sort_expr| {
100+
Ok(protobuf::PhysicalSortExprNode {
101+
expr: Some(Box::new(ctx.encode_expr(&sort_expr.expr)?)),
102+
asc: !sort_expr.options.descending,
103+
nulls_first: sort_expr.options.nulls_first,
104+
})
105+
})
106+
.collect::<Result<Vec<_>>>()?;
107+
108+
Ok(protobuf::PhysicalWindowExprNode {
109+
args,
110+
partition_by,
111+
order_by,
112+
window_frame: Some(encode_window_frame(window_frame.as_ref())?),
113+
window_function: Some(window_function),
114+
name: window_expr.name().to_string(),
115+
fun_definition,
116+
ignore_nulls,
117+
distinct,
118+
})
119+
}
120+
121+
pub(super) fn decode_physical_window_expr(
122+
proto: &protobuf::PhysicalWindowExprNode,
123+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
124+
input_schema: &Schema,
125+
) -> Result<Arc<dyn WindowExpr>> {
126+
let args = proto
127+
.args
128+
.iter()
129+
.map(|expr| ctx.decode_expr(expr, input_schema))
130+
.collect::<Result<Vec<_>>>()?;
131+
let partition_by = proto
132+
.partition_by
133+
.iter()
134+
.map(|expr| ctx.decode_expr(expr, input_schema))
135+
.collect::<Result<Vec<_>>>()?;
136+
let order_by = proto
137+
.order_by
138+
.iter()
139+
.map(|sort_expr| {
140+
let expr = sort_expr.expr.as_ref().ok_or_else(|| {
141+
internal_datafusion_err!(
142+
"Missing expr in window order_by sort expression"
143+
)
144+
})?;
145+
Ok(PhysicalSortExpr {
146+
expr: ctx.decode_expr(expr, input_schema)?,
147+
options: SortOptions {
148+
descending: !sort_expr.asc,
149+
nulls_first: sort_expr.nulls_first,
150+
},
151+
})
152+
})
153+
.collect::<Result<Vec<_>>>()?;
154+
let window_frame = proto
155+
.window_frame
156+
.as_ref()
157+
.map(decode_window_frame)
158+
.transpose()?
159+
.ok_or_else(|| {
160+
internal_datafusion_err!("Missing required field 'window_frame' in protobuf")
161+
})?;
162+
let function = match proto.window_function.as_ref() {
163+
Some(physical_window_expr_node::WindowFunction::UserDefinedAggrFunction(
164+
name,
165+
)) => WindowFunctionDefinition::AggregateUDF(
166+
ctx.decode_udaf(name, proto.fun_definition.as_deref())?,
167+
),
168+
Some(physical_window_expr_node::WindowFunction::UserDefinedWindowFunction(
169+
name,
170+
)) => WindowFunctionDefinition::WindowUDF(
171+
ctx.decode_udwf(name, proto.fun_definition.as_deref())?,
172+
),
173+
None => {
174+
return internal_err!("Missing required field 'window_function' in protobuf");
175+
}
176+
};
177+
178+
let name = proto.name.clone();
179+
// TODO: Remove extended_schema if functions are all UDAF
180+
let extended_schema = schema_add_window_field(&args, input_schema, &function, &name)?;
181+
create_window_expr(
182+
&function,
183+
name,
184+
&args,
185+
&partition_by,
186+
&order_by,
187+
Arc::new(window_frame),
188+
extended_schema,
189+
proto.ignore_nulls,
190+
proto.distinct,
191+
None,
192+
)
193+
}
194+
195+
fn encode_window_frame(window_frame: &WindowFrame) -> Result<protobuf::WindowFrame> {
196+
let units = match window_frame.units {
197+
WindowFrameUnits::Rows => protobuf::WindowFrameUnits::Rows,
198+
WindowFrameUnits::Range => protobuf::WindowFrameUnits::Range,
199+
WindowFrameUnits::Groups => protobuf::WindowFrameUnits::Groups,
200+
};
201+
Ok(protobuf::WindowFrame {
202+
window_frame_units: units.into(),
203+
start_bound: Some(encode_window_frame_bound(&window_frame.start_bound)?),
204+
end_bound: Some(protobuf::window_frame::EndBound::Bound(
205+
encode_window_frame_bound(&window_frame.end_bound)?,
206+
)),
207+
})
208+
}
209+
210+
fn encode_window_frame_bound(
211+
bound: &WindowFrameBound,
212+
) -> Result<protobuf::WindowFrameBound> {
213+
let encode_value = |value: &ScalarValue| -> Result<protobuf_common::ScalarValue> {
214+
Ok(value.try_into()?)
215+
};
216+
Ok(match bound {
217+
WindowFrameBound::CurrentRow => protobuf::WindowFrameBound {
218+
window_frame_bound_type: protobuf::WindowFrameBoundType::CurrentRow.into(),
219+
bound_value: None,
220+
},
221+
WindowFrameBound::Preceding(value) => protobuf::WindowFrameBound {
222+
window_frame_bound_type: protobuf::WindowFrameBoundType::Preceding.into(),
223+
bound_value: Some(encode_value(value)?),
224+
},
225+
WindowFrameBound::Following(value) => protobuf::WindowFrameBound {
226+
window_frame_bound_type: protobuf::WindowFrameBoundType::Following.into(),
227+
bound_value: Some(encode_value(value)?),
228+
},
229+
})
230+
}
231+
232+
fn decode_window_frame(window_frame: &protobuf::WindowFrame) -> Result<WindowFrame> {
233+
let units = protobuf::WindowFrameUnits::try_from(window_frame.window_frame_units)
234+
.map_err(|_| {
235+
internal_datafusion_err!(
236+
"Received a WindowFrame message with unknown WindowFrameUnits {}",
237+
window_frame.window_frame_units
238+
)
239+
})?;
240+
let units = match units {
241+
protobuf::WindowFrameUnits::Rows => WindowFrameUnits::Rows,
242+
protobuf::WindowFrameUnits::Range => WindowFrameUnits::Range,
243+
protobuf::WindowFrameUnits::Groups => WindowFrameUnits::Groups,
244+
};
245+
let start_bound =
246+
decode_window_frame_bound(window_frame.start_bound.as_ref().ok_or_else(
247+
|| internal_datafusion_err!("Missing start_bound in WindowFrame"),
248+
)?)?;
249+
let end_bound = window_frame
250+
.end_bound
251+
.as_ref()
252+
.map(|end_bound| match end_bound {
253+
protobuf::window_frame::EndBound::Bound(bound) => {
254+
decode_window_frame_bound(bound)
255+
}
256+
})
257+
.transpose()?
258+
.unwrap_or(WindowFrameBound::CurrentRow);
259+
Ok(WindowFrame::new_bounds(units, start_bound, end_bound))
260+
}
261+
262+
fn decode_window_frame_bound(
263+
bound: &protobuf::WindowFrameBound,
264+
) -> Result<WindowFrameBound> {
265+
let decode_value = |value: &protobuf_common::ScalarValue| -> Result<ScalarValue> {
266+
Ok(ScalarValue::try_from(value)?)
267+
};
268+
let bound_type = protobuf::WindowFrameBoundType::try_from(
269+
bound.window_frame_bound_type,
270+
)
271+
.map_err(|_| {
272+
internal_datafusion_err!(
273+
"Received a WindowFrameBound message with unknown WindowFrameBoundType {}",
274+
bound.window_frame_bound_type
275+
)
276+
})?;
277+
match bound_type {
278+
protobuf::WindowFrameBoundType::CurrentRow => Ok(WindowFrameBound::CurrentRow),
279+
protobuf::WindowFrameBoundType::Preceding => match &bound.bound_value {
280+
Some(value) => Ok(WindowFrameBound::Preceding(decode_value(value)?)),
281+
None => Ok(WindowFrameBound::Preceding(ScalarValue::UInt64(None))),
282+
},
283+
protobuf::WindowFrameBoundType::Following => match &bound.bound_value {
284+
Some(value) => Ok(WindowFrameBound::Following(decode_value(value)?)),
285+
None => Ok(WindowFrameBound::Following(ScalarValue::UInt64(None))),
286+
},
287+
}
288+
}

0 commit comments

Comments
 (0)