|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +//! Protobuf conversions shared by window execution plans. |
| 19 | +
|
| 20 | +use std::sync::Arc; |
| 21 | + |
| 22 | +use arrow::compute::SortOptions; |
| 23 | +use arrow::datatypes::Schema; |
| 24 | +use datafusion_common::{ |
| 25 | + Result, ScalarValue, internal_datafusion_err, internal_err, not_impl_err, |
| 26 | +}; |
| 27 | +use datafusion_expr::{ |
| 28 | + WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition, |
| 29 | +}; |
| 30 | +use datafusion_physical_expr::window::SlidingAggregateWindowExpr; |
| 31 | +use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; |
| 32 | +use datafusion_proto_common::protobuf_common; |
| 33 | +use datafusion_proto_models::protobuf::{self, physical_window_expr_node}; |
| 34 | + |
| 35 | +use super::{ |
| 36 | + PlainAggregateWindowExpr, StandardWindowExpr, WindowExpr, WindowUDFExpr, |
| 37 | + create_window_expr, schema_add_window_field, |
| 38 | +}; |
| 39 | + |
| 40 | +pub(super) fn encode_physical_window_expr( |
| 41 | + window_expr: &Arc<dyn WindowExpr>, |
| 42 | + ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, |
| 43 | +) -> Result<protobuf::PhysicalWindowExprNode> { |
| 44 | + let expr = window_expr.as_any(); |
| 45 | + let mut args = window_expr.expressions().to_vec(); |
| 46 | + let window_frame = window_expr.get_window_frame(); |
| 47 | + let (window_function, fun_definition, ignore_nulls, distinct) = |
| 48 | + if let Some(plain) = expr.downcast_ref::<PlainAggregateWindowExpr>() { |
| 49 | + let aggregate_expr = plain.get_aggregate_expr(); |
| 50 | + ( |
| 51 | + physical_window_expr_node::WindowFunction::UserDefinedAggrFunction( |
| 52 | + aggregate_expr.fun().name().to_string(), |
| 53 | + ), |
| 54 | + ctx.encode_udaf(aggregate_expr.fun())?, |
| 55 | + aggregate_expr.ignore_nulls(), |
| 56 | + aggregate_expr.is_distinct(), |
| 57 | + ) |
| 58 | + } else if let Some(sliding) = expr.downcast_ref::<SlidingAggregateWindowExpr>() { |
| 59 | + let aggregate_expr = sliding.get_aggregate_expr(); |
| 60 | + ( |
| 61 | + physical_window_expr_node::WindowFunction::UserDefinedAggrFunction( |
| 62 | + aggregate_expr.fun().name().to_string(), |
| 63 | + ), |
| 64 | + ctx.encode_udaf(aggregate_expr.fun())?, |
| 65 | + aggregate_expr.ignore_nulls(), |
| 66 | + aggregate_expr.is_distinct(), |
| 67 | + ) |
| 68 | + } else if let Some(standard) = expr.downcast_ref::<StandardWindowExpr>() { |
| 69 | + if let Some(window_udf) = standard |
| 70 | + .get_standard_func_expr() |
| 71 | + .as_any() |
| 72 | + .downcast_ref::<WindowUDFExpr>() |
| 73 | + { |
| 74 | + // `WindowUDFExpr::args` returns the full, unfiltered argument list so |
| 75 | + // every argument survives the round-trip. |
| 76 | + args = window_udf.args().to_vec(); |
| 77 | + ( |
| 78 | + physical_window_expr_node::WindowFunction::UserDefinedWindowFunction( |
| 79 | + window_udf.fun().name().to_string(), |
| 80 | + ), |
| 81 | + ctx.encode_udwf(window_udf.fun().as_ref())?, |
| 82 | + false, |
| 83 | + false, |
| 84 | + ) |
| 85 | + } else { |
| 86 | + return not_impl_err!( |
| 87 | + "User-defined window function not supported: {window_expr:?}" |
| 88 | + ); |
| 89 | + } |
| 90 | + } else { |
| 91 | + return not_impl_err!("WindowExpr not supported: {window_expr:?}"); |
| 92 | + }; |
| 93 | + |
| 94 | + let args = ctx.encode_expressions(&args)?; |
| 95 | + let partition_by = ctx.encode_expressions(window_expr.partition_by())?; |
| 96 | + let order_by = window_expr |
| 97 | + .order_by() |
| 98 | + .iter() |
| 99 | + .map(|sort_expr| { |
| 100 | + Ok(protobuf::PhysicalSortExprNode { |
| 101 | + expr: Some(Box::new(ctx.encode_expr(&sort_expr.expr)?)), |
| 102 | + asc: !sort_expr.options.descending, |
| 103 | + nulls_first: sort_expr.options.nulls_first, |
| 104 | + }) |
| 105 | + }) |
| 106 | + .collect::<Result<Vec<_>>>()?; |
| 107 | + |
| 108 | + Ok(protobuf::PhysicalWindowExprNode { |
| 109 | + args, |
| 110 | + partition_by, |
| 111 | + order_by, |
| 112 | + window_frame: Some(encode_window_frame(window_frame.as_ref())?), |
| 113 | + window_function: Some(window_function), |
| 114 | + name: window_expr.name().to_string(), |
| 115 | + fun_definition, |
| 116 | + ignore_nulls, |
| 117 | + distinct, |
| 118 | + }) |
| 119 | +} |
| 120 | + |
| 121 | +pub(super) fn decode_physical_window_expr( |
| 122 | + proto: &protobuf::PhysicalWindowExprNode, |
| 123 | + ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>, |
| 124 | + input_schema: &Schema, |
| 125 | +) -> Result<Arc<dyn WindowExpr>> { |
| 126 | + let args = proto |
| 127 | + .args |
| 128 | + .iter() |
| 129 | + .map(|expr| ctx.decode_expr(expr, input_schema)) |
| 130 | + .collect::<Result<Vec<_>>>()?; |
| 131 | + let partition_by = proto |
| 132 | + .partition_by |
| 133 | + .iter() |
| 134 | + .map(|expr| ctx.decode_expr(expr, input_schema)) |
| 135 | + .collect::<Result<Vec<_>>>()?; |
| 136 | + let order_by = proto |
| 137 | + .order_by |
| 138 | + .iter() |
| 139 | + .map(|sort_expr| { |
| 140 | + let expr = sort_expr.expr.as_ref().ok_or_else(|| { |
| 141 | + internal_datafusion_err!( |
| 142 | + "Missing expr in window order_by sort expression" |
| 143 | + ) |
| 144 | + })?; |
| 145 | + Ok(PhysicalSortExpr { |
| 146 | + expr: ctx.decode_expr(expr, input_schema)?, |
| 147 | + options: SortOptions { |
| 148 | + descending: !sort_expr.asc, |
| 149 | + nulls_first: sort_expr.nulls_first, |
| 150 | + }, |
| 151 | + }) |
| 152 | + }) |
| 153 | + .collect::<Result<Vec<_>>>()?; |
| 154 | + let window_frame = proto |
| 155 | + .window_frame |
| 156 | + .as_ref() |
| 157 | + .map(decode_window_frame) |
| 158 | + .transpose()? |
| 159 | + .ok_or_else(|| { |
| 160 | + internal_datafusion_err!("Missing required field 'window_frame' in protobuf") |
| 161 | + })?; |
| 162 | + let function = match proto.window_function.as_ref() { |
| 163 | + Some(physical_window_expr_node::WindowFunction::UserDefinedAggrFunction( |
| 164 | + name, |
| 165 | + )) => WindowFunctionDefinition::AggregateUDF( |
| 166 | + ctx.decode_udaf(name, proto.fun_definition.as_deref())?, |
| 167 | + ), |
| 168 | + Some(physical_window_expr_node::WindowFunction::UserDefinedWindowFunction( |
| 169 | + name, |
| 170 | + )) => WindowFunctionDefinition::WindowUDF( |
| 171 | + ctx.decode_udwf(name, proto.fun_definition.as_deref())?, |
| 172 | + ), |
| 173 | + None => { |
| 174 | + return internal_err!("Missing required field 'window_function' in protobuf"); |
| 175 | + } |
| 176 | + }; |
| 177 | + |
| 178 | + let name = proto.name.clone(); |
| 179 | + // TODO: Remove extended_schema if functions are all UDAF |
| 180 | + let extended_schema = schema_add_window_field(&args, input_schema, &function, &name)?; |
| 181 | + create_window_expr( |
| 182 | + &function, |
| 183 | + name, |
| 184 | + &args, |
| 185 | + &partition_by, |
| 186 | + &order_by, |
| 187 | + Arc::new(window_frame), |
| 188 | + extended_schema, |
| 189 | + proto.ignore_nulls, |
| 190 | + proto.distinct, |
| 191 | + None, |
| 192 | + ) |
| 193 | +} |
| 194 | + |
| 195 | +fn encode_window_frame(window_frame: &WindowFrame) -> Result<protobuf::WindowFrame> { |
| 196 | + let units = match window_frame.units { |
| 197 | + WindowFrameUnits::Rows => protobuf::WindowFrameUnits::Rows, |
| 198 | + WindowFrameUnits::Range => protobuf::WindowFrameUnits::Range, |
| 199 | + WindowFrameUnits::Groups => protobuf::WindowFrameUnits::Groups, |
| 200 | + }; |
| 201 | + Ok(protobuf::WindowFrame { |
| 202 | + window_frame_units: units.into(), |
| 203 | + start_bound: Some(encode_window_frame_bound(&window_frame.start_bound)?), |
| 204 | + end_bound: Some(protobuf::window_frame::EndBound::Bound( |
| 205 | + encode_window_frame_bound(&window_frame.end_bound)?, |
| 206 | + )), |
| 207 | + }) |
| 208 | +} |
| 209 | + |
| 210 | +fn encode_window_frame_bound( |
| 211 | + bound: &WindowFrameBound, |
| 212 | +) -> Result<protobuf::WindowFrameBound> { |
| 213 | + let encode_value = |value: &ScalarValue| -> Result<protobuf_common::ScalarValue> { |
| 214 | + Ok(value.try_into()?) |
| 215 | + }; |
| 216 | + Ok(match bound { |
| 217 | + WindowFrameBound::CurrentRow => protobuf::WindowFrameBound { |
| 218 | + window_frame_bound_type: protobuf::WindowFrameBoundType::CurrentRow.into(), |
| 219 | + bound_value: None, |
| 220 | + }, |
| 221 | + WindowFrameBound::Preceding(value) => protobuf::WindowFrameBound { |
| 222 | + window_frame_bound_type: protobuf::WindowFrameBoundType::Preceding.into(), |
| 223 | + bound_value: Some(encode_value(value)?), |
| 224 | + }, |
| 225 | + WindowFrameBound::Following(value) => protobuf::WindowFrameBound { |
| 226 | + window_frame_bound_type: protobuf::WindowFrameBoundType::Following.into(), |
| 227 | + bound_value: Some(encode_value(value)?), |
| 228 | + }, |
| 229 | + }) |
| 230 | +} |
| 231 | + |
| 232 | +fn decode_window_frame(window_frame: &protobuf::WindowFrame) -> Result<WindowFrame> { |
| 233 | + let units = protobuf::WindowFrameUnits::try_from(window_frame.window_frame_units) |
| 234 | + .map_err(|_| { |
| 235 | + internal_datafusion_err!( |
| 236 | + "Received a WindowFrame message with unknown WindowFrameUnits {}", |
| 237 | + window_frame.window_frame_units |
| 238 | + ) |
| 239 | + })?; |
| 240 | + let units = match units { |
| 241 | + protobuf::WindowFrameUnits::Rows => WindowFrameUnits::Rows, |
| 242 | + protobuf::WindowFrameUnits::Range => WindowFrameUnits::Range, |
| 243 | + protobuf::WindowFrameUnits::Groups => WindowFrameUnits::Groups, |
| 244 | + }; |
| 245 | + let start_bound = |
| 246 | + decode_window_frame_bound(window_frame.start_bound.as_ref().ok_or_else( |
| 247 | + || internal_datafusion_err!("Missing start_bound in WindowFrame"), |
| 248 | + )?)?; |
| 249 | + let end_bound = window_frame |
| 250 | + .end_bound |
| 251 | + .as_ref() |
| 252 | + .map(|end_bound| match end_bound { |
| 253 | + protobuf::window_frame::EndBound::Bound(bound) => { |
| 254 | + decode_window_frame_bound(bound) |
| 255 | + } |
| 256 | + }) |
| 257 | + .transpose()? |
| 258 | + .unwrap_or(WindowFrameBound::CurrentRow); |
| 259 | + Ok(WindowFrame::new_bounds(units, start_bound, end_bound)) |
| 260 | +} |
| 261 | + |
| 262 | +fn decode_window_frame_bound( |
| 263 | + bound: &protobuf::WindowFrameBound, |
| 264 | +) -> Result<WindowFrameBound> { |
| 265 | + let decode_value = |value: &protobuf_common::ScalarValue| -> Result<ScalarValue> { |
| 266 | + Ok(ScalarValue::try_from(value)?) |
| 267 | + }; |
| 268 | + let bound_type = protobuf::WindowFrameBoundType::try_from( |
| 269 | + bound.window_frame_bound_type, |
| 270 | + ) |
| 271 | + .map_err(|_| { |
| 272 | + internal_datafusion_err!( |
| 273 | + "Received a WindowFrameBound message with unknown WindowFrameBoundType {}", |
| 274 | + bound.window_frame_bound_type |
| 275 | + ) |
| 276 | + })?; |
| 277 | + match bound_type { |
| 278 | + protobuf::WindowFrameBoundType::CurrentRow => Ok(WindowFrameBound::CurrentRow), |
| 279 | + protobuf::WindowFrameBoundType::Preceding => match &bound.bound_value { |
| 280 | + Some(value) => Ok(WindowFrameBound::Preceding(decode_value(value)?)), |
| 281 | + None => Ok(WindowFrameBound::Preceding(ScalarValue::UInt64(None))), |
| 282 | + }, |
| 283 | + protobuf::WindowFrameBoundType::Following => match &bound.bound_value { |
| 284 | + Some(value) => Ok(WindowFrameBound::Following(decode_value(value)?)), |
| 285 | + None => Ok(WindowFrameBound::Following(ScalarValue::UInt64(None))), |
| 286 | + }, |
| 287 | + } |
| 288 | +} |
0 commit comments