|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +use arrow::datatypes::{DataType, Field, FieldRef}; |
| 19 | +use datafusion_common::types::logical_date; |
| 20 | +use datafusion_common::{ |
| 21 | + Result, ScalarValue, internal_err, types::logical_string, utils::take_function_args, |
| 22 | +}; |
| 23 | +use datafusion_expr::expr::ScalarFunction; |
| 24 | +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; |
| 25 | +use datafusion_expr::{ |
| 26 | + Coercion, ColumnarValue, Expr, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, |
| 27 | + Signature, TypeSignature, TypeSignatureClass, Volatility, |
| 28 | +}; |
| 29 | +use std::{any::Any, sync::Arc}; |
| 30 | + |
| 31 | +/// Wrapper around datafusion date_part function to handle |
| 32 | +/// Spark behavior returning day of the week 1-indexed instead of 0-indexed and different part aliases. |
| 33 | +/// <https://spark.apache.org/docs/latest/api/sql/index.html#date_part> |
| 34 | +#[derive(Debug, PartialEq, Eq, Hash)] |
| 35 | +pub struct SparkDatePart { |
| 36 | + signature: Signature, |
| 37 | + aliases: Vec<String>, |
| 38 | +} |
| 39 | + |
| 40 | +impl Default for SparkDatePart { |
| 41 | + fn default() -> Self { |
| 42 | + Self::new() |
| 43 | + } |
| 44 | +} |
| 45 | + |
| 46 | +impl SparkDatePart { |
| 47 | + pub fn new() -> Self { |
| 48 | + Self { |
| 49 | + signature: Signature::one_of( |
| 50 | + vec![ |
| 51 | + TypeSignature::Coercible(vec![ |
| 52 | + Coercion::new_exact(TypeSignatureClass::Native(logical_string())), |
| 53 | + Coercion::new_exact(TypeSignatureClass::Timestamp), |
| 54 | + ]), |
| 55 | + TypeSignature::Coercible(vec![ |
| 56 | + Coercion::new_exact(TypeSignatureClass::Native(logical_string())), |
| 57 | + Coercion::new_exact(TypeSignatureClass::Native(logical_date())), |
| 58 | + ]), |
| 59 | + ], |
| 60 | + Volatility::Immutable, |
| 61 | + ), |
| 62 | + aliases: vec![String::from("datepart")], |
| 63 | + } |
| 64 | + } |
| 65 | +} |
| 66 | + |
| 67 | +impl ScalarUDFImpl for SparkDatePart { |
| 68 | + fn as_any(&self) -> &dyn Any { |
| 69 | + self |
| 70 | + } |
| 71 | + |
| 72 | + fn name(&self) -> &str { |
| 73 | + "date_part" |
| 74 | + } |
| 75 | + |
| 76 | + fn aliases(&self) -> &[String] { |
| 77 | + &self.aliases |
| 78 | + } |
| 79 | + |
| 80 | + fn signature(&self) -> &Signature { |
| 81 | + &self.signature |
| 82 | + } |
| 83 | + |
| 84 | + fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> { |
| 85 | + internal_err!("Use return_field_from_args in this case instead.") |
| 86 | + } |
| 87 | + |
| 88 | + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> { |
| 89 | + let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); |
| 90 | + |
| 91 | + Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) |
| 92 | + } |
| 93 | + |
| 94 | + fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> { |
| 95 | + internal_err!("spark date_part should have been simplified to standard date_part") |
| 96 | + } |
| 97 | + |
| 98 | + fn simplify( |
| 99 | + &self, |
| 100 | + args: Vec<Expr>, |
| 101 | + _info: &SimplifyContext, |
| 102 | + ) -> Result<ExprSimplifyResult> { |
| 103 | + let [part_expr, date_expr] = take_function_args(self.name(), args)?; |
| 104 | + |
| 105 | + let part = match part_expr.as_literal() { |
| 106 | + Some(ScalarValue::Utf8(Some(v))) |
| 107 | + | Some(ScalarValue::Utf8View(Some(v))) |
| 108 | + | Some(ScalarValue::LargeUtf8(Some(v))) => v.to_lowercase(), |
| 109 | + _ => { |
| 110 | + return internal_err!( |
| 111 | + "First argument of `DATE_PART` must be non-null scalar Utf8" |
| 112 | + ); |
| 113 | + } |
| 114 | + }; |
| 115 | + |
| 116 | + // Map Spark-specific date part aliases to datafusion ones |
| 117 | + let part = match part.as_str() { |
| 118 | + "yearofweek" | "year_iso" => "isoyear", |
| 119 | + "dayofweek" => "dow", |
| 120 | + "dayofweek_iso" | "dow_iso" => "isodow", |
| 121 | + other => other, |
| 122 | + }; |
| 123 | + |
| 124 | + let part_expr = Expr::Literal(ScalarValue::new_utf8(part), None); |
| 125 | + |
| 126 | + let date_part_expr = Expr::ScalarFunction(ScalarFunction::new_udf( |
| 127 | + datafusion_functions::datetime::date_part(), |
| 128 | + vec![part_expr, date_expr], |
| 129 | + )); |
| 130 | + |
| 131 | + match part { |
| 132 | + // Add 1 for day-of-week parts to convert 0-indexed to 1-indexed |
| 133 | + "dow" | "isodow" => Ok(ExprSimplifyResult::Simplified( |
| 134 | + date_part_expr + Expr::Literal(ScalarValue::Int32(Some(1)), None), |
| 135 | + )), |
| 136 | + _ => Ok(ExprSimplifyResult::Simplified(date_part_expr)), |
| 137 | + } |
| 138 | + } |
| 139 | +} |
0 commit comments