Skip to content

Commit 38b2a8b

Browse files
sjhddhademelamiralamb
authored andcommitted
feat: implement Spark-compatible weekday function (apache#22740)
## Which issue does this PR close? - Closes apache#22599. ## Rationale for this change Implements the Spark `weekday` function as part of the datafusion-spark function library (#15914). Spark's `weekday` returns the day-of-week as a 0-indexed integer with Monday = 0 .. Sunday = 6, which differs from the existing `dayofweek` (1-indexed, Sunday = 1). This fills a gap for Spark-compatibility consumers. ## What changes are included in this PR? - New `SparkWeekDay` scalar UDF in `datafusion/spark/src/function/datetime/weekday.rs`, modeled on the sibling `monthname` function. - Accepts Date and (implicitly coerced) Timestamp input; returns `Int32`. - Uses Arrow's `DatePart::DayOfWeekMonday0` kernel, which is exactly 0=Monday .. 6=Sunday — a direct match for Spark semantics. - Null input propagates to null output for both scalar and array paths. - Registered in the datetime function `mod.rs` (UDF macro, doc export, `functions()` list). ## Are these changes tested? Yes: - Rust unit tests: return-field nullability, scalar evaluation (incl. null), and array evaluation. - sqllogictest coverage in `datafusion/sqllogictest/test_files/spark/datetime/weekday.slt`: scalar dates for all 7 weekdays, array input, TIMESTAMP / TIMESTAMP_NTZ / LTZ coercion, null handling, and argument-type / zero-argument error cases. The original PySpark 3.5.5 reference (`weekday('2009-07-30') = 3`) is preserved and validated. ## Are there any user-facing changes? Adds the new Spark-compatible scalar function `weekday`. No breaking changes. --- Note: I noticed there is an existing draft PR apache#22601 for the same function (last updated 2026-05-28). It appears to be a stale draft, so I've opened this as a complete, tested implementation — happy to defer or collaborate if the draft author is still active. Signed-off-by: sjhddh <[email protected]> Co-authored-by: sjhddh <[email protected]> Co-authored-by: Andrew Lamb <[email protected]>
1 parent edacd3a commit 38b2a8b

3 files changed

Lines changed: 305 additions & 2 deletions

File tree

‎datafusion/spark/src/function/datetime/mod.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ pub mod time_trunc;
3232
pub mod to_utc_timestamp;
3333
pub mod trunc;
3434
pub mod unix;
35+
pub mod weekday;
3536

3637
use datafusion_expr::ScalarUDF;
3738
use datafusion_functions::make_udf_function;
@@ -59,6 +60,7 @@ make_udf_function!(time_trunc::SparkTimeTrunc, time_trunc);
5960
make_udf_function!(to_utc_timestamp::SparkToUtcTimestamp, to_utc_timestamp);
6061
make_udf_function!(trunc::SparkTrunc, trunc);
6162
make_udf_function!(unix::SparkUnixDate, unix_date);
63+
make_udf_function!(weekday::SparkWeekDay, weekday);
6264
make_udf_function!(
6365
unix::SparkUnixTimestamp,
6466
unix_micros,
@@ -186,6 +188,11 @@ pub mod expr_fn {
186188
"Returns the number of seconds since epoch (1970-01-01 00:00:00 UTC) for the given timestamp `ts`.",
187189
ts
188190
));
191+
export_functions!((
192+
weekday,
193+
"Returns the day of the week for date/timestamp as an integer where Monday = 0, Tuesday = 1, ..., Sunday = 6.",
194+
arg1
195+
));
189196
}
190197

191198
pub fn functions() -> Vec<Arc<ScalarUDF>> {
@@ -212,5 +219,6 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {
212219
unix_micros(),
213220
unix_millis(),
214221
unix_seconds(),
222+
weekday(),
215223
]
216224
}
Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use std::sync::Arc;
19+
20+
use arrow::array::AsArray;
21+
use arrow::compute::{DatePart, date_part};
22+
use arrow::datatypes::{DataType, Field, FieldRef, Int32Type};
23+
use datafusion_common::types::{NativeType, logical_date};
24+
use datafusion_common::utils::take_function_args;
25+
use datafusion_common::{Result, ScalarValue, internal_err};
26+
use datafusion_expr::{
27+
Coercion, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
28+
Signature, TypeSignatureClass, Volatility,
29+
};
30+
31+
/// Spark-compatible `weekday` expression.
32+
/// Returns the day of the week for a date or timestamp as an integer index where
33+
/// Monday = 0, Tuesday = 1, ..., Sunday = 6.
34+
///
35+
/// Note: this differs from `dayofweek`, which is 1-indexed with Sunday = 1.
36+
///
37+
/// <https://spark.apache.org/docs/latest/api/sql/index.html#weekday>
38+
#[derive(Debug, PartialEq, Eq, Hash)]
39+
pub struct SparkWeekDay {
40+
signature: Signature,
41+
}
42+
43+
impl Default for SparkWeekDay {
44+
fn default() -> Self {
45+
Self::new()
46+
}
47+
}
48+
49+
impl SparkWeekDay {
50+
pub fn new() -> Self {
51+
Self {
52+
signature: Signature::coercible(
53+
vec![Coercion::new_implicit(
54+
TypeSignatureClass::Native(logical_date()),
55+
vec![TypeSignatureClass::Timestamp],
56+
NativeType::Date,
57+
)],
58+
Volatility::Immutable,
59+
),
60+
}
61+
}
62+
}
63+
64+
impl ScalarUDFImpl for SparkWeekDay {
65+
fn name(&self) -> &str {
66+
"weekday"
67+
}
68+
69+
fn signature(&self) -> &Signature {
70+
&self.signature
71+
}
72+
73+
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
74+
internal_err!("return_field_from_args should be used instead")
75+
}
76+
77+
fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
78+
let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
79+
Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable)))
80+
}
81+
82+
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
83+
let [arg] = take_function_args(self.name(), args.args)?;
84+
match arg {
85+
ColumnarValue::Scalar(scalar) => {
86+
if scalar.is_null() {
87+
return Ok(ColumnarValue::Scalar(ScalarValue::Int32(None)));
88+
}
89+
let arr = scalar.to_array_of_size(1)?;
90+
// `DayOfWeekMonday0` returns 0..=6 with Monday = 0, which
91+
// matches Spark `weekday` semantics exactly.
92+
let weekday_arr = date_part(&arr, DatePart::DayOfWeekMonday0)?;
93+
let value = weekday_arr.as_primitive::<Int32Type>().value(0);
94+
Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some(value))))
95+
}
96+
ColumnarValue::Array(arr) => {
97+
let weekday_arr = date_part(&arr, DatePart::DayOfWeekMonday0)?;
98+
Ok(ColumnarValue::Array(weekday_arr))
99+
}
100+
}
101+
}
102+
}
103+
104+
#[cfg(test)]
105+
mod tests {
106+
use super::*;
107+
use arrow::array::{Date32Array, Int32Array};
108+
109+
#[test]
110+
fn test_weekday_return_field_nullability_matches_input() {
111+
let func = SparkWeekDay::new();
112+
113+
let non_nullable_arg = Arc::new(Field::new("arg", DataType::Date32, false));
114+
let nullable_arg = Arc::new(Field::new("arg", DataType::Date32, true));
115+
116+
let non_nullable_out = func
117+
.return_field_from_args(ReturnFieldArgs {
118+
arg_fields: &[Arc::clone(&non_nullable_arg)],
119+
scalar_arguments: &[None],
120+
})
121+
.expect("non-nullable arg should succeed");
122+
assert_eq!(non_nullable_out.data_type(), &DataType::Int32);
123+
assert!(!non_nullable_out.is_nullable());
124+
125+
let nullable_out = func
126+
.return_field_from_args(ReturnFieldArgs {
127+
arg_fields: &[Arc::clone(&nullable_arg)],
128+
scalar_arguments: &[None],
129+
})
130+
.expect("nullable arg should succeed");
131+
assert_eq!(nullable_out.data_type(), &DataType::Int32);
132+
assert!(nullable_out.is_nullable());
133+
}
134+
135+
#[test]
136+
fn test_weekday_scalar() -> Result<()> {
137+
let func = SparkWeekDay::new();
138+
139+
// 2024-03-15 is a Friday -> Spark weekday = 4 (Mon=0).
140+
let result = func.invoke_with_args(ScalarFunctionArgs {
141+
args: vec![ColumnarValue::Scalar(ScalarValue::Date32(Some(19797)))],
142+
arg_fields: vec![Arc::new(Field::new("arg", DataType::Date32, true))],
143+
number_rows: 1,
144+
return_field: Arc::new(Field::new("weekday", DataType::Int32, true)),
145+
config_options: Arc::new(Default::default()),
146+
})?;
147+
match result {
148+
ColumnarValue::Scalar(ScalarValue::Int32(Some(v))) => assert_eq!(v, 4),
149+
other => panic!("unexpected result: {other:?}"),
150+
}
151+
152+
// NULL input -> NULL output.
153+
let result = func.invoke_with_args(ScalarFunctionArgs {
154+
args: vec![ColumnarValue::Scalar(ScalarValue::Date32(None))],
155+
arg_fields: vec![Arc::new(Field::new("arg", DataType::Date32, true))],
156+
number_rows: 1,
157+
return_field: Arc::new(Field::new("weekday", DataType::Int32, true)),
158+
config_options: Arc::new(Default::default()),
159+
})?;
160+
match result {
161+
ColumnarValue::Scalar(ScalarValue::Int32(None)) => {}
162+
other => panic!("unexpected result: {other:?}"),
163+
}
164+
165+
Ok(())
166+
}
167+
168+
#[test]
169+
fn test_weekday_array() -> Result<()> {
170+
let func = SparkWeekDay::new();
171+
172+
// 2024-01-01 Mon(0), 2024-01-06 Sat(5), 2024-01-07 Sun(6), NULL.
173+
let input = Date32Array::from(vec![Some(19723), Some(19728), Some(19729), None]);
174+
let result = func.invoke_with_args(ScalarFunctionArgs {
175+
args: vec![ColumnarValue::Array(Arc::new(input))],
176+
arg_fields: vec![Arc::new(Field::new("arg", DataType::Date32, true))],
177+
number_rows: 4,
178+
return_field: Arc::new(Field::new("weekday", DataType::Int32, true)),
179+
config_options: Arc::new(Default::default()),
180+
})?;
181+
match result {
182+
ColumnarValue::Array(arr) => {
183+
let expected = Int32Array::from(vec![Some(0), Some(5), Some(6), None]);
184+
assert_eq!(arr.as_primitive::<Int32Type>(), &expected);
185+
}
186+
other => panic!("unexpected result: {other:?}"),
187+
}
188+
189+
Ok(())
190+
}
191+
}

‎datafusion/sqllogictest/test_files/spark/datetime/weekday.slt‎

Lines changed: 106 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,109 @@
2323

2424
## Original Query: SELECT weekday('2009-07-30');
2525
## PySpark 3.5.5 Result: {'weekday(2009-07-30)': 3, 'typeof(weekday(2009-07-30))': 'int', 'typeof(2009-07-30)': 'string'}
26-
#query
27-
#SELECT weekday('2009-07-30'::string);
26+
# Spark `weekday` is 0-indexed with Monday = 0 .. Sunday = 6.
27+
# 2009-07-30 is a Thursday -> 3.
28+
query I
29+
SELECT weekday('2009-07-30'::DATE);
30+
----
31+
3
32+
33+
# All seven days of one week (2024-01-01 is a Monday).
34+
query I
35+
SELECT weekday('2024-01-01'::DATE);
36+
----
37+
0
38+
39+
query I
40+
SELECT weekday('2024-01-02'::DATE);
41+
----
42+
1
43+
44+
query I
45+
SELECT weekday('2024-01-03'::DATE);
46+
----
47+
2
48+
49+
query I
50+
SELECT weekday('2024-01-04'::DATE);
51+
----
52+
3
53+
54+
query I
55+
SELECT weekday('2024-01-05'::DATE);
56+
----
57+
4
58+
59+
query I
60+
SELECT weekday('2024-01-06'::DATE);
61+
----
62+
5
63+
64+
query I
65+
SELECT weekday('2024-01-07'::DATE);
66+
----
67+
6
68+
69+
# NULL handling
70+
query I
71+
SELECT weekday(NULL::DATE);
72+
----
73+
NULL
74+
75+
# Array input (mix of weekdays and NULL)
76+
query I
77+
SELECT weekday(d) FROM (VALUES ('2024-01-01'::DATE), ('2024-01-06'::DATE), ('2024-01-07'::DATE), (NULL::DATE)) AS t(d);
78+
----
79+
0
80+
5
81+
6
82+
NULL
83+
84+
# Timestamp input: Spark coerces TIMESTAMP/TIMESTAMP_NTZ to DATE before evaluation
85+
query I
86+
SELECT weekday('2009-07-30 12:34:56'::TIMESTAMP);
87+
----
88+
3
89+
90+
query I
91+
SELECT weekday(NULL::TIMESTAMP);
92+
----
93+
NULL
94+
95+
# Timestamp array input
96+
query I
97+
SELECT weekday(ts) FROM (VALUES
98+
('2024-01-01 01:02:03'::TIMESTAMP),
99+
('2024-01-06 10:20:30'::TIMESTAMP),
100+
('2024-01-07 23:59:59'::TIMESTAMP),
101+
(NULL::TIMESTAMP)
102+
) AS t(ts);
103+
----
104+
0
105+
5
106+
6
107+
NULL
108+
109+
# TIMESTAMP_NTZ (Timestamp without timezone) — explicit Microsecond precision
110+
query I
111+
SELECT weekday(arrow_cast('2009-07-30 09:15:00', 'Timestamp(Microsecond, None)'));
112+
----
113+
3
114+
115+
# TIMESTAMP with timezone (Spark TIMESTAMP / LTZ) — coerces to Date32
116+
query I
117+
SELECT weekday(arrow_cast('2024-01-07 03:00:00', 'Timestamp(Nanosecond, Some("UTC"))'));
118+
----
119+
6
120+
121+
# Error: wrong argument type (string without cast)
122+
statement error Function 'weekday' requires Date, but received String
123+
SELECT weekday('not-a-date');
124+
125+
# Error: wrong argument type (integer)
126+
statement error Function 'weekday' requires Date, but received Int64
127+
SELECT weekday(123);
128+
129+
# Error: no arguments
130+
statement error 'weekday' does not support zero arguments
131+
SELECT weekday();

0 commit comments

Comments
 (0)