Skip to content

Commit d2bd93c

Browse files
authored
fix(spark): accept STRING and TIMESTAMP start dates in next_day (apache#26127)
## Which issue does this PR close? - Closes apache#23889. ## Rationale for this change Spark's `NextDay` casts its first argument to DATE, so `next_day('2015-07-23', 'Mon')` and `next_day(TIMESTAMP '2015-07-23 12:12:12', 'Mon')` both work. `datafusion-spark` declared `Exact(Date32, Utf8)` and failed at planning for STRING and TIMESTAMP inputs. ## What changes are included in this PR? - `next_day` now uses a coercible signature: the first argument is implicitly cast to DATE from STRING or TIMESTAMP, and the day-of-week is any string type (numeric is still coerced to string). - The kernel is unchanged. - In `next_day.slt`, three existing `query error` expectations changed to the new planner messages (arity and non-date first argument), and the unparseable-string case now returns a cast error. They still fail at planning or execution as before. ## What is the testing strategy for this PR? New cases in `spark/datetime/next_day.slt`: string, string with a time part, timestamp literal, string column with NULL, timestamp column, and `LargeUtf8`/`Utf8View` day-of-week. I compared the expected results with `pyspark==4.0.0`, ANSI on and off. ## Are there any user-facing changes? `next_day` accepts more argument types. No API changes. ## Known limitations / unknowns - An unparseable date string always errors. Spark returns NULL with ANSI off and errors with ANSI on. This looks tied to the ANSI work in #23890 and apache#23929. - I did not verify timestamps with time zones near midnight against Spark's session-time-zone cast. - apache#23892 touches the same files, so I expect a small rebase if it merges first. - AI assistance was used to implement this. I reviewed and understand the change, and checked the results against Spark. --------- Signed-off-by: Harsh Ande <[email protected]>
1 parent 80dda10 commit d2bd93c

2 files changed

Lines changed: 103 additions & 9 deletions

File tree

‎datafusion/spark/src/function/datetime/next_day.rs‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,11 @@ use std::sync::Arc;
77
use arrow::array::{ArrayRef, AsArray, Date32Array, StringArrayType};
88
use arrow::datatypes::{DataType, Date32Type, Field, FieldRef};
99
use chrono::{Datelike, Weekday};
10+
use datafusion_common::types::{NativeType, logical_date, logical_string};
1011
use datafusion_common::{Result, ScalarValue, exec_err, internal_err};
1112
use datafusion_expr::{
12-
ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
13-
Volatility,
13+
Coercion, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
14+
Signature, TypeSignatureClass, Volatility,
1415
};
1516

1617
/// <https://spark.apache.org/docs/latest/api/sql/index.html#next_day>
@@ -28,8 +29,24 @@ impl Default for SparkNextDay {
2829
impl SparkNextDay {
2930
pub fn new() -> Self {
3031
Self {
31-
signature: Signature::exact(
32-
vec![DataType::Date32, DataType::Utf8],
32+
// Spark casts the start date to DATE, so STRING and TIMESTAMP are accepted
33+
signature: Signature::coercible(
34+
vec![
35+
Coercion::new_implicit(
36+
TypeSignatureClass::Native(logical_date()),
37+
vec![
38+
TypeSignatureClass::Native(logical_string()),
39+
TypeSignatureClass::Timestamp,
40+
],
41+
NativeType::Date,
42+
),
43+
Coercion::new_implicit(
44+
TypeSignatureClass::Native(logical_string()),
45+
// Preserve the previous Exact signature's weekday-to-string coercion.
46+
vec![TypeSignatureClass::Any],
47+
NativeType::String,
48+
),
49+
],
3350
Volatility::Immutable,
3451
),
3552
}

‎datafusion/sqllogictest/test_files/spark/datetime/next_day.slt‎

Lines changed: 82 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,29 +29,40 @@ SELECT next_day('2015-01-14'::DATE, ' MO '::string);
2929
----
3030
NULL
3131

32-
query error Failed to coerce arguments to satisfy a call to 'next_day' function
32+
query error Function 'next_day' expects 2 arguments but received 1
3333
SELECT next_day('2015-07-27'::DATE);
3434

35-
query error Failed to coerce arguments to satisfy a call to 'next_day' function
35+
query error Function 'next_day' expects 2 arguments but received 1
3636
SELECT next_day('Sun'::string);
3737

3838
query error 'next_day' does not support zero arguments
3939
SELECT next_day();
4040

41-
query error Failed to coerce arguments to satisfy a call to 'next_day' function
41+
query error Function 'next_day' requires Date, but received Int32
4242
SELECT next_day(1::int, 'Sun'::string);
4343

44-
query error Failed to coerce arguments to satisfy a call to 'next_day' function
44+
query error Function 'next_day' expects 2 arguments but received 3
4545
SELECT next_day('2015-07-27'::DATE, 'Sat'::string, 'Sun'::string);
4646

47-
query error Failed to coerce arguments to satisfy a call to 'next_day' function
47+
query error Cannot cast string 'invalid_date' to value of Date32 type
4848
SELECT next_day('invalid_date'::string, 'Mon'::string);
4949

5050
query D
5151
SELECT next_day('2000-01-01'::DATE, 2.0::float);
5252
----
5353
NULL
5454

55+
# Preserve implicit weekday-to-string conversions.
56+
query D
57+
SELECT next_day(DATE '2015-07-23', arrow_cast('Mon', 'Binary'));
58+
----
59+
2015-07-27
60+
61+
query D
62+
SELECT next_day(DATE '2015-07-23', true);
63+
----
64+
NULL
65+
5566
query D
5667
SELECT next_day('2020-01-01'::DATE, 'invalid_day'::string);
5768
----
@@ -87,3 +98,69 @@ query I
8798
SELECT arrow_cast(next_day(arrow_cast(95026230, 'Date32'), 'Tue'::string), 'Int32');
8899
----
89100
95026237
101+
102+
# https://github.com/apache/datafusion/issues/23889
103+
# Spark casts the start date to DATE, so STRING and TIMESTAMP are accepted
104+
query D
105+
SELECT next_day('2015-07-23'::string, 'Mon'::string);
106+
----
107+
2015-07-27
108+
109+
query D
110+
SELECT next_day('2015-07-23 12:12:12'::string, 'Mon'::string);
111+
----
112+
2015-07-27
113+
114+
query D
115+
SELECT next_day('2015-07-23 12:12:12'::timestamp, 'Mon'::string);
116+
----
117+
2015-07-27
118+
119+
# Timestamp-to-date conversion must round down before the Unix epoch.
120+
query D
121+
SELECT next_day(TIMESTAMP '1969-12-31 23:59:59', 'Thu');
122+
----
123+
1970-01-01
124+
125+
query D
126+
SELECT next_day(a, 'Mon'::string)
127+
FROM VALUES
128+
('2015-07-23'::string),
129+
(NULL::string) as t(a);
130+
----
131+
2015-07-27
132+
NULL
133+
134+
# Timestamp column with a day-of-week column
135+
query D
136+
SELECT next_day(a, b)
137+
FROM VALUES
138+
('2015-07-23 12:12:12'::timestamp, 'Mon'::string),
139+
('2015-07-23 23:59:59'::timestamp, 'Fri'::string),
140+
(NULL::timestamp, 'Mon'::string) as t(a, b);
141+
----
142+
2015-07-27
143+
2015-07-24
144+
NULL
145+
146+
# LargeUtf8 / Utf8View start dates
147+
query D
148+
SELECT next_day(arrow_cast('2015-07-23', 'LargeUtf8'), 'Mon');
149+
----
150+
2015-07-27
151+
152+
query D
153+
SELECT next_day(arrow_cast('2015-07-23', 'Utf8View'), 'Mon');
154+
----
155+
2015-07-27
156+
157+
# String start date and LargeUtf8 / Utf8View day-of-week
158+
query D
159+
SELECT next_day('2015-07-23', arrow_cast('Mon', 'LargeUtf8'));
160+
----
161+
2015-07-27
162+
163+
query D
164+
SELECT next_day('2015-07-23', arrow_cast('Mon', 'Utf8View'));
165+
----
166+
2015-07-27

0 commit comments

Comments
 (0)