Skip to content

Commit 8603ac8

Browse files
jbrockmendelclaude
andcommitted
BUG: return numpy scalars instead of ints from datetime64/timedelta64 astype(object)
convert_datetime_to_pyobject and convert_timedelta_to_pyobject returned raw Python ints for values that could not be represented as Python datetime.datetime or datetime.timedelta (sub-microsecond units, out-of-range years, leap seconds, overflowing timedelta days). This caused arr.astype(object) to silently produce integers, breaking roundtrips and surprising users. Replace those PyLong_FromLongLong fallbacks with PyArray_Scalar calls that return proper numpy.datetime64 / numpy.timedelta64 scalars. Because getitem now returns numpy scalars in those cases, int() and float() on datetime64/timedelta64 scalars would infinitely recurse through gentype_int -> array_int -> getitem -> gentype_int. Break the recursion by giving the two scalar types their own tp_as_number with nb_int/nb_float that raise TypeError, consistent with how int(datetime.datetime(...)) and int(datetime.timedelta(...)) already behave. Closes numpy#12550 Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
1 parent 758d986 commit 8603ac8

5 files changed

Lines changed: 172 additions & 20 deletions

File tree

‎numpy/_core/src/multiarray/_datetime.h‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -244,8 +244,9 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj,
244244
*
245245
* NaT (Not-a-time) is returned as None.
246246
* For D/W/Y/M (days or coarser), returns a datetime.date.
247-
* For μs/ms/s/m/h/D/W (microseconds or coarser), returns a datetime.datetime.
248-
* For ns/ps/fs/as (units shorter than microseconds), returns an integer.
247+
* For μs/ms/s/m/h (microseconds or coarser), returns a datetime.datetime.
248+
* For ns/ps/fs/as (units shorter than microseconds), or values outside the
249+
* range of Python's datetime, returns a numpy.datetime64 scalar.
249250
*/
250251
NPY_NO_EXPORT PyObject *
251252
convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta);
@@ -255,7 +256,9 @@ convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta);
255256
*
256257
* NaT (Not-a-time) is returned as None.
257258
* For μs/ms/s/m/h/D/W (microseconds or coarser), returns a datetime.timedelta.
258-
* For Y/M (non-linear units), generic units and ns/ps/fs/as (units shorter than microseconds), returns an integer.
259+
* For Y/M (non-linear units), generic units, ns/ps/fs/as (units shorter
260+
* than microseconds), or values outside datetime.timedelta range, returns
261+
* a numpy.timedelta64 scalar.
259262
*/
260263
NPY_NO_EXPORT PyObject *
261264
convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta);

‎numpy/_core/src/multiarray/datetime.c‎

Lines changed: 37 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2774,13 +2774,32 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj,
27742774
}
27752775
}
27762776

2777+
/*
2778+
* Creates a numpy datetime64 or timedelta64 scalar from a raw value
2779+
* and metadata. Used as a fallback when the value cannot be represented
2780+
* as a Python datetime.datetime or datetime.timedelta.
2781+
*/
2782+
static PyObject *
2783+
_make_datetime_or_timedelta_scalar(
2784+
int type_num, npy_int64 val, PyArray_DatetimeMetaData *meta)
2785+
{
2786+
PyArray_Descr *dtype = create_datetime_dtype(type_num, meta);
2787+
if (dtype == NULL) {
2788+
return NULL;
2789+
}
2790+
PyObject *ret = PyArray_Scalar(&val, dtype, NULL);
2791+
Py_DECREF(dtype);
2792+
return ret;
2793+
}
2794+
27772795
/*
27782796
* Converts a datetime into a PyObject *.
27792797
*
27802798
* NaT (Not-a-time) is returned as None.
27812799
* For D/W/Y/M (days or coarser), returns a datetime.date.
2782-
* For μs/ms/s/m/h/D/W (microseconds or coarser), returns a datetime.datetime.
2783-
* For ns/ps/fs/as (units shorter than microseconds), returns an integer.
2800+
* For μs/ms/s/m/h (microseconds or coarser), returns a datetime.datetime.
2801+
* For ns/ps/fs/as (units shorter than microseconds), or for values
2802+
* outside the range of Python's datetime, returns a numpy.datetime64 scalar.
27842803
*/
27852804
NPY_NO_EXPORT PyObject *
27862805
convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta)
@@ -2796,9 +2815,12 @@ convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta)
27962815
Py_RETURN_NONE;
27972816
}
27982817

2799-
/* If the type's precision is greater than microseconds, return an int */
2818+
/*
2819+
* If the type's precision is greater than microseconds,
2820+
* return a numpy datetime64 scalar.
2821+
*/
28002822
if (meta->base > NPY_FR_us) {
2801-
return PyLong_FromLongLong(dt);
2823+
return _make_datetime_or_timedelta_scalar(NPY_DATETIME, dt, meta);
28022824
}
28032825

28042826
/* Convert to a datetimestruct */
@@ -2809,10 +2831,10 @@ convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta)
28092831
/*
28102832
* If the year is outside the range of years supported by Python's
28112833
* datetime, or the datetime64 falls on a leap second,
2812-
* return a raw int.
2834+
* return a numpy datetime64 scalar.
28132835
*/
28142836
if (dts.year < 1 || dts.year > 9999 || dts.sec == 60) {
2815-
return PyLong_FromLongLong(dt);
2837+
return _make_datetime_or_timedelta_scalar(NPY_DATETIME, dt, meta);
28162838
}
28172839

28182840
/* If the type's precision is greater than days, return a datetime */
@@ -2963,7 +2985,9 @@ convert_timedelta_to_timedeltastruct(PyArray_DatetimeMetaData *meta,
29632985
*
29642986
* NaT (Not-a-time) is returned as None.
29652987
* For μs/ms/s/m/h/D/W (microseconds or coarser), returns a datetime.timedelta.
2966-
* For Y/M (non-linear units), generic units and ns/ps/fs/as (units shorter than microseconds), returns an integer.
2988+
* For Y/M (non-linear units), generic units, ns/ps/fs/as (units shorter
2989+
* than microseconds), or values outside datetime.timedelta range, returns
2990+
* a numpy.timedelta64 scalar.
29672991
*/
29682992
NPY_NO_EXPORT PyObject *
29692993
convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta)
@@ -2979,24 +3003,26 @@ convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta)
29793003

29803004
/*
29813005
* If the type's precision is greater than microseconds, is
2982-
* Y/M (nonlinear units), or is generic units, return an int
3006+
* Y/M (nonlinear units), or is generic units, return a
3007+
* numpy timedelta64 scalar.
29833008
*/
29843009
if (meta->base > NPY_FR_us ||
29853010
meta->base == NPY_FR_Y ||
29863011
meta->base == NPY_FR_M ||
29873012
meta->base == NPY_FR_GENERIC) {
2988-
return PyLong_FromLongLong(td);
3013+
return _make_datetime_or_timedelta_scalar(NPY_TIMEDELTA, td, meta);
29893014
}
29903015

29913016
if (convert_timedelta_to_timedeltastruct(meta, td, &tds) < 0) {
29923017
return NULL;
29933018
}
29943019

29953020
/*
2996-
* If it would overflow the datetime.timedelta days, return a raw int
3021+
* If it would overflow the datetime.timedelta days, return a
3022+
* numpy timedelta64 scalar.
29973023
*/
29983024
if (tds.day < -999999999 || tds.day > 999999999) {
2999-
return PyLong_FromLongLong(td);
3025+
return _make_datetime_or_timedelta_scalar(NPY_TIMEDELTA, td, meta);
30003026
}
30013027
else {
30023028
return PyDelta_FromDSU(tds.day, tds.sec, tds.us);

‎numpy/_core/src/multiarray/scalartypes.c.src‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3786,6 +3786,29 @@ static PyObject *
37863786
static PyNumberMethods @name@_arrtype_as_number;
37873787
/**end repeat**/
37883788

3789+
static PyNumberMethods datetime_arrtype_as_number;
3790+
static PyNumberMethods timedelta_arrtype_as_number;
3791+
3792+
static PyObject *
3793+
datetimetype_int(PyObject *self)
3794+
{
3795+
PyErr_Format(PyExc_TypeError,
3796+
"int() argument must be a string, a bytes-like object "
3797+
"or a real number, not '%.200s'",
3798+
Py_TYPE(self)->tp_name);
3799+
return NULL;
3800+
}
3801+
3802+
static PyObject *
3803+
datetimetype_float(PyObject *self)
3804+
{
3805+
PyErr_Format(PyExc_TypeError,
3806+
"float() argument must be a string or a real number, "
3807+
"not '%.200s'",
3808+
Py_TYPE(self)->tp_name);
3809+
return NULL;
3810+
}
3811+
37893812
/* Arithmetic methods -- only so we can override &, |, ^. */
37903813
NPY_NO_EXPORT PyNumberMethods bool_arrtype_as_number = {
37913814
.nb_bool = (inquiry)bool_arrtype_nonzero,
@@ -4476,6 +4499,21 @@ initialize_numeric_types(void)
44764499

44774500
/**end repeat**/
44784501

4502+
/*
4503+
* Datetime and timedelta types should not be convertible to int/float.
4504+
* Their getitem returns numpy scalars (datetime64/timedelta64), not
4505+
* Python scalars, so gentype_int/gentype_float would infinitely recurse.
4506+
*/
4507+
datetime_arrtype_as_number = gentype_as_number;
4508+
datetime_arrtype_as_number.nb_int = (unaryfunc)datetimetype_int;
4509+
datetime_arrtype_as_number.nb_float = (unaryfunc)datetimetype_float;
4510+
PyDatetimeArrType_Type.tp_as_number = &datetime_arrtype_as_number;
4511+
4512+
timedelta_arrtype_as_number = gentype_as_number;
4513+
timedelta_arrtype_as_number.nb_int = (unaryfunc)datetimetype_int;
4514+
timedelta_arrtype_as_number.nb_float = (unaryfunc)datetimetype_float;
4515+
PyTimedeltaArrType_Type.tp_as_number = &timedelta_arrtype_as_number;
4516+
44794517
PyStringArrType_Type.tp_alloc = NULL;
44804518
PyStringArrType_Type.tp_free = NULL;
44814519

‎numpy/_core/tests/test_array_coercion.py‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -425,15 +425,20 @@ def test_coercion_basic(self, dtype, scalar):
425425
[param(np.timedelta64(123, "ns"), id="timedelta64[ns]"),
426426
param(np.timedelta64(12, "generic"), id="timedelta64[generic]")])
427427
def test_coercion_timedelta_convert_to_number(self, dtype, scalar):
428-
# Only "ns" and "generic" timedeltas can be converted to numbers
429-
# so these are slightly special.
428+
# "ns" and "generic" timedeltas can be converted to numeric dtypes
429+
# via np.array(scalar, dtype=...) and .astype(), but integer item
430+
# assignment raises TypeError because it goes through int().
430431
arr = np.array(scalar, dtype=dtype)
431432
cast = np.array(scalar).astype(dtype)
432-
ass = np.ones((), dtype=dtype)
433-
ass[()] = scalar # raises, as would np.array([scalar], dtype=dtype)
434-
435433
assert_array_equal(arr, cast)
436-
assert_array_equal(cast, cast)
434+
435+
ass = np.ones((), dtype=dtype)
436+
if np.issubdtype(dtype, np.integer):
437+
with pytest.raises(TypeError):
438+
ass[()] = scalar
439+
else:
440+
ass[()] = scalar
441+
assert_array_equal(ass, cast)
437442

438443
@pytest.mark.parametrize("dtype", ["S6", "U6"])
439444
@pytest.mark.parametrize(["val", "unit"],

‎numpy/_core/tests/test_datetime.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -994,6 +994,86 @@ def test_pyobject_roundtrip(self):
994994
assert_equal(b.astype(object).astype(unit), b,
995995
f"Error roundtripping unit {unit}")
996996

997+
def test_astype_object_returns_scalar_not_int(self):
998+
# GH#12550: astype(object) should never return raw ints for
999+
# datetime64/timedelta64; it should return numpy scalars for
1000+
# values that can't be represented as Python datetime/timedelta.
1001+
1002+
# Sub-microsecond datetime64 should return datetime64 scalars
1003+
for unit in ["ns", "ps", "fs", "as"]:
1004+
dts = np.array([1000], dtype=f"M8[{unit}]")
1005+
obj = dts.astype(object)
1006+
assert isinstance(obj[0], np.datetime64), (
1007+
f"M8[{unit}] astype(object) returned {type(obj[0])}")
1008+
assert obj[0] == dts[0]
1009+
1010+
tds = dts.view(f"m8[{unit}]")
1011+
obj_td = tds.astype(object)
1012+
assert isinstance(obj_td[0], np.timedelta64), (
1013+
f"m8[{unit}] astype(object) returned {type(obj_td[0])}")
1014+
assert obj_td[0] == tds[0]
1015+
1016+
# Out-of-range years should return datetime64 scalars, not ints
1017+
for unit in ["us", "ms", "s"]:
1018+
dts = np.array(["0000-01-01", "10000-01-01"],
1019+
dtype="M8[Y]").astype(f"M8[{unit}]")
1020+
obj = dts.astype(object)
1021+
for idx in range(len(obj)):
1022+
assert isinstance(obj[idx], np.datetime64), (
1023+
f"OOB M8[{unit}] astype(object) returned {type(obj[idx])}")
1024+
assert obj[idx] == dts[idx]
1025+
1026+
# In-range values should still return Python datetime/timedelta
1027+
import datetime
1028+
dts_in_range = np.array(["2024-01-01T12:00:00"], dtype="M8[us]")
1029+
obj = dts_in_range.astype(object)
1030+
assert isinstance(obj[0], datetime.datetime)
1031+
1032+
tds_in_range = np.array([1000], dtype="m8[us]")
1033+
obj = tds_in_range.astype(object)
1034+
assert isinstance(obj[0], datetime.timedelta)
1035+
1036+
# NaT should still return None
1037+
nat = np.array(["NaT"], dtype="M8[ns]")
1038+
assert nat.astype(object)[0] is None
1039+
1040+
def test_item_returns_scalar_not_int(self):
1041+
# GH#12550: .item() should return numpy scalars for datetime64/
1042+
# timedelta64 values that can't be represented as Python types.
1043+
for unit in ["ns", "ps", "fs", "as"]:
1044+
dt = np.datetime64(1000, unit)
1045+
assert isinstance(dt.item(), np.datetime64)
1046+
1047+
td = np.timedelta64(1000, unit)
1048+
assert isinstance(td.item(), np.timedelta64)
1049+
1050+
# Out-of-range datetime should also return datetime64
1051+
oob = np.datetime64("0000-01-01", "us")
1052+
assert isinstance(oob.item(), np.datetime64)
1053+
1054+
# In-range should still return Python datetime/timedelta
1055+
import datetime
1056+
assert isinstance(np.datetime64("2024-01-01", "us").item(),
1057+
datetime.datetime)
1058+
assert isinstance(np.timedelta64(1000, "us").item(),
1059+
datetime.timedelta)
1060+
1061+
def test_datetime_timedelta_int_float_raises(self):
1062+
# GH#12550: int()/float() on datetime64/timedelta64 scalars should
1063+
# raise TypeError rather than returning raw integers.
1064+
for unit in ["ns", "us", "ms", "s"]:
1065+
dt = np.datetime64(123, unit)
1066+
with pytest.raises(TypeError):
1067+
int(dt)
1068+
with pytest.raises(TypeError):
1069+
float(dt)
1070+
1071+
td = np.timedelta64(123, unit)
1072+
with pytest.raises(TypeError):
1073+
int(td)
1074+
with pytest.raises(TypeError):
1075+
float(td)
1076+
9971077
def test_month_truncation(self):
9981078
# Make sure that months are truncating correctly
9991079
assert_equal(np.array('1945-03-01', dtype='M8[M]'),

0 commit comments

Comments
 (0)