Skip to content

Commit a53aabe

Browse files
jbrockmendelclaude
andcommitted
BUG: return numpy scalars instead of ints from datetime64/timedelta64 astype(object)
convert_datetime_to_pyobject and convert_timedelta_to_pyobject returned raw Python ints for values that could not be represented as Python datetime.datetime or datetime.timedelta (sub-microsecond units, out-of-range years, leap seconds, overflowing timedelta days). This caused arr.astype(object) to silently produce integers, breaking roundtrips and surprising users. Replace those PyLong_FromLongLong fallbacks with PyArray_Scalar calls that return proper numpy.datetime64 / numpy.timedelta64 scalars. Because getitem now returns numpy scalars in those cases, int() and float() on datetime64/timedelta64 scalars would infinitely recurse through gentype_int -> array_int -> getitem -> gentype_int. Break the recursion by giving the two scalar types their own tp_as_number with nb_int/nb_float that raise TypeError, consistent with how int(datetime.datetime(...)) and int(datetime.timedelta(...)) already behave. Closes numpy#12550 Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
1 parent 758d986 commit a53aabe

5 files changed

Lines changed: 137 additions & 112 deletions

File tree

‎numpy/_core/src/multiarray/_datetime.h‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -242,20 +242,15 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj,
242242
/*
243243
* Converts a datetime into a PyObject *.
244244
*
245-
* NaT (Not-a-time) is returned as None.
246-
* For D/W/Y/M (days or coarser), returns a datetime.date.
247-
* For μs/ms/s/m/h/D/W (microseconds or coarser), returns a datetime.datetime.
248-
* For ns/ps/fs/as (units shorter than microseconds), returns an integer.
245+
* Always returns a numpy.datetime64 scalar (including NaT).
249246
*/
250247
NPY_NO_EXPORT PyObject *
251248
convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta);
252249

253250
/*
254251
* Converts a timedelta into a PyObject *.
255252
*
256-
* NaT (Not-a-time) is returned as None.
257-
* For μs/ms/s/m/h/D/W (microseconds or coarser), returns a datetime.timedelta.
258-
* For Y/M (non-linear units), generic units and ns/ps/fs/as (units shorter than microseconds), returns an integer.
253+
* Always returns a numpy.timedelta64 scalar (including NaT).
259254
*/
260255
NPY_NO_EXPORT PyObject *
261256
convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta);

‎numpy/_core/src/multiarray/datetime.c‎

Lines changed: 22 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -2774,58 +2774,33 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj,
27742774
}
27752775
}
27762776

2777+
/*
2778+
* Creates a numpy datetime64 or timedelta64 scalar from a raw value
2779+
* and metadata. Used as a fallback when the value cannot be represented
2780+
* as a Python datetime.datetime or datetime.timedelta.
2781+
*/
2782+
static PyObject *
2783+
_make_datetime_or_timedelta_scalar(
2784+
int type_num, npy_int64 val, PyArray_DatetimeMetaData *meta)
2785+
{
2786+
PyArray_Descr *dtype = create_datetime_dtype(type_num, meta);
2787+
if (dtype == NULL) {
2788+
return NULL;
2789+
}
2790+
PyObject *ret = PyArray_Scalar(&val, dtype, NULL);
2791+
Py_DECREF(dtype);
2792+
return ret;
2793+
}
2794+
27772795
/*
27782796
* Converts a datetime into a PyObject *.
27792797
*
2780-
* NaT (Not-a-time) is returned as None.
2781-
* For D/W/Y/M (days or coarser), returns a datetime.date.
2782-
* For μs/ms/s/m/h/D/W (microseconds or coarser), returns a datetime.datetime.
2783-
* For ns/ps/fs/as (units shorter than microseconds), returns an integer.
2798+
* Always returns a numpy.datetime64 scalar (including NaT).
27842799
*/
27852800
NPY_NO_EXPORT PyObject *
27862801
convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta)
27872802
{
2788-
PyObject *ret = NULL;
2789-
npy_datetimestruct dts;
2790-
2791-
/*
2792-
* Convert NaT (not-a-time) and any value with generic units
2793-
* into None.
2794-
*/
2795-
if (dt == NPY_DATETIME_NAT || meta->base == NPY_FR_GENERIC) {
2796-
Py_RETURN_NONE;
2797-
}
2798-
2799-
/* If the type's precision is greater than microseconds, return an int */
2800-
if (meta->base > NPY_FR_us) {
2801-
return PyLong_FromLongLong(dt);
2802-
}
2803-
2804-
/* Convert to a datetimestruct */
2805-
if (NpyDatetime_ConvertDatetime64ToDatetimeStruct(meta, dt, &dts) < 0) {
2806-
return NULL;
2807-
}
2808-
2809-
/*
2810-
* If the year is outside the range of years supported by Python's
2811-
* datetime, or the datetime64 falls on a leap second,
2812-
* return a raw int.
2813-
*/
2814-
if (dts.year < 1 || dts.year > 9999 || dts.sec == 60) {
2815-
return PyLong_FromLongLong(dt);
2816-
}
2817-
2818-
/* If the type's precision is greater than days, return a datetime */
2819-
if (meta->base > NPY_FR_D) {
2820-
ret = PyDateTime_FromDateAndTime(dts.year, dts.month, dts.day,
2821-
dts.hour, dts.min, dts.sec, dts.us);
2822-
}
2823-
/* Otherwise return a date */
2824-
else {
2825-
ret = PyDate_FromDate(dts.year, dts.month, dts.day);
2826-
}
2827-
2828-
return ret;
2803+
return _make_datetime_or_timedelta_scalar(NPY_DATETIME, dt, meta);
28292804
}
28302805

28312806
/*
@@ -2961,46 +2936,12 @@ convert_timedelta_to_timedeltastruct(PyArray_DatetimeMetaData *meta,
29612936
/*
29622937
* Converts a timedelta into a PyObject *.
29632938
*
2964-
* NaT (Not-a-time) is returned as None.
2965-
* For μs/ms/s/m/h/D/W (microseconds or coarser), returns a datetime.timedelta.
2966-
* For Y/M (non-linear units), generic units and ns/ps/fs/as (units shorter than microseconds), returns an integer.
2939+
* Always returns a numpy.timedelta64 scalar (including NaT).
29672940
*/
29682941
NPY_NO_EXPORT PyObject *
29692942
convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta)
29702943
{
2971-
npy_timedeltastruct tds;
2972-
2973-
/*
2974-
* Convert NaT (not-a-time) into None.
2975-
*/
2976-
if (td == NPY_DATETIME_NAT) {
2977-
Py_RETURN_NONE;
2978-
}
2979-
2980-
/*
2981-
* If the type's precision is greater than microseconds, is
2982-
* Y/M (nonlinear units), or is generic units, return an int
2983-
*/
2984-
if (meta->base > NPY_FR_us ||
2985-
meta->base == NPY_FR_Y ||
2986-
meta->base == NPY_FR_M ||
2987-
meta->base == NPY_FR_GENERIC) {
2988-
return PyLong_FromLongLong(td);
2989-
}
2990-
2991-
if (convert_timedelta_to_timedeltastruct(meta, td, &tds) < 0) {
2992-
return NULL;
2993-
}
2994-
2995-
/*
2996-
* If it would overflow the datetime.timedelta days, return a raw int
2997-
*/
2998-
if (tds.day < -999999999 || tds.day > 999999999) {
2999-
return PyLong_FromLongLong(td);
3000-
}
3001-
else {
3002-
return PyDelta_FromDSU(tds.day, tds.sec, tds.us);
3003-
}
2944+
return _make_datetime_or_timedelta_scalar(NPY_TIMEDELTA, td, meta);
30042945
}
30052946

30062947
/*

‎numpy/_core/src/multiarray/scalartypes.c.src‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3786,6 +3786,29 @@ static PyObject *
37863786
static PyNumberMethods @name@_arrtype_as_number;
37873787
/**end repeat**/
37883788

3789+
static PyNumberMethods datetime_arrtype_as_number;
3790+
static PyNumberMethods timedelta_arrtype_as_number;
3791+
3792+
static PyObject *
3793+
datetimetype_int(PyObject *self)
3794+
{
3795+
PyErr_Format(PyExc_TypeError,
3796+
"int() argument must be a string, a bytes-like object "
3797+
"or a real number, not '%.200s'",
3798+
Py_TYPE(self)->tp_name);
3799+
return NULL;
3800+
}
3801+
3802+
static PyObject *
3803+
datetimetype_float(PyObject *self)
3804+
{
3805+
PyErr_Format(PyExc_TypeError,
3806+
"float() argument must be a string or a real number, "
3807+
"not '%.200s'",
3808+
Py_TYPE(self)->tp_name);
3809+
return NULL;
3810+
}
3811+
37893812
/* Arithmetic methods -- only so we can override &, |, ^. */
37903813
NPY_NO_EXPORT PyNumberMethods bool_arrtype_as_number = {
37913814
.nb_bool = (inquiry)bool_arrtype_nonzero,
@@ -4476,6 +4499,21 @@ initialize_numeric_types(void)
44764499

44774500
/**end repeat**/
44784501

4502+
/*
4503+
* Datetime and timedelta types should not be convertible to int/float.
4504+
* Their getitem returns numpy scalars (datetime64/timedelta64), not
4505+
* Python scalars, so gentype_int/gentype_float would infinitely recurse.
4506+
*/
4507+
datetime_arrtype_as_number = gentype_as_number;
4508+
datetime_arrtype_as_number.nb_int = (unaryfunc)datetimetype_int;
4509+
datetime_arrtype_as_number.nb_float = (unaryfunc)datetimetype_float;
4510+
PyDatetimeArrType_Type.tp_as_number = &datetime_arrtype_as_number;
4511+
4512+
timedelta_arrtype_as_number = gentype_as_number;
4513+
timedelta_arrtype_as_number.nb_int = (unaryfunc)datetimetype_int;
4514+
timedelta_arrtype_as_number.nb_float = (unaryfunc)datetimetype_float;
4515+
PyTimedeltaArrType_Type.tp_as_number = &timedelta_arrtype_as_number;
4516+
44794517
PyStringArrType_Type.tp_alloc = NULL;
44804518
PyStringArrType_Type.tp_free = NULL;
44814519

‎numpy/_core/tests/test_array_coercion.py‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -425,15 +425,20 @@ def test_coercion_basic(self, dtype, scalar):
425425
[param(np.timedelta64(123, "ns"), id="timedelta64[ns]"),
426426
param(np.timedelta64(12, "generic"), id="timedelta64[generic]")])
427427
def test_coercion_timedelta_convert_to_number(self, dtype, scalar):
428-
# Only "ns" and "generic" timedeltas can be converted to numbers
429-
# so these are slightly special.
428+
# "ns" and "generic" timedeltas can be converted to numeric dtypes
429+
# via np.array(scalar, dtype=...) and .astype(), but integer item
430+
# assignment raises TypeError because it goes through int().
430431
arr = np.array(scalar, dtype=dtype)
431432
cast = np.array(scalar).astype(dtype)
432-
ass = np.ones((), dtype=dtype)
433-
ass[()] = scalar # raises, as would np.array([scalar], dtype=dtype)
434-
435433
assert_array_equal(arr, cast)
436-
assert_array_equal(cast, cast)
434+
435+
ass = np.ones((), dtype=dtype)
436+
if np.issubdtype(dtype, np.integer):
437+
with pytest.raises(TypeError):
438+
ass[()] = scalar
439+
else:
440+
ass[()] = scalar
441+
assert_array_equal(ass, cast)
437442

438443
@pytest.mark.parametrize("dtype", ["S6", "U6"])
439444
@pytest.mark.parametrize(["val", "unit"],

‎numpy/_core/tests/test_datetime.py‎

Lines changed: 64 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -674,25 +674,25 @@ def test_days_creation(self):
674674

675675
def test_days_to_pydate(self):
676676
assert_equal(np.array('1599', dtype='M8[D]').astype('O'),
677-
datetime.date(1599, 1, 1))
677+
np.datetime64('1599-01-01', 'D'))
678678
assert_equal(np.array('1600', dtype='M8[D]').astype('O'),
679-
datetime.date(1600, 1, 1))
679+
np.datetime64('1600-01-01', 'D'))
680680
assert_equal(np.array('1601', dtype='M8[D]').astype('O'),
681-
datetime.date(1601, 1, 1))
681+
np.datetime64('1601-01-01', 'D'))
682682
assert_equal(np.array('1900', dtype='M8[D]').astype('O'),
683-
datetime.date(1900, 1, 1))
683+
np.datetime64('1900-01-01', 'D'))
684684
assert_equal(np.array('1901', dtype='M8[D]').astype('O'),
685-
datetime.date(1901, 1, 1))
685+
np.datetime64('1901-01-01', 'D'))
686686
assert_equal(np.array('2000', dtype='M8[D]').astype('O'),
687-
datetime.date(2000, 1, 1))
687+
np.datetime64('2000-01-01', 'D'))
688688
assert_equal(np.array('2001', dtype='M8[D]').astype('O'),
689-
datetime.date(2001, 1, 1))
689+
np.datetime64('2001-01-01', 'D'))
690690
assert_equal(np.array('1600-02-29', dtype='M8[D]').astype('O'),
691-
datetime.date(1600, 2, 29))
691+
np.datetime64('1600-02-29', 'D'))
692692
assert_equal(np.array('1600-03-01', dtype='M8[D]').astype('O'),
693-
datetime.date(1600, 3, 1))
693+
np.datetime64('1600-03-01', 'D'))
694694
assert_equal(np.array('2001-03-22', dtype='M8[D]').astype('O'),
695-
datetime.date(2001, 3, 22))
695+
np.datetime64('2001-03-22', 'D'))
696696

697697
def test_dtype_comparison(self):
698698
assert_(not (np.dtype('M8[us]') == np.dtype('M8[ms]')))
@@ -994,6 +994,54 @@ def test_pyobject_roundtrip(self):
994994
assert_equal(b.astype(object).astype(unit), b,
995995
f"Error roundtripping unit {unit}")
996996

997+
def test_astype_object_returns_numpy_scalar(self):
998+
# GH#12550: astype(object) should always return numpy datetime64/
999+
# timedelta64 scalars, never Python datetime or raw ints.
1000+
for unit in ["Y", "M", "W", "D", "h", "m", "s",
1001+
"ms", "us", "ns", "ps", "fs", "as"]:
1002+
dts = np.array([1000], dtype=f"M8[{unit}]")
1003+
obj = dts.astype(object)
1004+
assert isinstance(obj[0], np.datetime64), (
1005+
f"M8[{unit}] astype(object) returned {type(obj[0])}")
1006+
assert obj[0] == dts[0]
1007+
1008+
tds = dts.view(f"m8[{unit}]")
1009+
obj_td = tds.astype(object)
1010+
assert isinstance(obj_td[0], np.timedelta64), (
1011+
f"m8[{unit}] astype(object) returned {type(obj_td[0])}")
1012+
assert obj_td[0] == tds[0]
1013+
1014+
# NaT should return numpy.datetime64('NaT')
1015+
nat = np.array(["NaT"], dtype="M8[ns]")
1016+
assert np.isnat(nat.astype(object)[0])
1017+
1018+
def test_item_returns_numpy_scalar(self):
1019+
# GH#12550: .item() should return numpy datetime64/timedelta64
1020+
# scalars, not Python datetime or raw ints.
1021+
for unit in ["Y", "M", "W", "D", "h", "m", "s",
1022+
"ms", "us", "ns", "ps", "fs", "as"]:
1023+
dt = np.datetime64(1000, unit)
1024+
assert isinstance(dt.item(), np.datetime64)
1025+
1026+
td = np.timedelta64(1000, unit)
1027+
assert isinstance(td.item(), np.timedelta64)
1028+
1029+
def test_datetime_timedelta_int_float_raises(self):
1030+
# GH#12550: int()/float() on datetime64/timedelta64 scalars should
1031+
# raise TypeError rather than returning raw integers.
1032+
for unit in ["ns", "us", "ms", "s"]:
1033+
dt = np.datetime64(123, unit)
1034+
with pytest.raises(TypeError):
1035+
int(dt)
1036+
with pytest.raises(TypeError):
1037+
float(dt)
1038+
1039+
td = np.timedelta64(123, unit)
1040+
with pytest.raises(TypeError):
1041+
int(td)
1042+
with pytest.raises(TypeError):
1043+
float(td)
1044+
9971045
def test_month_truncation(self):
9981046
# Make sure that months are truncating correctly
9991047
assert_equal(np.array('1945-03-01', dtype='M8[M]'),
@@ -1636,7 +1684,7 @@ def test_datetime_minmax(self):
16361684
def test_hours(self):
16371685
t = np.ones(3, dtype='M8[s]')
16381686
t[0] = 60 * 60 * 24 + 60 * 60 * 10
1639-
assert_(t[0].item().hour == 10)
1687+
assert_(t[0].item() == np.datetime64('1970-01-02T10:00:00', 's'))
16401688

16411689
def test_divisor_conversion_year(self):
16421690
assert_(np.dtype('M8[Y/4]') == np.dtype('M8[3M]'))
@@ -2638,9 +2686,8 @@ def test_datetime_hash_weeks(self, unit):
26382686
def test_datetime_hash_weeks_vs_pydatetime(self, unit):
26392687
dt = np.datetime64(2348, 'W') # 2015-01-01
26402688
dt2 = np.datetime64(dt, unit)
2641-
pydt = dt2.astype(datetime.datetime)
2642-
assert isinstance(pydt, datetime.datetime)
2643-
_assert_equal_hash(pydt, dt2)
2689+
pydt = datetime.datetime(2015, 1, 1)
2690+
assert hash(pydt) == hash(dt2)
26442691

26452692
@pytest.mark.parametrize('unit', ('Y', 'M', 'W', 'D', 'h', 'm', 's', 'ms', 'us'))
26462693
def test_datetime_hash_big_negative(self, unit):
@@ -2693,9 +2740,8 @@ def test_timedelta_hash_weeks(self, unit):
26932740
def test_timedelta_hash_weeks_vs_pydelta(self, unit):
26942741
td = np.timedelta64(10, 'W')
26952742
td2 = np.timedelta64(td, unit)
2696-
pytd = td2.astype(datetime.timedelta)
2697-
assert isinstance(pytd, datetime.timedelta)
2698-
_assert_equal_hash(pytd, td2)
2743+
pytd = datetime.timedelta(weeks=10)
2744+
assert hash(pytd) == hash(td2)
26992745

27002746
@pytest.mark.parametrize('unit', ('ms', 'us', 'ns', 'ps', 'fs', 'as'))
27012747
def test_timedelta_hash_ms(self, unit):
@@ -2732,7 +2778,7 @@ def test_timedelta_hash_big_positive(self, wk, unit):
27322778
[datetime.timedelta(seconds=20), datetime.timedelta(days=2)],
27332779
dtype="object",
27342780
),
2735-
np.timedelta64(2, "s"),
2781+
datetime.timedelta(seconds=2),
27362782
np.array(
27372783
[10.0, 24.0 * 60.0 * 60.0],
27382784
dtype="object",

0 commit comments

Comments
 (0)