Skip to content

Commit bf3b995

Browse files
committed
BUG: datetime64 hash.
#3836
1 parent 3f11db4 commit bf3b995

6 files changed

Lines changed: 300 additions & 81 deletions

File tree

‎numpy/core/include/numpy/ndarraytypes.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -761,7 +761,7 @@ typedef struct {
761761
npy_int32 month, day, hour, min, sec, us, ps, as;
762762
} npy_datetimestruct;
763763

764-
/* This is not used internally. */
764+
/* This structure contains an exploded view of a timedelta value */
765765
typedef struct {
766766
npy_int64 day;
767767
npy_int32 sec, us, ps, as;

‎numpy/core/src/multiarray/_datetime.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,4 +371,10 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step,
371371
NPY_NO_EXPORT PyArray_Descr *
372372
find_object_datetime_type(PyObject *obj, int type_num);
373373

374+
NPY_NO_EXPORT npy_hash_t
375+
datetime_hash(PyArray_DatetimeMetaData *meta, npy_datetime dt);
376+
377+
NPY_NO_EXPORT npy_hash_t
378+
timedelta_hash(PyArray_DatetimeMetaData *meta, npy_timedelta td);
379+
374380
#endif

‎numpy/core/src/multiarray/datetime.c‎

Lines changed: 199 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2860,85 +2860,241 @@ convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta)
28602860
}
28612861

28622862
/*
2863-
* Converts a timedelta into a PyObject *.
2863+
* We require that if d is a PyDateTime, then
2864+
* hash(numpy.datetime64(d)) == hash(d).
2865+
* Where possible, convert dt to a PyDateTime and hash it.
28642866
*
2865-
* Not-a-time is returned as the string "NaT".
2866-
* For microseconds or coarser, returns a datetime.timedelta.
2867-
* For units finer than microseconds, returns an integer.
2867+
* NOTE: "equals" across PyDate, PyDateTime and np.datetime64 is not transitive:
2868+
* datetime.datetime(1970, 1, 1) == np.datetime64(0, 'us')
2869+
* np.datetime64(0, 'us') == np.datetime64(0, 'D')
2870+
* datetime.datetime(1970, 1, 1) != np.datetime64(0, 'D') # date, not datetime!
2871+
*
2872+
* But:
2873+
* datetime.date(1970, 1, 1) == np.datetime64(0, 'D')
2874+
*
2875+
* For hash(datetime64(0, 'D')) we could return either PyDate.hash or PyDateTime.hash.
2876+
* We choose PyDateTime.hash to match datetime64(0, 'us')
28682877
*/
2869-
NPY_NO_EXPORT PyObject *
2870-
convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta)
2878+
NPY_NO_EXPORT npy_hash_t
2879+
datetime_hash(PyArray_DatetimeMetaData *meta, npy_datetime dt)
28712880
{
2872-
npy_timedelta value;
2873-
int days = 0, seconds = 0, useconds = 0;
2881+
PyObject *obj;
2882+
npy_hash_t res = 0, obj_hash;
2883+
npy_datetimestruct dts;
28742884

2875-
/*
2876-
* Convert NaT (not-a-time) into None.
2877-
*/
2878-
if (td == NPY_DATETIME_NAT) {
2879-
Py_RETURN_NONE;
2885+
if (dt == NPY_DATETIME_NAT || meta->base == NPY_FR_GENERIC) {
2886+
return -2;
28802887
}
28812888

2882-
/*
2883-
* If the type's precision is greater than microseconds, is
2884-
* Y/M/B (nonlinear units), or is generic units, return an int
2885-
*/
2886-
if (meta->base > NPY_FR_us ||
2887-
meta->base == NPY_FR_Y ||
2888-
meta->base == NPY_FR_M ||
2889-
meta->base == NPY_FR_GENERIC) {
2890-
return PyLong_FromLongLong(td);
2889+
if (convert_datetime_to_datetimestruct(meta, dt, &dts) < 0) {
2890+
return -1;
28912891
}
28922892

2893-
value = td;
2893+
if (dts.year < 1 || dts.year > 9999) {
2894+
obj = PyBytes_FromStringAndSize((const char *)&dts, sizeof(dts));
2895+
} else {
2896+
res = dts.ps * 3 + dts.as;
2897+
2898+
obj = PyDateTime_FromDateAndTime(dts.year, dts.month, dts.day,
2899+
dts.hour, dts.min, dts.sec, dts.us);
2900+
}
2901+
2902+
if (obj == NULL) {
2903+
return -1;
2904+
}
2905+
2906+
obj_hash = PyObject_Hash(obj);
2907+
2908+
Py_DECREF(obj);
2909+
2910+
if (obj_hash == -1) {
2911+
return -1;
2912+
}
2913+
2914+
res += obj_hash;
2915+
2916+
return (res == -1) ? -2 : res; /* avoid -1 (error) */
2917+
}
2918+
2919+
static int
2920+
convert_timedelta_to_timedeltastruct(PyArray_DatetimeMetaData *meta,
2921+
npy_timedelta td,
2922+
npy_timedeltastruct *out)
2923+
{
2924+
memset(out, 0, sizeof(npy_timedeltastruct));
28942925

28952926
/* Apply the unit multiplier (TODO: overflow treatment...) */
2896-
value *= meta->num;
2927+
td *= meta->num;
28972928

28982929
/* Convert to days/seconds/useconds */
28992930
switch (meta->base) {
29002931
case NPY_FR_W:
2901-
days = value * 7;
2932+
out->day = td * 7;
29022933
break;
29032934
case NPY_FR_D:
2904-
days = value;
2935+
out->day = td;
29052936
break;
29062937
case NPY_FR_h:
2907-
days = extract_unit_64(&value, 24ULL);
2908-
seconds = value*60*60;
2938+
out->day = extract_unit_64(&td, 24LL);
2939+
out->sec = (npy_int32)(td * 60*60);
29092940
break;
29102941
case NPY_FR_m:
2911-
days = extract_unit_64(&value, 60ULL*24);
2912-
seconds = value*60;
2942+
out->day = extract_unit_64(&td, 60LL*24);
2943+
out->sec = (npy_int32)(td * 60);
29132944
break;
29142945
case NPY_FR_s:
2915-
days = extract_unit_64(&value, 60ULL*60*24);
2916-
seconds = value;
2946+
out->day = extract_unit_64(&td, 60LL*60*24);
2947+
out->sec = (npy_int32)td;
29172948
break;
29182949
case NPY_FR_ms:
2919-
days = extract_unit_64(&value, 1000ULL*60*60*24);
2920-
seconds = extract_unit_64(&value, 1000ULL);
2921-
useconds = value*1000;
2950+
out->day = extract_unit_64(&td, 1000LL*60*60*24);
2951+
out->sec = (npy_int32)extract_unit_64(&td, 1000LL);
2952+
out->us = (npy_int32)(td * 1000LL);
29222953
break;
29232954
case NPY_FR_us:
2924-
days = extract_unit_64(&value, 1000ULL*1000*60*60*24);
2925-
seconds = extract_unit_64(&value, 1000ULL*1000);
2926-
useconds = value;
2955+
out->day = extract_unit_64(&td, 1000LL*1000*60*60*24);
2956+
out->sec = (npy_int32)extract_unit_64(&td, 1000LL*1000);
2957+
out->us = (npy_int32)td;
29272958
break;
2928-
default:
2929-
// unreachable, handled by the `if` above
2930-
assert(NPY_FALSE);
2959+
case NPY_FR_ns:
2960+
out->day = extract_unit_64(&td, 1000LL*1000*1000*60*60*24);
2961+
out->sec = (npy_int32)extract_unit_64(&td, 1000LL*1000*1000);
2962+
out->us = (npy_int32)extract_unit_64(&td, 1000LL);
2963+
out->ps = (npy_int32)(td * 1000LL);
2964+
break;
2965+
case NPY_FR_ps:
2966+
out->day = extract_unit_64(&td, 1000LL*1000*1000*1000*60*60*24);
2967+
out->sec = (npy_int32)extract_unit_64(&td, 1000LL*1000*1000*1000);
2968+
out->us = (npy_int32)extract_unit_64(&td, 1000LL*1000);
2969+
out->ps = (npy_int32)td;
29312970
break;
2971+
case NPY_FR_fs:
2972+
out->sec = (npy_int32)extract_unit_64(&td, 1000LL*1000*1000*1000*1000);
2973+
out->us = (npy_int32)extract_unit_64(&td, 1000LL*1000*1000);
2974+
out->ps = (npy_int32)extract_unit_64(&td, 1000LL);
2975+
out->as = (npy_int32)(td * 1000LL);
2976+
break;
2977+
case NPY_FR_as:
2978+
out->sec = (npy_int32)extract_unit_64(&td, 1000LL*1000*1000*1000*1000*1000);
2979+
out->us = (npy_int32)extract_unit_64(&td, 1000LL*1000*1000*1000);
2980+
out->ps = (npy_int32)extract_unit_64(&td, 1000LL*1000);
2981+
out->as = (npy_int32)td;
2982+
break;
2983+
default:
2984+
PyErr_SetString(PyExc_RuntimeError,
2985+
"NumPy timedelta metadata is corrupted with invalid "
2986+
"base unit");
2987+
return -1;
29322988
}
2989+
2990+
return 0;
2991+
}
2992+
2993+
/*
2994+
* Converts a timedelta into a PyObject *.
2995+
*
2996+
* Not-a-time is returned as the string "NaT".
2997+
* For microseconds or coarser, returns a datetime.timedelta.
2998+
* For units finer than microseconds, returns an integer.
2999+
*/
3000+
NPY_NO_EXPORT PyObject *
3001+
convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta)
3002+
{
3003+
npy_timedeltastruct tds;
3004+
3005+
/*
3006+
* Convert NaT (not-a-time) into None.
3007+
*/
3008+
if (td == NPY_DATETIME_NAT) {
3009+
Py_RETURN_NONE;
3010+
}
3011+
3012+
/*
3013+
* If the type's precision is greater than microseconds, is
3014+
* Y/M/B (nonlinear units), or is generic units, return an int
3015+
*/
3016+
if (meta->base > NPY_FR_us ||
3017+
meta->base == NPY_FR_Y ||
3018+
meta->base == NPY_FR_M ||
3019+
meta->base == NPY_FR_GENERIC) {
3020+
return PyLong_FromLongLong(td);
3021+
}
3022+
3023+
if (convert_timedelta_to_timedeltastruct(meta, td, &tds) < 0) {
3024+
return NULL;
3025+
}
3026+
29333027
/*
29343028
* If it would overflow the datetime.timedelta days, return a raw int
29353029
*/
2936-
if (days < -999999999 || days > 999999999) {
3030+
if (tds.day < -999999999 || tds.day > 999999999) {
29373031
return PyLong_FromLongLong(td);
29383032
}
29393033
else {
2940-
return PyDelta_FromDSU(days, seconds, useconds);
3034+
return PyDelta_FromDSU(tds.day, tds.sec, tds.us);
3035+
}
3036+
}
3037+
3038+
/*
3039+
* We require that if d is a PyDelta, then
3040+
* hash(numpy.timedelta64(d)) == hash(d).
3041+
* Where possible, convert dt to a PyDelta and hash it.
3042+
*/
3043+
NPY_NO_EXPORT npy_hash_t
3044+
timedelta_hash(PyArray_DatetimeMetaData *meta, npy_timedelta td)
3045+
{
3046+
PyObject *obj;
3047+
npy_hash_t res = 0, obj_hash;
3048+
npy_timedeltastruct tds;
3049+
3050+
if (td == NPY_DATETIME_NAT) {
3051+
return -2;
3052+
}
3053+
3054+
if (meta->base == NPY_FR_GENERIC) {
3055+
/* generic compares equal to *every* other base, so no single hash works. */
3056+
PyErr_SetString(PyExc_ValueError, "Can't hash generic timedelta64");
3057+
return -1;
3058+
}
3059+
3060+
/* Y and M can be converted to each other but not to other units */
3061+
3062+
if (meta->base == NPY_FR_Y) {
3063+
res = (npy_hash_t)(td * 12);
3064+
return (res == -1) ? -2 : res; /* avoid -1 (error) */
3065+
}
3066+
if (meta->base == NPY_FR_M) {
3067+
res = (npy_hash_t)td;
3068+
return (res == -1) ? -2 : res; /* avoid -1 (error) */
3069+
}
3070+
3071+
if (convert_timedelta_to_timedeltastruct(meta, td, &tds) < 0) {
3072+
return -1;
3073+
}
3074+
3075+
if (tds.day < -999999999 || tds.day > 999999999) {
3076+
obj = PyBytes_FromStringAndSize((const char *)&tds, sizeof(tds));
3077+
} else {
3078+
res = tds.ps * 3 + tds.as;
3079+
3080+
obj = PyDelta_FromDSU(tds.day, tds.sec, tds.us);
3081+
}
3082+
3083+
if (obj == NULL) {
3084+
return -1;
29413085
}
3086+
3087+
obj_hash = PyObject_Hash(obj);
3088+
3089+
Py_DECREF(obj);
3090+
3091+
if (obj_hash == -1) {
3092+
return -1;
3093+
}
3094+
3095+
res += obj_hash;
3096+
3097+
return (res == -1) ? -2 : res; /* avoid -1 (error) */
29423098
}
29433099

29443100
/*

‎numpy/core/src/multiarray/scalartypes.c.src‎

Lines changed: 3 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2866,45 +2866,16 @@ static NPY_INLINE npy_hash_t
28662866
* #lname = datetime, timedelta#
28672867
* #name = Datetime, Timedelta#
28682868
*/
2869-
#if NPY_SIZEOF_HASH_T==NPY_SIZEOF_DATETIME
28702869
static npy_hash_t
28712870
@lname@_arrtype_hash(PyObject *obj)
28722871
{
2873-
npy_hash_t x = (npy_hash_t)(PyArrayScalar_VAL(obj, @name@));
2874-
if (x == -1) {
2875-
x = -2;
2876-
}
2877-
return x;
2872+
PyArray_Descr *dtype = PyArray_DescrFromScalar(obj);
2873+
PyArray_DatetimeMetaData *meta = get_datetime_metadata_from_dtype(dtype);
2874+
return @lname@_hash(meta, PyArrayScalar_VAL(obj, @name@));
28782875
}
2879-
#elif NPY_SIZEOF_LONGLONG==NPY_SIZEOF_DATETIME
2880-
static npy_hash_t
2881-
@lname@_arrtype_hash(PyObject *obj)
2882-
{
2883-
npy_hash_t y;
2884-
npy_longlong x = (PyArrayScalar_VAL(obj, @name@));
2885-
2886-
if ((x <= LONG_MAX)) {
2887-
y = (npy_hash_t) x;
2888-
}
2889-
else {
2890-
union Mask {
2891-
long hashvals[2];
2892-
npy_longlong v;
2893-
} both;
2894-
2895-
both.v = x;
2896-
y = both.hashvals[0] + (1000003)*both.hashvals[1];
2897-
}
2898-
if (y == -1) {
2899-
y = -2;
2900-
}
2901-
return y;
2902-
}
2903-
#endif
29042876
/**end repeat**/
29052877

29062878

2907-
29082879
/* Wrong thing to do for longdouble, but....*/
29092880

29102881
/**begin repeat

0 commit comments

Comments
 (0)