Skip to content

Commit 27bb4be

Browse files
jorisvandenbosscheMottl
authored andcommitted
apacheGH-48314: [Python] Compat with pandas 3.0 changed default datetime unit (apache#48319)
### Rationale for this change pandas 3.0 changes the default datetime/timedelta resolution from nanoseconds to microseconds. We already had mostly accounted for that in previous PRs, but pandas made this change the last days in a few additional places (eg `pd.date_range`), uncovering some more issues ### What changes are included in this PR? - Don't hardcode the nanosecond unit in the metadata and when recreating a datetime-tz column's Index - Update a few tests to account for those changes ### Are these changes tested? Yes ### Are there any user-facing changes? No * GitHub Issue: apache#48314 Lead-authored-by: Raúl Cumplido <[email protected]> Co-authored-by: Joris Van den Bossche <[email protected]> Signed-off-by: Raúl Cumplido <[email protected]>
1 parent 41697c8 commit 27bb4be

2 files changed

Lines changed: 22 additions & 8 deletions

File tree

‎python/pyarrow/pandas_compat.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ def get_extension_dtype_info(column):
140140
physical_dtype = str(cats.codes.dtype)
141141
elif hasattr(dtype, 'tz'):
142142
metadata = {'timezone': pa.lib.tzinfo_to_string(dtype.tz)}
143-
physical_dtype = 'datetime64[ns]'
143+
physical_dtype = f'datetime64[{dtype.unit}]'
144144
else:
145145
metadata = None
146146
physical_dtype = str(dtype)
@@ -1188,7 +1188,7 @@ def _reconstruct_columns_from_metadata(columns, column_indexes):
11881188
if _pandas_api.is_ge_v3():
11891189
# with pandas 3+, to_datetime returns a unit depending on the string
11901190
# data, so we restore it to the original unit from the metadata
1191-
level = level.as_unit(np.datetime_data(dtype)[0])
1191+
level = level.as_unit(np.datetime_data(numpy_dtype)[0])
11921192
# GH-41503: if the column index was decimal, restore to decimal
11931193
elif pandas_dtype == "decimal":
11941194
level = _pandas_api.pd.Index([decimal.Decimal(i) for i in level])

‎python/pyarrow/tests/test_pandas.py‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -212,13 +212,14 @@ def test_column_index_names_are_preserved(self):
212212
df.columns.names = ['a']
213213
_check_pandas_roundtrip(df, preserve_index=True)
214214

215-
def test_column_index_names_with_tz(self):
215+
@pytest.mark.parametrize("tz", [None, "Europe/Brussels"])
216+
def test_column_index_names_datetime(self, tz):
216217
# ARROW-13756
217218
# Bug if index is timezone aware DataTimeIndex
218219

219220
df = pd.DataFrame(
220221
np.random.randn(5, 3),
221-
columns=pd.date_range("2021-01-01", periods=3, freq="50D", tz="CET")
222+
columns=pd.date_range("2021-01-01", periods=3, freq="50D", tz=tz)
222223
)
223224
_check_pandas_roundtrip(df, preserve_index=True)
224225

@@ -447,11 +448,16 @@ def test_string_column_index(self):
447448
assert len(md) == 1
448449
assert md['encoding'] == 'UTF-8'
449450

450-
def test_datetimetz_column_index(self):
451+
@pytest.mark.parametrize('unit', ['us', 'ns'])
452+
def test_datetimetz_column_index(self, unit):
453+
ext_kwargs = {}
454+
if Version(pd.__version__) >= Version("2.0.0"):
455+
# unit argument not supported on date_range for pandas < 2.0.0
456+
ext_kwargs = {'unit': unit}
451457
df = pd.DataFrame(
452458
[(1, 'a', 2.0), (2, 'b', 3.0), (3, 'c', 4.0)],
453459
columns=pd.date_range(
454-
start='2017-01-01', periods=3, tz='America/New_York'
460+
start='2017-01-01', periods=3, tz='America/New_York', **ext_kwargs
455461
)
456462
)
457463
t = pa.Table.from_pandas(df, preserve_index=True)
@@ -460,7 +466,10 @@ def test_datetimetz_column_index(self):
460466
column_indexes, = js['column_indexes']
461467
assert column_indexes['name'] is None
462468
assert column_indexes['pandas_type'] == 'datetimetz'
463-
assert column_indexes['numpy_type'] == 'datetime64[ns]'
469+
if ext_kwargs:
470+
assert column_indexes['numpy_type'] == f'datetime64[{unit}]'
471+
else:
472+
assert column_indexes['numpy_type'] == 'datetime64[ns]'
464473

465474
md = column_indexes['metadata']
466475
assert md['timezone'] == 'America/New_York'
@@ -709,7 +718,12 @@ def test_mismatch_metadata_schema(self):
709718
# It is possible that the metadata and actual schema is not fully
710719
# matching (eg no timezone information for tz-aware column)
711720
# -> to_pandas() conversion should not fail on that
712-
df = pd.DataFrame({"datetime": pd.date_range("2020-01-01", periods=3)})
721+
ext_kwargs = {}
722+
if Version(pd.__version__) >= Version("2.0.0"):
723+
# unit argument not supported on date_range for pandas < 2.0.0
724+
ext_kwargs = {'unit': 'ns'}
725+
df = pd.DataFrame({"datetime": pd.date_range(
726+
"2020-01-01", periods=3, **ext_kwargs)})
713727

714728
# OPTION 1: casting after conversion
715729
table = pa.Table.from_pandas(df)

0 commit comments

Comments
 (0)