Skip to content

Commit ceca8cb

Browse files
Backport PR #71495 on branch 3.1.x (BUG: support python-dateutil 3.0 tzfile (GH#71459)) (#71861)
Co-authored-by: jbrockmendel <[email protected]>
1 parent 68fc76e commit ceca8cb

8 files changed

Lines changed: 167 additions & 19 deletions

File tree

‎.github/workflows/unit-tests.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ jobs:
289289
. ~/virtualenvs/pandas-dev/bin/activate
290290
python -m pip install --no-cache-dir -U pip wheel meson[ninja] meson-python
291291
python -m pip install numpy -Csetup-args="-Dallow-noblas=true"
292-
python -m pip install --no-cache-dir versioneer[toml] cython "python-dateutil<3" "pytest>=8.3.4" pytest-xdist>=3.6.1 pytest-cov
292+
python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil "pytest>=8.3.4" pytest-xdist>=3.6.1 pytest-cov
293293
python -m pip install --no-cache-dir --no-build-isolation -e . -Csetup-args="--werror" -Csetup-args=-Dc_args="-Wno-overflow -Wno-type-limits -Wno-shift-count-overflow -Wno-sign-compare"
294294
python -m pip list --no-cache-dir
295295
@@ -332,7 +332,7 @@ jobs:
332332
/opt/python/cp313-cp313/bin/python -m venv ~/virtualenvs/pandas-dev
333333
. ~/virtualenvs/pandas-dev/bin/activate
334334
python -m pip install --no-cache-dir -U pip wheel meson-python meson[ninja]
335-
python -m pip install --no-cache-dir versioneer[toml] cython numpy "python-dateutil<3" "pytest>=8.3.4" pytest-xdist>=3.6.1 pytest-cov
335+
python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil "pytest>=8.3.4" pytest-xdist>=3.6.1 pytest-cov
336336
python -m pip install --no-cache-dir --no-build-isolation -e . -Csetup-args="--werror"
337337
python -m pip list --no-cache-dir
338338
- name: Run Tests
@@ -480,7 +480,7 @@ jobs:
480480
python --version
481481
python -m pip install --upgrade pip wheel meson[ninja] meson-python
482482
python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
483-
python -m pip install versioneer[toml] "python-dateutil<3" tzdata cython "pytest>=8.3.4" pytest-xdist>=3.6.1 pytest-cov
483+
python -m pip install versioneer[toml] python-dateutil tzdata cython "pytest>=8.3.4" pytest-xdist>=3.6.1 pytest-cov
484484
python -m pip install -ve . --no-build-isolation --no-index --no-deps \
485485
-Csetup-args="--werror" \
486486
${{ matrix.os != 'windows-2025' && '-Csetup-args=-Dcpp_args="-Wno-missing-field-initializers"' || '' }} \
@@ -548,7 +548,7 @@ jobs:
548548
- name: Test pandas for Pyodide
549549
run: |
550550
source .venv-pyodide/bin/activate
551-
python -m pip install "python-dateutil<3" "pytest>=8.3.4" pytest-cov
551+
python -m pip install "pytest>=8.3.4" pytest-cov
552552
# do not import pandas from the checked out repo
553553
cd ../..
554554
# pytest>=9.2 fd capture fstats an unlinked tmpfile, which fails on Emscripten

‎doc/source/whatsnew/v3.1.1.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ Fixed regressions
2222
Bug fixes
2323
~~~~~~~~~
2424

25+
- Fixed ``AttributeError`` when using dateutil timezones such as ``"dateutil/US/Pacific"`` with python-dateutil 3.0 (:issue:`71459`)
2526

2627
.. ---------------------------------------------------------------------------
2728
.. _whatsnew_311.contributors:

‎pandas/_libs/tslibs/timezones.pxd‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ cdef tzinfo utc_stdlib
1010
cpdef bint is_utc(tzinfo tz)
1111
cdef bint is_tzlocal(tzinfo tz)
1212
cdef bint is_zoneinfo(tzinfo tz)
13+
cdef bint treat_tz_as_zoneinfo(tzinfo tz)
1314

1415
cdef bint treat_tz_as_pytz(tzinfo tz)
1516

‎pandas/_libs/tslibs/timezones.pyx‎

Lines changed: 56 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,37 @@ cdef bint treat_tz_as_pytz(tzinfo tz):
9090

9191

9292
cdef bint treat_tz_as_dateutil(tzinfo tz):
93+
# dateutil<3 tzfile; see treat_tz_as_zoneinfo for dateutil>=3
9394
return hasattr(tz, "_trans_list") and hasattr(tz, "_trans_idx")
9495

9596

97+
cdef bint treat_tz_as_zoneinfo(tzinfo tz):
98+
# dateutil>=3 tzfile is a backport of ZoneInfo, so it goes through the
99+
# same pure-python twin. GH#71459
100+
return is_zoneinfo(tz) or (
101+
isinstance(tz, _dateutil_tzfile) and not treat_tz_as_dateutil(tz)
102+
)
103+
104+
105+
cdef object get_dateutil_key(tzinfo tz):
106+
"""
107+
Return the key of a dateutil>=3 ``tzfile``, or None if it has no
108+
trustworthy key.
109+
110+
``tzfile(fileobj, key=...)`` accepts any key, so trust it only when
111+
``gettz``, which searches the same path as ``zoneinfo``, maps it to equal
112+
data.
113+
"""
114+
key = getattr(tz, "key", None)
115+
if key is None:
116+
return None
117+
found = dateutil_gettz(key)
118+
if found is not tz and found != tz:
119+
# tzfile compares by value
120+
return None
121+
return key
122+
123+
96124
# Returns str or tzinfo object
97125
cpdef inline object get_timezone(tzinfo tz):
98126
"""
@@ -118,7 +146,11 @@ cpdef inline object get_timezone(tzinfo tz):
118146
if zone is None:
119147
return tz
120148
return zone
121-
elif treat_tz_as_dateutil(tz):
149+
elif isinstance(tz, _dateutil_tzfile):
150+
key = get_dateutil_key(tz)
151+
if key is not None:
152+
# unlike _filename, the same on every machine
153+
return "dateutil/" + key
122154
if ".tar.gz" in tz._filename:
123155
raise ValueError(
124156
"Bad tz filename. Dateutil on python 3 on windows has a "
@@ -246,10 +278,10 @@ cpdef inline bint is_fixed_offset(tzinfo tz):
246278
return 1
247279
else:
248280
return 0
249-
elif is_zoneinfo(tz):
281+
elif treat_tz_as_zoneinfo(tz):
250282
tz_py = get_zoneinfo_twin(tz)
251283
if tz_py is None:
252-
# i.e. ZoneInfo.from_file; without a twin we cannot see the
284+
# e.g. ZoneInfo.from_file; without a twin we cannot see the
253285
# transitions, so report not-fixed and let callers take the
254286
# tzinfo-API path. GH#64379
255287
return 0
@@ -314,18 +346,34 @@ cdef tzinfo get_zoneinfo_twin(tzinfo tz):
314346
tzdata). In those cases we have no way to see the transitions and callers
315347
must fall back to the tzinfo API. GH#64379
316348
317-
Whether the key describes the object is decided by asking it to pickle.
349+
For a ZoneInfo, whether the key describes the object is decided by asking
350+
it to pickle.
318351
The stdlib documents that a ``from_file`` object cannot be pickled,
319352
because its data is not recoverable from its key.
320353
354+
A dateutil>=3 ``tzfile`` uses its key only when `get_dateutil_key` accepts
355+
it.
356+
321357
Parameters
322358
----------
323-
tz : ZoneInfo
359+
tz : ZoneInfo or dateutil.tz.tzfile
324360
325361
Returns
326362
-------
327363
zoneinfo._zoneinfo.ZoneInfo or None
328364
"""
365+
if not is_zoneinfo(tz):
366+
# dateutil tzfile timezone
367+
key = get_dateutil_key(tz)
368+
if key is None:
369+
return None
370+
try:
371+
return _ZoneInfo(key)
372+
except (zoneinfo.ZoneInfoNotFoundError, ValueError):
373+
# e.g. "America/New York", which dateutil resolves to
374+
# America/New_York and zoneinfo rejects
375+
return None
376+
329377
try:
330378
tz.__reduce__()
331379
except pickle.PicklingError:
@@ -337,14 +385,14 @@ cdef tzinfo get_zoneinfo_twin(tzinfo tz):
337385

338386
cdef tuple _get_zoneinfo_trans_and_deltas(tzinfo tz):
339387
"""
340-
Get transition times and UTC offsets for a ZoneInfo timezone.
388+
Get transition times and UTC offsets for a ZoneInfo or dateutil>=3 tzfile.
341389
342390
Uses zoneinfo's Python fallback implementation to get transition data,
343391
including future transitions generated from POSIX TZ rules.
344392
345393
Parameters
346394
----------
347-
tz : ZoneInfo
395+
tz : ZoneInfo or dateutil.tz.tzfile
348396
349397
Returns
350398
-------
@@ -529,7 +577,7 @@ cdef object get_dst_info(tzinfo tz):
529577
raise AssertionError("dateutil tzinfo is not a FixedOffset "
530578
"and has an empty `_trans_list`.", tz)
531579

532-
elif is_zoneinfo(tz):
580+
elif treat_tz_as_zoneinfo(tz):
533581
trans, deltas, is_fixed, has_tz_rule = _get_zoneinfo_trans_and_deltas(tz)
534582
typ = "fixed" if is_fixed else "zoneinfo"
535583

‎pandas/_libs/tslibs/tzconversion.pyx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ from pandas._libs.tslibs.timezones cimport (
5454
is_fixed_offset,
5555
is_tzlocal,
5656
is_utc,
57-
is_zoneinfo,
57+
treat_tz_as_zoneinfo,
5858
)
5959

6060

@@ -97,7 +97,7 @@ cdef class Localizer:
9797
elif is_tzlocal(tz):
9898
self.use_tzinfo_api = True
9999

100-
elif is_zoneinfo(tz) and get_zoneinfo_twin(tz) is None:
100+
elif treat_tz_as_zoneinfo(tz) and get_zoneinfo_twin(tz) is None:
101101
# GH#64379 no pure-python twin (e.g. ZoneInfo.from_file): put
102102
# last_trans below every value so all take the tzinfo-API arm.
103103
# Not use_tzinfo_api, which ignores ambiguous/nonexistent.

‎pandas/tests/indexes/datetimes/methods/test_tz_localize.py‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -531,12 +531,19 @@ def test_dti_tz_localize_nonexistent_shift_utc_overflow(self, tz, start_ts):
531531
dti.tz_localize(tz, nonexistent=shift)
532532

533533

534-
def test_dti_tz_localize_nonexistent_shift_past_last_transition():
534+
@pytest.mark.parametrize(
535+
"tz",
536+
["pytz/America/Phoenix", gettz("America/Phoenix"), ZoneInfo("America/Phoenix")],
537+
)
538+
def test_dti_tz_localize_nonexistent_shift_past_last_transition(tz):
535539
# GH#66550 a shift landing past the last cached DST transition indexed the
536540
# deltas array out of bounds, so the same call gave a different answer
537-
# each time
538-
tz = gettz("America/Boise")
539-
dti = pd.DatetimeIndex([pd.Timestamp("2015-03-08 02:30")])
541+
# each time. Phoenix stopped observing DST, so its transitions end for
542+
# every tz implementation; a negative UTC offset is needed to reach the bug.
543+
if isinstance(tz, str):
544+
pytz = pytest.importorskip("pytz")
545+
tz = pytz.timezone(tz.removeprefix("pytz/"))
546+
dti = pd.DatetimeIndex([pd.Timestamp("1967-04-30 02:30")])
540547
shift = pd.Timedelta(days=30_000)
541548

542549
result = dti.tz_localize(tz, nonexistent=shift)

‎pandas/tests/tslibs/test_timezones.py‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929

3030
import pandas as pd
3131
import pandas._testing as tm
32+
from pandas.util.version import Version
3233

3334

3435
@pytest.mark.single_cpu
@@ -589,6 +590,97 @@ def test_zoneinfo_from_file_without_key_ambiguous_infer():
589590
)
590591

591592

593+
_dateutil_lt_3 = Version(dateutil.__version__) < Version("3") # type: ignore[attr-defined]
594+
_skip_if_dateutil_lt_3 = pytest.mark.skipif(
595+
_dateutil_lt_3, reason="tzfile key added in 3.0"
596+
)
597+
598+
599+
@pytest.mark.parametrize(
600+
"make_tz",
601+
[
602+
lambda: dateutil.tz.gettz("US/Eastern"),
603+
# no key, so no zoneinfo twin
604+
pytest.param(
605+
lambda: dateutil.tz.tzfile(io.BytesIO(_tzdata_bytes("US/Eastern"))),
606+
marks=pytest.mark.skipif(
607+
_dateutil_lt_3,
608+
reason="dateutil<3 reads only the TZif v1 block, which is empty "
609+
"in slim tzdata files",
610+
),
611+
),
612+
# dateutil accepts this key but zoneinfo does not
613+
pytest.param(
614+
lambda: dateutil.tz.gettz("America/New York"),
615+
marks=pytest.mark.skipif(
616+
_dateutil_lt_3 and is_platform_windows(),
617+
reason="dateutil<3 resolves spaced keys only on a system TZPATH",
618+
),
619+
),
620+
# a key naming a different zone than the data
621+
pytest.param(
622+
lambda: dateutil.tz.tzfile(
623+
io.BytesIO(_tzdata_bytes("US/Eastern")),
624+
key="Europe/Paris", # type: ignore[call-arg]
625+
),
626+
marks=_skip_if_dateutil_lt_3,
627+
),
628+
],
629+
)
630+
@pytest.mark.parametrize("unit", ["s", "ns"])
631+
def test_dateutil_tzfile_matches_zoneinfo(make_tz, unit):
632+
# GH#71459 dateutil>=3 tzfile no longer has the attributes the dateutil
633+
# path reads, so every DST zone raised AttributeError
634+
tz = make_tz()
635+
expected_tz = zoneinfo.ZoneInfo("US/Eastern")
636+
637+
naive = pd.DatetimeIndex(
638+
["2025-03-09 02:30", "2025-11-02 01:30", "2025-07-01"]
639+
).as_unit(unit)
640+
kwargs = {"ambiguous": True, "nonexistent": "shift_forward"}
641+
result = naive.tz_localize(tz, **kwargs)
642+
expected = naive.tz_localize(expected_tz, **kwargs)
643+
tm.assert_numpy_array_equal(result.asi8, expected.asi8)
644+
645+
utc = pd.date_range("2025-01-01", "2025-12-31", freq="7h", tz="UTC", unit=unit)
646+
assert [ts.utcoffset() for ts in utc.tz_convert(tz)] == [
647+
ts.utcoffset() for ts in utc.tz_convert(expected_tz)
648+
]
649+
650+
# dateutil<3 has no rules past 2037, so check far-future dates against tz
651+
for wall in [datetime(2025, 6, 15, 12), datetime(2150, 7, 1)]:
652+
ts = pd.Timestamp(wall).as_unit(unit)
653+
assert ts.tz_localize(tz).timestamp() == wall.replace(tzinfo=tz).timestamp()
654+
655+
656+
@_skip_if_dateutil_lt_3
657+
def test_dateutil_tzfile_fast_path_detection():
658+
# GH#71459 is_fixed_offset needs the zoneinfo twin, which dateutil>=3
659+
# zones get only from a key that matches their data
660+
data = io.BytesIO(_tzdata_bytes("US/Eastern"))
661+
assert timezones.is_fixed_offset(dateutil.tz.gettz("Etc/GMT+5"))
662+
assert not timezones.is_fixed_offset(dateutil.tz.tzfile(data, key="Etc/GMT+5"))
663+
664+
665+
@_skip_if_dateutil_lt_3
666+
def test_dateutil_tzfile_get_timezone_uses_key():
667+
# GH#71459 _filename is a machine-specific path, so Timestamp repr and
668+
# serialized zones use the key instead, when it matches the data
669+
tz = dateutil.tz.gettz("US/Eastern")
670+
assert timezones.get_timezone(tz) == "dateutil/US/Eastern"
671+
ts = pd.Timestamp("2025-01-01", tz=tz)
672+
assert "tz='dateutil/US/Eastern'" in repr(ts)
673+
674+
# a key that doesn't match the data is ignored; filename= is passed
675+
# because _filename otherwise defaults to the key
676+
mislabeled = dateutil.tz.tzfile(
677+
io.BytesIO(_tzdata_bytes("US/Eastern")),
678+
filename="mislabeled",
679+
key="Europe/Paris",
680+
)
681+
assert timezones.get_timezone(mislabeled) == "dateutil/mislabeled"
682+
683+
592684
def test_zoneinfo_fast_path_detection():
593685
# GH#64379 is_fixed_offset needs the pure-python twin, so it shows which
594686
# ZoneInfo objects keep the cached transition fast path.

‎pyproject.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,6 @@ build-verbosity = 1
133133
config-settings = { compile-args = ["-v"] }
134134
environment = {LDFLAGS="-Wl,--strip-all"}
135135
test-extras = "test"
136-
test-requires = ["python-dateutil<3"]
137136
test-command = """
138137
python -c 'import pandas as pd; \
139138
pd.test(extra_args=["-m not clipboard and not single_cpu and not slow and not network and not db", "--numprocesses=2", "--dist=worksteal", "--no-strict-data-files"]); \

0 commit comments

Comments
 (0)