Skip to content

Commit 2a54711

Browse files
meeseeksmachineJeongmin Giljorisvandenbossche
authored
Backport PR #64379 on branch 3.0.x (PERF: improve performance with ZoneInfo timezones (get internal transitions/deltas to avoid slow python path)) (#65480)
Co-authored-by: Jeongmin Gil <[email protected]> Co-authored-by: Joris Van den Bossche <[email protected]>
1 parent 036bb7c commit 2a54711

3 files changed

Lines changed: 159 additions & 4 deletions

File tree

‎pandas/_libs/tslibs/timezones.pyx‎

Lines changed: 106 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ from datetime import (
33
timezone,
44
)
55
import zoneinfo
6+
from zoneinfo._zoneinfo import ZoneInfo as _ZoneInfo
67

78
from pandas.compat._optional import import_optional_dependency
89

@@ -210,6 +211,8 @@ cdef object tz_cache_key(tzinfo tz):
210211
"of passing a timezone object. See "
211212
"https://github.com/pandas-dev/pandas/pull/7362")
212213
return "dateutil" + tz._filename
214+
elif is_zoneinfo(tz):
215+
return "zoneinfo/" + tz.key
213216
else:
214217
return None
215218

@@ -238,7 +241,11 @@ cpdef inline bint is_fixed_offset(tzinfo tz):
238241
else:
239242
return 0
240243
elif is_zoneinfo(tz):
241-
return 0
244+
tz_py = _ZoneInfo(tz.key)
245+
if tz_py._fixed_offset:
246+
return 1
247+
else:
248+
return 0
242249
# This also implicitly accepts datetime.timezone objects which are
243250
# considered fixed
244251
return 1
@@ -278,6 +285,99 @@ cdef int64_t[::1] unbox_utcoffsets(object transinfo):
278285
# Daylight Savings
279286

280287

288+
cdef tuple _get_zoneinfo_trans_and_deltas(tzinfo tz):
289+
"""
290+
Get transition times and UTC offsets for a ZoneInfo timezone.
291+
292+
Uses zoneinfo's Python fallback implementation to get transition data,
293+
including future transitions generated from POSIX TZ rules.
294+
295+
Parameters
296+
----------
297+
tz : ZoneInfo
298+
299+
Returns
300+
-------
301+
trans : ndarray[int64_t]
302+
Nanosecond UTC times of DST transitions.
303+
deltas : ndarray[int64_t]
304+
Nanosecond UTC offsets corresponding to DST transitions.
305+
is_fixed : bint
306+
True if this is a fixed-offset timezone with no transitions.
307+
"""
308+
cdef:
309+
int64_t fixed_offset_seconds, last_hist_ts, start_utc, end_utc
310+
list trans_utc, deltas_seconds, future_trans
311+
int year, last_year, std_offset, dst_offset
312+
313+
tz_py = _ZoneInfo(tz.key)
314+
315+
if tz_py._fixed_offset:
316+
fixed_offset_seconds = int(tz_py._tz_after.utcoff.total_seconds())
317+
trans = np.array([NPY_NAT + 1], dtype=np.int64)
318+
deltas = np.array([fixed_offset_seconds], dtype="i8") * 1_000_000_000
319+
return trans, deltas, True
320+
321+
trans_utc = list(tz_py._trans_utc)
322+
deltas_seconds = [int(info.utcoff.total_seconds()) for info in tz_py._ttinfos]
323+
324+
has_future_dst = (
325+
hasattr(tz_py, "_tz_after")
326+
and tz_py._tz_after is not None
327+
and hasattr(tz_py._tz_after, "transitions")
328+
)
329+
if has_future_dst:
330+
# ZoneInfo's _trans_utc only contains historical data (typically up to
331+
# the late 1990s for many timezones). POSIX TZ rules stored in _tz_after
332+
# are required to generate transitions for current and future dates.
333+
# Without this block, DST-observing timezones like Europe/Amsterdam
334+
# would return incorrect offsets for any date after ~1996.
335+
tz_after = tz_py._tz_after
336+
std_offset = int(tz_after.std.utcoff.total_seconds())
337+
dst_offset = int(tz_after.dst.utcoff.total_seconds())
338+
339+
if trans_utc:
340+
last_hist_ts = trans_utc[-1]
341+
try:
342+
last_year = datetime.fromtimestamp(last_hist_ts, timezone.utc).year
343+
except (OSError, OverflowError, ValueError):
344+
last_year = 1970
345+
else:
346+
last_hist_ts = 0
347+
last_year = 1970
348+
349+
future_trans = []
350+
for year in range(last_year, 2100):
351+
try:
352+
year_trans = tz_after.transitions(year)
353+
if not year_trans:
354+
break
355+
start_local, end_local = year_trans
356+
start_utc = start_local - std_offset
357+
end_utc = end_local - dst_offset
358+
if start_utc > last_hist_ts:
359+
future_trans.append((start_utc, dst_offset))
360+
if end_utc > last_hist_ts:
361+
future_trans.append((end_utc, std_offset))
362+
except Exception:
363+
break
364+
365+
future_trans.sort()
366+
367+
for t, d in future_trans:
368+
trans_utc.append(t)
369+
deltas_seconds.append(d)
370+
371+
trans = np.array(trans_utc, dtype="i8") * 1_000_000_000
372+
trans = np.hstack([np.array([NPY_NAT + 1], dtype=np.int64), trans])
373+
374+
first_offset_seconds = int(tz_py._tti_before.utcoff.total_seconds())
375+
deltas = np.array(deltas_seconds, dtype="i8") * 1_000_000_000
376+
deltas = np.hstack([[first_offset_seconds * 1_000_000_000], deltas])
377+
378+
return trans, deltas, False
379+
380+
281381
cdef object get_dst_info(tzinfo tz):
282382
"""
283383
Returns
@@ -340,6 +440,11 @@ cdef object get_dst_info(tzinfo tz):
340440
# (under the just-deleted code that returned empty arrays)
341441
raise AssertionError("dateutil tzinfo is not a FixedOffset "
342442
"and has an empty `_trans_list`.", tz)
443+
444+
elif is_zoneinfo(tz):
445+
trans, deltas, is_fixed = _get_zoneinfo_trans_and_deltas(tz)
446+
typ = "fixed" if is_fixed else "zoneinfo"
447+
343448
else:
344449
# static tzinfo, we can get here with pytz.StaticTZInfo
345450
# which are not caught by treat_tz_as_pytz

‎pandas/_libs/tslibs/tzconversion.pyx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ cdef class Localizer:
8282
if is_utc(tz) or tz is None:
8383
self.use_utc = True
8484

85-
elif is_tzlocal(tz) or is_zoneinfo(tz):
85+
elif is_tzlocal(tz):
8686
self.use_tzlocal = True
8787

8888
else:
@@ -108,7 +108,7 @@ cdef class Localizer:
108108
self.ntrans = self.trans.shape[0]
109109
self.deltas = deltas
110110

111-
if typ != "pytz" and typ != "dateutil":
111+
if typ != "pytz" and typ != "dateutil" and typ != "zoneinfo":
112112
# static/fixed; in this case we know that len(delta) == 1
113113
self.use_fixed = True
114114
self.delta = deltas[0]

‎pandas/tests/tslibs/test_timezones.py‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import zoneinfo
1010

1111
import dateutil.tz
12+
import numpy as np
1213
import pytest
1314

1415
from pandas._libs.tslibs import (
@@ -17,7 +18,11 @@
1718
)
1819
from pandas.compat import is_platform_windows
1920

20-
from pandas import Timestamp
21+
from pandas import (
22+
Timestamp,
23+
date_range,
24+
)
25+
import pandas._testing as tm
2126

2227

2328
@pytest.mark.single_cpu
@@ -172,6 +177,21 @@ def test_maybe_get_tz_invalid_types():
172177
timezones.maybe_get_tz(Timestamp("2021-01-01", tz="UTC"))
173178

174179

180+
@pytest.mark.parametrize("tz_name", ["UTC", "GMT", "Etc/GMT+1", "Etc/GMT-5"])
181+
def test_zoneinfo_fixed_offset(tz_name):
182+
# GH#64363
183+
zoneinfo = pytest.importorskip("zoneinfo")
184+
tz = zoneinfo.ZoneInfo(tz_name)
185+
assert timezones.is_fixed_offset(tz)
186+
187+
188+
def test_zoneinfo_not_fixed_offset_with_historical_transition():
189+
# GH#64363
190+
zoneinfo = pytest.importorskip("zoneinfo")
191+
tz = zoneinfo.ZoneInfo("Africa/Lusaka")
192+
assert not timezones.is_fixed_offset(tz)
193+
194+
175195
def test_maybe_get_tz_offset_only():
176196
# see gh-36004
177197

@@ -194,6 +214,36 @@ def test_maybe_get_tz_offset_only():
194214
assert tz == timezone(-timedelta(hours=2, minutes=45))
195215

196216

217+
def test_zoneinfo_utc_to_local_post_2037():
218+
# GH#64363 - verify that ZoneInfo DST transitions after 2037
219+
# (generated from POSIX TZ string rules) produce correct local times.
220+
tz = zoneinfo.ZoneInfo("US/Pacific")
221+
utc_times = date_range("2040-07-01", periods=24, freq="h", tz="UTC")
222+
local = utc_times.tz_convert(tz)
223+
224+
expected_hours = np.array(
225+
[
226+
datetime(2040, 7, 1, hour, tzinfo=timezone.utc).astimezone(tz).hour
227+
for hour in range(24)
228+
],
229+
dtype=np.int32,
230+
)
231+
tm.assert_numpy_array_equal(local.hour.to_numpy(), expected_hours)
232+
233+
234+
@pytest.mark.parametrize("key", ["US/Eastern", "Africa/Lusaka", "Asia/Qyzylorda"])
235+
def test_zoneinfo_utc_to_local_pre_first_transition(key):
236+
# GH#64363 - verify that ZoneInfo offsets before the first historical
237+
# transition (LMT era) match what ZoneInfo itself returns. The "before"
238+
# offset must be utcoff[0] from the TZ data (matching CPython's C
239+
# implementation), NOT the first non-DST transition offset.
240+
tz = zoneinfo.ZoneInfo(key)
241+
ts = Timestamp("1850-01-01", tz="UTC").tz_convert(tz)
242+
243+
expected = datetime(1850, 1, 1, tzinfo=timezone.utc).astimezone(tz)
244+
assert ts.minute == expected.minute
245+
246+
197247
def test_normalize_pytz_timezone():
198248
pytz = pytest.importorskip("pytz")
199249

0 commit comments

Comments
 (0)