Skip to content

Commit 9cbd884

Browse files
jorisvandenbosscheBenlite777jbrockmendelParth1353fangchenli
authored
[backport 3.0.x] Backport of some already merged regression fixes (#68447)
Co-authored-by: Benhur Stephen <[email protected]> Co-authored-by: jbrockmendel <[email protected]> Co-authored-by: Parth Saini <[email protected]> Co-authored-by: Fangchen Li <[email protected]>
1 parent 8e24e9a commit 9cbd884

10 files changed

Lines changed: 425 additions & 12 deletions

File tree

‎doc/source/whatsnew/v3.0.6.rst‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,11 @@ including other versions of pandas.
1414
Fixed regressions
1515
~~~~~~~~~~~~~~~~~
1616

17-
17+
- Bug in :meth:`DataFrame.plot.bar` and :meth:`DataFrame.plot.barh` raising ``AttributeError`` for a regularly-spaced :class:`DatetimeIndex` whose ``freq`` attribute is unset (:issue:`66771`)
18+
- Bug in :meth:`Series.str.split`, :meth:`Series.str.rsplit`, :meth:`Series.str.partition`, and :meth:`Series.str.rpartition` on :class:`CategoricalDtype` Series with ``expand=False`` returning stringified lists/tuples instead of actual lists/tuples (:issue:`66341`)
19+
- Bug in :meth:`Timestamp.tz_convert` and :meth:`Timestamp.tz_localize` and their :class:`Index` and :class:`Series` counterparts returning a UTC offset a few minutes off for dates before the specified timezone adopted a standard time (typically early 20th century); this only affected older systems whose ``tzdata`` was compiled with a toolchain from between 2013 and 2018, such as Amazon Linux 2 and RHEL 8 and its derivatives (:issue:`67066`)
20+
- Fixed bug in :func:`read_csv` with the ``c`` engine and ``low_memory=True`` where a mixed-dtype column selected by ``usecols`` raised ``IndexError`` instead of emitting a :class:`DtypeWarning` unless it was the first column in the file, and where that warning named the wrong column when the file had an index column (:issue:`67375`)
21+
- Fixed bug in :func:`read_csv` with the ``c`` engine and ``low_memory=True`` where the number reported next to each column name in the mixed-type :class:`DtypeWarning` was a running count of the warned columns rather than the column's position in the file (:issue:`67375`)
1822

1923
.. ---------------------------------------------------------------------------
2024
.. _whatsnew_306.bug_fixes:

‎pandas/_libs/tslibs/timezones.pyx‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,10 @@ cdef int64_t[::1] unbox_utcoffsets(object transinfo):
284284
# ----------------------------------------------------------------------
285285
# Daylight Savings
286286

287+
cdef datetime _UTC_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
288+
# Smallest whole second representable as int64 nanoseconds.
289+
cdef int64_t _MIN_TRANS_SECONDS = -(-(NPY_NAT + 1) // 1_000_000_000)
290+
287291

288292
cdef tuple _get_zoneinfo_trans_and_deltas(tzinfo tz):
289293
"""
@@ -342,7 +346,9 @@ cdef tuple _get_zoneinfo_trans_and_deltas(tzinfo tz):
342346
if trans_utc:
343347
last_hist_ts = trans_utc[-1]
344348
try:
345-
last_year = datetime.fromtimestamp(last_hist_ts, timezone.utc).year
349+
# NB: not fromtimestamp, which goes through the platform
350+
# time_t and overflows past 2038 on 32-bit. GH#67826
351+
last_year = (_UTC_EPOCH + timedelta(seconds=last_hist_ts)).year
346352
except (OSError, OverflowError, ValueError):
347353
last_year = 1970
348354
else:
@@ -371,10 +377,19 @@ cdef tuple _get_zoneinfo_trans_and_deltas(tzinfo tz):
371377
trans_utc.append(t)
372378
deltas_seconds.append(d)
373379

380+
first_offset_seconds = int(tz_py._tti_before.utcoff.total_seconds())
381+
382+
# Certain older systems (toolchain from between 2013 and 2018f) have TZif
383+
# files that start with a "big bang" transition at -2**59 seconds, which
384+
# wraps when scaled to nanoseconds and leaves trans unsorted; the
385+
# NPY_NAT + 1 sentinel below stands in for it. GH#67066
386+
if trans_utc and trans_utc[0] < _MIN_TRANS_SECONDS:
387+
del trans_utc[0]
388+
del deltas_seconds[0]
389+
374390
trans = np.array(trans_utc, dtype="i8") * 1_000_000_000
375391
trans = np.hstack([np.array([NPY_NAT + 1], dtype=np.int64), trans])
376392

377-
first_offset_seconds = int(tz_py._tti_before.utcoff.total_seconds())
378393
deltas = np.array(deltas_seconds, dtype="i8") * 1_000_000_000
379394
deltas = np.hstack([[first_offset_seconds * 1_000_000_000], deltas])
380395

‎pandas/core/strings/accessor.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -891,7 +891,7 @@ def split(
891891
regex = True
892892
result = self._data.array._str_split(pat, n, expand, regex)
893893
if self._data.dtype == "category":
894-
dtype = self._data.dtype.categories.dtype
894+
dtype = self._data.dtype.categories.dtype if expand else object
895895
else:
896896
dtype = object if self._data.dtype == object else None
897897
return self._wrap_result(
@@ -1022,7 +1022,10 @@ def rsplit(self, pat=None, *, n=-1, expand: bool = False):
10221022
2 NaN NaN
10231023
"""
10241024
result = self._data.array._str_rsplit(pat, n=n)
1025-
dtype = object if self._data.dtype == object else None
1025+
if self._data.dtype == "category":
1026+
dtype = self._data.dtype.categories.dtype if expand else object
1027+
else:
1028+
dtype = object if self._data.dtype == object else None
10261029
return self._wrap_result(
10271030
result, expand=expand, returns_string=expand, dtype=dtype
10281031
)
@@ -1112,7 +1115,7 @@ def partition(self, sep: str = " ", expand: bool = True):
11121115
"""
11131116
result = self._data.array._str_partition(sep, expand)
11141117
if self._data.dtype == "category":
1115-
dtype = self._data.dtype.categories.dtype
1118+
dtype = self._data.dtype.categories.dtype if expand else object
11161119
else:
11171120
dtype = object if self._data.dtype == object else None
11181121
return self._wrap_result(
@@ -1204,7 +1207,7 @@ def rpartition(self, sep: str = " ", expand: bool = True):
12041207
"""
12051208
result = self._data.array._str_rpartition(sep, expand)
12061209
if self._data.dtype == "category":
1207-
dtype = self._data.dtype.categories.dtype
1210+
dtype = self._data.dtype.categories.dtype if expand else object
12081211
else:
12091212
dtype = object if self._data.dtype == object else None
12101213
return self._wrap_result(

‎pandas/io/parsers/c_parser_wrapper.py‎

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,29 @@ def _set_noconvert_columns(self) -> None:
200200
for col in noconvert_columns:
201201
self._reader.set_noconvert(col)
202202

203+
def _low_memory_column_labels(
204+
self, chunks: list[dict[int, ArrayLike]]
205+
) -> dict[int, Hashable]:
206+
"""
207+
Map the field positions ``read_low_memory`` keys its chunks by to labels.
208+
209+
The positions are those of the source row, so they skip whatever
210+
``usecols`` dropped and start past a leading implicit index column.
211+
``orig_names`` holds exactly the named columns that remain, in the same
212+
order, which is how ``read`` renames these keys further down. A leading
213+
implicit index column has no name of its own and maps to ``None``.
214+
"""
215+
assert self.orig_names is not None
216+
positions = sorted(chunks[0])
217+
named = dict(
218+
zip(
219+
positions[self._reader.leading_cols :],
220+
self.orig_names,
221+
strict=False,
222+
)
223+
)
224+
return {position: named.get(position) for position in positions}
225+
203226
def read(
204227
self,
205228
nrows: int | None = None,
@@ -214,7 +237,9 @@ def read(
214237
if self.low_memory:
215238
chunks = self._reader.read_low_memory(nrows)
216239
# destructive to chunks
217-
data = _concatenate_chunks(chunks, self.names)
240+
data = _concatenate_chunks(
241+
chunks, self._low_memory_column_labels(chunks)
242+
)
218243
else:
219244
data = self._reader.read(nrows)
220245
except StopIteration:
@@ -334,13 +359,19 @@ def _filter_usecols(usecols, names: SequenceT) -> SequenceT | list[Hashable]:
334359

335360

336361
def _concatenate_chunks(
337-
chunks: list[dict[int, ArrayLike]], column_names: list[str]
362+
chunks: list[dict[int, ArrayLike]],
363+
column_names: Sequence[Hashable] | Mapping[int, Hashable],
338364
) -> dict:
339365
"""
340366
Concatenate chunks of data read with low_memory=True.
341367
342368
The tricky part is handling Categoricals, where different chunks
343369
may have different inferred categories.
370+
371+
``column_names`` names the keys of ``chunks`` for the mixed-dtype warning,
372+
so it has to be indexable by whatever those keys are: the low_memory reader
373+
keys its chunks by field position and passes a mapping keyed the same way,
374+
while the parallel reader keys its own by name.
344375
"""
345376
names = list(chunks[0].keys())
346377
warning_columns = []
@@ -358,11 +389,14 @@ def _concatenate_chunks(
358389
else:
359390
result[name] = concat_compat(arrs)
360391
if len(non_cat_dtypes) > 1 and result[name].dtype == np.dtype(object):
361-
warning_columns.append(column_names[name])
392+
warning_columns.append((name, column_names[name]))
362393

363394
if warning_columns:
364395
warning_names = ", ".join(
365-
[f"{index}: {name}" for index, name in enumerate(warning_columns)]
396+
[
397+
f"{position}: {label}" if label is not None else f"{position}"
398+
for position, label in warning_columns
399+
]
366400
)
367401
warning_message = " ".join(
368402
[

‎pandas/plotting/_matplotlib/core.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
from pandas.plotting._matplotlib.style import get_standard_colors
6868
from pandas.plotting._matplotlib.timeseries import (
6969
format_dateaxis,
70+
get_period_offset,
7071
maybe_convert_index,
7172
prepare_ts_data,
7273
use_dynamic_x,
@@ -86,6 +87,7 @@
8687
from matplotlib.axis import Axis
8788
from matplotlib.figure import Figure
8889

90+
from pandas._libs.tslibs import BaseOffset
8991
from pandas._typing import (
9092
IndexLabel,
9193
NDFrameT,
@@ -1876,12 +1878,20 @@ def __init__(
18761878
self.tick_pos = np.array(
18771879
PeriodConverter.convert_from_freq(
18781880
self._get_xticks(),
1879-
data.index.freq,
1881+
self._ts_freq,
18801882
)
18811883
)
18821884
else:
18831885
self.tick_pos = np.arange(len(data))
18841886

1887+
@cache_readonly
1888+
def _ts_freq(self) -> BaseOffset:
1889+
freq = get_period_offset(self._get_ax(0), self.data.index)
1890+
# only evaluated when _is_ts_plot() is True, which resolves the freq
1891+
# the same way and is False unless it resolves to a period alias
1892+
assert freq is not None
1893+
return freq
1894+
18851895
@cache_readonly
18861896
def ax_pos(self) -> np.ndarray:
18871897
return self.tick_pos - self.tickoffset

‎pandas/plotting/_matplotlib/timeseries.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,30 @@ def use_dynamic_x(ax: Axes, index: Index) -> bool:
255255
return True
256256

257257

258+
def get_period_offset(ax: Axes, index: Index) -> BaseOffset | None:
259+
"""
260+
Resolve the period frequency to plot `index` with on `ax`, or None.
261+
262+
The freq is resolved the same way :func:`use_dynamic_x` resolves it, since
263+
that is what decides whether the dynamic date axis is used at all: the
264+
index ``freq`` attribute may be unset even when the index is regular (the
265+
freq is then inferred), and an already-decorated axes carries the freq of
266+
whatever was plotted on it first. The result is normalized to a period
267+
offset, because an axes stores its freq as a period alias string.
268+
"""
269+
freq = _get_index_freq(index)
270+
if freq is None:
271+
freq = _get_ax_freq(ax)
272+
if freq is None:
273+
return None
274+
275+
freq_str = _get_period_alias(freq)
276+
if freq_str is None:
277+
return None
278+
freq_str = OFFSET_TO_PERIOD_FREQSTR.get(freq_str, freq_str)
279+
return to_offset(freq_str, is_period=True)
280+
281+
258282
def _get_index_freq(index: Index) -> BaseOffset | None:
259283
freq = getattr(index, "freq", None)
260284
if freq is None:

‎pandas/tests/io/parser/test_c_parser_only.py‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,10 @@
1818
import numpy as np
1919
import pytest
2020

21+
from pandas._libs import parsers as libparsers
2122
from pandas.compat import WASM
2223
from pandas.errors import (
24+
DtypeWarning,
2325
ParserError,
2426
ParserWarning,
2527
)
@@ -642,3 +644,143 @@ def test_float_precision_options(c_parser_only):
642644

643645
with pytest.raises(ValueError, match=msg):
644646
parser.read_csv(StringIO(s), float_precision="junk")
647+
648+
649+
# small enough that DEFAULT_BUFFER_HEURISTIC // table_width leaves one line per
650+
# buffer, so the rows below land in separate low_memory chunks
651+
_MIXED_DTYPE_HEURISTIC = 2**3
652+
653+
654+
def _mixed_dtype_data(n_leading_names: int = 3) -> str:
655+
"""
656+
Build a csv whose middle chunk makes the second of three fields mixed-dtype.
657+
658+
Naming fewer than the three fields per row leaves an unnamed leading field,
659+
which the reader takes as an implicit index, shifting which name that field
660+
carries.
661+
"""
662+
names = ["a", "b", "c"][:n_leading_names]
663+
rows = [f"{i},{i},{i}" for i in range(_MIXED_DTYPE_HEURISTIC - 1)]
664+
return ",".join(names) + "\n" + "\n".join([*rows, "7,x,7", "8,y,8", *rows]) + "\n"
665+
666+
667+
@pytest.mark.parametrize("usecols", [["b"], ["b", "c"], [1], [1, 2]])
668+
def test_mixed_dtype_warning_with_usecols(c_parser_only, monkeypatch, usecols):
669+
# GH#67375: read_low_memory keys its chunks by field position in the source
670+
# row, so usecols leaves gaps in those keys. The mixed-dtype warning used
671+
# them to index the column names directly, which raised IndexError for any
672+
# selected column that was not still at its original position.
673+
parser = c_parser_only
674+
data = _mixed_dtype_data()
675+
676+
with monkeypatch.context() as m:
677+
m.setattr(libparsers, "DEFAULT_BUFFER_HEURISTIC", _MIXED_DTYPE_HEURISTIC)
678+
if parser.low_memory:
679+
result = parser.read_csv_check_warnings(
680+
DtypeWarning,
681+
r"Columns \(1: b\) have mixed types\. "
682+
"Specify dtype option on import or set low_memory=False.",
683+
StringIO(data),
684+
usecols=usecols,
685+
)
686+
else:
687+
result = parser.read_csv(StringIO(data), usecols=usecols)
688+
689+
expected = ["b"] if len(usecols) == 1 else ["b", "c"]
690+
assert result.columns.tolist() == expected
691+
692+
693+
def test_mixed_dtype_warning_with_index_col(c_parser_only, monkeypatch):
694+
# GH#67375: an index column shifts the parsed field positions past the data
695+
# columns, so the same positional lookup named the column after the mixed
696+
# one instead of naming "b".
697+
parser = c_parser_only
698+
data = _mixed_dtype_data()
699+
700+
with monkeypatch.context() as m:
701+
m.setattr(libparsers, "DEFAULT_BUFFER_HEURISTIC", _MIXED_DTYPE_HEURISTIC)
702+
if parser.low_memory:
703+
result = parser.read_csv_check_warnings(
704+
DtypeWarning,
705+
r"Columns \(1: b\) have mixed types\. "
706+
"Specify dtype option on import or set low_memory=False.",
707+
StringIO(data),
708+
index_col=0,
709+
)
710+
else:
711+
result = parser.read_csv(StringIO(data), index_col=0)
712+
713+
assert result.columns.tolist() == ["b", "c"]
714+
assert result.index.name == "a"
715+
716+
717+
def test_mixed_dtype_warning_with_implicit_index(c_parser_only, monkeypatch):
718+
# GH#67375: a header shorter than the rows makes the reader take the first
719+
# field as an implicit index. That field has no name, so the parsed
720+
# positions run one ahead of the column names and the warning named "b"
721+
# while the mixed column was "a".
722+
parser = c_parser_only
723+
data = _mixed_dtype_data(n_leading_names=2)
724+
725+
with monkeypatch.context() as m:
726+
m.setattr(libparsers, "DEFAULT_BUFFER_HEURISTIC", _MIXED_DTYPE_HEURISTIC)
727+
if parser.low_memory:
728+
result = parser.read_csv_check_warnings(
729+
DtypeWarning,
730+
r"Columns \(1: a\) have mixed types\. "
731+
"Specify dtype option on import or set low_memory=False.",
732+
StringIO(data),
733+
)
734+
else:
735+
result = parser.read_csv(StringIO(data))
736+
737+
assert result.columns.tolist() == ["a", "b"]
738+
739+
740+
def test_mixed_dtype_warning_reports_source_positions(c_parser_only, monkeypatch):
741+
# GH#67375: the number before each name is the column's position in the
742+
# source row -- as GH#58174 asked for and as the pre-GH#58250 message
743+
# printed -- not a running count of the mixed columns. With two mixed
744+
# columns at positions 1 and 3 the two readings disagree.
745+
parser = c_parser_only
746+
rows = [f"{i},{i},{i},{i}" for i in range(_MIXED_DTYPE_HEURISTIC - 1)]
747+
data = "a,b,c,d\n" + "\n".join([*rows, "7,x,7,x", "8,y,8,y", *rows]) + "\n"
748+
749+
with monkeypatch.context() as m:
750+
m.setattr(libparsers, "DEFAULT_BUFFER_HEURISTIC", _MIXED_DTYPE_HEURISTIC)
751+
if parser.low_memory:
752+
result = parser.read_csv_check_warnings(
753+
DtypeWarning,
754+
r"Columns \(1: b, 3: d\) have mixed types\. "
755+
"Specify dtype option on import or set low_memory=False.",
756+
StringIO(data),
757+
)
758+
else:
759+
result = parser.read_csv(StringIO(data))
760+
761+
assert result.columns.tolist() == ["a", "b", "c", "d"]
762+
763+
764+
def test_mixed_dtype_warning_with_mixed_implicit_index(c_parser_only, monkeypatch):
765+
# GH#67375: the implicit index column is parsed like any other field and can
766+
# itself be mixed, but it has no name to report. Naming it by position, as
767+
# the message did before GH#58250 added names, beats borrowing the name of
768+
# the first data column -- which is not the column that is mixed.
769+
parser = c_parser_only
770+
rows = [f"{i},{i},{i},{i}" for i in range(_MIXED_DTYPE_HEURISTIC - 1)]
771+
data = "a,b,c\n" + "\n".join([*rows, "x,7,7,7", "y,8,8,8", *rows]) + "\n"
772+
773+
with monkeypatch.context() as m:
774+
m.setattr(libparsers, "DEFAULT_BUFFER_HEURISTIC", _MIXED_DTYPE_HEURISTIC)
775+
if parser.low_memory:
776+
result = parser.read_csv_check_warnings(
777+
DtypeWarning,
778+
r"Columns \(0\) have mixed types\. "
779+
"Specify dtype option on import or set low_memory=False.",
780+
StringIO(data),
781+
)
782+
else:
783+
result = parser.read_csv(StringIO(data))
784+
785+
assert result.columns.tolist() == ["a", "b", "c"]
786+
assert result.index.name is None

0 commit comments

Comments
 (0)