Skip to content

Commit bf4c182

Browse files
Backport PR #65410 on branch 3.0.x (TST: also convert str index to object in pyarrow->pandas conversion for infer_string=0 CI) (#65489)
Co-authored-by: Joris Van den Bossche <[email protected]>
1 parent dd02d75 commit bf4c182

2 files changed

Lines changed: 51 additions & 15 deletions

File tree

‎pandas/io/_util.py‎

Lines changed: 50 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from typing import (
55
TYPE_CHECKING,
66
Literal,
7+
cast,
78
)
89
import zoneinfo
910

@@ -181,24 +182,59 @@ def _post_convert_dtypes(
181182
):
182183
# Convert any StringDtype columns back to object dtype (pyarrow always
183184
# uses string dtype even when the infer_string option is False)
184-
for col, dtype in zip(df.columns, df.dtypes, strict=True):
185-
if isinstance(dtype, pd.StringDtype) and dtype.na_value is np.nan:
186-
df[col] = df[col].astype("object").fillna(None)
187-
if isinstance(dtype, pd.CategoricalDtype):
188-
cat_dtype = dtype.categories.dtype
189-
if (
190-
isinstance(cat_dtype, pd.StringDtype)
191-
and cat_dtype.na_value is np.nan
192-
):
193-
cat_dtype = pd.CategoricalDtype(
194-
categories=dtype.categories.astype("object"),
195-
ordered=dtype.ordered,
196-
)
197-
df[col] = df[col].astype(cat_dtype)
185+
for i in range(len(df.columns)):
186+
new_col = _maybe_convert_string_to_object(df.iloc[:, i])
187+
if new_col is not None:
188+
df.isetitem(i, new_col)
189+
190+
new_idx = _maybe_convert_string_index_to_object(df.index)
191+
if new_idx is not None:
192+
df.index = new_idx
193+
new_cols = _maybe_convert_string_index_to_object(df.columns)
194+
if new_cols is not None:
195+
df.columns = new_cols
198196

199197
return df
200198

201199

200+
def _maybe_convert_string_to_object(
201+
data: pd.Series | pd.Index,
202+
) -> pd.Series | pd.Index | None:
203+
if isinstance(data.dtype, pd.StringDtype) and data.dtype.na_value is np.nan:
204+
return data.astype("object").fillna(None)
205+
elif isinstance(data.dtype, pd.CategoricalDtype):
206+
cat_dtype = data.dtype.categories.dtype
207+
if isinstance(cat_dtype, pd.StringDtype) and cat_dtype.na_value is np.nan:
208+
cat_dtype = pd.CategoricalDtype(
209+
categories=data.dtype.categories.astype("object"),
210+
ordered=data.dtype.ordered,
211+
)
212+
return data.astype(cat_dtype)
213+
214+
# no conversion needed
215+
return None
216+
217+
218+
def _maybe_convert_string_index_to_object(index: pd.Index) -> pd.Index | None:
219+
if isinstance(index, pd.MultiIndex):
220+
if any(
221+
isinstance(level.dtype, pd.StringDtype) and level.dtype.na_value is np.nan
222+
for level in index.levels
223+
):
224+
new_levels = []
225+
for level in index.levels:
226+
new_level = _maybe_convert_string_to_object(level)
227+
if new_level is not None:
228+
new_levels.append(new_level)
229+
else:
230+
new_levels.append(level)
231+
return index.set_levels(new_levels)
232+
return None
233+
234+
else:
235+
return cast("pd.Index | None", _maybe_convert_string_to_object(index))
236+
237+
202238
def _normalize_pytz_timezone(tz: dt.tzinfo) -> dt.tzinfo:
203239
"""
204240
If the input tz is a pytz timezone, attempt to convert it to "default"

‎pandas/tests/io/test_parquet.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,8 +218,8 @@ def compare(repeat):
218218
if "string_with_nan" in expected:
219219
expected.loc[1, "string_with_nan"] = None
220220
tm.assert_frame_equal(
221-
expected,
222221
actual,
222+
expected,
223223
check_names=check_names,
224224
check_like=check_like,
225225
check_dtype=check_dtype,

0 commit comments

Comments
 (0)