|
4 | 4 | from typing import ( |
5 | 5 | TYPE_CHECKING, |
6 | 6 | Literal, |
| 7 | + cast, |
7 | 8 | ) |
8 | 9 | import zoneinfo |
9 | 10 |
|
@@ -181,24 +182,59 @@ def _post_convert_dtypes( |
181 | 182 | ): |
182 | 183 | # Convert any StringDtype columns back to object dtype (pyarrow always |
183 | 184 | # uses string dtype even when the infer_string option is False) |
184 | | - for col, dtype in zip(df.columns, df.dtypes, strict=True): |
185 | | - if isinstance(dtype, pd.StringDtype) and dtype.na_value is np.nan: |
186 | | - df[col] = df[col].astype("object").fillna(None) |
187 | | - if isinstance(dtype, pd.CategoricalDtype): |
188 | | - cat_dtype = dtype.categories.dtype |
189 | | - if ( |
190 | | - isinstance(cat_dtype, pd.StringDtype) |
191 | | - and cat_dtype.na_value is np.nan |
192 | | - ): |
193 | | - cat_dtype = pd.CategoricalDtype( |
194 | | - categories=dtype.categories.astype("object"), |
195 | | - ordered=dtype.ordered, |
196 | | - ) |
197 | | - df[col] = df[col].astype(cat_dtype) |
| 185 | + for i in range(len(df.columns)): |
| 186 | + new_col = _maybe_convert_string_to_object(df.iloc[:, i]) |
| 187 | + if new_col is not None: |
| 188 | + df.isetitem(i, new_col) |
| 189 | + |
| 190 | + new_idx = _maybe_convert_string_index_to_object(df.index) |
| 191 | + if new_idx is not None: |
| 192 | + df.index = new_idx |
| 193 | + new_cols = _maybe_convert_string_index_to_object(df.columns) |
| 194 | + if new_cols is not None: |
| 195 | + df.columns = new_cols |
198 | 196 |
|
199 | 197 | return df |
200 | 198 |
|
201 | 199 |
|
| 200 | +def _maybe_convert_string_to_object( |
| 201 | + data: pd.Series | pd.Index, |
| 202 | +) -> pd.Series | pd.Index | None: |
| 203 | + if isinstance(data.dtype, pd.StringDtype) and data.dtype.na_value is np.nan: |
| 204 | + return data.astype("object").fillna(None) |
| 205 | + elif isinstance(data.dtype, pd.CategoricalDtype): |
| 206 | + cat_dtype = data.dtype.categories.dtype |
| 207 | + if isinstance(cat_dtype, pd.StringDtype) and cat_dtype.na_value is np.nan: |
| 208 | + cat_dtype = pd.CategoricalDtype( |
| 209 | + categories=data.dtype.categories.astype("object"), |
| 210 | + ordered=data.dtype.ordered, |
| 211 | + ) |
| 212 | + return data.astype(cat_dtype) |
| 213 | + |
| 214 | + # no conversion needed |
| 215 | + return None |
| 216 | + |
| 217 | + |
| 218 | +def _maybe_convert_string_index_to_object(index: pd.Index) -> pd.Index | None: |
| 219 | + if isinstance(index, pd.MultiIndex): |
| 220 | + if any( |
| 221 | + isinstance(level.dtype, pd.StringDtype) and level.dtype.na_value is np.nan |
| 222 | + for level in index.levels |
| 223 | + ): |
| 224 | + new_levels = [] |
| 225 | + for level in index.levels: |
| 226 | + new_level = _maybe_convert_string_to_object(level) |
| 227 | + if new_level is not None: |
| 228 | + new_levels.append(new_level) |
| 229 | + else: |
| 230 | + new_levels.append(level) |
| 231 | + return index.set_levels(new_levels) |
| 232 | + return None |
| 233 | + |
| 234 | + else: |
| 235 | + return cast("pd.Index | None", _maybe_convert_string_to_object(index)) |
| 236 | + |
| 237 | + |
202 | 238 | def _normalize_pytz_timezone(tz: dt.tzinfo) -> dt.tzinfo: |
203 | 239 | """ |
204 | 240 | If the input tz is a pytz timezone, attempt to convert it to "default" |
|
0 commit comments