Skip to content

Commit 4f68223

Browse files
[backport 3.0.x] BUG: fix setting with bool column mask into 1-column DataFrame with EA dtype (#68820) (#68961)
1 parent e3df0e8 commit 4f68223

5 files changed

Lines changed: 171 additions & 4 deletions

File tree

‎doc/source/whatsnew/v3.0.6.rst‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ Fixed regressions
2020
- Bug in full-slice assignment (``arr[:] = value``) into a PyArrow-backed array where the result shared memory with ``value``, so later modifying ``value`` also changed the assigned-to object (:issue:`67990`)
2121
- Fixed bug in :func:`read_csv` with the ``c`` engine and ``low_memory=True`` where a mixed-dtype column selected by ``usecols`` raised ``IndexError`` instead of emitting a :class:`DtypeWarning` unless it was the first column in the file, and where that warning named the wrong column when the file had an index column (:issue:`67375`)
2222
- Fixed bug in :func:`read_csv` with the ``c`` engine and ``low_memory=True`` where the number reported next to each column name in the mixed-type :class:`DtypeWarning` was a running count of the warned columns rather than the column's position in the file (:issue:`67375`)
23+
- Fixed a bug in setting with ``.loc`` or ``.iloc`` using a boolean mask to
24+
select the columns into a 1-column DataFrame with an extension dtype, which
25+
surfaced as a regression for a DataFrame with string data (:issue:`66527`,
26+
:issue:`66255`)
2327
- Regression in :meth:`Series.interpolate` with ``method="linear"`` and a pyarrow-backed dtype leaving consecutive and trailing missing values unfilled, and truncating interpolated values for integer dtypes; the result now matches the equivalent masked (e.g. ``Int64``) dtype, including the upcast to ``float64[pyarrow]`` (:issue:`65345`)
2428

2529
.. ---------------------------------------------------------------------------
@@ -31,6 +35,7 @@ Bug fixes
3135
- Fixed memory leak in :func:`read_csv` with ``engine="c"`` when a string or ``category`` column failed to decode, for example invalid UTF-8 under ``encoding="utf-8"`` and the default ``encoding_errors="strict"`` (:issue:`67931`)
3236

3337

38+
3439
.. ---------------------------------------------------------------------------
3540
.. _whatsnew_306.contributors:
3641

‎pandas/core/indexing.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2328,10 +2328,10 @@ def _ensure_iterable_column_indexer(self, column_indexer):
23282328
ilocs = [column_indexer]
23292329
elif isinstance(column_indexer, slice):
23302330
ilocs = range(len(self.obj.columns))[column_indexer]
2331-
elif (
2332-
isinstance(column_indexer, np.ndarray) and column_indexer.dtype.kind == "b"
2333-
):
2334-
ilocs = np.arange(len(column_indexer))[column_indexer]
2331+
elif com.is_bool_indexer(column_indexer):
2332+
ilocs = np.arange(len(column_indexer))[
2333+
np.asarray(column_indexer, dtype=bool)
2334+
]
23352335
else:
23362336
ilocs = column_indexer
23372337
return ilocs

‎pandas/core/internals/blocks.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2020,6 +2020,12 @@ def _unwrap_setitem_indexer(self, indexer):
20202020
elif com.is_null_slice(indexer[1]):
20212021
indexer = indexer[0]
20222022

2023+
elif com.is_bool_indexer(indexer[1]) and len(indexer[1]) == 1:
2024+
if indexer[1][0]:
2025+
indexer = indexer[0]
2026+
else:
2027+
indexer = []
2028+
20232029
elif is_list_like(indexer[1]) and indexer[1][0] == 0:
20242030
indexer = indexer[0]
20252031

‎pandas/tests/indexing/test_iloc.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
Interval,
2020
NaT,
2121
Series,
22+
StringDtype,
2223
Timestamp,
2324
array,
2425
concat,
@@ -1590,3 +1591,62 @@ def test_setitem_pyarrow_int_series(self):
15901591

15911592
expected = Series([7, 8, 3], dtype="int64[pyarrow]")
15921593
tm.assert_series_equal(ser, expected)
1594+
1595+
1596+
@pytest.mark.parametrize(
1597+
"dtype",
1598+
[
1599+
"Int64",
1600+
# string dtypes
1601+
("python", np.nan),
1602+
pytest.param(("pyarrow", np.nan), marks=td.skip_if_no("pyarrow")),
1603+
],
1604+
)
1605+
@pytest.mark.parametrize(
1606+
"box",
1607+
[
1608+
list,
1609+
np.array,
1610+
pytest.param(
1611+
lambda x: Series(x, index=["a"]),
1612+
marks=pytest.mark.xfail(reason="Series indexer fails"),
1613+
),
1614+
Index,
1615+
array,
1616+
],
1617+
# [np.array],
1618+
ids=["list", "ndarray", "Series", "Index", "array"],
1619+
)
1620+
def test_iloc_setitem_single_column_frame_ea_dtype(dtype, box):
1621+
# https://github.com/pandas-dev/pandas/issues/66527
1622+
# column boolean mask that sets into the single column of a 1-column df
1623+
if isinstance(dtype, tuple):
1624+
dtype = StringDtype(*dtype)
1625+
df = DataFrame({"a": array([1, 2, 3], dtype=dtype)})
1626+
1627+
# setting with a 2d dataframe
1628+
df.iloc[:, box([True])] = df * 2
1629+
1630+
expected = DataFrame({"a": array([1, 2, 3], dtype=dtype) * 2})
1631+
tm.assert_frame_equal(df, expected)
1632+
1633+
df.iloc[[0, 1], box([True])] = df.iloc[[0, 1], :] * 2
1634+
1635+
arr = array([1, 2, 3], dtype=dtype) * 2
1636+
arr[[0, 1]] = arr[[0, 1]] * 2
1637+
expected = DataFrame({"a": arr})
1638+
tm.assert_frame_equal(df, expected)
1639+
1640+
# setting with a scalar
1641+
df = DataFrame({"a": array([1, 2, 3], dtype=dtype)})
1642+
scalar = df.iloc[1, 0]
1643+
1644+
df.iloc[:, box([True])] = scalar
1645+
1646+
expected = DataFrame({"a": array([scalar] * 3, dtype=dtype)})
1647+
tm.assert_frame_equal(df, expected)
1648+
1649+
df.iloc[[0, 1], box([True])] = scalar * 2
1650+
1651+
expected = DataFrame({"a": array([scalar * 2, scalar * 2, scalar], dtype=dtype)})
1652+
tm.assert_frame_equal(df, expected)

‎pandas/tests/indexing/test_loc.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3478,3 +3478,99 @@ def test_loc_setitem_matching_index(self):
34783478
result = s[["a", "b"]]
34793479
expected = Series([np.nan, np.nan], index=["a", "b"])
34803480
tm.assert_series_equal(result, expected)
3481+
3482+
3483+
@pytest.mark.parametrize(
3484+
"dtype",
3485+
[
3486+
"Int64",
3487+
# string dtypes
3488+
("python", np.nan),
3489+
pytest.param(("pyarrow", np.nan), marks=td.skip_if_no("pyarrow")),
3490+
],
3491+
)
3492+
@pytest.mark.parametrize(
3493+
"box",
3494+
[list, np.array, lambda x: Series(x, index=["a"]), Index, pd.array],
3495+
ids=["list", "ndarray", "Series", "Index", "pd.array"],
3496+
)
3497+
def test_loc_setitem_single_column_frame_ea_dtype(dtype, box):
3498+
# https://github.com/pandas-dev/pandas/issues/66527
3499+
# column boolean mask that sets into the single column of a 1-column df
3500+
if isinstance(dtype, tuple):
3501+
dtype = pd.StringDtype(*dtype)
3502+
df = DataFrame({"a": pd.array([1, 2, 3], dtype=dtype)})
3503+
3504+
# setting with a 2d dataframe
3505+
df.loc[:, box([True])] = df * 2
3506+
3507+
expected = DataFrame({"a": pd.array([1, 2, 3], dtype=dtype) * 2})
3508+
tm.assert_frame_equal(df, expected)
3509+
3510+
df.loc[[0, 1], box([True])] = df.loc[[0, 1], :] * 2
3511+
3512+
arr = pd.array([1, 2, 3], dtype=dtype) * 2
3513+
arr[[0, 1]] = arr[[0, 1]] * 2
3514+
expected = DataFrame({"a": arr})
3515+
tm.assert_frame_equal(df, expected)
3516+
3517+
# setting with a scalar
3518+
df = DataFrame({"a": pd.array([1, 2, 3], dtype=dtype)})
3519+
scalar = df.iloc[1, 0]
3520+
3521+
df.loc[:, box([True])] = scalar
3522+
3523+
expected = DataFrame({"a": pd.array([scalar] * 3, dtype=dtype)})
3524+
tm.assert_frame_equal(df, expected)
3525+
3526+
df.loc[[0, 1], box([True])] = scalar * 2
3527+
3528+
expected = DataFrame({"a": pd.array([scalar * 2, scalar * 2, scalar], dtype=dtype)})
3529+
tm.assert_frame_equal(df, expected)
3530+
3531+
3532+
@pytest.mark.parametrize(
3533+
"dtype",
3534+
["float64", "Float64"],
3535+
)
3536+
@pytest.mark.parametrize(
3537+
"box",
3538+
[list, np.array, lambda x: Series(x, index=["a"]), Index, pd.array],
3539+
ids=["list", "ndarray", "Series", "Index", "pd.array"],
3540+
)
3541+
def test_loc_setitem_empty_boolean_column_mask(dtype, box):
3542+
# https://github.com/pandas-dev/pandas/issues/66255
3543+
df = DataFrame({"a": [1, 2, 3, np.nan]}, dtype=dtype)
3544+
df_orig = df.copy()
3545+
3546+
# setting scalar
3547+
df.loc[:, box([False])] = 100
3548+
3549+
tm.assert_frame_equal(df, df_orig)
3550+
3551+
3552+
@pytest.mark.parametrize(
3553+
"dtype",
3554+
[
3555+
"float64",
3556+
pytest.param(
3557+
"Float64",
3558+
marks=pytest.mark.xfail(reason="Setting frame causes AssertionError"),
3559+
),
3560+
],
3561+
)
3562+
@pytest.mark.parametrize(
3563+
"box",
3564+
# Series boolean key needs to be aligned with the indexed axis
3565+
[list, np.array, lambda x: Series(x, index=["a"]), Index, pd.array],
3566+
ids=["list", "ndarray", "Series", "Index", "pd.array"],
3567+
)
3568+
def test_loc_setitem_empty_boolean_column_mask_frame_value(dtype, box):
3569+
# https://github.com/pandas-dev/pandas/issues/66255
3570+
df = DataFrame({"a": [1, 2, 3, np.nan]}, dtype=dtype)
3571+
df_orig = df.copy()
3572+
3573+
# setting frame
3574+
df.loc[:, box([False])] = df * 2
3575+
3576+
tm.assert_frame_equal(df, df_orig)

0 commit comments

Comments
 (0)