Skip to content

Commit 8399cf6

Browse files
dancsiclaude
andauthored
fix: Match categories added after is_in prepares a string haystack (#29744)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 549dfaf commit 8399cf6

2 files changed

Lines changed: 19 additions & 2 deletions

File tree

‎crates/polars-ops/src/series/ops/is_in/mod.rs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -280,11 +280,19 @@ fn categorical_haystack<T: PolarsCategoricalType>(
280280
let out = match (needle_dtype, flat.dtype()) {
281281
(DataType::Enum(_, mapping) | DataType::Categorical(_, mapping), DataType::String) => {
282282
let ca = flat.str().unwrap();
283-
// Strings without a category can never match; only nulls stay null.
283+
// Later batches can add these labels, so reserve their categories now.
284+
let is_enum = needle_dtype.is_enum();
284285
ca.iter()
285286
.filter_map(|opt_s| match opt_s {
286287
None => Some(None),
287-
Some(s) => mapping.get_cat(s).map(|c| Some(T::Native::from_cat(c))),
288+
Some(s) => {
289+
let cat = if is_enum {
290+
mapping.get_cat(s)
291+
} else {
292+
mapping.insert_cat(s).ok()
293+
};
294+
cat.map(|c| Some(T::Native::from_cat(c)))
295+
},
288296
})
289297
.collect_ca(PlSmallStr::EMPTY)
290298
},

‎py-polars/tests/unit/operations/test_is_in.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1401,6 +1401,15 @@ def test_is_in_categorical_needle_keeps_its_categories() -> None:
14011401
assert q.collect()["c"].to_list() == [True, False]
14021402

14031403

1404+
def test_is_in_categorical_needle_matches_categories_of_later_batches_29743() -> None:
1405+
# Fresh categories only gain "b" in the last batch, after the haystack is prepared.
1406+
cats = pl.Categories.random()
1407+
lf = pl.LazyFrame({"s": ["a"] * 10_000 + ["b"]})
1408+
q = lf.select(pl.col("s").cast(pl.Categorical(cats)).is_in(["b"]).sum())
1409+
with pl.Config(streaming_chunk_size=100):
1410+
assert q.collect(engine="streaming").item() == 1
1411+
1412+
14041413
def test_is_in_native_pair_does_not_skip_row_groups_by_category_order() -> None:
14051414
# The Enum orders "z" before "a", so comparing String statistics with the
14061415
# haystack's Enum min/max would wrongly skip the row group holding "z".

0 commit comments

Comments
 (0)