File tree Expand file tree Collapse file tree
crates/polars-ops/src/series/ops/is_in
py-polars/tests/unit/operations Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -280,11 +280,19 @@ fn categorical_haystack<T: PolarsCategoricalType>(
280280 let out = match ( needle_dtype, flat. dtype ( ) ) {
281281 ( DataType :: Enum ( _, mapping) | DataType :: Categorical ( _, mapping) , DataType :: String ) => {
282282 let ca = flat. str ( ) . unwrap ( ) ;
283- // Strings without a category can never match; only nulls stay null.
283+ // Later batches can add these labels, so reserve their categories now.
284+ let is_enum = needle_dtype. is_enum ( ) ;
284285 ca. iter ( )
285286 . filter_map ( |opt_s| match opt_s {
286287 None => Some ( None ) ,
287- Some ( s) => mapping. get_cat ( s) . map ( |c| Some ( T :: Native :: from_cat ( c) ) ) ,
288+ Some ( s) => {
289+ let cat = if is_enum {
290+ mapping. get_cat ( s)
291+ } else {
292+ mapping. insert_cat ( s) . ok ( )
293+ } ;
294+ cat. map ( |c| Some ( T :: Native :: from_cat ( c) ) )
295+ } ,
288296 } )
289297 . collect_ca ( PlSmallStr :: EMPTY )
290298 } ,
Original file line number Diff line number Diff line change @@ -1401,6 +1401,15 @@ def test_is_in_categorical_needle_keeps_its_categories() -> None:
14011401 assert q .collect ()["c" ].to_list () == [True , False ]
14021402
14031403
1404+ def test_is_in_categorical_needle_matches_categories_of_later_batches_29743 () -> None :
1405+ # Fresh categories only gain "b" in the last batch, after the haystack is prepared.
1406+ cats = pl .Categories .random ()
1407+ lf = pl .LazyFrame ({"s" : ["a" ] * 10_000 + ["b" ]})
1408+ q = lf .select (pl .col ("s" ).cast (pl .Categorical (cats )).is_in (["b" ]).sum ())
1409+ with pl .Config (streaming_chunk_size = 100 ):
1410+ assert q .collect (engine = "streaming" ).item () == 1
1411+
1412+
14041413def test_is_in_native_pair_does_not_skip_row_groups_by_category_order () -> None :
14051414 # The Enum orders "z" before "a", so comparing String statistics with the
14061415 # haystack's Enum min/max would wrongly skip the row group holding "z".
You can’t perform that action at this time.
0 commit comments