Skip to content

Commit b1618e0

Browse files
authored
GH-44366: [Python][Acero] RecordBatch.filter on expression raises error if result set is empty (#46057)
### Rationale for this change When filtering a RecordBatch using an expression that results in zero matching rows, an IndexError was raised due to an attempt to access the first batch of an empty result. This change ensures that such cases are handled gracefully by returning an empty RecordBatch with the correct schema. ### What changes are included in this PR? Fixed a bug in _filter_table where filtering all rows from a RecordBatch would raise an IndexError. Added a helper function _empty_record_batch_from_schema to construct an empty RecordBatch with the correct schema. ### Are these changes tested? Yes, the behavior was verified manually. Let me know if you'd like a dedicated test case added to the test suite. ### Are there any user-facing changes? No changes to public APIs. The fix ensures more robust handling of edge cases internally. - Github issue: #44366 * GitHub Issue: #44366 Authored-by: Koen Vossen <[email protected]> Signed-off-by: Raúl Cumplido <[email protected]>
1 parent 577fa89 commit b1618e0

2 files changed

Lines changed: 29 additions & 3 deletions

File tree

‎python/pyarrow/acero.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
# distutils: language = c++
2323
# cython: language_level = 3
2424

25-
from pyarrow.lib import Table, RecordBatch
25+
from pyarrow.lib import Table, RecordBatch, array
2626
from pyarrow.compute import Expression, field
2727

2828
try:
@@ -362,7 +362,7 @@ def _filter_table(table, expression):
362362
363363
Returns
364364
-------
365-
Table
365+
Table or RecordBatch
366366
"""
367367
is_batch = False
368368
if isinstance(table, RecordBatch):
@@ -375,7 +375,11 @@ def _filter_table(table, expression):
375375
])
376376
result = decl.to_table(use_threads=True)
377377
if is_batch:
378-
result = result.combine_chunks().to_batches()[0]
378+
if result.num_rows > 0:
379+
result = result.combine_chunks().to_batches()[0]
380+
else:
381+
arrays = [array([], type=field.type) for field in result.schema]
382+
result = RecordBatch.from_arrays(arrays, schema=result.schema)
379383
return result
380384

381385

‎python/pyarrow/tests/test_compute.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1415,6 +1415,17 @@ def test_filter_record_batch():
14151415
expected = pa.record_batch([pa.array(["a", None, "e"])], names=["a'"])
14161416
assert result.equals(expected)
14171417

1418+
# GH-46057: filtering all rows should return empty RecordBatch with same schema
1419+
mask_empty_result = batch.filter(pa.array([False] * batch.num_rows))
1420+
assert mask_empty_result.num_rows == 0
1421+
assert isinstance(mask_empty_result, pa.RecordBatch)
1422+
assert mask_empty_result.schema.equals(batch.schema)
1423+
1424+
expr_empty_result = batch.filter(pc.field("a'") == "zzz")
1425+
assert expr_empty_result.num_rows == 0
1426+
assert isinstance(expr_empty_result, pa.RecordBatch)
1427+
assert expr_empty_result.schema.equals(batch.schema)
1428+
14181429

14191430
def test_filter_table():
14201431
table = pa.table([pa.array(["a", None, "c", "d", "e"])], names=["a"])
@@ -1434,6 +1445,17 @@ def test_filter_table():
14341445
result = table.filter(mask, null_selection_behavior="emit_null")
14351446
assert result.equals(expected_null)
14361447

1448+
# GH-46057: filtering all rows should return empty table with same schema
1449+
mask_empty_result = table.filter(pa.array([False] * table.num_rows))
1450+
assert mask_empty_result.num_rows == 0
1451+
assert isinstance(mask_empty_result, pa.Table)
1452+
assert mask_empty_result.schema.equals(table.schema)
1453+
1454+
expr_empty_result = table.filter(pc.field("a") == "zzz")
1455+
assert expr_empty_result.num_rows == 0
1456+
assert isinstance(expr_empty_result, pa.Table)
1457+
assert expr_empty_result.schema.equals(table.schema)
1458+
14371459

14381460
def test_filter_errors():
14391461
arr = pa.chunked_array([["a", None], ["c", "d", "e"]])

0 commit comments

Comments
 (0)