filter_is_not_null / filtered_point_of_view returns incorrect results #7681
Description
- Surfaced this issue while working on: rerun_py.dataframe: Add APIs for
using_index_values,fill_latest_at, andfilter_is_not_null#7680
Position3D is not in the result set even though it should be.
Can be reproduced on top of #7680
Simple repro in python:
import rerun as rr
import tempfile
rr.init("rerun_example_test_recording")
rr.set_time_sequence("my_index", 1)
rr.log("points", rr.Points3D([[1, 2, 3], [4, 5, 6], [7, 8, 9]]))
rr.set_time_sequence("my_index", 7)
rr.log("points", rr.Points3D([[10, 11, 12]], colors=[[255, 0, 0]]))
with tempfile.TemporaryDirectory() as tmpdir:
rrd = tmpdir + "/tmp.rrd"
rr.save(rrd)
recording = rr.dataframe.load_recording(rrd)
color_col = rr.dataframe.ComponentColumnSelector("points", rr.components.Color)
view = recording.view(index="my_index", contents="points")
# Baseline
table = view.select().read_all()
print(80 * "=")
print(table)
print(80 * "=")
# my_index, log_time, log_tick, points, colors
assert table.num_columns == 5
assert table.num_rows == 2
assert table.column("my_index").combine_chunks()[1].as_py() == 7
assert table.column("/points:Color").combine_chunks()[1][0].as_py() == 4278190335
assert table.column("/points:Position3D").combine_chunks()[1][0].as_py() == [10, 11, 12]
# Filtered to Color is not null
table = view.filter_is_not_null(color_col).select().read_all()
print(80 * "=")
print(table)
print(80 * "=")
# my_index, log_time, log_tick, points, colors
assert table.num_columns == 5
assert table.num_rows == 1
assert table.column("my_index").combine_chunks()[0].as_py() == 7
assert table.column("/points:Color").combine_chunks()[0][0].as_py() == 4278190335
assert table.column("/points:Position3D").combine_chunks()[0][0].as_py() == [10, 11, 12]Output:
================================================================================
pyarrow.Table
log_tick: int64
log_time: timestamp[ns]
my_index: int64
/points:Color: list<item: uint32>
child 0, item: uint32
/points:Position3D: list<item: fixed_size_list<item: float not null>[3]>
child 0, item: fixed_size_list<item: float not null>[3]
child 0, item: float not null
----
log_tick: [[1]]
log_time: [[2024-10-10 16:22:00.557968899]]
my_index: [[7]]
/points:Color: [[[4278190335]]]
/points:Position3D: [[null]]
================================================================================
Traceback (most recent call last):
File "/home/jleibs/rerun/repro.py", line 50, in <module>
assert table.column("/points:Position3D").combine_chunks()[0][0].as_py() == [10, 11, 12]
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^
File "pyarrow/scalar.pxi", line 691, in pyarrow.lib.ListScalar.__getitem__
TypeError: 'NoneType' object is not subscriptable