Skip to content

Commit d0d4786

Browse files
authored
Treat NaN floats at the same position as equal in Record.__eq__ (apache#4280)
* Treat NaN floats at the same position as equal in `Record.__eq__` `Record.__eq__` compared the underlying lists, so two separately decoded NaN values compared unequal while their `str`-based hashes matched. `DeleteFileIndex` keys partition-scoped position deletes by partition `Record`, so those deletes never applied to data files in a NaN identity partition, and scans returned deleted rows. Commit validation missed concurrently deleted files in NaN partitions for the same reason. Compare elementwise when the lists differ, counting NaN floats at the same position as equal. The hash is unchanged. Closes apache#4279 * Compare nested records once in `Record.__eq__` Drop the list fast path so nested records are not compared twice when they differ. * Simplify `Record.__eq__` with a `_same_value` helper Move the per-position comparison into a helper so `__eq__` reads as a single `all()` over the fields.
1 parent 40ebd54 commit d0d4786

4 files changed

Lines changed: 85 additions & 2 deletions

File tree

‎pyiceberg/typedef.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
# under the License.
1717
from __future__ import annotations
1818

19+
import math
1920
from abc import abstractmethod
2021
from collections.abc import Callable
2122
from datetime import date, datetime, time
@@ -173,6 +174,14 @@ class IcebergRootModel(RootModel[T], Generic[T]):
173174
model_config = ConfigDict(frozen=True)
174175

175176

177+
def _is_nan(value: Any) -> bool:
178+
return isinstance(value, float) and math.isnan(value)
179+
180+
181+
def _same_value(a: Any, b: Any) -> bool:
182+
return a is b or a == b or (_is_nan(a) and _is_nan(b))
183+
184+
176185
class Record(StructProtocol):
177186
__slots__ = ("_data",)
178187
_data: list[Any]
@@ -196,8 +205,10 @@ def __getitem__(self, pos: int) -> Any:
196205
return self._data[pos]
197206

198207
def __eq__(self, other: Any) -> bool:
199-
"""Return the equality of two instances of the Record class."""
200-
return self._data == other._data if isinstance(other, Record) else False
208+
"""Compare records, treating NaN floats at the same position as equal."""
209+
if not isinstance(other, Record) or len(self._data) != len(other._data):
210+
return False
211+
return all(_same_value(a, b) for a, b in zip(self._data, other._data, strict=True))
201212

202213
def __repr__(self) -> str:
203214
"""Return the string representation of the Record class."""

‎tests/table/test_delete_file_index.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,3 +233,10 @@ def test_record_equality_for_partition_lookup() -> None:
233233

234234
assert len(index.for_data_file(1, data_file, partition_b)) == 1
235235
assert len(index.for_data_file(1, data_file, partition_c)) == 0
236+
237+
238+
def test_record_equality_for_nan_partition_lookup() -> None:
239+
index = DeleteFileIndex()
240+
index.add_delete_file(_create_partition_delete(sequence_number=2, partition=Record(float("nan"))), Record(float("nan")))
241+
242+
assert len(index.for_data_file(1, _create_data_file(), Record(float("nan")))) == 1

‎tests/table/test_validate.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,13 @@
3030
_added_delete_files,
3131
_deleted_data_files,
3232
_validate_added_data_files,
33+
_validate_data_files_exist,
3334
_validate_deleted_data_files,
3435
_validate_no_new_delete_files,
3536
_validate_no_new_deletes_for_data_files,
3637
_validation_history,
3738
)
39+
from pyiceberg.typedef import Record
3840

3941

4042
@pytest.fixture
@@ -235,6 +237,37 @@ class DummyEntry:
235237
)
236238

237239

240+
def test_validate_data_files_exist_raises_on_conflict_in_nan_partition(
241+
table_v2_with_extensive_snapshots_and_manifests: tuple[Table, dict[int, list[ManifestFile]]],
242+
) -> None:
243+
table, mock_manifests = table_v2_with_extensive_snapshots_and_manifests
244+
oldest_snapshot = table.snapshots()[0]
245+
newest_snapshot = cast(Snapshot, table.current_snapshot())
246+
247+
snapshots = table.snapshots()
248+
snapshots[-2] = snapshots[-2].model_copy(update={"summary": Summary(operation=Operation.DELETE)})
249+
table.metadata = table.metadata.model_copy(update={"snapshots": snapshots})
250+
251+
def data_file() -> DataFile:
252+
return DataFile.from_args(file_path="s3://bucket/data.parquet", partition=Record(float("nan")), spec_id=0)
253+
254+
deleted_entry = ManifestEntry.from_args(
255+
status=ManifestEntryStatus.DELETED, snapshot_id=snapshots[-2].snapshot_id, data_file=data_file()
256+
)
257+
258+
with (
259+
patch("pyiceberg.table.snapshots.Snapshot.manifests", new=lambda self, io: mock_manifests.get(self.snapshot_id, [])),
260+
patch("pyiceberg.manifest.ManifestFile.fetch_manifest_entry", return_value=[deleted_entry]),
261+
):
262+
with pytest.raises(ValidationException, match="Data files were concurrently deleted"):
263+
_validate_data_files_exist(
264+
table=table,
265+
starting_snapshot=newest_snapshot,
266+
data_files={data_file()},
267+
parent_snapshot=oldest_snapshot,
268+
)
269+
270+
238271
@pytest.mark.parametrize("operation", [Operation.APPEND, Operation.OVERWRITE])
239272
def test_validate_added_data_files_conflicting_count(
240273
table_v2_with_extensive_snapshots_and_manifests: tuple[Table, dict[int, list[ManifestFile]]],

‎tests/test_typedef.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,35 @@ def computed(self) -> int:
6565

6666
with pytest.raises(TypeError, match="Unexpected RecordWithProperty fields: computed"):
6767
RecordWithProperty._bind(StructType(), computed=1)
68+
69+
70+
def test_record_equality_with_nan() -> None:
71+
assert Record(1, float("nan")) == Record(1, float("nan"))
72+
assert Record(Record(float("nan"))) == Record(Record(float("nan")))
73+
assert len({Record(float("nan")), Record(float("nan"))}) == 1
74+
75+
assert Record(float("nan")) != Record(1.0)
76+
assert Record(1, float("nan")) != Record(2, float("nan"))
77+
78+
79+
def test_record_equality_compares_nested_records_once() -> None:
80+
calls = 0
81+
82+
class Leaf:
83+
def __init__(self, value: int) -> None:
84+
self.value = value
85+
86+
def __eq__(self, other: object) -> bool:
87+
nonlocal calls
88+
calls += 1
89+
return isinstance(other, Leaf) and self.value == other.value
90+
91+
depth = 20
92+
for suffix in [(), (float("nan"),)]:
93+
left, right = Record(Leaf(1)), Record(Leaf(2))
94+
for _ in range(depth - 1):
95+
left, right = Record(left, *suffix), Record(right, *suffix)
96+
97+
calls = 0
98+
assert left != right
99+
assert calls <= depth, suffix

0 commit comments

Comments
 (0)