Skip to content

Commit b0965d0

Browse files
committed
add referenced_data_file property
1 parent 72a6426 commit b0965d0

4 files changed

Lines changed: 68 additions & 1 deletion

File tree

‎pyiceberg/manifest.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,14 @@ def equality_ids(self) -> Optional[List[int]]:
515515
def sort_order_id(self) -> Optional[int]:
516516
return self._data[15]
517517

518+
@property
519+
def first_row_id(self) -> Optional[int]:
520+
return self._data[16] if len(self._data) > 16 else None
521+
522+
@property
523+
def referenced_data_file(self) -> Optional[str]:
524+
return self._data[17] if len(self._data) > 17 else None
525+
518526
# Spec ID should not be stored in the file
519527
_spec_id: int
520528

‎pyiceberg/table/delete_file_index.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,16 @@ def _get_referenced_data_file(self, data_file: DataFile) -> Optional[str]:
442442
Returns:
443443
The referenced data file path or None if not available
444444
"""
445-
if data_file.content != DataFileContent.POSITION_DELETES or not (data_file.lower_bounds and data_file.upper_bounds):
445+
if data_file.content != DataFileContent.POSITION_DELETES:
446+
return None
447+
448+
# First, check the referenced_data_file field (table format v3)
449+
referenced_file = data_file.referenced_data_file
450+
if referenced_file is not None:
451+
return referenced_file
452+
453+
# Fall back to extracting from bounds (table format v1/v2)
454+
if not (data_file.lower_bounds and data_file.upper_bounds):
446455
return None
447456

448457
lower_bound = data_file.lower_bounds.get(PATH_FIELD_ID)

‎tests/conftest.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3038,6 +3038,26 @@ def create_partition_positional_delete_entry(
30383038
return ManifestEntry.from_args(status=ManifestEntryStatus.DELETED, sequence_number=sequence_number, data_file=delete_file)
30393039

30403040

3041+
def create_positional_delete_entry_with_referenced_data_file(
3042+
sequence_number: int = 1, file_path: str = "s3://bucket/data.parquet", spec_id: int = 0, partition: Optional[Record] = None
3043+
) -> ManifestEntry:
3044+
"""Create a positional delete entry with referenced_data_file field set (for table format version 3)."""
3045+
delete_file = DataFile.from_args(
3046+
_table_format_version=3,
3047+
content=DataFileContent.POSITION_DELETES,
3048+
file_path=f"s3://bucket/pos-delete-{sequence_number}.parquet",
3049+
file_format=FileFormat.PARQUET,
3050+
partition=partition or Record(),
3051+
record_count=10,
3052+
file_size_in_bytes=100,
3053+
referenced_data_file=file_path,
3054+
# Note: intentionally NOT setting lower_bounds/upper_bounds to test the referenced_data_file path
3055+
)
3056+
delete_file._spec_id = spec_id
3057+
3058+
return ManifestEntry.from_args(status=ManifestEntryStatus.DELETED, sequence_number=sequence_number, data_file=delete_file)
3059+
3060+
30413061
def create_deletion_vector_entry(
30423062
sequence_number: int = 1, file_path: str = "s3://bucket/data.parquet", spec_id: int = 0
30433063
) -> ManifestEntry:
@@ -3067,6 +3087,8 @@ def create_equality_delete_file(
30673087
upper_bounds: Optional[Dict[int, Any]] = None,
30683088
value_counts: Optional[Dict[int, Any]] = None,
30693089
null_value_counts: Optional[Dict[int, Any]] = None,
3090+
first_row_id: Optional[int] = None,
3091+
referenced_data_file: Optional[str] = None,
30703092
spec_id: int = 0,
30713093
) -> DataFile:
30723094
partition_record = partition
@@ -3082,6 +3104,8 @@ def create_equality_delete_file(
30823104
upper_bounds=upper_bounds,
30833105
value_counts=value_counts,
30843106
null_value_counts=null_value_counts,
3107+
first_row_id=first_row_id,
3108+
referenced_data_file=referenced_data_file,
30853109
)
30863110
data_file._spec_id = spec_id
30873111
return data_file
@@ -3096,6 +3120,8 @@ def create_data_file(
30963120
upper_bounds: Optional[Dict[int, Any]] = None,
30973121
value_counts: Optional[Dict[int, Any]] = None,
30983122
null_value_counts: Optional[Dict[int, Any]] = None,
3123+
first_row_id: Optional[int] = None,
3124+
referenced_data_file: Optional[str] = None,
30993125
spec_id: int = 0,
31003126
) -> DataFile:
31013127
# Set default value counts and null value counts if not provided
@@ -3114,6 +3140,8 @@ def create_data_file(
31143140
upper_bounds=upper_bounds,
31153141
value_counts=value_counts,
31163142
null_value_counts=null_value_counts,
3143+
first_row_id=first_row_id,
3144+
referenced_data_file=referenced_data_file,
31173145
)
31183146
data_file._spec_id = spec_id
31193147
return data_file

‎tests/table/test_delete_file_index.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
create_manifest_entry_with_delete_file,
3838
create_partition_positional_delete_entry,
3939
create_positional_delete_entry,
40+
create_positional_delete_entry_with_referenced_data_file,
4041
)
4142

4243

@@ -514,3 +515,24 @@ def test_all_delete_types(self, id_data_schema: Schema) -> None:
514515
# Verify that no position deletes are included
515516
pos_deletes = [d for d in deletes if d.content == DataFileContent.POSITION_DELETES and d.file_format != FileFormat.PUFFIN]
516517
assert len(pos_deletes) == 1
518+
519+
def test_referenced_data_file_without_bounds(self, id_data_schema: Schema) -> None:
520+
"""Test that position deletes with referenced_data_file field (but no bounds) are indexed correctly."""
521+
target_file_path = "s3://bucket/data-target.parquet"
522+
other_file_path = "s3://bucket/data-other.parquet"
523+
524+
pos_delete_entry = create_positional_delete_entry_with_referenced_data_file(
525+
sequence_number=5, file_path=target_file_path, spec_id=0
526+
)
527+
528+
delete_index = DeleteFileIndex(id_data_schema, {0: PartitionSpec()})
529+
delete_index.add_delete_file(pos_delete_entry)
530+
531+
target_data_file = create_data_file(file_path=target_file_path)
532+
target_deletes = delete_index.for_data_file(0, target_data_file)
533+
assert len(target_deletes) == 1
534+
assert target_deletes[0].file_path == pos_delete_entry.data_file.file_path
535+
536+
other_data_file = create_data_file(file_path=other_file_path)
537+
other_deletes = delete_index.for_data_file(0, other_data_file)
538+
assert len(other_deletes) == 0

0 commit comments

Comments
 (0)