Skip to content

Commit 3d8c004

Browse files
committed
feat: Add metadata-only REPLACE API via _RewriteFiles snapshot producer
Adds a _RewriteFiles snapshot producer that implements Operation.REPLACE for data file compaction (bin-packing, sort, format migration). This mirrors Java Iceberg's BaseRewriteFiles and is accessed via: with table.transaction() as tx: with tx.update_snapshot().replace() as rewrite: rewrite.delete_data_file(old_file) rewrite.append_data_file(new_file) Key behaviors: - Validates all files-to-delete exist in the table - Enforces added_records <= deleted_records invariant - No-op on empty input (no snapshot produced) - Reuses unaffected manifests, rewrites only dirty ones - Passes through delete-file manifests unchanged - Scoped _validate_concurrency: only checks no new delete files conflict with replaced data (concurrent appends are NOT conflicts) - Retry-safe via _refresh_for_retry clearing @cached_property Scoped to data-file rewriting only. Delete-file rewriting, dataSequenceNumber override, and validateFromSnapshot are documented as future additive extensions. Fixes apache#3130
1 parent c86eb8e commit 3d8c004

4 files changed

Lines changed: 1109 additions & 46 deletions

File tree

‎pyiceberg/table/snapshots.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,7 @@ def _partition_summary(self, update_metrics: UpdateMetrics) -> str:
351351

352352

353353
def update_snapshot_summaries(summary: Summary, previous_summary: Mapping[str, str] | None = None) -> Summary:
354-
if summary.operation not in {Operation.APPEND, Operation.OVERWRITE, Operation.DELETE}:
354+
if summary.operation not in {Operation.APPEND, Operation.OVERWRITE, Operation.DELETE, Operation.REPLACE}:
355355
raise ValueError(f"Operation not implemented: {summary.operation}")
356356

357357
if not previous_summary:

‎pyiceberg/table/update/snapshot.py‎

Lines changed: 164 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,46 @@ def _calculate_added_rows(self, manifests: list[ManifestFile]) -> int:
216216
added_rows += manifest.added_rows_count
217217
return added_rows
218218

219+
def _get_existing_manifests(self, should_use_manifest_pruning: bool = False) -> list[ManifestFile]:
220+
"""Filter existing manifests and rewrite those containing deleted data files."""
221+
existing_files: list[ManifestFile] = []
222+
manifest_evaluators: dict[int, Callable[[ManifestFile], bool]] = KeyDefaultDict(self._build_manifest_evaluator)
223+
224+
if snapshot := self._transaction.table_metadata.snapshot_by_name(name=self._target_branch):
225+
for manifest_file in snapshot.manifests(io=self._io):
226+
if should_use_manifest_pruning and not manifest_evaluators[manifest_file.partition_spec_id](manifest_file):
227+
existing_files.append(manifest_file)
228+
continue
229+
230+
entries_to_write: list[ManifestEntry] = []
231+
found_deleted_entries = False
232+
233+
for entry in manifest_file.fetch_manifest_entry(io=self._io, discard_deleted=True):
234+
if entry.data_file in self._deleted_data_files:
235+
found_deleted_entries = True
236+
else:
237+
entries_to_write.append(entry)
238+
239+
if not found_deleted_entries:
240+
existing_files.append(manifest_file)
241+
continue
242+
243+
if len(entries_to_write) > 0:
244+
with self.new_manifest_writer(self.spec(manifest_file.partition_spec_id)) as writer:
245+
for entry in entries_to_write:
246+
writer.add_entry(
247+
ManifestEntry.from_args(
248+
status=ManifestEntryStatus.EXISTING,
249+
snapshot_id=entry.snapshot_id,
250+
sequence_number=entry.sequence_number,
251+
file_sequence_number=entry.file_sequence_number,
252+
data_file=entry.data_file,
253+
)
254+
)
255+
existing_files.append(writer.to_manifest_file())
256+
257+
return existing_files
258+
219259
@abstractmethod
220260
def _deleted_entries(self) -> list[ManifestEntry]: ...
221261

@@ -770,49 +810,7 @@ class _OverwriteFiles(_SnapshotProducer["_OverwriteFiles"]):
770810

771811
def _existing_manifests(self) -> list[ManifestFile]:
772812
"""Determine if there are any existing manifest files."""
773-
existing_files = []
774-
775-
manifest_evaluators: dict[int, Callable[[ManifestFile], bool]] = KeyDefaultDict(self._build_manifest_evaluator)
776-
if snapshot := self._transaction.table_metadata.snapshot_by_name(name=self._target_branch):
777-
for manifest_file in snapshot.manifests(io=self._io):
778-
# Manifest does not contain rows that match the files to delete partitions
779-
if not manifest_evaluators[manifest_file.partition_spec_id](manifest_file):
780-
existing_files.append(manifest_file)
781-
continue
782-
783-
entries_to_write: set[ManifestEntry] = set()
784-
found_deleted_entries: set[ManifestEntry] = set()
785-
786-
for entry in manifest_file.fetch_manifest_entry(io=self._io, discard_deleted=True):
787-
if entry.data_file in self._deleted_data_files:
788-
found_deleted_entries.add(entry)
789-
else:
790-
entries_to_write.add(entry)
791-
792-
# Is the intercept the empty set?
793-
if len(found_deleted_entries) == 0:
794-
existing_files.append(manifest_file)
795-
continue
796-
797-
# Delete all files from manifest
798-
if len(entries_to_write) == 0:
799-
continue
800-
801-
# We have to rewrite the manifest file without the deleted data files
802-
with self.new_manifest_writer(self.spec(manifest_file.partition_spec_id)) as writer:
803-
for entry in entries_to_write:
804-
writer.add_entry(
805-
ManifestEntry.from_args(
806-
status=ManifestEntryStatus.EXISTING,
807-
snapshot_id=entry.snapshot_id,
808-
sequence_number=entry.sequence_number,
809-
file_sequence_number=entry.file_sequence_number,
810-
data_file=entry.data_file,
811-
)
812-
)
813-
existing_files.append(writer.to_manifest_file())
814-
815-
return existing_files
813+
return self._get_existing_manifests(should_use_manifest_pruning=True)
816814

817815
def _deleted_entries(self) -> list[ManifestEntry]:
818816
"""To determine if we need to record any deleted entries.
@@ -852,6 +850,120 @@ def _get_entries(manifest: ManifestFile) -> list[ManifestEntry]:
852850
return []
853851

854852

853+
class _RewriteFiles(_SnapshotProducer["_RewriteFiles"]):
854+
"""A snapshot producer that rewrites data files.
855+
856+
Produces a REPLACE snapshot that swaps existing data files for new ones without
857+
changing the logical contents of the table. This is the metadata-only operation
858+
used by compaction (bin-packing, sort, format migration).
859+
860+
Current scope:
861+
- Data file rewriting only (delete + add DataFiles)
862+
- Validates: files-to-delete exist, added_records <= deleted_records,
863+
no new delete files conflict with replaced data files
864+
865+
Future work (additive — no structural changes needed):
866+
- Delete-file rewriting (add _deleted_delete_files set + separate manifest handling)
867+
- dataSequenceNumber override (pin new files' seq to match replaced, for eq-delete safety)
868+
- validateFromSnapshot (expose _starting_snapshot_id setter for long-running planners)
869+
- ignoreEqualityDeletes in validation (coupled with dataSequenceNumber)
870+
"""
871+
872+
def _commit(self) -> UpdatesAndRequirements:
873+
# Only produce a commit when there is something to rewrite
874+
if self._deleted_data_files or self._added_data_files:
875+
# Grab the entries that we actually found in the table's manifests
876+
deleted_entries = self._deleted_entries()
877+
found_deleted_files = {entry.data_file for entry in deleted_entries}
878+
879+
# If the user asked to delete files that aren't in the table, abort.
880+
if len(found_deleted_files) != len(self._deleted_data_files):
881+
raise ValidationException("Cannot commit, missing data files to be rewritten that are not in the table")
882+
883+
added_records = sum(f.record_count for f in self._added_data_files)
884+
deleted_records = sum(entry.data_file.record_count for entry in deleted_entries)
885+
886+
if added_records > deleted_records:
887+
raise ValidationException(
888+
f"Invalid replace: records added ({added_records}) exceeds records removed ({deleted_records})"
889+
)
890+
891+
return super()._commit()
892+
else:
893+
return (), ()
894+
895+
@cached_property
896+
def _cached_deleted_entries(self) -> list[ManifestEntry]:
897+
"""Build manifest entries marking deleted data files with DELETED status."""
898+
if self._parent_snapshot_id is not None:
899+
previous_snapshot = self._transaction.table_metadata.snapshot_by_id(self._parent_snapshot_id)
900+
if previous_snapshot is None:
901+
raise ValueError(f"Could not find the previous snapshot: {self._parent_snapshot_id}")
902+
903+
executor = ExecutorFactory.get_or_create()
904+
905+
def _get_entries(manifest: ManifestFile) -> list[ManifestEntry]:
906+
return [
907+
ManifestEntry.from_args(
908+
status=ManifestEntryStatus.DELETED,
909+
snapshot_id=self.snapshot_id,
910+
sequence_number=entry.sequence_number,
911+
file_sequence_number=entry.file_sequence_number,
912+
data_file=entry.data_file,
913+
)
914+
for entry in manifest.fetch_manifest_entry(self._io, discard_deleted=True)
915+
if entry.data_file.content == DataFileContent.DATA and entry.data_file in self._deleted_data_files
916+
]
917+
918+
list_of_entries = executor.map(_get_entries, previous_snapshot.manifests(self._io))
919+
return list(itertools.chain(*list_of_entries))
920+
else:
921+
return []
922+
923+
def _deleted_entries(self) -> list[ManifestEntry]:
924+
return self._cached_deleted_entries
925+
926+
def _existing_manifests(self) -> list[ManifestFile]:
927+
return self._get_existing_manifests()
928+
929+
def _validate_concurrency(self) -> None:
930+
"""Validate that concurrent changes do not conflict with this replace.
931+
932+
Unlike overwrite/delete, a replace operation only needs to validate that no new
933+
delete files have been added that would apply to the data files being replaced.
934+
Concurrent data file additions (appends) do NOT conflict with a replace because
935+
the replace only touches files it explicitly planned to rewrite.
936+
937+
This matches Java's BaseRewriteFiles.validate() which only calls
938+
validateNoNewDeletesForDataFiles, not validateAddedDataFiles or
939+
validateDeletedDataFiles.
940+
"""
941+
from pyiceberg.table.update.validate import _validate_no_new_deletes_for_data_files
942+
943+
if self._commit_window is None or self._commit_window.is_empty():
944+
return
945+
946+
catalog_head = self._commit_window.head
947+
starting_snapshot = self._commit_window.base
948+
949+
if catalog_head is None:
950+
return
951+
952+
if self._deleted_data_files:
953+
table = self._transaction._table
954+
conflict_detection_filter = self._predicate if self._predicate != AlwaysFalse() else None
955+
_validate_no_new_deletes_for_data_files(
956+
table, catalog_head, conflict_detection_filter, self._deleted_data_files, starting_snapshot
957+
)
958+
959+
def _refresh_for_retry(self) -> None:
960+
"""Reset state for a retry attempt, clearing the cached deleted entries."""
961+
super()._refresh_for_retry()
962+
# Clear @cached_property so it recomputes against the refreshed parent snapshot.
963+
if "_cached_deleted_entries" in self.__dict__:
964+
del self.__dict__["_cached_deleted_entries"]
965+
966+
855967
class UpdateSnapshot:
856968
_transaction: Transaction
857969
_io: FileIO
@@ -909,6 +1021,15 @@ def delete(self) -> _DeleteFiles:
9091021
snapshot_properties=self._snapshot_properties,
9101022
)
9111023

1024+
def replace(self) -> _RewriteFiles:
1025+
return _RewriteFiles(
1026+
operation=Operation.REPLACE,
1027+
transaction=self._transaction,
1028+
io=self._io,
1029+
branch=self._branch,
1030+
snapshot_properties=self._snapshot_properties,
1031+
)
1032+
9121033

9131034
class _ManifestMergeManager(Generic[U]):
9141035
_target_size_bytes: int

0 commit comments

Comments
 (0)