@@ -216,6 +216,46 @@ def _calculate_added_rows(self, manifests: list[ManifestFile]) -> int:
216216 added_rows += manifest .added_rows_count
217217 return added_rows
218218
219+ def _get_existing_manifests (self , should_use_manifest_pruning : bool = False ) -> list [ManifestFile ]:
220+ """Filter existing manifests and rewrite those containing deleted data files."""
221+ existing_files : list [ManifestFile ] = []
222+ manifest_evaluators : dict [int , Callable [[ManifestFile ], bool ]] = KeyDefaultDict (self ._build_manifest_evaluator )
223+
224+ if snapshot := self ._transaction .table_metadata .snapshot_by_name (name = self ._target_branch ):
225+ for manifest_file in snapshot .manifests (io = self ._io ):
226+ if should_use_manifest_pruning and not manifest_evaluators [manifest_file .partition_spec_id ](manifest_file ):
227+ existing_files .append (manifest_file )
228+ continue
229+
230+ entries_to_write : list [ManifestEntry ] = []
231+ found_deleted_entries = False
232+
233+ for entry in manifest_file .fetch_manifest_entry (io = self ._io , discard_deleted = True ):
234+ if entry .data_file in self ._deleted_data_files :
235+ found_deleted_entries = True
236+ else :
237+ entries_to_write .append (entry )
238+
239+ if not found_deleted_entries :
240+ existing_files .append (manifest_file )
241+ continue
242+
243+ if len (entries_to_write ) > 0 :
244+ with self .new_manifest_writer (self .spec (manifest_file .partition_spec_id )) as writer :
245+ for entry in entries_to_write :
246+ writer .add_entry (
247+ ManifestEntry .from_args (
248+ status = ManifestEntryStatus .EXISTING ,
249+ snapshot_id = entry .snapshot_id ,
250+ sequence_number = entry .sequence_number ,
251+ file_sequence_number = entry .file_sequence_number ,
252+ data_file = entry .data_file ,
253+ )
254+ )
255+ existing_files .append (writer .to_manifest_file ())
256+
257+ return existing_files
258+
219259 @abstractmethod
220260 def _deleted_entries (self ) -> list [ManifestEntry ]: ...
221261
@@ -770,49 +810,7 @@ class _OverwriteFiles(_SnapshotProducer["_OverwriteFiles"]):
770810
771811 def _existing_manifests (self ) -> list [ManifestFile ]:
772812 """Determine if there are any existing manifest files."""
773- existing_files = []
774-
775- manifest_evaluators : dict [int , Callable [[ManifestFile ], bool ]] = KeyDefaultDict (self ._build_manifest_evaluator )
776- if snapshot := self ._transaction .table_metadata .snapshot_by_name (name = self ._target_branch ):
777- for manifest_file in snapshot .manifests (io = self ._io ):
778- # Manifest does not contain rows that match the files to delete partitions
779- if not manifest_evaluators [manifest_file .partition_spec_id ](manifest_file ):
780- existing_files .append (manifest_file )
781- continue
782-
783- entries_to_write : set [ManifestEntry ] = set ()
784- found_deleted_entries : set [ManifestEntry ] = set ()
785-
786- for entry in manifest_file .fetch_manifest_entry (io = self ._io , discard_deleted = True ):
787- if entry .data_file in self ._deleted_data_files :
788- found_deleted_entries .add (entry )
789- else :
790- entries_to_write .add (entry )
791-
792- # Is the intercept the empty set?
793- if len (found_deleted_entries ) == 0 :
794- existing_files .append (manifest_file )
795- continue
796-
797- # Delete all files from manifest
798- if len (entries_to_write ) == 0 :
799- continue
800-
801- # We have to rewrite the manifest file without the deleted data files
802- with self .new_manifest_writer (self .spec (manifest_file .partition_spec_id )) as writer :
803- for entry in entries_to_write :
804- writer .add_entry (
805- ManifestEntry .from_args (
806- status = ManifestEntryStatus .EXISTING ,
807- snapshot_id = entry .snapshot_id ,
808- sequence_number = entry .sequence_number ,
809- file_sequence_number = entry .file_sequence_number ,
810- data_file = entry .data_file ,
811- )
812- )
813- existing_files .append (writer .to_manifest_file ())
814-
815- return existing_files
813+ return self ._get_existing_manifests (should_use_manifest_pruning = True )
816814
817815 def _deleted_entries (self ) -> list [ManifestEntry ]:
818816 """To determine if we need to record any deleted entries.
@@ -852,6 +850,120 @@ def _get_entries(manifest: ManifestFile) -> list[ManifestEntry]:
852850 return []
853851
854852
853+ class _RewriteFiles (_SnapshotProducer ["_RewriteFiles" ]):
854+ """A snapshot producer that rewrites data files.
855+
856+ Produces a REPLACE snapshot that swaps existing data files for new ones without
857+ changing the logical contents of the table. This is the metadata-only operation
858+ used by compaction (bin-packing, sort, format migration).
859+
860+ Current scope:
861+ - Data file rewriting only (delete + add DataFiles)
862+ - Validates: files-to-delete exist, added_records <= deleted_records,
863+ no new delete files conflict with replaced data files
864+
865+ Future work (additive — no structural changes needed):
866+ - Delete-file rewriting (add _deleted_delete_files set + separate manifest handling)
867+ - dataSequenceNumber override (pin new files' seq to match replaced, for eq-delete safety)
868+ - validateFromSnapshot (expose _starting_snapshot_id setter for long-running planners)
869+ - ignoreEqualityDeletes in validation (coupled with dataSequenceNumber)
870+ """
871+
872+ def _commit (self ) -> UpdatesAndRequirements :
873+ # Only produce a commit when there is something to rewrite
874+ if self ._deleted_data_files or self ._added_data_files :
875+ # Grab the entries that we actually found in the table's manifests
876+ deleted_entries = self ._deleted_entries ()
877+ found_deleted_files = {entry .data_file for entry in deleted_entries }
878+
879+ # If the user asked to delete files that aren't in the table, abort.
880+ if len (found_deleted_files ) != len (self ._deleted_data_files ):
881+ raise ValidationException ("Cannot commit, missing data files to be rewritten that are not in the table" )
882+
883+ added_records = sum (f .record_count for f in self ._added_data_files )
884+ deleted_records = sum (entry .data_file .record_count for entry in deleted_entries )
885+
886+ if added_records > deleted_records :
887+ raise ValidationException (
888+ f"Invalid replace: records added ({ added_records } ) exceeds records removed ({ deleted_records } )"
889+ )
890+
891+ return super ()._commit ()
892+ else :
893+ return (), ()
894+
895+ @cached_property
896+ def _cached_deleted_entries (self ) -> list [ManifestEntry ]:
897+ """Build manifest entries marking deleted data files with DELETED status."""
898+ if self ._parent_snapshot_id is not None :
899+ previous_snapshot = self ._transaction .table_metadata .snapshot_by_id (self ._parent_snapshot_id )
900+ if previous_snapshot is None :
901+ raise ValueError (f"Could not find the previous snapshot: { self ._parent_snapshot_id } " )
902+
903+ executor = ExecutorFactory .get_or_create ()
904+
905+ def _get_entries (manifest : ManifestFile ) -> list [ManifestEntry ]:
906+ return [
907+ ManifestEntry .from_args (
908+ status = ManifestEntryStatus .DELETED ,
909+ snapshot_id = self .snapshot_id ,
910+ sequence_number = entry .sequence_number ,
911+ file_sequence_number = entry .file_sequence_number ,
912+ data_file = entry .data_file ,
913+ )
914+ for entry in manifest .fetch_manifest_entry (self ._io , discard_deleted = True )
915+ if entry .data_file .content == DataFileContent .DATA and entry .data_file in self ._deleted_data_files
916+ ]
917+
918+ list_of_entries = executor .map (_get_entries , previous_snapshot .manifests (self ._io ))
919+ return list (itertools .chain (* list_of_entries ))
920+ else :
921+ return []
922+
923+ def _deleted_entries (self ) -> list [ManifestEntry ]:
924+ return self ._cached_deleted_entries
925+
926+ def _existing_manifests (self ) -> list [ManifestFile ]:
927+ return self ._get_existing_manifests ()
928+
929+ def _validate_concurrency (self ) -> None :
930+ """Validate that concurrent changes do not conflict with this replace.
931+
932+ Unlike overwrite/delete, a replace operation only needs to validate that no new
933+ delete files have been added that would apply to the data files being replaced.
934+ Concurrent data file additions (appends) do NOT conflict with a replace because
935+ the replace only touches files it explicitly planned to rewrite.
936+
937+ This matches Java's BaseRewriteFiles.validate() which only calls
938+ validateNoNewDeletesForDataFiles, not validateAddedDataFiles or
939+ validateDeletedDataFiles.
940+ """
941+ from pyiceberg .table .update .validate import _validate_no_new_deletes_for_data_files
942+
943+ if self ._commit_window is None or self ._commit_window .is_empty ():
944+ return
945+
946+ catalog_head = self ._commit_window .head
947+ starting_snapshot = self ._commit_window .base
948+
949+ if catalog_head is None :
950+ return
951+
952+ if self ._deleted_data_files :
953+ table = self ._transaction ._table
954+ conflict_detection_filter = self ._predicate if self ._predicate != AlwaysFalse () else None
955+ _validate_no_new_deletes_for_data_files (
956+ table , catalog_head , conflict_detection_filter , self ._deleted_data_files , starting_snapshot
957+ )
958+
959+ def _refresh_for_retry (self ) -> None :
960+ """Reset state for a retry attempt, clearing the cached deleted entries."""
961+ super ()._refresh_for_retry ()
962+ # Clear @cached_property so it recomputes against the refreshed parent snapshot.
963+ if "_cached_deleted_entries" in self .__dict__ :
964+ del self .__dict__ ["_cached_deleted_entries" ]
965+
966+
855967class UpdateSnapshot :
856968 _transaction : Transaction
857969 _io : FileIO
@@ -909,6 +1021,15 @@ def delete(self) -> _DeleteFiles:
9091021 snapshot_properties = self ._snapshot_properties ,
9101022 )
9111023
1024+ def replace (self ) -> _RewriteFiles :
1025+ return _RewriteFiles (
1026+ operation = Operation .REPLACE ,
1027+ transaction = self ._transaction ,
1028+ io = self ._io ,
1029+ branch = self ._branch ,
1030+ snapshot_properties = self ._snapshot_properties ,
1031+ )
1032+
9121033
9131034class _ManifestMergeManager (Generic [U ]):
9141035 _target_size_bytes : int
0 commit comments