Skip to content

Commit 919f598

Browse files
committed
Support rewrite_manifests table maintenance
Add table.maintenance.rewrite_manifests(), which merges the current snapshot's data manifests into fewer manifests sized by commit.manifest.target-size-bytes. Entries are rewritten as EXISTING and keep their sequence numbers; delete manifests and manifests that need no merging are kept as-is. The result is committed as a replace snapshot whose totals carry over unchanged. V3 tables are rejected for now: rewriting must preserve the first-row-id of rewritten manifests, which needs the read side of row lineage (apache#3621). Closes apache#3629
1 parent d0ee9d8 commit 919f598

5 files changed

Lines changed: 290 additions & 6 deletions

File tree

‎pyiceberg/table/maintenance.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424

2525
if TYPE_CHECKING:
2626
from pyiceberg.table import Table
27-
from pyiceberg.table.update.snapshot import ExpireSnapshots
27+
from pyiceberg.table.update.snapshot import ExpireSnapshots, RewriteManifests
2828

2929

3030
class MaintenanceTable:
@@ -43,3 +43,17 @@ def expire_snapshots(self) -> ExpireSnapshots:
4343
from pyiceberg.table.update.snapshot import ExpireSnapshots
4444

4545
return ExpireSnapshots(transaction=Transaction(self.tbl, autocommit=True))
46+
47+
def rewrite_manifests(self) -> RewriteManifests:
48+
"""Return a RewriteManifests operation that merges the current snapshot's data manifests.
49+
50+
Entries are rewritten as EXISTING, keeping their sequence numbers; delete
51+
manifests are kept as-is. The result is committed as a `replace` snapshot.
52+
53+
Returns:
54+
RewriteManifests operation; call commit() to execute it.
55+
"""
56+
from pyiceberg.table import Transaction
57+
from pyiceberg.table.update.snapshot import RewriteManifests
58+
59+
return RewriteManifests(transaction=Transaction(self.tbl, autocommit=True), io=self.tbl.io)

‎pyiceberg/table/snapshots.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,7 @@ def _partition_summary(self, update_metrics: UpdateMetrics) -> str:
351351

352352

353353
def update_snapshot_summaries(summary: Summary, previous_summary: Mapping[str, str] | None = None) -> Summary:
354-
if summary.operation not in {Operation.APPEND, Operation.OVERWRITE, Operation.DELETE}:
354+
if summary.operation not in {Operation.APPEND, Operation.OVERWRITE, Operation.DELETE, Operation.REPLACE}:
355355
raise ValueError(f"Operation not implemented: {summary.operation}")
356356

357357
if not previous_summary:

‎pyiceberg/table/update/snapshot.py‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1312,3 +1312,114 @@ def older_than(self, dt: datetime) -> ExpireSnapshots:
13121312
if snapshot.timestamp_ms < expire_from and snapshot.snapshot_id not in protected_ids:
13131313
self._snapshot_ids_to_expire.add(snapshot.snapshot_id)
13141314
return self
1315+
1316+
1317+
class RewriteManifests(_SnapshotProducer["RewriteManifests"]):
1318+
"""Rewrite the current snapshot's data manifests without changing data.
1319+
1320+
Live entries from the rewritten data manifests are regrouped into new
1321+
manifests sized by `commit.manifest.target-size-bytes`, written as EXISTING
1322+
entries that keep their sequence numbers. Delete manifests are kept as-is.
1323+
The result is committed as a `replace` snapshot.
1324+
"""
1325+
1326+
_rewritten_count: int
1327+
_created_count: int
1328+
_kept_count: int
1329+
_entries_processed: int
1330+
1331+
def __init__(
1332+
self,
1333+
transaction: Transaction,
1334+
io: FileIO,
1335+
commit_uuid: uuid.UUID | None = None,
1336+
snapshot_properties: dict[str, str] = EMPTY_DICT,
1337+
branch: str | None = MAIN_BRANCH,
1338+
) -> None:
1339+
super().__init__(Operation.REPLACE, transaction, io, commit_uuid, snapshot_properties, branch)
1340+
if transaction.table_metadata.format_version >= 3:
1341+
raise NotImplementedError(
1342+
"Rewriting manifests is not yet supported for V3 tables: "
1343+
"the first-row-id of rewritten manifests must be preserved, "
1344+
"see: https://github.com/apache/iceberg-python/issues/3621"
1345+
)
1346+
self._rewritten_count = 0
1347+
self._created_count = 0
1348+
self._kept_count = 0
1349+
self._entries_processed = 0
1350+
1351+
def _deleted_entries(self) -> list[ManifestEntry]:
1352+
return []
1353+
1354+
def _target_size_bytes(self) -> int:
1355+
from pyiceberg.table import TableProperties
1356+
1357+
return property_as_int( # type: ignore
1358+
self._transaction.table_metadata.properties,
1359+
TableProperties.MANIFEST_TARGET_SIZE_BYTES,
1360+
TableProperties.MANIFEST_TARGET_SIZE_BYTES_DEFAULT,
1361+
)
1362+
1363+
def _group_by_target_size(self, manifests: list[ManifestFile]) -> list[list[ManifestFile]]:
1364+
"""Pack manifests into groups whose source sizes add up to roughly the target size."""
1365+
target_size = self._target_size_bytes()
1366+
groups: list[list[ManifestFile]] = []
1367+
current_group: list[ManifestFile] = []
1368+
current_size = 0
1369+
for manifest in manifests:
1370+
if current_group and current_size + manifest.manifest_length > target_size:
1371+
groups.append(current_group)
1372+
current_group = []
1373+
current_size = 0
1374+
current_group.append(manifest)
1375+
current_size += manifest.manifest_length
1376+
if current_group:
1377+
groups.append(current_group)
1378+
return groups
1379+
1380+
def _existing_manifests(self) -> list[ManifestFile]:
1381+
snapshot = self._transaction.table_metadata.snapshot_by_name(self._target_branch or MAIN_BRANCH)
1382+
if snapshot is None:
1383+
return []
1384+
1385+
data_manifests_by_spec: defaultdict[int, list[ManifestFile]] = defaultdict(list)
1386+
kept_manifests: list[ManifestFile] = []
1387+
for manifest in snapshot.manifests(self._io):
1388+
if manifest.content == ManifestContent.DATA:
1389+
data_manifests_by_spec[manifest.partition_spec_id].append(manifest)
1390+
else:
1391+
kept_manifests.append(manifest)
1392+
1393+
new_manifests: list[ManifestFile] = []
1394+
for spec_id, manifests in data_manifests_by_spec.items():
1395+
for group in self._group_by_target_size(manifests):
1396+
if len(group) == 1:
1397+
# nothing to merge; keep the manifest as-is
1398+
kept_manifests.append(group[0])
1399+
continue
1400+
with self.new_manifest_writer(self.spec(spec_id)) as writer:
1401+
for manifest in group:
1402+
for entry in manifest.fetch_manifest_entry(self._io, discard_deleted=True):
1403+
writer.existing(entry)
1404+
self._entries_processed += 1
1405+
new_manifests.append(writer.to_manifest_file())
1406+
self._rewritten_count += len(group)
1407+
self._created_count += 1
1408+
1409+
self._kept_count = len(kept_manifests)
1410+
self.snapshot_properties = {
1411+
**self.snapshot_properties,
1412+
"manifests-created": str(self._created_count),
1413+
"manifests-kept": str(self._kept_count),
1414+
"manifests-replaced": str(self._rewritten_count),
1415+
"entries-processed": str(self._entries_processed),
1416+
}
1417+
return new_manifests + kept_manifests
1418+
1419+
def rewrites_needed(self) -> bool:
1420+
"""Return whether the current snapshot has more than one data manifest to merge."""
1421+
snapshot = self._transaction.table_metadata.snapshot_by_name(self._target_branch or MAIN_BRANCH)
1422+
if snapshot is None:
1423+
return False
1424+
data_manifests = [m for m in snapshot.manifests(self._io) if m.content == ManifestContent.DATA]
1425+
return len(data_manifests) > 1
Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
from pathlib import Path
18+
19+
import pyarrow as pa
20+
import pytest
21+
22+
from pyiceberg.catalog import Catalog
23+
from pyiceberg.catalog.memory import InMemoryCatalog
24+
from pyiceberg.manifest import ManifestContent
25+
from pyiceberg.table import Table
26+
from pyiceberg.table.snapshots import Operation
27+
28+
29+
@pytest.fixture
30+
def catalog(tmp_path: Path) -> Catalog:
31+
catalog = InMemoryCatalog("test.rewrite_manifests", warehouse=f"file://{tmp_path}")
32+
catalog.create_namespace("default")
33+
return catalog
34+
35+
36+
def _arrow_table(offset: int = 0) -> pa.Table:
37+
return pa.table({"id": pa.array([offset + 1, offset + 2, offset + 3], type=pa.int64())})
38+
39+
40+
def _create_table_with_appends(catalog: Catalog, appends: int = 3) -> Table:
41+
table = catalog.create_table("default.test_rewrite", schema=pa.schema([pa.field("id", pa.int64())]))
42+
for i in range(appends):
43+
table.append(_arrow_table(offset=i * 3))
44+
return table
45+
46+
47+
def _data_manifests(table: Table) -> list:
48+
snapshot = table.current_snapshot()
49+
assert snapshot is not None
50+
return [m for m in snapshot.manifests(table.io) if m.content == ManifestContent.DATA]
51+
52+
53+
def test_rewrite_manifests_merges_data_manifests(catalog: Catalog) -> None:
54+
table = _create_table_with_appends(catalog, appends=3)
55+
assert len(_data_manifests(table)) == 3
56+
rows_before = table.scan().to_arrow().sort_by("id")
57+
58+
table.maintenance.rewrite_manifests().commit()
59+
60+
table = catalog.load_table("default.test_rewrite")
61+
manifests = _data_manifests(table)
62+
assert len(manifests) == 1
63+
# entries are rewritten as EXISTING
64+
assert manifests[0].existing_files_count == 3
65+
assert manifests[0].added_files_count == 0
66+
67+
# data is unchanged
68+
assert table.scan().to_arrow().sort_by("id") == rows_before
69+
70+
snapshot = table.current_snapshot()
71+
assert snapshot is not None
72+
assert snapshot.summary is not None
73+
assert snapshot.summary.operation == Operation.REPLACE
74+
assert snapshot.summary["manifests-created"] == "1"
75+
assert snapshot.summary["manifests-replaced"] == "3"
76+
assert snapshot.summary["entries-processed"] == "3"
77+
# totals carry over unchanged
78+
assert snapshot.summary["total-data-files"] == "3"
79+
assert snapshot.summary["total-records"] == "9"
80+
81+
82+
def test_rewrite_manifests_preserves_sequence_numbers(catalog: Catalog) -> None:
83+
table = _create_table_with_appends(catalog, appends=3)
84+
entries_before = {
85+
entry.data_file.file_path: entry.sequence_number
86+
for manifest in _data_manifests(table)
87+
for entry in manifest.fetch_manifest_entry(table.io, discard_deleted=True)
88+
}
89+
90+
table.maintenance.rewrite_manifests().commit()
91+
92+
table = catalog.load_table("default.test_rewrite")
93+
manifests = _data_manifests(table)
94+
entries_after = {
95+
entry.data_file.file_path: entry.sequence_number
96+
for manifest in manifests
97+
for entry in manifest.fetch_manifest_entry(table.io, discard_deleted=True)
98+
}
99+
assert entries_after == entries_before
100+
# the merged manifest keeps the min sequence number of its entries
101+
assert manifests[0].min_sequence_number == min(entries_before.values())
102+
103+
104+
def test_rewrite_manifests_single_manifest_is_noop_kept(catalog: Catalog) -> None:
105+
table = _create_table_with_appends(catalog, appends=1)
106+
manifest_path_before = _data_manifests(table)[0].manifest_path
107+
108+
table.maintenance.rewrite_manifests().commit()
109+
110+
table = catalog.load_table("default.test_rewrite")
111+
manifests = _data_manifests(table)
112+
assert len(manifests) == 1
113+
# a single manifest is kept as-is, not rewritten
114+
assert manifests[0].manifest_path == manifest_path_before
115+
116+
snapshot = table.current_snapshot()
117+
assert snapshot is not None
118+
assert snapshot.summary is not None
119+
assert snapshot.summary["manifests-created"] == "0"
120+
assert snapshot.summary["manifests-replaced"] == "0"
121+
122+
123+
def test_rewrite_manifests_respects_target_size(catalog: Catalog) -> None:
124+
table = _create_table_with_appends(catalog, appends=4)
125+
manifest_length = _data_manifests(table)[0].manifest_length
126+
127+
# allow roughly two source manifests per group
128+
with table.transaction() as tx:
129+
tx.set_properties({"commit.manifest.target-size-bytes": str(manifest_length * 2)})
130+
131+
table = catalog.load_table("default.test_rewrite")
132+
table.maintenance.rewrite_manifests().commit()
133+
134+
table = catalog.load_table("default.test_rewrite")
135+
manifests = _data_manifests(table)
136+
assert len(manifests) == 2
137+
assert all(m.existing_files_count == 2 for m in manifests)
138+
139+
140+
def test_rewrites_needed(catalog: Catalog) -> None:
141+
table = _create_table_with_appends(catalog, appends=1)
142+
assert table.maintenance.rewrite_manifests().rewrites_needed() is False
143+
144+
table.append(_arrow_table(offset=3))
145+
table = catalog.load_table("default.test_rewrite")
146+
assert table.maintenance.rewrite_manifests().rewrites_needed() is True

‎tests/table/test_snapshots.py‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -399,10 +399,23 @@ def test_merge_snapshot_summaries_overwrite_summary() -> None:
399399
assert actual.additional_properties == expected
400400

401401

402-
def test_invalid_operation() -> None:
403-
with pytest.raises(ValueError) as e:
404-
update_snapshot_summaries(summary=Summary(Operation.REPLACE))
405-
assert "Operation not implemented: Operation.REPLACE" in str(e.value)
402+
def test_replace_operation_carries_totals() -> None:
403+
actual = update_snapshot_summaries(
404+
summary=Summary(Operation.REPLACE),
405+
previous_summary={
406+
"total-data-files": "3",
407+
"total-delete-files": "0",
408+
"total-records": "9",
409+
"total-files-size": "1234",
410+
"total-position-deletes": "0",
411+
"total-equality-deletes": "0",
412+
},
413+
)
414+
415+
# a replace operation does not change any of the totals
416+
assert actual["total-data-files"] == "3"
417+
assert actual["total-records"] == "9"
418+
assert actual["total-files-size"] == "1234"
406419

407420

408421
def test_invalid_type() -> None:

0 commit comments

Comments
 (0)