Skip to content

Commit 44631e0

Browse files
committed
Support upgrading tables to format version 3
Allow upgrade_table_version(3) and initialize next-row-id to 0 on upgrade, per the spec's Row Lineage for Upgraded Tables. Persisting the upgraded metadata still requires V3 metadata serialization (apache#3551). Closes apache#3622
1 parent 1a87aa3 commit 44631e0

5 files changed

Lines changed: 34 additions & 7 deletions

File tree

‎pyiceberg/table/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
from pyiceberg.table.inspect import InspectTable
4949
from pyiceberg.table.locations import LocationProvider, load_location_provider
5050
from pyiceberg.table.maintenance import MaintenanceTable
51-
from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER, TableMetadata
51+
from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER, SUPPORTED_TABLE_FORMAT_VERSION, TableMetadata
5252
from pyiceberg.table.name_mapping import NameMapping
5353
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef
5454
from pyiceberg.table.snapshots import (
@@ -300,7 +300,7 @@ def upgrade_table_version(self, format_version: TableVersion) -> Transaction:
300300
Returns:
301301
The alter table builder.
302302
"""
303-
if format_version not in {1, 2}:
303+
if not 1 <= format_version <= SUPPORTED_TABLE_FORMAT_VERSION:
304304
raise ValueError(f"Unsupported table format version: {format_version}")
305305

306306
if format_version < self.table_metadata.format_version:

‎pyiceberg/table/metadata.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,10 @@
6666
INITIAL_SPEC_ID = 0
6767
DEFAULT_SCHEMA_ID = 0
6868

69-
SUPPORTED_TABLE_FORMAT_VERSION = 2
69+
SUPPORTED_TABLE_FORMAT_VERSION = 3
70+
71+
INITIAL_ROW_ID = 0
72+
"""The initial value of `next-row-id` for newly upgraded V3 tables."""
7073

7174

7275
def cleanup_snapshot_id(data: dict[str, Any]) -> dict[str, Any]:

‎pyiceberg/table/update/__init__.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
from pyiceberg.exceptions import CommitFailedException
3030
from pyiceberg.partitioning import PARTITION_FIELD_ID_START, PartitionSpec
3131
from pyiceberg.schema import Schema
32-
from pyiceberg.table.metadata import SUPPORTED_TABLE_FORMAT_VERSION, TableMetadata, TableMetadataUtil
32+
from pyiceberg.table.metadata import INITIAL_ROW_ID, SUPPORTED_TABLE_FORMAT_VERSION, TableMetadata, TableMetadataUtil
3333
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType
3434
from pyiceberg.table.snapshots import (
3535
MetadataLogEntry,
@@ -321,7 +321,10 @@ def _(
321321
elif update.format_version == base_metadata.format_version:
322322
return base_metadata
323323

324-
updated_metadata = base_metadata.model_copy(update={"format_version": update.format_version})
324+
updates: dict[str, Any] = {"format_version": update.format_version}
325+
if update.format_version >= 3 and getattr(base_metadata, "next_row_id", None) is None:
326+
updates["next_row_id"] = INITIAL_ROW_ID
327+
updated_metadata = base_metadata.model_copy(update=updates)
325328

326329
context.add_update(update)
327330
return TableMetadataUtil._construct_without_validation(updated_metadata)

‎tests/integration/test_reads.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -969,10 +969,16 @@ def test_upgrade_table_version(catalog: Catalog) -> None:
969969
transaction.upgrade_table_version(format_version=1)
970970
assert "Cannot downgrade v2 table to v1" in str(e.value)
971971

972+
with table_test_table_version.transaction() as transaction:
973+
transaction.upgrade_table_version(format_version=3)
974+
975+
assert table_test_table_version.format_version == 3
976+
assert table_test_table_version.metadata.next_row_id == 0
977+
972978
with pytest.raises(ValueError) as e:
973979
with table_test_table_version.transaction() as transaction:
974-
transaction.upgrade_table_version(format_version=3)
975-
assert "Unsupported table format version: 3" in str(e.value)
980+
transaction.upgrade_table_version(format_version=4) # type: ignore[arg-type]
981+
assert "Unsupported table format version: 4" in str(e.value)
976982

977983

978984
@pytest.mark.integration

‎tests/table/test_init.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
SetPropertiesUpdate,
8484
SetSnapshotRefUpdate,
8585
SetStatisticsUpdate,
86+
UpgradeFormatVersionUpdate,
8687
_apply_table_update,
8788
_TableMetadataUpdateContext,
8889
update_table_metadata,
@@ -1240,6 +1241,20 @@ def test_update_metadata_with_multiple_updates(table_v1: Table) -> None:
12401241
assert new_metadata.properties == {"owner": "test", "test_a": "test_a1"}
12411242

12421243

1244+
def test_update_metadata_upgrade_to_v3(table_v2: Table) -> None:
1245+
from pyiceberg.table.metadata import TableMetadataV3
1246+
1247+
new_metadata = update_table_metadata(table_v2.metadata, (UpgradeFormatVersionUpdate(format_version=3),))
1248+
assert isinstance(new_metadata, TableMetadataV3)
1249+
assert new_metadata.format_version == 3
1250+
assert new_metadata.next_row_id == 0
1251+
1252+
# rebuild the metadata to trigger validation
1253+
validated = TableMetadataUtil.parse_obj(copy(new_metadata.model_dump()))
1254+
assert validated.format_version == 3
1255+
assert validated.next_row_id == 0
1256+
1257+
12431258
def test_update_metadata_schema_immutability(
12441259
table_v2_with_fixed_and_decimal_types: TableMetadataV2,
12451260
) -> None:

0 commit comments

Comments
 (0)