Skip to content

Commit 4f3e899

Browse files
moomindaniclaude
andcommitted
Support upgrading tables to format version 3
Allow upgrade_table_version(3) and initialize next-row-id to 0 on upgrade, per the spec's Row Lineage for Upgraded Tables. Persisting the upgraded metadata still requires V3 metadata serialization (apache#3551). Closes apache#3622 Co-authored-by: Claude Fable 5 <[email protected]>
1 parent ec1413d commit 4f3e899

5 files changed

Lines changed: 34 additions & 7 deletions

File tree

‎pyiceberg/table/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
from pyiceberg.table.inspect import InspectTable
4949
from pyiceberg.table.locations import LocationProvider, load_location_provider
5050
from pyiceberg.table.maintenance import MaintenanceTable
51-
from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER, TableMetadata
51+
from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER, SUPPORTED_TABLE_FORMAT_VERSION, TableMetadata
5252
from pyiceberg.table.name_mapping import NameMapping
5353
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef
5454
from pyiceberg.table.snapshots import Snapshot, SnapshotLogEntry
@@ -293,7 +293,7 @@ def upgrade_table_version(self, format_version: TableVersion) -> Transaction:
293293
Returns:
294294
The alter table builder.
295295
"""
296-
if format_version not in {1, 2}:
296+
if not 1 <= format_version <= SUPPORTED_TABLE_FORMAT_VERSION:
297297
raise ValueError(f"Unsupported table format version: {format_version}")
298298

299299
if format_version < self.table_metadata.format_version:

‎pyiceberg/table/metadata.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,10 @@
6666
INITIAL_SPEC_ID = 0
6767
DEFAULT_SCHEMA_ID = 0
6868

69-
SUPPORTED_TABLE_FORMAT_VERSION = 2
69+
SUPPORTED_TABLE_FORMAT_VERSION = 3
70+
71+
INITIAL_ROW_ID = 0
72+
"""The initial value of `next-row-id` for newly upgraded V3 tables."""
7073

7174

7275
def cleanup_snapshot_id(data: dict[str, Any]) -> dict[str, Any]:

‎pyiceberg/table/update/__init__.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
from pyiceberg.exceptions import CommitFailedException
3030
from pyiceberg.partitioning import PARTITION_FIELD_ID_START, PartitionSpec
3131
from pyiceberg.schema import Schema
32-
from pyiceberg.table.metadata import SUPPORTED_TABLE_FORMAT_VERSION, TableMetadata, TableMetadataUtil
32+
from pyiceberg.table.metadata import INITIAL_ROW_ID, SUPPORTED_TABLE_FORMAT_VERSION, TableMetadata, TableMetadataUtil
3333
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType
3434
from pyiceberg.table.snapshots import (
3535
MetadataLogEntry,
@@ -321,7 +321,10 @@ def _(
321321
elif update.format_version == base_metadata.format_version:
322322
return base_metadata
323323

324-
updated_metadata = base_metadata.model_copy(update={"format_version": update.format_version})
324+
updates: dict[str, Any] = {"format_version": update.format_version}
325+
if update.format_version >= 3 and getattr(base_metadata, "next_row_id", None) is None:
326+
updates["next_row_id"] = INITIAL_ROW_ID
327+
updated_metadata = base_metadata.model_copy(update=updates)
325328

326329
context.add_update(update)
327330
return TableMetadataUtil._construct_without_validation(updated_metadata)

‎tests/integration/test_reads.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -968,10 +968,16 @@ def test_upgrade_table_version(catalog: Catalog) -> None:
968968
transaction.upgrade_table_version(format_version=1)
969969
assert "Cannot downgrade v2 table to v1" in str(e.value)
970970

971+
with table_test_table_version.transaction() as transaction:
972+
transaction.upgrade_table_version(format_version=3)
973+
974+
assert table_test_table_version.format_version == 3
975+
assert table_test_table_version.metadata.next_row_id == 0
976+
971977
with pytest.raises(ValueError) as e:
972978
with table_test_table_version.transaction() as transaction:
973-
transaction.upgrade_table_version(format_version=3)
974-
assert "Unsupported table format version: 3" in str(e.value)
979+
transaction.upgrade_table_version(format_version=4) # type: ignore[arg-type]
980+
assert "Unsupported table format version: 4" in str(e.value)
975981

976982

977983
@pytest.mark.integration

‎tests/table/test_init.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@
8282
SetPropertiesUpdate,
8383
SetSnapshotRefUpdate,
8484
SetStatisticsUpdate,
85+
UpgradeFormatVersionUpdate,
8586
_apply_table_update,
8687
_TableMetadataUpdateContext,
8788
update_table_metadata,
@@ -1017,6 +1018,20 @@ def test_update_metadata_with_multiple_updates(table_v1: Table) -> None:
10171018
assert new_metadata.properties == {"owner": "test", "test_a": "test_a1"}
10181019

10191020

1021+
def test_update_metadata_upgrade_to_v3(table_v2: Table) -> None:
1022+
from pyiceberg.table.metadata import TableMetadataV3
1023+
1024+
new_metadata = update_table_metadata(table_v2.metadata, (UpgradeFormatVersionUpdate(format_version=3),))
1025+
assert isinstance(new_metadata, TableMetadataV3)
1026+
assert new_metadata.format_version == 3
1027+
assert new_metadata.next_row_id == 0
1028+
1029+
# rebuild the metadata to trigger validation
1030+
validated = TableMetadataUtil.parse_obj(copy(new_metadata.model_dump()))
1031+
assert validated.format_version == 3
1032+
assert validated.next_row_id == 0
1033+
1034+
10201035
def test_update_metadata_schema_immutability(
10211036
table_v2_with_fixed_and_decimal_types: TableMetadataV2,
10221037
) -> None:

0 commit comments

Comments
 (0)