Skip to content

Commit 826da07

Browse files
avison9Fokko
andauthored
Validate each branch against its own commits on retry (apache#4317)
* Assert every ref a transaction writes, not just the first Transaction._stage de-duplicated requirements by type, so a transaction writing two branches only sent AssertRefSnapshotId for the first one and a concurrent change to the second was overwritten. Key ref assertions by ref name so each ref keeps its base assertion, as Java does. Closes apache#4183 * Validate each branch against its own commits on retry The retry path and the empty-output validation path resolved one CommitWindow from the first snapshot producer and reused it for every producer, so in a multi-branch transaction a delete or overwrite on a later branch was validated against the first branch's commits and could skip validation entirely. Resolve one window per target branch, based on the first producer on that branch, since later producers on the same branch start from a snapshot staged by this transaction. Closes apache#4316 --------- Co-authored-by: Fokko Driesprong <[email protected]>
1 parent 8e1f4a3 commit 826da07

2 files changed

Lines changed: 86 additions & 14 deletions

File tree

‎pyiceberg/table/__init__.py‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1197,17 +1197,10 @@ def commit_transaction(self) -> Table:
11971197
elif self._snapshot_producers:
11981198
# An empty staged output (e.g. a delete whose plan matched nothing) skips the
11991199
# commit loop above, so run concurrency validation explicitly before reporting success.
1200-
from pyiceberg.table.update.snapshot import CommitWindow
1201-
12021200
try:
12031201
self._table.refresh()
1204-
commit_window = CommitWindow.resolve(
1205-
self._table.metadata,
1206-
self._snapshot_producers[0]._starting_snapshot_id,
1207-
self._snapshot_producers[0]._target_branch,
1208-
)
1202+
self._resolve_commit_windows()
12091203
for producer in self._snapshot_producers:
1210-
producer._commit_window = commit_window
12111204
producer._validate_concurrency()
12121205
except Exception:
12131206
for producer in self._snapshot_producers:
@@ -1245,20 +1238,30 @@ def _cleanup_uncommitted_manifests(self) -> None:
12451238
for producer in self._snapshot_producers:
12461239
producer._cleanup_uncommitted()
12471240

1241+
def _resolve_commit_windows(self) -> None:
1242+
"""Resolve each producer's commit window against the refreshed metadata of its own branch.
1243+
1244+
The window starts where the transaction first wrote to the branch. Later producers on the same
1245+
branch start from a snapshot staged by this transaction, which the catalog has never seen.
1246+
"""
1247+
from pyiceberg.table.update.snapshot import CommitWindow
1248+
1249+
windows: dict[str | None, CommitWindow] = {}
1250+
for producer in self._snapshot_producers:
1251+
branch = producer._target_branch
1252+
if branch not in windows:
1253+
windows[branch] = CommitWindow.resolve(self._table.metadata, producer._starting_snapshot_id, branch)
1254+
producer._commit_window = windows[branch]
1255+
12481256
def _rebuild_snapshot_updates(self) -> None:
12491257
"""Rebuild snapshot updates for retry by re-executing registered producers."""
12501258
from pyiceberg.table.update import AddSnapshotUpdate, AssertRefSnapshotId, SetSnapshotRefUpdate
1251-
from pyiceberg.table.update.snapshot import CommitWindow
12521259

12531260
self._updates = tuple(u for u in self._updates if not isinstance(u, (AddSnapshotUpdate, SetSnapshotRefUpdate)))
12541261
self._requirements = tuple(r for r in self._requirements if not isinstance(r, AssertRefSnapshotId))
12551262

1256-
starting_id = self._snapshot_producers[0]._starting_snapshot_id if self._snapshot_producers else None
1257-
target_branch = self._snapshot_producers[0]._target_branch if self._snapshot_producers else None
1258-
commit_window = CommitWindow.resolve(self._table.metadata, starting_id, target_branch)
1259-
1263+
self._resolve_commit_windows()
12601264
for producer in self._snapshot_producers:
1261-
producer._commit_window = commit_window
12621265
producer._refresh_for_retry()
12631266
producer._validate_concurrency()
12641267
updates, requirements = producer._commit()

‎tests/table/test_commit_retry.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1493,3 +1493,72 @@ def test_transaction_stages_one_ref_assertion_per_ref(catalog: Catalog) -> None:
14931493
ref_assertions = {r.ref: r.snapshot_id for r in tx._requirements if isinstance(r, AssertRefSnapshotId)}
14941494
assert ref_assertions == {"b": base_snapshot_id, "main": base_snapshot_id}
14951495
assert len([r for r in tx._requirements if isinstance(r, AssertRefSnapshotId)]) == 2
1496+
1497+
1498+
def _table_with_branch(catalog: Catalog, identifier: str) -> Any:
1499+
import pyarrow as pa
1500+
1501+
tbl = catalog.create_table(
1502+
identifier,
1503+
schema=_test_schema(),
1504+
properties={TableProperties.COMMIT_MIN_RETRY_WAIT_MS: "1", TableProperties.COMMIT_MAX_RETRY_WAIT_MS: "2"},
1505+
)
1506+
tbl.append(pa.table({"x": [0, 1]}))
1507+
snapshot_id = tbl.metadata.current_snapshot_id
1508+
assert snapshot_id is not None
1509+
tbl.manage_snapshots().create_branch(snapshot_id=snapshot_id, branch_name="b").commit()
1510+
return tbl
1511+
1512+
1513+
def test_retry_validates_each_branch_against_its_own_commits(catalog: Catalog) -> None:
1514+
"""A retried delete on a second branch is validated against that branch, not the first producer's branch."""
1515+
import pyarrow as pa
1516+
1517+
catalog.create_namespace("default")
1518+
tx = _table_with_branch(catalog, "default.retry_window_per_branch").transaction()
1519+
tx.append(pa.table({"x": [9]}))
1520+
tx.delete("x == 1", branch="b")
1521+
1522+
# A row matching the delete lands on b after the delete was planned; main is untouched.
1523+
catalog.load_table("default.retry_window_per_branch").append(pa.table({"x": [1]}), branch="b")
1524+
1525+
with pytest.raises(ValidationException):
1526+
tx.commit_transaction()
1527+
1528+
refreshed = catalog.load_table("default.retry_window_per_branch")
1529+
assert sorted(refreshed.scan().use_ref("b").to_arrow()["x"].to_pylist()) == [0, 1, 1]
1530+
assert sorted(refreshed.scan().to_arrow()["x"].to_pylist()) == [0, 1]
1531+
1532+
1533+
def test_empty_plan_validation_checks_each_branch_against_its_own_commits(catalog: Catalog) -> None:
1534+
"""Empty-plan deletes on two branches are each validated against their own branch."""
1535+
import pyarrow as pa
1536+
1537+
catalog.create_namespace("default")
1538+
tx = _table_with_branch(catalog, "default.empty_plan_window_per_branch").transaction()
1539+
with pytest.warns(UserWarning): # neither delete matches anything at staging time
1540+
tx.delete("x > 45")
1541+
tx.delete("x > 45", branch="b")
1542+
1543+
catalog.load_table("default.empty_plan_window_per_branch").append(pa.table({"x": [50]}), branch="b")
1544+
1545+
with pytest.raises(ValidationException):
1546+
tx.commit_transaction()
1547+
1548+
1549+
def test_retry_with_two_writes_to_one_branch_succeeds(catalog: Catalog) -> None:
1550+
"""The second write to a branch starts from a staged snapshot; its retry window must use the committed base."""
1551+
import pyarrow as pa
1552+
1553+
catalog.create_namespace("default")
1554+
tx = _table_with_branch(catalog, "default.retry_two_writes_one_branch").transaction()
1555+
tx.append(pa.table({"x": [9]}))
1556+
tx.delete("x == 0")
1557+
1558+
# Non-conflicting concurrent append to the same branch forces a retry.
1559+
catalog.load_table("default.retry_two_writes_one_branch").append(pa.table({"x": [5]}))
1560+
1561+
tx.commit_transaction()
1562+
1563+
rows = sorted(catalog.load_table("default.retry_two_writes_one_branch").scan().to_arrow()["x"].to_pylist())
1564+
assert rows == [1, 5, 9]

0 commit comments

Comments
 (0)