Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Select an option

  • Save rambleraptor/216ca4dfcf74ef10f3a49015dbed877f to your computer and use it in GitHub Desktop.

Select an option

Save rambleraptor/216ca4dfcf74ef10f3a49015dbed877f to your computer and use it in GitHub Desktop.
import tempfile
from pathlib import Path
import pyarrow as pa
import pyiceberg.table as table_module
from pyiceberg.catalog.sql import SqlCatalog
from pyiceberg.exceptions import ValidationException
from pyiceberg.schema import Schema
from pyiceberg.types import LongType, NestedField
SCHEMA = Schema(NestedField(1, "x", LongType(), required=False))
def make_catalog(warehouse: Path) -> SqlCatalog:
return SqlCatalog(
"demo",
uri=f"sqlite:///{warehouse / 'catalog.db'}",
warehouse=f"file://{warehouse}",
)
def run_scenario(catalog: SqlCatalog, name: str, *, burst: bool) -> bool:
"""Returns True if the victim delete raised ValidationException."""
ident = f"default.{name}"
catalog.create_table(ident, schema=SCHEMA)
# S0: baseline the victim transaction reads from
setup = catalog.load_table(ident)
setup.append(pa.table({"x": [1, 2, 3]}))
victim = catalog.load_table(ident) # tbl2, based on S0
# S1: concurrent CONFLICTING commit (adds x == 1, matching the victim's delete filter)
writer1 = catalog.load_table(ident)
writer1.append(pa.table({"x": [1]}))
# In the burst case, inject a second, COMPATIBLE commit (S2: x == 99) during the
# victim's retry sleep. After this, the head is S2, not the conflicting S1.
injected = {"done": False}
original_sleep = table_module.time.sleep
def fake_sleep(_seconds: float) -> None:
if burst and not injected["done"]:
injected["done"] = True
writer2 = catalog.load_table(ident) # loads S1
writer2.append(pa.table({"x": [99]})) # commits S2 (non-conflicting)
table_module.time.sleep = fake_sleep
try:
# SERIALIZABLE is the default for write.delete.isolation-level.
victim.delete("x == 1")
return False
except ValidationException:
return True
finally:
table_module.time.sleep = original_sleep
def main() -> None:
with tempfile.TemporaryDirectory() as tmp:
catalog = make_catalog(Path(tmp))
catalog.create_namespace("default")
control_raised = run_scenario(catalog, "control", burst=False)
burst_raised = run_scenario(catalog, "burst", burst=True)
print(f"CONTROL (conflict is the head): ValidationException raised? {control_raised}")
print(f"BURST (conflict below head) : ValidationException raised? {burst_raised}")
print()
assert control_raised, "control should detect the conflict"
if not burst_raised:
print("BUG REPRODUCED: the conflicting commit S1 was missed because the")
print("validation window only inspected the head (S2). Serializable isolation")
print("silently failed to abort.")
else:
print("Fixed: the full window was validated and the conflict was detected.")
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment