Skip to content

Commit 19ddb77

Browse files
moomindanicayetanobv
authored andcommitted
Support writing V3 manifests and manifest lists
Add ManifestWriterV3 and ManifestListWriterV3. The manifest writer uses the V3 record layout so the V3-only data file fields (first_row_id, referenced_data_file, content_offset, content_size_in_bytes) are written, rebinding V2-layout records when needed. The manifest list writer implements the spec's First Row ID Assignment: preserving existing first_row_id values, never assigning one to delete manifests, and assigning a running value advanced by existing and added row counts to unassigned data manifests. It also exposes next_row_id for the commit path to update the table's next-row-id. Closes apache#3620
1 parent ef95e63 commit 19ddb77

2 files changed

Lines changed: 256 additions & 1 deletion

File tree

‎pyiceberg/manifest.py‎

Lines changed: 127 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -883,7 +883,14 @@ def key_metadata(self) -> bytes | None:
883883

884884
@property
885885
def first_row_id(self) -> int | None:
886-
return self._data[15]
886+
# Only present when the record is bound to the V3 manifest list schema
887+
return self._data[15] if len(self._data) > 15 else None
888+
889+
@first_row_id.setter
890+
def first_row_id(self, value: int | None) -> None:
891+
if len(self._data) <= 15:
892+
raise ValueError("Cannot set first_row_id on a manifest file not bound to the V3 schema")
893+
self._data[15] = value
887894

888895
def has_added_files(self) -> bool:
889896
return self.added_files_count is None or self.added_files_count > 0
@@ -1330,6 +1337,51 @@ def prepare_entry(self, entry: ManifestEntry) -> ManifestEntry:
13301337
return entry
13311338

13321339

1340+
class ManifestWriterV3(ManifestWriterV2):
1341+
"""Writes V3 manifest files.
1342+
1343+
The writer inherits the V2 sequence-number semantics; the V3 manifest entry
1344+
schema additionally carries `first_row_id`, `referenced_data_file`,
1345+
`content_offset` and `content_size_in_bytes` on the data file struct.
1346+
"""
1347+
1348+
@property
1349+
def version(self) -> TableVersion:
1350+
return 3
1351+
1352+
def new_writer(self) -> AvroOutputFile[ManifestEntry]:
1353+
# Use the V3 record layout so the V3-only data file fields are written
1354+
return AvroOutputFile[ManifestEntry](
1355+
output_file=self._output_file,
1356+
file_schema=self._with_partition(3),
1357+
record_schema=self._with_partition(3),
1358+
schema_name="manifest_entry",
1359+
metadata=self._meta,
1360+
)
1361+
1362+
def _wrap_data_file(self, data_file: DataFile) -> DataFile:
1363+
"""Rebind a data file to the V3 record layout."""
1364+
if len(data_file._data) >= len(DATA_FILE_TYPE[3].fields):
1365+
return data_file
1366+
args = {
1367+
field.name: value for field, value in zip(DATA_FILE_TYPE[DEFAULT_READ_VERSION].fields, data_file._data, strict=True)
1368+
}
1369+
return DataFile.from_args(_table_format_version=3, **args)
1370+
1371+
def prepare_entry(self, entry: ManifestEntry) -> ManifestEntry:
1372+
entry = super().prepare_entry(entry)
1373+
if len(entry.data_file._data) < len(DATA_FILE_TYPE[3].fields):
1374+
entry = ManifestEntry.from_args(
1375+
_table_format_version=3,
1376+
status=entry.status,
1377+
snapshot_id=entry.snapshot_id,
1378+
sequence_number=entry.sequence_number,
1379+
file_sequence_number=entry.file_sequence_number,
1380+
data_file=self._wrap_data_file(entry.data_file),
1381+
)
1382+
return entry
1383+
1384+
13331385
def write_manifest(
13341386
format_version: TableVersion,
13351387
spec: PartitionSpec,
@@ -1342,6 +1394,8 @@ def write_manifest(
13421394
return ManifestWriterV1(spec, schema, output_file, snapshot_id, avro_compression)
13431395
elif format_version == 2:
13441396
return ManifestWriterV2(spec, schema, output_file, snapshot_id, avro_compression)
1397+
elif format_version == 3:
1398+
return ManifestWriterV3(spec, schema, output_file, snapshot_id, avro_compression)
13451399
else:
13461400
raise ValueError(f"Cannot write manifest for table version: {format_version}")
13471401

@@ -1465,19 +1519,91 @@ def prepare_manifest(self, manifest_file: ManifestFile) -> ManifestFile:
14651519
return wrapped_manifest_file
14661520

14671521

1522+
class ManifestListWriterV3(ManifestListWriterV2):
1523+
"""Writes V3 manifest lists, assigning `first_row_id` to data manifests.
1524+
1525+
Follows the spec's First Row ID Assignment: existing `first_row_id` values are
1526+
preserved, delete manifests are never assigned one, and data manifests without a
1527+
`first_row_id` are assigned a running value starting at the snapshot's
1528+
`first-row-id`, advanced by each assigned manifest's existing and added row counts.
1529+
"""
1530+
1531+
_next_row_id: int
1532+
1533+
def __init__(
1534+
self,
1535+
output_file: OutputFile,
1536+
snapshot_id: int,
1537+
parent_snapshot_id: int | None,
1538+
sequence_number: int,
1539+
compression: AvroCompressionCodec,
1540+
first_row_id: int,
1541+
):
1542+
super().__init__(output_file, snapshot_id, parent_snapshot_id, sequence_number, compression)
1543+
self._format_version = 3
1544+
self._meta = {
1545+
**self._meta,
1546+
"first-row-id": str(first_row_id),
1547+
"format-version": "3",
1548+
}
1549+
self._next_row_id = first_row_id
1550+
1551+
def __enter__(self) -> ManifestListWriter:
1552+
"""Open the writer for writing, using the V3 record layout so `first_row_id` is written."""
1553+
self._writer = AvroOutputFile[ManifestFile](
1554+
output_file=self._output_file,
1555+
record_schema=MANIFEST_LIST_FILE_SCHEMAS[3],
1556+
file_schema=MANIFEST_LIST_FILE_SCHEMAS[3],
1557+
schema_name="manifest_file",
1558+
metadata=self._meta,
1559+
)
1560+
self._writer.__enter__()
1561+
return self
1562+
1563+
@property
1564+
def next_row_id(self) -> int:
1565+
"""The row ID after the last assigned one; the table's `next-row-id` after this snapshot."""
1566+
return self._next_row_id
1567+
1568+
def _wrap(self, manifest_file: ManifestFile) -> ManifestFile:
1569+
"""Rebind a manifest file to the V3 record layout."""
1570+
if len(manifest_file._data) >= len(MANIFEST_LIST_FILE_SCHEMAS[3].fields):
1571+
return copy(manifest_file)
1572+
args = {
1573+
field.name: value
1574+
for field, value in zip(MANIFEST_LIST_FILE_SCHEMAS[DEFAULT_READ_VERSION].fields, manifest_file._data, strict=True)
1575+
}
1576+
return ManifestFile.from_args(_table_format_version=3, **args)
1577+
1578+
def prepare_manifest(self, manifest_file: ManifestFile) -> ManifestFile:
1579+
wrapped_manifest_file = super().prepare_manifest(self._wrap(manifest_file))
1580+
1581+
if wrapped_manifest_file.content == ManifestContent.DATA and wrapped_manifest_file.first_row_id is None:
1582+
wrapped_manifest_file.first_row_id = self._next_row_id
1583+
self._next_row_id += (wrapped_manifest_file.existing_rows_count or 0) + (wrapped_manifest_file.added_rows_count or 0)
1584+
return wrapped_manifest_file
1585+
1586+
14681587
def write_manifest_list(
14691588
format_version: TableVersion,
14701589
output_file: OutputFile,
14711590
snapshot_id: int,
14721591
parent_snapshot_id: int | None,
14731592
sequence_number: int | None,
14741593
avro_compression: AvroCompressionCodec,
1594+
first_row_id: int | None = None,
14751595
) -> ManifestListWriter:
14761596
if format_version == 1:
14771597
return ManifestListWriterV1(output_file, snapshot_id, parent_snapshot_id, avro_compression)
14781598
elif format_version == 2:
14791599
if sequence_number is None:
14801600
raise ValueError(f"Sequence-number is required for V2 tables: {sequence_number}")
14811601
return ManifestListWriterV2(output_file, snapshot_id, parent_snapshot_id, sequence_number, avro_compression)
1602+
elif format_version == 3:
1603+
if sequence_number is None:
1604+
raise ValueError(f"Sequence-number is required for V3 tables: {sequence_number}")
1605+
if first_row_id is None:
1606+
raise ValueError(f"First-row-id is required for V3 tables: {first_row_id}")
1607+
return ManifestListWriterV3(output_file, snapshot_id, parent_snapshot_id, sequence_number, avro_compression, first_row_id)
14821608
else:
14831609
raise ValueError(f"Cannot write manifest list for table version: {format_version}")

‎tests/utils/test_manifest.py‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1477,3 +1477,132 @@ def test_negative_manifest_cache_size_raises_value_error(monkeypatch: pytest.Mon
14771477
finally:
14781478
monkeypatch.delenv("PYICEBERG_MANIFEST_CACHE_SIZE", raising=False)
14791479
importlib.reload(manifest_module)
1480+
1481+
1482+
@pytest.mark.parametrize("compression", ["null", "deflate"])
1483+
def test_write_manifest_v3(compression: AvroCompressionCodec) -> None:
1484+
io = load_file_io()
1485+
test_schema = Schema(NestedField(1, "foo", IntegerType(), False))
1486+
1487+
v3_data_file = DataFile.from_args(
1488+
_table_format_version=3,
1489+
content=DataFileContent.DATA,
1490+
file_path="/data/file-v3.parquet",
1491+
file_format=FileFormat.PARQUET,
1492+
partition=Record(),
1493+
record_count=100,
1494+
file_size_in_bytes=1024,
1495+
first_row_id=1000,
1496+
)
1497+
v2_data_file = DataFile.from_args(
1498+
content=DataFileContent.DATA,
1499+
file_path="/data/file-v2.parquet",
1500+
file_format=FileFormat.PARQUET,
1501+
partition=Record(),
1502+
record_count=50,
1503+
file_size_in_bytes=512,
1504+
)
1505+
1506+
with TemporaryDirectory() as tmp_dir:
1507+
path = tmp_dir + "/manifest-v3.avro"
1508+
with write_manifest(
1509+
format_version=3,
1510+
spec=UNPARTITIONED_PARTITION_SPEC,
1511+
schema=test_schema,
1512+
output_file=io.new_output(path),
1513+
snapshot_id=25,
1514+
avro_compression=compression,
1515+
) as writer:
1516+
writer.add(ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=25, data_file=v3_data_file))
1517+
# a data file bound to the default (V2) layout is rebound to the V3 layout
1518+
writer.add(ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=25, data_file=v2_data_file))
1519+
1520+
_verify_metadata_with_fastavro(path, {"format-version": "3", "content": "data"})
1521+
1522+
with open(path, "rb") as f:
1523+
entries = list(fastavro.reader(f))
1524+
1525+
assert len(entries) == 2
1526+
assert entries[0]["data_file"]["first_row_id"] == 1000
1527+
assert entries[1]["data_file"]["first_row_id"] is None
1528+
for entry in entries:
1529+
for v3_field in ("first_row_id", "referenced_data_file", "content_offset", "content_size_in_bytes"):
1530+
assert v3_field in entry["data_file"]
1531+
1532+
1533+
@pytest.mark.parametrize("compression", ["null", "deflate"])
1534+
def test_write_manifest_list_v3_assigns_first_row_id(compression: AvroCompressionCodec) -> None:
1535+
io = load_file_io()
1536+
1537+
def manifest(path: str, content: ManifestContent, first_row_id: int | None = None, **counts: int) -> ManifestFile:
1538+
args: dict[str, Any] = {
1539+
"manifest_path": path,
1540+
"manifest_length": 100,
1541+
"partition_spec_id": 0,
1542+
"content": content,
1543+
"sequence_number": 1,
1544+
"min_sequence_number": 1,
1545+
"added_snapshot_id": 25,
1546+
"added_files_count": 1,
1547+
"existing_files_count": 1,
1548+
"deleted_files_count": 0,
1549+
"added_rows_count": counts.get("added", 0),
1550+
"existing_rows_count": counts.get("existing", 0),
1551+
"deleted_rows_count": 0,
1552+
}
1553+
if first_row_id is not None:
1554+
return ManifestFile.from_args(_table_format_version=3, first_row_id=first_row_id, **args)
1555+
# bound to the default (V2) layout to exercise rebinding in the writer
1556+
return ManifestFile.from_args(**args)
1557+
1558+
unassigned_data = manifest("/m1.avro", ManifestContent.DATA, added=100, existing=25)
1559+
preserved_data = manifest("/m2.avro", ManifestContent.DATA, first_row_id=77, added=10)
1560+
deletes = manifest("/m3.avro", ManifestContent.DELETES, added=10)
1561+
second_unassigned_data = manifest("/m4.avro", ManifestContent.DATA, added=5)
1562+
1563+
with TemporaryDirectory() as tmp_dir:
1564+
path = tmp_dir + "/manifest-list-v3.avro"
1565+
with write_manifest_list(
1566+
format_version=3,
1567+
output_file=io.new_output(path),
1568+
snapshot_id=25,
1569+
parent_snapshot_id=19,
1570+
sequence_number=2,
1571+
avro_compression=compression,
1572+
first_row_id=1000,
1573+
) as writer:
1574+
writer.add_manifests([unassigned_data, preserved_data, deletes, second_unassigned_data])
1575+
1576+
# 1000 + (100 + 25) + (5): assigned manifests advance by added + existing rows
1577+
assert writer.next_row_id == 1130 # type: ignore[attr-defined]
1578+
1579+
_verify_metadata_with_fastavro(
1580+
path,
1581+
{
1582+
"snapshot-id": "25",
1583+
"parent-snapshot-id": "19",
1584+
"sequence-number": "2",
1585+
"first-row-id": "1000",
1586+
"format-version": "3",
1587+
},
1588+
)
1589+
1590+
with open(path, "rb") as f:
1591+
records = list(fastavro.reader(f))
1592+
1593+
assert [r["first_row_id"] for r in records] == [1000, 77, None, 1125]
1594+
1595+
1596+
def test_write_manifest_list_v3_requires_first_row_id() -> None:
1597+
io = load_file_io()
1598+
with TemporaryDirectory() as tmp_dir:
1599+
with pytest.raises(ValueError, match="First-row-id is required for V3 tables"):
1600+
write_manifest_list(
1601+
format_version=3,
1602+
output_file=io.new_output(tmp_dir + "/manifest-list.avro"),
1603+
snapshot_id=25,
1604+
parent_snapshot_id=19,
1605+
sequence_number=2,
1606+
avro_compression="null",
1607+
first_row_id=None,
1608+
)

0 commit comments

Comments
 (0)