Skip to content

Commit 23b426b

Browse files
committed
Support writing V3 manifests and manifest lists
Add ManifestWriterV3 and ManifestListWriterV3. The manifest writer uses the V3 record layout so the V3-only data file fields (first_row_id, referenced_data_file, content_offset, content_size_in_bytes) are written, rebinding V2-layout records when needed. The manifest list writer implements the spec's First Row ID Assignment: preserving existing first_row_id values, never assigning one to delete manifests, and assigning a running value advanced by existing and added row counts to unassigned data manifests. It also exposes next_row_id for the commit path to update the table's next-row-id. Closes apache#3620
1 parent 068aae5 commit 23b426b

2 files changed

Lines changed: 256 additions & 1 deletion

File tree

‎pyiceberg/manifest.py‎

Lines changed: 127 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -883,7 +883,14 @@ def key_metadata(self) -> bytes | None:
883883

884884
@property
885885
def first_row_id(self) -> int | None:
886-
return self._data[15]
886+
# Only present when the record is bound to the V3 manifest list schema
887+
return self._data[15] if len(self._data) > 15 else None
888+
889+
@first_row_id.setter
890+
def first_row_id(self, value: int | None) -> None:
891+
if len(self._data) <= 15:
892+
raise ValueError("Cannot set first_row_id on a manifest file not bound to the V3 schema")
893+
self._data[15] = value
887894

888895
def has_added_files(self) -> bool:
889896
return self.added_files_count is None or self.added_files_count > 0
@@ -1330,6 +1337,51 @@ def prepare_entry(self, entry: ManifestEntry) -> ManifestEntry:
13301337
return entry
13311338

13321339

1340+
class ManifestWriterV3(ManifestWriterV2):
1341+
"""Writes V3 manifest files.
1342+
1343+
The writer inherits the V2 sequence-number semantics; the V3 manifest entry
1344+
schema additionally carries `first_row_id`, `referenced_data_file`,
1345+
`content_offset` and `content_size_in_bytes` on the data file struct.
1346+
"""
1347+
1348+
@property
1349+
def version(self) -> TableVersion:
1350+
return 3
1351+
1352+
def new_writer(self) -> AvroOutputFile[ManifestEntry]:
1353+
# Use the V3 record layout so the V3-only data file fields are written
1354+
return AvroOutputFile[ManifestEntry](
1355+
output_file=self._output_file,
1356+
file_schema=self._with_partition(3),
1357+
record_schema=self._with_partition(3),
1358+
schema_name="manifest_entry",
1359+
metadata=self._meta,
1360+
)
1361+
1362+
def _wrap_data_file(self, data_file: DataFile) -> DataFile:
1363+
"""Rebind a data file to the V3 record layout."""
1364+
if len(data_file._data) >= len(DATA_FILE_TYPE[3].fields):
1365+
return data_file
1366+
args = {
1367+
field.name: value for field, value in zip(DATA_FILE_TYPE[DEFAULT_READ_VERSION].fields, data_file._data, strict=True)
1368+
}
1369+
return DataFile.from_args(_table_format_version=3, **args)
1370+
1371+
def prepare_entry(self, entry: ManifestEntry) -> ManifestEntry:
1372+
entry = super().prepare_entry(entry)
1373+
if len(entry.data_file._data) < len(DATA_FILE_TYPE[3].fields):
1374+
entry = ManifestEntry.from_args(
1375+
_table_format_version=3,
1376+
status=entry.status,
1377+
snapshot_id=entry.snapshot_id,
1378+
sequence_number=entry.sequence_number,
1379+
file_sequence_number=entry.file_sequence_number,
1380+
data_file=self._wrap_data_file(entry.data_file),
1381+
)
1382+
return entry
1383+
1384+
13331385
def write_manifest(
13341386
format_version: TableVersion,
13351387
spec: PartitionSpec,
@@ -1342,6 +1394,8 @@ def write_manifest(
13421394
return ManifestWriterV1(spec, schema, output_file, snapshot_id, avro_compression)
13431395
elif format_version == 2:
13441396
return ManifestWriterV2(spec, schema, output_file, snapshot_id, avro_compression)
1397+
elif format_version == 3:
1398+
return ManifestWriterV3(spec, schema, output_file, snapshot_id, avro_compression)
13451399
else:
13461400
raise ValueError(f"Cannot write manifest for table version: {format_version}")
13471401

@@ -1465,19 +1519,91 @@ def prepare_manifest(self, manifest_file: ManifestFile) -> ManifestFile:
14651519
return wrapped_manifest_file
14661520

14671521

1522+
class ManifestListWriterV3(ManifestListWriterV2):
1523+
"""Writes V3 manifest lists, assigning `first_row_id` to data manifests.
1524+
1525+
Follows the spec's First Row ID Assignment: existing `first_row_id` values are
1526+
preserved, delete manifests are never assigned one, and data manifests without a
1527+
`first_row_id` are assigned a running value starting at the snapshot's
1528+
`first-row-id`, advanced by each assigned manifest's existing and added row counts.
1529+
"""
1530+
1531+
_next_row_id: int
1532+
1533+
def __init__(
1534+
self,
1535+
output_file: OutputFile,
1536+
snapshot_id: int,
1537+
parent_snapshot_id: int | None,
1538+
sequence_number: int,
1539+
compression: AvroCompressionCodec,
1540+
first_row_id: int,
1541+
):
1542+
super().__init__(output_file, snapshot_id, parent_snapshot_id, sequence_number, compression)
1543+
self._format_version = 3
1544+
self._meta = {
1545+
**self._meta,
1546+
"first-row-id": str(first_row_id),
1547+
"format-version": "3",
1548+
}
1549+
self._next_row_id = first_row_id
1550+
1551+
def __enter__(self) -> ManifestListWriter:
1552+
"""Open the writer for writing, using the V3 record layout so `first_row_id` is written."""
1553+
self._writer = AvroOutputFile[ManifestFile](
1554+
output_file=self._output_file,
1555+
record_schema=MANIFEST_LIST_FILE_SCHEMAS[3],
1556+
file_schema=MANIFEST_LIST_FILE_SCHEMAS[3],
1557+
schema_name="manifest_file",
1558+
metadata=self._meta,
1559+
)
1560+
self._writer.__enter__()
1561+
return self
1562+
1563+
@property
1564+
def next_row_id(self) -> int:
1565+
"""The row ID after the last assigned one; the table's `next-row-id` after this snapshot."""
1566+
return self._next_row_id
1567+
1568+
def _wrap(self, manifest_file: ManifestFile) -> ManifestFile:
1569+
"""Rebind a manifest file to the V3 record layout."""
1570+
if len(manifest_file._data) >= len(MANIFEST_LIST_FILE_SCHEMAS[3].fields):
1571+
return copy(manifest_file)
1572+
args = {
1573+
field.name: value
1574+
for field, value in zip(MANIFEST_LIST_FILE_SCHEMAS[DEFAULT_READ_VERSION].fields, manifest_file._data, strict=True)
1575+
}
1576+
return ManifestFile.from_args(_table_format_version=3, **args)
1577+
1578+
def prepare_manifest(self, manifest_file: ManifestFile) -> ManifestFile:
1579+
wrapped_manifest_file = super().prepare_manifest(self._wrap(manifest_file))
1580+
1581+
if wrapped_manifest_file.content == ManifestContent.DATA and wrapped_manifest_file.first_row_id is None:
1582+
wrapped_manifest_file.first_row_id = self._next_row_id
1583+
self._next_row_id += (wrapped_manifest_file.existing_rows_count or 0) + (wrapped_manifest_file.added_rows_count or 0)
1584+
return wrapped_manifest_file
1585+
1586+
14681587
def write_manifest_list(
14691588
format_version: TableVersion,
14701589
output_file: OutputFile,
14711590
snapshot_id: int,
14721591
parent_snapshot_id: int | None,
14731592
sequence_number: int | None,
14741593
avro_compression: AvroCompressionCodec,
1594+
first_row_id: int | None = None,
14751595
) -> ManifestListWriter:
14761596
if format_version == 1:
14771597
return ManifestListWriterV1(output_file, snapshot_id, parent_snapshot_id, avro_compression)
14781598
elif format_version == 2:
14791599
if sequence_number is None:
14801600
raise ValueError(f"Sequence-number is required for V2 tables: {sequence_number}")
14811601
return ManifestListWriterV2(output_file, snapshot_id, parent_snapshot_id, sequence_number, avro_compression)
1602+
elif format_version == 3:
1603+
if sequence_number is None:
1604+
raise ValueError(f"Sequence-number is required for V3 tables: {sequence_number}")
1605+
if first_row_id is None:
1606+
raise ValueError(f"First-row-id is required for V3 tables: {first_row_id}")
1607+
return ManifestListWriterV3(output_file, snapshot_id, parent_snapshot_id, sequence_number, avro_compression, first_row_id)
14821608
else:
14831609
raise ValueError(f"Cannot write manifest list for table version: {format_version}")

‎tests/utils/test_manifest.py‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1477,3 +1477,132 @@ def test_negative_manifest_cache_size_raises_value_error(monkeypatch: pytest.Mon
14771477
finally:
14781478
monkeypatch.delenv("PYICEBERG_MANIFEST_CACHE_SIZE", raising=False)
14791479
importlib.reload(manifest_module)
1480+
1481+
1482+
@pytest.mark.parametrize("compression", ["null", "deflate"])
1483+
def test_write_manifest_v3(compression: AvroCompressionCodec) -> None:
1484+
io = load_file_io()
1485+
test_schema = Schema(NestedField(1, "foo", IntegerType(), False))
1486+
1487+
v3_data_file = DataFile.from_args(
1488+
_table_format_version=3,
1489+
content=DataFileContent.DATA,
1490+
file_path="/data/file-v3.parquet",
1491+
file_format=FileFormat.PARQUET,
1492+
partition=Record(),
1493+
record_count=100,
1494+
file_size_in_bytes=1024,
1495+
first_row_id=1000,
1496+
)
1497+
v2_data_file = DataFile.from_args(
1498+
content=DataFileContent.DATA,
1499+
file_path="/data/file-v2.parquet",
1500+
file_format=FileFormat.PARQUET,
1501+
partition=Record(),
1502+
record_count=50,
1503+
file_size_in_bytes=512,
1504+
)
1505+
1506+
with TemporaryDirectory() as tmp_dir:
1507+
path = tmp_dir + "/manifest-v3.avro"
1508+
with write_manifest(
1509+
format_version=3,
1510+
spec=UNPARTITIONED_PARTITION_SPEC,
1511+
schema=test_schema,
1512+
output_file=io.new_output(path),
1513+
snapshot_id=25,
1514+
avro_compression=compression,
1515+
) as writer:
1516+
writer.add(ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=25, data_file=v3_data_file))
1517+
# a data file bound to the default (V2) layout is rebound to the V3 layout
1518+
writer.add(ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=25, data_file=v2_data_file))
1519+
1520+
_verify_metadata_with_fastavro(path, {"format-version": "3", "content": "data"})
1521+
1522+
with open(path, "rb") as f:
1523+
entries = list(fastavro.reader(f))
1524+
1525+
assert len(entries) == 2
1526+
assert entries[0]["data_file"]["first_row_id"] == 1000
1527+
assert entries[1]["data_file"]["first_row_id"] is None
1528+
for entry in entries:
1529+
for v3_field in ("first_row_id", "referenced_data_file", "content_offset", "content_size_in_bytes"):
1530+
assert v3_field in entry["data_file"]
1531+
1532+
1533+
@pytest.mark.parametrize("compression", ["null", "deflate"])
1534+
def test_write_manifest_list_v3_assigns_first_row_id(compression: AvroCompressionCodec) -> None:
1535+
io = load_file_io()
1536+
1537+
def manifest(path: str, content: ManifestContent, first_row_id: int | None = None, **counts: int) -> ManifestFile:
1538+
args: dict[str, Any] = {
1539+
"manifest_path": path,
1540+
"manifest_length": 100,
1541+
"partition_spec_id": 0,
1542+
"content": content,
1543+
"sequence_number": 1,
1544+
"min_sequence_number": 1,
1545+
"added_snapshot_id": 25,
1546+
"added_files_count": 1,
1547+
"existing_files_count": 1,
1548+
"deleted_files_count": 0,
1549+
"added_rows_count": counts.get("added", 0),
1550+
"existing_rows_count": counts.get("existing", 0),
1551+
"deleted_rows_count": 0,
1552+
}
1553+
if first_row_id is not None:
1554+
return ManifestFile.from_args(_table_format_version=3, first_row_id=first_row_id, **args)
1555+
# bound to the default (V2) layout to exercise rebinding in the writer
1556+
return ManifestFile.from_args(**args)
1557+
1558+
unassigned_data = manifest("/m1.avro", ManifestContent.DATA, added=100, existing=25)
1559+
preserved_data = manifest("/m2.avro", ManifestContent.DATA, first_row_id=77, added=10)
1560+
deletes = manifest("/m3.avro", ManifestContent.DELETES, added=10)
1561+
second_unassigned_data = manifest("/m4.avro", ManifestContent.DATA, added=5)
1562+
1563+
with TemporaryDirectory() as tmp_dir:
1564+
path = tmp_dir + "/manifest-list-v3.avro"
1565+
with write_manifest_list(
1566+
format_version=3,
1567+
output_file=io.new_output(path),
1568+
snapshot_id=25,
1569+
parent_snapshot_id=19,
1570+
sequence_number=2,
1571+
avro_compression=compression,
1572+
first_row_id=1000,
1573+
) as writer:
1574+
writer.add_manifests([unassigned_data, preserved_data, deletes, second_unassigned_data])
1575+
1576+
# 1000 + (100 + 25) + (5): assigned manifests advance by added + existing rows
1577+
assert writer.next_row_id == 1130 # type: ignore[attr-defined]
1578+
1579+
_verify_metadata_with_fastavro(
1580+
path,
1581+
{
1582+
"snapshot-id": "25",
1583+
"parent-snapshot-id": "19",
1584+
"sequence-number": "2",
1585+
"first-row-id": "1000",
1586+
"format-version": "3",
1587+
},
1588+
)
1589+
1590+
with open(path, "rb") as f:
1591+
records = list(fastavro.reader(f))
1592+
1593+
assert [r["first_row_id"] for r in records] == [1000, 77, None, 1125]
1594+
1595+
1596+
def test_write_manifest_list_v3_requires_first_row_id() -> None:
1597+
io = load_file_io()
1598+
with TemporaryDirectory() as tmp_dir:
1599+
with pytest.raises(ValueError, match="First-row-id is required for V3 tables"):
1600+
write_manifest_list(
1601+
format_version=3,
1602+
output_file=io.new_output(tmp_dir + "/manifest-list.avro"),
1603+
snapshot_id=25,
1604+
parent_snapshot_id=19,
1605+
sequence_number=2,
1606+
avro_compression="null",
1607+
first_row_id=None,
1608+
)

0 commit comments

Comments
 (0)