Skip to content

Commit c068b18

Browse files
committed
Support writing V3 manifests and manifest lists
Add ManifestWriterV3 and ManifestListWriterV3. The manifest writer uses the V3 record layout so the V3-only data file fields (first_row_id, referenced_data_file, content_offset, content_size_in_bytes) are written, rebinding V2-layout records when needed. The manifest list writer implements the spec's First Row ID Assignment: preserving existing first_row_id values, never assigning one to delete manifests, and assigning a running value advanced by existing and added row counts to unassigned data manifests. It also exposes next_row_id for the commit path to update the table's next-row-id. Closes apache#3620
1 parent 4e6033d commit c068b18

2 files changed

Lines changed: 256 additions & 1 deletion

File tree

‎pyiceberg/manifest.py‎

Lines changed: 127 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -871,7 +871,14 @@ def key_metadata(self) -> bytes | None:
871871

872872
@property
873873
def first_row_id(self) -> int | None:
874-
return self._data[15]
874+
# Only present when the record is bound to the V3 manifest list schema
875+
return self._data[15] if len(self._data) > 15 else None
876+
877+
@first_row_id.setter
878+
def first_row_id(self, value: int | None) -> None:
879+
if len(self._data) <= 15:
880+
raise ValueError("Cannot set first_row_id on a manifest file not bound to the V3 schema")
881+
self._data[15] = value
875882

876883
def has_added_files(self) -> bool:
877884
return self.added_files_count is None or self.added_files_count > 0
@@ -1318,6 +1325,51 @@ def prepare_entry(self, entry: ManifestEntry) -> ManifestEntry:
13181325
return entry
13191326

13201327

1328+
class ManifestWriterV3(ManifestWriterV2):
1329+
"""Writes V3 manifest files.
1330+
1331+
The writer inherits the V2 sequence-number semantics; the V3 manifest entry
1332+
schema additionally carries `first_row_id`, `referenced_data_file`,
1333+
`content_offset` and `content_size_in_bytes` on the data file struct.
1334+
"""
1335+
1336+
@property
1337+
def version(self) -> TableVersion:
1338+
return 3
1339+
1340+
def new_writer(self) -> AvroOutputFile[ManifestEntry]:
1341+
# Use the V3 record layout so the V3-only data file fields are written
1342+
return AvroOutputFile[ManifestEntry](
1343+
output_file=self._output_file,
1344+
file_schema=self._with_partition(3),
1345+
record_schema=self._with_partition(3),
1346+
schema_name="manifest_entry",
1347+
metadata=self._meta,
1348+
)
1349+
1350+
def _wrap_data_file(self, data_file: DataFile) -> DataFile:
1351+
"""Rebind a data file to the V3 record layout."""
1352+
if len(data_file._data) >= len(DATA_FILE_TYPE[3].fields):
1353+
return data_file
1354+
args = {
1355+
field.name: value for field, value in zip(DATA_FILE_TYPE[DEFAULT_READ_VERSION].fields, data_file._data, strict=True)
1356+
}
1357+
return DataFile.from_args(_table_format_version=3, **args)
1358+
1359+
def prepare_entry(self, entry: ManifestEntry) -> ManifestEntry:
1360+
entry = super().prepare_entry(entry)
1361+
if len(entry.data_file._data) < len(DATA_FILE_TYPE[3].fields):
1362+
entry = ManifestEntry.from_args(
1363+
_table_format_version=3,
1364+
status=entry.status,
1365+
snapshot_id=entry.snapshot_id,
1366+
sequence_number=entry.sequence_number,
1367+
file_sequence_number=entry.file_sequence_number,
1368+
data_file=self._wrap_data_file(entry.data_file),
1369+
)
1370+
return entry
1371+
1372+
13211373
def write_manifest(
13221374
format_version: TableVersion,
13231375
spec: PartitionSpec,
@@ -1330,6 +1382,8 @@ def write_manifest(
13301382
return ManifestWriterV1(spec, schema, output_file, snapshot_id, avro_compression)
13311383
elif format_version == 2:
13321384
return ManifestWriterV2(spec, schema, output_file, snapshot_id, avro_compression)
1385+
elif format_version == 3:
1386+
return ManifestWriterV3(spec, schema, output_file, snapshot_id, avro_compression)
13331387
else:
13341388
raise ValueError(f"Cannot write manifest for table version: {format_version}")
13351389

@@ -1453,19 +1507,91 @@ def prepare_manifest(self, manifest_file: ManifestFile) -> ManifestFile:
14531507
return wrapped_manifest_file
14541508

14551509

1510+
class ManifestListWriterV3(ManifestListWriterV2):
1511+
"""Writes V3 manifest lists, assigning `first_row_id` to data manifests.
1512+
1513+
Follows the spec's First Row ID Assignment: existing `first_row_id` values are
1514+
preserved, delete manifests are never assigned one, and data manifests without a
1515+
`first_row_id` are assigned a running value starting at the snapshot's
1516+
`first-row-id`, advanced by each assigned manifest's existing and added row counts.
1517+
"""
1518+
1519+
_next_row_id: int
1520+
1521+
def __init__(
1522+
self,
1523+
output_file: OutputFile,
1524+
snapshot_id: int,
1525+
parent_snapshot_id: int | None,
1526+
sequence_number: int,
1527+
compression: AvroCompressionCodec,
1528+
first_row_id: int,
1529+
):
1530+
super().__init__(output_file, snapshot_id, parent_snapshot_id, sequence_number, compression)
1531+
self._format_version = 3
1532+
self._meta = {
1533+
**self._meta,
1534+
"first-row-id": str(first_row_id),
1535+
"format-version": "3",
1536+
}
1537+
self._next_row_id = first_row_id
1538+
1539+
def __enter__(self) -> ManifestListWriter:
1540+
"""Open the writer for writing, using the V3 record layout so `first_row_id` is written."""
1541+
self._writer = AvroOutputFile[ManifestFile](
1542+
output_file=self._output_file,
1543+
record_schema=MANIFEST_LIST_FILE_SCHEMAS[3],
1544+
file_schema=MANIFEST_LIST_FILE_SCHEMAS[3],
1545+
schema_name="manifest_file",
1546+
metadata=self._meta,
1547+
)
1548+
self._writer.__enter__()
1549+
return self
1550+
1551+
@property
1552+
def next_row_id(self) -> int:
1553+
"""The row ID after the last assigned one; the table's `next-row-id` after this snapshot."""
1554+
return self._next_row_id
1555+
1556+
def _wrap(self, manifest_file: ManifestFile) -> ManifestFile:
1557+
"""Rebind a manifest file to the V3 record layout."""
1558+
if len(manifest_file._data) >= len(MANIFEST_LIST_FILE_SCHEMAS[3].fields):
1559+
return copy(manifest_file)
1560+
args = {
1561+
field.name: value
1562+
for field, value in zip(MANIFEST_LIST_FILE_SCHEMAS[DEFAULT_READ_VERSION].fields, manifest_file._data, strict=True)
1563+
}
1564+
return ManifestFile.from_args(_table_format_version=3, **args)
1565+
1566+
def prepare_manifest(self, manifest_file: ManifestFile) -> ManifestFile:
1567+
wrapped_manifest_file = super().prepare_manifest(self._wrap(manifest_file))
1568+
1569+
if wrapped_manifest_file.content == ManifestContent.DATA and wrapped_manifest_file.first_row_id is None:
1570+
wrapped_manifest_file.first_row_id = self._next_row_id
1571+
self._next_row_id += (wrapped_manifest_file.existing_rows_count or 0) + (wrapped_manifest_file.added_rows_count or 0)
1572+
return wrapped_manifest_file
1573+
1574+
14561575
def write_manifest_list(
14571576
format_version: TableVersion,
14581577
output_file: OutputFile,
14591578
snapshot_id: int,
14601579
parent_snapshot_id: int | None,
14611580
sequence_number: int | None,
14621581
avro_compression: AvroCompressionCodec,
1582+
first_row_id: int | None = None,
14631583
) -> ManifestListWriter:
14641584
if format_version == 1:
14651585
return ManifestListWriterV1(output_file, snapshot_id, parent_snapshot_id, avro_compression)
14661586
elif format_version == 2:
14671587
if sequence_number is None:
14681588
raise ValueError(f"Sequence-number is required for V2 tables: {sequence_number}")
14691589
return ManifestListWriterV2(output_file, snapshot_id, parent_snapshot_id, sequence_number, avro_compression)
1590+
elif format_version == 3:
1591+
if sequence_number is None:
1592+
raise ValueError(f"Sequence-number is required for V3 tables: {sequence_number}")
1593+
if first_row_id is None:
1594+
raise ValueError(f"First-row-id is required for V3 tables: {first_row_id}")
1595+
return ManifestListWriterV3(output_file, snapshot_id, parent_snapshot_id, sequence_number, avro_compression, first_row_id)
14701596
else:
14711597
raise ValueError(f"Cannot write manifest list for table version: {format_version}")

‎tests/utils/test_manifest.py‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1324,3 +1324,132 @@ def test_negative_manifest_cache_size_raises_value_error(monkeypatch: pytest.Mon
13241324
finally:
13251325
monkeypatch.delenv("PYICEBERG_MANIFEST_CACHE_SIZE", raising=False)
13261326
importlib.reload(manifest_module)
1327+
1328+
1329+
@pytest.mark.parametrize("compression", ["null", "deflate"])
1330+
def test_write_manifest_v3(compression: AvroCompressionCodec) -> None:
1331+
io = load_file_io()
1332+
test_schema = Schema(NestedField(1, "foo", IntegerType(), False))
1333+
1334+
v3_data_file = DataFile.from_args(
1335+
_table_format_version=3,
1336+
content=DataFileContent.DATA,
1337+
file_path="/data/file-v3.parquet",
1338+
file_format=FileFormat.PARQUET,
1339+
partition=Record(),
1340+
record_count=100,
1341+
file_size_in_bytes=1024,
1342+
first_row_id=1000,
1343+
)
1344+
v2_data_file = DataFile.from_args(
1345+
content=DataFileContent.DATA,
1346+
file_path="/data/file-v2.parquet",
1347+
file_format=FileFormat.PARQUET,
1348+
partition=Record(),
1349+
record_count=50,
1350+
file_size_in_bytes=512,
1351+
)
1352+
1353+
with TemporaryDirectory() as tmp_dir:
1354+
path = tmp_dir + "/manifest-v3.avro"
1355+
with write_manifest(
1356+
format_version=3,
1357+
spec=UNPARTITIONED_PARTITION_SPEC,
1358+
schema=test_schema,
1359+
output_file=io.new_output(path),
1360+
snapshot_id=25,
1361+
avro_compression=compression,
1362+
) as writer:
1363+
writer.add(ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=25, data_file=v3_data_file))
1364+
# a data file bound to the default (V2) layout is rebound to the V3 layout
1365+
writer.add(ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=25, data_file=v2_data_file))
1366+
1367+
_verify_metadata_with_fastavro(path, {"format-version": "3", "content": "data"})
1368+
1369+
with open(path, "rb") as f:
1370+
entries = list(fastavro.reader(f))
1371+
1372+
assert len(entries) == 2
1373+
assert entries[0]["data_file"]["first_row_id"] == 1000
1374+
assert entries[1]["data_file"]["first_row_id"] is None
1375+
for entry in entries:
1376+
for v3_field in ("first_row_id", "referenced_data_file", "content_offset", "content_size_in_bytes"):
1377+
assert v3_field in entry["data_file"]
1378+
1379+
1380+
@pytest.mark.parametrize("compression", ["null", "deflate"])
1381+
def test_write_manifest_list_v3_assigns_first_row_id(compression: AvroCompressionCodec) -> None:
1382+
io = load_file_io()
1383+
1384+
def manifest(path: str, content: ManifestContent, first_row_id: int | None = None, **counts: int) -> ManifestFile:
1385+
args: dict[str, Any] = {
1386+
"manifest_path": path,
1387+
"manifest_length": 100,
1388+
"partition_spec_id": 0,
1389+
"content": content,
1390+
"sequence_number": 1,
1391+
"min_sequence_number": 1,
1392+
"added_snapshot_id": 25,
1393+
"added_files_count": 1,
1394+
"existing_files_count": 1,
1395+
"deleted_files_count": 0,
1396+
"added_rows_count": counts.get("added", 0),
1397+
"existing_rows_count": counts.get("existing", 0),
1398+
"deleted_rows_count": 0,
1399+
}
1400+
if first_row_id is not None:
1401+
return ManifestFile.from_args(_table_format_version=3, first_row_id=first_row_id, **args)
1402+
# bound to the default (V2) layout to exercise rebinding in the writer
1403+
return ManifestFile.from_args(**args)
1404+
1405+
unassigned_data = manifest("/m1.avro", ManifestContent.DATA, added=100, existing=25)
1406+
preserved_data = manifest("/m2.avro", ManifestContent.DATA, first_row_id=77, added=10)
1407+
deletes = manifest("/m3.avro", ManifestContent.DELETES, added=10)
1408+
second_unassigned_data = manifest("/m4.avro", ManifestContent.DATA, added=5)
1409+
1410+
with TemporaryDirectory() as tmp_dir:
1411+
path = tmp_dir + "/manifest-list-v3.avro"
1412+
with write_manifest_list(
1413+
format_version=3,
1414+
output_file=io.new_output(path),
1415+
snapshot_id=25,
1416+
parent_snapshot_id=19,
1417+
sequence_number=2,
1418+
avro_compression=compression,
1419+
first_row_id=1000,
1420+
) as writer:
1421+
writer.add_manifests([unassigned_data, preserved_data, deletes, second_unassigned_data])
1422+
1423+
# 1000 + (100 + 25) + (5): assigned manifests advance by added + existing rows
1424+
assert writer.next_row_id == 1130 # type: ignore[attr-defined]
1425+
1426+
_verify_metadata_with_fastavro(
1427+
path,
1428+
{
1429+
"snapshot-id": "25",
1430+
"parent-snapshot-id": "19",
1431+
"sequence-number": "2",
1432+
"first-row-id": "1000",
1433+
"format-version": "3",
1434+
},
1435+
)
1436+
1437+
with open(path, "rb") as f:
1438+
records = list(fastavro.reader(f))
1439+
1440+
assert [r["first_row_id"] for r in records] == [1000, 77, None, 1125]
1441+
1442+
1443+
def test_write_manifest_list_v3_requires_first_row_id() -> None:
1444+
io = load_file_io()
1445+
with TemporaryDirectory() as tmp_dir:
1446+
with pytest.raises(ValueError, match="First-row-id is required for V3 tables"):
1447+
write_manifest_list(
1448+
format_version=3,
1449+
output_file=io.new_output(tmp_dir + "/manifest-list.avro"),
1450+
snapshot_id=25,
1451+
parent_snapshot_id=19,
1452+
sequence_number=2,
1453+
avro_compression="null",
1454+
first_row_id=None,
1455+
)

0 commit comments

Comments
 (0)