Skip to content

Commit 3af5d72

Browse files
committed
Support writing V3 manifests and manifest lists
Add ManifestWriterV3 and ManifestListWriterV3. The manifest writer uses the V3 record layout so the V3-only data file fields (first_row_id, referenced_data_file, content_offset, content_size_in_bytes) are written, rebinding V2-layout records when needed. The manifest list writer implements the spec's First Row ID Assignment: preserving existing first_row_id values, never assigning one to delete manifests, and assigning a running value advanced by existing and added row counts to unassigned data manifests. It also exposes next_row_id for the commit path to update the table's next-row-id. Closes apache#3620
1 parent 03ad0d6 commit 3af5d72

2 files changed

Lines changed: 259 additions & 0 deletions

File tree

‎pyiceberg/manifest.py‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -853,6 +853,17 @@ def partitions(self) -> list[PartitionFieldSummary] | None:
853853
def key_metadata(self) -> bytes | None:
854854
return self._data[14]
855855

856+
@property
857+
def first_row_id(self) -> int | None:
858+
# Only present when the record is bound to the V3 manifest list schema
859+
return self._data[15] if len(self._data) > 15 else None
860+
861+
@first_row_id.setter
862+
def first_row_id(self, value: int | None) -> None:
863+
if len(self._data) <= 15:
864+
raise ValueError("Cannot set first_row_id on a manifest file not bound to the V3 schema")
865+
self._data[15] = value
866+
856867
def has_added_files(self) -> bool:
857868
return self.added_files_count is None or self.added_files_count > 0
858869

@@ -1286,6 +1297,51 @@ def prepare_entry(self, entry: ManifestEntry) -> ManifestEntry:
12861297
return entry
12871298

12881299

1300+
class ManifestWriterV3(ManifestWriterV2):
1301+
"""Writes V3 manifest files.
1302+
1303+
The writer inherits the V2 sequence-number semantics; the V3 manifest entry
1304+
schema additionally carries `first_row_id`, `referenced_data_file`,
1305+
`content_offset` and `content_size_in_bytes` on the data file struct.
1306+
"""
1307+
1308+
@property
1309+
def version(self) -> TableVersion:
1310+
return 3
1311+
1312+
def new_writer(self) -> AvroOutputFile[ManifestEntry]:
1313+
# Use the V3 record layout so the V3-only data file fields are written
1314+
return AvroOutputFile[ManifestEntry](
1315+
output_file=self._output_file,
1316+
file_schema=self._with_partition(3),
1317+
record_schema=self._with_partition(3),
1318+
schema_name="manifest_entry",
1319+
metadata=self._meta,
1320+
)
1321+
1322+
def _wrap_data_file(self, data_file: DataFile) -> DataFile:
1323+
"""Rebind a data file to the V3 record layout."""
1324+
if len(data_file._data) >= len(DATA_FILE_TYPE[3].fields):
1325+
return data_file
1326+
args = {
1327+
field.name: value for field, value in zip(DATA_FILE_TYPE[DEFAULT_READ_VERSION].fields, data_file._data, strict=True)
1328+
}
1329+
return DataFile.from_args(_table_format_version=3, **args)
1330+
1331+
def prepare_entry(self, entry: ManifestEntry) -> ManifestEntry:
1332+
entry = super().prepare_entry(entry)
1333+
if len(entry.data_file._data) < len(DATA_FILE_TYPE[3].fields):
1334+
entry = ManifestEntry.from_args(
1335+
_table_format_version=3,
1336+
status=entry.status,
1337+
snapshot_id=entry.snapshot_id,
1338+
sequence_number=entry.sequence_number,
1339+
file_sequence_number=entry.file_sequence_number,
1340+
data_file=self._wrap_data_file(entry.data_file),
1341+
)
1342+
return entry
1343+
1344+
12891345
def write_manifest(
12901346
format_version: TableVersion,
12911347
spec: PartitionSpec,
@@ -1298,6 +1354,8 @@ def write_manifest(
12981354
return ManifestWriterV1(spec, schema, output_file, snapshot_id, avro_compression)
12991355
elif format_version == 2:
13001356
return ManifestWriterV2(spec, schema, output_file, snapshot_id, avro_compression)
1357+
elif format_version == 3:
1358+
return ManifestWriterV3(spec, schema, output_file, snapshot_id, avro_compression)
13011359
else:
13021360
raise ValueError(f"Cannot write manifest for table version: {format_version}")
13031361

@@ -1421,19 +1479,91 @@ def prepare_manifest(self, manifest_file: ManifestFile) -> ManifestFile:
14211479
return wrapped_manifest_file
14221480

14231481

1482+
class ManifestListWriterV3(ManifestListWriterV2):
1483+
"""Writes V3 manifest lists, assigning `first_row_id` to data manifests.
1484+
1485+
Follows the spec's First Row ID Assignment: existing `first_row_id` values are
1486+
preserved, delete manifests are never assigned one, and data manifests without a
1487+
`first_row_id` are assigned a running value starting at the snapshot's
1488+
`first-row-id`, advanced by each assigned manifest's existing and added row counts.
1489+
"""
1490+
1491+
_next_row_id: int
1492+
1493+
def __init__(
1494+
self,
1495+
output_file: OutputFile,
1496+
snapshot_id: int,
1497+
parent_snapshot_id: int | None,
1498+
sequence_number: int,
1499+
compression: AvroCompressionCodec,
1500+
first_row_id: int,
1501+
):
1502+
super().__init__(output_file, snapshot_id, parent_snapshot_id, sequence_number, compression)
1503+
self._format_version = 3
1504+
self._meta = {
1505+
**self._meta,
1506+
"first-row-id": str(first_row_id),
1507+
"format-version": "3",
1508+
}
1509+
self._next_row_id = first_row_id
1510+
1511+
def __enter__(self) -> ManifestListWriter:
1512+
"""Open the writer for writing, using the V3 record layout so `first_row_id` is written."""
1513+
self._writer = AvroOutputFile[ManifestFile](
1514+
output_file=self._output_file,
1515+
record_schema=MANIFEST_LIST_FILE_SCHEMAS[3],
1516+
file_schema=MANIFEST_LIST_FILE_SCHEMAS[3],
1517+
schema_name="manifest_file",
1518+
metadata=self._meta,
1519+
)
1520+
self._writer.__enter__()
1521+
return self
1522+
1523+
@property
1524+
def next_row_id(self) -> int:
1525+
"""The row ID after the last assigned one; the table's `next-row-id` after this snapshot."""
1526+
return self._next_row_id
1527+
1528+
def _wrap(self, manifest_file: ManifestFile) -> ManifestFile:
1529+
"""Rebind a manifest file to the V3 record layout."""
1530+
if len(manifest_file._data) >= len(MANIFEST_LIST_FILE_SCHEMAS[3].fields):
1531+
return copy(manifest_file)
1532+
args = {
1533+
field.name: value
1534+
for field, value in zip(MANIFEST_LIST_FILE_SCHEMAS[DEFAULT_READ_VERSION].fields, manifest_file._data, strict=True)
1535+
}
1536+
return ManifestFile.from_args(_table_format_version=3, **args)
1537+
1538+
def prepare_manifest(self, manifest_file: ManifestFile) -> ManifestFile:
1539+
wrapped_manifest_file = super().prepare_manifest(self._wrap(manifest_file))
1540+
1541+
if wrapped_manifest_file.content == ManifestContent.DATA and wrapped_manifest_file.first_row_id is None:
1542+
wrapped_manifest_file.first_row_id = self._next_row_id
1543+
self._next_row_id += (wrapped_manifest_file.existing_rows_count or 0) + (wrapped_manifest_file.added_rows_count or 0)
1544+
return wrapped_manifest_file
1545+
1546+
14241547
def write_manifest_list(
14251548
format_version: TableVersion,
14261549
output_file: OutputFile,
14271550
snapshot_id: int,
14281551
parent_snapshot_id: int | None,
14291552
sequence_number: int | None,
14301553
avro_compression: AvroCompressionCodec,
1554+
first_row_id: int | None = None,
14311555
) -> ManifestListWriter:
14321556
if format_version == 1:
14331557
return ManifestListWriterV1(output_file, snapshot_id, parent_snapshot_id, avro_compression)
14341558
elif format_version == 2:
14351559
if sequence_number is None:
14361560
raise ValueError(f"Sequence-number is required for V2 tables: {sequence_number}")
14371561
return ManifestListWriterV2(output_file, snapshot_id, parent_snapshot_id, sequence_number, avro_compression)
1562+
elif format_version == 3:
1563+
if sequence_number is None:
1564+
raise ValueError(f"Sequence-number is required for V3 tables: {sequence_number}")
1565+
if first_row_id is None:
1566+
raise ValueError(f"First-row-id is required for V3 tables: {first_row_id}")
1567+
return ManifestListWriterV3(output_file, snapshot_id, parent_snapshot_id, sequence_number, avro_compression, first_row_id)
14381568
else:
14391569
raise ValueError(f"Cannot write manifest list for table version: {format_version}")

‎tests/utils/test_manifest.py‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1188,3 +1188,132 @@ def test_negative_manifest_cache_size_raises_value_error(monkeypatch: pytest.Mon
11881188
finally:
11891189
monkeypatch.delenv("PYICEBERG_MANIFEST_CACHE_SIZE", raising=False)
11901190
importlib.reload(manifest_module)
1191+
1192+
1193+
@pytest.mark.parametrize("compression", ["null", "deflate"])
1194+
def test_write_manifest_v3(compression: AvroCompressionCodec) -> None:
1195+
io = load_file_io()
1196+
test_schema = Schema(NestedField(1, "foo", IntegerType(), False))
1197+
1198+
v3_data_file = DataFile.from_args(
1199+
_table_format_version=3,
1200+
content=DataFileContent.DATA,
1201+
file_path="/data/file-v3.parquet",
1202+
file_format=FileFormat.PARQUET,
1203+
partition=Record(),
1204+
record_count=100,
1205+
file_size_in_bytes=1024,
1206+
first_row_id=1000,
1207+
)
1208+
v2_data_file = DataFile.from_args(
1209+
content=DataFileContent.DATA,
1210+
file_path="/data/file-v2.parquet",
1211+
file_format=FileFormat.PARQUET,
1212+
partition=Record(),
1213+
record_count=50,
1214+
file_size_in_bytes=512,
1215+
)
1216+
1217+
with TemporaryDirectory() as tmp_dir:
1218+
path = tmp_dir + "/manifest-v3.avro"
1219+
with write_manifest(
1220+
format_version=3,
1221+
spec=UNPARTITIONED_PARTITION_SPEC,
1222+
schema=test_schema,
1223+
output_file=io.new_output(path),
1224+
snapshot_id=25,
1225+
avro_compression=compression,
1226+
) as writer:
1227+
writer.add(ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=25, data_file=v3_data_file))
1228+
# a data file bound to the default (V2) layout is rebound to the V3 layout
1229+
writer.add(ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=25, data_file=v2_data_file))
1230+
1231+
_verify_metadata_with_fastavro(path, {"format-version": "3", "content": "data"})
1232+
1233+
with open(path, "rb") as f:
1234+
entries = list(fastavro.reader(f))
1235+
1236+
assert len(entries) == 2
1237+
assert entries[0]["data_file"]["first_row_id"] == 1000
1238+
assert entries[1]["data_file"]["first_row_id"] is None
1239+
for entry in entries:
1240+
for v3_field in ("first_row_id", "referenced_data_file", "content_offset", "content_size_in_bytes"):
1241+
assert v3_field in entry["data_file"]
1242+
1243+
1244+
@pytest.mark.parametrize("compression", ["null", "deflate"])
1245+
def test_write_manifest_list_v3_assigns_first_row_id(compression: AvroCompressionCodec) -> None:
1246+
io = load_file_io()
1247+
1248+
def manifest(path: str, content: ManifestContent, first_row_id: int | None = None, **counts: int) -> ManifestFile:
1249+
args: dict[str, Any] = {
1250+
"manifest_path": path,
1251+
"manifest_length": 100,
1252+
"partition_spec_id": 0,
1253+
"content": content,
1254+
"sequence_number": 1,
1255+
"min_sequence_number": 1,
1256+
"added_snapshot_id": 25,
1257+
"added_files_count": 1,
1258+
"existing_files_count": 1,
1259+
"deleted_files_count": 0,
1260+
"added_rows_count": counts.get("added", 0),
1261+
"existing_rows_count": counts.get("existing", 0),
1262+
"deleted_rows_count": 0,
1263+
}
1264+
if first_row_id is not None:
1265+
return ManifestFile.from_args(_table_format_version=3, first_row_id=first_row_id, **args)
1266+
# bound to the default (V2) layout to exercise rebinding in the writer
1267+
return ManifestFile.from_args(**args)
1268+
1269+
unassigned_data = manifest("/m1.avro", ManifestContent.DATA, added=100, existing=25)
1270+
preserved_data = manifest("/m2.avro", ManifestContent.DATA, first_row_id=77, added=10)
1271+
deletes = manifest("/m3.avro", ManifestContent.DELETES, added=10)
1272+
second_unassigned_data = manifest("/m4.avro", ManifestContent.DATA, added=5)
1273+
1274+
with TemporaryDirectory() as tmp_dir:
1275+
path = tmp_dir + "/manifest-list-v3.avro"
1276+
with write_manifest_list(
1277+
format_version=3,
1278+
output_file=io.new_output(path),
1279+
snapshot_id=25,
1280+
parent_snapshot_id=19,
1281+
sequence_number=2,
1282+
avro_compression=compression,
1283+
first_row_id=1000,
1284+
) as writer:
1285+
writer.add_manifests([unassigned_data, preserved_data, deletes, second_unassigned_data])
1286+
1287+
# 1000 + (100 + 25) + (5): assigned manifests advance by added + existing rows
1288+
assert writer.next_row_id == 1130 # type: ignore[attr-defined]
1289+
1290+
_verify_metadata_with_fastavro(
1291+
path,
1292+
{
1293+
"snapshot-id": "25",
1294+
"parent-snapshot-id": "19",
1295+
"sequence-number": "2",
1296+
"first-row-id": "1000",
1297+
"format-version": "3",
1298+
},
1299+
)
1300+
1301+
with open(path, "rb") as f:
1302+
records = list(fastavro.reader(f))
1303+
1304+
assert [r["first_row_id"] for r in records] == [1000, 77, None, 1125]
1305+
1306+
1307+
def test_write_manifest_list_v3_requires_first_row_id() -> None:
1308+
io = load_file_io()
1309+
with TemporaryDirectory() as tmp_dir:
1310+
with pytest.raises(ValueError, match="First-row-id is required for V3 tables"):
1311+
write_manifest_list(
1312+
format_version=3,
1313+
output_file=io.new_output(tmp_dir + "/manifest-list.avro"),
1314+
snapshot_id=25,
1315+
parent_snapshot_id=19,
1316+
sequence_number=2,
1317+
avro_compression="null",
1318+
first_row_id=None,
1319+
)

0 commit comments

Comments
 (0)