@@ -853,6 +853,17 @@ def partitions(self) -> list[PartitionFieldSummary] | None:
853853 def key_metadata (self ) -> bytes | None :
854854 return self ._data [14 ]
855855
856+ @property
857+ def first_row_id (self ) -> int | None :
858+ # Only present when the record is bound to the V3 manifest list schema
859+ return self ._data [15 ] if len (self ._data ) > 15 else None
860+
861+ @first_row_id .setter
862+ def first_row_id (self , value : int | None ) -> None :
863+ if len (self ._data ) <= 15 :
864+ raise ValueError ("Cannot set first_row_id on a manifest file not bound to the V3 schema" )
865+ self ._data [15 ] = value
866+
856867 def has_added_files (self ) -> bool :
857868 return self .added_files_count is None or self .added_files_count > 0
858869
@@ -1286,6 +1297,51 @@ def prepare_entry(self, entry: ManifestEntry) -> ManifestEntry:
12861297 return entry
12871298
12881299
1300+ class ManifestWriterV3 (ManifestWriterV2 ):
1301+ """Writes V3 manifest files.
1302+
1303+ The writer inherits the V2 sequence-number semantics; the V3 manifest entry
1304+ schema additionally carries `first_row_id`, `referenced_data_file`,
1305+ `content_offset` and `content_size_in_bytes` on the data file struct.
1306+ """
1307+
1308+ @property
1309+ def version (self ) -> TableVersion :
1310+ return 3
1311+
1312+ def new_writer (self ) -> AvroOutputFile [ManifestEntry ]:
1313+ # Use the V3 record layout so the V3-only data file fields are written
1314+ return AvroOutputFile [ManifestEntry ](
1315+ output_file = self ._output_file ,
1316+ file_schema = self ._with_partition (3 ),
1317+ record_schema = self ._with_partition (3 ),
1318+ schema_name = "manifest_entry" ,
1319+ metadata = self ._meta ,
1320+ )
1321+
1322+ def _wrap_data_file (self , data_file : DataFile ) -> DataFile :
1323+ """Rebind a data file to the V3 record layout."""
1324+ if len (data_file ._data ) >= len (DATA_FILE_TYPE [3 ].fields ):
1325+ return data_file
1326+ args = {
1327+ field .name : value for field , value in zip (DATA_FILE_TYPE [DEFAULT_READ_VERSION ].fields , data_file ._data , strict = True )
1328+ }
1329+ return DataFile .from_args (_table_format_version = 3 , ** args )
1330+
1331+ def prepare_entry (self , entry : ManifestEntry ) -> ManifestEntry :
1332+ entry = super ().prepare_entry (entry )
1333+ if len (entry .data_file ._data ) < len (DATA_FILE_TYPE [3 ].fields ):
1334+ entry = ManifestEntry .from_args (
1335+ _table_format_version = 3 ,
1336+ status = entry .status ,
1337+ snapshot_id = entry .snapshot_id ,
1338+ sequence_number = entry .sequence_number ,
1339+ file_sequence_number = entry .file_sequence_number ,
1340+ data_file = self ._wrap_data_file (entry .data_file ),
1341+ )
1342+ return entry
1343+
1344+
12891345def write_manifest (
12901346 format_version : TableVersion ,
12911347 spec : PartitionSpec ,
@@ -1298,6 +1354,8 @@ def write_manifest(
12981354 return ManifestWriterV1 (spec , schema , output_file , snapshot_id , avro_compression )
12991355 elif format_version == 2 :
13001356 return ManifestWriterV2 (spec , schema , output_file , snapshot_id , avro_compression )
1357+ elif format_version == 3 :
1358+ return ManifestWriterV3 (spec , schema , output_file , snapshot_id , avro_compression )
13011359 else :
13021360 raise ValueError (f"Cannot write manifest for table version: { format_version } " )
13031361
@@ -1421,19 +1479,91 @@ def prepare_manifest(self, manifest_file: ManifestFile) -> ManifestFile:
14211479 return wrapped_manifest_file
14221480
14231481
1482+ class ManifestListWriterV3 (ManifestListWriterV2 ):
1483+ """Writes V3 manifest lists, assigning `first_row_id` to data manifests.
1484+
1485+ Follows the spec's First Row ID Assignment: existing `first_row_id` values are
1486+ preserved, delete manifests are never assigned one, and data manifests without a
1487+ `first_row_id` are assigned a running value starting at the snapshot's
1488+ `first-row-id`, advanced by each assigned manifest's existing and added row counts.
1489+ """
1490+
1491+ _next_row_id : int
1492+
1493+ def __init__ (
1494+ self ,
1495+ output_file : OutputFile ,
1496+ snapshot_id : int ,
1497+ parent_snapshot_id : int | None ,
1498+ sequence_number : int ,
1499+ compression : AvroCompressionCodec ,
1500+ first_row_id : int ,
1501+ ):
1502+ super ().__init__ (output_file , snapshot_id , parent_snapshot_id , sequence_number , compression )
1503+ self ._format_version = 3
1504+ self ._meta = {
1505+ ** self ._meta ,
1506+ "first-row-id" : str (first_row_id ),
1507+ "format-version" : "3" ,
1508+ }
1509+ self ._next_row_id = first_row_id
1510+
1511+ def __enter__ (self ) -> ManifestListWriter :
1512+ """Open the writer for writing, using the V3 record layout so `first_row_id` is written."""
1513+ self ._writer = AvroOutputFile [ManifestFile ](
1514+ output_file = self ._output_file ,
1515+ record_schema = MANIFEST_LIST_FILE_SCHEMAS [3 ],
1516+ file_schema = MANIFEST_LIST_FILE_SCHEMAS [3 ],
1517+ schema_name = "manifest_file" ,
1518+ metadata = self ._meta ,
1519+ )
1520+ self ._writer .__enter__ ()
1521+ return self
1522+
1523+ @property
1524+ def next_row_id (self ) -> int :
1525+ """The row ID after the last assigned one; the table's `next-row-id` after this snapshot."""
1526+ return self ._next_row_id
1527+
1528+ def _wrap (self , manifest_file : ManifestFile ) -> ManifestFile :
1529+ """Rebind a manifest file to the V3 record layout."""
1530+ if len (manifest_file ._data ) >= len (MANIFEST_LIST_FILE_SCHEMAS [3 ].fields ):
1531+ return copy (manifest_file )
1532+ args = {
1533+ field .name : value
1534+ for field , value in zip (MANIFEST_LIST_FILE_SCHEMAS [DEFAULT_READ_VERSION ].fields , manifest_file ._data , strict = True )
1535+ }
1536+ return ManifestFile .from_args (_table_format_version = 3 , ** args )
1537+
1538+ def prepare_manifest (self , manifest_file : ManifestFile ) -> ManifestFile :
1539+ wrapped_manifest_file = super ().prepare_manifest (self ._wrap (manifest_file ))
1540+
1541+ if wrapped_manifest_file .content == ManifestContent .DATA and wrapped_manifest_file .first_row_id is None :
1542+ wrapped_manifest_file .first_row_id = self ._next_row_id
1543+ self ._next_row_id += (wrapped_manifest_file .existing_rows_count or 0 ) + (wrapped_manifest_file .added_rows_count or 0 )
1544+ return wrapped_manifest_file
1545+
1546+
14241547def write_manifest_list (
14251548 format_version : TableVersion ,
14261549 output_file : OutputFile ,
14271550 snapshot_id : int ,
14281551 parent_snapshot_id : int | None ,
14291552 sequence_number : int | None ,
14301553 avro_compression : AvroCompressionCodec ,
1554+ first_row_id : int | None = None ,
14311555) -> ManifestListWriter :
14321556 if format_version == 1 :
14331557 return ManifestListWriterV1 (output_file , snapshot_id , parent_snapshot_id , avro_compression )
14341558 elif format_version == 2 :
14351559 if sequence_number is None :
14361560 raise ValueError (f"Sequence-number is required for V2 tables: { sequence_number } " )
14371561 return ManifestListWriterV2 (output_file , snapshot_id , parent_snapshot_id , sequence_number , avro_compression )
1562+ elif format_version == 3 :
1563+ if sequence_number is None :
1564+ raise ValueError (f"Sequence-number is required for V3 tables: { sequence_number } " )
1565+ if first_row_id is None :
1566+ raise ValueError (f"First-row-id is required for V3 tables: { first_row_id } " )
1567+ return ManifestListWriterV3 (output_file , snapshot_id , parent_snapshot_id , sequence_number , avro_compression , first_row_id )
14381568 else :
14391569 raise ValueError (f"Cannot write manifest list for table version: { format_version } " )
0 commit comments