@@ -883,7 +883,14 @@ def key_metadata(self) -> bytes | None:
883883
884884 @property
885885 def first_row_id (self ) -> int | None :
886- return self ._data [15 ]
886+ # Only present when the record is bound to the V3 manifest list schema
887+ return self ._data [15 ] if len (self ._data ) > 15 else None
888+
889+ @first_row_id .setter
890+ def first_row_id (self , value : int | None ) -> None :
891+ if len (self ._data ) <= 15 :
892+ raise ValueError ("Cannot set first_row_id on a manifest file not bound to the V3 schema" )
893+ self ._data [15 ] = value
887894
888895 def has_added_files (self ) -> bool :
889896 return self .added_files_count is None or self .added_files_count > 0
@@ -1330,6 +1337,51 @@ def prepare_entry(self, entry: ManifestEntry) -> ManifestEntry:
13301337 return entry
13311338
13321339
1340+ class ManifestWriterV3 (ManifestWriterV2 ):
1341+ """Writes V3 manifest files.
1342+
1343+ The writer inherits the V2 sequence-number semantics; the V3 manifest entry
1344+ schema additionally carries `first_row_id`, `referenced_data_file`,
1345+ `content_offset` and `content_size_in_bytes` on the data file struct.
1346+ """
1347+
1348+ @property
1349+ def version (self ) -> TableVersion :
1350+ return 3
1351+
1352+ def new_writer (self ) -> AvroOutputFile [ManifestEntry ]:
1353+ # Use the V3 record layout so the V3-only data file fields are written
1354+ return AvroOutputFile [ManifestEntry ](
1355+ output_file = self ._output_file ,
1356+ file_schema = self ._with_partition (3 ),
1357+ record_schema = self ._with_partition (3 ),
1358+ schema_name = "manifest_entry" ,
1359+ metadata = self ._meta ,
1360+ )
1361+
1362+ def _wrap_data_file (self , data_file : DataFile ) -> DataFile :
1363+ """Rebind a data file to the V3 record layout."""
1364+ if len (data_file ._data ) >= len (DATA_FILE_TYPE [3 ].fields ):
1365+ return data_file
1366+ args = {
1367+ field .name : value for field , value in zip (DATA_FILE_TYPE [DEFAULT_READ_VERSION ].fields , data_file ._data , strict = True )
1368+ }
1369+ return DataFile .from_args (_table_format_version = 3 , ** args )
1370+
1371+ def prepare_entry (self , entry : ManifestEntry ) -> ManifestEntry :
1372+ entry = super ().prepare_entry (entry )
1373+ if len (entry .data_file ._data ) < len (DATA_FILE_TYPE [3 ].fields ):
1374+ entry = ManifestEntry .from_args (
1375+ _table_format_version = 3 ,
1376+ status = entry .status ,
1377+ snapshot_id = entry .snapshot_id ,
1378+ sequence_number = entry .sequence_number ,
1379+ file_sequence_number = entry .file_sequence_number ,
1380+ data_file = self ._wrap_data_file (entry .data_file ),
1381+ )
1382+ return entry
1383+
1384+
13331385def write_manifest (
13341386 format_version : TableVersion ,
13351387 spec : PartitionSpec ,
@@ -1342,6 +1394,8 @@ def write_manifest(
13421394 return ManifestWriterV1 (spec , schema , output_file , snapshot_id , avro_compression )
13431395 elif format_version == 2 :
13441396 return ManifestWriterV2 (spec , schema , output_file , snapshot_id , avro_compression )
1397+ elif format_version == 3 :
1398+ return ManifestWriterV3 (spec , schema , output_file , snapshot_id , avro_compression )
13451399 else :
13461400 raise ValueError (f"Cannot write manifest for table version: { format_version } " )
13471401
@@ -1465,19 +1519,91 @@ def prepare_manifest(self, manifest_file: ManifestFile) -> ManifestFile:
14651519 return wrapped_manifest_file
14661520
14671521
1522+ class ManifestListWriterV3 (ManifestListWriterV2 ):
1523+ """Writes V3 manifest lists, assigning `first_row_id` to data manifests.
1524+
1525+ Follows the spec's First Row ID Assignment: existing `first_row_id` values are
1526+ preserved, delete manifests are never assigned one, and data manifests without a
1527+ `first_row_id` are assigned a running value starting at the snapshot's
1528+ `first-row-id`, advanced by each assigned manifest's existing and added row counts.
1529+ """
1530+
1531+ _next_row_id : int
1532+
1533+ def __init__ (
1534+ self ,
1535+ output_file : OutputFile ,
1536+ snapshot_id : int ,
1537+ parent_snapshot_id : int | None ,
1538+ sequence_number : int ,
1539+ compression : AvroCompressionCodec ,
1540+ first_row_id : int ,
1541+ ):
1542+ super ().__init__ (output_file , snapshot_id , parent_snapshot_id , sequence_number , compression )
1543+ self ._format_version = 3
1544+ self ._meta = {
1545+ ** self ._meta ,
1546+ "first-row-id" : str (first_row_id ),
1547+ "format-version" : "3" ,
1548+ }
1549+ self ._next_row_id = first_row_id
1550+
1551+ def __enter__ (self ) -> ManifestListWriter :
1552+ """Open the writer for writing, using the V3 record layout so `first_row_id` is written."""
1553+ self ._writer = AvroOutputFile [ManifestFile ](
1554+ output_file = self ._output_file ,
1555+ record_schema = MANIFEST_LIST_FILE_SCHEMAS [3 ],
1556+ file_schema = MANIFEST_LIST_FILE_SCHEMAS [3 ],
1557+ schema_name = "manifest_file" ,
1558+ metadata = self ._meta ,
1559+ )
1560+ self ._writer .__enter__ ()
1561+ return self
1562+
1563+ @property
1564+ def next_row_id (self ) -> int :
1565+ """The row ID after the last assigned one; the table's `next-row-id` after this snapshot."""
1566+ return self ._next_row_id
1567+
1568+ def _wrap (self , manifest_file : ManifestFile ) -> ManifestFile :
1569+ """Rebind a manifest file to the V3 record layout."""
1570+ if len (manifest_file ._data ) >= len (MANIFEST_LIST_FILE_SCHEMAS [3 ].fields ):
1571+ return copy (manifest_file )
1572+ args = {
1573+ field .name : value
1574+ for field , value in zip (MANIFEST_LIST_FILE_SCHEMAS [DEFAULT_READ_VERSION ].fields , manifest_file ._data , strict = True )
1575+ }
1576+ return ManifestFile .from_args (_table_format_version = 3 , ** args )
1577+
1578+ def prepare_manifest (self , manifest_file : ManifestFile ) -> ManifestFile :
1579+ wrapped_manifest_file = super ().prepare_manifest (self ._wrap (manifest_file ))
1580+
1581+ if wrapped_manifest_file .content == ManifestContent .DATA and wrapped_manifest_file .first_row_id is None :
1582+ wrapped_manifest_file .first_row_id = self ._next_row_id
1583+ self ._next_row_id += (wrapped_manifest_file .existing_rows_count or 0 ) + (wrapped_manifest_file .added_rows_count or 0 )
1584+ return wrapped_manifest_file
1585+
1586+
14681587def write_manifest_list (
14691588 format_version : TableVersion ,
14701589 output_file : OutputFile ,
14711590 snapshot_id : int ,
14721591 parent_snapshot_id : int | None ,
14731592 sequence_number : int | None ,
14741593 avro_compression : AvroCompressionCodec ,
1594+ first_row_id : int | None = None ,
14751595) -> ManifestListWriter :
14761596 if format_version == 1 :
14771597 return ManifestListWriterV1 (output_file , snapshot_id , parent_snapshot_id , avro_compression )
14781598 elif format_version == 2 :
14791599 if sequence_number is None :
14801600 raise ValueError (f"Sequence-number is required for V2 tables: { sequence_number } " )
14811601 return ManifestListWriterV2 (output_file , snapshot_id , parent_snapshot_id , sequence_number , avro_compression )
1602+ elif format_version == 3 :
1603+ if sequence_number is None :
1604+ raise ValueError (f"Sequence-number is required for V3 tables: { sequence_number } " )
1605+ if first_row_id is None :
1606+ raise ValueError (f"First-row-id is required for V3 tables: { first_row_id } " )
1607+ return ManifestListWriterV3 (output_file , snapshot_id , parent_snapshot_id , sequence_number , avro_compression , first_row_id )
14821608 else :
14831609 raise ValueError (f"Cannot write manifest list for table version: { format_version } " )
0 commit comments