@@ -871,7 +871,14 @@ def key_metadata(self) -> bytes | None:
871871
872872 @property
873873 def first_row_id (self ) -> int | None :
874- return self ._data [15 ]
874+ # Only present when the record is bound to the V3 manifest list schema
875+ return self ._data [15 ] if len (self ._data ) > 15 else None
876+
877+ @first_row_id .setter
878+ def first_row_id (self , value : int | None ) -> None :
879+ if len (self ._data ) <= 15 :
880+ raise ValueError ("Cannot set first_row_id on a manifest file not bound to the V3 schema" )
881+ self ._data [15 ] = value
875882
876883 def has_added_files (self ) -> bool :
877884 return self .added_files_count is None or self .added_files_count > 0
@@ -1318,6 +1325,51 @@ def prepare_entry(self, entry: ManifestEntry) -> ManifestEntry:
13181325 return entry
13191326
13201327
1328+ class ManifestWriterV3 (ManifestWriterV2 ):
1329+ """Writes V3 manifest files.
1330+
1331+ The writer inherits the V2 sequence-number semantics; the V3 manifest entry
1332+ schema additionally carries `first_row_id`, `referenced_data_file`,
1333+ `content_offset` and `content_size_in_bytes` on the data file struct.
1334+ """
1335+
1336+ @property
1337+ def version (self ) -> TableVersion :
1338+ return 3
1339+
1340+ def new_writer (self ) -> AvroOutputFile [ManifestEntry ]:
1341+ # Use the V3 record layout so the V3-only data file fields are written
1342+ return AvroOutputFile [ManifestEntry ](
1343+ output_file = self ._output_file ,
1344+ file_schema = self ._with_partition (3 ),
1345+ record_schema = self ._with_partition (3 ),
1346+ schema_name = "manifest_entry" ,
1347+ metadata = self ._meta ,
1348+ )
1349+
1350+ def _wrap_data_file (self , data_file : DataFile ) -> DataFile :
1351+ """Rebind a data file to the V3 record layout."""
1352+ if len (data_file ._data ) >= len (DATA_FILE_TYPE [3 ].fields ):
1353+ return data_file
1354+ args = {
1355+ field .name : value for field , value in zip (DATA_FILE_TYPE [DEFAULT_READ_VERSION ].fields , data_file ._data , strict = True )
1356+ }
1357+ return DataFile .from_args (_table_format_version = 3 , ** args )
1358+
1359+ def prepare_entry (self , entry : ManifestEntry ) -> ManifestEntry :
1360+ entry = super ().prepare_entry (entry )
1361+ if len (entry .data_file ._data ) < len (DATA_FILE_TYPE [3 ].fields ):
1362+ entry = ManifestEntry .from_args (
1363+ _table_format_version = 3 ,
1364+ status = entry .status ,
1365+ snapshot_id = entry .snapshot_id ,
1366+ sequence_number = entry .sequence_number ,
1367+ file_sequence_number = entry .file_sequence_number ,
1368+ data_file = self ._wrap_data_file (entry .data_file ),
1369+ )
1370+ return entry
1371+
1372+
13211373def write_manifest (
13221374 format_version : TableVersion ,
13231375 spec : PartitionSpec ,
@@ -1330,6 +1382,8 @@ def write_manifest(
13301382 return ManifestWriterV1 (spec , schema , output_file , snapshot_id , avro_compression )
13311383 elif format_version == 2 :
13321384 return ManifestWriterV2 (spec , schema , output_file , snapshot_id , avro_compression )
1385+ elif format_version == 3 :
1386+ return ManifestWriterV3 (spec , schema , output_file , snapshot_id , avro_compression )
13331387 else :
13341388 raise ValueError (f"Cannot write manifest for table version: { format_version } " )
13351389
@@ -1453,19 +1507,91 @@ def prepare_manifest(self, manifest_file: ManifestFile) -> ManifestFile:
14531507 return wrapped_manifest_file
14541508
14551509
1510+ class ManifestListWriterV3 (ManifestListWriterV2 ):
1511+ """Writes V3 manifest lists, assigning `first_row_id` to data manifests.
1512+
1513+ Follows the spec's First Row ID Assignment: existing `first_row_id` values are
1514+ preserved, delete manifests are never assigned one, and data manifests without a
1515+ `first_row_id` are assigned a running value starting at the snapshot's
1516+ `first-row-id`, advanced by each assigned manifest's existing and added row counts.
1517+ """
1518+
1519+ _next_row_id : int
1520+
1521+ def __init__ (
1522+ self ,
1523+ output_file : OutputFile ,
1524+ snapshot_id : int ,
1525+ parent_snapshot_id : int | None ,
1526+ sequence_number : int ,
1527+ compression : AvroCompressionCodec ,
1528+ first_row_id : int ,
1529+ ):
1530+ super ().__init__ (output_file , snapshot_id , parent_snapshot_id , sequence_number , compression )
1531+ self ._format_version = 3
1532+ self ._meta = {
1533+ ** self ._meta ,
1534+ "first-row-id" : str (first_row_id ),
1535+ "format-version" : "3" ,
1536+ }
1537+ self ._next_row_id = first_row_id
1538+
1539+ def __enter__ (self ) -> ManifestListWriter :
1540+ """Open the writer for writing, using the V3 record layout so `first_row_id` is written."""
1541+ self ._writer = AvroOutputFile [ManifestFile ](
1542+ output_file = self ._output_file ,
1543+ record_schema = MANIFEST_LIST_FILE_SCHEMAS [3 ],
1544+ file_schema = MANIFEST_LIST_FILE_SCHEMAS [3 ],
1545+ schema_name = "manifest_file" ,
1546+ metadata = self ._meta ,
1547+ )
1548+ self ._writer .__enter__ ()
1549+ return self
1550+
1551+ @property
1552+ def next_row_id (self ) -> int :
1553+ """The row ID after the last assigned one; the table's `next-row-id` after this snapshot."""
1554+ return self ._next_row_id
1555+
1556+ def _wrap (self , manifest_file : ManifestFile ) -> ManifestFile :
1557+ """Rebind a manifest file to the V3 record layout."""
1558+ if len (manifest_file ._data ) >= len (MANIFEST_LIST_FILE_SCHEMAS [3 ].fields ):
1559+ return copy (manifest_file )
1560+ args = {
1561+ field .name : value
1562+ for field , value in zip (MANIFEST_LIST_FILE_SCHEMAS [DEFAULT_READ_VERSION ].fields , manifest_file ._data , strict = True )
1563+ }
1564+ return ManifestFile .from_args (_table_format_version = 3 , ** args )
1565+
1566+ def prepare_manifest (self , manifest_file : ManifestFile ) -> ManifestFile :
1567+ wrapped_manifest_file = super ().prepare_manifest (self ._wrap (manifest_file ))
1568+
1569+ if wrapped_manifest_file .content == ManifestContent .DATA and wrapped_manifest_file .first_row_id is None :
1570+ wrapped_manifest_file .first_row_id = self ._next_row_id
1571+ self ._next_row_id += (wrapped_manifest_file .existing_rows_count or 0 ) + (wrapped_manifest_file .added_rows_count or 0 )
1572+ return wrapped_manifest_file
1573+
1574+
14561575def write_manifest_list (
14571576 format_version : TableVersion ,
14581577 output_file : OutputFile ,
14591578 snapshot_id : int ,
14601579 parent_snapshot_id : int | None ,
14611580 sequence_number : int | None ,
14621581 avro_compression : AvroCompressionCodec ,
1582+ first_row_id : int | None = None ,
14631583) -> ManifestListWriter :
14641584 if format_version == 1 :
14651585 return ManifestListWriterV1 (output_file , snapshot_id , parent_snapshot_id , avro_compression )
14661586 elif format_version == 2 :
14671587 if sequence_number is None :
14681588 raise ValueError (f"Sequence-number is required for V2 tables: { sequence_number } " )
14691589 return ManifestListWriterV2 (output_file , snapshot_id , parent_snapshot_id , sequence_number , avro_compression )
1590+ elif format_version == 3 :
1591+ if sequence_number is None :
1592+ raise ValueError (f"Sequence-number is required for V3 tables: { sequence_number } " )
1593+ if first_row_id is None :
1594+ raise ValueError (f"First-row-id is required for V3 tables: { first_row_id } " )
1595+ return ManifestListWriterV3 (output_file , snapshot_id , parent_snapshot_id , sequence_number , avro_compression , first_row_id )
14701596 else :
14711597 raise ValueError (f"Cannot write manifest list for table version: { format_version } " )
0 commit comments