@@ -1296,3 +1296,100 @@ def test_negative_manifest_cache_size_raises_value_error(monkeypatch: pytest.Mon
12961296 finally :
12971297 monkeypatch .delenv ("PYICEBERG_MANIFEST_CACHE_SIZE" , raising = False )
12981298 importlib .reload (manifest_module )
1299+
1300+
1301+ def test_write_manifest_equality_ids_int_schema (tmp_path : Path ) -> None :
1302+ io = PyArrowFileIO ()
1303+ schema = Schema (NestedField (1 , "x" , IntegerType ()))
1304+ df = DataFile .from_args (
1305+ content = DataFileContent .DATA ,
1306+ file_path = "file:///tmp/test.parquet" ,
1307+ file_format = FileFormat .PARQUET ,
1308+ partition = (),
1309+ record_count = 10 ,
1310+ file_size_in_bytes = 100 ,
1311+ equality_ids = [1 , 2 , 3 ],
1312+ )
1313+ manifest_path = f"file://{ tmp_path / 'test_equality_ids.avro' } "
1314+ with write_manifest (
1315+ format_version = 2 ,
1316+ spec = UNPARTITIONED_PARTITION_SPEC ,
1317+ schema = schema ,
1318+ output_file = io .new_output (manifest_path ),
1319+ snapshot_id = 12345 ,
1320+ avro_compression = "null" ,
1321+ ) as writer :
1322+ writer .add_entry (
1323+ ManifestEntry .from_args (
1324+ status = ManifestEntryStatus .ADDED ,
1325+ snapshot_id = 12345 ,
1326+ sequence_number = 1 ,
1327+ file_sequence_number = 1 ,
1328+ data_file = df ,
1329+ )
1330+ )
1331+
1332+ with open (tmp_path / "test_equality_ids.avro" , "rb" ) as f :
1333+ reader = fastavro .reader (f )
1334+ writer_schema = reader .writer_schema
1335+ fields = {f ["name" ]: f for f in writer_schema ["fields" ]}
1336+ df_fields = {f ["name" ]: f for f in fields ["data_file" ]["type" ]["fields" ]}
1337+ assert df_fields ["equality_ids" ]["type" ][1 ]["items" ] == "int"
1338+
1339+
1340+ def test_read_manifest_legacy_equality_ids_long_schema (tmp_path : Path ) -> None :
1341+ io = PyArrowFileIO ()
1342+ schema = Schema (NestedField (1 , "x" , IntegerType ()))
1343+ df = DataFile .from_args (
1344+ content = DataFileContent .DATA ,
1345+ file_path = "file:///tmp/test.parquet" ,
1346+ file_format = FileFormat .PARQUET ,
1347+ partition = (),
1348+ record_count = 10 ,
1349+ file_size_in_bytes = 100 ,
1350+ equality_ids = [1 , 2 , 3 ],
1351+ )
1352+ manifest_path = f"file://{ tmp_path / 'test_legacy.avro' } "
1353+ with write_manifest (
1354+ format_version = 2 ,
1355+ spec = UNPARTITIONED_PARTITION_SPEC ,
1356+ schema = schema ,
1357+ output_file = io .new_output (manifest_path ),
1358+ snapshot_id = 12345 ,
1359+ avro_compression = "null" ,
1360+ ) as writer :
1361+ writer .add_entry (
1362+ ManifestEntry .from_args (
1363+ status = ManifestEntryStatus .ADDED ,
1364+ snapshot_id = 12345 ,
1365+ sequence_number = 1 ,
1366+ file_sequence_number = 1 ,
1367+ data_file = df ,
1368+ )
1369+ )
1370+
1371+ with open (tmp_path / "test_legacy.avro" , "rb" ) as f :
1372+ reader = fastavro .reader (f )
1373+ records = list (reader )
1374+ writer_schema = reader .writer_schema
1375+ for field in writer_schema ["fields" ]:
1376+ if field ["name" ] == "data_file" :
1377+ for df_field in field ["type" ]["fields" ]:
1378+ if df_field ["name" ] == "equality_ids" :
1379+ df_field ["type" ][1 ]["items" ] = "long"
1380+
1381+ legacy_file_path = tmp_path / "test_legacy_modified.avro"
1382+ with open (legacy_file_path , "wb" ) as f :
1383+ fastavro .writer (f , writer_schema , records )
1384+
1385+ mf = ManifestFile .from_args (
1386+ manifest_path = f"file://{ legacy_file_path } " ,
1387+ manifest_length = 1000 ,
1388+ partition_spec_id = 0 ,
1389+ added_snapshot_id = 12345 ,
1390+ sequence_number = 1 ,
1391+ partitions = [],
1392+ )
1393+ entries = mf .fetch_manifest_entry (io )
1394+ assert len (entries ) == 1
1395+ assert entries [0 ].data_file .equality_ids == [1 , 2 , 3 ]
0 commit comments