Skip to content

Commit 4b4ac26

Browse files
committed
Manifest: Use int for equality_ids in manifest schema per Iceberg spec (apache#3840)
1 parent 58749a3 commit 4b4ac26

3 files changed

Lines changed: 103 additions & 3 deletions

File tree

‎pyiceberg/avro/resolver.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -461,7 +461,10 @@ def primitive(self, primitive: PrimitiveType, expected_primitive: IcebergType |
461461

462462
# ensure that the type can be projected to the expected
463463
if primitive != expected_primitive:
464-
promote(primitive, expected_primitive)
464+
if isinstance(primitive, LongType) and isinstance(expected_primitive, IntegerType):
465+
pass
466+
else:
467+
promote(primitive, expected_primitive)
465468

466469
return super().primitive(primitive, expected_primitive)
467470

‎pyiceberg/manifest.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -295,7 +295,7 @@ def __repr__(self) -> str:
295295
NestedField(
296296
field_id=135,
297297
name="equality_ids",
298-
field_type=ListType(element_id=136, element_type=LongType(), element_required=True),
298+
field_type=ListType(element_id=136, element_type=IntegerType(), element_required=True),
299299
required=False,
300300
doc="Field ids used to determine row equality in equality delete files.",
301301
),
@@ -390,7 +390,7 @@ def __repr__(self) -> str:
390390
NestedField(
391391
field_id=135,
392392
name="equality_ids",
393-
field_type=ListType(element_id=136, element_type=LongType(), element_required=True),
393+
field_type=ListType(element_id=136, element_type=IntegerType(), element_required=True),
394394
required=False,
395395
doc="Field ids used to determine row equality in equality delete files.",
396396
),

‎tests/utils/test_manifest.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1296,3 +1296,100 @@ def test_negative_manifest_cache_size_raises_value_error(monkeypatch: pytest.Mon
12961296
finally:
12971297
monkeypatch.delenv("PYICEBERG_MANIFEST_CACHE_SIZE", raising=False)
12981298
importlib.reload(manifest_module)
1299+
1300+
1301+
def test_write_manifest_equality_ids_int_schema(tmp_path: Path) -> None:
1302+
io = PyArrowFileIO()
1303+
schema = Schema(NestedField(1, "x", IntegerType()))
1304+
df = DataFile.from_args(
1305+
content=DataFileContent.DATA,
1306+
file_path="file:///tmp/test.parquet",
1307+
file_format=FileFormat.PARQUET,
1308+
partition=(),
1309+
record_count=10,
1310+
file_size_in_bytes=100,
1311+
equality_ids=[1, 2, 3],
1312+
)
1313+
manifest_path = f"file://{tmp_path / 'test_equality_ids.avro'}"
1314+
with write_manifest(
1315+
format_version=2,
1316+
spec=UNPARTITIONED_PARTITION_SPEC,
1317+
schema=schema,
1318+
output_file=io.new_output(manifest_path),
1319+
snapshot_id=12345,
1320+
avro_compression="null",
1321+
) as writer:
1322+
writer.add_entry(
1323+
ManifestEntry.from_args(
1324+
status=ManifestEntryStatus.ADDED,
1325+
snapshot_id=12345,
1326+
sequence_number=1,
1327+
file_sequence_number=1,
1328+
data_file=df,
1329+
)
1330+
)
1331+
1332+
with open(tmp_path / "test_equality_ids.avro", "rb") as f:
1333+
reader = fastavro.reader(f)
1334+
writer_schema = reader.writer_schema
1335+
fields = {f["name"]: f for f in writer_schema["fields"]}
1336+
df_fields = {f["name"]: f for f in fields["data_file"]["type"]["fields"]}
1337+
assert df_fields["equality_ids"]["type"][1]["items"] == "int"
1338+
1339+
1340+
def test_read_manifest_legacy_equality_ids_long_schema(tmp_path: Path) -> None:
1341+
io = PyArrowFileIO()
1342+
schema = Schema(NestedField(1, "x", IntegerType()))
1343+
df = DataFile.from_args(
1344+
content=DataFileContent.DATA,
1345+
file_path="file:///tmp/test.parquet",
1346+
file_format=FileFormat.PARQUET,
1347+
partition=(),
1348+
record_count=10,
1349+
file_size_in_bytes=100,
1350+
equality_ids=[1, 2, 3],
1351+
)
1352+
manifest_path = f"file://{tmp_path / 'test_legacy.avro'}"
1353+
with write_manifest(
1354+
format_version=2,
1355+
spec=UNPARTITIONED_PARTITION_SPEC,
1356+
schema=schema,
1357+
output_file=io.new_output(manifest_path),
1358+
snapshot_id=12345,
1359+
avro_compression="null",
1360+
) as writer:
1361+
writer.add_entry(
1362+
ManifestEntry.from_args(
1363+
status=ManifestEntryStatus.ADDED,
1364+
snapshot_id=12345,
1365+
sequence_number=1,
1366+
file_sequence_number=1,
1367+
data_file=df,
1368+
)
1369+
)
1370+
1371+
with open(tmp_path / "test_legacy.avro", "rb") as f:
1372+
reader = fastavro.reader(f)
1373+
records = list(reader)
1374+
writer_schema = reader.writer_schema
1375+
for field in writer_schema["fields"]:
1376+
if field["name"] == "data_file":
1377+
for df_field in field["type"]["fields"]:
1378+
if df_field["name"] == "equality_ids":
1379+
df_field["type"][1]["items"] = "long"
1380+
1381+
legacy_file_path = tmp_path / "test_legacy_modified.avro"
1382+
with open(legacy_file_path, "wb") as f:
1383+
fastavro.writer(f, writer_schema, records)
1384+
1385+
mf = ManifestFile.from_args(
1386+
manifest_path=f"file://{legacy_file_path}",
1387+
manifest_length=1000,
1388+
partition_spec_id=0,
1389+
added_snapshot_id=12345,
1390+
sequence_number=1,
1391+
partitions=[],
1392+
)
1393+
entries = mf.fetch_manifest_entry(io)
1394+
assert len(entries) == 1
1395+
assert entries[0].data_file.equality_ids == [1, 2, 3]

0 commit comments

Comments
 (0)