Skip to content

Commit 05b4e0f

Browse files
committed
Tolerate legacy long equality_ids in resolver and write int in manifest (apache#3840)
1 parent f8a3d37 commit 05b4e0f

5 files changed

Lines changed: 167 additions & 116 deletions

File tree

‎pyiceberg/avro/resolver.py‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,6 @@
1414
# KIND, either express or implied. See the License for the
1515
# specific language governing permissions and limitations
1616
# under the License.
17-
# pylint: disable=arguments-renamed,unused-argument
18-
import warnings
1917
from collections.abc import Callable
2018
from enum import Enum
2119

@@ -379,6 +377,9 @@ def visit_geography(self, geography_type: "GeographyType", partner: IcebergType
379377
return BinaryWriter()
380378

381379

380+
_MANIFEST_DATA_FILE_EQUALITY_IDS_ELEMENT_ID = 136
381+
382+
382383
class ReadSchemaResolver(PrimitiveWithPartnerVisitor[IcebergType, Reader]):
383384
__slots__ = ("read_types", "read_enums", "context")
384385
read_types: dict[int, Callable[..., StructProtocol]]
@@ -462,13 +463,14 @@ def primitive(self, primitive: PrimitiveType, expected_primitive: IcebergType |
462463

463464
# ensure that the type can be projected to the expected
464465
if primitive != expected_primitive:
465-
if isinstance(primitive, LongType) and isinstance(expected_primitive, IntegerType):
466-
warnings.warn(
467-
"Encountered non-compliant manifest with long equality_ids (spec requires int). "
468-
"Support for legacy long equality_ids is deprecated and will be removed in a future release.",
469-
DeprecationWarning,
470-
stacklevel=2,
471-
)
466+
is_manifest_data_file = getattr(self.read_types.get(2), "__name__", "") == "DataFile"
467+
if (
468+
is_manifest_data_file
469+
and _MANIFEST_DATA_FILE_EQUALITY_IDS_ELEMENT_ID in self.context
470+
and isinstance(primitive, LongType)
471+
and isinstance(expected_primitive, IntegerType)
472+
):
473+
pass
472474
else:
473475
promote(primitive, expected_primitive)
474476

‎pyiceberg/manifest.py‎

Lines changed: 18 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -433,9 +433,7 @@ def __repr__(self) -> str:
433433
}
434434

435435

436-
def data_file_with_partition(
437-
partition_type: StructType, format_version: TableVersion, legacy_equality_ids: bool = False
438-
) -> StructType:
436+
def data_file_with_partition(partition_type: StructType, format_version: TableVersion) -> StructType:
439437
data_file_partition_type = StructType(
440438
*[
441439
NestedField(
@@ -448,32 +446,20 @@ def data_file_with_partition(
448446
]
449447
)
450448

451-
fields = []
452-
for field in DATA_FILE_TYPE[format_version].fields:
453-
if field.field_id == 102:
454-
fields.append(
455-
NestedField(
456-
field_id=102,
457-
name="partition",
458-
field_type=data_file_partition_type,
459-
required=True,
460-
doc="Partition data tuple, schema based on the partition spec",
461-
)
462-
)
463-
elif field.field_id == 135 and legacy_equality_ids:
464-
fields.append(
465-
NestedField(
466-
field_id=135,
467-
name="equality_ids",
468-
field_type=ListType(element_id=136, element_type=LongType(), element_required=True),
469-
required=False,
470-
doc="Field ids used to determine row equality in equality delete files.",
471-
)
449+
return StructType(
450+
*[
451+
NestedField(
452+
field_id=102,
453+
name="partition",
454+
field_type=data_file_partition_type,
455+
required=True,
456+
doc="Partition data tuple, schema based on the partition spec",
472457
)
473-
else:
474-
fields.append(field)
475-
476-
return StructType(*fields)
458+
if field.field_id == 102
459+
else field
460+
for field in DATA_FILE_TYPE[format_version].fields
461+
]
462+
)
477463

478464

479465
class DataFile(Record):
@@ -1092,7 +1078,6 @@ class ManifestWriter(ABC):
10921078
_min_sequence_number: int | None
10931079
_partitions: list[Record]
10941080
_compression: AvroCompressionCodec
1095-
_legacy_equality_ids: bool
10961081

10971082
def __init__(
10981083
self,
@@ -1101,7 +1086,6 @@ def __init__(
11011086
output_file: OutputFile,
11021087
snapshot_id: int,
11031088
avro_compression: AvroCompressionCodec,
1104-
legacy_equality_ids: bool = False,
11051089
) -> None:
11061090
self.closed = False
11071091
self._spec = spec
@@ -1118,7 +1102,6 @@ def __init__(
11181102
self._min_sequence_number = None
11191103
self._partitions = []
11201104
self._compression = avro_compression
1121-
self._legacy_equality_ids = legacy_equality_ids
11221105

11231106
def __enter__(self) -> ManifestWriter:
11241107
"""Open the writer."""
@@ -1164,7 +1147,6 @@ def _with_partition(self, format_version: TableVersion) -> Schema:
11641147
data_file_type = data_file_with_partition(
11651148
format_version=format_version,
11661149
partition_type=self._spec.partition_type(self._schema),
1167-
legacy_equality_ids=self._legacy_equality_ids,
11681150
)
11691151
return manifest_entry_schema_with_data_file(format_version=format_version, data_file=data_file_type)
11701152

@@ -1277,9 +1259,8 @@ def __init__(
12771259
output_file: OutputFile,
12781260
snapshot_id: int,
12791261
avro_compression: AvroCompressionCodec,
1280-
legacy_equality_ids: bool = False,
12811262
):
1282-
super().__init__(spec, schema, output_file, snapshot_id, avro_compression, legacy_equality_ids=legacy_equality_ids)
1263+
super().__init__(spec, schema, output_file, snapshot_id, avro_compression)
12831264

12841265
def content(self) -> ManifestContent:
12851266
return ManifestContent.DATA
@@ -1300,9 +1281,8 @@ def __init__(
13001281
output_file: OutputFile,
13011282
snapshot_id: int,
13021283
avro_compression: AvroCompressionCodec,
1303-
legacy_equality_ids: bool = False,
13041284
):
1305-
super().__init__(spec, schema, output_file, snapshot_id, avro_compression, legacy_equality_ids=legacy_equality_ids)
1285+
super().__init__(spec, schema, output_file, snapshot_id, avro_compression)
13061286

13071287
def content(self) -> ManifestContent:
13081288
return ManifestContent.DATA
@@ -1334,12 +1314,11 @@ def write_manifest(
13341314
output_file: OutputFile,
13351315
snapshot_id: int,
13361316
avro_compression: AvroCompressionCodec,
1337-
legacy_equality_ids: bool = False,
13381317
) -> ManifestWriter:
13391318
if format_version == 1:
1340-
return ManifestWriterV1(spec, schema, output_file, snapshot_id, avro_compression, legacy_equality_ids=legacy_equality_ids)
1319+
return ManifestWriterV1(spec, schema, output_file, snapshot_id, avro_compression)
13411320
elif format_version == 2:
1342-
return ManifestWriterV2(spec, schema, output_file, snapshot_id, avro_compression, legacy_equality_ids=legacy_equality_ids)
1321+
return ManifestWriterV2(spec, schema, output_file, snapshot_id, avro_compression)
13431322
else:
13441323
raise ValueError(f"Cannot write manifest for table version: {format_version}")
13451324

‎pyiceberg/table/__init__.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -167,9 +167,6 @@ class TableProperties:
167167
WRITE_AVRO_COMPRESSION = "write.avro.compression-codec"
168168
WRITE_AVRO_COMPRESSION_DEFAULT = "gzip"
169169

170-
WRITE_MANIFEST_LEGACY_LONG_EQUALITY_IDS = "write.manifest.legacy-long-equality-ids"
171-
WRITE_MANIFEST_LEGACY_LONG_EQUALITY_IDS_DEFAULT = False
172-
173170
DEFAULT_WRITE_METRICS_MODE = "write.metadata.metrics.default"
174171
DEFAULT_WRITE_METRICS_MODE_DEFAULT = "truncate(16)"
175172

‎tests/avro/test_resolver.py‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -418,3 +418,88 @@ def test_writer_missing_optional_in_read_schema() -> None:
418418
expected = StructWriter(field_writers=((None, OptionWriter(option=StringWriter())),))
419419

420420
assert actual == expected
421+
422+
423+
def test_resolver_long_to_int_isolated_to_manifest_equality_ids() -> None:
424+
from pyiceberg.manifest import DataFile
425+
426+
# 1. User table schema: Long -> Int must fail even if field ID is 136
427+
user_file_schema = Schema(NestedField(field_id=136, name="col", field_type=LongType(), required=True))
428+
user_read_schema = Schema(NestedField(field_id=136, name="col", field_type=IntegerType(), required=True))
429+
430+
with pytest.raises(ResolveError, match="Cannot promote long to int"):
431+
resolve_reader(file_schema=user_file_schema, read_schema=user_read_schema)
432+
433+
# 2. Manifest DataFile schema with equality_ids (field 135, element 136): Long -> Int must succeed
434+
manifest_file_schema = Schema(
435+
NestedField(
436+
field_id=2,
437+
name="data_file",
438+
field_type=StructType(
439+
NestedField(
440+
field_id=135,
441+
name="equality_ids",
442+
field_type=ListType(element_id=136, element_type=LongType(), element_required=True),
443+
required=False,
444+
)
445+
),
446+
required=True,
447+
)
448+
)
449+
manifest_read_schema = Schema(
450+
NestedField(
451+
field_id=2,
452+
name="data_file",
453+
field_type=StructType(
454+
NestedField(
455+
field_id=135,
456+
name="equality_ids",
457+
field_type=ListType(element_id=136, element_type=IntegerType(), element_required=True),
458+
required=False,
459+
)
460+
),
461+
required=True,
462+
)
463+
)
464+
465+
# Without read_types={2: DataFile}, it fails because it's not recognized as a manifest data_file
466+
with pytest.raises(ResolveError, match="Cannot promote long to int"):
467+
resolve_reader(file_schema=manifest_file_schema, read_schema=manifest_read_schema)
468+
469+
# With read_types={2: DataFile}, equality_ids Long -> Int succeeds!
470+
reader = resolve_reader(
471+
file_schema=manifest_file_schema,
472+
read_schema=manifest_read_schema,
473+
read_types={2: DataFile},
474+
)
475+
assert reader is not None
476+
477+
# 3. Manifest DataFile schema with other field (e.g. record_count field 103):
478+
# Long -> Int must FAIL even with read_types={2: DataFile}
479+
manifest_record_count_file_schema = Schema(
480+
NestedField(
481+
field_id=2,
482+
name="data_file",
483+
field_type=StructType(
484+
NestedField(field_id=103, name="record_count", field_type=LongType(), required=True),
485+
),
486+
required=True,
487+
)
488+
)
489+
manifest_record_count_read_schema = Schema(
490+
NestedField(
491+
field_id=2,
492+
name="data_file",
493+
field_type=StructType(
494+
NestedField(field_id=103, name="record_count", field_type=IntegerType(), required=True),
495+
),
496+
required=True,
497+
)
498+
)
499+
500+
with pytest.raises(ResolveError, match="Cannot promote long to int"):
501+
resolve_reader(
502+
file_schema=manifest_record_count_file_schema,
503+
read_schema=manifest_record_count_read_schema,
504+
read_types={2: DataFile},
505+
)

0 commit comments

Comments
 (0)