Skip to content

Commit a42a60f

Browse files
committed
Build nested rows for normalized Optional and sequence array element types
BatchConverter.from_typehints normalizes the element type, so PyarrowArrayBatchConverter saw Beam UnionConstraint and sequence constraints, which typing.get_origin does not recognize, and returned struct values as dicts for Optional[Row] and List[Row] elements.
1 parent 1b1633c commit a42a60f

2 files changed

Lines changed: 31 additions & 2 deletions

File tree

‎sdks/python/apache_beam/typehints/arrow_type_compatibility.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import pyarrow as pa
3939

4040
from apache_beam.portability.api import schema_pb2
41+
from apache_beam.typehints import typehints
4142
from apache_beam.typehints.batch import BatchConverter
4243
from apache_beam.typehints.native_type_compatibility import match_dataclass_for_row
4344
from apache_beam.typehints.native_type_compatibility import match_is_named_tuple
@@ -320,8 +321,16 @@ def _from_arrow_value_fn(typ) -> Optional[Callable[[Any], Any]]:
320321
return _row_from_dict_fn(
321322
typ, [(field.name, field.type) for field in dataclasses.fields(typ)])
322323

323-
origin = get_origin(typ)
324-
args = get_args(typ)
324+
# BatchConverter.from_typehints normalizes typing hints to Beam ones.
325+
if isinstance(typ, typehints.UnionConstraint):
326+
origin, args = Union, tuple(typ.union_types)
327+
elif isinstance(typ,
328+
(typehints.ListConstraint,
329+
typehints.ABCSequenceTypeConstraint,
330+
typehints.IterableTypeConstraint)):
331+
origin, args = list, (typ.inner_type, )
332+
else:
333+
origin, args = get_origin(typ), get_args(typ)
325334
if origin is Union:
326335
not_none = [arg for arg in args if arg is not type(None)]
327336
return _from_arrow_value_fn(not_none[0]) if len(not_none) == 1 else None

‎sdks/python/apache_beam/typehints/arrow_type_compatibility_test.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,26 @@ def test_explode_array_batch_builds_rows(self):
278278
self.assertEqual(exploded, [Address(city='SF', zip=1), None])
279279
self.assertIsInstance(exploded[0], Address)
280280

281+
@parameterized.expand([
282+
(
283+
'optional',
284+
Optional[Address],
285+
'address', [Address(city='SF', zip=1), None]),
286+
(
287+
'sequence',
288+
Optional[Sequence[Optional[Address]]],
289+
'history', [[Address(city='NY', zip=2), None], None]),
290+
])
291+
def test_explode_array_batch_builds_nested_rows(
292+
self, _, element_type, column, expected):
293+
# from_typehints normalizes these hints to Beam type constraints.
294+
converter = BatchConverter.from_typehints(
295+
element_type=element_type, batch_type=pa.Array)
296+
297+
exploded = list(converter.explode_batch(self.BATCH.column(column)))
298+
299+
self.assertEqual(exploded, expected)
300+
281301

282302
class ArrowBatchConverterErrorsTest(unittest.TestCase):
283303
@parameterized.expand([

0 commit comments

Comments
 (0)