Skip to content

Commit ea76574

Browse files
apacheGH-43683: [Python] Use pandas StringDtype when enabled (pandas 3+)
1 parent 0f7b5e5 commit ea76574

4 files changed

Lines changed: 35 additions & 3 deletions

File tree

‎python/pyarrow/array.pxi‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ def _handle_arrow_array_protocol(obj, type, mask, size):
117117
"return a pyarrow Array or ChunkedArray.")
118118
if isinstance(res, ChunkedArray) and res.num_chunks==1:
119119
res = res.chunk(0)
120+
if type is not None and res.type != type:
121+
res = res.cast(type)
120122
return res
121123

122124

‎python/pyarrow/pandas-shim.pxi‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,16 @@ cdef class _PandasAPIShim(object):
174174
self._check_import()
175175
return self._is_ge_v3
176176

177+
def uses_string_dtype(self):
178+
if self.is_ge_v3():
179+
return True
180+
try:
181+
if self.pd.options.future.infer_string:
182+
return True
183+
except:
184+
pass
185+
return False
186+
177187
@property
178188
def categorical_type(self):
179189
self._check_import()

‎python/pyarrow/pandas_compat.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -848,6 +848,16 @@ def _get_extension_dtypes(table, columns_metadata, types_mapper=None):
848848
if _pandas_api.extension_dtype is None:
849849
return ext_columns
850850

851+
# for pandas 3.0+, use pandas' new default string dtype
852+
if _pandas_api.uses_string_dtype():
853+
for field in table.schema:
854+
if (
855+
pa.types.is_string(field.type)
856+
or pa.types.is_large_string(field.type)
857+
or pa.types.is_string_view(field.type)
858+
):
859+
ext_columns[field.name] = _pandas_api.pd.StringDtype(na_value=np.nan)
860+
851861
# infer the extension columns from the pandas metadata
852862
for col_meta in columns_metadata:
853863
try:
@@ -1133,7 +1143,12 @@ def _reconstruct_columns_from_metadata(columns, column_indexes):
11331143
# GH-41503: if the column index was decimal, restore to decimal
11341144
elif pandas_dtype == "decimal":
11351145
level = _pandas_api.pd.Index([decimal.Decimal(i) for i in level])
1136-
elif level.dtype != dtype:
1146+
elif (
1147+
level.dtype == "str" and "mixed" in pandas_dtype and numpy_dtype == "object"
1148+
):
1149+
new_levels.append(level)
1150+
continue
1151+
elif level.dtype != dtype and not level.dtype == "str":
11371152
level = level.astype(dtype)
11381153
# ARROW-9096: if original DataFrame was upcast we keep that
11391154
if level.dtype != numpy_dtype and pandas_dtype != "datetimetz":

‎python/pyarrow/tests/test_pandas.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -411,7 +411,9 @@ def test_string_column_index(self):
411411
column_indexes, = js['column_indexes']
412412
assert column_indexes['name'] == 'stringz'
413413
assert column_indexes['name'] == column_indexes['field_name']
414-
assert column_indexes['numpy_type'] == 'object'
414+
assert column_indexes['numpy_type'] == (
415+
'str' if _pandas_api.uses_string_dtype() else 'object'
416+
)
415417
assert column_indexes['pandas_type'] == 'unicode'
416418

417419
md = column_indexes['metadata']
@@ -1680,7 +1682,10 @@ def test_pandas_unicode(self):
16801682
repeats = 1000
16811683
values = ['foo', None, 'bar', 'mañana', np.nan]
16821684
df = pd.DataFrame({'strings': values * repeats})
1683-
field = pa.field('strings', pa.string())
1685+
field = pa.field(
1686+
'strings',
1687+
pa.large_string() if _pandas_api.uses_string_dtype() else pa.string()
1688+
)
16841689
schema = pa.schema([field])
16851690
ex_values = ['foo', None, 'bar', 'mañana', None]
16861691
expected = pd.DataFrame({'strings': ex_values * repeats})

0 commit comments

Comments
 (0)