Skip to content

Commit 54d7862

Browse files
authored
Merge pull request #9541 from bmorris3/fix-issue-9280
Adding pandas Int64 support [fix #9280]
2 parents e88753e + c899c17 commit 54d7862

3 files changed

Lines changed: 64 additions & 17 deletions

File tree

‎CHANGES.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,9 @@ astropy.table
143143
- Added new ``Table`` method ``.round()``, which rounds numeric columns to the
144144
specified number of decimals. [#9862]
145145

146+
- Updated ``to_pandas()`` and ``from_pandas()`` to use and support Pandas
147+
nullable integer data type for masked integer data. [#9541]
148+
146149
- The HDF5 writer, ``write_table_hdf5()``, now allows passing through
147150
additional keyword arguments to the ``h5py.Group.create_dataset()``. [#9602]
148151

‎astropy/table/table.py‎

Lines changed: 35 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3293,7 +3293,7 @@ def group_by(self, keys):
32933293
"""
32943294
return groups.table_group_by(self, keys)
32953295

3296-
def to_pandas(self, index=None):
3296+
def to_pandas(self, index=None, use_nullable_int=True):
32973297
"""
32983298
Return a :class:`pandas.DataFrame` instance
32993299
@@ -3304,20 +3304,28 @@ def to_pandas(self, index=None):
33043304
then no DataFrame index will be specified. If ``index`` is the name of
33053305
a column in the table then that will be the DataFrame index.
33063306
3307-
In additional to vanilla columns or masked columns, this supports Table
3307+
In addition to vanilla columns or masked columns, this supports Table
33083308
mixin columns like Quantity, Time, or SkyCoord. In many cases these
33093309
objects have no analog in pandas and will be converted to a "encoded"
33103310
representation using only Column or MaskedColumn. The exception is
33113311
Time or TimeDelta columns, which will be converted to the corresponding
33123312
representation in pandas using ``np.datetime64`` or ``np.timedelta64``.
33133313
See the example below.
33143314
3315+
Parameters
3316+
----------
3317+
index : None, bool, str
3318+
Specify DataFrame index mode
3319+
use_nullable_int : bool, default=True
3320+
Convert integer MaskedColumn to pandas nullable integer type.
3321+
If ``use_nullable_int=False`` or the pandas version does not support
3322+
nullable integer types (version < 0.24), then the column is converted
3323+
to float with NaN for missing elements and a warning is issued.
3324+
33153325
Returns
33163326
-------
33173327
dataframe : :class:`pandas.DataFrame`
33183328
A pandas :class:`pandas.DataFrame` instance
3319-
index : None, bool, str
3320-
Specify DataFrame index mode
33213329
33223330
Raises
33233331
------
@@ -3353,7 +3361,7 @@ def to_pandas(self, index=None):
33533361
2002-01-01 2.0 6.0 8.0 00:03:20
33543362
33553363
"""
3356-
from pandas import DataFrame
3364+
from pandas import DataFrame, Series
33573365

33583366
if index is not False:
33593367
if index in (None, True):
@@ -3417,18 +3425,26 @@ def _encode_mixins(tbl):
34173425
for name, column in tbl.columns.items():
34183426
if isinstance(column, MaskedColumn) and np.any(column.mask):
34193427
if column.dtype.kind in ['i', 'u']:
3420-
out[name] = column.astype(float).filled(np.nan)
3421-
warnings.warn(
3422-
"converted column '{}' from integer to float".format(
3423-
name), TableReplaceWarning, stacklevel=3)
3428+
pd_dtype = str(column.dtype)
3429+
if use_nullable_int:
3430+
# Convert int64 to Int64, uint32 to UInt32, etc for nullable types
3431+
pd_dtype = pd_dtype.replace('i', 'I').replace('u', 'U')
3432+
out[name] = Series(column, dtype=pd_dtype)
3433+
3434+
# If pandas is older than 0.24 the type may have turned to float
3435+
if column.dtype.kind != out[name].dtype.kind:
3436+
warnings.warn(
3437+
f"converted column '{name}' from {column.dtype} to {out[name].dtype}",
3438+
TableReplaceWarning, stacklevel=3)
34243439
elif column.dtype.kind in ['f', 'c']:
3425-
out[name] = column.filled(np.nan)
3440+
out[name] = column
34263441
else:
34273442
out[name] = column.astype(object).filled(np.nan)
34283443
else:
34293444
out[name] = column
34303445

3431-
if out[name].dtype.byteorder not in ('=', '|'):
3446+
if (hasattr(out[name].dtype, 'byteorder')
3447+
and out[name].dtype.byteorder not in ('=', '|')):
34323448
out[name] = out[name].byteswap().newbyteorder()
34333449

34343450
kwargs = {'index': out.pop(index)} if index else {}
@@ -3526,6 +3542,14 @@ def from_pandas(cls, dataframe, index=False, units=None):
35263542

35273543
for name, column, data, mask, unit in zip(names, columns, datas, masks, units):
35283544

3545+
if column.dtype.kind in ['u', 'i'] and np.any(mask):
3546+
# Special-case support for pandas nullable int
3547+
np_dtype = str(column.dtype).lower()
3548+
data = np.zeros(shape=column.shape, dtype=np_dtype)
3549+
data[~mask] = column[~mask]
3550+
out[name] = MaskedColumn(data=data, name=name, mask=mask, unit=unit, copy=False)
3551+
continue
3552+
35293553
if data.dtype.kind == 'O':
35303554
# If all elements of an object array are string-like or np.nan
35313555
# then coerce back to a native numpy str/unicode array.

‎astropy/table/tests/test_table.py‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1845,6 +1845,20 @@ def test_simple(self):
18451845
else:
18461846
assert t[column].byteswap().newbyteorder().dtype == t2[column].dtype
18471847

1848+
@pytest.mark.parametrize('unsigned', ['u', ''])
1849+
@pytest.mark.parametrize('bits', [8, 16, 32, 64])
1850+
def test_nullable_int(self, unsigned, bits):
1851+
np_dtype = f'{unsigned}int{bits}'
1852+
c = MaskedColumn([1, 2], mask=[False, True], dtype=np_dtype)
1853+
t = Table([c])
1854+
df = t.to_pandas()
1855+
pd_dtype = np_dtype.replace('i', 'I').replace('u', 'U')
1856+
assert str(df['col0'].dtype) == pd_dtype
1857+
t2 = Table.from_pandas(df)
1858+
assert str(t2['col0'].dtype) == np_dtype
1859+
assert np.all(t2['col0'].mask == [False, True])
1860+
assert np.all(t2['col0'] == c)
1861+
18481862
def test_2d(self):
18491863

18501864
t = table.Table()
@@ -1952,7 +1966,8 @@ def test_from_pandas_index(self):
19521966
assert t2.colnames == ['tm', 'x']
19531967
assert np.allclose(t2['tm'].jyear, tm.jyear)
19541968

1955-
def test_masking(self):
1969+
@pytest.mark.parametrize('use_nullable_int', [True, False])
1970+
def test_masking(self, use_nullable_int):
19561971

19571972
t = table.Table(masked=True)
19581973

@@ -1973,22 +1988,27 @@ def test_masking(self):
19731988
2584288728310999296]
19741989
t['Source'].mask = [False, False, False]
19751990

1976-
with pytest.warns(TableReplaceWarning,
1977-
match="converted column 'a' from integer to float"):
1978-
d = t.to_pandas()
1991+
if use_nullable_int: # Default
1992+
# No warning with the default use_nullable_int=True
1993+
d = t.to_pandas(use_nullable_int=use_nullable_int)
1994+
else:
1995+
with pytest.warns(TableReplaceWarning,
1996+
match=r"converted column 'a' from int(32|64) to float64"):
1997+
d = t.to_pandas(use_nullable_int=use_nullable_int)
19791998

19801999
t2 = table.Table.from_pandas(d)
19812000

19822001
for name, column in t.columns.items():
19832002
assert np.all(column.data == t2[name].data)
19842003
if hasattr(t2[name], 'mask'):
19852004
assert np.all(column.mask == t2[name].mask)
1986-
# Masked integer type comes back as float. Nothing we can do about this.
2005+
19872006
if column.dtype.kind == 'i':
1988-
if np.any(column.mask):
2007+
if np.any(column.mask) and not use_nullable_int:
19892008
assert t2[name].dtype.kind == 'f'
19902009
else:
19912010
assert t2[name].dtype.kind == 'i'
2011+
19922012
assert_array_equal(column.data,
19932013
t2[name].data.astype(column.dtype))
19942014
else:

0 commit comments

Comments
 (0)