Skip to content

Commit 2dc8991

Browse files
committed
Add FixedWidthTwoLine Reader to guess list
As discussed in astropy#3025 and astropy#3099, it would be useful if tables that are copied from the screen (where the defaul print format is something like) ``` col1 col2 ---- ---- 1.0 2.0 3.0 44.0 ``` into a file could be read automatically with the guessing process. closes astropy#3099 closes astropy#3025
1 parent 76659fe commit 2dc8991

7 files changed

Lines changed: 82 additions & 17 deletions

File tree

‎CHANGES.rst‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,10 @@ API Changes
166166
``parser``, which allows the user to specify which parser
167167
BeautifulSoup should use as a backend. [#2815]
168168

169+
- Add ``FixedWidthTwoLine`` reader to guessing. This will allows to read tables
170+
that a copied from screen output like ``print my_table`` to be read
171+
automatically. Discussed in #3025 and #3099 [#3109]
172+
169173
- ``astropy.io.fits``
170174

171175
- ``astropy.io.misc``
@@ -271,6 +275,10 @@ Bug Fixes
271275
- In the ``CommentedHeader`` the ``data_start`` parameter now defaults to
272276
``0``, which is the first uncommented line. Discussed in #2692. [#3054]
273277

278+
- Position lines in ``FixedWidthTwoLine`` reader could consist of many characters.
279+
Now, only one character in addition to the delimiter is allowed. This bug was
280+
discovered as part of [#3109]
281+
274282
- ``astropy.io.fits``
275283

276284
- Setting ``memmap=True`` in ``fits.open`` and related functions now raises a ValueError if opening a file in memory-mapped mode is impossible. [#2298]

‎astropy/io/ascii/fixedwidth.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,8 @@ def get_cols(self, lines):
133133
# slice col_ends but expects inclusive col_ends on input (for
134134
# more intuitive user interface).
135135
line = self.get_line(lines, position_line)
136+
if len(set(line) - set(self.splitter.delimiter)) != 1:
137+
raise InconsistentTableError('Position line should only contain delimiters and one other character, e.g. "--- ------- ---".')
136138
vals, self.col_starts, col_ends = self.get_fixedwidth_params(line)
137139
self.col_ends = [x - 1 for x in col_ends]
138140

‎astropy/io/ascii/setup_package.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ def get_package_data():
7575
't/whitespace.dat',
7676
't/simple_csv.csv',
7777
't/simple_csv_missing.csv',
78+
't/fixed_width_2_line.txt',
7879
]
7980
}
8081

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Col1 Col2 Col3 Col4
2+
---- --------- ---- ----
3+
1.2 "hello" 1 a
4+
2.4 's worlds 2 2

‎astropy/io/ascii/tests/test_fixedwidth.py‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,10 @@
88
from io import StringIO
99

1010
from ... import ascii
11+
from ..core import InconsistentTableError
1112
from .common import (assert_equal, assert_almost_equal,
1213
setup_function, teardown_function)
13-
14+
from ....tests.helper import pytest
1415

1516
def assert_equal_splitlines(arg1, arg2):
1617
assert_equal(arg1.splitlines(), arg2.splitlines())
@@ -384,6 +385,21 @@ def test_read_twoline_human():
384385
assert_equal(dat[1][1], "'s worlds")
385386

386387

388+
def test_read_twoline_fail():
389+
"""The position line shall consist of only one character in addition to the delimiter."""
390+
table = """
391+
| Col1 | Col2 |
392+
|------|==========|
393+
| 1.2 | "hello" |
394+
| 2.4 | 's worlds|
395+
"""
396+
with pytest.raises(InconsistentTableError) as excinfo:
397+
dat = ascii.read(table, Reader=ascii.FixedWidthTwoLine,
398+
delimiter='|', guess=False)
399+
assert 'Position line should only contain delimiters and one other character' in str(excinfo.value)
400+
401+
402+
387403
def test_write_twoline_normal():
388404
"""Write a table as a normal fixed width table."""
389405
out = StringIO()

‎astropy/io/ascii/tests/test_read.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -767,6 +767,10 @@ def get_testfiles(name=None):
767767
'name': 't/latex2.tex',
768768
'nrows': 3,
769769
'opts': {'Reader': ascii.AASTex}},
770+
{'cols': ('Col1', 'Col2', 'Col3', 'Col4'),
771+
'name': 't/fixed_width_2_line.txt',
772+
'nrows': 2,
773+
'opts': {'Reader': ascii.FixedWidthTwoLine}},
770774
]
771775

772776
try:
@@ -866,3 +870,17 @@ def test_guess_fail():
866870
ascii.read('asfdasdf\n1 2 3', format='html')
867871

868872
assert "** To figure out why the table did not read, use guess=False and" in str(err.value)
873+
874+
def test_pformat_roundtrip():
875+
"""Check that the screen output of ``print tab`` can be read. See #3025."""
876+
"""Read a table with empty values and ensure that corresponding entries are masked"""
877+
table = '\n'.join(['a,b,c,d',
878+
'1,3,1.11,1',
879+
'2, 2, 4.0 , ss '])
880+
dat = ascii.read(table)
881+
out = ascii.read(dat.pformat())
882+
assert len(dat) == len(out)
883+
assert dat.colnames == out.colnames
884+
for c in dat.colnames:
885+
assert np.all(dat[c] == out[c])
886+

‎astropy/io/ascii/ui.py‎

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
from . import html
2525
from . import fastbasic
2626
from . import cparser
27+
from . import fixedwidth
2728

2829
from ...table import Table
2930

@@ -169,18 +170,19 @@ def _guess(table, read_kwargs, format, fast_reader):
169170

170171
# Keep a trace of all failed guesses kwarg
171172
failed_kwargs = []
172-
fast_kwargs = []
173+
full_list_guess = _get_guess_kwargs_list(read_kwargs)
173174

174-
first_kwargs = [read_kwargs.copy()]
175175
if fast_reader and format is not None and 'fast_{0}'.format(format) in \
176176
core.FAST_CLASSES:
177177
# If a fast version of the reader is available, try that before the slow version
178178
fast_kwargs = read_kwargs.copy()
179179
fast_kwargs['Reader'] = core.FAST_CLASSES['fast_{0}'.format(format)]
180-
first_kwargs = [fast_kwargs] + first_kwargs
180+
full_list_guess = [fast_kwargs] + full_list_guess
181+
else:
182+
fast_kwargs = None
181183

182184
# First try guessing
183-
for guess_kwargs in first_kwargs + _get_guess_kwargs_list():
185+
for guess_kwargs in full_list_guess:
184186
guess_kwargs_ok = True # guess_kwargs are consistent with user_kwargs?
185187
for key, val in read_kwargs.items():
186188
# Do guess_kwargs.update(read_kwargs) except that if guess_args has
@@ -237,18 +239,32 @@ def _guess(table, read_kwargs, format, fast_reader):
237239
lines.extend(msg)
238240
raise core.InconsistentTableError('\n'.join(lines))
239241

240-
def _get_guess_kwargs_list():
241-
guess_kwargs_list = [dict(Reader=basic.Rdb),
242-
dict(Reader=fastbasic.FastTab),
243-
dict(Reader=basic.Tab),
244-
dict(Reader=cds.Cds),
245-
dict(Reader=daophot.Daophot),
246-
dict(Reader=sextractor.SExtractor),
247-
dict(Reader=ipac.Ipac),
248-
dict(Reader=latex.Latex),
249-
dict(Reader=latex.AASTex),
250-
dict(Reader=html.HTML)
251-
]
242+
def _get_guess_kwargs_list(read_kwargs):
243+
guess_kwargs_list = []
244+
# First try readers that accept the common arguments with the input arguments
245+
# (Unless there are not arguments - we try that in the next step anyway.)
246+
# FixedWidthTwoLine would also be read by Basic, so it needs to come first.
247+
if len(read_kwargs) > 0:
248+
for reader in [fixedwidth.FixedWidthTwoLine,
249+
basic.Basic]:
250+
first_kwargs = read_kwargs.copy()
251+
first_kwargs.update(dict(Reader=reader))
252+
guess_kwargs_list.append(first_kwargs)
253+
# Then try a list of readers with default arguments
254+
guess_kwargs_list.extend([dict(Reader=fixedwidth.FixedWidthTwoLine),
255+
dict(Reader=fastbasic.FastBasic),
256+
dict(Reader=basic.Basic),
257+
dict(Reader=basic.Rdb),
258+
dict(Reader=fastbasic.FastTab),
259+
dict(Reader=basic.Tab),
260+
dict(Reader=cds.Cds),
261+
dict(Reader=daophot.Daophot),
262+
dict(Reader=sextractor.SExtractor),
263+
dict(Reader=ipac.Ipac),
264+
dict(Reader=latex.Latex),
265+
dict(Reader=latex.AASTex),
266+
dict(Reader=html.HTML)
267+
])
252268
for Reader in (basic.CommentedHeader, fastbasic.FastBasic, basic.Basic,
253269
fastbasic.FastNoHeader, basic.NoHeader):
254270
for delimiter in ("|", ",", " ", "\s"):

0 commit comments

Comments
 (0)