Skip to content

Commit a2d2628

Browse files
committed
Add FixedWidthTwoLine Reader to guess list
As discussed in astropy#3025 and astropy#3099, it would be useful if tables that are copied from the screen (where the defaul print format is something like) ``` col1 col2 ---- ---- 1.0 2.0 3.0 44.0 ``` into a file could be read automatically with the guessing process. closes astropy#3099 closes astropy#3025
1 parent 1fc5831 commit a2d2628

7 files changed

Lines changed: 83 additions & 17 deletions

File tree

‎CHANGES.rst‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,10 @@ API Changes
196196
``parser``, which allows the user to specify which parser
197197
BeautifulSoup should use as a backend. [#2815]
198198

199+
- Add ``FixedWidthTwoLine`` reader to guessing. This will allows to read tables
200+
that a copied from screen output like ``print my_table`` to be read
201+
automatically. Discussed in #3025 and #3099 [#3109]
202+
199203
- ``astropy.io.fits``
200204

201205
- ``astropy.io.misc``
@@ -315,6 +319,10 @@ Bug Fixes
315319
- In the ``CommentedHeader`` the ``data_start`` parameter now defaults to
316320
``0``, which is the first uncommented line. Discussed in #2692. [#3054]
317321

322+
- Position lines in ``FixedWidthTwoLine`` reader could consist of many characters.
323+
Now, only one character in addition to the delimiter is allowed. This bug was
324+
discovered as part of [#3109]
325+
318326
- ``astropy.io.fits``
319327

320328
- Setting ``memmap=True`` in ``fits.open`` and related functions now raises a ValueError if opening a file in memory-mapped mode is impossible. [#2298]

‎astropy/io/ascii/fixedwidth.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,8 @@ def get_cols(self, lines):
133133
# slice col_ends but expects inclusive col_ends on input (for
134134
# more intuitive user interface).
135135
line = self.get_line(lines, position_line)
136+
if len(set(line) - set(self.splitter.delimiter)) != 1:
137+
raise InconsistentTableError('Position line should only contain delimiters and one other character, e.g. "--- ------- ---".')
136138
vals, self.col_starts, col_ends = self.get_fixedwidth_params(line)
137139
self.col_ends = [x - 1 for x in col_ends]
138140

‎astropy/io/ascii/setup_package.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ def get_package_data():
7575
't/whitespace.dat',
7676
't/simple_csv.csv',
7777
't/simple_csv_missing.csv',
78+
't/fixed_width_2_line.txt',
7879
]
7980
}
8081

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Col1 Col2 Col3 Col4
2+
---- --------- ---- ----
3+
1.2 "hello" 1 a
4+
2.4 's worlds 2 2

‎astropy/io/ascii/tests/test_fixedwidth.py‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,10 @@
88
from io import StringIO
99

1010
from ... import ascii
11+
from ..core import InconsistentTableError
1112
from .common import (assert_equal, assert_almost_equal,
1213
setup_function, teardown_function)
13-
14+
from ....tests.helper import pytest
1415

1516
def assert_equal_splitlines(arg1, arg2):
1617
assert_equal(arg1.splitlines(), arg2.splitlines())
@@ -384,6 +385,21 @@ def test_read_twoline_human():
384385
assert_equal(dat[1][1], "'s worlds")
385386

386387

388+
def test_read_twoline_fail():
389+
"""The position line shall consist of only one character in addition to the delimiter."""
390+
table = """
391+
| Col1 | Col2 |
392+
|------|==========|
393+
| 1.2 | "hello" |
394+
| 2.4 | 's worlds|
395+
"""
396+
with pytest.raises(InconsistentTableError) as excinfo:
397+
dat = ascii.read(table, Reader=ascii.FixedWidthTwoLine,
398+
delimiter='|', guess=False)
399+
assert 'Position line should only contain delimiters and one other character' in str(excinfo.value)
400+
401+
402+
387403
def test_write_twoline_normal():
388404
"""Write a table as a normal fixed width table."""
389405
out = StringIO()

‎astropy/io/ascii/tests/test_read.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -760,6 +760,10 @@ def get_testfiles(name=None):
760760
'name': 't/latex2.tex',
761761
'nrows': 3,
762762
'opts': {'Reader': ascii.AASTex}},
763+
{'cols': ('Col1', 'Col2', 'Col3', 'Col4'),
764+
'name': 't/fixed_width_2_line.txt',
765+
'nrows': 2,
766+
'opts': {'Reader': ascii.FixedWidthTwoLine}},
763767
]
764768

765769
try:
@@ -860,9 +864,24 @@ def test_guess_fail():
860864

861865
assert "** To figure out why the table did not read, use guess=False and" in str(err.value)
862866

867+
863868
def test_guessing_file_object():
864869
"""
865870
Test guessing a file object. Fixes #3013 and similar issue noted in #3019.
866871
"""
867872
t = ascii.read(open('t/ipac.dat.bz2', 'rb'))
868873
assert t.colnames == ['ra','dec','sai','v2','sptype']
874+
875+
876+
def test_pformat_roundtrip():
877+
"""Check that the screen output of ``print tab`` can be read. See #3025."""
878+
"""Read a table with empty values and ensure that corresponding entries are masked"""
879+
table = '\n'.join(['a,b,c,d',
880+
'1,3,1.11,1',
881+
'2, 2, 4.0 , ss '])
882+
dat = ascii.read(table)
883+
out = ascii.read(dat.pformat())
884+
assert len(dat) == len(out)
885+
assert dat.colnames == out.colnames
886+
for c in dat.colnames:
887+
assert np.all(dat[c] == out[c])

‎astropy/io/ascii/ui.py‎

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
from . import html
2525
from . import fastbasic
2626
from . import cparser
27+
from . import fixedwidth
2728

2829
from ...table import Table
2930
from ...utils.data import get_readable_fileobj
@@ -180,18 +181,19 @@ def _guess(table, read_kwargs, format, fast_reader):
180181

181182
# Keep a trace of all failed guesses kwarg
182183
failed_kwargs = []
183-
fast_kwargs = []
184+
full_list_guess = _get_guess_kwargs_list(read_kwargs)
184185

185-
first_kwargs = [read_kwargs.copy()]
186186
if fast_reader and format is not None and 'fast_{0}'.format(format) in \
187187
core.FAST_CLASSES:
188188
# If a fast version of the reader is available, try that before the slow version
189189
fast_kwargs = read_kwargs.copy()
190190
fast_kwargs['Reader'] = core.FAST_CLASSES['fast_{0}'.format(format)]
191-
first_kwargs = [fast_kwargs] + first_kwargs
191+
full_list_guess = [fast_kwargs] + full_list_guess
192+
else:
193+
fast_kwargs = None
192194

193195
# First try guessing
194-
for guess_kwargs in first_kwargs + _get_guess_kwargs_list():
196+
for guess_kwargs in full_list_guess:
195197
guess_kwargs_ok = True # guess_kwargs are consistent with user_kwargs?
196198
for key, val in read_kwargs.items():
197199
# Do guess_kwargs.update(read_kwargs) except that if guess_args has
@@ -248,18 +250,32 @@ def _guess(table, read_kwargs, format, fast_reader):
248250
lines.extend(msg)
249251
raise core.InconsistentTableError('\n'.join(lines))
250252

251-
def _get_guess_kwargs_list():
252-
guess_kwargs_list = [dict(Reader=basic.Rdb),
253-
dict(Reader=fastbasic.FastTab),
254-
dict(Reader=basic.Tab),
255-
dict(Reader=cds.Cds),
256-
dict(Reader=daophot.Daophot),
257-
dict(Reader=sextractor.SExtractor),
258-
dict(Reader=ipac.Ipac),
259-
dict(Reader=latex.Latex),
260-
dict(Reader=latex.AASTex),
261-
dict(Reader=html.HTML)
262-
]
253+
def _get_guess_kwargs_list(read_kwargs):
254+
guess_kwargs_list = []
255+
# First try readers that accept the common arguments with the input arguments
256+
# (Unless there are not arguments - we try that in the next step anyway.)
257+
# FixedWidthTwoLine would also be read by Basic, so it needs to come first.
258+
if len(read_kwargs) > 0:
259+
for reader in [fixedwidth.FixedWidthTwoLine,
260+
basic.Basic]:
261+
first_kwargs = read_kwargs.copy()
262+
first_kwargs.update(dict(Reader=reader))
263+
guess_kwargs_list.append(first_kwargs)
264+
# Then try a list of readers with default arguments
265+
guess_kwargs_list.extend([dict(Reader=fixedwidth.FixedWidthTwoLine),
266+
dict(Reader=fastbasic.FastBasic),
267+
dict(Reader=basic.Basic),
268+
dict(Reader=basic.Rdb),
269+
dict(Reader=fastbasic.FastTab),
270+
dict(Reader=basic.Tab),
271+
dict(Reader=cds.Cds),
272+
dict(Reader=daophot.Daophot),
273+
dict(Reader=sextractor.SExtractor),
274+
dict(Reader=ipac.Ipac),
275+
dict(Reader=latex.Latex),
276+
dict(Reader=latex.AASTex),
277+
dict(Reader=html.HTML)
278+
])
263279
for Reader in (basic.CommentedHeader, fastbasic.FastBasic, basic.Basic,
264280
fastbasic.FastNoHeader, basic.NoHeader):
265281
for delimiter in ("|", ",", " ", "\s"):

0 commit comments

Comments
 (0)