Skip to content

Commit 5d0f923

Browse files
authored
Merge pull request #26 from xovishnukosuri/fix/underscore-removal-in-bilingual-atf
Strip underscores from tokens in bilingual ATF texts
2 parents 5c640e4 + eb21e69 commit 5d0f923

2 files changed

Lines changed: 72 additions & 21 deletions

File tree

‎atf2conll_convertor/convertor.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,5 +134,7 @@ def __parse(self, linenumber, line):
134134
form = cleanTokensToProcess[i]
135135
form_clean = form.replace('#', '').replace('[', '').replace(']', '').replace('<', '').replace('>',
136136
'').replace(
137-
'!', '').replace('?', '')
137+
'!', '').replace('?', '').replace('_', '')
138+
if not form_clean:
139+
continue
138140
self.tokens.append((ID, form_clean))

‎tests/test_cli.py‎

Lines changed: 69 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,78 @@
1+
import os
2+
import tempfile
13
import pytest
2-
from click.testing import CliRunner
3-
from atf2conll_convertor import cli
4+
from atf2conll_convertor.convertor import ATFCONLConvertor
45

56

6-
@pytest.fixture
7-
def runner():
8-
return CliRunner()
7+
ATF_SIMPLE = """\
8+
&P101049 = AnOr 01, 058
9+
#atf: lang sux
10+
@tablet
11+
@obverse
12+
1. 2(disz) ma2 gur
13+
@reverse
14+
1. a-pi4-sal4{ki}-ta
15+
"""
916

17+
ATF_BILINGUAL = """\
18+
&P222243 = Test Bilingual
19+
#atf: lang akk
20+
@tablet
21+
@obverse
22+
1. a-na _lu2 dumu_-szu2 iqbi
23+
2. _sze gur_ ba-an-si
24+
"""
1025

11-
def test_cli(runner):
12-
result = runner.invoke(cli.main)
13-
assert result.exit_code == 0
14-
assert not result.exception
15-
assert result.output.strip() == 'Hello, world.'
1626

27+
def make_convertor(atf_content):
28+
with tempfile.NamedTemporaryFile(mode='w', suffix='.atf', delete=False, encoding='utf-8') as f:
29+
f.write(atf_content)
30+
fname = f.name
31+
outfolder = os.path.join(os.path.dirname(fname), 'output')
32+
os.makedirs(outfolder, exist_ok=True)
33+
return fname, ATFCONLConvertor(fname, verbose=False)
1734

18-
def test_cli_with_option(runner):
19-
result = runner.invoke(cli.main, ['--as-cowboy'])
20-
assert not result.exception
21-
assert result.exit_code == 0
22-
assert result.output.strip() == 'Howdy, world.'
2335

36+
def test_basic_conversion():
37+
fname, c = make_convertor(ATF_SIMPLE)
38+
try:
39+
c.convert()
40+
assert c.tokens == [
41+
('o.1.1', '2(disz)'),
42+
('o.1.2', 'ma2'),
43+
('o.1.3', 'gur'),
44+
('r.1.1', 'a-pi4-sal4{ki}-ta'),
45+
]
46+
finally:
47+
os.unlink(fname)
2448

25-
def test_cli_with_arg(runner):
26-
result = runner.invoke(cli.main, ['Jayanth'])
27-
assert result.exit_code == 0
28-
assert not result.exception
29-
assert result.output.strip() == 'Hello, Jayanth.'
49+
50+
def test_underscore_removal():
51+
fname, c = make_convertor(ATF_BILINGUAL)
52+
try:
53+
c.convert()
54+
forms = [form for _, form in c.tokens]
55+
for form in forms:
56+
assert '_' not in form, f'Underscore found in output token: {form!r}'
57+
assert forms == ['a-na', 'lu2', 'dumu-szu2', 'iqbi', 'sze', 'gur', 'ba-an-si']
58+
finally:
59+
os.unlink(fname)
60+
61+
62+
def test_special_chars_removed():
63+
atf = """\
64+
&P000001 = Test
65+
#atf: lang sux
66+
@tablet
67+
@obverse
68+
1. sze# [ba]-hul <du> mu!
69+
"""
70+
fname, c = make_convertor(atf)
71+
try:
72+
c.convert()
73+
forms = [form for _, form in c.tokens]
74+
for form in forms:
75+
for ch in ('#', '[', ']', '<', '>', '!', '?'):
76+
assert ch not in form, f'{ch!r} found in output token: {form!r}'
77+
finally:
78+
os.unlink(fname)

0 commit comments

Comments
 (0)