Skip to content
This repository was archived by the owner on Jul 11, 2022. It is now read-only.

Commit 9138a75

Browse files
committed
Fix parsing of unaligned standalone comments
Fixes pytest-dev#99 Fixes pytest-dev#112
1 parent 1909d8c commit 9138a75

4 files changed

Lines changed: 31 additions & 5 deletions

File tree

‎README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,10 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
491491

492492
## Change Log
493493

494+
### 18.4a2 (unreleased)
495+
496+
* fixed parsing of unaligned standalone comments (#99, #112)
497+
494498
### 18.4a1
495499

496500
* added `--quiet` (#78)

‎blib2to3/pgen2/driver.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,12 @@ def parse_tokens(self, tokens, debug=False):
7777
self.logger.debug("Stop.")
7878
break
7979
prefix = ""
80+
if type == token.INDENT:
81+
if _prefix.startswith(value):
82+
# Don't double-indent. Since we're delaying the prefix that
83+
# would normally belong to INDENT, we need to put the value
84+
# at the end versus at the beginning.
85+
_prefix = _prefix[len(value):] + value
8086
if type in {token.INDENT, token.DEDENT}:
8187
prefix = _prefix
8288
lineno, column = end

‎blib2to3/pgen2/tokenize.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -412,10 +412,6 @@ def generate_tokens(readline):
412412
yield (NL, line[pos:], (lnum, pos), (lnum, len(line)), line)
413413
continue
414414

415-
if column > indents[-1]: # count indents
416-
indents.append(column)
417-
yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
418-
419415
if line[pos] == '#': # skip comments
420416
comment_token = line[pos:].rstrip('\r\n')
421417
nl_pos = pos + len(comment_token)
@@ -425,6 +421,10 @@ def generate_tokens(readline):
425421
(lnum, nl_pos), (lnum, len(line)), line)
426422
continue
427423

424+
if column > indents[-1]: # count indents
425+
indents.append(column)
426+
yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
427+
428428
while column < indents[-1]: # count dedents
429429
if column not in indents:
430430
raise IndentationError(

‎tests/comments2.py‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,14 @@
2323
'Generator',
2424
]
2525

26+
if 'PYTHON' in os.environ:
27+
add_compiler(compiler_from_env())
28+
else:
29+
# for compiler in compilers.values():
30+
# add_compiler(compiler)
31+
add_compiler(compilers[(7.0, 32)])
32+
# add_compiler(compilers[(7.1, 64)])
33+
2634
# Comment before function.
2735
def inline_comments_in_brackets_ruin_everything():
2836
if typedargslist:
@@ -97,7 +105,7 @@ def inline_comments_in_brackets_ruin_everything():
97105
# and round and round we go
98106
# and round and round we go
99107

100-
# let's return
108+
# let's return
101109
return Node(
102110
syms.simple_stmt,
103111
[
@@ -144,6 +152,14 @@ def inline_comments_in_brackets_ruin_everything():
144152
"Generator",
145153
]
146154

155+
if "PYTHON" in os.environ:
156+
add_compiler(compiler_from_env())
157+
else:
158+
# for compiler in compilers.values():
159+
# add_compiler(compiler)
160+
add_compiler(compilers[(7.0, 32)])
161+
# add_compiler(compilers[(7.1, 64)])
162+
147163
# Comment before function.
148164

149165

0 commit comments

Comments
 (0)