Skip to content

Commit ba97203

Browse files
Honor options.transform_source with the native parser
The native (Rust) parser path in mypy/parse.py silently ignored options.transform_source: the transform was only applied in the old-parser branch. Additionally, with the native parser the source was never read in the first place during (parallel) builds, since State.requires_read() did not account for the transform. Apply the transform up front in parse() so both parsers honor it, and make requires_read() return True when a transform is configured, so the source is read (and then transformed) before the native parser sees it. Fixes python#21222
1 parent 8d30064 commit ba97203

3 files changed

Lines changed: 76 additions & 4 deletions

File tree

‎mypy/build.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3145,6 +3145,10 @@ def requires_read(self) -> bool:
31453145
* Non-trivial shadow file mapping.
31463146
* Various "fake" files like namespace packages, etc.
31473147
"""
3148+
if self.options.transform_source is not None:
3149+
# The transform is a Python callable that the native parser cannot
3150+
# apply itself, so the source must be read (and transformed) first.
3151+
return True
31483152
if (
31493153
self.manager.shadow_map
31503154
and self.manager.maybe_swap_for_shadow_path(self.xpath) != self.xpath

‎mypy/parse.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ def parse(
3131
`source` must not be `None` if the old parser is used. The new parser will read and
3232
parse contents from path `fnam` if `source` is `None`.
3333
"""
34+
if options.transform_source is not None and source is not None:
35+
source = options.transform_source(source)
3436
if options.native_parser:
3537
import mypy.nativeparse
3638

@@ -50,8 +52,6 @@ def parse(
5052

5153
if source is None:
5254
raise ValueError("Source cannot be `None` when using the old parser")
53-
if options.transform_source is not None:
54-
source = options.transform_source(source)
5555
import mypy.fastparse
5656

5757
return mypy.fastparse.parse(source, fnam=fnam, module=module, errors=errors, options=options)

‎mypy/test/testparse.py‎

Lines changed: 70 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,22 @@
22

33
from __future__ import annotations
44

5+
import os
6+
import shutil
57
import sys
8+
import tempfile
69

10+
import pytest
711
from pytest import skip
812

9-
from mypy import defaults
13+
from mypy import build, defaults
1014
from mypy.config_parser import parse_mypy_comments
1115
from mypy.errors import CompileError, Errors
16+
from mypy.modulefinder import BuildSource
1217
from mypy.options import Options
1318
from mypy.parse import parse
1419
from mypy.test.data import DataDrivenTestCase, DataSuite
15-
from mypy.test.helpers import assert_string_arrays_equal, find_test_files, parse_options
20+
from mypy.test.helpers import Suite, assert_string_arrays_equal, find_test_files, parse_options
1621
from mypy.test.update_data import update_testcase_output
1722
from mypy.util import get_mypy_comments
1823

@@ -129,3 +134,66 @@ def test_parse_error(testcase: DataDrivenTestCase) -> None:
129134
e.messages,
130135
f"Invalid compiler output ({testcase.file}, line {testcase.line})",
131136
)
137+
138+
139+
class TransformSourceSuite(Suite):
140+
"""Tests for options.transform_source (see #21222).
141+
142+
transform_source is a Python callable, so it cannot be exercised via the
143+
data-driven test cases; these are plain unit tests instead.
144+
"""
145+
146+
def parse_with_transform(self, native_parser: bool) -> int:
147+
"""Parse a trivial module with transform_source set.
148+
149+
Return the number of top-level definitions in the resulting AST.
150+
"""
151+
options = Options()
152+
options.native_parser = native_parser
153+
options.transform_source = lambda source: source + "reveal_type(1)\n"
154+
errors = Errors(options)
155+
tree = parse(
156+
"x = 1\n", fnam="main", module="__main__", errors=errors, options=options, eager=True
157+
)
158+
assert not errors.is_errors()
159+
return len(tree.defs)
160+
161+
def test_transform_source_native_parser(self) -> None:
162+
pytest.importorskip("ast_serialize")
163+
assert Options().native_parser
164+
# The transform appends a statement, so we expect two top-level definitions.
165+
assert self.parse_with_transform(native_parser=True) == 2
166+
167+
def test_transform_source_old_parser(self) -> None:
168+
# The transform appends a statement, so we expect two top-level definitions.
169+
assert self.parse_with_transform(native_parser=False) == 2
170+
171+
def test_transform_source_parallel_build(self) -> None:
172+
"""transform_source must be honored with parallel (native parser) checking.
173+
174+
With more than one file the native parser parses in parallel threads, and
175+
the source must be read (and transformed) instead of letting the parser
176+
use the file on disk directly.
177+
"""
178+
pytest.importorskip("ast_serialize")
179+
assert Options().native_parser
180+
tmpdir = tempfile.mkdtemp(prefix="mypy-test-transform-")
181+
self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True)
182+
for name in ("a.py", "b.py"):
183+
with open(os.path.join(tmpdir, name), "w") as f:
184+
f.write("x: int = 1\n")
185+
options = Options()
186+
# The transform introduces a type error on line 3 of each file.
187+
options.transform_source = lambda source: source + "\nbad: str = 1\n"
188+
options.incremental = False
189+
options.cache_dir = os.devnull
190+
options.show_traceback = True
191+
sources = [
192+
BuildSource(os.path.join(tmpdir, name), name[:-3], None) for name in ("a.py", "b.py")
193+
]
194+
try:
195+
result = build.build(sources=sources, options=options)
196+
messages = result.errors
197+
except CompileError as e:
198+
messages = e.messages
199+
assert len([m for m in messages if ":3: error" in m]) == 2

0 commit comments

Comments
 (0)