Skip to content

Commit 4c5c588

Browse files
authored
Avoid eagerly reading sources for singe-file batches (#21996)
Fixes #21514 Currently, `parse_file()` (that is still used with parallel parsing if there is only one file in batch) eagerly reads file, computes hash, and parses inline configuration. This is not needed, since Ruff parser already does this. This PR unifies the logic in `parse_file()` with `parse_all()` that already avoids this overhead. Note that few tests changed because Ruff parser ignores `# coding:` declaration and always uses UTF-8.
1 parent 7e00ff0 commit 4c5c588

3 files changed

Lines changed: 38 additions & 18 deletions

File tree

‎mypy/build.py‎

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1032,9 +1032,7 @@ def parse_all(self, states: list[State], post_parse: bool = True) -> None:
10321032

10331033
parallel_states = []
10341034
for state in states:
1035-
if not self.fscache.exists(state.xpath, real_only=True) or (
1036-
self.shadow_map and self.maybe_swap_for_shadow_path(state.xpath) != state.xpath
1037-
):
1035+
if state.requires_read():
10381036
state.source = state.get_source()
10391037
if state.tree is not None:
10401038
# The file was already parsed.
@@ -1077,13 +1075,6 @@ def parse_all(self, states: list[State], post_parse: bool = True) -> None:
10771075
for state in parallel_states:
10781076
assert state.tree is not None
10791077
if state in parallel_parsed_states_set:
1080-
if state.tree.raw_data is not None:
1081-
# source_hash was already extracted above, but raw_data
1082-
# may have been preserved for workers (imports_only=True).
1083-
pass
1084-
elif state.source_hash is None:
1085-
# At least namespace packages may not have source.
1086-
state.get_source()
10871078
state.early_errors = list(self.errors.error_info_map.get(state.xpath, []))
10881079
state.semantic_analysis_pass1()
10891080
self.ast_cache[state.id] = (state.tree, state.early_errors, state.source_hash)
@@ -3146,6 +3137,26 @@ def fix_cross_refs(self) -> None:
31463137

31473138
# Methods for processing modules from source code.
31483139

3140+
def requires_read(self) -> bool:
3141+
"""Do we need to force reading the file source before parsing?
3142+
3143+
Ruff parser can do all the work faster, but there are situation where
3144+
we cannot delegate the work to it (because it needs actual file on disk):
3145+
* Non-trivial shadow file mapping.
3146+
* Various "fake" files like namespace packages, etc.
3147+
"""
3148+
if (
3149+
self.manager.shadow_map
3150+
and self.manager.maybe_swap_for_shadow_path(self.xpath) != self.xpath
3151+
):
3152+
return True
3153+
return (
3154+
self.manager.fscache.isdir(self.xpath)
3155+
or not self.manager.fscache.exists(self.xpath, real_only=True)
3156+
# For a better error in case one tries to parse a .pyd file (which is a DLL).
3157+
or self.xpath.endswith(".pyd")
3158+
)
3159+
31493160
def get_source(self) -> str:
31503161
"""Get module source and parse inline mypy configurations."""
31513162
manager = self.manager
@@ -3211,10 +3222,10 @@ def parse_file(self, *, temporary: bool = False, raw_data: FileRawData | None =
32113222
# The file was already parsed.
32123223
return
32133224

3214-
if raw_data is None:
3225+
if not self.options.native_parser or raw_data is None and self.requires_read():
32153226
source = self.get_source()
32163227
else:
3217-
source = ""
3228+
source = None
32183229
manager = self.manager
32193230
# Can we reuse a previously parsed AST? This avoids redundant work in daemon.
32203231
if self.id not in manager.ast_cache:
@@ -3235,6 +3246,9 @@ def parse_file(self, *, temporary: bool = False, raw_data: FileRawData | None =
32353246
# New parser returns serialized trees that need to be de-serialized.
32363247
if self.tree.raw_data is not None:
32373248
assert raw_data is None
3249+
# Same as above, apply inline configuration first.
3250+
self.source_hash = self.tree.raw_data.source_hash
3251+
self.apply_inline_configuration(self.tree.raw_data.mypy_comments)
32383252
self.tree = load_from_raw(
32393253
self.xpath,
32403254
self.id,

‎test-data/unit/cmdline.test‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,11 +112,17 @@ undef
112112
sub.pkg contains __init__.py but is not a valid Python package name
113113
== Return code: 2
114114

115-
[case testBadFileEncoding]
115+
[case testBadFileEncodingIgnored]
116116
# cmd: mypy a.py
117117
[file a.py]
118118
# coding: uft-8
119119
[out]
120+
121+
[case testBadFileEncodingOldParser]
122+
# cmd: mypy a.py --no-native-parser
123+
[file a.py]
124+
# coding: uft-8
125+
[out]
120126
a.py: error: Cannot decode file: unknown encoding: uft-8
121127
== Return code: 2
122128

‎test-data/unit/fine-grained-blockers.test‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -490,7 +490,7 @@ a.py:1: error: Simple statements must be separated by newlines or semicolons
490490
b.py:2: error: Incompatible return value type (got "str", expected "int")
491491
==
492492

493-
[case testDecodeErrorBlocker1-posix]
493+
[case testDecodeErrorNonBlocker1-posix]
494494
import a
495495
a.f(1)
496496
[file a.py]
@@ -502,11 +502,11 @@ def f(x: int) -> None: ...
502502
def f(x: str) -> None: ...
503503
[out]
504504
==
505-
a.py: error: Cannot decode file: 'ascii' codec can't decode byte 0xc3 in position 16: ordinal not in range(128)
505+
main:2: error: Module has no attribute "f"
506506
==
507507
main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
508508

509-
[case testDecodeErrorBlocker2-windows]
509+
[case testDecodeErrorNonBlocker2-windows]
510510
import a
511511
a.f(1)
512512
[file a.py]
@@ -518,11 +518,11 @@ def f(x: int) -> None: ...
518518
def f(x: str) -> None: ...
519519
[out]
520520
==
521-
a.py: error: Cannot decode file: 'ascii' codec can't decode byte 0xc3 in position 17: ordinal not in range(128)
521+
main:2: error: Module has no attribute "f"
522522
==
523523
main:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
524524

525-
[case testDecodeErrorBlockerOnInitialRun-posix-skip]
525+
[case testDecodeErrorNonBlockerOnInitialRun-posix]
526526
# Note that there's no test variant for Windows, since the above Windows test case is good enough.
527527
import a
528528
a.f(1)

0 commit comments

Comments
 (0)