Repository navigation
Expand file tree
/
Copy pathbuild.py
More file actions
2726 lines (2354 loc) · 115 KB
/
Copy pathbuild.py
File metadata and controls
2726 lines (2354 loc) · 115 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""Facilities to analyze entire programs, including imported modules.
Parse and analyze the source files of a program in the correct order
(based on file dependencies), and collect the results.
This module only directs a build, which is performed in multiple passes per
file. The individual passes are implemented in separate modules.
The function build() is the main interface to this module.
"""
# TODO: More consistent terminology, e.g. path/fnam, module/id, state/file
import binascii
import collections
import contextlib
from distutils.sysconfig import get_python_lib
import gc
import hashlib
import json
import os.path
import re
import site
import stat
import sys
import time
from os.path import dirname, basename
import errno
from typing import (AbstractSet, Any, cast, Dict, Iterable, Iterator, List,
Mapping, NamedTuple, Optional, Set, Tuple, Union, Callable)
# Can't use TYPE_CHECKING because it's not in the Python 3.5.1 stdlib
MYPY = False
if MYPY:
from typing import Deque
from mypy.nodes import (MODULE_REF, MypyFile, Node, ImportBase, Import, ImportFrom, ImportAll)
from mypy.semanal_pass1 import SemanticAnalyzerPass1
from mypy.semanal import SemanticAnalyzerPass2, apply_semantic_analyzer_patches
from mypy.semanal_pass3 import SemanticAnalyzerPass3
from mypy.checker import TypeChecker
from mypy.indirection import TypeIndirectionVisitor
from mypy.errors import Errors, CompileError, report_internal_error
from mypy.util import DecodeError
from mypy.report import Reports
from mypy import moduleinfo
from mypy.fixup import fixup_module_pass_one, fixup_module_pass_two
from mypy.nodes import Expression
from mypy.options import Options
from mypy.parse import parse
from mypy.stats import dump_type_stats
from mypy.types import Type
from mypy.version import __version__
from mypy.plugin import Plugin, DefaultPlugin, ChainedPlugin
from mypy.defaults import PYTHON3_VERSION_MIN
from mypy.server.deps import get_dependencies
from mypy.fscache import FileSystemCache, FileSystemMetaCache
# Switch to True to produce debug output related to fine-grained incremental
# mode only that is useful during development. This produces only a subset of
# output compared to --verbose output. We use a global flag to enable this so
# that it's easy to enable this when running tests.
DEBUG_FINE_GRAINED = False
PYTHON_EXTENSIONS = ['.pyi', '.py']
Graph = Dict[str, 'State']
def getmtime(name: str) -> int:
return int(os.path.getmtime(name))
# TODO: Get rid of BuildResult. We might as well return a BuildManager.
class BuildResult:
"""The result of a successful build.
Attributes:
manager: The build manager.
files: Dictionary from module name to related AST node.
types: Dictionary from parse tree node to its inferred type.
used_cache: Whether the build took advantage of a pre-existing cache
errors: List of error messages.
"""
def __init__(self, manager: 'BuildManager', graph: Graph) -> None:
self.manager = manager
self.graph = graph
self.files = manager.modules
self.types = manager.all_types # Non-empty for tests only or if dumping deps
self.used_cache = manager.cache_enabled
self.errors = [] # type: List[str] # Filled in by build if desired
class BuildSource:
def __init__(self, path: Optional[str], module: Optional[str],
text: Optional[str]) -> None:
self.path = path
self.module = module or '__main__'
self.text = text
def __repr__(self) -> str:
return '<BuildSource path=%r module=%r has_text=%s>' % (self.path,
self.module,
self.text is not None)
class BuildSourceSet:
"""Efficiently test a file's membership in the set of build sources."""
def __init__(self, sources: List[BuildSource]) -> None:
self.source_text_present = False
self.source_modules = set() # type: Set[str]
self.source_paths = set() # type: Set[str]
for source in sources:
if source.text is not None:
self.source_text_present = True
elif source.path:
self.source_paths.add(source.path)
else:
self.source_modules.add(source.module)
def is_source(self, file: MypyFile) -> bool:
if file.path and file.path in self.source_paths:
return True
elif file._fullname in self.source_modules:
return True
elif file.path is None and self.source_text_present:
return True
else:
return False
def build(sources: List[BuildSource],
options: Options,
alt_lib_path: Optional[str] = None,
bin_dir: Optional[str] = None,
flush_errors: Optional[Callable[[List[str], bool], None]] = None,
fscache: Optional[FileSystemCache] = None,
) -> BuildResult:
"""Analyze a program.
A single call to build performs parsing, semantic analysis and optionally
type checking for the program *and* all imported modules, recursively.
Return BuildResult if successful or only non-blocking errors were found;
otherwise raise CompileError.
If a flush_errors callback is provided, all error messages will be
passed to it and the errors and messages fields of BuildResult and
CompileError (respectively) will be empty. Otherwise those fields will
report any error messages.
Args:
sources: list of sources to build
options: build options
alt_lib_path: an additional directory for looking up library modules
(takes precedence over other directories)
bin_dir: directory containing the mypy script, used for finding data
directories; if omitted, use '.' as the data directory
flush_errors: optional function to flush errors after a file is processed
fscache: optionally a file-system cacher
"""
# If we were not given a flush_errors, we use one that will populate those
# fields for callers that want the traditional API.
messages = []
def default_flush_errors(new_messages: List[str], is_serious: bool) -> None:
messages.extend(new_messages)
flush_errors = flush_errors or default_flush_errors
try:
result = _build(sources, options, alt_lib_path, bin_dir,
flush_errors, fscache)
result.errors = messages
return result
except CompileError as e:
# CompileErrors raised from an errors object carry all of the
# messages that have not been reported out by error streaming.
# Patch it up to contain either none or all none of the messages,
# depending on whether we are flushing errors.
serious = not e.use_stdout
flush_errors(e.messages, serious)
e.messages = messages
raise
def _build(sources: List[BuildSource],
options: Options,
alt_lib_path: Optional[str],
bin_dir: Optional[str],
flush_errors: Callable[[List[str], bool], None],
fscache: Optional[FileSystemCache],
) -> BuildResult:
# This seems the most reasonable place to tune garbage collection.
gc.set_threshold(50000)
data_dir = default_data_dir(bin_dir)
fscache = fscache or FileSystemCache(options.python_version)
# Determine the default module search path.
lib_path = default_lib_path(data_dir,
options.python_version,
custom_typeshed_dir=options.custom_typeshed_dir)
if options.use_builtins_fixtures:
# Use stub builtins (to speed up test cases and to make them easier to
# debug). This is a test-only feature, so assume our files are laid out
# as in the source tree.
root_dir = dirname(dirname(__file__))
lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub'))
else:
for source in sources:
if source.path:
# Include directory of the program file in the module search path.
dir = remove_cwd_prefix_from_path(fscache, dirname(source.path))
if dir not in lib_path:
lib_path.insert(0, dir)
# Do this even if running as a file, for sanity (mainly because with
# multiple builds, there could be a mix of files/modules, so its easier
# to just define the semantics that we always add the current director
# to the lib_path
# TODO: Don't do this in some cases; for motivation see see
# https://github.com/python/mypy/issues/4195#issuecomment-341915031
lib_path.insert(0, os.getcwd())
# Prepend a config-defined mypy path.
lib_path[:0] = options.mypy_path
# Add MYPYPATH environment variable to front of library path, if defined.
lib_path[:0] = mypy_path()
# If provided, insert the caller-supplied extra module path to the
# beginning (highest priority) of the search path.
if alt_lib_path:
lib_path.insert(0, alt_lib_path)
reports = Reports(data_dir, options.report_dirs)
source_set = BuildSourceSet(sources)
errors = Errors(options.show_error_context, options.show_column_numbers)
plugin = load_plugins(options, errors)
# Construct a build manager object to hold state during the build.
#
# Ignore current directory prefix in error messages.
manager = BuildManager(data_dir, lib_path,
ignore_prefix=os.getcwd(),
source_set=source_set,
reports=reports,
options=options,
version_id=__version__,
plugin=plugin,
errors=errors,
flush_errors=flush_errors,
fscache=fscache)
try:
graph = dispatch(sources, manager)
return BuildResult(manager, graph)
finally:
manager.log("Build finished in %.3f seconds with %d modules, and %d errors" %
(time.time() - manager.start_time,
len(manager.modules),
manager.errors.num_messages()))
# Finish the HTML or XML reports even if CompileError was raised.
reports.finish()
def default_data_dir(bin_dir: Optional[str]) -> str:
"""Returns directory containing typeshed directory
Args:
bin_dir: directory containing the mypy script
"""
if not bin_dir:
if os.name == 'nt':
prefixes = [os.path.join(sys.prefix, 'Lib')]
try:
prefixes.append(os.path.join(site.getuserbase(), 'lib'))
except AttributeError:
# getuserbase in not available in virtualenvs
prefixes.append(os.path.join(get_python_lib(), 'lib'))
for parent in prefixes:
data_dir = os.path.join(parent, 'mypy')
if os.path.exists(data_dir):
return data_dir
mypy_package = os.path.dirname(__file__)
parent = os.path.dirname(mypy_package)
if (os.path.basename(parent) == 'site-packages' or
os.path.basename(parent) == 'dist-packages'):
# Installed in site-packages or dist-packages, but invoked with python3 -m mypy;
# __file__ is .../blah/lib/python3.N/site-packages/mypy/build.py
# or .../blah/lib/python3.N/dist-packages/mypy/build.py (Debian)
# or .../blah/lib64/python3.N/dist-packages/mypy/build.py (Gentoo)
# or .../blah/lib/site-packages/mypy/build.py (Windows)
# blah may be a virtualenv or /usr/local. We want .../blah/lib/mypy.
lib = parent
for i in range(2):
lib = os.path.dirname(lib)
if os.path.basename(lib) in ('lib', 'lib32', 'lib64'):
return os.path.join(os.path.dirname(lib), 'lib/mypy')
subdir = os.path.join(parent, 'lib', 'mypy')
if os.path.isdir(subdir):
# If installed via buildout, the __file__ is
# somewhere/mypy/__init__.py and what we want is
# somewhere/lib/mypy.
return subdir
# Default to directory containing this file's parent.
return parent
base = os.path.basename(bin_dir)
dir = os.path.dirname(bin_dir)
if (sys.platform == 'win32' and base.lower() == 'scripts'
and not os.path.isdir(os.path.join(dir, 'typeshed'))):
# Installed, on Windows.
return os.path.join(dir, 'Lib', 'mypy')
elif base == 'scripts':
# Assume that we have a repo check out or unpacked source tarball.
return dir
elif base == 'bin':
# Installed to somewhere (can be under /usr/local or anywhere).
return os.path.join(dir, 'lib', 'mypy')
elif base == 'python3':
# Assume we installed python3 with brew on os x
return os.path.join(os.path.dirname(dir), 'lib', 'mypy')
elif dir.endswith('python-exec'):
# Gentoo uses a python wrapper in /usr/lib to which mypy is a symlink.
return os.path.join(os.path.dirname(dir), 'mypy')
else:
# Don't know where to find the data files!
raise RuntimeError("Broken installation: can't determine base dir")
def mypy_path() -> List[str]:
path_env = os.getenv('MYPYPATH')
if not path_env:
return []
return path_env.split(os.pathsep)
def default_lib_path(data_dir: str,
pyversion: Tuple[int, int],
custom_typeshed_dir: Optional[str]) -> List[str]:
"""Return default standard library search paths."""
# IDEA: Make this more portable.
path = [] # type: List[str]
if custom_typeshed_dir:
typeshed_dir = custom_typeshed_dir
else:
auto = os.path.join(data_dir, 'stubs-auto')
if os.path.isdir(auto):
data_dir = auto
typeshed_dir = os.path.join(data_dir, "typeshed")
if pyversion[0] == 3:
# We allow a module for e.g. version 3.5 to be in 3.4/. The assumption
# is that a module added with 3.4 will still be present in Python 3.5.
versions = ["%d.%d" % (pyversion[0], minor)
for minor in reversed(range(PYTHON3_VERSION_MIN[1], pyversion[1] + 1))]
else:
# For Python 2, we only have stubs for 2.7
versions = ["2.7"]
# E.g. for Python 3.5, try 3.5/, 3.4/, 3.3/, 3/, 2and3/.
for v in versions + [str(pyversion[0]), '2and3']:
for lib_type in ['stdlib', 'third_party']:
stubdir = os.path.join(typeshed_dir, lib_type, v)
if os.path.isdir(stubdir):
path.append(stubdir)
# Add fallback path that can be used if we have a broken installation.
if sys.platform != 'win32':
path.append('/usr/local/lib/mypy')
if not path:
print("Could not resolve typeshed subdirectories. If you are using mypy\n"
"from source, you need to run \"git submodule update --init\".\n"
"Otherwise your mypy install is broken.\nPython executable is located at "
"{0}.\nMypy located at {1}".format(sys.executable, data_dir), file=sys.stderr)
sys.exit(1)
return path
CacheMeta = NamedTuple('CacheMeta',
[('id', str),
('path', str),
('mtime', int),
('size', int),
('hash', str),
('dependencies', List[str]), # names of imported modules
('data_mtime', int), # mtime of data_json
('data_json', str), # path of <id>.data.json
('suppressed', List[str]), # dependencies that weren't imported
('child_modules', List[str]), # all submodules of the given module
('options', Optional[Dict[str, object]]), # build options
# dep_prios and dep_lines are in parallel with
# dependencies + suppressed.
('dep_prios', List[int]),
('dep_lines', List[int]),
('interface_hash', str), # hash representing the public interface
('version_id', str), # mypy version for cache invalidation
('ignore_all', bool), # if errors were ignored
])
# NOTE: dependencies + suppressed == all reachable imports;
# suppressed contains those reachable imports that were prevented by
# silent mode or simply not found.
def cache_meta_from_dict(meta: Dict[str, Any], data_json: str) -> CacheMeta:
sentinel = None # type: Any # Values to be validated by the caller
return CacheMeta(
meta.get('id', sentinel),
meta.get('path', sentinel),
int(meta['mtime']) if 'mtime' in meta else sentinel,
meta.get('size', sentinel),
meta.get('hash', sentinel),
meta.get('dependencies', []),
int(meta['data_mtime']) if 'data_mtime' in meta else sentinel,
data_json,
meta.get('suppressed', []),
meta.get('child_modules', []),
meta.get('options'),
meta.get('dep_prios', []),
meta.get('dep_lines', []),
meta.get('interface_hash', ''),
meta.get('version_id', sentinel),
meta.get('ignore_all', True),
)
# Priorities used for imports. (Here, top-level includes inside a class.)
# These are used to determine a more predictable order in which the
# nodes in an import cycle are processed.
PRI_HIGH = 5 # top-level "from X import blah"
PRI_MED = 10 # top-level "import X"
PRI_LOW = 20 # either form inside a function
PRI_MYPY = 25 # inside "if MYPY" or "if TYPE_CHECKING"
PRI_INDIRECT = 30 # an indirect dependency
PRI_ALL = 99 # include all priorities
def import_priority(imp: ImportBase, toplevel_priority: int) -> int:
"""Compute import priority from an import node."""
if not imp.is_top_level:
# Inside a function
return PRI_LOW
if imp.is_mypy_only:
# Inside "if MYPY" or "if typing.TYPE_CHECKING"
return max(PRI_MYPY, toplevel_priority)
# A regular import; priority determined by argument.
return toplevel_priority
def load_plugins(options: Options, errors: Errors) -> Plugin:
"""Load all configured plugins.
Return a plugin that encapsulates all plugins chained together. Always
at least include the default plugin (it's last in the chain).
"""
default_plugin = DefaultPlugin(options) # type: Plugin
if not options.config_file:
return default_plugin
line = find_config_file_line_number(options.config_file, 'mypy', 'plugins')
if line == -1:
line = 1 # We need to pick some line number that doesn't look too confusing
def plugin_error(message: str) -> None:
errors.report(line, 0, message)
errors.raise_error()
custom_plugins = [] # type: List[Plugin]
errors.set_file(options.config_file, None)
for plugin_path in options.plugins:
# Plugin paths are relative to the config file location.
plugin_path = os.path.join(os.path.dirname(options.config_file), plugin_path)
if not os.path.isfile(plugin_path):
plugin_error("Can't find plugin '{}'".format(plugin_path))
plugin_dir = os.path.dirname(plugin_path)
fnam = os.path.basename(plugin_path)
if not fnam.endswith('.py'):
plugin_error("Plugin '{}' does not have a .py extension".format(fnam))
module_name = fnam[:-3]
import importlib
sys.path.insert(0, plugin_dir)
try:
m = importlib.import_module(module_name)
except Exception:
print('Error importing plugin {}\n'.format(plugin_path))
raise # Propagate to display traceback
finally:
assert sys.path[0] == plugin_dir
del sys.path[0]
if not hasattr(m, 'plugin'):
plugin_error('Plugin \'{}\' does not define entry point function "plugin"'.format(
plugin_path))
try:
plugin_type = getattr(m, 'plugin')(__version__)
except Exception:
print('Error calling the plugin(version) entry point of {}\n'.format(plugin_path))
raise # Propagate to display traceback
if not isinstance(plugin_type, type):
plugin_error(
'Type object expected as the return value of "plugin"; got {!r} (in {})'.format(
plugin_type, plugin_path))
if not issubclass(plugin_type, Plugin):
plugin_error(
'Return value of "plugin" must be a subclass of "mypy.plugin.Plugin" '
'(in {})'.format(plugin_path))
try:
custom_plugins.append(plugin_type(options))
except Exception:
print('Error constructing plugin instance of {}\n'.format(plugin_type.__name__))
raise # Propagate to display traceback
# Custom plugins take precedence over the default plugin.
return ChainedPlugin(options, custom_plugins + [default_plugin])
def find_config_file_line_number(path: str, section: str, setting_name: str) -> int:
"""Return the approximate location of setting_name within mypy config file.
Return -1 if can't determine the line unambiguously.
"""
in_desired_section = False
try:
results = []
with open(path) as f:
for i, line in enumerate(f):
line = line.strip()
if line.startswith('[') and line.endswith(']'):
current_section = line[1:-1].strip()
in_desired_section = (current_section == section)
elif in_desired_section and re.match(r'{}\s*='.format(setting_name), line):
results.append(i + 1)
if len(results) == 1:
return results[0]
except OSError:
pass
return -1
class BuildManager:
"""This class holds shared state for building a mypy program.
It is used to coordinate parsing, import processing, semantic
analysis and type checking. The actual build steps are carried
out by dispatch().
Attributes:
data_dir: Mypy data directory (contains stubs)
lib_path: Library path for looking up modules
modules: Mapping of module ID to MypyFile (shared by the passes)
semantic_analyzer:
Semantic analyzer, pass 2
semantic_analyzer_pass3:
Semantic analyzer, pass 3
all_types: Map {Expression: Type} collected from all modules (tests only)
options: Build options
missing_modules: Set of modules that could not be imported encountered so far
stale_modules: Set of modules that needed to be rechecked (only used by tests)
version_id: The current mypy version (based on commit id when possible)
plugin: Active mypy plugin(s)
errors: Used for reporting all errors
flush_errors: A function for processing errors after each SCC
cache_enabled: Whether cache is being read. This is set based on options,
but is disabled if fine-grained cache loading fails
and after an initial fine-grained load. This doesn't
determine whether we write cache files or not.
stats: Dict with various instrumentation numbers
fscache: A file system cacher
"""
def __init__(self, data_dir: str,
lib_path: List[str],
ignore_prefix: str,
source_set: BuildSourceSet,
reports: Reports,
options: Options,
version_id: str,
plugin: Plugin,
errors: Errors,
flush_errors: Callable[[List[str], bool], None],
fscache: FileSystemCache,
) -> None:
self.start_time = time.time()
self.data_dir = data_dir
self.errors = errors
self.errors.set_ignore_prefix(ignore_prefix)
self.lib_path = tuple(lib_path)
self.source_set = source_set
self.reports = reports
self.options = options
self.version_id = version_id
self.modules = {} # type: Dict[str, MypyFile]
self.missing_modules = set() # type: Set[str]
self.plugin = plugin
self.semantic_analyzer = SemanticAnalyzerPass2(self.modules, self.missing_modules,
lib_path, self.errors, self.plugin)
self.semantic_analyzer_pass3 = SemanticAnalyzerPass3(self.modules, self.errors,
self.semantic_analyzer)
self.all_types = {} # type: Dict[Expression, Type] # Used by tests only
self.indirection_detector = TypeIndirectionVisitor()
self.stale_modules = set() # type: Set[str]
self.rechecked_modules = set() # type: Set[str]
self.plugin = plugin
self.flush_errors = flush_errors
self.cache_enabled = options.incremental and (
not options.fine_grained_incremental or options.use_fine_grained_cache)
self.stats = {} # type: Dict[str, Any] # Values are ints or floats
self.fscache = fscache
self.find_module_cache = FindModuleCache(self.fscache)
def use_fine_grained_cache(self) -> bool:
return self.cache_enabled and self.options.use_fine_grained_cache
def maybe_swap_for_shadow_path(self, path: str) -> str:
if (self.options.shadow_file and
os.path.samefile(self.options.shadow_file[0], path)):
path = self.options.shadow_file[1]
return path
def get_stat(self, path: str) -> os.stat_result:
return self.fscache.stat(self.maybe_swap_for_shadow_path(path))
def all_imported_modules_in_file(self,
file: MypyFile) -> List[Tuple[int, str, int]]:
"""Find all reachable import statements in a file.
Return list of tuples (priority, module id, import line number)
for all modules imported in file; lower numbers == higher priority.
Can generate blocking errors on bogus relative imports.
"""
def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str:
"""Function to correct for relative imports."""
file_id = file.fullname()
rel = imp.relative
if rel == 0:
return imp.id
if os.path.basename(file.path).startswith('__init__.'):
rel -= 1
if rel != 0:
file_id = ".".join(file_id.split(".")[:-rel])
new_id = file_id + "." + imp.id if imp.id else file_id
if not new_id:
self.errors.set_file(file.path, file.name())
self.errors.report(imp.line, 0,
"No parent module -- cannot perform relative import",
blocker=True)
return new_id
res = [] # type: List[Tuple[int, str, int]]
for imp in file.imports:
if not imp.is_unreachable:
if isinstance(imp, Import):
pri = import_priority(imp, PRI_MED)
ancestor_pri = import_priority(imp, PRI_LOW)
for id, _ in imp.ids:
ancestor_parts = id.split(".")[:-1]
ancestors = []
for part in ancestor_parts:
ancestors.append(part)
res.append((ancestor_pri, ".".join(ancestors), imp.line))
res.append((pri, id, imp.line))
elif isinstance(imp, ImportFrom):
cur_id = correct_rel_imp(imp)
pos = len(res)
all_are_submodules = True
# Also add any imported names that are submodules.
pri = import_priority(imp, PRI_MED)
for name, __ in imp.names:
sub_id = cur_id + '.' + name
if self.is_module(sub_id):
res.append((pri, sub_id, imp.line))
else:
all_are_submodules = False
# If all imported names are submodules, don't add
# cur_id as a dependency. Otherwise (i.e., if at
# least one imported name isn't a submodule)
# cur_id is also a dependency, and we should
# insert it *before* any submodules.
if not all_are_submodules:
pri = import_priority(imp, PRI_HIGH)
res.insert(pos, ((pri, cur_id, imp.line)))
elif isinstance(imp, ImportAll):
pri = import_priority(imp, PRI_HIGH)
res.append((pri, correct_rel_imp(imp), imp.line))
return res
def is_module(self, id: str) -> bool:
"""Is there a file in the file system corresponding to module id?"""
return self.find_module_cache.find_module(id, self.lib_path) is not None
def parse_file(self, id: str, path: str, source: str, ignore_errors: bool) -> MypyFile:
"""Parse the source of a file with the given name.
Raise CompileError if there is a parse error.
"""
num_errs = self.errors.num_messages()
tree = parse(source, path, id, self.errors, options=self.options)
tree._fullname = id
self.add_stats(files_parsed=1,
modules_parsed=int(not tree.is_stub),
stubs_parsed=int(tree.is_stub))
if self.errors.num_messages() != num_errs:
self.log("Bailing due to parse errors")
self.errors.raise_error()
self.errors.set_file_ignored_lines(path, tree.ignored_lines, ignore_errors)
return tree
def report_file(self,
file: MypyFile,
type_map: Dict[Expression, Type],
options: Options) -> None:
if self.source_set.is_source(file):
self.reports.file(file, type_map, options)
def log(self, *message: str) -> None:
if self.options.verbosity >= 1:
if message:
print('LOG: ', *message, file=sys.stderr)
else:
print(file=sys.stderr)
sys.stderr.flush()
def log_fine_grained(self, *message: str) -> None:
if self.options.verbosity >= 1:
self.log('fine-grained:', *message)
elif DEBUG_FINE_GRAINED:
# Output log in a simplified format that is quick to browse.
if message:
print(*message, file=sys.stderr)
else:
print(file=sys.stderr)
sys.stderr.flush()
def trace(self, *message: str) -> None:
if self.options.verbosity >= 2:
print('TRACE:', *message, file=sys.stderr)
sys.stderr.flush()
def add_stats(self, **kwds: Any) -> None:
for key, value in kwds.items():
if key in self.stats:
self.stats[key] += value
else:
self.stats[key] = value
def stats_summary(self) -> Mapping[str, object]:
return self.stats
def remove_cwd_prefix_from_path(fscache: FileSystemCache, p: str) -> str:
"""Remove current working directory prefix from p, if present.
Also crawl up until a directory without __init__.py is found.
If the result would be empty, return '.' instead.
"""
cur = os.getcwd()
# Add separator to the end of the path, unless one is already present.
if basename(cur) != '':
cur += os.sep
# Compute root path.
while (p and
(fscache.isfile(os.path.join(p, '__init__.py')) or
fscache.isfile(os.path.join(p, '__init__.pyi')))):
dir, base = os.path.split(p)
if not base:
break
p = dir
# Remove current directory prefix from the path, if present.
if p.startswith(cur):
p = p[len(cur):]
# Avoid returning an empty path; replace that with '.'.
if p == '':
p = '.'
return p
class FindModuleCache:
"""Module finder with integrated cache.
Module locations and some intermediate results are cached internally
and can be cleared with the clear() method.
All file system accesses are performed through a FileSystemCache,
which is not ever cleared by this class. If necessary it must be
cleared by client code.
"""
def __init__(self, fscache: Optional[FileSystemMetaCache] = None) -> None:
self.fscache = fscache or FileSystemMetaCache()
# Cache find_module: (id, lib_path) -> result.
self.results = {} # type: Dict[Tuple[str, Tuple[str, ...]], Optional[str]]
# Cache some repeated work within distinct find_module calls: finding which
# elements of lib_path have even the subdirectory they'd need for the module
# to exist. This is shared among different module ids when they differ only
# in the last component.
self.dirs = {} # type: Dict[Tuple[str, Tuple[str, ...]], List[str]]
def clear(self) -> None:
self.results.clear()
self.dirs.clear()
def _find_module(self, id: str, lib_path: Tuple[str, ...]) -> Optional[str]:
fscache = self.fscache
# If we're looking for a module like 'foo.bar.baz', it's likely that most of the
# many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover
# that only once and cache it for when we look for modules like 'foo.bar.blah'
# that will require the same subdirectory.
components = id.split('.')
dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar'
if (dir_chain, lib_path) not in self.dirs:
dirs = []
for pathitem in lib_path:
# e.g., '/usr/lib/python3.4/foo/bar'
dir = os.path.normpath(os.path.join(pathitem, dir_chain))
if fscache.isdir(dir):
dirs.append(dir)
self.dirs[dir_chain, lib_path] = dirs
candidate_base_dirs = self.dirs[dir_chain, lib_path]
# If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now
# contains just the subdirectories 'foo/bar' that actually exist under the
# elements of lib_path. This is probably much shorter than lib_path itself.
# Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories.
seplast = os.sep + components[-1] # so e.g. '/baz'
sepinit = os.sep + '__init__'
for base_dir in candidate_base_dirs:
base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz'
# Prefer package over module, i.e. baz/__init__.py* over baz.py*.
for extension in PYTHON_EXTENSIONS:
path = base_path + sepinit + extension
if fscache.isfile_case(path) and verify_module(fscache, id, path):
return path
# No package, look for module.
for extension in PYTHON_EXTENSIONS:
path = base_path + extension
if fscache.isfile_case(path) and verify_module(fscache, id, path):
return path
return None
def find_module(self, id: str, lib_path_arg: Iterable[str]) -> Optional[str]:
"""Return the path of the module source file, or None if not found."""
lib_path = tuple(lib_path_arg)
key = (id, lib_path)
if key not in self.results:
self.results[key] = self._find_module(id, lib_path)
return self.results[key]
def find_modules_recursive(self, module: str, lib_path: List[str]) -> List[BuildSource]:
module_path = self.find_module(module, lib_path)
if not module_path:
return []
result = [BuildSource(module_path, module, None)]
if module_path.endswith(('__init__.py', '__init__.pyi')):
# Subtle: this code prefers the .pyi over the .py if both
# exists, and also prefers packages over modules if both x/
# and x.py* exist. How? We sort the directory items, so x
# comes before x.py and x.pyi. But the preference for .pyi
# over .py is encoded in find_module(); even though we see
# x.py before x.pyi, find_module() will find x.pyi first. We
# use hits to avoid adding it a second time when we see x.pyi.
# This also avoids both x.py and x.pyi when x/ was seen first.
hits = set() # type: Set[str]
for item in sorted(self.fscache.listdir(os.path.dirname(module_path))):
abs_path = os.path.join(os.path.dirname(module_path), item)
if os.path.isdir(abs_path) and \
(os.path.isfile(os.path.join(abs_path, '__init__.py')) or
os.path.isfile(os.path.join(abs_path, '__init__.pyi'))):
hits.add(item)
result += self.find_modules_recursive(module + '.' + item, lib_path)
elif item != '__init__.py' and item != '__init__.pyi' and \
item.endswith(('.py', '.pyi')):
mod = item.split('.')[0]
if mod not in hits:
hits.add(mod)
result += self.find_modules_recursive(module + '.' + mod, lib_path)
return result
def verify_module(fscache: FileSystemMetaCache, id: str, path: str) -> bool:
"""Check that all packages containing id have a __init__ file."""
if path.endswith(('__init__.py', '__init__.pyi')):
path = dirname(path)
for i in range(id.count('.')):
path = dirname(path)
if not any(fscache.isfile_case(os.path.join(path, '__init__{}'.format(extension)))
for extension in PYTHON_EXTENSIONS):
return False
return True
def get_cache_names(id: str, path: str, manager: BuildManager) -> Tuple[str, str]:
"""Return the file names for the cache files.
Args:
id: module ID
path: module path (used to recognize packages)
cache_dir: cache directory
pyversion: Python version (major, minor)
Returns:
A tuple with the file names to be used for the meta JSON and the
data JSON, respectively.
"""
cache_dir = manager.options.cache_dir
pyversion = manager.options.python_version
prefix = os.path.join(cache_dir, '%d.%d' % pyversion, *id.split('.'))
is_package = os.path.basename(path).startswith('__init__.py')
if is_package:
prefix = os.path.join(prefix, '__init__')
return (prefix + '.meta.json', prefix + '.data.json')
def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[CacheMeta]:
"""Find cache data for a module.
Args:
id: module ID
path: module path
manager: the build manager (for pyversion, log/trace, and build options)
Returns:
A CacheMeta instance if the cache data was found and appears
valid; otherwise None.
"""
# TODO: May need to take more build options into account
meta_json, data_json = get_cache_names(id, path, manager)
manager.trace('Looking for {} at {}'.format(id, meta_json))
try:
with open(meta_json, 'r') as f:
meta_str = f.read()
manager.trace('Meta {} {}'.format(id, meta_str.rstrip()))
meta = json.loads(meta_str) # TODO: Errors
except IOError:
manager.log('Could not load cache for {}: could not find {}'.format(id, meta_json))
return None
if not isinstance(meta, dict):
manager.log('Could not load cache for {}: meta cache is not a dict: {}'
.format(id, repr(meta)))
return None
m = cache_meta_from_dict(meta, data_json)
# Don't check for path match, that is dealt with in validate_meta().
if (m.id != id or
m.mtime is None or m.size is None or
m.dependencies is None or m.data_mtime is None):
manager.log('Metadata abandoned for {}: attributes are missing'.format(id))
return None
# Ignore cache if generated by an older mypy version.
if ((m.version_id != manager.version_id and not manager.options.skip_version_check)
or m.options is None
or len(m.dependencies) + len(m.suppressed) != len(m.dep_prios)
or len(m.dependencies) + len(m.suppressed) != len(m.dep_lines)):
manager.log('Metadata abandoned for {}: new attributes are missing'.format(id))
return None
# Ignore cache if (relevant) options aren't the same.
# Note that it's fine to mutilate cached_options since it's only used here.
cached_options = m.options
current_options = manager.options.clone_for_module(id).select_options_affecting_cache()
if manager.options.quick_and_dirty:
# In quick_and_dirty mode allow non-quick_and_dirty cache files.
cached_options['quick_and_dirty'] = True
if manager.options.skip_version_check:
# When we're lax about version we're also lax about platform.
cached_options['platform'] = current_options['platform']
if 'debug_cache' in cached_options:
# Older versions included debug_cache, but it's silly to compare it.
del cached_options['debug_cache']
if cached_options != current_options:
manager.log('Metadata abandoned for {}: options differ'.format(id))
if manager.options.verbosity >= 2:
for key in sorted(set(cached_options) | set(current_options)):
if cached_options.get(key) != current_options.get(key):
manager.trace(' {}: {} != {}'
.format(key, cached_options.get(key), current_options.get(key)))
return None
manager.add_stats(fresh_metas=1)
return m
def random_string() -> str: