"""Facilities to analyze entire programs, including imported modules.

Parse and analyze the source files of a program in the correct order

(based on file dependencies), and collect the results.

This module only directs a build, which is performed in multiple passes per

file. The individual passes are implemented in separate modules.

The function build() is the main interface to this module.

"""

# TODO: More consistent terminology, e.g. path/fnam, module/id, state/file

from __future__ import annotations

import collections

import contextlib

import errno

import gc

import json

import os

import platform

import re

import stat

import sys

import time

import types

from typing import (

TYPE_CHECKING,

AbstractSet,

Any,

Callable,

ClassVar,

Dict,

Iterable,

Iterator,

Mapping,

NamedTuple,

NoReturn,

Sequence,

TextIO,

TypeVar,

)

from typing_extensions import Final, TypeAlias as _TypeAlias

from mypy_extensions import TypedDict

import mypy.semanal_main

from mypy.checker import TypeChecker

from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error

from mypy.indirection import TypeIndirectionVisitor

from mypy.messages import MessageBuilder

from mypy.nodes import Import, ImportAll, ImportBase, ImportFrom, MypyFile, SymbolTable, TypeInfo

from mypy.partially_defined import PossiblyUndefinedVariableVisitor

from mypy.semanal import SemanticAnalyzer

from mypy.semanal_pass1 import SemanticAnalyzerPreAnalysis

from mypy.util import (

DecodeError,

decode_python_encoding,

get_mypy_comments,

get_top_two_prefixes,

hash_digest,

is_stub_package_file,

is_sub_path,

is_typeshed_file,

module_prefix,

read_py_file,

time_ref,

time_spent_us,

)

if TYPE_CHECKING:

from mypy.report import Reports # Avoid unconditional slow import

from mypy import errorcodes as codes

from mypy.config_parser import parse_mypy_comments

from mypy.fixup import fixup_module

from mypy.freetree import free_tree

from mypy.fscache import FileSystemCache

from mypy.metastore import FilesystemMetadataStore, MetadataStore, SqliteMetadataStore

from mypy.modulefinder import (

BuildSource as BuildSource,

BuildSourceSet as BuildSourceSet,

FindModuleCache,

ModuleNotFoundReason,

ModuleSearchResult,

SearchPaths,

compute_search_paths,

)

from mypy.nodes import Expression

from mypy.options import Options

from mypy.parse import parse

from mypy.plugin import ChainedPlugin, Plugin, ReportConfigContext

from mypy.plugins.default import DefaultPlugin

from mypy.renaming import LimitedVariableRenameVisitor, VariableRenameVisitor

from mypy.stats import dump_type_stats

from mypy.stubinfo import (

is_legacy_bundled_package,

legacy_bundled_packages,

non_bundled_packages,

stub_package_name,

)

from mypy.types import Type

from mypy.typestate import reset_global_state, type_state

from mypy.version import __version__

# Switch to True to produce debug output related to fine-grained incremental

# mode only that is useful during development. This produces only a subset of

# output compared to --verbose output. We use a global flag to enable this so

# that it's easy to enable this when running tests.

DEBUG_FINE_GRAINED: Final = False

# These modules are special and should always come from typeshed.

CORE_BUILTIN_MODULES: Final = {

"builtins",

"typing",

"types",

"typing_extensions",

"mypy_extensions",

"_importlib_modulespec",

"sys",

"abc",

}

Graph: _TypeAlias = Dict[str, "State"]

# TODO: Get rid of BuildResult. We might as well return a BuildManager.

class BuildResult:

"""The result of a successful build.

Attributes:

manager: The build manager.

files: Dictionary from module name to related AST node.

types: Dictionary from parse tree node to its inferred type.

used_cache: Whether the build took advantage of a pre-existing cache

errors: List of error messages.

"""

def __init__(self, manager: BuildManager, graph: Graph) -> None:

self.manager = manager

self.graph = graph

self.files = manager.modules

self.types = manager.all_types # Non-empty if export_types True in options

self.used_cache = manager.cache_enabled

self.errors: list[str] = [] # Filled in by build if desired

def build(

sources: list[BuildSource],

options: Options,

alt_lib_path: str | None = None,

flush_errors: Callable[[list[str], bool], None] | None = None,

fscache: FileSystemCache | None = None,

stdout: TextIO | None = None,

stderr: TextIO | None = None,

extra_plugins: Sequence[Plugin] | None = None,

) -> BuildResult:

"""Analyze a program.

A single call to build performs parsing, semantic analysis and optionally

type checking for the program *and* all imported modules, recursively.

Return BuildResult if successful or only non-blocking errors were found;

otherwise raise CompileError.

If a flush_errors callback is provided, all error messages will be

passed to it and the errors and messages fields of BuildResult and

CompileError (respectively) will be empty. Otherwise those fields will

report any error messages.

Args:

sources: list of sources to build

options: build options

alt_lib_path: an additional directory for looking up library modules

(takes precedence over other directories)

flush_errors: optional function to flush errors after a file is processed

fscache: optionally a file-system cacher

"""

# If we were not given a flush_errors, we use one that will populate those

# fields for callers that want the traditional API.

messages = []

def default_flush_errors(new_messages: list[str], is_serious: bool) -> None:

messages.extend(new_messages)

flush_errors = flush_errors or default_flush_errors

stdout = stdout or sys.stdout

stderr = stderr or sys.stderr

extra_plugins = extra_plugins or []

try:

result = _build(

sources, options, alt_lib_path, flush_errors, fscache, stdout, stderr, extra_plugins

)

result.errors = messages

return result

except CompileError as e:

# CompileErrors raised from an errors object carry all of the

# messages that have not been reported out by error streaming.

# Patch it up to contain either none or all none of the messages,

# depending on whether we are flushing errors.

serious = not e.use_stdout

flush_errors(e.messages, serious)

e.messages = messages

raise

def _build(

sources: list[BuildSource],

options: Options,

alt_lib_path: str | None,

flush_errors: Callable[[list[str], bool], None],

fscache: FileSystemCache | None,

stdout: TextIO,

stderr: TextIO,

extra_plugins: Sequence[Plugin],

) -> BuildResult:

if platform.python_implementation() == "CPython":

# This seems the most reasonable place to tune garbage collection.

gc.set_threshold(150 * 1000)

data_dir = default_data_dir()

fscache = fscache or FileSystemCache()

search_paths = compute_search_paths(sources, options, data_dir, alt_lib_path)

reports = None

if options.report_dirs:

# Import lazily to avoid slowing down startup.

from mypy.report import Reports

reports = Reports(data_dir, options.report_dirs)

source_set = BuildSourceSet(sources)

cached_read = fscache.read

errors = Errors(

options.show_error_context,

options.show_column_numbers,

options.hide_error_codes,

options.pretty,

options.show_error_end,

lambda path: read_py_file(path, cached_read),

options.show_absolute_path,

options.many_errors_threshold,

options,

)

plugin, snapshot = load_plugins(options, errors, stdout, extra_plugins)

# Add catch-all .gitignore to cache dir if we created it

cache_dir_existed = os.path.isdir(options.cache_dir)

# Construct a build manager object to hold state during the build.

#

# Ignore current directory prefix in error messages.

manager = BuildManager(

data_dir,

search_paths,

ignore_prefix=os.getcwd(),

source_set=source_set,

reports=reports,

options=options,

version_id=__version__,

plugin=plugin,

plugins_snapshot=snapshot,

errors=errors,

flush_errors=flush_errors,

fscache=fscache,

stdout=stdout,

stderr=stderr,

)

manager.trace(repr(options))

reset_global_state()

try:

graph = dispatch(sources, manager, stdout)

if not options.fine_grained_incremental:

type_state.reset_all_subtype_caches()

if options.timing_stats is not None:

dump_timing_stats(options.timing_stats, graph)

if options.line_checking_stats is not None:

dump_line_checking_stats(options.line_checking_stats, graph)

return BuildResult(manager, graph)

finally:

t0 = time.time()

manager.metastore.commit()

manager.add_stats(cache_commit_time=time.time() - t0)

manager.log(

"Build finished in %.3f seconds with %d modules, and %d errors"

% (

time.time() - manager.start_time,

len(manager.modules),

manager.errors.num_messages(),

)

)

manager.dump_stats()

if reports is not None:

# Finish the HTML or XML reports even if CompileError was raised.

reports.finish()

if not cache_dir_existed and os.path.isdir(options.cache_dir):

add_catch_all_gitignore(options.cache_dir)

exclude_from_backups(options.cache_dir)

if os.path.isdir(options.cache_dir):

record_missing_stub_packages(options.cache_dir, manager.missing_stub_packages)

def default_data_dir() -> str:

"""Returns directory containing typeshed directory."""

return os.path.dirname(__file__)

def normpath(path: str, options: Options) -> str:

"""Convert path to absolute; but to relative in bazel mode.

(Bazel's distributed cache doesn't like filesystem metadata to

end up in output files.)

"""

# TODO: Could we always use relpath? (A worry in non-bazel

# mode would be that a moved file may change its full module

# name without changing its size, mtime or hash.)

if options.bazel:

return os.path.relpath(path)

else:

return os.path.abspath(path)

class CacheMeta(NamedTuple):

id: str

path: str

mtime: int

size: int

hash: str

dependencies: list[str] # names of imported modules

data_mtime: int # mtime of data_json

data_json: str # path of <id>.data.json

suppressed: list[str] # dependencies that weren't imported

options: dict[str, object] | None # build options

# dep_prios and dep_lines are in parallel with dependencies + suppressed

dep_prios: list[int]

dep_lines: list[int]

interface_hash: str # hash representing the public interface

version_id: str # mypy version for cache invalidation

ignore_all: bool # if errors were ignored

plugin_data: Any # config data from plugins

# NOTE: dependencies + suppressed == all reachable imports;

# suppressed contains those reachable imports that were prevented by

# silent mode or simply not found.

# Metadata for the fine-grained dependencies file associated with a module.

FgDepMeta = TypedDict("FgDepMeta", {"path": str, "mtime": int})

def cache_meta_from_dict(meta: dict[str, Any], data_json: str) -> CacheMeta:

"""Build a CacheMeta object from a json metadata dictionary

Args:

meta: JSON metadata read from the metadata cache file

data_json: Path to the .data.json file containing the AST trees

"""

sentinel: Any = None # Values to be validated by the caller

return CacheMeta(

meta.get("id", sentinel),

meta.get("path", sentinel),

int(meta["mtime"]) if "mtime" in meta else sentinel,

meta.get("size", sentinel),

meta.get("hash", sentinel),

meta.get("dependencies", []),

int(meta["data_mtime"]) if "data_mtime" in meta else sentinel,

data_json,

meta.get("suppressed", []),

meta.get("options"),

meta.get("dep_prios", []),

meta.get("dep_lines", []),

meta.get("interface_hash", ""),

meta.get("version_id", sentinel),

meta.get("ignore_all", True),

meta.get("plugin_data", None),

)

# Priorities used for imports. (Here, top-level includes inside a class.)

# These are used to determine a more predictable order in which the

# nodes in an import cycle are processed.

PRI_HIGH: Final = 5 # top-level "from X import blah"

PRI_MED: Final = 10 # top-level "import X"

PRI_LOW: Final = 20 # either form inside a function

PRI_MYPY: Final = 25 # inside "if MYPY" or "if TYPE_CHECKING"

PRI_INDIRECT: Final = 30 # an indirect dependency

PRI_ALL: Final = 99 # include all priorities

def import_priority(imp: ImportBase, toplevel_priority: int) -> int:

"""Compute import priority from an import node."""

if not imp.is_top_level:

# Inside a function

return PRI_LOW

if imp.is_mypy_only:

# Inside "if MYPY" or "if typing.TYPE_CHECKING"

return max(PRI_MYPY, toplevel_priority)

# A regular import; priority determined by argument.

return toplevel_priority

def load_plugins_from_config(

options: Options, errors: Errors, stdout: TextIO

) -> tuple[list[Plugin], dict[str, str]]:

"""Load all configured plugins.

Return a list of all the loaded plugins from the config file.

The second return value is a snapshot of versions/hashes of loaded user

plugins (for cache validation).

"""

import importlib

snapshot: dict[str, str] = {}

if not options.config_file:

return [], snapshot

line = find_config_file_line_number(options.config_file, "mypy", "plugins")

if line == -1:

line = 1 # We need to pick some line number that doesn't look too confusing

def plugin_error(message: str) -> NoReturn:

errors.report(line, 0, message)

errors.raise_error(use_stdout=False)

custom_plugins: list[Plugin] = []

errors.set_file(options.config_file, None, options)

for plugin_path in options.plugins:

func_name = "plugin"

plugin_dir: str | None = None

if ":" in os.path.basename(plugin_path):

plugin_path, func_name = plugin_path.rsplit(":", 1)

if plugin_path.endswith(".py"):

# Plugin paths can be relative to the config file location.

plugin_path = os.path.join(os.path.dirname(options.config_file), plugin_path)

if not os.path.isfile(plugin_path):

plugin_error(f'Can\'t find plugin "{plugin_path}"')

# Use an absolute path to avoid populating the cache entry

# for 'tmp' during tests, since it will be different in

# different tests.

plugin_dir = os.path.abspath(os.path.dirname(plugin_path))

fnam = os.path.basename(plugin_path)

module_name = fnam[:-3]

sys.path.insert(0, plugin_dir)

elif re.search(r"[\\/]", plugin_path):

fnam = os.path.basename(plugin_path)

plugin_error(f'Plugin "{fnam}" does not have a .py extension')

else:

module_name = plugin_path

try:

module = importlib.import_module(module_name)

except Exception as exc:

plugin_error(f'Error importing plugin "{plugin_path}": {exc}')

finally:

if plugin_dir is not None:

assert sys.path[0] == plugin_dir

del sys.path[0]

if not hasattr(module, func_name):

plugin_error(

'Plugin "{}" does not define entry point function "{}"'.format(

plugin_path, func_name

)

)

try:

plugin_type = getattr(module, func_name)(__version__)

except Exception:

print(f"Error calling the plugin(version) entry point of {plugin_path}\n", file=stdout)

raise # Propagate to display traceback

if not isinstance(plugin_type, type):

plugin_error(

'Type object expected as the return value of "plugin"; got {!r} (in {})'.format(

plugin_type, plugin_path

)

)

if not issubclass(plugin_type, Plugin):

plugin_error(

'Return value of "plugin" must be a subclass of "mypy.plugin.Plugin" '

"(in {})".format(plugin_path)

)

try:

custom_plugins.append(plugin_type(options))

snapshot[module_name] = take_module_snapshot(module)

except Exception:

print(f"Error constructing plugin instance of {plugin_type.__name__}\n", file=stdout)

raise # Propagate to display traceback

return custom_plugins, snapshot

def load_plugins(

options: Options, errors: Errors, stdout: TextIO, extra_plugins: Sequence[Plugin]

) -> tuple[Plugin, dict[str, str]]:

"""Load all configured plugins.

Return a plugin that encapsulates all plugins chained together. Always

at least include the default plugin (it's last in the chain).

The second return value is a snapshot of versions/hashes of loaded user

plugins (for cache validation).

"""

custom_plugins, snapshot = load_plugins_from_config(options, errors, stdout)

custom_plugins += extra_plugins

default_plugin: Plugin = DefaultPlugin(options)

if not custom_plugins:

return default_plugin, snapshot

# Custom plugins take precedence over the default plugin.

return ChainedPlugin(options, custom_plugins + [default_plugin]), snapshot

def take_module_snapshot(module: types.ModuleType) -> str:

"""Take plugin module snapshot by recording its version and hash.

We record _both_ hash and the version to detect more possible changes

(e.g. if there is a change in modules imported by a plugin).

"""

if hasattr(module, "__file__"):

assert module.__file__ is not None

with open(module.__file__, "rb") as f:

digest = hash_digest(f.read())

else:

digest = "unknown"

ver = getattr(module, "__version__", "none")

return f"{ver}:{digest}"

def find_config_file_line_number(path: str, section: str, setting_name: str) -> int:

"""Return the approximate location of setting_name within mypy config file.

Return -1 if can't determine the line unambiguously.

"""

in_desired_section = False

try:

results = []

with open(path, encoding="UTF-8") as f:

for i, line in enumerate(f):

line = line.strip()

if line.startswith("[") and line.endswith("]"):

current_section = line[1:-1].strip()

in_desired_section = current_section == section

elif in_desired_section and re.match(rf"{setting_name}\s*=", line):

results.append(i + 1)

if len(results) == 1:

return results[0]

except OSError:

pass

return -1

class BuildManager:

"""This class holds shared state for building a mypy program.

It is used to coordinate parsing, import processing, semantic

analysis and type checking. The actual build steps are carried

out by dispatch().

Attributes:

data_dir: Mypy data directory (contains stubs)

search_paths: SearchPaths instance indicating where to look for modules

modules: Mapping of module ID to MypyFile (shared by the passes)

semantic_analyzer:

Semantic analyzer, pass 2

all_types: Map {Expression: Type} from all modules (enabled by export_types)

options: Build options

missing_modules: Set of modules that could not be imported encountered so far

stale_modules: Set of modules that needed to be rechecked (only used by tests)

fg_deps_meta: Metadata for fine-grained dependencies caches associated with modules

fg_deps: A fine-grained dependency map

version_id: The current mypy version (based on commit id when possible)

plugin: Active mypy plugin(s)

plugins_snapshot:

Snapshot of currently active user plugins (versions and hashes)

old_plugins_snapshot:

Plugins snapshot from previous incremental run (or None in

non-incremental mode and if cache was not found)

errors: Used for reporting all errors

flush_errors: A function for processing errors after each SCC

cache_enabled: Whether cache is being read. This is set based on options,

but is disabled if fine-grained cache loading fails

and after an initial fine-grained load. This doesn't

determine whether we write cache files or not.

quickstart_state:

A cache of filename -> mtime/size/hash info used to avoid

needing to hash source files when using a cache with mismatching mtimes

stats: Dict with various instrumentation numbers, it is used

not only for debugging, but also required for correctness,

in particular to check consistency of the fine-grained dependency cache.

fscache: A file system cacher

ast_cache: AST cache to speed up mypy daemon

"""

def __init__(

self,

data_dir: str,

search_paths: SearchPaths,

ignore_prefix: str,

source_set: BuildSourceSet,

reports: Reports | None,

options: Options,

version_id: str,

plugin: Plugin,

plugins_snapshot: dict[str, str],

errors: Errors,

flush_errors: Callable[[list[str], bool], None],

fscache: FileSystemCache,

stdout: TextIO,

stderr: TextIO,

) -> None:

self.stats: dict[str, Any] = {} # Values are ints or floats

self.stdout = stdout

self.stderr = stderr

self.start_time = time.time()

self.data_dir = data_dir

self.errors = errors

self.errors.set_ignore_prefix(ignore_prefix)

self.search_paths = search_paths

self.source_set = source_set

self.reports = reports

self.options = options

self.version_id = version_id

self.modules: dict[str, MypyFile] = {}

self.missing_modules: set[str] = set()

self.fg_deps_meta: dict[str, FgDepMeta] = {}

# fg_deps holds the dependencies of every module that has been

# processed. We store this in BuildManager so that we can compute

# dependencies as we go, which allows us to free ASTs and type information,

# saving a ton of memory on net.

self.fg_deps: dict[str, set[str]] = {}

# Always convert the plugin to a ChainedPlugin so that it can be manipulated if needed

if not isinstance(plugin, ChainedPlugin):

plugin = ChainedPlugin(options, [plugin])

self.plugin = plugin

# Set of namespaces (module or class) that are being populated during semantic

# analysis and may have missing definitions.

self.incomplete_namespaces: set[str] = set()

self.semantic_analyzer = SemanticAnalyzer(

self.modules,

self.missing_modules,

self.incomplete_namespaces,

self.errors,

self.plugin,

)

self.all_types: dict[Expression, Type] = {} # Enabled by export_types

self.indirection_detector = TypeIndirectionVisitor()

self.stale_modules: set[str] = set()

self.rechecked_modules: set[str] = set()

self.flush_errors = flush_errors

has_reporters = reports is not None and reports.reporters

self.cache_enabled = (

options.incremental

and (not options.fine_grained_incremental or options.use_fine_grained_cache)

and not has_reporters

)

self.fscache = fscache

self.find_module_cache = FindModuleCache(

self.search_paths, self.fscache, self.options, source_set=self.source_set

)

for module in CORE_BUILTIN_MODULES:

if options.use_builtins_fixtures:

continue

if module == "_importlib_modulespec":

continue

path = self.find_module_cache.find_module(module)

if not isinstance(path, str):

raise CompileError(

[f"Failed to find builtin module {module}, perhaps typeshed is broken?"]

)

if is_typeshed_file(options.abs_custom_typeshed_dir, path) or is_stub_package_file(

path

):

continue

raise CompileError(

[

f'mypy: "{os.path.relpath(path)}" shadows library module "{module}"',

f'note: A user-defined top-level module with name "{module}" is not supported',

]

)

self.metastore = create_metastore(options)

# a mapping from source files to their corresponding shadow files

# for efficient lookup

self.shadow_map: dict[str, str] = {}

if self.options.shadow_file is not None:

self.shadow_map = {

source_file: shadow_file for (source_file, shadow_file) in self.options.shadow_file

}

# a mapping from each file being typechecked to its possible shadow file

self.shadow_equivalence_map: dict[str, str | None] = {}

self.plugin = plugin

self.plugins_snapshot = plugins_snapshot

self.old_plugins_snapshot = read_plugins_snapshot(self)

self.quickstart_state = read_quickstart_file(options, self.stdout)

# Fine grained targets (module top levels and top level functions) processed by

# the semantic analyzer, used only for testing. Currently used only by the new

# semantic analyzer. Tuple of module and target name.

self.processed_targets: list[tuple[str, str]] = []

# Missing stub packages encountered.

self.missing_stub_packages: set[str] = set()

# Cache for mypy ASTs that have completed semantic analysis

# pass 1. When multiple files are added to the build in a

# single daemon increment, only one of the files gets added

# per step and the others are discarded. This gets repeated

# until all the files have been added. This means that a

# new file can be processed O(n**2) times. This cache

# avoids most of this redundant work.

self.ast_cache: dict[str, tuple[MypyFile, list[ErrorInfo]]] = {}

def dump_stats(self) -> None:

if self.options.dump_build_stats:

print("Stats:")

for key, value in sorted(self.stats_summary().items()):

print(f"{key + ':':24}{value}")

def use_fine_grained_cache(self) -> bool:

return self.cache_enabled and self.options.use_fine_grained_cache

def maybe_swap_for_shadow_path(self, path: str) -> str:

if not self.shadow_map:

return path

path = normpath(path, self.options)

previously_checked = path in self.shadow_equivalence_map

if not previously_checked:

for source, shadow in self.shadow_map.items():

if self.fscache.samefile(path, source):

self.shadow_equivalence_map[path] = shadow

break

else:

self.shadow_equivalence_map[path] = None

shadow_file = self.shadow_equivalence_map.get(path)

return shadow_file if shadow_file else path

def get_stat(self, path: str) -> os.stat_result:

return self.fscache.stat(self.maybe_swap_for_shadow_path(path))

def getmtime(self, path: str) -> int:

"""Return a file's mtime; but 0 in bazel mode.

(Bazel's distributed cache doesn't like filesystem metadata to

end up in output files.)

"""

if self.options.bazel:

return 0

else:

return int(self.metastore.getmtime(path))

def all_imported_modules_in_file(self, file: MypyFile) -> list[tuple[int, str, int]]:

"""Find all reachable import statements in a file.

Return list of tuples (priority, module id, import line number)

for all modules imported in file; lower numbers == higher priority.

Can generate blocking errors on bogus relative imports.

"""

def correct_rel_imp(imp: ImportFrom | ImportAll) -> str:

"""Function to correct for relative imports."""

file_id = file.fullname

rel = imp.relative

if rel == 0:

return imp.id

if os.path.basename(file.path).startswith("__init__."):

rel -= 1

if rel != 0:

file_id = ".".join(file_id.split(".")[:-rel])

new_id = file_id + "." + imp.id if imp.id else file_id

if not new_id:

self.errors.set_file(file.path, file.name, self.options)

self.errors.report(

imp.line, 0, "No parent module -- cannot perform relative import", blocker=True

)

return new_id

res: list[tuple[int, str, int]] = []

for imp in file.imports:

if not imp.is_unreachable:

if isinstance(imp, Import):

pri = import_priority(imp, PRI_MED)

ancestor_pri = import_priority(imp, PRI_LOW)

for id, _ in imp.ids:

res.append((pri, id, imp.line))

ancestor_parts = id.split(".")[:-1]

ancestors = []

for part in ancestor_parts:

ancestors.append(part)

res.append((ancestor_pri, ".".join(ancestors), imp.line))

elif isinstance(imp, ImportFrom):

cur_id = correct_rel_imp(imp)

all_are_submodules = True

# Also add any imported names that are submodules.

pri = import_priority(imp, PRI_MED)

for name, __ in imp.names:

sub_id = cur_id + "." + name

if self.is_module(sub_id):

res.append((pri, sub_id, imp.line))

else:

all_are_submodules = False

# Add cur_id as a dependency, even if all of the

# imports are submodules. Processing import from will try

# to look through cur_id, so we should depend on it.

# As a workaround for for some bugs in cycle handling (#4498),

# if all of the imports are submodules, do the import at a lower

# priority.

pri = import_priority(imp, PRI_HIGH if not all_are_submodules else PRI_LOW)

res.append((pri, cur_id, imp.line))

elif isinstance(imp, ImportAll):

pri = import_priority(imp, PRI_HIGH)

res.append((pri, correct_rel_imp(imp), imp.line))

# Sort such that module (e.g. foo.bar.baz) comes before its ancestors (e.g. foo

# and foo.bar) so that, if FindModuleCache finds the target module in a

# package marked with py.typed underneath a namespace package installed in

# site-packages, (gasp), that cache's knowledge of the ancestors

# (aka FindModuleCache.ns_ancestors) can be primed when it is asked to find

# the parent.

res.sort(key=lambda x: -x[1].count("."))

return res

def is_module(self, id: str) -> bool:

"""Is there a file in the file system corresponding to module id?"""

return find_module_simple(id, self) is not None

def parse_file(

self, id: str, path: str, source: str, ignore_errors: bool, options: Options

) -> MypyFile:

"""Parse the source of a file with the given name.

Raise CompileError if there is a parse error.

"""

t0 = time.time()

tree = parse(source, path, id, self.errors, options=options)

tree._fullname = id

self.add_stats(

files_parsed=1,

modules_parsed=int(not tree.is_stub),

stubs_parsed=int(tree.is_stub),

parse_time=time.time() - t0,

)

if self.errors.is_blockers():

self.log("Bailing due to parse errors")

self.errors.raise_error()

self.errors.set_file_ignored_lines(path, tree.ignored_lines, ignore_errors)

return tree

def load_fine_grained_deps(self, id: str) -> dict[str, set[str]]:

t0 = time.time()

if id in self.fg_deps_meta:

# TODO: Assert deps file wasn't changed.

deps = json.loads(self.metastore.read(self.fg_deps_meta[id]["path"]))

else:

deps = {}

val = {k: set(v) for k, v in deps.items()}

self.add_stats(load_fg_deps_time=time.time() - t0)

return val

def report_file(

self, file: MypyFile, type_map: dict[Expression, Type], options: Options

) -> None:

if self.reports is not None and self.source_set.is_source(file):

self.reports.file(file, self.modules, type_map, options)

def verbosity(self) -> int:

return self.options.verbosity

def log(self, *message: str) -> None:

if self.verbosity() >= 1:

if message:

print("LOG: ", *message, file=self.stderr)

else:

print(file=self.stderr)

self.stderr.flush()

def log_fine_grained(self, *message: str) -> None:

import mypy.build

if self.verbosity() >= 1:

self.log("fine-grained:", *message)

elif mypy.build.DEBUG_FINE_GRAINED:

# Output log in a simplified format that is quick to browse.

if message:

print(*message, file=self.stderr)

else:

print(file=self.stderr)

self.stderr.flush()

def trace(self, *message: str) -> None:

if self.verbosity() >= 2:

print("TRACE:", *message, file=self.stderr)

self.stderr.flush()

def add_stats(self, **kwds: Any) -> None:

for key, value in kwds.items():

if key in self.stats:

self.stats[key] += value

else:

self.stats[key] = value

def stats_summary(self) -> Mapping[str, object]:

return self.stats

def deps_to_json(x: dict[str, set[str]]) -> str:

return json.dumps({k: list(v) for k, v in x.items()}, separators=(",", ":"))

# File for storing metadata about all the fine-grained dependency caches

DEPS_META_FILE: Final = "@deps.meta.json"

# File for storing fine-grained dependencies that didn't a parent in the build

DEPS_ROOT_FILE: Final = "@root.deps.json"

# The name of the fake module used to store fine-grained dependencies that

# have no other place to go.

FAKE_ROOT_MODULE: Final = "@root"

def write_deps_cache(

rdeps: dict[str, dict[str, set[str]]], manager: BuildManager, graph: Graph

) -> None:

"""Write cache files for fine-grained dependencies.

Serialize fine-grained dependencies map for fine grained mode.

Dependencies on some module 'm' is stored in the dependency cache

file m.deps.json. This entails some spooky action at a distance:

if module 'n' depends on 'm', that produces entries in m.deps.json.

When there is a dependency on a module that does not exist in the

build, it is stored with its first existing parent module. If no

such module exists, it is stored with the fake module FAKE_ROOT_MODULE.

This means that the validity of the fine-grained dependency caches

are a global property, so we store validity checking information for

fine-grained dependencies in a global cache file:

* We take a snapshot of current sources to later check consistency

between the fine-grained dependency cache and module cache metadata

* We store the mtime of all of the dependency files to verify they

haven't changed

"""

metastore = manager.metastore

error = False

fg_deps_meta = manager.fg_deps_meta.copy()

for id in rdeps:

if id != FAKE_ROOT_MODULE:

_, _, deps_json = get_cache_names(id, graph[id].xpath, manager.options)

else:

deps_json = DEPS_ROOT_FILE

assert deps_json

manager.log("Writing deps cache", deps_json)

if not manager.metastore.write(deps_json, deps_to_json(rdeps[id])):

manager.log(f"Error writing fine-grained deps JSON file {deps_json}")

error = True

else:

fg_deps_meta[id] = {"path": deps_json, "mtime": manager.getmtime(deps_json)}

meta_snapshot: dict[str, str] = {}

for id, st in graph.items():

# If we didn't parse a file (so it doesn't have a

# source_hash), then it must be a module with a fresh cache,

# so use the hash from that.

if st.source_hash:

hash = st.source_hash

else:

assert st.meta, "Module must be either parsed or cached"

hash = st.meta.hash

meta_snapshot[id] = hash

meta = {"snapshot": meta_snapshot, "deps_meta": fg_deps_meta}

if not metastore.write(DEPS_META_FILE, json.dumps(meta, separators=(",", ":"))):

manager.log(f"Error writing fine-grained deps meta JSON file {DEPS_META_FILE}")

error = True

if error:

manager.errors.set_file(_cache_dir_prefix(manager.options), None, manager.options)

manager.errors.report(0, 0, "Error writing fine-grained dependencies cache", blocker=True)

def invert_deps(deps: dict[str, set[str]], graph: Graph) -> dict[str, dict[str, set[str]]]:

"""Splits fine-grained dependencies based on the module of the trigger.