Skip to content

Commit 78cab6e

Browse files
authored
fix: reduce memory use during reporting. #2229 (#2231)
1 parent b4c1fda commit 78cab6e

5 files changed

Lines changed: 25 additions & 29 deletions

File tree

‎CHANGES.rst‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,12 @@ upgrading your version of coverage.py.
2323
Unreleased
2424
----------
2525

26-
Nothing yet.
26+
- Fix: one of the performance improvements in 7.15.1 (pull 2215) dramatically
27+
increased memory use during reporting for large projects. Now we use a
28+
different approach that is both faster and slimmer than 7.15.0. Fixes `issue
29+
2229`_.
30+
31+
.. _issue 2229: https://github.com/coveragepy/coveragepy/issues/2229
2732

2833

2934
.. start-releases

‎coverage/control.py‎

Lines changed: 3 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -304,8 +304,6 @@ def __init__( # pylint: disable=too-many-arguments
304304
self._file_mapper: Callable[[str], str] = abs_file
305305
self._data_suffix = self._run_suffix = None
306306
self._exclude_re: dict[str, str] = {}
307-
self._analysis_cache: dict[TMorf, Analysis] = {}
308-
self._file_reporter_cache: dict[TMorf, FileReporter] = {}
309307
self._old_sigterm: Callable[[int, FrameType | None], Any] | None = None
310308

311309
# State machine variables:
@@ -567,7 +565,6 @@ def load(self) -> None:
567565
if not should_skip:
568566
assert self._data is not None
569567
self._data.read()
570-
self._clear_analysis_caches()
571568

572569
def _init_for_start(self) -> None:
573570
"""Initialization for start()"""
@@ -780,7 +777,6 @@ def erase(self) -> None:
780777
self._data.erase(parallel=self.config.parallel)
781778
self._data = None
782779
self._inited_for_start = False
783-
self._clear_analysis_caches()
784780

785781
def switch_context(self, new_context: str) -> None:
786782
"""Switch to a new dynamic context.
@@ -913,7 +909,6 @@ def combine(
913909
keep=keep,
914910
message=self._message,
915911
)
916-
self._clear_analysis_caches()
917912

918913
def get_data(self) -> CoverageData:
919914
"""Get the collected data.
@@ -935,7 +930,6 @@ def get_data(self) -> CoverageData:
935930
self._collector.plugin_was_disabled(plugin)
936931

937932
if self._collector.flush_data():
938-
self._clear_analysis_caches()
939933
self._post_save_work()
940934

941935
assert self._data is not None
@@ -1004,27 +998,18 @@ def analysis2(
1004998
analysis.missing_formatted(),
1005999
)
10061000

1007-
def _analyze(self, morf: TMorf) -> Analysis:
1001+
def _analyze(self, morf: TMorf, file_reporter: FileReporter | None = None) -> Analysis:
10081002
"""Analyze a module or file. Private for now."""
1009-
analysis = self._analysis_cache.get(morf)
1010-
if analysis is not None:
1011-
return analysis
1012-
10131003
self._init()
10141004
self._post_init()
10151005

10161006
data = self.get_data()
1017-
file_reporter = self._get_file_reporter(morf)
1007+
if file_reporter is None:
1008+
file_reporter = self._get_file_reporter(morf)
10181009
filename = self._file_mapper(file_reporter.filename)
10191010
analysis = analysis_from_file_reporter(data, self.config.precision, file_reporter, filename)
1020-
self._analysis_cache[morf] = analysis
10211011
return analysis
10221012

1023-
def _clear_analysis_caches(self) -> None:
1024-
"""Forget cached analyses and file reporters, because data changed."""
1025-
self._analysis_cache.clear()
1026-
self._file_reporter_cache.clear()
1027-
10281013
def branch_stats(self, morf: TMorf) -> dict[TLineNo, tuple[int, int]]:
10291014
"""Get branch statistics about a module.
10301015
@@ -1041,9 +1026,6 @@ def branch_stats(self, morf: TMorf) -> dict[TLineNo, tuple[int, int]]:
10411026

10421027
def _get_file_reporter(self, morf: TMorf) -> FileReporter:
10431028
"""Get a FileReporter for a module or file name."""
1044-
cached = self._file_reporter_cache.get(morf)
1045-
if cached is not None:
1046-
return cached
10471029
assert self._data is not None
10481030
plugin = None
10491031
file_reporter: str | FileReporter = "python"
@@ -1068,7 +1050,6 @@ def _get_file_reporter(self, morf: TMorf) -> FileReporter:
10681050
file_reporter = PythonFileReporter(morf, self)
10691051

10701052
assert isinstance(file_reporter, FileReporter)
1071-
self._file_reporter_cache[morf] = file_reporter
10721053
return file_reporter
10731054

10741055
def _get_file_reporters(
@@ -1093,7 +1074,6 @@ def _get_file_reporters(
10931074
if not isinstance(morfs, (list, tuple, set)):
10941075
morfs = [morfs] # type: ignore[list-item]
10951076

1096-
morfs = sorted(morfs, key=lambda m: m if isinstance(m, str) else m.__name__)
10971077
return [(self._get_file_reporter(morf), morf) for morf in morfs]
10981078

10991079
def _prepare_data_for_reporting(self) -> None:

‎coverage/parser.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -336,8 +336,10 @@ def _analyze_ast(self) -> None:
336336
`_all_arcs` is the set of arcs in the code.
337337
338338
"""
339-
assert self._ast_root is not None
340-
aaa = AstArcAnalyzer(self.filename, self._ast_root, self.raw_statements, self.multiline_map)
339+
ast_root = self._ast_root
340+
if ast_root is None:
341+
ast_root = ast.parse(self.text)
342+
aaa = AstArcAnalyzer(self.filename, ast_root, self.raw_statements, self.multiline_map)
341343
aaa.analyze()
342344
arcs = aaa.arcs
343345
self._with_jump_fixers = aaa.with_jump_fixers()
@@ -353,6 +355,10 @@ def _analyze_ast(self) -> None:
353355

354356
self._missing_arc_fragments = aaa.missing_arc_fragments
355357

358+
# The AST is large and no longer needed: everything derived from it is
359+
# memoized above. Release it so long-lived parsers don't hold it.
360+
self._ast_root = None
361+
356362
def fix_with_jumps(self, arcs: Iterable[TArc]) -> set[TArc]:
357363
"""Adjust arcs to fix jumps leaving `with` statements.
358364

‎coverage/python.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,6 @@ def __init__(self, morf: TMorf, coverage: Coverage | None = None) -> None:
172172

173173
self._source: str | None = None
174174
self._parser: PythonParser | None = None
175-
self._excluded = None
176175

177176
def __repr__(self) -> str:
178177
return f"<PythonFileReporter {self.filename!r}>"

‎coverage/report_core.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,9 +93,15 @@ def get_analysis_to_report(
9393
if not fr_morfs:
9494
raise NoDataError("No data to report.")
9595

96-
for fr, morf in sorted(fr_morfs):
96+
# fr_morfs is a complete list of all files. FileReporters start very small,
97+
# but when you use one, it becomes large. We don't want to hold onto them
98+
# longer than we need to, so instead of a for loop, we'll pop and discard
99+
# the file reporters as we go.
100+
fr_morfs.sort(reverse=True)
101+
while fr_morfs:
102+
fr, morf = fr_morfs.pop()
97103
try:
98-
analysis = coverage._analyze(morf)
104+
analysis = coverage._analyze(morf, file_reporter=fr)
99105
except NotPython:
100106
# Only report errors for .py files, and only if we didn't
101107
# explicitly suppress those errors.

0 commit comments

Comments
 (0)