Skip to content

Commit 6033660

Browse files
fix: update the data file hash when a parallel data file is saved again (#2339)
A parallel data file got the hash of its data in its name on the first save, and later saves added data without changing the name. Combining could then skip the file as a duplicate of another one that had the same data at the first save, and delete it.
1 parent 99024b5 commit 6033660

4 files changed

Lines changed: 93 additions & 3 deletions

File tree

‎CHANGES.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,12 @@ Unreleased
5757
``line-rate`` values like ``8.333e-05`` or ``1``. This is now fixed, closing
5858
`issue 2293`_ and `issue 2294`_.
5959

60+
- Fix: a parallel data file saved more than once kept the hash of its first
61+
save in its file name. Combining could then skip it as a duplicate of another
62+
file that had the same data at that point, and delete it, losing the data
63+
recorded after the first save. Each save now updates the hash. This had been
64+
a problem since 7.13.1. Fixes `issue 2338`_.
65+
6066
- Fix: using the C tracer, the refcount of ``None`` was mismanaged during
6167
function calls, including comprehensions. This was unlikely to cause an
6268
actual problem, but is now fixed. Thanks, Andrew Dalke.
@@ -104,6 +110,7 @@ Unreleased
104110
.. _issue 2294: https://github.com/coveragepy/coveragepy/issues/2294
105111
.. _issue 2314: https://github.com/coveragepy/coveragepy/issues/2314
106112
.. _issue 2335: https://github.com/coveragepy/coveragepy/issues/2335
113+
.. _issue 2338: https://github.com/coveragepy/coveragepy/issues/2338
107114

108115

109116
.. start-releases

‎coverage/sqldata.py‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -936,19 +936,36 @@ def read(self) -> None:
936936
with self._connect():
937937
self._have_used = True
938938

939+
def _data_hash(self) -> str:
940+
"""The short hash of the data so far, for the data file name."""
941+
return base64.b64encode(self._hasher.digest(), altchars=b"01").decode()[:NHASH]
942+
939943
def write(self) -> None:
940944
"""Ensure the data is written to the data file."""
945+
unhashed_filename = None
941946
if self._our_suffix and not self._wrote_hash:
947+
unhashed_filename = self._filename
948+
elif (
949+
self._our_suffix
950+
and (m := filename_match(self._filename))
951+
and m["hash"]
952+
and not self._filename.endswith(f".H{self._data_hash()}h")
953+
):
954+
# More data was recorded after the hash went into the file name.
955+
# Update the name, or combining would skip this file as a
956+
# duplicate of another one that only shared its earlier data.
957+
unhashed_filename = self._filename[: m.start("hash") - len(".H")]
958+
if unhashed_filename is not None:
942959
self._debug_dataio("Finishing data file", self._filename)
943960
with self._connect() as con:
944961
con.execute_void(
945-
"INSERT OR IGNORE INTO meta (key, value) VALUES ('hash', ?)",
962+
"INSERT OR REPLACE INTO meta (key, value) VALUES ('hash', ?)",
946963
(self._hasher.hexdigest(),),
947964
)
948965
self.close()
949-
data_hash = base64.b64encode(self._hasher.digest(), altchars=b"01").decode()[:NHASH]
966+
data_hash = self._data_hash()
950967
current_filename = self._filename
951-
self._filename += f".H{data_hash}h"
968+
self._filename = unhashed_filename + f".H{data_hash}h"
952969
self._debug_dataio("Renaming data file to", self._filename)
953970
os.rename(current_filename, self._filename)
954971
self._wrote_hash = True

‎tests/test_api.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -553,6 +553,28 @@ def test_combining_with_a_used_coverage(self) -> None:
553553
assert self.stdout() == ""
554554
self.check_code1_code2(cov)
555555

556+
def test_combining_after_saving_twice(self) -> None:
557+
# Two parallel runs measure the same code and save while still running,
558+
# then each measures different code and saves again. Combining must
559+
# keep both.
560+
self.make_code1_code2()
561+
self.make_file("code3.py", "code3 = 1\n")
562+
for later in ["code2", "code3"]:
563+
cov = coverage.Coverage(data_suffix=True)
564+
with cov.collect():
565+
import_local_file("code1")
566+
cov.save()
567+
import_local_file(later)
568+
cov.save()
569+
self.assert_file_count(".coverage.*", 2)
570+
571+
cov = coverage.Coverage()
572+
cov.combine()
573+
self.check_code1_code2(cov)
574+
_, statements, missing, _ = cov.analysis("code3.py")
575+
assert statements == [1]
576+
assert missing == []
577+
556578
def test_ordered_combine(self) -> None:
557579
# https://github.com/coveragepy/coveragepy/issues/649
558580
# The order of the [paths] setting used to matter. Now the

‎tests/test_data.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from coverage.exceptions import ConfigError, DataError, NoDataError
2929
from coverage.files import PathAliases, canonical_filename
3030
from coverage.misc import first
31+
from coverage.sqldata import good_filename_match
3132
from coverage.types import FilePathClasses, FilePathType, TArc, TLineNo
3233
from tests import osinfo
3334
from tests.coveragetest import CoverageTest
@@ -879,6 +880,49 @@ def test_true_suffix(self) -> None:
879880
# In addition to being different, the suffixes have the pid in them.
880881
assert all(str(os.getpid()) in fn for fn in data_files2)
881882

883+
def test_true_suffix_written_twice(self) -> None:
884+
# Two files start with the same data, then record different data and
885+
# are written again. Both have to be combined, so the second write has
886+
# to update the hash in the file name.
887+
for more_lines in [LINES_2, {"d.py": {4}}]:
888+
covdata = DebugCoverageData(suffix=True)
889+
covdata.add_lines(LINES_1)
890+
covdata.write()
891+
covdata.add_lines(more_lines)
892+
covdata.write()
893+
data_files = glob.glob(".coverage.*")
894+
assert len(data_files) == 2
895+
assert all(good_filename_match(fn)["hash"] for fn in data_files)
896+
897+
covdata3 = DebugCoverageData()
898+
combine_parallel_data(covdata3)
899+
assert_line_counts(covdata3, {"a.py": 3, "b.py": 1, "c.py": 1, "d.py": 1})
900+
self.assert_file_count(".coverage.*", 0)
901+
902+
def test_true_suffix_written_twice_same_data(self) -> None:
903+
# Files with the same data are still skipped as duplicates when they
904+
# were written more than once.
905+
data_files = []
906+
for _ in range(2):
907+
covdata = DebugCoverageData(suffix=True)
908+
covdata.add_lines(LINES_1)
909+
covdata.write()
910+
covdata.add_lines(LINES_2)
911+
covdata.write()
912+
data_files.append(covdata.data_filename())
913+
classifier = DataFileClassifier()
914+
assert classifier.classify(data_files[0]) == "combine"
915+
assert classifier.classify(data_files[1]) == "skip"
916+
917+
def test_true_suffix_written_again_without_new_data(self) -> None:
918+
covdata = DebugCoverageData(suffix=True)
919+
covdata.add_lines(LINES_1)
920+
covdata.write()
921+
data_file = covdata.data_filename()
922+
covdata.write()
923+
assert covdata.data_filename() == data_file
924+
self.assert_file_count(".coverage.*", 1)
925+
882926
def test_combining(self) -> None:
883927
self.assert_file_count(".coverage.*", 0)
884928

0 commit comments

Comments
 (0)