Skip to content

Commit ff7e800

Browse files
authored
Merge pull request #4886 from pschanely/atomic-constants-cache
Write the local constants cache atomically
2 parents 9c55f97 + e57fd12 commit ff7e800

3 files changed

Lines changed: 59 additions & 8 deletions

File tree

‎hypothesis/RELEASE.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
RELEASE_TYPE: patch
2+
3+
This patch makes writes to the cache of constants collected from local source
4+
files atomic. Previously, a concurrent process sharing the same ``.hypothesis``
5+
directory, such as another :pypi:`pytest-xdist` worker, could read a partially
6+
written cache file, which silently changed the data generated for a given
7+
:func:`@seed <hypothesis.seed>` (:issue:`4885`).

‎hypothesis/src/hypothesis/internal/constants_ast.py‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212
import hashlib
1313
import inspect
1414
import math
15+
import os
1516
import sys
17+
import tempfile
1618
from ast import Constant, Expr, NodeVisitor, UnaryOp, USub
1719
from collections.abc import Iterator, MutableSet
1820
from functools import lru_cache
@@ -235,14 +237,19 @@ def constants_from_module(module: ModuleType, *, limit: bool = True) -> Constant
235237

236238
try:
237239
cache_dir.create_if_missing()
238-
cache_p.write_text(
239-
f"# file: {module_file}\n# hypothesis_version: {hypothesis.__version__}\n\n"
240-
# somewhat arbitrary sort order. The cache file doesn't *have* to be
241-
# stable... but it is aesthetically pleasing, and means we could rely
242-
# on it in the future!
243-
+ _constants_file_str(constants),
244-
encoding="utf-8",
245-
)
240+
# Write atomically, so that a concurrent process (e.g. another
241+
# pytest-xdist worker) never reads a partially written cache file.
242+
# A truncated file still parses, and would silently drop constants.
243+
fd, tmp_path = tempfile.mkstemp(dir=cache_dir.path, prefix=".tmp-")
244+
with os.fdopen(fd, "w", encoding="utf-8") as f:
245+
f.write(
246+
f"# file: {module_file}\n# hypothesis_version: {hypothesis.__version__}\n\n"
247+
# somewhat arbitrary sort order. The cache file doesn't *have* to be
248+
# stable... but it is aesthetically pleasing, and means we could rely
249+
# on it in the future!
250+
+ _constants_file_str(constants)
251+
)
252+
os.replace(tmp_path, cache_p)
246253
except Exception: # pragma: no cover
247254
pass
248255

‎hypothesis/tests/cover/test_constants_ast.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,19 @@
99
# obtain one at https://mozilla.org/MPL/2.0/.
1010

1111
import ast
12+
import hashlib
1213
import inspect
1314
import subprocess
1415
import sys
16+
import tempfile
1517
import textwrap
18+
from pathlib import Path
1619
from types import ModuleType
1720

1821
import pytest
1922

2023
from hypothesis import example, given, note, settings, strategies as st
24+
from hypothesis.configuration import set_hypothesis_home_dir, storage_directory
2125
from hypothesis.internal.compat import PYPY
2226
from hypothesis.internal.constants_ast import (
2327
Constants,
@@ -317,3 +321,36 @@ def test_module_too_large(tmp_path):
317321
module.__file__ = str(p)
318322
assert constants_from_module(module) == Constants()
319323
assert constants_from_module(module, limit=False) == Constants(integers={constant})
324+
325+
326+
@skipif_threading # concurrent writes to the same file
327+
def test_cache_file_is_written_via_a_temporary_file(tmp_path, monkeypatch):
328+
constant = 18710297
329+
p = tmp_path / "cached_constants.py"
330+
p.write_text(f"a = {constant}\n", encoding="utf-8")
331+
module = ModuleType("cached_constants")
332+
module.__file__ = str(p)
333+
334+
temp_paths = []
335+
real_mkstemp = tempfile.mkstemp
336+
337+
def recording_mkstemp(*args, **kwargs):
338+
fd, path = real_mkstemp(*args, **kwargs)
339+
temp_paths.append(path)
340+
return fd, path
341+
342+
monkeypatch.setattr(tempfile, "mkstemp", recording_mkstemp)
343+
# use an empty storage directory, so the cache file can't already exist
344+
previous_home_dir = storage_directory().path
345+
set_hypothesis_home_dir(tmp_path / "hypothesis_home")
346+
try:
347+
assert constants_from_module(module) == Constants(integers={constant})
348+
source_hash = hashlib.sha1(p.read_bytes()).hexdigest()[:16]
349+
cache_p = storage_directory("constants").path / source_hash
350+
cached = _constants_from_source(cache_p.read_bytes(), limit=True)
351+
finally:
352+
set_hypothesis_home_dir(previous_home_dir)
353+
354+
assert cached == Constants(integers={constant})
355+
assert len(temp_paths) == 1
356+
assert not Path(temp_paths[0]).exists()

0 commit comments

Comments
 (0)