Skip to content

Commit 082ed70

Browse files
Merge pull request #15255 from pytest-dev/claude/project-thread-eoi3d6
tmpdir: hand out numbered dirs from a per-prefix counter
2 parents ae37a01 + 22705a5 commit 082ed70

3 files changed

Lines changed: 155 additions & 4 deletions

File tree

‎changelog/15254.improvement.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
:fixture:`tmp_path` and :meth:`TempPathFactory.mktemp <pytest.TempPathFactory.mktemp>` no longer list the whole base temporary directory on every call, which made sessions with many temporary directories quadratically slow. Numbers are now handed out from a per-prefix counter, skipping names that already exist.

‎src/_pytest/tmpdir.py‎

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,12 @@
1717
from typing import final
1818
from typing import Literal
1919

20+
from .pathlib import _force_symlink
2021
from .pathlib import cleanup_dead_symlinks
22+
from .pathlib import find_suffixes
2123
from .pathlib import LOCK_TIMEOUT
22-
from .pathlib import make_numbered_dir
2324
from .pathlib import make_numbered_dir_with_cleanup
25+
from .pathlib import parse_num
2426
from .pathlib import rm_rf
2527
from _pytest.compat import get_user_id
2628
from _pytest.config import Config
@@ -79,6 +81,9 @@ def __init__(
7981
# Register cleanups for session finish. Also called atexit as a last
8082
# resort if sessionfinish for some reason doesn't happen.
8183
self._exit_stack = ExitStack()
84+
# Lower-cased prefix -> next number to try, so numbered directories
85+
# do not require listing basetemp on every call.
86+
self._next_number: dict[str, int] = {}
8287

8388
@classmethod
8489
def from_config(
@@ -122,9 +127,10 @@ def mktemp(self, basename: str, numbered: bool = True) -> Path:
122127
123128
:param numbered:
124129
If ``True``, ensure the directory is unique by adding a numbered
125-
suffix greater than any existing one: ``basename="foo-"`` and ``numbered=True``
130+
suffix: ``basename="foo-"`` and ``numbered=True``
126131
means that this function will create directories named ``"foo-0"``,
127-
``"foo-1"``, ``"foo-2"`` and so on.
132+
``"foo-1"``, ``"foo-2"`` and so on. Names that already exist in the
133+
base temporary directory are skipped.
128134
129135
:returns:
130136
The path to the new directory.
@@ -134,10 +140,48 @@ def mktemp(self, basename: str, numbered: bool = True) -> Path:
134140
p = self.getbasetemp().joinpath(basename)
135141
p.mkdir(mode=0o700)
136142
else:
137-
p = make_numbered_dir(root=self.getbasetemp(), prefix=basename, mode=0o700)
143+
p = self._make_counted_dir(basename)
138144
self._trace("mktemp", p)
139145
return p
140146

147+
def _make_counted_dir(self, prefix: str) -> Path:
148+
"""Create ``<prefix><n>`` in basetemp, using a per-prefix counter.
149+
150+
Counting starts at 0 without listing basetemp, as basetemp is created
151+
by this factory. mkdir decides uniqueness:
152+
names it fails on are skipped, a run of 10 failures makes basetemp be
153+
listed once to jump past it, and 20 failures raise OSError.
154+
"""
155+
root = self.getbasetemp()
156+
key = prefix.lower()
157+
number = self._next_number.get(key, 0)
158+
collisions = 0
159+
while True:
160+
path = root.joinpath(f"{prefix}{number}")
161+
try:
162+
path.mkdir(mode=0o700)
163+
except OSError as e:
164+
# Usually FileExistsError; on Windows also PermissionError for
165+
# a name that is still being deleted.
166+
collisions += 1
167+
if collisions == 10:
168+
listed = max(
169+
map(parse_num, find_suffixes(root, prefix)), default=-1
170+
)
171+
number = max(number, listed) + 1
172+
elif collisions == 20:
173+
raise OSError(
174+
f"could not create numbered dir with prefix {prefix} in {root}"
175+
f" after {collisions} tries"
176+
) from e
177+
else:
178+
number += 1
179+
else:
180+
break
181+
self._next_number[key] = number + 1
182+
_force_symlink(root, prefix + "current", path)
183+
return path
184+
141185
def getbasetemp(self) -> Path:
142186
"""Return the base temporary directory, creating it if needed.
143187

‎testing/test_tmpdir.py‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
from _pytest.pathlib import on_rm_rf_error
2323
from _pytest.pathlib import register_cleanup_lock_removal
2424
from _pytest.pathlib import rm_rf
25+
from _pytest.pathlib import symlink_or_skip
2526
from _pytest.pytester import Pytester
2627
from _pytest.tmpdir import get_user
2728
from _pytest.tmpdir import TempPathFactory
@@ -70,6 +71,111 @@ def test_mktemp(self, tmp_path: Path) -> None:
7071
assert str(tmp2.relative_to(t.getbasetemp())).startswith("this")
7172
assert tmp2 != tmp
7273

74+
def test_mktemp_does_not_list_basetemp(
75+
self, tmp_path: Path, monkeypatch: MonkeyPatch
76+
) -> None:
77+
"""Mktemp counts per prefix instead of listing basetemp (#15254)."""
78+
config = cast(Config, FakeConfig(tmp_path))
79+
t = TempPathFactory.from_config(config, _ispytest=True)
80+
(t.getbasetemp() / "world1").mkdir()
81+
# Listing basetemp would fail.
82+
monkeypatch.setattr("_pytest.tmpdir.find_suffixes", None)
83+
names = [t.mktemp("world").name for _ in range(3)]
84+
# The existing "world1" is skipped.
85+
assert names == ["world0", "world2", "world3"]
86+
87+
@pytest.mark.parametrize("kind", ["dir", "file", "dangling_symlink"])
88+
def test_counted_dir_skips_unexpected_entries(
89+
self, tmp_path: Path, kind: str
90+
) -> None:
91+
"""Something else taking the next name is skipped (#15254)."""
92+
config = cast(Config, FakeConfig(tmp_path))
93+
t = TempPathFactory.from_config(config, _ispytest=True)
94+
first = t._make_counted_dir("test_a")
95+
taken = t.getbasetemp() / "test_a1"
96+
if kind == "dir":
97+
taken.mkdir()
98+
elif kind == "file":
99+
taken.touch()
100+
else:
101+
symlink_or_skip(t.getbasetemp() / "missing", taken)
102+
names = [t._make_counted_dir("test_a").name for _ in range(2)]
103+
assert first.name == "test_a0"
104+
assert names == ["test_a2", "test_a3"]
105+
106+
def test_counted_dir_jumps_past_long_run(
107+
self, tmp_path: Path, monkeypatch: MonkeyPatch
108+
) -> None:
109+
"""A long run of taken names is skipped with one listing (#15254)."""
110+
config = cast(Config, FakeConfig(tmp_path))
111+
t = TempPathFactory.from_config(config, _ispytest=True)
112+
for i in range(50):
113+
(t.getbasetemp() / f"test_a{i}").mkdir()
114+
listed: list[str] = []
115+
orig = pathlib.find_suffixes
116+
117+
def find_suffixes(root: Path, prefix: str):
118+
listed.append(prefix)
119+
return orig(root, prefix)
120+
121+
monkeypatch.setattr("_pytest.tmpdir.find_suffixes", find_suffixes)
122+
names = [t._make_counted_dir("test_a").name for _ in range(2)]
123+
assert names == ["test_a50", "test_a51"]
124+
assert listed == ["test_a"]
125+
126+
def test_counted_dir_overlapping_prefixes(self, tmp_path: Path) -> None:
127+
"""Prefixes "test_a" and "test_a1" share names like "test_a10" (#15254)."""
128+
config = cast(Config, FakeConfig(tmp_path))
129+
t = TempPathFactory.from_config(config, _ispytest=True)
130+
names = [
131+
t._make_counted_dir(prefix).name
132+
for _ in range(30)
133+
for prefix in ("test_a", "test_a1")
134+
]
135+
assert len(set(names)) == len(names)
136+
137+
def test_counted_dir_gives_up(
138+
self, tmp_path: Path, monkeypatch: MonkeyPatch
139+
) -> None:
140+
"""A filesystem that reports every name as taken raises instead of
141+
looping forever (#15254)."""
142+
config = cast(Config, FakeConfig(tmp_path))
143+
t = TempPathFactory.from_config(config, _ispytest=True)
144+
t.getbasetemp()
145+
146+
def mkdir(self: Path, mode: int = 0o777) -> None:
147+
raise FileExistsError(self)
148+
149+
monkeypatch.setattr(Path, "mkdir", mkdir)
150+
with pytest.raises(OSError, match="after 20 tries"):
151+
t._make_counted_dir("test_a")
152+
153+
def test_counted_dir_retries_other_errors(
154+
self, tmp_path: Path, monkeypatch: MonkeyPatch
155+
) -> None:
156+
"""Other OSErrors are retried too, e.g. PermissionError on Windows for a
157+
name that is still being deleted (#15254)."""
158+
config = cast(Config, FakeConfig(tmp_path))
159+
t = TempPathFactory.from_config(config, _ispytest=True)
160+
t.getbasetemp()
161+
orig_mkdir = Path.mkdir
162+
163+
def mkdir(self: Path, mode: int = 0o777) -> None:
164+
if self.name == "test_a0":
165+
raise PermissionError(self)
166+
orig_mkdir(self, mode)
167+
168+
monkeypatch.setattr(Path, "mkdir", mkdir)
169+
assert t._make_counted_dir("test_a").name == "test_a1"
170+
171+
def always_fail(self: Path, mode: int = 0o777) -> None:
172+
raise PermissionError(self)
173+
174+
monkeypatch.setattr(Path, "mkdir", always_fail)
175+
with pytest.raises(OSError, match="after 20 tries") as excinfo:
176+
t._make_counted_dir("test_b")
177+
assert isinstance(excinfo.value.__cause__, PermissionError)
178+
73179
def test_tmppath_relative_basetemp_absolute(
74180
self, tmp_path: Path, monkeypatch: MonkeyPatch
75181
) -> None:

0 commit comments

Comments
 (0)