|
25 | 25 |
|
26 | 26 | import py.path |
27 | 27 | import pytest |
| 28 | +from _pytest.monkeypatch import MonkeyPatch |
28 | 29 |
|
29 | 30 | from astropy.utils import data |
30 | 31 | from astropy.config import paths |
|
34 | 35 | CacheDamaged, |
35 | 36 | conf, |
36 | 37 | _cache, |
| 38 | + _cache_lock, |
37 | 39 | compute_hash, |
38 | 40 | download_file, |
39 | 41 | cache_contents, |
@@ -190,23 +192,30 @@ def no_mkdir(p): |
190 | 192 |
|
191 | 193 |
|
192 | 194 | _shelve_possible_backends = ["dbm.dumb", "dbm.ndbm", "dbm.gnu"] |
| 195 | +for n in _shelve_possible_backends: |
| 196 | + if n not in dbm._modules: |
| 197 | + try: |
| 198 | + dbm._modules[n] = __import__(n, fromlist=['open']) |
| 199 | + except ImportError: |
| 200 | + pass |
193 | 201 |
|
194 | 202 |
|
195 | 203 | @contextlib.contextmanager |
196 | 204 | def shelve_backend(name): |
197 | 205 | """Ensure that shelve has access only to one backend.""" |
198 | | - defaultmod = dbm._defaultmod |
199 | | - modules = dbm._modules |
200 | | - try: |
| 206 | + # WARNING: there is something weird and fragile about this. |
| 207 | + # Using it can cause weird order-dependent test failures in |
| 208 | + # the second shelve_backend test if the two meet certain |
| 209 | + # mysterious criteria. |
| 210 | + with MonkeyPatch().context() as mp: |
201 | 211 | try: |
202 | | - dbm._defaultmod = __import__(name, fromlist=['open']) |
203 | | - except ImportError: |
204 | | - pytest.skip(f"module {name} not available") |
205 | | - dbm._modules = {name: dbm._defaultmod} |
| 212 | + m = dbm._modules[name] |
| 213 | + except KeyError: |
| 214 | + pytest.skip(f"Backend {name} not available") |
| 215 | + |
| 216 | + mp.setattr(dbm, "_defaultmod", m) |
| 217 | + mp.setattr(dbm, "_modules", {name: m}) |
206 | 218 | yield |
207 | | - finally: |
208 | | - dbm._defaultmod = defaultmod |
209 | | - dbm._modules = modules |
210 | 219 |
|
211 | 220 |
|
212 | 221 | @pytest.mark.remote_data(source="astropy") |
@@ -286,15 +295,21 @@ def test_download_file_threaded_many(b, temp_cache, valid_urls): |
286 | 295 | also does this but coverage tools lose track of which paths are explored. |
287 | 296 |
|
288 | 297 | """ |
| 298 | + u, c = next(valid_urls) |
| 299 | + # Ensure download cache exists in appropriate format |
289 | 300 | with shelve_backend(b): |
290 | | - urls = list(islice(valid_urls, N_THREAD_HAMMER)) |
291 | | - with ThreadPoolExecutor(max_workers=len(urls)) as P: |
292 | | - r = list(P.map(lambda u: download_file(u, cache=True), |
293 | | - [u for (u, c) in urls])) |
294 | | - check_download_cache() |
295 | | - assert len(r) == len(urls) |
296 | | - for r, (u, c) in zip(r, urls): |
297 | | - assert get_file_contents(r) == c |
| 301 | + download_file(u, cache=True) |
| 302 | + clear_download_cache(u) |
| 303 | + |
| 304 | + # Run test outside of non-thread-safe context manager |
| 305 | + urls = list(islice(valid_urls, N_THREAD_HAMMER)) |
| 306 | + with ThreadPoolExecutor(max_workers=len(urls)) as P: |
| 307 | + r = list(P.map(lambda u: download_file(u, cache=True), |
| 308 | + [u for (u, c) in urls])) |
| 309 | + check_download_cache() |
| 310 | + assert len(r) == len(urls) |
| 311 | + for r, (u, c) in zip(r, urls): |
| 312 | + assert get_file_contents(r) == c |
298 | 313 |
|
299 | 314 |
|
300 | 315 | def test_download_file_threaded_many_partial_success( |
@@ -1790,9 +1805,16 @@ def test_cache_with_different_shelve_backends(b, temp_cache, valid_urls): |
1790 | 1805 | uc = list(islice(valid_urls, FEW)) |
1791 | 1806 | for u, c in uc: |
1792 | 1807 | download_file(u, cache=True) |
| 1808 | + assert is_url_in_cache(u) |
| 1809 | + |
| 1810 | + for u, _ in uc: |
| 1811 | + assert is_url_in_cache(u) |
1793 | 1812 |
|
1794 | 1813 | check_download_cache() |
1795 | 1814 |
|
| 1815 | + for u, _ in uc: |
| 1816 | + assert is_url_in_cache(u) |
| 1817 | + |
1796 | 1818 | for u, c in uc: |
1797 | 1819 | assert get_file_contents( |
1798 | 1820 | download_file(u, cache=True, sources=[])) == c |
@@ -1837,3 +1859,30 @@ def test_wrong_backend_reports_useful_error(b1b2, temp_cache, valid_urls): |
1837 | 1859 | assert b1 in str(e.value) |
1838 | 1860 | with pytest.raises(CacheDamaged): |
1839 | 1861 | check_download_cache() |
| 1862 | + clear_download_cache() |
| 1863 | + clear_download_cache() |
| 1864 | + |
| 1865 | + |
| 1866 | +# What happens when the lock can't be obtained in a timely manner? |
| 1867 | +def test_lock_unavailable_raises_runtimeerror_and_clear_frees_lock( |
| 1868 | + temp_cache, valid_urls): |
| 1869 | + u, c = next(valid_urls) |
| 1870 | + download_file(u, cache=True) |
| 1871 | + with _cache_lock("astropy", need_write=True): |
| 1872 | + with conf.set_temp("download_cache_lock_attempts", 0): |
| 1873 | + with pytest.raises(RuntimeError): |
| 1874 | + with _cache_lock("astropy", need_write=True): |
| 1875 | + pass |
| 1876 | + # Ensure that the cache doesn't accidentally get unlocked! |
| 1877 | + with pytest.raises(RuntimeError): |
| 1878 | + with _cache_lock("astropy", need_write=True): |
| 1879 | + pass |
| 1880 | + # Download |
| 1881 | + with pytest.raises(RuntimeError): |
| 1882 | + is_url_in_cache(u), "if locked, act like cache empty" |
| 1883 | + with pytest.raises(RuntimeError): |
| 1884 | + download_file(u, cache=True, sources=[]) |
| 1885 | + with pytest.raises(RuntimeError): |
| 1886 | + download_file(u, cache=True) |
| 1887 | + clear_download_cache() # breaks lock |
| 1888 | + is_url_in_cache(u) |
0 commit comments