|
25 | 25 |
|
26 | 26 | import py.path |
27 | 27 | import pytest |
| 28 | +from _pytest.monkeypatch import MonkeyPatch |
28 | 29 |
|
29 | 30 | from astropy.utils import data |
30 | 31 | from astropy.config import paths |
|
34 | 35 | CacheDamaged, |
35 | 36 | conf, |
36 | 37 | _cache, |
| 38 | + _cache_lock, |
37 | 39 | compute_hash, |
38 | 40 | download_file, |
39 | 41 | cache_contents, |
@@ -191,23 +193,30 @@ def no_mkdir(p): |
191 | 193 |
|
192 | 194 |
|
193 | 195 | _shelve_possible_backends = ["dbm.dumb", "dbm.ndbm", "dbm.gnu"] |
| 196 | +for n in _shelve_possible_backends: |
| 197 | + if n not in dbm._modules: |
| 198 | + try: |
| 199 | + dbm._modules[n] = __import__(n, fromlist=['open']) |
| 200 | + except ImportError: |
| 201 | + pass |
194 | 202 |
|
195 | 203 |
|
196 | 204 | @contextlib.contextmanager |
197 | 205 | def shelve_backend(name): |
198 | 206 | """Ensure that shelve has access only to one backend.""" |
199 | | - defaultmod = dbm._defaultmod |
200 | | - modules = dbm._modules |
201 | | - try: |
| 207 | + # WARNING: there is something weird and fragile about this. |
| 208 | + # Using it can cause weird order-dependent test failures in |
| 209 | + # the second shelve_backend test if the two meet certain |
| 210 | + # mysterious criteria. |
| 211 | + with MonkeyPatch().context() as mp: |
202 | 212 | try: |
203 | | - dbm._defaultmod = __import__(name, fromlist=['open']) |
204 | | - except ImportError: |
205 | | - pytest.skip(f"module {name} not available") |
206 | | - dbm._modules = {name: dbm._defaultmod} |
| 213 | + m = dbm._modules[name] |
| 214 | + except KeyError: |
| 215 | + pytest.skip(f"Backend {name} not available") |
| 216 | + |
| 217 | + mp.setattr(dbm, "_defaultmod", m) |
| 218 | + mp.setattr(dbm, "_modules", {name: m}) |
207 | 219 | yield |
208 | | - finally: |
209 | | - dbm._defaultmod = defaultmod |
210 | | - dbm._modules = modules |
211 | 220 |
|
212 | 221 |
|
213 | 222 | @pytest.mark.remote_data(source="astropy") |
@@ -287,15 +296,21 @@ def test_download_file_threaded_many(b, temp_cache, valid_urls): |
287 | 296 | also does this but coverage tools lose track of which paths are explored. |
288 | 297 |
|
289 | 298 | """ |
| 299 | + u, c = next(valid_urls) |
| 300 | + # Ensure download cache exists in appropriate format |
290 | 301 | with shelve_backend(b): |
291 | | - urls = list(islice(valid_urls, N_THREAD_HAMMER)) |
292 | | - with ThreadPoolExecutor(max_workers=len(urls)) as P: |
293 | | - r = list(P.map(lambda u: download_file(u, cache=True), |
294 | | - [u for (u, c) in urls])) |
295 | | - check_download_cache() |
296 | | - assert len(r) == len(urls) |
297 | | - for r, (u, c) in zip(r, urls): |
298 | | - assert get_file_contents(r) == c |
| 302 | + download_file(u, cache=True) |
| 303 | + clear_download_cache(u) |
| 304 | + |
| 305 | + # Run test outside of non-thread-safe context manager |
| 306 | + urls = list(islice(valid_urls, N_THREAD_HAMMER)) |
| 307 | + with ThreadPoolExecutor(max_workers=len(urls)) as P: |
| 308 | + r = list(P.map(lambda u: download_file(u, cache=True), |
| 309 | + [u for (u, c) in urls])) |
| 310 | + check_download_cache() |
| 311 | + assert len(r) == len(urls) |
| 312 | + for r, (u, c) in zip(r, urls): |
| 313 | + assert get_file_contents(r) == c |
299 | 314 |
|
300 | 315 |
|
301 | 316 | def test_download_file_threaded_many_partial_success( |
@@ -1791,9 +1806,16 @@ def test_cache_with_different_shelve_backends(b, temp_cache, valid_urls): |
1791 | 1806 | uc = list(islice(valid_urls, FEW)) |
1792 | 1807 | for u, c in uc: |
1793 | 1808 | download_file(u, cache=True) |
| 1809 | + assert is_url_in_cache(u) |
| 1810 | + |
| 1811 | + for u, _ in uc: |
| 1812 | + assert is_url_in_cache(u) |
1794 | 1813 |
|
1795 | 1814 | check_download_cache() |
1796 | 1815 |
|
| 1816 | + for u, _ in uc: |
| 1817 | + assert is_url_in_cache(u) |
| 1818 | + |
1797 | 1819 | for u, c in uc: |
1798 | 1820 | assert get_file_contents( |
1799 | 1821 | download_file(u, cache=True, sources=[])) == c |
@@ -1841,3 +1863,30 @@ def test_wrong_backend_reports_useful_error(b1b2, temp_cache, valid_urls): |
1841 | 1863 | with pytest.raises(CacheDamaged): |
1842 | 1864 | with pytest.warns(WrongDBMModuleWarning): |
1843 | 1865 | check_download_cache() |
| 1866 | + clear_download_cache() |
| 1867 | + clear_download_cache() |
| 1868 | + |
| 1869 | + |
| 1870 | +# What happens when the lock can't be obtained in a timely manner? |
| 1871 | +def test_lock_unavailable_raises_runtimeerror_and_clear_frees_lock( |
| 1872 | + temp_cache, valid_urls): |
| 1873 | + u, c = next(valid_urls) |
| 1874 | + download_file(u, cache=True) |
| 1875 | + with _cache_lock("astropy", need_write=True): |
| 1876 | + with conf.set_temp("download_cache_lock_attempts", 0): |
| 1877 | + with pytest.raises(RuntimeError): |
| 1878 | + with _cache_lock("astropy", need_write=True): |
| 1879 | + pass |
| 1880 | + # Ensure that the cache doesn't accidentally get unlocked! |
| 1881 | + with pytest.raises(RuntimeError): |
| 1882 | + with _cache_lock("astropy", need_write=True): |
| 1883 | + pass |
| 1884 | + # Download |
| 1885 | + with pytest.raises(RuntimeError): |
| 1886 | + is_url_in_cache(u), "if locked, act like cache empty" |
| 1887 | + with pytest.raises(RuntimeError): |
| 1888 | + download_file(u, cache=True, sources=[]) |
| 1889 | + with pytest.raises(RuntimeError): |
| 1890 | + download_file(u, cache=True) |
| 1891 | + clear_download_cache() # breaks lock |
| 1892 | + is_url_in_cache(u) |
0 commit comments