|
4 | 4 | import pytest |
5 | 5 |
|
6 | 6 | from fsspec.implementations.local import LocalFileSystem, make_path_posix |
| 7 | +from fsspec.implementations.memory import MemoryFileSystem |
7 | 8 |
|
8 | 9 |
|
9 | 10 | def test_1(m): |
@@ -407,3 +408,95 @@ def test_open_path_windows(m): |
407 | 408 | f.write(b"some\nlines\nof\ntext") |
408 | 409 |
|
409 | 410 | assert m.read_text(path) == "some\nlines\nof\ntext" |
| 411 | + |
| 412 | + |
| 413 | +def test_find_matches_generic(m): |
| 414 | + # MemoryFileSystem overrides find() with a single-pass implementation; make |
| 415 | + # sure it agrees with the generic ls()-based AbstractFileSystem.find() across |
| 416 | + # roots, maxdepth, withdirs and detail. |
| 417 | + from fsspec.spec import AbstractFileSystem |
| 418 | + |
| 419 | + for path in [ |
| 420 | + "/data/a/f1.txt", |
| 421 | + "/data/a/f2.txt", |
| 422 | + "/data/a/b/deep.txt", |
| 423 | + "/data/a/b/c/deepest.txt", |
| 424 | + "/data/x.txt", |
| 425 | + "/data/y/z.txt", |
| 426 | + "/other/o.txt", |
| 427 | + ]: |
| 428 | + m.pipe_file(path, b"hello") |
| 429 | + m.mkdir("/data/emptydir") # empty (pseudo) directory |
| 430 | + m.mkdir("/data/a/b/emptysub") |
| 431 | + |
| 432 | + for root in ["", "/data", "/data/a", "/data/a/b", "/data/x.txt", "/nope"]: |
| 433 | + for maxdepth in [None, 1, 2, 3]: |
| 434 | + for withdirs in [False, True]: |
| 435 | + for detail in [False, True]: |
| 436 | + got = m.find( |
| 437 | + root, maxdepth=maxdepth, withdirs=withdirs, detail=detail |
| 438 | + ) |
| 439 | + expected = AbstractFileSystem.find( |
| 440 | + m, root, maxdepth=maxdepth, withdirs=withdirs, detail=detail |
| 441 | + ) |
| 442 | + assert got == expected, (root, maxdepth, withdirs, detail) |
| 443 | + |
| 444 | + |
| 445 | +def test_find_snapshots_store_before_iterating(m): |
| 446 | + # `store` is a class attribute shared by every MemoryFileSystem instance, so |
| 447 | + # another instance can add or remove a path while find() is walking it. |
| 448 | + # ls() iterates over a snapshot for this reason; find() must too, or it dies |
| 449 | + # with "dictionary changed size during iteration". |
| 450 | + for f in range(5): |
| 451 | + m.pipe_file(f"/data/file{f}.txt", b"x") |
| 452 | + |
| 453 | + class MutatesStoreOnRead: |
| 454 | + """Stands in for a real entry, but writes to the store when read. |
| 455 | +
|
| 456 | + Reading ``size`` happens inside find()'s loop, so this reproduces a |
| 457 | + concurrent write landing mid-iteration without needing a second thread. |
| 458 | + """ |
| 459 | + |
| 460 | + def __init__(self, real, store): |
| 461 | + self._real = real |
| 462 | + self._store = store |
| 463 | + self._fired = False |
| 464 | + |
| 465 | + @property |
| 466 | + def created(self): |
| 467 | + return self._real.created |
| 468 | + |
| 469 | + @property |
| 470 | + def size(self): |
| 471 | + if not self._fired: |
| 472 | + self._fired = True |
| 473 | + self._store["/data/concurrent.txt"] = self._real |
| 474 | + return self._real.size |
| 475 | + |
| 476 | + real = m.store["/data/file0.txt"] |
| 477 | + m.store["/data/file0.txt"] = MutatesStoreOnRead(real, m.store) |
| 478 | + |
| 479 | + out = m.find("/data") |
| 480 | + |
| 481 | + # The snapshot is taken before the concurrent write, so the new path is not |
| 482 | + # part of this result; the point is that find() completes instead of raising. |
| 483 | + assert "/data/file4.txt" in out |
| 484 | + assert len(out) == 5 |
| 485 | + |
| 486 | + |
| 487 | +def test_find_does_not_scan_per_directory(m): |
| 488 | + # Regression guard: the old find() called ls() once per directory and each |
| 489 | + # ls() re-scanned the whole (global) store, giving O(n_dirs * n_files) work. |
| 490 | + # The single-pass implementation must not call ls() at all, so total work |
| 491 | + # stays O(n_files) regardless of how many directories there are. |
| 492 | + from unittest import mock |
| 493 | + |
| 494 | + for d in range(20): |
| 495 | + for f in range(5): |
| 496 | + m.pipe_file(f"/data/dir{d}/file{f}.txt", b"x") |
| 497 | + |
| 498 | + with mock.patch.object(MemoryFileSystem, "ls", wraps=m.ls) as spy: |
| 499 | + out = m.find("/data") |
| 500 | + |
| 501 | + assert len(out) == 100 |
| 502 | + assert spy.call_count == 0 |
0 commit comments