Repository navigation
Expand file tree
/
Copy path_local.py
More file actions
446 lines (369 loc) · 14.6 KB
/
Copy path_local.py
File metadata and controls
446 lines (369 loc) · 14.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
from __future__ import annotations
import asyncio
import contextlib
import io
import os
import shutil
import sys
import time
import uuid
from pathlib import Path
from typing import TYPE_CHECKING, BinaryIO, Literal, Self
from zarr.abc.store import (
ByteRequest,
OffsetByteRequest,
RangeByteRequest,
Store,
SuffixByteRequest,
)
from zarr.core.buffer import Buffer
from zarr.core.buffer.core import default_buffer_prototype
from zarr.core.common import AccessModeLiteral, concurrent_map
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Callable, Iterable, Iterator
from zarr.core.buffer import BufferPrototype
def _get(path: Path, prototype: BufferPrototype, byte_range: ByteRequest | None) -> Buffer:
if byte_range is None:
return prototype.buffer.from_bytes(path.read_bytes())
with path.open("rb") as f:
size = f.seek(0, io.SEEK_END)
if isinstance(byte_range, RangeByteRequest):
f.seek(byte_range.start)
return prototype.buffer.from_bytes(f.read(byte_range.end - f.tell()))
elif isinstance(byte_range, OffsetByteRequest):
f.seek(byte_range.offset)
elif isinstance(byte_range, SuffixByteRequest):
f.seek(max(0, size - byte_range.suffix))
else:
raise TypeError(f"Unexpected byte_range, got {byte_range}.")
return prototype.buffer.from_bytes(f.read())
if sys.platform == "win32":
# Per the os.rename docs:
# On Windows, if dst exists a FileExistsError is always raised.
_safe_move = os.rename
else:
# On Unix, os.rename silently replace files, so instead we use os.link like
# atomicwrites:
# https://github.com/untitaker/python-atomicwrites/blob/1.4.1/atomicwrites/__init__.py#L59-L60
# This also raises FileExistsError if dst exists.
def _safe_move(src: Path, dst: Path) -> None:
os.link(src, dst)
os.unlink(src)
# Windows error codes that MoveFileEx reports when the destination could not be
# superseded. Replacing a name that was itself replaced moments earlier
# intermittently fails this way with no second process and no open handle
# involved, and a retry clears it in well under a millisecond. Zarr v2 hit the
# same thing and fixed it in #698.
#
# ERROR_ACCESS_DENIED is also what Windows reports for conditions that will not
# clear (the destination is a directory, is read-only, or is ACL-denied). Those
# are retried too and surface the same error after the bounded delay below.
#
# Nothing else is retried. In particular the FileExistsError that the exclusive
# path relies on to report an existing node is ERROR_ALREADY_EXISTS (183), so it
# still propagates on the first attempt.
_TRANSIENT_WINERRORS = frozenset(
{
5, # ERROR_ACCESS_DENIED
32, # ERROR_SHARING_VIOLATION
}
)
# Delay before each retry. The sequence sums to well under a second so that a
# genuine failure still surfaces promptly; in practice most transient failures
# clear on the first retry.
_RETRY_DELAYS = (0.001, 0.005, 0.02, 0.05, 0.2)
def _move_with_retry(tmp_path: Path, path: Path, move: Callable[[Path, Path], object]) -> None:
"""Run `move(tmp_path, path)`, retrying while the destination is busy.
This is a single attempt on every platform but Windows, without needing to
test for one: only `winerror` values are ever retried, and off Windows an
`OSError` does not carry one.
"""
for delay in _RETRY_DELAYS:
try:
move(tmp_path, path)
except OSError as e:
if getattr(e, "winerror", None) not in _TRANSIENT_WINERRORS:
raise
time.sleep(delay)
else:
return
move(tmp_path, path)
@contextlib.contextmanager
def _atomic_write(
path: Path,
mode: Literal["r+b", "wb"],
exclusive: bool = False,
) -> Iterator[BinaryIO]:
tmp_path = path.with_suffix(f".{uuid.uuid4().hex}.partial")
try:
with tmp_path.open(mode) as f:
yield f
if exclusive:
_move_with_retry(tmp_path, path, _safe_move)
else:
_move_with_retry(tmp_path, path, Path.replace)
except Exception:
tmp_path.unlink(missing_ok=True)
raise
def _put(path: Path, value: Buffer, exclusive: bool = False) -> int:
path.parent.mkdir(parents=True, exist_ok=True)
# write takes any object supporting the buffer protocol
view = value.as_buffer_like()
with _atomic_write(path, "wb", exclusive=exclusive) as f:
return f.write(view)
# The helpers below do the blocking filesystem work behind LocalStore's async methods.
# Each async method runs exactly one of them via `asyncio.to_thread` so that the event
# loop is never stalled on disk I/O; the synchronous methods call them directly.
def _ensure_root(root: Path, *, create: bool) -> None:
if create:
root.mkdir(parents=True, exist_ok=True)
if not root.exists():
raise FileNotFoundError(f"{root} does not exist")
def _clear(root: Path) -> None:
shutil.rmtree(root)
root.mkdir()
def _delete(path: Path) -> None:
if path.is_dir():
shutil.rmtree(path)
else:
path.unlink(missing_ok=True)
def _delete_dir(path: Path, prefix: str) -> None:
if path.is_dir():
shutil.rmtree(path)
elif path.is_file():
raise ValueError(f"delete_dir was passed a {prefix=!r} that is a file")
# A non-existent directory is a no-op; test_group:test_create_creates_parents relies on it.
def _exists(path: Path) -> bool:
return path.is_file()
def _getsize(path: Path) -> int:
return path.stat().st_size
def _list_files(root: Path, prefix: str) -> list[str]:
"""Keys (paths relative to `root`, POSIX style) of every file under `root / prefix`."""
to_strip = root.as_posix() + "/"
return [p.as_posix().removeprefix(to_strip) for p in (root / prefix).rglob("*") if p.is_file()]
def _list_dir(base: Path) -> list[str]:
try:
return [p.name for p in base.iterdir()]
except (FileNotFoundError, NotADirectoryError):
return []
def _move(src: Path, dest_root: Path) -> None:
dest_root.parent.mkdir(parents=True, exist_ok=True)
if dest_root.exists():
raise FileExistsError(f"Destination root {dest_root} already exists.")
shutil.move(src, dest_root)
class LocalStore(Store):
"""
Store for the local file system.
Parameters
----------
root : str or Path
Directory to use as root of store.
read_only : bool
Whether the store is read-only
Attributes
----------
supports_writes
supports_deletes
supports_listing
root
"""
supports_writes: bool = True
supports_deletes: bool = True
supports_listing: bool = True
root: Path
def __init__(self, root: Path | str, *, read_only: bool = False) -> None:
super().__init__(read_only=read_only)
if isinstance(root, str):
root = Path(root)
if not isinstance(root, Path):
raise TypeError(
f"'root' must be a string or Path instance. Got an instance of {type(root)} instead."
)
self.root = root
def with_read_only(self, read_only: bool = False) -> Self:
# docstring inherited
return type(self)(
root=self.root,
read_only=read_only,
)
@classmethod
async def open(
cls, root: Path | str, *, read_only: bool = False, mode: AccessModeLiteral | None = None
) -> Self:
"""
Create and open the store.
Parameters
----------
root : str or Path
Directory to use as root of store.
read_only : bool
Whether the store is read-only
mode :
Mode in which to create the store. This only affects opening the store,
and the final read-only state of the store is controlled through the
read_only parameter.
Returns
-------
Store
The opened store instance.
"""
# If mode = 'r+', want to open in read only mode (fail if exists),
# but return a writeable store
if mode is not None:
read_only_creation = mode in ["r", "r+"]
else:
read_only_creation = read_only
store = cls(root, read_only=read_only_creation)
await store._open()
# Set read_only state
store = store.with_read_only(read_only)
await store._open()
return store
async def _open(self, *, mode: AccessModeLiteral | None = None) -> None:
await asyncio.to_thread(_ensure_root, self.root, create=not self.read_only)
return await super()._open()
async def _ensure_open(self) -> None:
# docstring inherited
if not self._is_open:
# Concurrent lazy opens (every `set` of a `set_many`, say) can all get here
# before any of them finishes, so each one verifies the root. That is
# idempotent, and so is setting the flag, so the calls cannot conflict.
# Going through `self._open()` instead would make every call after the first
# fail, because `Store._open` raises on a store that is already open.
# As in `_ensure_open_sync`, a subclass's `_open` override is not run here.
await asyncio.to_thread(_ensure_root, self.root, create=not self.read_only)
self._is_open = True
async def clear(self) -> None:
# docstring inherited
self._check_writable()
await asyncio.to_thread(_clear, self.root)
def __str__(self) -> str:
return f"file://{self.root.as_posix()}"
def __repr__(self) -> str:
return f"LocalStore('{self}')"
def __eq__(self, other: object) -> bool:
return isinstance(other, type(self)) and self.root == other.root
# -------------------------------------------------------------------
# Synchronous store methods
# -------------------------------------------------------------------
def _ensure_open_sync(self) -> None:
if not self._is_open:
_ensure_root(self.root, create=not self.read_only)
self._is_open = True
def get_sync(
self,
key: str,
*,
prototype: BufferPrototype | None = None,
byte_range: ByteRequest | None = None,
) -> Buffer | None:
if prototype is None:
prototype = default_buffer_prototype()
self._ensure_open_sync()
path = self.root / key
try:
return _get(path, prototype, byte_range)
except (FileNotFoundError, IsADirectoryError, NotADirectoryError):
return None
def set_sync(self, key: str, value: Buffer) -> None:
self._ensure_open_sync()
self._check_writable()
if not isinstance(value, Buffer):
raise TypeError(
f"LocalStore.set(): `value` must be a Buffer instance. "
f"Got an instance of {type(value)} instead."
)
path = self.root / key
_put(path, value)
def delete_sync(self, key: str) -> None:
self._ensure_open_sync()
self._check_writable()
_delete(self.root / key)
async def get(
self,
key: str,
prototype: BufferPrototype | None = None,
byte_range: ByteRequest | None = None,
) -> Buffer | None:
# docstring inherited
if prototype is None:
prototype = default_buffer_prototype()
await self._ensure_open()
path = self.root / key
try:
return await asyncio.to_thread(_get, path, prototype, byte_range)
except (FileNotFoundError, IsADirectoryError, NotADirectoryError):
return None
async def get_partial_values(
self,
prototype: BufferPrototype,
key_ranges: Iterable[tuple[str, ByteRequest | None]],
) -> list[Buffer | None]:
# docstring inherited
args = []
for key, byte_range in key_ranges:
path = self.root / key
args.append((_get, path, prototype, byte_range))
return await concurrent_map(args, asyncio.to_thread, limit=None) # TODO: fix limit
async def set(self, key: str, value: Buffer) -> None:
# docstring inherited
return await self._set(key, value)
async def set_if_not_exists(self, key: str, value: Buffer) -> None:
# docstring inherited
try:
return await self._set(key, value, exclusive=True)
except FileExistsError:
pass
async def _set(self, key: str, value: Buffer, exclusive: bool = False) -> None:
await self._ensure_open()
self._check_writable()
if not isinstance(value, Buffer):
raise TypeError(
f"LocalStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead."
)
path = self.root / key
await asyncio.to_thread(_put, path, value, exclusive=exclusive)
async def delete(self, key: str) -> None:
"""
Remove a key from the store.
Parameters
----------
key : str
Notes
-----
If ``key`` is a directory within this store, the entire directory
at ``store.root / key`` is deleted.
"""
# docstring inherited
self._check_writable()
await asyncio.to_thread(_delete, self.root / key)
async def delete_dir(self, prefix: str) -> None:
# docstring inherited
self._check_writable()
await asyncio.to_thread(_delete_dir, self.root / prefix, prefix)
async def exists(self, key: str) -> bool:
# docstring inherited
return await asyncio.to_thread(_exists, self.root / key)
async def list(self) -> AsyncIterator[str]:
# docstring inherited
for key in await asyncio.to_thread(_list_files, self.root, ""):
yield key
async def list_prefix(self, prefix: str) -> AsyncIterator[str]:
# docstring inherited
for key in await asyncio.to_thread(_list_files, self.root, prefix.rstrip("/")):
yield key
async def list_dir(self, prefix: str) -> AsyncIterator[str]:
# docstring inherited
for name in await asyncio.to_thread(_list_dir, self.root / prefix):
yield name
async def move(self, dest_root: Path | str) -> None:
"""
Move the store to another path. The old root directory is deleted.
"""
if isinstance(dest_root, str):
dest_root = Path(dest_root)
await asyncio.to_thread(_move, self.root, dest_root)
self.root = dest_root
async def getsize(self, key: str) -> int:
# docstring inherited
return await asyncio.to_thread(_getsize, self.root / key)