|
12 | 12 | ``` |
13 | 13 |
|
14 | 14 | Selection construction does not read source values: `result()`, `__array__`, |
15 | | -and eager `__getitem__` perform reads. Tokenization hashes plain NumPy source |
16 | | -data or delegates to an explicit source hook. `.lazy` operations inspect |
| 15 | +and eager `__getitem__` perform reads. Source tokenization is delegated to |
| 16 | +Dask and may inspect source values. `.lazy` operations inspect |
17 | 17 | selection metadata and may copy or process supplied index arrays. Composition |
18 | 18 | does not accumulate wrapper layers: a view of a view is still a single transform |
19 | 19 | and retains its reader. |
|
137 | 137 | import hashlib |
138 | 138 | import json |
139 | 139 | import math |
140 | | -import mmap |
141 | 140 | import operator |
142 | 141 | from collections.abc import Sequence |
143 | 142 | from dataclasses import dataclass, field |
@@ -521,66 +520,6 @@ def _validate_prepared_parts(parts: Sequence[Partition], out_shape: tuple[int, . |
521 | 520 | raise ValueError("prepared parts do not tile the view exactly") |
522 | 521 |
|
523 | 522 |
|
524 | | -# --------------------------------------------------------------------------- # |
525 | | -# Tokenization |
526 | | -# --------------------------------------------------------------------------- # |
527 | | - |
528 | | - |
529 | | -def _validate_token_dtype(dtype: np.dtype[Any]) -> None: |
530 | | - """Require dtype attributes with a deterministic, value-based representation.""" |
531 | | - if dtype.metadata is not None: |
532 | | - raise TypeError("Dtype metadata requires an explicit source __dask_tokenize__ hook.") |
533 | | - if dtype.subdtype is not None: |
534 | | - _validate_token_dtype(dtype.subdtype[0]) |
535 | | - if dtype.fields is not None: |
536 | | - for field in dtype.fields.values(): |
537 | | - if len(field) == 3 and type(field[2]) is not str: |
538 | | - raise TypeError( |
539 | | - "Non-string field titles require an explicit source __dask_tokenize__ hook." |
540 | | - ) |
541 | | - _validate_token_dtype(field[0]) |
542 | | - |
543 | | - |
544 | | -def _numpy_values_token(array: np.ndarray[Any, Any]) -> Any: |
545 | | - """Hash field values in C order, excluding structured padding bytes.""" |
546 | | - if array.dtype.fields is not None: |
547 | | - return tuple(_numpy_values_token(array[name]) for name in array.dtype.names or ()) |
548 | | - return hashlib.sha256(array.tobytes(order="C")).hexdigest() |
549 | | - |
550 | | - |
551 | | -def _wrapped_token(array: Any) -> Any: |
552 | | - """Identify supported source contents without converting foreign arrays. |
553 | | -
|
554 | | - Explicit hooks own their versioning, determinism, and I/O contract. |
555 | | - Supported plain NumPy arrays are hashed field by field on every call. Other |
556 | | - sources must provide a hook; neither serialization nor identity is a safe |
557 | | - substitute for a source's value/version contract. |
558 | | - """ |
559 | | - hook = getattr(array, "__dask_tokenize__", None) |
560 | | - if hook is not None: |
561 | | - return hook() |
562 | | - if type(array) is not np.ndarray or array.dtype.hasobject: |
563 | | - msg = ( |
564 | | - "Tokenization requires a source __dask_tokenize__ hook or a plain " |
565 | | - "NumPy ndarray without object fields. For Dask from_array, use " |
566 | | - "name=False to bypass content tokenization." |
567 | | - ) |
568 | | - raise TypeError(msg) |
569 | | - base: Any = array.base |
570 | | - while isinstance(base, (np.ndarray, memoryview, mmap.mmap)): |
571 | | - if isinstance(base, (np.memmap, mmap.mmap)): |
572 | | - msg = "Memory-mapped sources require an explicit __dask_tokenize__ hook." |
573 | | - raise TypeError(msg) |
574 | | - base = base.obj if isinstance(base, memoryview) else base.base |
575 | | - _validate_token_dtype(array.dtype) |
576 | | - return ( |
577 | | - "numpy.ndarray", |
578 | | - array.shape, |
579 | | - repr(array.dtype), |
580 | | - _numpy_values_token(array), |
581 | | - ) |
582 | | - |
583 | | - |
584 | 523 | # --------------------------------------------------------------------------- # |
585 | 524 | # The wrapper |
586 | 525 | # --------------------------------------------------------------------------- # |
@@ -1245,27 +1184,18 @@ def __dask_tokenize__(self) -> Any: |
1245 | 1184 | tokens produce equal tokens; arbitrary semantically equivalent mappings |
1246 | 1185 | are not guaranteed to serialize identically. |
1247 | 1186 |
|
1248 | | - Supported plain NumPy arrays are hashed in full on each call. Structured |
1249 | | - fields are hashed separately, excluding padding; overlapping fields |
1250 | | - are each visited. Time scales with the bytes visited, and temporary |
1251 | | - memory with the largest field buffer. Dtype metadata and non-string |
1252 | | - field titles require an explicit source hook, including in nested dtypes. |
1253 | | - Known NumPy/mmap backing is rejected through ndarray base and |
1254 | | - memoryview object chains; arbitrary buffer provenance is not inferred. |
1255 | | - Array subclasses, object arrays, and foreign sources require a source `__dask_tokenize__` hook; unsupported sources |
1256 | | - raise `TypeError`. Hooks own determinism, versioning, and any I/O, and |
1257 | | - hook exceptions propagate. Installing Dask does not change this policy. |
1258 | | -
|
1259 | | - The reader and partitioning are omitted under the contract that they |
1260 | | - preserve values. A token describes the source at tokenization time; |
1261 | | - mutation after graph construction does not invalidate existing Dask |
1262 | | - keys. Concurrent mutation during hashing is unsupported. This method |
1263 | | - does not provide a persistent cache identity or a source snapshot. |
| 1187 | + Dask tokenizes the wrapped source using its normal dispatch and |
| 1188 | + determinism policy. This may read or hash source values. Dask is |
| 1189 | + imported only when this method is called and is otherwise optional. |
| 1190 | + The reader and partitioning are omitted because they must preserve |
| 1191 | + values. Mutating a source does not update keys in existing Dask graphs. |
1264 | 1192 | """ |
| 1193 | + from dask.base import tokenize # pyright: ignore[reportMissingImports] |
| 1194 | + |
1265 | 1195 | canonical = json.dumps(self._transform.to_json(), sort_keys=True) |
1266 | 1196 | return ( |
1267 | 1197 | type(self).__qualname__, |
1268 | | - _wrapped_token(self._array), |
| 1198 | + tokenize(self._array), |
1269 | 1199 | hashlib.sha256(canonical.encode()).hexdigest(), |
1270 | 1200 | ) |
1271 | 1201 |
|
|
0 commit comments