Skip to content

Commit f281ac3

Browse files
ieivanovclaude
andauthored
Implement version-specific default chunk and shard sizes in iohub utils (#403)
* add sharding defaults to `create_empty_plate` * add tests * tighten hypothesis index strategies to sorted, unique lists apply_transform_czyx_setup and process_single_position_setup now draw integer time_indices / channel_indices with unique=True and .map(sorted). That matches the pattern real callers pass and keeps the oindex selectors on the zarrs fast path -- unsorted or duplicate integer arrays cause ZarrsCodecPipeline to raise UnsupportedVIndexingError and fall back to BatchedCodecPipeline, which hits zarr-developers/zarr-python#2834 on sharded v0.5 stores (tracked in iohub#404). Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]> * move V04_MAX_CHUNK_SIZE_BYTES to compat * update sharding test --------- Co-authored-by: Claude Opus 4.7 (1M context) <[email protected]>
1 parent efff544 commit f281ac3

3 files changed

Lines changed: 324 additions & 24 deletions

File tree

‎src/iohub/core/compat.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@
77
NGFF_TO_ZARR_FORMAT: dict[str, int] = {"0.4": 2, "0.5": 3}
88
ZARR_FORMAT_TO_NGFF: dict[int, str] = {2: "0.4", 3: "0.5"}
99

10+
#: Maximum chunk size in bytes for the v0.4 default-chunk calculation.
11+
#: v0.4 (Zarr v2) does not support sharding, so the default chunk shape
12+
#: ``(1, 1, Z, Y, X)`` has Z halved until the single-chunk byte size fits
13+
#: under this cap.
14+
V04_MAX_CHUNK_SIZE_BYTES: float = 500e6
15+
1016

1117
def zarr_format_for_version(version: NGFFVersion) -> ZarrFormat:
1218
"""Map NGFF version string to zarr format integer."""

‎src/iohub/ngff/utils.py‎

Lines changed: 47 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,15 @@
1515
import numpy as np
1616
from numpy.typing import DTypeLike, NDArray
1717

18+
from iohub.core.compat import V04_MAX_CHUNK_SIZE_BYTES
1819
from iohub.ngff import open_ome_zarr
1920
from iohub.ngff.nodes import TransformationMeta
2021

2122

23+
#: Default ZYX chunk size for OME-Zarr v0.5 stores: ~2 MB at uint16 / ~4 MB at float32.
24+
_V05_DEFAULT_ZYX_CHUNKS: tuple[int, int, int] = (16, 256, 256)
25+
26+
2227
def create_empty_plate(
2328
store_path: Path,
2429
position_keys: list[tuple[str, str, str]],
@@ -29,7 +34,6 @@ def create_empty_plate(
2934
version: Literal["0.4", "0.5"] = "0.5",
3035
scale: tuple[float, ...] = (1, 1, 1, 1, 1),
3136
dtype: DTypeLike = np.float32,
32-
max_chunk_size_bytes: float = 500e6,
3337
) -> None:
3438
"""
3539
Create a new HCS Plate in OME-Zarr format if the plate does not exist.
@@ -50,22 +54,24 @@ def create_empty_plate(
5054
shape : tuple[int, ...]
5155
TCZYX shape of the plate.
5256
chunks : tuple[int, ...], optional
53-
TCZYX chunk size of the plate. If None, the chunk size is calculated
54-
based on the shape to be less than max_chunk_size_bytes.
57+
TCZYX chunk size of the plate. If None, a version-specific default
58+
is used:
59+
- "0.4": ``(1, 1, Z, Y, X)`` capped in Z to 500 MB.
60+
- "0.5": ``(1, 1, 16, 256, 256)`` clamped to ``shape``.
5561
Defaults to None.
5662
shards_ratio : tuple[int, ...], optional
57-
TCZYX shards ratio of the plate.
58-
If None, no sharding is applied.
63+
TCZYX shards ratio of the plate (shard size = chunks * shards_ratio).
64+
If None, a version-specific default is used:
65+
- "0.4": no sharding (Zarr v2 does not support it).
66+
- "0.5": ratio that yields a shard size of ``(1, 1, Z, Y, X)``.
5967
Defaults to None.
6068
version : Literal["0.4", "0.5"], optional
6169
OME-Zarr version to use for the plate.
62-
Defaults to "0.4".
70+
Defaults to "0.5".
6371
scale : tuple[float, ...], optional
6472
TCZYX scale of the plate. Defaults to (1, 1, 1, 1, 1).
6573
dtype : DTypeLike, optional
6674
Data type of the plate. Defaults to np.float32.
67-
max_chunk_size_bytes : float, optional
68-
Maximum chunk size in bytes. Defaults to 500e6 (500MB).
6975
7076
Examples
7177
--------
@@ -101,20 +107,16 @@ def create_empty_plate(
101107
102108
Notes
103109
-----
104-
- If `chunks` is not provided, the function calculates an appropriate
105-
chunk size to keep the chunks under the specified `max_chunk_size_bytes`.
110+
- If `chunks` is not provided, a version-specific default is used (see
111+
parameter description).
106112
- The function ensures that positions and channels are appended to an
107113
existing plate if they are not already present.
108114
"""
109-
# Limiting the chunking to 500MB
110115
if chunks is None:
111-
chunk_zyx_shape = _limit_zyx_chunk_size(
112-
shape,
113-
np.dtype(dtype).itemsize,
114-
max_chunk_size_bytes,
115-
)
116+
chunks = _default_chunks(shape, dtype, version)
116117

117-
chunks = 2 * (1,) + chunk_zyx_shape
118+
if shards_ratio is None and version == "0.5":
119+
shards_ratio = _default_shards_ratio(shape, chunks)
118120

119121
# Create plate
120122
output_plate = open_ome_zarr(
@@ -478,3 +480,31 @@ def _clamp_chunks_to_shape(shape: tuple[int, ...], chunks: tuple[int, ...]) -> t
478480
Chunk sizes clamped to at most the dimension size.
479481
"""
480482
return tuple(min(c, d) for c, d in zip(chunks, shape, strict=False))
483+
484+
485+
def _default_chunks(
486+
shape: tuple[int, ...],
487+
dtype: DTypeLike,
488+
version: Literal["0.4", "0.5"],
489+
) -> tuple[int, ...]:
490+
"""Version-specific default TCZYX chunk size for a plate array."""
491+
if version == "0.4":
492+
chunk_zyx_shape = _limit_zyx_chunk_size(
493+
shape,
494+
np.dtype(dtype).itemsize,
495+
V04_MAX_CHUNK_SIZE_BYTES,
496+
)
497+
return (1, 1, *chunk_zyx_shape)
498+
# v0.5: DCA-aligned small chunks, clamped to the array shape.
499+
return _clamp_chunks_to_shape(shape, (1, 1, *_V05_DEFAULT_ZYX_CHUNKS))
500+
501+
502+
def _default_shards_ratio(
503+
shape: tuple[int, ...],
504+
chunks: tuple[int, ...],
505+
) -> tuple[int, ...]:
506+
"""Shards ratio yielding a shard of (1, 1, Z, Y, X)."""
507+
ratios = [1, 1]
508+
for dim, chunk in zip(shape[-3:], chunks[-3:], strict=False):
509+
ratios.append(-(-dim // chunk))
510+
return tuple(ratios)

0 commit comments

Comments
 (0)