Last active
March 14, 2025 16:34
-
-
Save eschechter/26c06a5f05e15af69b335c27a59dcfa9 to your computer and use it in GitHub Desktop.
Convert zarr stored on s3 from v2 format to v3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Script to migrate v2 zarr stored on s3 to zarr v3 | |
| # NOTE: This script is experimental and should be used cautiously. | |
| # It will leave existing zarr v2 metadata in place, so the zarr can still be opened | |
| # with zarr_format=2 after conversion, but further experimentation is needed to make sure | |
| # the zarr is consistent when opened with zarr_format=3 | |
| # TODO: Make script compatible with other fsspec filesystems | |
| import json | |
| from enum import Enum | |
| import s3fs | |
| V3_METADATA_NAME = "zarr.json" | |
| # paths to ignore when looking for arrays | |
| GROUP_JSON_FILES = [".zmetadata", ".zgroup", ".zattrs", "zarr.json"] | |
| # convert zarr v2 dtype to v3 data_type | |
| DTYPE_TO_DATA_TYPE = { | |
| "|b1": "bool", | |
| "bool": "bool", | |
| "|i1": "int8", | |
| "<i2": "int16", | |
| "<i4": "int32", | |
| "<i8": "int64", | |
| "|u1": "uint8", | |
| "<u2": "uint16", | |
| "<u4": "uint32", | |
| "<u8": "uint64", | |
| "<f4": "float32", | |
| "<f8": "float64", | |
| } | |
| # Possible zarr v3 data_types | |
| class DataType(Enum): | |
| bool = "bool" | |
| int8 = "int8" | |
| int16 = "int16" | |
| int32 = "int32" | |
| int64 = "int64" | |
| uint8 = "uint8" | |
| uint16 = "uint16" | |
| uint32 = "uint32" | |
| uint64 = "uint64" | |
| float32 = "float32" | |
| float64 = "float64" | |
| @property | |
| def byte_count(self) -> int: | |
| data_type_byte_counts = { | |
| DataType.bool: 1, | |
| DataType.int8: 1, | |
| DataType.int16: 2, | |
| DataType.int32: 4, | |
| DataType.int64: 8, | |
| DataType.uint8: 1, | |
| DataType.uint16: 2, | |
| DataType.uint32: 4, | |
| DataType.uint64: 8, | |
| DataType.float32: 4, | |
| DataType.float64: 8, | |
| } | |
| return data_type_byte_counts[self] | |
| BLOSC_SHUFFLE_INT_TO_STRING = { | |
| 0: "noshuffle", | |
| 1: "shuffle", | |
| 2: "bitshuffle", | |
| } | |
| def _load_dict_from_path(s3_path: str) -> dict: | |
| """Load a dict from an s3_path to a JSON file | |
| Args: | |
| s3_path (str): full s3 path to json file | |
| Returns: | |
| dict: content of s3_path as dict | |
| """ | |
| return json.loads(s3fs.S3FileSystem().cat(s3_path)) | |
| def _write_zarr_json_to_path(metadata: dict, s3_path: str): | |
| """Write out v3-style metadata to zarr.json file | |
| Args: | |
| metadata (dict): v3-compatible metadata | |
| s3_path (str): prefix for zarr.json file to write | |
| Raises: | |
| ValueError: Raised if zarr.json already exists at s3_path | |
| """ | |
| fs = s3fs.S3FileSystem() | |
| path_to_write = f"{s3_path}/zarr.json" | |
| if fs.exists(path_to_write): | |
| raise ValueError(f"zarr.json file already exists at {s3_path}") | |
| with fs.open(path_to_write, "w") as f: | |
| json.dump(metadata, f) | |
| def _convert_group(s3_zarr_group_path: str): | |
| """Load group-level zarr v2 metadata and write out corresonding zarr.json | |
| Args: | |
| s3_zarr_group_path (str): root prefix of zarr to convert | |
| """ | |
| zattrs = _load_dict_from_path(f"{s3_zarr_group_path}/.zattrs") | |
| group_zarr_json = {"zarr_format": 3} | |
| group_zarr_json["node_type"] = "group" | |
| group_zarr_json["attributes"] = zattrs | |
| _write_zarr_json_to_path(group_zarr_json, s3_zarr_group_path) | |
| def _convert_array(s3_zarr_array_path: str): | |
| """Load array-level zarr v2 metadata and write out corresonding zarr.json | |
| Args: | |
| s3_zarr_array_path (str): prefix of array tp convert | |
| Raises: | |
| NotImplementedError: If array is compressed using compressor other than | |
| blosc or zstd | |
| """ | |
| zattrs = _load_dict_from_path(f"{s3_zarr_array_path}/.zattrs") | |
| zarray = _load_dict_from_path(f"{s3_zarr_array_path}/.zarray") | |
| array_zarr_json = {"zarr_format": 3} | |
| array_zarr_json["node_type"] = "array" | |
| array_zarr_json["attributes"] = zattrs | |
| data_type = DataType[DTYPE_TO_DATA_TYPE[zarray["dtype"]]] | |
| array_zarr_json["data_type"] = data_type.name | |
| codecs = [] | |
| if zarray["order"] == "F": | |
| codecs.append({"name": "transpose", "configuration": {"order": "F"}}) | |
| codecs.append({"name": "bytes", "configuration": {"endian": "little"}}) | |
| if zarray.get("compressor"): | |
| if zarray["compressor"]["id"] == "blosc": | |
| compressor_metadata = { | |
| "name": "blosc", | |
| "configuration": { | |
| "typesize": data_type.byte_count, | |
| "cname": zarray["compressor"]["cname"], | |
| "clevel": zarray["compressor"]["clevel"], | |
| "shuffle": BLOSC_SHUFFLE_INT_TO_STRING[ | |
| zarray["compressor"].get("shuffle", 0) | |
| ], | |
| "blocksize": zarray["compressor"].get("blocksize", 0), | |
| }, | |
| } | |
| codecs.append(compressor_metadata) | |
| elif zarray["compressor"]["id"] == "zstd": | |
| compressor_metadata = { | |
| "name": "zstd", | |
| "configuration": {"level": zarray["compressor"]["level"]}, | |
| } | |
| codecs.append(compressor_metadata) | |
| else: | |
| raise NotImplementedError("only blosc and zstd compression supported") | |
| array_zarr_json["codecs"] = codecs | |
| array_zarr_json["chunk_grid"] = { | |
| "configuration": {"chunk_shape": zarray["chunks"]}, | |
| "name": "regular", | |
| } | |
| array_zarr_json["fill_value"] = ( | |
| 0 if not zarray.get("fill_value") else zarray["fill_value"] | |
| ) | |
| # TODO: make compatible with chunk separators other than "." | |
| array_zarr_json["chunk_key_encoding"] = { | |
| "configuration": {"separator": "."}, | |
| "name": "v2", | |
| } | |
| array_zarr_json["shape"] = zarray["shape"] | |
| array_zarr_json["dimension_names"] = zattrs["_ARRAY_DIMENSIONS"] | |
| _write_zarr_json_to_path(array_zarr_json, s3_zarr_array_path) | |
| def convert_zarr_v2_to_v3(s3_zarr_group_path: str): | |
| """Main entrypoint for script. Converts group and array level metadata | |
| from v2 zarr format to v3 | |
| Args: | |
| s3_zarr_group_path (str): Path to zarr | |
| """ | |
| _convert_group(s3_zarr_group_path) | |
| fs = s3fs.S3FileSystem() | |
| array_paths = [ | |
| path | |
| for path in fs.ls(s3_zarr_group_path, detail=False) | |
| if path.split("/")[-1] not in GROUP_JSON_FILES | |
| ] | |
| for array_path in array_paths: | |
| _convert_array(array_path) | |
| if __name__ == "__main__": | |
| convert_zarr_v2_to_v3("s3://path/to.zarr") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment