Skip to content

Instantly share code, notes, and snippets.

@eschechter
Last active March 14, 2025 16:34
Show Gist options
  • Select an option

  • Save eschechter/26c06a5f05e15af69b335c27a59dcfa9 to your computer and use it in GitHub Desktop.

Select an option

Save eschechter/26c06a5f05e15af69b335c27a59dcfa9 to your computer and use it in GitHub Desktop.
Convert zarr stored on s3 from v2 format to v3
# Script to migrate v2 zarr stored on s3 to zarr v3
# NOTE: This script is experimental and should be used cautiously.
# It will leave existing zarr v2 metadata in place, so the zarr can still be opened
# with zarr_format=2 after conversion, but further experimentation is needed to make sure
# the zarr is consistent when opened with zarr_format=3
# TODO: Make script compatible with other fsspec filesystems
import json
from enum import Enum
import s3fs
V3_METADATA_NAME = "zarr.json"
# paths to ignore when looking for arrays
GROUP_JSON_FILES = [".zmetadata", ".zgroup", ".zattrs", "zarr.json"]
# convert zarr v2 dtype to v3 data_type
DTYPE_TO_DATA_TYPE = {
"|b1": "bool",
"bool": "bool",
"|i1": "int8",
"<i2": "int16",
"<i4": "int32",
"<i8": "int64",
"|u1": "uint8",
"<u2": "uint16",
"<u4": "uint32",
"<u8": "uint64",
"<f4": "float32",
"<f8": "float64",
}
# Possible zarr v3 data_types
class DataType(Enum):
bool = "bool"
int8 = "int8"
int16 = "int16"
int32 = "int32"
int64 = "int64"
uint8 = "uint8"
uint16 = "uint16"
uint32 = "uint32"
uint64 = "uint64"
float32 = "float32"
float64 = "float64"
@property
def byte_count(self) -> int:
data_type_byte_counts = {
DataType.bool: 1,
DataType.int8: 1,
DataType.int16: 2,
DataType.int32: 4,
DataType.int64: 8,
DataType.uint8: 1,
DataType.uint16: 2,
DataType.uint32: 4,
DataType.uint64: 8,
DataType.float32: 4,
DataType.float64: 8,
}
return data_type_byte_counts[self]
BLOSC_SHUFFLE_INT_TO_STRING = {
0: "noshuffle",
1: "shuffle",
2: "bitshuffle",
}
def _load_dict_from_path(s3_path: str) -> dict:
"""Load a dict from an s3_path to a JSON file
Args:
s3_path (str): full s3 path to json file
Returns:
dict: content of s3_path as dict
"""
return json.loads(s3fs.S3FileSystem().cat(s3_path))
def _write_zarr_json_to_path(metadata: dict, s3_path: str):
"""Write out v3-style metadata to zarr.json file
Args:
metadata (dict): v3-compatible metadata
s3_path (str): prefix for zarr.json file to write
Raises:
ValueError: Raised if zarr.json already exists at s3_path
"""
fs = s3fs.S3FileSystem()
path_to_write = f"{s3_path}/zarr.json"
if fs.exists(path_to_write):
raise ValueError(f"zarr.json file already exists at {s3_path}")
with fs.open(path_to_write, "w") as f:
json.dump(metadata, f)
def _convert_group(s3_zarr_group_path: str):
"""Load group-level zarr v2 metadata and write out corresonding zarr.json
Args:
s3_zarr_group_path (str): root prefix of zarr to convert
"""
zattrs = _load_dict_from_path(f"{s3_zarr_group_path}/.zattrs")
group_zarr_json = {"zarr_format": 3}
group_zarr_json["node_type"] = "group"
group_zarr_json["attributes"] = zattrs
_write_zarr_json_to_path(group_zarr_json, s3_zarr_group_path)
def _convert_array(s3_zarr_array_path: str):
"""Load array-level zarr v2 metadata and write out corresonding zarr.json
Args:
s3_zarr_array_path (str): prefix of array tp convert
Raises:
NotImplementedError: If array is compressed using compressor other than
blosc or zstd
"""
zattrs = _load_dict_from_path(f"{s3_zarr_array_path}/.zattrs")
zarray = _load_dict_from_path(f"{s3_zarr_array_path}/.zarray")
array_zarr_json = {"zarr_format": 3}
array_zarr_json["node_type"] = "array"
array_zarr_json["attributes"] = zattrs
data_type = DataType[DTYPE_TO_DATA_TYPE[zarray["dtype"]]]
array_zarr_json["data_type"] = data_type.name
codecs = []
if zarray["order"] == "F":
codecs.append({"name": "transpose", "configuration": {"order": "F"}})
codecs.append({"name": "bytes", "configuration": {"endian": "little"}})
if zarray.get("compressor"):
if zarray["compressor"]["id"] == "blosc":
compressor_metadata = {
"name": "blosc",
"configuration": {
"typesize": data_type.byte_count,
"cname": zarray["compressor"]["cname"],
"clevel": zarray["compressor"]["clevel"],
"shuffle": BLOSC_SHUFFLE_INT_TO_STRING[
zarray["compressor"].get("shuffle", 0)
],
"blocksize": zarray["compressor"].get("blocksize", 0),
},
}
codecs.append(compressor_metadata)
elif zarray["compressor"]["id"] == "zstd":
compressor_metadata = {
"name": "zstd",
"configuration": {"level": zarray["compressor"]["level"]},
}
codecs.append(compressor_metadata)
else:
raise NotImplementedError("only blosc and zstd compression supported")
array_zarr_json["codecs"] = codecs
array_zarr_json["chunk_grid"] = {
"configuration": {"chunk_shape": zarray["chunks"]},
"name": "regular",
}
array_zarr_json["fill_value"] = (
0 if not zarray.get("fill_value") else zarray["fill_value"]
)
# TODO: make compatible with chunk separators other than "."
array_zarr_json["chunk_key_encoding"] = {
"configuration": {"separator": "."},
"name": "v2",
}
array_zarr_json["shape"] = zarray["shape"]
array_zarr_json["dimension_names"] = zattrs["_ARRAY_DIMENSIONS"]
_write_zarr_json_to_path(array_zarr_json, s3_zarr_array_path)
def convert_zarr_v2_to_v3(s3_zarr_group_path: str):
"""Main entrypoint for script. Converts group and array level metadata
from v2 zarr format to v3
Args:
s3_zarr_group_path (str): Path to zarr
"""
_convert_group(s3_zarr_group_path)
fs = s3fs.S3FileSystem()
array_paths = [
path
for path in fs.ls(s3_zarr_group_path, detail=False)
if path.split("/")[-1] not in GROUP_JSON_FILES
]
for array_path in array_paths:
_convert_array(array_path)
if __name__ == "__main__":
convert_zarr_v2_to_v3("s3://path/to.zarr")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment