Skip to content

Commit 55e725d

Browse files
author
Tarik Onalan
committed
Write subshape tests and remove old meta test
1 parent 637248c commit 55e725d

2 files changed

Lines changed: 181 additions & 43 deletions

File tree

‎zarr/tests/test_meta.py‎

Lines changed: 0 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -116,49 +116,6 @@ def test_encode_decode_array_2():
116116
assert [df.get_config()] == meta_dec['filters']
117117

118118

119-
def test_encode_decode_array_shape():
120-
121-
meta = dict(
122-
shape=(100,),
123-
chunks=(10,),
124-
dtype=np.dtype('(10, 10)f8'),
125-
compressor=Zlib(1).get_config(),
126-
fill_value=None,
127-
filters=None,
128-
order='C'
129-
)
130-
131-
meta_json = '''{
132-
"chunks": [10],
133-
"compressor": {"id": "zlib", "level": 1},
134-
"dtype": "<f8",
135-
"fill_value": null,
136-
"filters": null,
137-
"order": "C",
138-
"shape": [100, 10, 10],
139-
"zarr_format": %s
140-
}''' % ZARR_FORMAT
141-
142-
# test encoding
143-
meta_enc = encode_array_metadata(meta)
144-
assert_json_equal(meta_json, meta_enc)
145-
146-
# test decoding
147-
meta_dec = decode_array_metadata(meta_enc)
148-
assert ZARR_FORMAT == meta_dec['zarr_format']
149-
# NOTE(onalant): https://github.com/zarr-developers/zarr/pull/296#issuecomment-417608487
150-
# To maintain consistency with numpy unstructured arrays, unpack dimensions into shape.
151-
# assert meta['shape'] == meta_dec['shape']
152-
assert meta['chunks'] == meta_dec['chunks']
153-
# NOTE(onalant): https://github.com/zarr-developers/zarr/pull/296#issuecomment-417608487
154-
# To maintain consistency with numpy unstructured arrays, unpack dimensions into shape.
155-
# assert meta['dtype'] == meta_dec['dtype']
156-
assert meta['compressor'] == meta_dec['compressor']
157-
assert meta['order'] == meta_dec['order']
158-
assert meta_dec['fill_value'] is None
159-
assert meta_dec['filters'] is None
160-
161-
162119
def test_encode_decode_fill_values_nan():
163120

164121
fills = (

‎zarr/tests/test_subshapes.py‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
# -*- coding: utf-8 -*-
2+
from __future__ import absolute_import, print_function, division
3+
import unittest
4+
from tempfile import mkdtemp, mktemp
5+
import json
6+
import atexit
7+
import shutil
8+
import pickle
9+
import os
10+
import warnings
11+
12+
13+
import numpy as np
14+
from numpy.testing import assert_array_equal, assert_array_almost_equal
15+
import pytest
16+
17+
18+
from zarr.codecs import Zlib
19+
from zarr.compat import binary_type, text_type, PY2
20+
from zarr.creation import array, open_array
21+
from zarr.meta import (decode_array_metadata, encode_dtype, decode_dtype, ZARR_FORMAT,
22+
encode_array_metadata)
23+
24+
from zarr.storage import DirectoryStore
25+
26+
27+
if PY2: # pragma: py3 no cover
28+
warnings.resetwarnings()
29+
warnings.simplefilter('always')
30+
31+
32+
def assert_json_equal(expect, actual):
33+
if isinstance(expect, binary_type): # pragma: py3 no cover
34+
expect = text_type(expect, 'ascii')
35+
if isinstance(actual, binary_type):
36+
actual = text_type(actual, 'ascii')
37+
ej = json.loads(expect)
38+
aj = json.loads(actual)
39+
assert ej == aj
40+
41+
42+
class TestUnstructured(unittest.TestCase):
43+
44+
dtype = np.dtype('(10, 10)f8')
45+
46+
def test_encode_decode(self):
47+
48+
meta = dict(
49+
shape=(100,),
50+
chunks=(10,),
51+
dtype=TestUnstructured.dtype,
52+
compressor=Zlib(1).get_config(),
53+
fill_value=None,
54+
filters=None,
55+
order='C'
56+
)
57+
58+
meta_json = '''{
59+
"chunks": [10],
60+
"compressor": {"id": "zlib", "level": 1},
61+
"dtype": "<f8",
62+
"fill_value": null,
63+
"filters": null,
64+
"order": "C",
65+
"shape": [100, 10, 10],
66+
"zarr_format": %s
67+
}''' % ZARR_FORMAT
68+
69+
# test encoding
70+
meta_enc = encode_array_metadata(meta)
71+
assert_json_equal(meta_json, meta_enc)
72+
73+
# test decoding
74+
meta_dec = decode_array_metadata(meta_enc)
75+
assert ZARR_FORMAT == meta_dec['zarr_format']
76+
# NOTE(onalant): https://github.com/zarr-developers/zarr/pull/296#issuecomment-417608487
77+
# To maintain consistency with numpy unstructured arrays, unpack dimensions into shape.
78+
# assert meta['shape'] == meta_dec['shape']
79+
assert meta['chunks'] == meta_dec['chunks']
80+
# NOTE(onalant): https://github.com/zarr-developers/zarr/pull/296#issuecomment-417608487
81+
# To maintain consistency with numpy unstructured arrays, unpack dimensions into shape.
82+
# assert meta['dtype'] == meta_dec['dtype']
83+
assert meta['compressor'] == meta_dec['compressor']
84+
assert meta['order'] == meta_dec['order']
85+
assert meta_dec['fill_value'] is None
86+
assert meta_dec['filters'] is None
87+
88+
89+
def test_write_read(self):
90+
91+
path = mkdtemp()
92+
atexit.register(shutil.rmtree, path)
93+
shape = (100, 100)
94+
sbytes = np.random.bytes(np.product(shape) * TestUnstructured.dtype.itemsize)
95+
shape += TestUnstructured.dtype.shape
96+
s = np.frombuffer(sbytes, dtype=TestUnstructured.dtype).reshape(shape)
97+
98+
store = DirectoryStore(path)
99+
z = array(s, store=store)
100+
101+
assert(s.shape == z.shape)
102+
assert(s.tobytes() == z[:].tobytes())
103+
104+
del store
105+
del z
106+
107+
store = DirectoryStore(path)
108+
z = open_array(store)
109+
110+
assert(s.shape == z.shape)
111+
assert(s.tobytes() == z[:].tobytes())
112+
113+
114+
class TestStructured(unittest.TestCase):
115+
116+
dtype = np.dtype('i8, (10, 10)f8, (5, 10, 15)u1')
117+
118+
def test_encode_decode(self):
119+
120+
meta = dict(
121+
shape=(100,),
122+
chunks=(10,),
123+
dtype=np.dtype('i8, (10, 10)f8, (5, 10, 15)u1'),
124+
compressor=Zlib(1).get_config(),
125+
fill_value=None,
126+
filters=None,
127+
order='C'
128+
)
129+
130+
meta_json = '''{
131+
"chunks": [10],
132+
"compressor": {"id": "zlib", "level": 1},
133+
"dtype": [["f0", "<i8"], ["f1", "<f8", [10, 10]], ["f2", "|u1", [5, 10, 15]]],
134+
"fill_value": null,
135+
"filters": null,
136+
"order": "C",
137+
"shape": [100],
138+
"zarr_format": %s
139+
}''' % ZARR_FORMAT
140+
141+
# test encoding
142+
meta_enc = encode_array_metadata(meta)
143+
assert_json_equal(meta_json, meta_enc)
144+
145+
# test decoding
146+
meta_dec = decode_array_metadata(meta_enc)
147+
assert ZARR_FORMAT == meta_dec['zarr_format']
148+
assert meta['shape'] == meta_dec['shape']
149+
assert meta['chunks'] == meta_dec['chunks']
150+
assert meta['dtype'] == meta_dec['dtype']
151+
assert meta['compressor'] == meta_dec['compressor']
152+
assert meta['order'] == meta_dec['order']
153+
assert meta_dec['fill_value'] is None
154+
assert meta_dec['filters'] is None
155+
156+
157+
def test_write_read(self):
158+
159+
path = mkdtemp()
160+
atexit.register(shutil.rmtree, path)
161+
shape = (100, 100)
162+
sbytes = np.random.bytes(np.product(shape) * TestUnstructured.dtype.itemsize)
163+
shape += TestUnstructured.dtype.shape
164+
s = np.frombuffer(sbytes, dtype=TestUnstructured.dtype).reshape(shape)
165+
166+
store = DirectoryStore(path)
167+
z = array(s, store=store)
168+
169+
assert(s.shape == z.shape)
170+
assert(s.tobytes() == z[:].tobytes())
171+
172+
del store
173+
del z
174+
175+
store = DirectoryStore(path)
176+
z = open_array(store)
177+
178+
assert(s.shape == z.shape)
179+
assert(s.tobytes() == z[:].tobytes())
180+
181+

0 commit comments

Comments
 (0)