Error with combining kerchunk mappings with MultiZarrToZarr #483

Activity

1. a-nu-ragn commented on Jul 25, 2024

   

   a-nu-ragn

   on Jul 25, 2024

   Contributor

   More actions

   Here is the contents of the mzz variable:

   That is the not the content of the mzz variable. It is the list of kerchunk'd references
2. martindurant commented on Jul 25, 2024

   

   martindurant

   on Jul 25, 2024

   Member

   More actions

   This has come up before, I'll try to find it.

   For completeness, here is what mzz looks like for me:

   {'version': 1,
    'refs': {'.zgroup': '{"zarr_format":2}',
     'y/.zarray': '{\n    "chunks": [\n        2\n    ],\n    "compressor": null,\n    "dtype": "<i8",\n    "fill_value": null,\n    "filters": null,\n    "order": "C",\n    "shape": [\n        2\n    ],\n    "zarr_format": 2\n}',
     'y/0': '\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00',
     'y/.zattrs': '{\n    "_ARRAY_DIMENSIONS": [\n        "y"\n    ]\n}',
     'A/.zarray': '{"chunks":[1000,1],"compressor":null,"dtype":"<f8","fill_value":"NaN","filters":null,"order":"C","shape":[1000,2],"zarr_format":2}',
     'A/.zattrs': '{"_ARRAY_DIMENSIONS":["x","y"]}',
     'A/0.0': ['ds1.nc', 17256, 8000],
     'x/.zarray': '{"chunks":[1000],"compressor":null,"dtype":"<i8","fill_value":null,"filters":null,"order":"C","shape":[1000],"zarr_format":2}',
     'x/.zattrs': '{"_ARRAY_DIMENSIONS":["x"]}',
     'x/0': ['ds1.nc', 983, 8000],
     'A/0.1': ['ds2.nc', 17256, 8000]}}

   and here is the exception

   File ~/conda/envs/py310/lib/python3.10/site-packages/xarray/core/indexing.py:554, in LazilyIndexedArray.get_duck_array(self)
       553 def get_duck_array(self):
   --> 554     array = self.array[self.key]
       555     # self.array[self.key] is now a numpy array when
       556     # self.array is a BackendArray subclass
       557     # and self.key is BasicIndexer((slice(None, None, None),))
       558     # so we need the explicit check for ExplicitlyIndexed
       559     if isinstance(array, ExplicitlyIndexed):
   
   File ~/conda/envs/py310/lib/python3.10/site-packages/xarray/backends/zarr.py:94, in ZarrArrayWrapper.__getitem__(self, key)
        92 array = self.get_array()
        93 if isinstance(key, indexing.BasicIndexer):
   ---> 94     return array[key.tuple]
        95 elif isinstance(key, indexing.VectorizedIndexer):
        96     return array.vindex[
        97         indexing._arrayize_vectorized_indexer(key, self.shape).tuple
        98     ]
   
   File ~/code/zarr/zarr/core.py:798, in Array.__getitem__(self, selection)
       796     result = self.vindex[selection]
       797 elif is_pure_orthogonal_indexing(pure_selection, self.ndim):
   --> 798     result = self.get_orthogonal_selection(pure_selection, fields=fields)
       799 else:
       800     result = self.get_basic_selection(pure_selection, fields=fields)
   
   File ~/code/zarr/zarr/core.py:1080, in Array.get_orthogonal_selection(self, selection, out, fields)
      1077 # setup indexer
      1078 indexer = OrthogonalIndexer(selection, self)
   -> 1080 return self._get_selection(indexer=indexer, out=out, fields=fields)
   
   File ~/code/zarr/zarr/core.py:1343, in Array._get_selection(self, indexer, out, fields)
      1340 if math.prod(out_shape) > 0:
      1341     # allow storage to get multiple items at once
      1342     lchunk_coords, lchunk_selection, lout_selection = zip(*indexer)
   -> 1343     self._chunk_getitems(
      1344         lchunk_coords,
      1345         lchunk_selection,
      1346         out,
      1347         lout_selection,
      1348         drop_axes=indexer.drop_axes,
      1349         fields=fields,
      1350     )
      1351 if out.shape:
      1352     return out
   
   File ~/code/zarr/zarr/core.py:2183, in Array._chunk_getitems(self, lchunk_coords, lchunk_selection, out, lout_selection, drop_axes, fields)
      2181 for ckey, chunk_select, out_select in zip(ckeys, lchunk_selection, lout_selection):
      2182     if ckey in cdatas:
   -> 2183         self._process_chunk(
      2184             out,
      2185             cdatas[ckey],
      2186             chunk_select,
      2187             drop_axes,
      2188             out_is_ndarray,
      2189             fields,
      2190             out_select,
      2191             partial_read_decode=partial_read_decode,
      2192         )
      2193     else:
      2194         # check exception type
      2195         if self._fill_value is not None:
   
   File ~/code/zarr/zarr/core.py:2057, in Array._process_chunk(self, out, cdata, chunk_selection, drop_axes, out_is_ndarray, fields, out_selection, partial_read_decode)
      2055             chunk = ensure_ndarray_like(cdata).view(self._dtype)
      2056             chunk = chunk.reshape(self._chunks, order=self._order)
   -> 2057             np.copyto(dest, chunk)
      2058         return
      2060 # decode chunk
   
   ValueError: could not broadcast input array from shape (1000,1) into shape (1000,)
   

   👍React with 👍1Reacted by John Ragland