Write errors with latest Pyarrow release #1000

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[3], line 1
----> 1 blobs.write("test.zarr")

File ~/code/spatialdata/src/spatialdata/_utils.py:263, in _deprecation_alias.<locals>.deprecation_decorator.<locals>.wrapper(*args, **kwargs)
    261     raise ValueError("version for deprecation must be specified")
    262 rename_kwargs(f.__name__, kwargs, alias_copy, class_name, library, version)
--> 263 return f(*args, **kwargs)

File ~/code/spatialdata/src/spatialdata/_core/spatialdata.py:1177, in SpatialData.write(self, file_path, overwrite, consolidate_metadata, update_sdata_path, sdata_formats)
   1174 store.close()
   1176 for element_type, element_name, element in self.gen_elements():
-> 1177     self._write_element(
   1178         element=element,
   1179         zarr_container_path=file_path,
   1180         element_type=element_type,
   1181         element_name=element_name,
   1182         overwrite=False,
   1183         parsed_formats=parsed,
   1184     )
   1186 if self.path != file_path and update_sdata_path:
   1187     self.path = file_path

File ~/code/spatialdata/src/spatialdata/_core/spatialdata.py:1242, in SpatialData._write_element(self, element, zarr_container_path, element_type, element_name, overwrite, parsed_formats)
   1235     write_labels(
   1236         labels=element,
   1237         group=root_group,
   1238         name=element_name,
   1239         element_format=parsed_formats["raster"],
   1240     )
   1241 elif element_type == "points":
-> 1242     write_points(
   1243         points=element,
   1244         group=element_group,
   1245         element_format=parsed_formats["points"],
   1246     )
   1247 elif element_type == "shapes":
   1248     write_shapes(
   1249         shapes=element,
   1250         group=element_group,
   1251         element_format=parsed_formats["shapes"],
   1252     )

File ~/code/spatialdata/src/spatialdata/_io/io_points.py:83, in write_points(points, group, group_type, element_format)
     80         c = c.cat.as_known()
     81         points[column_name] = c
---> 83 points.to_parquet(path)
     85 attrs = element_format.attrs_to_dict(points.attrs)
     86 attrs["version"] = element_format.spatialdata_format_version

File ~/miniforge3/envs/scverse/lib/python3.13/site-packages/dask/dataframe/core.py:5646, in DataFrame.to_parquet(self, path, *args, **kwargs)
   5643 """See dd.to_parquet docstring for more information"""
   5644 from dask.dataframe.io import to_parquet
-> 5646 return to_parquet(self, path, *args, **kwargs)

File ~/miniforge3/envs/scverse/lib/python3.13/site-packages/dask/dataframe/io/parquet/core.py:952, in to_parquet(df, path, engine, compression, write_index, append, overwrite, ignore_divisions, partition_on, storage_options, custom_metadata, write_metadata_file, compute, compute_kwargs, schema, name_function, filesystem, **kwargs)
    943     raise ValueError(
    944         "User-defined key/value metadata (custom_metadata) can not "
    945         "contain a b'pandas' key.  This key is reserved by Pandas, "
    946         "and overwriting the corresponding value can render the "
    947         "entire dataset unreadable."
    948     )
    950 # Engine-specific initialization steps to write the dataset.
    951 # Possibly create parquet metadata, and load existing stuff if appending
--> 952 i_offset, fmd, metadata_file_exists, extra_write_kwargs = engine.initialize_write(
    953     df,
    954     fs,
    955     path,
    956     append=append,
    957     ignore_divisions=ignore_divisions,
    958     partition_on=partition_on,
    959     division_info=division_info,
    960     index_cols=index_cols,
    961     schema=schema,
    962     custom_metadata=custom_metadata,
    963     **kwargs,
    964 )
    966 # By default we only write a metadata file when appending if one already
    967 # exists
    968 if append and write_metadata_file is None:

File ~/miniforge3/envs/scverse/lib/python3.13/site-packages/dask/dataframe/io/parquet/arrow.py:724, in ArrowDatasetEngine.initialize_write(cls, df, fs, path, append, partition_on, ignore_divisions, division_info, schema, index_cols, **kwargs)
    708 @classmethod
    709 def initialize_write(
    710     cls,
   (...)    720     **kwargs,
    721 ):
    722     if schema == "infer" or isinstance(schema, dict):
    723         # Start with schema from _meta_nonempty
--> 724         inferred_schema = pyarrow_schema_dispatch(
    725             df._meta_nonempty.set_index(index_cols)
    726             if index_cols
    727             else df._meta_nonempty
    728         ).remove_metadata()
    730         # Use dict to update our inferred schema
    731         if isinstance(schema, dict):

File ~/miniforge3/envs/scverse/lib/python3.13/site-packages/dask/utils.py:772, in Dispatch.__call__(self, arg, *args, **kwargs)
    768 """
    769 Call the corresponding method based on type of argument.
    770 """
    771 meth = self.dispatch(type(arg))
--> 772 return meth(arg, *args, **kwargs)

File ~/miniforge3/envs/scverse/lib/python3.13/site-packages/dask/dataframe/backends.py:216, in get_pyarrow_schema_pandas(obj, preserve_index)
    212 @pyarrow_schema_dispatch.register((pd.DataFrame,))
    213 def get_pyarrow_schema_pandas(obj, preserve_index=None):
    214     import pyarrow as pa
--> 216     return pa.Schema.from_pandas(obj, preserve_index=preserve_index)

File ~/miniforge3/envs/scverse/lib/python3.13/site-packages/pyarrow/types.pxi:3148, in pyarrow.lib.Schema.from_pandas()

File ~/miniforge3/envs/scverse/lib/python3.13/site-packages/pyarrow/pandas_compat.py:580, in dataframe_to_types(df, preserve_index, columns)
    577             type_ = pa.array(c, from_pandas=True).type
    578     types.append(type_)
--> 580 metadata = construct_metadata(
    581     columns_to_convert, df, column_names, index_columns, index_descriptors,
    582     preserve_index, types, column_field_names=column_field_names
    583 )
    585 return all_names, types, metadata

File ~/miniforge3/envs/scverse/lib/python3.13/site-packages/pyarrow/pandas_compat.py:281, in construct_metadata(columns_to_convert, df, column_names, index_levels, index_descriptors, preserve_index, types, column_field_names)
    276     index_descriptors = index_column_metadata = column_indexes = []
    278 attributes = df.attrs if hasattr(df, "attrs") else {}
    280 return {
--> 281     b'pandas': json.dumps({
    282         'index_columns': index_descriptors,
    283         'column_indexes': column_indexes,
    284         'columns': column_metadata + index_column_metadata,
    285         'attributes': attributes,
    286         'creator': {
    287             'library': 'pyarrow',
    288             'version': pa.__version__
    289         },
    290         'pandas_version': _pandas_api.version
    291     }).encode('utf8')
    292 }

File ~/miniforge3/envs/scverse/lib/python3.13/json/__init__.py:231, in dumps(obj, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)
    226 # cached encoder
    227 if (not skipkeys and ensure_ascii and
    228     check_circular and allow_nan and
    229     cls is None and indent is None and separators is None and
    230     default is None and not sort_keys and not kw):
--> 231     return _default_encoder.encode(obj)
    232 if cls is None:
    233     cls = JSONEncoder

File ~/miniforge3/envs/scverse/lib/python3.13/json/encoder.py:200, in JSONEncoder.encode(self, o)
    196         return encode_basestring(o)
    197 # This doesn't pass the iterator directly to ''.join() because the
    198 # exceptions aren't as detailed.  The list call should be roughly
    199 # equivalent to the PySequence_Fast that ''.join() would do.
--> 200 chunks = self.iterencode(o, _one_shot=True)
    201 if not isinstance(chunks, (list, tuple)):
    202     chunks = list(chunks)

File ~/miniforge3/envs/scverse/lib/python3.13/json/encoder.py:261, in JSONEncoder.iterencode(self, o, _one_shot)
    256 else:
    257     _iterencode = _make_iterencode(
    258         markers, self.default, _encoder, indent, floatstr,
    259         self.key_separator, self.item_separator, self.sort_keys,
    260         self.skipkeys, _one_shot)
--> 261 return _iterencode(o, 0)

File ~/miniforge3/envs/scverse/lib/python3.13/json/encoder.py:180, in JSONEncoder.default(self, o)
    161 def default(self, o):
    162     """Implement this method in a subclass such that it returns
    163     a serializable object for ``o``, or calls the base implementation
    164     (to raise a ``TypeError``).
   (...)    178 
    179     """
--> 180     raise TypeError(f'Object of type {o.__class__.__name__} '
    181                     f'is not JSON serializable')

TypeError: Object of type Identity is not JSON serializable