Skip to content

Commit 50ded8f

Browse files
lhoupertclaude
andcommitted
refactor(s1-rtc): address review — round-trip test, orbit-builder dedup, typed attrs
- Add an end-to-end NaN-masking round-trip test (open with `use_zarr_fill_value_as_mask=True`, `to_masked_array()`), mirroring the S2 guarantee in `test_array_attrs.py`. Proves `_FillValue` actually masks nodata, not just that the attribute is present. - Unify the two orbit-creation paths — new store (`create_s1_store`) and new orbit added to an existing store (`ingest_s1tiling_acquisition`) — into a shared `_build_orbit_group` helper. Removes ~60 lines of duplication and fixes the inline path's missing `proj:code` on level groups (guarded by a new test). Net: `s1_ingest.py` shrinks while gaining the fix. - Model the backscatter band CF attrs as a `S1BackscatterAttrsJSON` TypedDict (PR review request), alongside the existing `Standard*CoordAttrsJSON` types. - Document the internal `xarray.backends.zarr.FillValueCoder` import. Co-Authored-By: Claude Opus 4.8 <[email protected]> Claude-Session: https://claude.ai/code/session_011sBdb17MWSAn7VEPSnqKJU
1 parent 66a5d10 commit 50ded8f

3 files changed

Lines changed: 100 additions & 64 deletions

File tree

‎src/eopf_geozarr/conversion/s1_ingest.py‎

Lines changed: 37 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,28 @@
1919
from dataclasses import dataclass
2020
from math import ceil
2121
from pathlib import Path
22+
from typing import TYPE_CHECKING, cast
2223

2324
import numpy as np
2425
import rasterio
2526
import structlog
2627
import zarr
2728
import zarr.codecs
2829
from pyproj import CRS as PyprojCRS
30+
31+
# `FillValueCoder` is xarray's internal CF fill-value encoder: the canonical way to produce
32+
# the base64 `_FillValue` xarray reads back under `use_zarr_fill_value_as_mask` (xarray
33+
# #11345); the S2 path relies on the same mechanism. Internal API — revisit if xarray moves it.
2934
from xarray.backends.zarr import FillValueCoder
3035
from zarr_cm import geo_proj
3136
from zarr_cm import multiscales as multiscales_cm
3237
from zarr_cm import spatial as spatial_cm
3338

3439
from eopf_geozarr.conversion.utils import calculate_aligned_chunk_size
3540

41+
if TYPE_CHECKING:
42+
from eopf_geozarr.data_api.geozarr.types import S1BackscatterAttrsJSON
43+
3644
log = structlog.get_logger()
3745

3846
# =============================================================================
@@ -318,7 +326,7 @@ def _create_spatial_coordinate_arrays(
318326
# zarr-direct, so the attribute must be set explicitly.
319327
FLOAT32_NAN_FILL_VALUE = FillValueCoder.encode(np.nan, np.dtype("float32"))
320328
# CF metadata for the backscatter bands (vv/vh).
321-
BACKSCATTER_CF_ATTRS = {
329+
BACKSCATTER_CF_ATTRS: S1BackscatterAttrsJSON = {
322330
"standard_name": "surface_backwards_scattering_coefficient_of_radar_wave",
323331
"units": "1",
324332
"_FillValue": FLOAT32_NAN_FILL_VALUE,
@@ -405,23 +413,24 @@ def _create_band_arrays(level_group: zarr.Group, level_h: int, level_w: int) ->
405413
dimension_names=["time", "y", "x"],
406414
)
407415
if name in ("vv", "vh"):
408-
band.attrs.update(BACKSCATTER_CF_ATTRS)
416+
# cast: zarr's `attrs.update` is typed for its JSON union, which a TypedDict
417+
# (invariant) doesn't satisfy structurally; the values are JSON-safe.
418+
band.attrs.update(cast("dict", BACKSCATTER_CF_ATTRS))
409419

410420

411-
def create_s1_store(
412-
store_path: str | Path,
413-
orbit_direction: str,
414-
metadata: S1TilingMetadata,
421+
def _build_orbit_group(
422+
root: zarr.Group, orbit_direction: str, metadata: S1TilingMetadata
415423
) -> zarr.Group:
416-
"""Create a new S1 GRD RTC Zarr V3 store with full conventions metadata.
424+
"""Create one orbit group (asc/desc) with full GeoZarr metadata.
417425
418-
Returns the root group.
426+
Builds the multiscale level groups (bands + spatial coordinates + grid-mapping +
427+
per-level ``time``) and the native-resolution per-acquisition metadata coordinates.
428+
Shared by ``create_s1_store`` (new store) and ``ingest_s1tiling_acquisition`` (new
429+
orbit added to an existing store) so the two creation paths cannot drift — the inline
430+
path previously omitted ``proj:code`` on level groups.
419431
"""
420432
layout = compute_multiscales_layout(metadata.shape, metadata.spatial_transform)
421-
422-
root = zarr.open_group(str(store_path), mode="w-", zarr_format=3)
423433
orbit_group = root.create_group(orbit_direction)
424-
425434
orbit_group.attrs.update(
426435
{
427436
"zarr_conventions": ZARR_CONVENTIONS,
@@ -479,6 +488,20 @@ def create_s1_store(
479488
fill_value="",
480489
dimension_names=["time"],
481490
)
491+
return orbit_group
492+
493+
494+
def create_s1_store(
495+
store_path: str | Path,
496+
orbit_direction: str,
497+
metadata: S1TilingMetadata,
498+
) -> zarr.Group:
499+
"""Create a new S1 GRD RTC Zarr V3 store with full conventions metadata.
500+
501+
Returns the root group.
502+
"""
503+
root = zarr.open_group(str(store_path), mode="w-", zarr_format=3)
504+
_build_orbit_group(root, orbit_direction, metadata)
482505

483506
log.info(
484507
"Created S1 store",
@@ -588,59 +611,9 @@ def ingest_s1tiling_acquisition(
588611
else:
589612
root = zarr.open_group(str(store_path), mode="r+", zarr_format=3)
590613
if orbit_direction not in root:
591-
# Create orbit direction group in existing store
592-
orbit_group = root.create_group(orbit_direction)
593-
layout = compute_multiscales_layout(meta.shape, meta.spatial_transform)
594-
orbit_group.attrs.update(
595-
{
596-
"zarr_conventions": ZARR_CONVENTIONS,
597-
"multiscales": {
598-
"layout": layout,
599-
"resampling_method": "average",
600-
},
601-
"proj:code": meta.crs,
602-
"spatial:dimensions": ["y", "x"],
603-
"spatial:bbox": meta.bounds,
604-
}
605-
)
606-
for level_entry in layout:
607-
level_name = level_entry["asset"]
608-
level_h, level_w = level_entry["spatial:shape"]
609-
level_group = orbit_group.create_group(level_name)
610-
level_group.attrs.update(
611-
{
612-
"spatial:shape": [level_h, level_w],
613-
"spatial:transform": level_entry["spatial:transform"],
614-
}
615-
)
616-
_create_band_arrays(level_group, level_h, level_w)
617-
_create_spatial_coordinate_arrays(
618-
level_group, level_h, level_w, level_entry["spatial:transform"]
619-
)
620-
_add_grid_mapping(level_group, meta.crs)
621-
# `time` coordinate on every level (datetime `.sel` at any scale, #192).
622-
_create_time_coordinate_array(level_group)
623-
r10m = orbit_group["r10m"]
624-
for name, dtype, fill in [
625-
("absolute_orbit", "int32", 0),
626-
("relative_orbit", "int32", 0),
627-
]:
628-
r10m.create_array(
629-
name,
630-
shape=(0,),
631-
dtype=dtype,
632-
chunks=(512,),
633-
fill_value=fill,
634-
dimension_names=["time"],
635-
)
636-
r10m.create_array(
637-
"platform",
638-
shape=(0,),
639-
dtype="<U4",
640-
chunks=(512,),
641-
fill_value="",
642-
dimension_names=["time"],
643-
)
614+
# Create the new orbit group in the existing store (same builder as a fresh
615+
# store, so per-level metadata — incl. `proj:code` — stays consistent).
616+
_build_orbit_group(root, orbit_direction, meta)
644617
else:
645618
# Validate consistency on append
646619
orbit_group = root[orbit_direction]

‎src/eopf_geozarr/data_api/geozarr/types.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,18 @@ class StandardYCoordAttrsJSON(TypedDict):
6060
_ARRAY_DIMENSIONS: list[Literal["y"]]
6161

6262

63+
class S1BackscatterAttrsJSON(TypedDict):
64+
"""CF attributes for the Sentinel-1 RTC backscatter bands (vv/vh).
65+
66+
``_FillValue`` is the ``FillValueCoder``-encoded base64 NaN that lets xarray mask
67+
nodata under ``use_zarr_fill_value_as_mask`` despite xarray #11345 (data-model #172).
68+
"""
69+
70+
standard_name: Literal["surface_backwards_scattering_coefficient_of_radar_wave"]
71+
units: Literal["1"]
72+
_FillValue: str
73+
74+
6375
class TileMatrixJSON(TypedDict):
6476
id: str
6577
scaleDenominator: float

‎tests/test_s1_rtc_ingest.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,57 @@ def test_ingested_bands_declare_cf_fill_value(
408408
)
409409
assert attrs.get("units") == "1"
410410

411+
def test_new_orbit_level_groups_carry_proj_code(
412+
self, s1_geotiff_dir: Path, s1_store_path: Path
413+
) -> None:
414+
"""A second orbit added to an existing store must get the same per-level metadata
415+
as the store-creating orbit — incl. ``proj:code`` on every level group. The inline
416+
new-orbit path previously omitted it (drift vs ``create_s1_store``)."""
417+
vv, vh, mask = self._get_acq_paths(s1_geotiff_dir, "20230115t061234")
418+
ingest_s1tiling_acquisition(vv, vh, mask, s1_store_path, "ascending")
419+
ingest_s1tiling_acquisition(vv, vh, mask, s1_store_path, "descending")
420+
root = zarr.open_group(str(s1_store_path), mode="r", zarr_format=3)
421+
for orbit in ("ascending", "descending"):
422+
for level_name, _, _ in OVERVIEW_CHAIN:
423+
attrs = dict(root[orbit][level_name].attrs)
424+
assert attrs.get("proj:code") == CRS, f"{orbit}/{level_name} missing proj:code"
425+
426+
def test_fill_value_masking_roundtrip(self, tmp_path: Path, s1_store_path: Path) -> None:
427+
"""End-to-end: NaN nodata in vv comes back masked when the cube is reopened with
428+
``use_zarr_fill_value_as_mask=True`` — the behaviour the CF ``_FillValue`` attribute
429+
exists to enable despite xarray #11345. Mirrors the S2 guarantee in
430+
``tests/test_array_attrs.py::test_fill_value_masking_roundtrip``.
431+
"""
432+
stamp = "20230115t061234"
433+
rng = np.random.default_rng(0)
434+
vv_data = rng.uniform(0.0, 1.0, (SIZE, SIZE)).astype(np.float32)
435+
vv_data[0:16, 0:16] = np.nan # nodata patch
436+
vh_data = rng.uniform(0.0, 0.5, (SIZE, SIZE)).astype(np.float32)
437+
mask_data = np.ones((SIZE, SIZE), dtype=np.uint8)
438+
vv = tmp_path / f"s1a_32TQM_vv_ASC_037_{stamp}_GammaNaughtRTC.tif"
439+
vh = tmp_path / f"s1a_32TQM_vh_ASC_037_{stamp}_GammaNaughtRTC.tif"
440+
mask = tmp_path / f"s1a_32TQM_vv_ASC_037_{stamp}_GammaNaughtRTC_BorderMask.tif"
441+
_create_synthetic_geotiff(vv, vv_data, tags=ACQ1_TAGS)
442+
_create_synthetic_geotiff(vh, vh_data, tags=ACQ1_TAGS)
443+
_create_synthetic_geotiff(mask, mask_data, tags=ACQ1_TAGS)
444+
ingest_s1tiling_acquisition(vv, vh, mask, s1_store_path, "ascending")
445+
446+
ds = xr.open_dataset(
447+
str(s1_store_path / "ascending" / "r10m"),
448+
engine="zarr",
449+
consolidated=False,
450+
decode_times=False,
451+
decode_coords=False,
452+
use_zarr_fill_value_as_mask=True,
453+
)
454+
try:
455+
masked = ds["vv"].to_masked_array()
456+
assert np.ma.is_masked(masked), "NaN nodata must be masked via `_FillValue`"
457+
assert masked.mask[0, 0, 0], "nodata cell must be masked"
458+
assert not masked.mask[0, -1, -1], "valid cell must not be masked"
459+
finally:
460+
ds.close()
461+
411462
def test_preserves_data_integrity(self, s1_geotiff_dir: Path, s1_store_path: Path) -> None:
412463
vv, vh, mask = self._get_acq_paths(s1_geotiff_dir, "20230115t061234")
413464
ingest_s1tiling_acquisition(vv, vh, mask, s1_store_path, "ascending")

0 commit comments

Comments
 (0)