Skip to content

Commit d69bd45

Browse files
authored
Merge pull request #4 from EOPF-Explorer/port-open-source-to-olci
fix(s3-olci): open OLCI sources with native-chunk-aligned reads
2 parents 8828401 + 5ea5662 commit d69bd45

3 files changed

Lines changed: 31 additions & 12 deletions

File tree

‎src/eopf_geozarr/cli.py‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -244,10 +244,8 @@ def convert_command(args: argparse.Namespace) -> None:
244244
# the generic path, so close it and re-open raw, matching
245245
# convert_s3_olci_optimized_command.
246246
dt.close()
247-
dt_raw = xr.open_datatree(
247+
dt_raw = open_source_datatree(
248248
str(input_path),
249-
engine="zarr",
250-
chunks="auto",
251249
storage_options=storage_options,
252250
mask_and_scale=False,
253251
)
@@ -1406,14 +1404,7 @@ def add_s3_olci_optimization_commands(subparsers: argparse._SubParsersAction) ->
14061404

14071405
def convert_s3_olci_optimized_command(args: argparse.Namespace) -> None:
14081406
"""Execute S3 OLCI optimized conversion command."""
1409-
storage_options = get_storage_options(str(args.input_path))
1410-
dt_input = xr.open_datatree(
1411-
str(args.input_path),
1412-
engine="zarr",
1413-
chunks="auto",
1414-
storage_options=storage_options,
1415-
mask_and_scale=False,
1416-
)
1407+
dt_input = open_source_datatree(str(args.input_path), mask_and_scale=False)
14171408
convert_olci_optimized(
14181409
dt_input,
14191410
output_path=args.output_path,

‎src/eopf_geozarr/conversion/open_source.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ def open_source_datatree(
2626
storage_options: S3FsOptions | dict[str, Any] | None = None,
2727
cache_dir: str | None = None,
2828
engine: str = "zarr",
29+
mask_and_scale: bool = True,
2930
) -> xr.DataTree:
3031
"""Open a source datatree with dask chunks matching the native zarr chunks.
3132
@@ -46,6 +47,10 @@ def open_source_datatree(
4647
ephemeral directory.
4748
engine : str
4849
xarray backend engine, default ``"zarr"``.
50+
mask_and_scale : bool
51+
Forwarded to ``xr.open_datatree``. Pass ``False`` to keep variables
52+
packed (e.g. uint16 with CF scale_factor/add_offset in ``.attrs``)
53+
instead of CF-decoding to floats, as the OLCI converter requires.
4954
5055
Returns
5156
-------
@@ -61,6 +66,7 @@ def open_source_datatree(
6166
engine=engine,
6267
chunks={},
6368
storage_options=storage_options,
69+
mask_and_scale=mask_and_scale,
6470
)
6571
# Imported lazily: CacheStore is zarr's experimental API, so a relocation
6672
# in a future zarr release must not break importing this package.
@@ -87,4 +93,9 @@ def open_source_datatree(
8793
cached = CacheStore(source, cache_store=LocalStore(source_cache))
8894
# xarray's open_datatree accepts a zarr store at runtime, but its stub does
8995
# not list Store among the accepted input types.
90-
return xr.open_datatree(cached, engine=engine, chunks={}) # pyright: ignore[reportArgumentType]
96+
return xr.open_datatree(
97+
cached, # pyright: ignore[reportArgumentType]
98+
engine=engine,
99+
chunks={},
100+
mask_and_scale=mask_and_scale,
101+
)

‎tests/test_open_source.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,23 @@ def fake_open_datatree(path: object, **kwargs: object) -> None:
8787
assert captured["chunks"] == {}
8888

8989

90+
def test_mask_and_scale_is_forwarded_to_xarray(monkeypatch) -> None:
91+
captured: dict[str, object] = {}
92+
93+
def fake_open_datatree(path: object, **kwargs: object) -> None:
94+
captured.update(kwargs)
95+
raise InterruptedError("stop before any I/O")
96+
97+
monkeypatch.setattr("xarray.open_datatree", fake_open_datatree)
98+
with pytest.raises(InterruptedError):
99+
open_source_datatree("/some/store.zarr", storage_options={})
100+
assert captured["mask_and_scale"] is True
101+
102+
with pytest.raises(InterruptedError):
103+
open_source_datatree("/some/store.zarr", storage_options={}, mask_and_scale=False)
104+
assert captured["mask_and_scale"] is False
105+
106+
90107
@pytest.fixture
91108
def http_source(source_store) -> Iterator[str]:
92109
"""Serve the source store over local HTTP (mimics the EODC https source)."""

0 commit comments

Comments
 (0)