-
Notifications
You must be signed in to change notification settings - Fork 60
Description
Used example recipe and changed make_url. It makes the proper urls that when executed in a browser, do download the NetCDF data, e.g.,
Index({DimIndex(name='time', index=0, sequence_len=2, operation=<CombineOp.CONCAT: 2>)}) http://tds.coaps.fsu.edu/thredds/fileServer/samos/data/research/ZCYL5/2021/ZCYL5_20210101v30001.nc Index({DimIndex(name='time', index=1, sequence_len=2, operation=<CombineOp.CONCAT: 2>)}) http://tds.coaps.fsu.edu/thredds/fileServer/samos/data/research/ZCYL5/2021/ZCYL5_20210102v30001.nc
(Template URL: See here and copy/paste #3 (HTTPServer)
When it runs, it seems to access first file, but has errors like:
FileNotFoundError: [Errno 2] No such file or directory: '/tmp/tmp3zzmk1oe/ri5TAkGy/.zmetadata'
# ---
# jupyter:
# jupytext:
# text_representation:
# extension: .py
# format_name: light
# format_version: '1.5'
# jupytext_version: 1.13.7
# kernelspec:
# display_name: Python 3 (ipykernel)
# language: python
# name: python3
# ---
# +
# start coding here
# -
# ## Format Function
# +
# def make_url(time):
# yyyy = time.strftime('%Y')
# yyyymmdd = time.strftime('%Y%m%d')
# return (
# 'https://coastwatch.noaa.gov/pub/socd/lsa/rads/sla/daily/dt'
# f'/{yyyy}/rads_global_dt_sla_{yyyymmdd}_001.nc'
# )
# http://tds.coaps.fsu.edu/thredds/fileServer/samos/data/research/ZCYL5/2021/ZCYL5_20210101v30001.nc
def make_url(time):
yyyy = time.strftime('%Y')
yyyymmdd = time.strftime('%Y%m%d')
return (
f'http://tds.coaps.fsu.edu/thredds/fileServer/samos/data/research/ZCYL5/{yyyy}/ZCYL5_{yyyymmdd}v30001.nc'
)
# +
#dates[0]
# +
#make_url(dates[0])
# -
# ## Combine Dimension
# +
import pandas as pd
dates = pd.date_range('2021-01-01', '2021-01-02', freq='D')
# print the first 4 dates
dates[:]
make_url(dates[0])
# +
from pangeo_forge_recipes.patterns import ConcatDim
# only one day in each file --> nitems_per_file=1
time_concat_dim = ConcatDim("time", dates, nitems_per_file=1)
time_concat_dim
# -
# ## FilePattern
# +
from pangeo_forge_recipes.patterns import FilePattern
pattern = FilePattern(make_url, time_concat_dim)
pattern
# + [markdown] tags=[]
# ### Iterate through FilePattern
# -
for index, url in pattern.items() :
print(index)
print(url)
if '20120103' in url:
break
# ## Create Recipe Object
# +
from pangeo_forge_recipes.recipes import XarrayZarrRecipe
recipe = XarrayZarrRecipe(pattern, inputs_per_chunk=20)
recipe
# -
# ## Setup Logging
# +
from pangeo_forge_recipes.recipes import setup_logging
setup_logging()
# -
# ## Prune (built in smaller copy)
recipe_pruned = recipe.copy_pruned() # Removes all but first two items
# +
## Run!
# -
run_function = recipe_pruned.to_function()
run_function()
# ## Check the output
import xarray as xr
recipe_pruned.storage_config
# (NOT YET MODIFIED FOR SAMOS variables)
sla_zarr = xr.open_zarr(recipe.target_mapper, consolidated=True)
sla_zarr
sla_zarr['sla'].isel(time=1).plot(robust=True)