Tests using ncgen_from_cdl fail on OS X #5097
Description
🐛 Bug Report
Hi there! While running Iris tests locally on OS X (Intel processor), I have noticed that all tests using ncgen_from_cdl from iris/tests/stock/netcdf.py fail (see example for one of the tests below). This seems to be due to the order in which parameters are provided to ncgen via the subprocess call. OS X seems to be more "strict" here than Linux in requiring first options/flags and then other parameters.
How To Reproduce
On OS X, running one of the tests that make use of ncgen, like test_um_stash_source from lib/iris/tests/test_netcdf.py:
pytest lib/iris/tests/test_netcdf.py -k "test_um_stash_source"Gives:
Error output
================================================================ test session starts =================================================================
platform darwin -- Python 3.10.8, pytest-7.2.0, pluggy-1.0.0
rootdir: /Users/fnattino/Programs/iris, configfile: pyproject.toml
plugins: xdist-3.1.0
collected 68 items / 67 deselected / 1 selected
lib/iris/tests/test_netcdf.py F [100%]
====================================================================== FAILURES ======================================================================
________________________________________________________ TestNetCDFLoad.test_um_stash_source _________________________________________________________
self = <iris.tests.test_netcdf.TestNetCDFLoad testMethod=test_um_stash_source>
def test_um_stash_source(self):
"""Test that um_stash_source is converted into a STASH code"""
# Note: using a CDL string as a test data reference, rather than a binary file.
ref_cdl = """
netcdf cm_attr {
dimensions:
axv = 3 ;
ayv = 2 ;
variables:
int64 qqv(ayv, axv) ;
qqv:long_name = "qq" ;
qqv:ancillary_variables = "my_av" ;
qqv:cell_measures = "area: my_areas" ;
qqv:um_stash_source = "m01s02i003" ;
int64 ayv(ayv) ;
ayv:long_name = "y" ;
int64 axv(axv) ;
axv:units = "1" ;
axv:long_name = "x" ;
double my_av(axv) ;
my_av:long_name = "refs" ;
double my_areas(ayv, axv) ;
my_areas:long_name = "areas" ;
data:
axv = 11, 12, 13;
ayv = 21, 22;
my_areas = 110., 120., 130., 221., 231., 241.;
}
"""
self.tmpdir = tempfile.mkdtemp()
cdl_path = os.path.join(self.tmpdir, "tst.cdl")
nc_path = os.path.join(self.tmpdir, "tst.nc")
# Create a temporary netcdf file from the CDL string.
> ncgen_from_cdl(ref_cdl, cdl_path, nc_path)
lib/iris/tests/test_netcdf.py:367:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
lib/iris/tests/stock/netcdf.py:63: in ncgen_from_cdl
subprocess.run(call_args, check=True, **call_kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
input = None, capture_output = False, timeout = None, check = True
popenargs = (['/opt/miniconda3/envs/iris-dev/bin/ncgen', '/var/folders/t6/r2gjczqj7bb8798wr4g1p87m0000gn/T/tmp6x6tefqq/tst.cdl', '-k3', '-o', '/var/folders/t6/r2gjczqj7bb8798wr4g1p87m0000gn/T/tmp6x6tefqq/tst.nc'],)
kwargs = {}, process = <Popen: returncode: 6 args: ['/opt/miniconda3/envs/iris-dev/bin/ncgen', '/va...>, stdout = None, stderr = None, retcode = 6
def run(*popenargs,
input=None, capture_output=False, timeout=None, check=False, **kwargs):
"""Run command with arguments and return a CompletedProcess instance.
The returned instance will have attributes args, returncode, stdout and
stderr. By default, stdout and stderr are not captured, and those attributes
will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them,
or pass capture_output=True to capture both.
If check is True and the exit code was non-zero, it raises a
CalledProcessError. The CalledProcessError object will have the return code
in the returncode attribute, and output & stderr attributes if those streams
were captured.
If timeout is given, and the process takes too long, a TimeoutExpired
exception will be raised.
There is an optional argument "input", allowing you to
pass bytes or a string to the subprocess's stdin. If you use this argument
you may not also use the Popen constructor's "stdin" argument, as
it will be used internally.
By default, all communication is in bytes, and therefore any "input" should
be bytes, and the stdout and stderr will be bytes. If in text mode, any
"input" should be a string, and stdout and stderr will be strings decoded
according to locale encoding, or by "encoding" if set. Text mode is
triggered by setting any of text, encoding, errors or universal_newlines.
The other arguments are the same as for the Popen constructor.
"""
if input is not None:
if kwargs.get('stdin') is not None:
raise ValueError('stdin and input arguments may not both be used.')
kwargs['stdin'] = PIPE
if capture_output:
if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None:
raise ValueError('stdout and stderr arguments may not be used '
'with capture_output.')
kwargs['stdout'] = PIPE
kwargs['stderr'] = PIPE
with Popen(*popenargs, **kwargs) as process:
try:
stdout, stderr = process.communicate(input, timeout=timeout)
except TimeoutExpired as exc:
process.kill()
if _mswindows:
# Windows accumulates the output in a single blocking
# read() call run on child threads, with the timeout
# being done in a join() on those threads. communicate()
# _after_ kill() is required to collect that and add it
# to the exception.
exc.stdout, exc.stderr = process.communicate()
else:
# POSIX _communicate already populated the output so
# far into the TimeoutExpired exception.
process.wait()
raise
except: # Including KeyboardInterrupt, communicate handled that.
process.kill()
# We don't call process.wait() as .__exit__ does that for us.
raise
retcode = process.poll()
if check and retcode:
> raise CalledProcessError(retcode, process.args,
output=stdout, stderr=stderr)
E subprocess.CalledProcessError: Command '['/opt/miniconda3/envs/iris-dev/bin/ncgen', '/var/folders/t6/r2gjczqj7bb8798wr4g1p87m0000gn/T/tmp6x6tefqq/tst.cdl', '-k3', '-o', '/var/folders/t6/r2gjczqj7bb8798wr4g1p87m0000gn/T/tmp6x6tefqq/tst.nc']' returned non-zero exit status 6.
/opt/miniconda3/envs/iris-dev/lib/python3.10/subprocess.py:526: CalledProcessError
---------------------------------------------------------------- Captured stderr call ----------------------------------------------------------------
ncgen: only one input file argument permitted
============================================================== short test summary info ===============================================================
FAILED lib/iris/tests/test_netcdf.py::TestNetCDFLoad::test_um_stash_source - subprocess.CalledProcessError: Command '['/opt/miniconda3/envs/iris-dev/bin/ncgen', '/var/folders/t6/r2gjczqj7bb8798wr4g1p87m0000gn/T/tmp6x6tefqq...
========================================================== 1 failed, 67 deselected in 3.52s ==========================================================Expected behaviour
The key error message in the log above seems to be:
ncgen: only one input file argument permitted
which I also get by running directly ncgen from command line. Note that if I change the order of the options and the CDL filename solves the issue:
ncgen tst.cdl -k3 -o tst.nc # ERROR: `ncgen: only one input file argument permitted`
ncgen -k3 -o tst.nc tst.cdl # works fineAnd indeed if I modify:
iris/lib/iris/tests/stock/netcdf.py
Line 54 in dfbc5c2
to:
call_args = [NCGEN_PATHSTR, "-k3", "-o", nc_path, cdl_path]the test above (and all other tests using `ncgen_from_cdl`) passes
$ pytest lib/iris/tests/test_netcdf.py -k "test_um_stash_source"
================================================================ test session starts =================================================================
platform darwin -- Python 3.10.8, pytest-7.2.0, pluggy-1.0.0
rootdir: /Users/fnattino/Programs/iris, configfile: pyproject.toml
plugins: xdist-3.1.0
collected 68 items / 67 deselected / 1 selected
lib/iris/tests/test_netcdf.py . [100%]
========================================================== 1 passed, 67 deselected in 3.66s ==========================================================
Changing the order as above should not affect the behaviour on Linux, so I would be happy to suggest this via a PR!
Environment
- OS & Version: OS X 13.0.1
- Python environment from
conda list(environment built fromrequirements/ci/iris.ymlfile):
Full environment here
# packages in environment at /opt/miniconda3/envs/iris-dev:
#
# Name Version Build Channel
alabaster 0.7.12 py_0 conda-forge
antlr-python-runtime 4.7.2 py310h2ec42d9_1003 conda-forge
atk-1.0 2.38.0 h1d18e73_1 conda-forge
attrs 22.1.0 pyh71513ae_1 conda-forge
babel 2.11.0 pyhd8ed1ab_0 conda-forge
beautifulsoup4 4.11.1 pyha770c72_0 conda-forge
brotli 1.0.9 hb7f2c08_8 conda-forge
brotli-bin 1.0.9 hb7f2c08_8 conda-forge
brotlipy 0.7.0 py310h90acd4f_1005 conda-forge
bzip2 1.0.8 h0d85af4_4 conda-forge
c-ares 1.18.1 h0d85af4_0 conda-forge
ca-certificates 2022.9.24 h033912b_0 conda-forge
cairo 1.16.0 h904041c_1014 conda-forge
cartopy 0.21.0 py310h578c2b2_3 conda-forge
certifi 2022.9.24 pyhd8ed1ab_0 conda-forge
cf-units 3.1.1 py310h936d966_2 conda-forge
cffi 1.15.1 py310ha78151a_2 conda-forge
cfgv 3.3.1 pyhd8ed1ab_0 conda-forge
cftime 1.6.2 py310h936d966_1 conda-forge
charset-normalizer 2.1.1 pyhd8ed1ab_0 conda-forge
click 8.1.3 unix_pyhd8ed1ab_2 conda-forge
cloudpickle 2.2.0 pyhd8ed1ab_0 conda-forge
colorama 0.4.6 pyhd8ed1ab_0 conda-forge
contourpy 1.0.6 py310ha23aa8a_0 conda-forge
cryptography 38.0.4 py310hdd0c95c_0 conda-forge
curl 7.86.0 h581aaea_1 conda-forge
cycler 0.11.0 pyhd8ed1ab_0 conda-forge
dask-core 2022.12.0 pyhd8ed1ab_0 conda-forge
distlib 0.3.6 pyhd8ed1ab_0 conda-forge
docutils 0.17.1 py310h2ec42d9_3 conda-forge
esmf 8.2.0 mpi_mpich_hcd927a8_102 conda-forge
esmpy 8.2.0 mpi_mpich_py310hb1b2199_101 conda-forge
exceptiongroup 1.0.4 pyhd8ed1ab_0 conda-forge
execnet 1.9.0 pyhd8ed1ab_0 conda-forge
expat 2.5.0 hf0c8a7f_0 conda-forge
filelock 3.8.2 pyhd8ed1ab_0 conda-forge
font-ttf-dejavu-sans-mono 2.37 hab24e00_0 conda-forge
font-ttf-inconsolata 3.000 h77eed37_0 conda-forge
font-ttf-source-code-pro 2.038 h77eed37_0 conda-forge
font-ttf-ubuntu 0.83 hab24e00_0 conda-forge
fontconfig 2.14.1 h5bb23bf_0 conda-forge
fonts-conda-ecosystem 1 0 conda-forge
fonts-conda-forge 1 0 conda-forge
fonttools 4.38.0 py310h90acd4f_1 conda-forge
freetype 2.12.1 h3f81eb7_1 conda-forge
fribidi 1.0.10 hbcb3906_0 conda-forge
fsspec 2022.11.0 pyhd8ed1ab_0 conda-forge
gdk-pixbuf 2.42.8 h3648f77_1 conda-forge
geos 3.11.1 hf0c8a7f_0 conda-forge
gettext 0.21.1 h8a4c099_0 conda-forge
giflib 5.2.1 hbcb3906_2 conda-forge
graphite2 1.3.13 h2e338ed_1001 conda-forge
graphviz 6.0.2 hc51f7b9_0 conda-forge
gtk2 2.24.33 h7c1209e_2 conda-forge
gts 0.7.6 hccb3bdf_2 conda-forge
harfbuzz 5.3.0 h08f8713_0 conda-forge
hdf4 4.2.15 h7aa5921_5 conda-forge
hdf5 1.12.2 mpi_mpich_hc154f39_0 conda-forge
icu 70.1 h96cf925_0 conda-forge
identify 2.5.9 pyhd8ed1ab_0 conda-forge
idna 3.4 pyhd8ed1ab_0 conda-forge
imagehash 4.3.1 pyhd8ed1ab_0 conda-forge
imagesize 1.4.1 pyhd8ed1ab_0 conda-forge
importlib-metadata 5.1.0 pyha770c72_0 conda-forge
iniconfig 1.1.1 pyh9f0ad1d_0 conda-forge
iris-sample-data 2.4.0 pyhd8ed1ab_0 conda-forge
jinja2 3.1.2 pyhd8ed1ab_1 conda-forge
jpeg 9e hac89ed1_2 conda-forge
kiwisolver 1.4.4 py310ha23aa8a_1 conda-forge
krb5 1.19.3 hb98e516_0 conda-forge
lcms2 2.14 h90f4b2a_0 conda-forge
lerc 4.0.0 hb486fe8_0 conda-forge
libblas 3.9.0 16_osx64_openblas conda-forge
libbrotlicommon 1.0.9 hb7f2c08_8 conda-forge
libbrotlidec 1.0.9 hb7f2c08_8 conda-forge
libbrotlienc 1.0.9 hb7f2c08_8 conda-forge
libcblas 3.9.0 16_osx64_openblas conda-forge
libcurl 7.86.0 h581aaea_1 conda-forge
libcxx 14.0.6 hccf4f1f_0 conda-forge
libdeflate 1.14 hb7f2c08_0 conda-forge
libedit 3.1.20191231 h0678c8f_2 conda-forge
libev 4.33 haf1e3a3_1 conda-forge
libffi 3.4.2 h0d85af4_5 conda-forge
libgd 2.3.3 h1e214de_3 conda-forge
libgfortran 5.0.0 9_5_0_h97931a8_26 conda-forge
libgfortran5 11.3.0 h082f757_26 conda-forge
libglib 2.74.1 h4c723e1_1 conda-forge
libiconv 1.17 hac89ed1_0 conda-forge
liblapack 3.9.0 16_osx64_openblas conda-forge
libmo_unpack 3.1.2 h0a44026_1001 conda-forge
libnetcdf 4.8.1 mpi_mpich_hc6a5375_6 conda-forge
libnghttp2 1.47.0 h5aae05b_1 conda-forge
libopenblas 0.3.21 openmp_h429af6e_3 conda-forge
libpng 1.6.39 ha978bb4_0 conda-forge
librsvg 2.54.4 h3d48ba6_0 conda-forge
libsqlite 3.40.0 ha978bb4_0 conda-forge
libssh2 1.10.0 h47af595_3 conda-forge
libtiff 4.4.0 hdb44e8a_4 conda-forge
libtool 2.4.6 he49afe7_1008 conda-forge
libwebp 1.2.4 hfa4350a_0 conda-forge
libwebp-base 1.2.4 h775f41a_0 conda-forge
libxcb 1.13 h0d85af4_1004 conda-forge
libxml2 2.10.3 hb9e07b5_0 conda-forge
libzip 1.9.2 h6db710c_1 conda-forge
libzlib 1.2.13 hfd90126_4 conda-forge
llvm-openmp 15.0.6 h61d9ccf_0 conda-forge
locket 1.0.0 pyhd8ed1ab_0 conda-forge
markupsafe 2.1.1 py310h90acd4f_2 conda-forge
matplotlib 3.6.2 py310h2ec42d9_0 conda-forge
matplotlib-base 3.6.2 py310he725631_0 conda-forge
mo_pack 0.2.0 py310h936d966_1008 conda-forge
mpi 1.0 mpich conda-forge
mpi4py 3.1.4 py310hb3ae6ab_0 conda-forge
mpich 4.0.3 hd33e60e_100 conda-forge
munkres 1.1.4 pyh9f0ad1d_0 conda-forge
nc-time-axis 1.4.1 pyhd8ed1ab_0 conda-forge
ncurses 6.3 h96cf925_1 conda-forge
netcdf-fortran 4.6.0 mpi_mpich_hddcf434_1 conda-forge
netcdf4 1.6.0 nompi_py310h6892ea4_102 conda-forge
nodeenv 1.7.0 pyhd8ed1ab_0 conda-forge
numpy 1.23.5 py310h1b7c290_0 conda-forge
openjpeg 2.5.0 h5d0d7b0_1 conda-forge
openssl 3.0.7 hfd90126_1 conda-forge
packaging 21.3 pyhd8ed1ab_0 conda-forge
pandas 1.5.2 py310hecf8f37_0 conda-forge
pango 1.50.12 h7fca291_0 conda-forge
partd 1.3.0 pyhd8ed1ab_0 conda-forge
pcre2 10.40 h1c4e4bc_0 conda-forge
pillow 9.2.0 py310hffcf78b_3 conda-forge
pip 22.3.1 pyhd8ed1ab_0 conda-forge
pixman 0.40.0 hbcb3906_0 conda-forge
platformdirs 2.5.2 pyhd8ed1ab_1 conda-forge
pluggy 1.0.0 pyhd8ed1ab_5 conda-forge
pockets 0.9.1 py_0 conda-forge
pre-commit 2.20.0 py310h2ec42d9_1 conda-forge
proj 9.1.0 hcbd9701_0 conda-forge
psutil 5.9.4 py310h90acd4f_0 conda-forge
pthread-stubs 0.4 hc929b4f_1001 conda-forge
pycparser 2.21 pyhd8ed1ab_0 conda-forge
pydata-sphinx-theme 0.8.1 pyhd8ed1ab_0 conda-forge
pygments 2.13.0 pyhd8ed1ab_0 conda-forge
pyopenssl 22.1.0 pyhd8ed1ab_0 conda-forge
pyparsing 3.0.9 pyhd8ed1ab_0 conda-forge
pyproj 3.4.0 py310h8c678d5_2 conda-forge
pyshp 2.3.1 pyhd8ed1ab_0 conda-forge
pysocks 1.7.1 pyha2e5f31_6 conda-forge
pytest 7.2.0 pyhd8ed1ab_2 conda-forge
pytest-xdist 3.1.0 pyhd8ed1ab_0 conda-forge
python 3.10.8 he7542f4_0_cpython conda-forge
python-dateutil 2.8.2 pyhd8ed1ab_0 conda-forge
python-stratify 0.2.post0 py310h936d966_3 conda-forge
python-xxhash 3.0.0 py310h90acd4f_2 conda-forge
python_abi 3.10 3_cp310 conda-forge
pytz 2022.6 pyhd8ed1ab_0 conda-forge
pywavelets 1.3.0 py310h936d966_2 conda-forge
pyyaml 6.0 py310h90acd4f_5 conda-forge
readline 8.1.2 h3899abd_0 conda-forge
requests 2.28.1 pyhd8ed1ab_1 conda-forge
scipy 1.9.3 py310h240c617_2 conda-forge
scitools-iris 3.5.0.dev6+dirty dev_0 <develop>
setuptools 65.5.1 pyhd8ed1ab_0 conda-forge
setuptools-scm 7.0.5 pyhd8ed1ab_1 conda-forge
shapely 1.8.5 py310h4e43f2a_2 conda-forge
six 1.16.0 pyh6c4a22f_0 conda-forge
snowballstemmer 2.2.0 pyhd8ed1ab_0 conda-forge
soupsieve 2.3.2.post1 pyhd8ed1ab_0 conda-forge
sphinx 4.5.0 pyh6c4a22f_0 conda-forge
sphinx-copybutton 0.5.0 pyhd8ed1ab_0 conda-forge
sphinx-gallery 0.11.1 pyhd8ed1ab_0 conda-forge
sphinx-panels 0.6.0 pyhd8ed1ab_0 conda-forge
sphinxcontrib-applehelp 1.0.2 py_0 conda-forge
sphinxcontrib-devhelp 1.0.2 py_0 conda-forge
sphinxcontrib-htmlhelp 2.0.0 pyhd8ed1ab_0 conda-forge
sphinxcontrib-jsmath 1.0.1 py_0 conda-forge
sphinxcontrib-napoleon 0.7 py_0 conda-forge
sphinxcontrib-qthelp 1.0.3 py_0 conda-forge
sphinxcontrib-serializinghtml 1.1.5 pyhd8ed1ab_2 conda-forge
sqlite 3.40.0 h9ae0607_0 conda-forge
tk 8.6.12 h5dbffcc_0 conda-forge
toml 0.10.2 pyhd8ed1ab_0 conda-forge
tomli 2.0.1 pyhd8ed1ab_0 conda-forge
toolz 0.12.0 pyhd8ed1ab_0 conda-forge
tornado 6.2 py310h90acd4f_1 conda-forge
typing-extensions 4.4.0 hd8ed1ab_0 conda-forge
typing_extensions 4.4.0 pyha770c72_0 conda-forge
tzdata 2022g h191b570_0 conda-forge
udunits2 2.2.28 h06ef574_0 conda-forge
ukkonen 1.0.1 py310ha23aa8a_3 conda-forge
unicodedata2 15.0.0 py310h90acd4f_0 conda-forge
urllib3 1.26.13 pyhd8ed1ab_0 conda-forge
virtualenv 20.17.0 py310h2ec42d9_0 conda-forge
wheel 0.38.4 pyhd8ed1ab_0 conda-forge
xorg-libxau 1.0.9 h35c211d_0 conda-forge
xorg-libxdmcp 1.1.3 h35c211d_0 conda-forge
xxhash 0.8.0 h35c211d_3 conda-forge
xz 5.2.6 h775f41a_0 conda-forge
yaml 0.2.5 h0d85af4_2 conda-forge
zipp 3.11.0 pyhd8ed1ab_0 conda-forge
zlib 1.2.13 hfd90126_4 conda-forge
zstd 1.5.2 hfa58983_4 conda-forge