Skip to content

Commit

Permalink
per #1566, added explicit version numbers for each Python package tha…
Browse files Browse the repository at this point in the history
…t is installed in the Docker Conda environments used for the automated tests to ensure that the same environment will be generated even if the default version of the package differs from the env used to create them
  • Loading branch information
georgemccabe committed Jul 25, 2022
1 parent 162b063 commit 996ea16
Show file tree
Hide file tree
Showing 13 changed files with 90 additions and 105 deletions.
17 changes: 8 additions & 9 deletions scripts/docker/docker_env/scripts/cfgrib_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,10 @@
# Notes: Adds Python packages needed to read GRIB data into Xarray and
# so it can easily be processed with MetPy
# Python Packages:
# TODO: update version numbers
# metpy==1.0.1
# netcdf4==1.5.6
# cfgrib==0.9.9.0
# pygrib==2.1.3
# metpy==1.3.0
# netcdf4==1.5.8
# cfgrib==0.9.10.1
# pygrib==2.1.4
#
# Other Content: None
################################################################################
Expand All @@ -19,7 +18,7 @@
ENV_NAME=cfgrib.v5

conda create -y --name ${ENV_NAME} -c conda-forge python=3.8.6
conda install -y --name ${ENV_NAME} -c conda-forge metpy #==1.0.1
conda install -y --name ${ENV_NAME} -c conda-forge netcdf4 #==1.5.6
conda install -y --name ${ENV_NAME} -c conda-forge cfgrib #==0.9.9.0
conda install -y --name ${ENV_NAME} -c conda-forge pygrib #==2.1.3
conda install -y --name ${ENV_NAME} -c conda-forge metpy==1.3.0
conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.5.8
conda install -y --name ${ENV_NAME} -c conda-forge cfgrib==0.9.10.1
conda install -y --name ${ENV_NAME} -c conda-forge pygrib==2.1.4
13 changes: 6 additions & 7 deletions scripts/docker/docker_env/scripts/cycloneplotter_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,9 @@
# Notes: Adds packages needed to run CyclonePlotter wrapper
# Added pandas because it is used by tc_and_extra_tc use case
# Python Packages:
# TODO: update versions
# cartopy==
# matplotlib==
# pandas==
# cartopy==0.20.3
# matplotlib==3.5.2
# pandas==1.4.3
#
# Other Content: None
################################################################################
Expand All @@ -21,9 +20,9 @@ ENV_NAME=cycloneplotter.v5
BASE_ENV=metplus_base.v5

conda create -y --clone ${BASE_ENV} --name ${ENV_NAME}
conda install -y --name ${ENV_NAME} -c conda-forge cartopy
conda install -y --name ${ENV_NAME} -c conda-forge matplotlib
conda install -y --name ${ENV_NAME} -c conda-forge pandas
conda install -y --name ${ENV_NAME} -c conda-forge cartopy==0.20.3
conda install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.5.2
conda install -y --name ${ENV_NAME} -c conda-forge pandas==1.4.3

yum -y install wget
wget https://raw.githubusercontent.com/SciTools/cartopy/master/tools/cartopy_feature_download.py
Expand Down
9 changes: 4 additions & 5 deletions scripts/docker/docker_env/scripts/diff_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,9 @@
# Last Updated: 2022-07-08 ([email protected])
# Notes: Adds packages needed to run differences tests to compare output to
# truth data.
# TODO: update version numbers
# Python Packages:
# pillow==?
# pdf2image==?
# pillow==9.2.0
# pdf2image==1.16.0
#
# Other Content:
# poppler-utils
Expand All @@ -21,8 +20,8 @@ ENV_NAME=diff.v5
BASE_ENV=netcdf4.v5

conda create -y --clone ${BASE_ENV} --name ${ENV_NAME}
conda install -y --name ${ENV_NAME} -c conda-forge pillow
conda install -y --name ${ENV_NAME} -c conda-forge pillow==9.2.0

yum -y install poppler-utils

conda install -y --name ${ENV_NAME} -c conda-forge pdf2image
conda install -y --name ${ENV_NAME} -c conda-forge pdf2image==1.16.0
5 changes: 2 additions & 3 deletions scripts/docker/docker_env/scripts/h5py_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@
# Last Updated: 2022-06-15 ([email protected])
# Notes: Adds Python interface to the HDF5 binary format
# Python Packages:
# TODO: update versions
# h5py==2.10.0
# h5py==3.6.0
#
# Other Content: None
################################################################################
Expand All @@ -18,4 +17,4 @@ ENV_NAME=h5py.v5
BASE_ENV=py_embed_base.v5

conda create -y --clone ${BASE_ENV} --name ${ENV_NAME}
conda install -y --name ${ENV_NAME} -c conda-forge h5py #==2.10.0
conda install -y --name ${ENV_NAME} -c conda-forge h5py==3.6.0
16 changes: 8 additions & 8 deletions scripts/docker/docker_env/scripts/icecover_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@
# Last Updated: 2022-07-08 ([email protected])
# Notes: Adds Python packages required for ice cover use case
# Python Packages:
# TODO: update package versions
# pyproj==
# pyresample==
# scikit-learn==
# xarray==2022.3.0
# pyresample==1.24.1
# scikit-learn==1.1.1
# pyproj==3.3.1
#
# Other Content: None
################################################################################
Expand All @@ -20,7 +20,7 @@ ENV_NAME=icecover.v5
BASE_ENV=py_embed_base.v5

conda create -y --clone ${BASE_ENV} --name ${ENV_NAME}
conda install -y --name ${ENV_NAME} -c conda-forge xarray
conda install -y --name ${ENV_NAME} -c conda-forge pyresample
conda install -y --name ${ENV_NAME} -c conda-forge scikit-learn
conda install -y --name ${ENV_NAME} -c conda-forge pyproj
conda install -y --name ${ENV_NAME} -c conda-forge xarray==2022.3.0
conda install -y --name ${ENV_NAME} -c conda-forge pyresample==1.24.1
conda install -y --name ${ENV_NAME} -c conda-forge scikit-learn==1.1.1
conda install -y --name ${ENV_NAME} -c conda-forge pyproj==3.3.1
11 changes: 5 additions & 6 deletions scripts/docker/docker_env/scripts/metdataio_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,9 @@
# Last Updated: 2022-07-13 ([email protected])
# Notes: Adds Python packages needed to run METdbLoad from METdataio
# Python Packages:
# TODO: update versions
# lxml==3.8.0
# lxml==4.9.1
# pymysql==1.0.2
# pandas==1.1.4
# pandas==1.4.3
#
# Other Content: None
################################################################################
Expand All @@ -21,6 +20,6 @@ BASE_ENV=metplus_base.v5

conda create -y --clone ${BASE_ENV} --name ${ENV_NAME}

conda install -y --name ${ENV_NAME} -c conda-forge lxml #==3.8.0
conda install -y --name ${ENV_NAME} -c conda-forge pymysql #==1.0.2
conda install -y --name ${ENV_NAME} -c conda-forge pandas #==1.1.4
conda install -y --name ${ENV_NAME} -c conda-forge lxml==4.9.1
conda install -y --name ${ENV_NAME} -c conda-forge pymysql==1.0.2
conda install -y --name ${ENV_NAME} -c conda-forge pandas==1.4.3
56 changes: 25 additions & 31 deletions scripts/docker/docker_env/scripts/metplotpy_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,20 @@
# Environment: metplotpy
# Last Updated: 2022-06-14 ([email protected])
# Notes: Adds Python packages needed to run METplotpy and METcalcpy
# Uses pip to install kaleido because
# could not install via Conda (glibc conflict)
# Python Packages:
# UPDATE THESE VERSION NUMBERS!
# matplotlib==3.3.0
# scipy==1.5.1
# plotly==4.9.0
# cartopy==0.18.0
# eofs==1.3.0
# cmocean==2.0
# xarray==0.17
# netcdf4==1.5.6
# pyyaml==?
# matplotlib==3.5.2
# scipy==1.8.1
# plotly==5.9.0
# xarray==2022.3.0
# netcdf4==1.6.0
# pyyaml==6.0
# statsmodels==0.13.2
# python-kaleido==0.2.1
# imageio==2.19.3
# imutils==0.5.4
# scikit-image==0.19.3
# pint==0.19.2
# metpy=1.3.1
#
# Other Content: None
################################################################################
Expand All @@ -31,25 +31,19 @@ BASE_ENV=metplus_base.v5

conda create -y --clone ${BASE_ENV} --name ${ENV_NAME}

conda install -y --name ${ENV_NAME} -c conda-forge matplotlib #==3.3.0
conda install -y --name ${ENV_NAME} -c conda-forge scipy #==1.5.1
conda install -y --name ${ENV_NAME} -c conda-forge plotly #==4.9.0
#conda install -y --name ${ENV_NAME} -c conda-forge cartopy #==0.18.0
#conda install -y --name ${ENV_NAME} -c conda-forge eofs #==1.3.0
#conda install -y --name ${ENV_NAME} -c conda-forge cmocean #==2.0
conda install -y --name ${ENV_NAME} -c conda-forge xarray #==0.17
conda install -y --name ${ENV_NAME} -c conda-forge netcdf4 #==1.5.6
conda install -y --name ${ENV_NAME} -c conda-forge pyyaml
conda install -y --name ${ENV_NAME} -c conda-forge statsmodels #==0.11.1
conda install -y --name ${ENV_NAME} -c conda-forge python-kaleido

conda install -y --name ${ENV_NAME} -c conda-forge imageio
conda install -y --name ${ENV_NAME} -c conda-forge imutils
#opencv-python?
conda install -y --name ${ENV_NAME} -c conda-forge scikit-image
conda install -y --name ${ENV_NAME} -c conda-forge pint
conda install -y --name ${ENV_NAME} -c conda-forge metpy
#/usr/local/envs/${ENV_NAME}/bin/pip3 install kaleido==0.2.1
conda install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.5.2
conda install -y --name ${ENV_NAME} -c conda-forge scipy==1.8.1
conda install -y --name ${ENV_NAME} -c conda-forge plotly==5.9.0
conda install -y --name ${ENV_NAME} -c conda-forge xarray==2022.3.0
conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.6.0
conda install -y --name ${ENV_NAME} -c conda-forge pyyaml==6.0
conda install -y --name ${ENV_NAME} -c conda-forge statsmodels==0.13.2
conda install -y --name ${ENV_NAME} -c conda-forge python-kaleido==0.2.1
conda install -y --name ${ENV_NAME} -c conda-forge imageio==2.19.3
conda install -y --name ${ENV_NAME} -c conda-forge imutils==0.5.4
conda install -y --name ${ENV_NAME} -c conda-forge scikit-image==0.19.3
conda install -y --name ${ENV_NAME} -c conda-forge pint==0.19.2
conda install -y --name ${ENV_NAME} -c conda-forge metpy=1.3.1

yum -y install wget
wget https://raw.githubusercontent.com/SciTools/cartopy/master/tools/cartopy_feature_download.py
Expand Down
5 changes: 2 additions & 3 deletions scripts/docker/docker_env/scripts/netcdf4_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@
# Last Updated: 2022-06-16 ([email protected])
# Notes: Adds NetCDF4 Python package
# Python Packages:
# TODO: update version numbers
# netcdf4==1.5.6
# netcdf4==1.5.8
#
# Other Content: None
################################################################################
Expand All @@ -18,4 +17,4 @@ ENV_NAME=netcdf4.v5
BASE_ENV=metplus_base.v5

conda create -y --clone ${BASE_ENV} --name ${ENV_NAME}
conda install -y --name ${ENV_NAME} -c conda-forge netcdf4 #==1.5.6
conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.5.8
9 changes: 4 additions & 5 deletions scripts/docker/docker_env/scripts/py_embed_base_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,8 @@
# Notes: Move logic to create METplus base env to script so it can be called
# on a local machine to create the environment
# Python Packages:
# TODO: update package list
# xarray==
# netcdf4==
# xarray==2022.3.0
# netcdf4==1.5.8
#
# Other Content: None
################################################################################
Expand All @@ -17,5 +16,5 @@
ENV_NAME=py_embed_base.v5

conda create -y --name ${ENV_NAME} -c conda-forge python=3.8.6
conda install -y --name ${ENV_NAME} -c conda-forge xarray #==0.16.2
conda install -y --name ${ENV_NAME} -c conda-forge netcdf4 #==1.5.6
conda install -y --name ${ENV_NAME} -c conda-forge xarray==2022.3.0
conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.5.8
9 changes: 4 additions & 5 deletions scripts/docker/docker_env/scripts/pygrib_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,8 @@
# Last Updated: 2022-06-16 ([email protected])
# Notes: Adds Python packages needed to read GRIB data
# Python Packages:
# TODO: update version numbers!
# pygrib==2.0.2
# metpy==1.0.1
# pygrib==2.1.4
# metpy==1.3.0
#
# Other Content: None
################################################################################
Expand All @@ -21,5 +20,5 @@ BASE_ENV=py_embed_base.v5

conda create -y --clone ${BASE_ENV} --name ${ENV_NAME}

conda install -y --name ${ENV_NAME} -c conda-forge pygrib #==2.0.2
conda install -y --name ${ENV_NAME} -c conda-forge metpy #==1.0.1
conda install -y --name ${ENV_NAME} -c conda-forge pygrib==2.1.4
conda install -y --name ${ENV_NAME} -c conda-forge metpy==1.3.0
23 changes: 11 additions & 12 deletions scripts/docker/docker_env/scripts/spacetime_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,12 @@
# Last Updated: 2022-06-16 ([email protected])
# Notes: Adds Python packages needed to generate coherence spectra (METplotpy)
# Python Packages:
# TODO: update package list
# netCDF4==1.5.4
# xarray==0.18.2
# scipy==1.5.3
# matplotlib==3.2.2
# netCDF4==1.5.8
# xarray==2022.3.0
# scipy==1.8.1
# matplotlib==3.5.2
# pyngl==1.6.1
# pyyaml==5.3.1
# pyyaml==6.0
#
# Other Content: None
################################################################################
Expand All @@ -21,9 +20,9 @@ ENV_NAME=spacetime.v5

conda create -y --name ${ENV_NAME} -c conda-forge python=3.8.6

conda install -y --name ${ENV_NAME} -c conda-forge netCDF4 #==1.5.4
conda install -y --name ${ENV_NAME} -c conda-forge xarray #==0.18.2
conda install -y --name ${ENV_NAME} -c conda-forge scipy #==1.5.3
conda install -y --name ${ENV_NAME} -c conda-forge matplotlib #==3.2.2
conda install -y --name ${ENV_NAME} -c conda-forge pyngl #==1.6.1
conda install -y --name ${ENV_NAME} -c conda-forge pyyaml #==5.3.1
conda install -y --name ${ENV_NAME} -c conda-forge netCDF4==1.5.8
conda install -y --name ${ENV_NAME} -c conda-forge xarray==2022.3.0
conda install -y --name ${ENV_NAME} -c conda-forge scipy==1.8.1
conda install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.5.2
conda install -y --name ${ENV_NAME} -c conda-forge pyngl==1.6.1
conda install -y --name ${ENV_NAME} -c conda-forge pyyaml==6.0
13 changes: 6 additions & 7 deletions scripts/docker/docker_env/scripts/weatherregime_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,10 @@
# Notes: Adds Python packages needed to run weather regime use case
# METplotpy and METcalcpy
# Python Packages:
# TODO: update versions
# All packages from metplotpy_env
# scikit-learn==0.24.2
# All packages from metplotpy.v5
# scikit-learn==1.1.1
# eofs==1.4.0
# cmocean==
# cmocean==2.0
#
# Other Content: None
################################################################################
Expand All @@ -23,9 +22,9 @@ BASE_ENV=metplotpy.v5


conda create -y --clone ${BASE_ENV} --name ${ENV_NAME}
conda install -y --name ${ENV_NAME} -c conda-forge scikit-learn #==0.24.2
conda install -y --name ${ENV_NAME} -c conda-forge eofs #==1.4.0
conda install -y --name ${ENV_NAME} -c conda-forge cmocean
conda install -y --name ${ENV_NAME} -c conda-forge scikit-learn==1.1.1
conda install -y --name ${ENV_NAME} -c conda-forge eofs==1.4.0
conda install -y --name ${ENV_NAME} -c conda-forge cmocean==2.0

rm cartopy_feature_download.py
yum -y install wget
Expand Down
9 changes: 5 additions & 4 deletions scripts/docker/docker_env/scripts/xesmf_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
# Last Updated: 2022-06-16 ([email protected])
# Notes: Adds Python package to read Tripolar grids
# Python Packages:
# TODO: update package versions and add others
# netcdf4==1.5.8
# xarray==2022.3.0
# xesmf==0.3.0
#
# Other Content: None
Expand All @@ -18,6 +19,6 @@ ENV_NAME=xesmf.v5
BASE_ENV=metplus_base.v5

conda create -y --clone ${BASE_ENV} --name ${ENV_NAME}
conda install -y --name ${ENV_NAME} -c conda-forge netcdf4
conda install -y --name ${ENV_NAME} -c conda-forge xarray
conda install -y --name ${ENV_NAME} -c conda-forge xesmf
conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.5.8
conda install -y --name ${ENV_NAME} -c conda-forge xarray==2022.3.0
conda install -y --name ${ENV_NAME} -c conda-forge xesmf==0.3.0

0 comments on commit 996ea16

Please sign in to comment.