Build miniconda and conda environments.

README.md

@@ -33,6 +33,10 @@ WCOSS-Cray  | Hang-Lei
 Cheyenne    | Dom Heinzeller
 Gaea        | Dom Heinzeller
 
+## Contributors
+
+Mark Potts, Steve Lawrence, Ed Hartnett, Guoqing Ge, Raffaele Montuoro, David Huber
+
 ## Prerequisites:
 
 The prerequisites of building hpc-stack are:
@@ -101,6 +105,8 @@ some of the parameter settings available.
 
 - **MAKE_VERBOSE:** Print out extra information to the log files during the build
 
+- **VENVTYPE:** Set the type of python environment to build.  Value depends on whether using `pip` or `conda`.  Set `VENVTYPE=pyvenv` when using `pip` and `VENVTYPE=condaenv` when using `Miniconda` for creating virtual environments. Default is `pyvenv`
+
 The next step is to choose what components of the stack you wish to
 build.  This is done by editing the file `stack/stack_custom.yaml`
 which defines the software packages to be built along with their
@@ -143,6 +149,9 @@ script.
   - [JSON Schema Validator for C++](https://github.com/pboettch/json-schema-validator)
   - [pybind11](https://github.com/pybind/pybind11)
   - [MADIS](https://madis-data.ncep.noaa.gov)
+  - [SQLite](https://www.sqlite.org)
+  - [PROJ](https://proj.org)
+  - [GEOS](https://www.osgeo.org/projects/geos)
 
 * UFS Dependencies
   - [ESMF](https://www.earthsystemcog.org/projects/esmf/)
@@ -179,7 +188,8 @@ script.
   - [fckit](https://github.com/ecmwf/fckit.git)
   - [atlas](https://github.com/ecmwf/atlas.git)
 
-* Python Virtual Environments
+* Python and Virtual Environments
+  - [Miniconda3](https://docs.conda.io/en/latest/)
   - [r2d2](https://github.com/jcsda-internal/r2d2.git)
 
 **IMPORTANT: Steps 1, 2, and 3 need to be repeated for each
@@ -306,13 +316,15 @@ template
 2. define a new section in the `yaml` file for that library/package in
 config directory
 
-3. if the package is a python virtual environment, add a `requirements.txt` file listing the python packages required to install the package in `pyvenv/package_name.txt`
+3. if the package is a python virtual environment, add a `requirements.txt` or `environment.yml` file containing the listing the python packages required to install the package.  These files should be named and placed in `pyvenv/package_name.txt` and `pyvenv/package_name.yml`.  `VENVTYPE=pyvenv` will use the `pyvenv/package_name.txt` and `VENVTYPE=condaenv` will use `pyvenv/package_name.yml`
 
 4. Add a call to the new build script in `build_stack.sh`
 
 5. Create a new module template at the appropriate place in the
 modulefiles directory, using exising files as a template
 
+6. Update `README.md` to include the name of the new library or package
+
 ### Configuring for a new HPC
 
 If you want to port this to a new HPC, you need to follow these steps:

build_stack.sh

@@ -185,8 +185,9 @@ build_lib ncio
 
 build_lib madis
 
-# Python virtual environments
+# Python and associate virtual environments
 
+build_lib miniconda3
 build_lib r2d2
 
 # JEDI 3rd party dependencies

config/config_hera.sh

@@ -17,6 +17,7 @@ export   MAKE_CLEAN=N
 export DOWNLOAD_ONLY=N
 export STACK_EXIT_ON_FAIL=Y
 export WGET="wget -nv"
+export VENVTYPE="condaenv"
 
 # Load these basic modules for Hera
 module purge

config/config_s4.sh

@@ -17,6 +17,7 @@ export   MAKE_CLEAN=N
 export DOWNLOAD_ONLY=N
 export STACK_EXIT_ON_FAIL=Y
 export WGET="wget -nv"
+export VENVTYPE="condaenv"
 
 # Load these basic modules for S4
 module purge

libs/build_condaenv.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+set -eux
+
+name=$1
+
+var="STACK_${name}_version"
+set +u
+stack_version=${!var}
+set -u
+version=${2:-$stack_version}
+
+var="STACK_${name}_requirements"
+set +u
+stack_rqmts=${!var}
+set -u
+rqmts=${stack_rqmts:-"$name.yml"}
+
+# Check for conda environment file
+rqmts_file=${HPC_STACK_ROOT}/pyvenv/$rqmts
+[[ ! -f $rqmts_file ]] && ( echo "Unable to find environment file: $rqmts \nABORT!"; exit 1 )
+
+python=$(echo $HPC_PYTHON | sed 's/\//-/g')
+
+if $MODULES; then
+  set +x
+  source $MODULESHOME/init/bash
+  module load hpc-$HPC_PYTHON
+  module list
+  set -x
+  prefix="${PREFIX:-"/opt/modules"}/$python/$name/$version"
+  if [[ -d $prefix ]]; then
+    [[ $OVERWRITE =~ [yYtT] ]] && ( echo "WARNING: $prefix EXISTS: OVERWRITING!"; $SUDO rm -rf $prefix ) \
+                               || ( echo "WARNING: $prefix EXISTS, SKIPPING"; exit 1 )
+  fi
+else
+  nameUpper=$(echo $name | tr [a-z] [A-Z])
+  eval prefix="\${${nameUpper}_ROOT:-'/usr/local'}"
+fi
+
+cd ${HPC_STACK_ROOT}/${PKGDIR:-"pkg"}
+
+software=${name}-${version}
+mkdir -p $software
+
+# Activate conda's base environment to get a few details such as python_version
+set +x
+echo "executing ... conda activate base"
+conda activate base
+set -x
+
+# Determine python version; 3.x
+python_version=$(python3 -c 'import sys;print(sys.version_info[0],".",sys.version_info[1],sep="")')
+
+# Support python version >= 3.6
+min_python_version=3.6
+if (( $(echo "$min_python_version >= $python_version" | bc -l) )); then
+  echo "Must have python version ($python_version) >= ${min_python_version}. ABORT!"
+  exit 1
+fi
+
+# Deactivate base environment before building new conda environment
+set +x
+echo "executing ... conda deactivate"
+conda deactivate
+set -x
+
+# Create the conda environment
+set +x
+echo "executing ... conda env create -q --file $rqmts_file"
+conda env create -q --file $rqmts_file
+set -x
+
+# Activate conda environment and get list
+set +x
+echo "executing ... conda activate $name"
+conda activate $name
+echo "executing ... conda env list"
+conda env list
+set -x
+
+# generate modulefile from template
+$MODULES && update_modules python $name $version $python_version
+echo $name $version $rqmts_file >> ${HPC_STACK_ROOT}/hpc-stack-contents.log

libs/build_miniconda3.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Download and install Miniconda3
+# https://docs.conda.io/projects/conda/en/latest/user-guide/getting-started.html
+
+set -eux
+
+name="miniconda3"
+version=${2:-${STACK_miniconda3_version:-"latest"}}
+pyversion=${3:-${STACK_miniconda3_pyversion:-}}
+
+if $MODULES; then
+  prefix="${PREFIX:-"/opt/modules"}/core/$name/$version"
+  if [[ -d $prefix ]]; then
+    [[ $OVERWRITE =~ [yYtT] ]] && ( echo "WARNING: $prefix EXISTS: OVERWRITING!"; $SUDO rm -rf $prefix; $SUDO mkdir $prefix ) \
+                               || ( echo "WARNING: $prefix EXISTS, SKIPPING"; exit 1 )
+  fi
+else
+  prefix=${MINICONDA3_ROOT:-"/usr/local"}
+fi
+
+cd ${HPC_STACK_ROOT}/${PKGDIR:-"pkg"}
+
+host=$(uname -s)
+
+if [[ "$host" == "Darwin" ]]; then
+  os="MacOSX"
+elif [[ "$host" == "Linux" ]]; then
+  os="Linux"
+else
+  echo "Unsupported host $host for Miniconda3; ABORT!"
+  exit 1
+fi
+
+software=$name-$version
+pkg_version=$version
+[[ -n ${pyversion:-} ]] && pkg_version=$pyversion_$version
+installer="Miniconda3-${pkg_version}-${os}-x86_64.sh"
+URL="https://repo.anaconda.com/miniconda/$installer"
+
+[[ -d $software ]] || ( mkdir -p $software; $WGET $URL -O $software/$installer )
+[[ ${DOWNLOAD_ONLY} =~ [yYtT] ]] && exit 0
+
+$SUDO bash $software/$installer -b -p $prefix -s
+set +x
+echo "sourcing conda.sh"
+source $prefix/etc/profile.d/conda.sh
+echo "setting conda default threads to 4"
+conda config --system --set default_threads 4
+echo "disabling conda auto updates"
+conda config --system --set auto_update_conda False
+echo "install $version of conda"
+conda install -yq conda=$version
+set -x
+
+# generate modulefile from template
+$MODULES && update_modules core $name $version
+echo $name $version $URL >> ${HPC_STACK_ROOT}/hpc-stack-contents.log

libs/build_pyvenv.sh

@@ -14,7 +14,11 @@ var="STACK_${name}_requirements"
 set +u
 stack_rqmts=${!var}
 set -u
-rqmts=${stack_rqmts:-"requirements.txt"}
+rqmts=${stack_rqmts:-"$name.txt"}
+
+# Check for requirements file
+rqmts_file=${HPC_STACK_ROOT}/pyvenv/$rqmts
+[[ ! -f $rqmts_file ]] && ( echo "Unable to find requirements file: $rqmts \nABORT!"; exit 1 )
 
 python=$(echo $HPC_PYTHON | sed 's/\//-/g')
 
@@ -49,10 +53,6 @@ if (( $(echo "$min_python_version >= $python_version" | bc -l) )); then
   exit 1
 fi
 
-# Check for requirements file
-rqmts_file=${HPC_STACK_ROOT}/pyvenv/$rqmts
-[[ ! -f $rqmts_file ]] && ( echo "Unable to find requirements file: $rqmts \nABORT!"; exit 1 )
-
 # Download the requirements of the virtual environment
 if [[ ${DOWNLOAD_ONLY} =~ [yYtT] ]]; then
   pip download -r $rqmts_file -d $software

modulefiles/core/miniconda3/miniconda3.lua

@@ -13,13 +13,45 @@ conflict("intelpython")
 
 local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules"
 
-local mpath = pathJoin(opt,"modulefiles/core",pkgName,pkgVersion)
+local mpath = pathJoin(opt,"modulefiles/python",pkgName,pkgVersion)
 prepend_path("MODULEPATH", mpath)
 
-local base = pathJoin(opt,pkgName,pkgVersion)
-
-prepend_path("PATH", pathJoin(base,"bin"))
-prepend_path("PYTHONPATH", pathJoin(base,"lib/python@PYTHON_VERSION@/site-packages"))
+local base = pathJoin(opt,"core",pkgName,pkgVersion)
+
+pushenv("CONDA_ENVS_PATH", pathJoin(base,"envs"))
+pushenv("CONDA_PKGS_PATH", pathJoin(base,"pkgs"))
+
+-- These are conda functions defined in conda.sh
+local funcs = "conda __conda_activate __conda_hashr __conda_reactivate __add_sys_prefix_to_path"
+
+-- Line #: What does it do?
+-- 1: source conda.sh from the installation path
+-- 2: export conda functions silently(> dev/null)
+local load_cmd = "source " .. pathJoin(base, "etc/profile.d/conda.sh") .. "; \
+export -f " .. funcs .. " > /dev/null"
+
+-- Line #: What does it do?
+-- 1: deactivate all conda envs
+-- 2: unset the conda funcs
+-- 3: define local variable prefix as path to Miniconda installation
+-- 4: remove from PATH all paths to $prefix
+-- 5: unset CONDA_ env. variables that are set via sourcing conda.sh
+-- 6: unset previously set variable $prefix
+local unload_cmd="for i in $(seq ${CONDA_SHLVL:=0}); do conda deactivate; done; \
+unset -f " .. funcs .. "; \
+prefix=" .. base .. "; \
+export PATH=$(echo $PATH | tr ':' '\\n' | grep . | grep -v $prefix | tr '\\n' ':' | sed 's/:$//'); \
+unset $(env | grep -o \"[^=]*CONDA[^=]*\" | grep -v 'CONDA_ENVS_PATH\\|CONDA_PKGS_DIRS'); \
+unset prefix"
+
+-- source conda on load, deactivate on unload
+if (mode() == "load") then
+  execute{cmd=load_cmd, modeA={"load"}}
+else
+  if (mode() == "unload") then
+    execute{cmd=unload_cmd, modeA={"unload"}}
+  end
+end
 
 whatis("Name: ".. pkgName)
 whatis("Version: " .. pkgVersion)

modulefiles/core/python/python.lua

@@ -13,7 +13,7 @@ conflict("intelpython")
 
 local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules"
 
-local mpath = pathJoin(opt,"modulefiles/core",pkgName,pkgVersion)
+local mpath = pathJoin(opt,"modulefiles/python",pkgName,pkgVersion)
 prepend_path("MODULEPATH", mpath)
 
 local base = pathJoin(opt,pkgName,pkgVersion)

modulefiles/python/pythonName/pythonVersion/condaenv/condaenv.lua

@@ -0,0 +1,32 @@
+help([[
+]])
+
+local pkgName    = myModuleName()
+local pkgVersion = myModuleVersion()
+local pkgNameVer = myModuleFullName()
+
+local hierA          = hierarchyA(pkgNameVer,1)
+local pythonNameVer  = hierA[1]
+local pythonNameVerD = pythonNameVer:gsub("/","-")
+
+conflict(pkgName)
+
+local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules"
+
+local base = pathJoin(opt,pythonNameVerD,pkgName,pkgVersion)
+
+-- activate on load, deactivate on unload
+if (mode() == "load") then
+  local load_cmd = "conda activate " .. pkgName
+  execute{cmd=load_cmd, modeA={"load"}}
+else
+  if (mode() == "unload") then
+    local unload_cmd = "conda deactivate"
+    execute{cmd=unload_cmd, modeA={"unload"}}
+  end
+end
+
+whatis("Name: " .. pkgName)
+whatis("Version: " .. pkgVersion)
+whatis("Category: Software")
+whatis("Description: " .. pkgName .. " Conda Environment")

pyvenv/hofx.txt

@@ -11,14 +11,11 @@ python-dateutil >= 2.8.1
 pytz >= 2020.4
 PyYAML >= 5.3.1
 s3transfer >= 0.3.3
-sdist >= 0.0.0
 six >= 1.15.0
 urllib3 >= 1.26.2
 wheel >= 0.36.2
 xarray >= 0.16.2
-
 netcdf4 >= 1.5.5.1
-h5netcdf >= 0.8.1
 h5py >= 3.1.0
 git+https://github.com/jcsda-internal/solo.git
 git+https://github.com/jcsda-internal/r2d2.git

pyvenv/hofx.yml

@@ -0,0 +1,27 @@
+name: hofx
+channels:
+  - defaults
+dependencies:
+  - boto3 >= 1.16.38
+  - botocore >= 1.19.38
+  - cftime >= 1.3.0
+  - click >= 7.1.2
+  - Cython >= 0.29.21
+  - jmespath >= 0.10.0
+  - numpy >= 1.19.4
+  - pandas >= 1.1.5
+  - pip >= 20.3.3
+  - python-dateutil >= 2.8.1
+  - pytz >= 2020.4
+  - PyYAML >= 5.3.1
+  - s3transfer >= 0.3.3
+  - six >= 1.15.0
+  - urllib3 >= 1.26.2
+  - wheel >= 0.36.2
+  - xarray >= 0.16.2
+  - netcdf4 >= 1.5.5.1
+  - h5py >= 3.1.0
+  - pip:
+    - git+https://github.com/jcsda-internal/solo.git
+    - git+https://github.com/jcsda-internal/r2d2.git
+    - git+https://github.com/noaa-emc/hofxcs.git