From d0428a5fae7ee8a756066784bcdfb8db04ff8330 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 12 Jul 2021 12:02:02 -0400 Subject: [PATCH 1/5] add conda ability in the stack --- README.md | 12 +++- build_stack.sh | 3 +- config/config_mac.sh | 3 +- libs/build_condaenv.sh | 58 +++++++++++++++++++ libs/build_miniconda3.sh | 45 ++++++++++++++ modulefiles/core/miniconda3/miniconda3.lua | 41 +++++++++++-- modulefiles/core/python/python.lua | 2 +- .../pythonVersion/condaenv/condaenv.lua | 32 ++++++++++ stack/stack_mac.yaml | 4 ++ 9 files changed, 191 insertions(+), 9 deletions(-) create mode 100755 libs/build_condaenv.sh create mode 100755 libs/build_miniconda3.sh create mode 100644 modulefiles/python/pythonName/pythonVersion/condaenv/condaenv.lua diff --git a/README.md b/README.md index 8c74dad0..25963ed8 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ WCOSS-Cray | Hang-Lei Cheyenne | Dom Heinzeller Gaea | Dom Heinzeller +## Contributors + +Mark Potts, Steve Lawrence, Ed Hartnett, Guoqing Ge, Raffaele Montuoro, David Huber + ## Prerequisites: The prerequisites of building hpc-stack are: @@ -143,6 +147,9 @@ script. - [JSON Schema Validator for C++](https://github.com/pboettch/json-schema-validator) - [pybind11](https://github.com/pybind/pybind11) - [MADIS](https://madis-data.ncep.noaa.gov) + - [SQLite](https://www.sqlite.org) + - [PROJ](https://proj.org) + - [GEOS](https://www.osgeo.org/projects/geos) * UFS Dependencies - [ESMF](https://www.earthsystemcog.org/projects/esmf/) @@ -179,7 +186,8 @@ script. - [fckit](https://github.com/ecmwf/fckit.git) - [atlas](https://github.com/ecmwf/atlas.git) -* Python Virtual Environments +* Python and Virtual Environments + - [Miniconda3](https://docs.conda.io/en/latest/) - [r2d2](https://github.com/jcsda-internal/r2d2.git) **IMPORTANT: Steps 1, 2, and 3 need to be repeated for each @@ -313,6 +321,8 @@ config directory 5. Create a new module template at the appropriate place in the modulefiles directory, using exising files as a template +6. Update `README.md` to include the name of the new library or package + ### Configuring for a new HPC If you want to port this to a new HPC, you need to follow these steps: diff --git a/build_stack.sh b/build_stack.sh index 8492aac6..02af0c7d 100755 --- a/build_stack.sh +++ b/build_stack.sh @@ -185,8 +185,9 @@ build_lib ncio build_lib madis -# Python virtual environments +# Python and associate virtual environments +build_lib miniconda3 build_lib r2d2 # JEDI 3rd party dependencies diff --git a/config/config_mac.sh b/config/config_mac.sh index 64d5f6cf..c49dbfa3 100644 --- a/config/config_mac.sh +++ b/config/config_mac.sh @@ -3,7 +3,8 @@ # Compiler/MPI combination export HPC_COMPILER="clang/12.0.5" export HPC_MPI="mpich/3.3.2" -export HPC_PYTHON="python/3.9.6" +#export HPC_PYTHON="python/3.9.6" +export HPC_PYTHON="miniconda3/4.7.12" # Build options export USE_SUDO=N diff --git a/libs/build_condaenv.sh b/libs/build_condaenv.sh new file mode 100755 index 00000000..52ae3170 --- /dev/null +++ b/libs/build_condaenv.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +set -eux + +name=$1 + +var="STACK_${name}_version" +set +u +stack_version=${!var} +set -u +version=${2:-$stack_version} + +var="STACK_${name}_environment" +set +u +stack_rqmts=${!var} +set -u +rqmts=${stack_rqmts:-"environments.yml"} + +python=$(echo $HPC_PYTHON | sed 's/\//-/g') + +if $MODULES; then + set +x + source $MODULESHOME/init/bash + module load hpc-$HPC_PYTHON + module list + set -x + prefix="${PREFIX:-"/opt/modules"}/$python/$name/$version" + if [[ -d $prefix ]]; then + [[ $OVERWRITE =~ [yYtT] ]] && ( echo "WARNING: $prefix EXISTS: OVERWRITING!"; $SUDO rm -rf $prefix ) \ + || ( echo "WARNING: $prefix EXISTS, SKIPPING"; exit 1 ) + fi +else + nameUpper=$(echo $name | tr [a-z] [A-Z]) + eval prefix="\${${nameUpper}_ROOT:-'/usr/local'}" +fi + +cd ${HPC_STACK_ROOT}/${PKGDIR:-"pkg"} + +# Check for conda environment file +rqmts_file=${HPC_STACK_ROOT}/pyvenv/$rqmts +[[ ! -f $rqmts_file ]] && ( echo "Unable to find enviroment file: $rqmts \nABORT!"; exit 1 ) + +# Determine python version; 3.x +python_version=$(python3 -c 'import sys;print(sys.version_info[0],".",sys.version_info[1],sep="")') + +# Support python version >= 3.6 +min_python_version=3.6 +if (( $(echo "$min_python_version >= $python_version" | bc -l) )); then + echo "Must have python version ($python_version) >= ${min_python_version}. ABORT!" + exit 1 +fi + +# Create the conda environment +conda env create --file $rqmts_file -p $prefix + +# generate modulefile from template +$MODULES && update_modules python $name $version $python_version +echo $name $version $rqmts_file >> ${HPC_STACK_ROOT}/hpc-stack-contents.log diff --git a/libs/build_miniconda3.sh b/libs/build_miniconda3.sh new file mode 100755 index 00000000..5849a177 --- /dev/null +++ b/libs/build_miniconda3.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Download and install Miniconda3 +# https://docs.conda.io/projects/conda/en/latest/user-guide/getting-started.html + +set -eux + +name="miniconda3" +version=${2:-${STACK_miniconda3_version:-"latest"}} + +if $MODULES; then + prefix="${PREFIX:-"/opt/modules"}/core/$name/$version" + if [[ -d $prefix ]]; then + [[ $OVERWRITE =~ [yYtT] ]] && ( echo "WARNING: $prefix EXISTS: OVERWRITING!"; $SUDO rm -rf $prefix; $SUDO mkdir $prefix ) \ + || ( echo "WARNING: $prefix EXISTS, SKIPPING"; exit 1 ) + fi +else + prefix=${MINICONDA3_ROOT:-"/usr/local"} +fi + +cd ${HPC_STACK_ROOT}/${PKGDIR:-"pkg"} + +host=$(uname -s) + +if [[ "$host" == "Darwin" ]]; then + os="MacOSX" +elif [[ "$host" == "Linux" ]]; then + os="Linux" +else + echo "Unsupported host $host for Miniconda3; ABORT!" + exit 1 +fi + +software=$name-$version +installer="Miniconda3-${version}-${os}-x86_64.sh" +URL="https://repo.anaconda.com/miniconda/$installer" + +[[ -d $software ]] || ( mkdir -p $software; $WGET $URL -O $software/$installer ) +[[ ${DOWNLOAD_ONLY} =~ [yYtT] ]] && exit 0 + +$SUDO bash $software/$installer -b -p $prefix + +# generate modulefile from template +$MODULES && update_modules core $name $version +echo $name $version $URL >> ${HPC_STACK_ROOT}/hpc-stack-contents.log diff --git a/modulefiles/core/miniconda3/miniconda3.lua b/modulefiles/core/miniconda3/miniconda3.lua index be0a8dc5..9021e185 100644 --- a/modulefiles/core/miniconda3/miniconda3.lua +++ b/modulefiles/core/miniconda3/miniconda3.lua @@ -13,13 +13,44 @@ conflict("intelpython") local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules" -local mpath = pathJoin(opt,"modulefiles/core",pkgName,pkgVersion) +local mpath = pathJoin(opt,"modulefiles/python",pkgName,pkgVersion) prepend_path("MODULEPATH", mpath) -local base = pathJoin(opt,pkgName,pkgVersion) - -prepend_path("PATH", pathJoin(base,"bin")) -prepend_path("PYTHONPATH", pathJoin(base,"lib/python@PYTHON_VERSION@/site-packages")) +local base = pathJoin(opt,"core",pkgName,pkgVersion) + +pushenv("CONDA_ENVS_PATH", pathJoin(base,"envs")) +pushenv("CONDA_PKGS_PATH", pathJoin(base,"pkgs")) + +-- These are conda functions defined in conda.sh +local funcs = "conda __conda_activate __conda_hashr __conda_reactivate __add_sys_prefix_to_path" + +-- Line #: What does it do? +-- 1: source conda.sh from the installation path +-- 2: export conda functions +local load_cmd = "source " .. pathJoin(base, "etc/profile.d/conda.sh") .. "; \ +export -f " .. funcs + +-- Line #: What does it do? +-- 1: deactivate all conda envs +-- 2: unset the conda funcs +-- 3: define local variable prefix as path to Miniconda installation +-- 4: remove from PATH all paths to $prefix +-- 5: unset CONDA_ env. variables that are set via sourcing conda.sh +-- 6: unset previously set variable $prefix +local unload_cmd="for i in $(seq ${CONDA_SHLVL:=0}); do conda deactivate; done; \ +unset -f " .. funcs .. "; \ +prefix=" .. base .. "; \ +export PATH=$(echo ${PATH} | tr ':' '\n' | grep . | grep -v $prefix | tr '\n' ':' | sed 's/:$//'); \ +unset $(env | grep CONDA | grep -ev 'CONDA_ENVS_PATH|CONDA_PKGS_PATH' | cut -d= -f1)" + +-- source conda on load, deactivate on unload +if (mode() == "load") then + execute{cmd=load_cmd, modeA={"load"}} +else + if (mode() == "unload") then + execute{cmd=unload_cmd, modeA={"unload"}} + end +end whatis("Name: ".. pkgName) whatis("Version: " .. pkgVersion) diff --git a/modulefiles/core/python/python.lua b/modulefiles/core/python/python.lua index 2f3ec6f4..47b6cef1 100644 --- a/modulefiles/core/python/python.lua +++ b/modulefiles/core/python/python.lua @@ -13,7 +13,7 @@ conflict("intelpython") local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules" -local mpath = pathJoin(opt,"modulefiles/core",pkgName,pkgVersion) +local mpath = pathJoin(opt,"modulefiles/python",pkgName,pkgVersion) prepend_path("MODULEPATH", mpath) local base = pathJoin(opt,pkgName,pkgVersion) diff --git a/modulefiles/python/pythonName/pythonVersion/condaenv/condaenv.lua b/modulefiles/python/pythonName/pythonVersion/condaenv/condaenv.lua new file mode 100644 index 00000000..36846e0d --- /dev/null +++ b/modulefiles/python/pythonName/pythonVersion/condaenv/condaenv.lua @@ -0,0 +1,32 @@ +help([[ +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +local hierA = hierarchyA(pkgNameVer,1) +local pythonNameVer = hierA[1] +local pythonNameVerD = pythonNameVer:gsub("/","-") + +conflict(pkgName) + +local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules" + +local base = pathJoin(opt,pythonNameVerD,pkgName,pkgVersion) + +-- activate on load, deactivate on unload +if (mode() == "load") then + local load_cmd = "conda activate " .. pkgName + execute{cmd=load_cmd, modeA={"load"}} +else + if (mode() == "unload") then + local unload_cmd = "conda deactivate " .. pkgName + execute{cmd=unload_cmd, modeA={"unload"}} + end +end + +whatis("Name: " .. pkgName) +whatis("Version: " .. pkgVersion) +whatis("Category: Software") +whatis("Description: " .. pkgName .. " Conda Environment") diff --git a/stack/stack_mac.yaml b/stack/stack_mac.yaml index d300ca0b..bea2c0eb 100644 --- a/stack/stack_mac.yaml +++ b/stack/stack_mac.yaml @@ -339,6 +339,10 @@ proj: version: 7.1.0 cmake_opts: "-DENABLE_CURL=OFF -DBUILD_PROJSYNC=OFF" +miniconda3: + build: YES + version: 4.7.12 + r2d2: build: NO version: 1.0.0 From e3d9bf0d58b701946d8f04d49bf41884e23b1ddc Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 13 Jul 2021 22:39:10 -0400 Subject: [PATCH 2/5] define venvtype, add ability to build condaenv and pyvenv. fix miniconda3 modulefile --- config/config_mac.sh | 1 + modulefiles/core/miniconda3/miniconda3.lua | 9 +++++---- stack_helpers.sh | 9 ++++++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/config/config_mac.sh b/config/config_mac.sh index c49dbfa3..b5b83b5c 100644 --- a/config/config_mac.sh +++ b/config/config_mac.sh @@ -18,6 +18,7 @@ export MAKE_CLEAN=N export DOWNLOAD_ONLY=N export STACK_EXIT_ON_FAIL=Y export WGET="wget -nv" +export VENVTYPE="condaenv" # Conda Env. OR Python Virtual Env. # gfortran-10 needs the following #export STACK_esmf_FFLAGS="-fallow-argument-mismatch -fallow-invalid-boz" diff --git a/modulefiles/core/miniconda3/miniconda3.lua b/modulefiles/core/miniconda3/miniconda3.lua index 9021e185..7ff9fdb9 100644 --- a/modulefiles/core/miniconda3/miniconda3.lua +++ b/modulefiles/core/miniconda3/miniconda3.lua @@ -26,9 +26,9 @@ local funcs = "conda __conda_activate __conda_hashr __conda_reactivate __add_sys -- Line #: What does it do? -- 1: source conda.sh from the installation path --- 2: export conda functions +-- 2: export conda functions silently(> dev/null) local load_cmd = "source " .. pathJoin(base, "etc/profile.d/conda.sh") .. "; \ -export -f " .. funcs +export -f " .. funcs .. " > /dev/null" -- Line #: What does it do? -- 1: deactivate all conda envs @@ -40,8 +40,9 @@ export -f " .. funcs local unload_cmd="for i in $(seq ${CONDA_SHLVL:=0}); do conda deactivate; done; \ unset -f " .. funcs .. "; \ prefix=" .. base .. "; \ -export PATH=$(echo ${PATH} | tr ':' '\n' | grep . | grep -v $prefix | tr '\n' ':' | sed 's/:$//'); \ -unset $(env | grep CONDA | grep -ev 'CONDA_ENVS_PATH|CONDA_PKGS_PATH' | cut -d= -f1)" +export PATH=$(echo $PATH | tr ':' '\\n' | grep . | grep -v $prefix | tr '\\n' ':' | sed 's/:$//'); \ +unset $(env | grep -o \"[^=]*CONDA[^=]*\" | grep -v 'CONDA_ENVS_PATH\\|CONDA_PKGS_DIRS'); \ +unset prefix" -- source conda on load, deactivate on unload if (mode() == "load") then diff --git a/stack_helpers.sh b/stack_helpers.sh index af90bca9..e65c9480 100755 --- a/stack_helpers.sh +++ b/stack_helpers.sh @@ -201,7 +201,14 @@ function build_lib() { if [[ ${is_nceplib:-} =~ [yYtT] ]]; then ${HPC_STACK_ROOT}/libs/build_nceplibs.sh "$1" 2>&1 | tee "$log" elif [[ ${is_pyvenv:-} =~ [yYtT] ]]; then - ${HPC_STACK_ROOT}/libs/build_pyvenv.sh "$1" 2>&1 | tee "$log" + if [[ "$VENVTYPE" == "pyvenv" ]]; then + ${HPC_STACK_ROOT}/libs/build_pyvenv.sh "$1" 2>&1 | tee "$log" + elif [[ "$VENVTYPE" == "condaenv" ]]; then + ${HPC_STACK_ROOT}/libs/build_condaenv.sh "$1" 2>&1 | tee "$log" + else + echo "BUILD FAIL! Lib: $1 Error: Unknown VENVTYPE = $VENVTYPE" + [[ ${STACK_EXIT_ON_FAIL} =~ [yYtT] ]] && exit 999 + fi else ${HPC_STACK_ROOT}/libs/build_$1.sh 2>&1 | tee "$log" fi From df7e51fa0d0bba4a007d8a13527a2f116878e347 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 15 Jul 2021 10:25:42 -0400 Subject: [PATCH 3/5] add condaenv ymls. --- README.md | 4 +- libs/build_condaenv.sh | 38 ++++++++++++++++--- libs/build_miniconda3.sh | 17 ++++++++- libs/build_pyvenv.sh | 10 ++--- .../pythonVersion/condaenv/condaenv.lua | 2 +- pyvenv/hofx.txt | 3 -- pyvenv/hofx.yml | 27 +++++++++++++ pyvenv/r2d2.txt | 3 -- pyvenv/r2d2.yml | 26 +++++++++++++ pyvenv/ufswm.txt | 1 - pyvenv/ufswm.yml | 11 ++++++ stack/stack_mac.yaml | 4 +- stack_helpers.sh | 6 ++- 13 files changed, 126 insertions(+), 26 deletions(-) create mode 100644 pyvenv/hofx.yml create mode 100644 pyvenv/r2d2.yml create mode 100644 pyvenv/ufswm.yml diff --git a/README.md b/README.md index 25963ed8..8d3ce4a7 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,8 @@ some of the parameter settings available. - **MAKE_VERBOSE:** Print out extra information to the log files during the build +- **VENVTYPE:** Set the type of python environment to build. Value depends on whether using `pip` or `conda`. Set `VENVTYPE=pyvenv` when using `pip` and `VENVTYPE=condaenv` when using `Miniconda` for creating virtual environments. + The next step is to choose what components of the stack you wish to build. This is done by editing the file `stack/stack_custom.yaml` which defines the software packages to be built along with their @@ -314,7 +316,7 @@ template 2. define a new section in the `yaml` file for that library/package in config directory -3. if the package is a python virtual environment, add a `requirements.txt` file listing the python packages required to install the package in `pyvenv/package_name.txt` +3. if the package is a python virtual environment, add a `requirements.txt` or `environment.yml` file containing the listing the python packages required to install the package. These files should be named and placed in `pyvenv/package_name.txt` and `pyvenv/package_name.yml`. `VENVTYPE=pyvenv` will use the `pyvenv/package_name.txt` and `VENVTYPE=condaenv` will use `pyvenv/package_name.yml` 4. Add a call to the new build script in `build_stack.sh` diff --git a/libs/build_condaenv.sh b/libs/build_condaenv.sh index 52ae3170..fb6f9d9c 100755 --- a/libs/build_condaenv.sh +++ b/libs/build_condaenv.sh @@ -10,11 +10,15 @@ stack_version=${!var} set -u version=${2:-$stack_version} -var="STACK_${name}_environment" +var="STACK_${name}_requirements" set +u stack_rqmts=${!var} set -u -rqmts=${stack_rqmts:-"environments.yml"} +rqmts=${stack_rqmts:-"$name.yml"} + +# Check for conda environment file +rqmts_file=${HPC_STACK_ROOT}/pyvenv/$rqmts +[[ ! -f $rqmts_file ]] && ( echo "Unable to find environment file: $rqmts \nABORT!"; exit 1 ) python=$(echo $HPC_PYTHON | sed 's/\//-/g') @@ -36,9 +40,14 @@ fi cd ${HPC_STACK_ROOT}/${PKGDIR:-"pkg"} -# Check for conda environment file -rqmts_file=${HPC_STACK_ROOT}/pyvenv/$rqmts -[[ ! -f $rqmts_file ]] && ( echo "Unable to find enviroment file: $rqmts \nABORT!"; exit 1 ) +software=${name}-${version} +mkdir -p $software + +# Activate conda's base environment to get a few details such as python_version +set +x +echo "executing ... conda activate base" +conda activate base +set -x # Determine python version; 3.x python_version=$(python3 -c 'import sys;print(sys.version_info[0],".",sys.version_info[1],sep="")') @@ -50,8 +59,25 @@ if (( $(echo "$min_python_version >= $python_version" | bc -l) )); then exit 1 fi +# Deactivate base environment before building new conda environment +set +x +echo "executing ... conda deactivate" +conda deactivate +set -x + # Create the conda environment -conda env create --file $rqmts_file -p $prefix +set +x +echo "executing ... conda env create -q --file $rqmts_file" +conda env create -q --file $rqmts_file +set -x + +# Activate conda environment and get list +set +x +echo "executing ... conda activate $name" +conda activate $name +echo "executing ... conda env list" +conda env list +set -x # generate modulefile from template $MODULES && update_modules python $name $version $python_version diff --git a/libs/build_miniconda3.sh b/libs/build_miniconda3.sh index 5849a177..f9f46e28 100755 --- a/libs/build_miniconda3.sh +++ b/libs/build_miniconda3.sh @@ -7,6 +7,7 @@ set -eux name="miniconda3" version=${2:-${STACK_miniconda3_version:-"latest"}} +pyversion=${3:-${STACK_miniconda3_pyversion:-}} if $MODULES; then prefix="${PREFIX:-"/opt/modules"}/core/$name/$version" @@ -32,13 +33,25 @@ else fi software=$name-$version -installer="Miniconda3-${version}-${os}-x86_64.sh" +pkg_version=$version +[[ -n ${pyversion:-} ]] && pkg_version=$pyversion_$version +installer="Miniconda3-${pkg_version}-${os}-x86_64.sh" URL="https://repo.anaconda.com/miniconda/$installer" [[ -d $software ]] || ( mkdir -p $software; $WGET $URL -O $software/$installer ) [[ ${DOWNLOAD_ONLY} =~ [yYtT] ]] && exit 0 -$SUDO bash $software/$installer -b -p $prefix +$SUDO bash $software/$installer -b -p $prefix -s +set +x +echo "sourcing conda.sh" +source $prefix/etc/profile.d/conda.sh +echo "setting conda default threads to 4" +conda config --system --set default_threads 4 +echo "disabling conda auto updates" +conda config --system --set auto_update_conda False +echo "install $version of conda" +conda install -yq conda=$version +set -x # generate modulefile from template $MODULES && update_modules core $name $version diff --git a/libs/build_pyvenv.sh b/libs/build_pyvenv.sh index 1ccaa9e1..26d2ca71 100755 --- a/libs/build_pyvenv.sh +++ b/libs/build_pyvenv.sh @@ -14,7 +14,11 @@ var="STACK_${name}_requirements" set +u stack_rqmts=${!var} set -u -rqmts=${stack_rqmts:-"requirements.txt"} +rqmts=${stack_rqmts:-"$name.txt"} + +# Check for requirements file +rqmts_file=${HPC_STACK_ROOT}/pyvenv/$rqmts +[[ ! -f $rqmts_file ]] && ( echo "Unable to find requirements file: $rqmts \nABORT!"; exit 1 ) python=$(echo $HPC_PYTHON | sed 's/\//-/g') @@ -49,10 +53,6 @@ if (( $(echo "$min_python_version >= $python_version" | bc -l) )); then exit 1 fi -# Check for requirements file -rqmts_file=${HPC_STACK_ROOT}/pyvenv/$rqmts -[[ ! -f $rqmts_file ]] && ( echo "Unable to find requirements file: $rqmts \nABORT!"; exit 1 ) - # Download the requirements of the virtual environment if [[ ${DOWNLOAD_ONLY} =~ [yYtT] ]]; then pip download -r $rqmts_file -d $software diff --git a/modulefiles/python/pythonName/pythonVersion/condaenv/condaenv.lua b/modulefiles/python/pythonName/pythonVersion/condaenv/condaenv.lua index 36846e0d..5c354172 100644 --- a/modulefiles/python/pythonName/pythonVersion/condaenv/condaenv.lua +++ b/modulefiles/python/pythonName/pythonVersion/condaenv/condaenv.lua @@ -21,7 +21,7 @@ if (mode() == "load") then execute{cmd=load_cmd, modeA={"load"}} else if (mode() == "unload") then - local unload_cmd = "conda deactivate " .. pkgName + local unload_cmd = "conda deactivate" execute{cmd=unload_cmd, modeA={"unload"}} end end diff --git a/pyvenv/hofx.txt b/pyvenv/hofx.txt index abc6e57a..514e6e8d 100644 --- a/pyvenv/hofx.txt +++ b/pyvenv/hofx.txt @@ -11,14 +11,11 @@ python-dateutil >= 2.8.1 pytz >= 2020.4 PyYAML >= 5.3.1 s3transfer >= 0.3.3 -sdist >= 0.0.0 six >= 1.15.0 urllib3 >= 1.26.2 wheel >= 0.36.2 xarray >= 0.16.2 - netcdf4 >= 1.5.5.1 -h5netcdf >= 0.8.1 h5py >= 3.1.0 git+https://github.com/jcsda-internal/solo.git git+https://github.com/jcsda-internal/r2d2.git diff --git a/pyvenv/hofx.yml b/pyvenv/hofx.yml new file mode 100644 index 00000000..5359ed21 --- /dev/null +++ b/pyvenv/hofx.yml @@ -0,0 +1,27 @@ +name: hofx +channels: + - defaults +dependencies: + - boto3 >= 1.16.38 + - botocore >= 1.19.38 + - cftime >= 1.3.0 + - click >= 7.1.2 + - Cython >= 0.29.21 + - jmespath >= 0.10.0 + - numpy >= 1.19.4 + - pandas >= 1.1.5 + - pip >= 20.3.3 + - python-dateutil >= 2.8.1 + - pytz >= 2020.4 + - PyYAML >= 5.3.1 + - s3transfer >= 0.3.3 + - six >= 1.15.0 + - urllib3 >= 1.26.2 + - wheel >= 0.36.2 + - xarray >= 0.16.2 + - netcdf4 >= 1.5.5.1 + - h5py >= 3.1.0 + - pip: + - git+https://github.com/jcsda-internal/solo.git + - git+https://github.com/jcsda-internal/r2d2.git + - git+https://github.com/noaa-emc/hofxcs.git diff --git a/pyvenv/r2d2.txt b/pyvenv/r2d2.txt index d5f7b69a..fb42bdcf 100644 --- a/pyvenv/r2d2.txt +++ b/pyvenv/r2d2.txt @@ -11,14 +11,11 @@ python-dateutil >= 2.8.1 pytz >= 2020.4 PyYAML >= 5.3.1 s3transfer >= 0.3.3 -sdist >= 0.0.0 six >= 1.15.0 urllib3 >= 1.26.2 wheel >= 0.36.2 xarray >= 0.16.2 - netcdf4 >= 1.5.5.1 -h5netcdf >= 0.8.1 h5py >= 3.1.0 git+https://github.com/jcsda-internal/solo.git git+https://github.com/jcsda-internal/r2d2.git diff --git a/pyvenv/r2d2.yml b/pyvenv/r2d2.yml new file mode 100644 index 00000000..3f049d7b --- /dev/null +++ b/pyvenv/r2d2.yml @@ -0,0 +1,26 @@ +name: r2d2 +channels: + - defaults +dependencies: + - boto3 >= 1.16.38 + - botocore >= 1.19.38 + - cftime >= 1.3.0 + - click >= 7.1.2 + - Cython >= 0.29.21 + - jmespath >= 0.10.0 + - numpy >= 1.19.4 + - pandas >= 1.1.5 + - pip >= 20.3.3 + - python-dateutil >= 2.8.1 + - pytz >= 2020.4 + - PyYAML >= 5.3.1 + - s3transfer >= 0.3.3 + - six >= 1.15.0 + - urllib3 >= 1.26.2 + - wheel >= 0.36.2 + - xarray >= 0.16.2 + - netcdf4 >= 1.5.5.1 + - h5py >= 3.1.0 + - pip: + - git+https://github.com/jcsda-internal/solo.git + - git+https://github.com/jcsda-internal/r2d2.git diff --git a/pyvenv/ufswm.txt b/pyvenv/ufswm.txt index e33397a7..e45e10c8 100644 --- a/pyvenv/ufswm.txt +++ b/pyvenv/ufswm.txt @@ -3,6 +3,5 @@ Cython >= 0.29.21 numpy >= 1.19.4 pandas >= 1.1.5 python-dateutil >= 2.8.1 -h5netcdf >= 0.8.1 h5py >= 3.1.0 netcdf4 >= 1.5.5.1 diff --git a/pyvenv/ufswm.yml b/pyvenv/ufswm.yml new file mode 100644 index 00000000..e1e7ffd4 --- /dev/null +++ b/pyvenv/ufswm.yml @@ -0,0 +1,11 @@ +name: ufswm +channels: + - defaults +dependencies: + - cftime >= 1.3.0 + - Cython >= 0.29.21 + - numpy >= 1.19.4 + - pandas >= 1.1.5 + - python-dateutil >= 2.8.1 + - h5py >= 3.1.0 + - netcdf4 >= 1.5.5.1 diff --git a/stack/stack_mac.yaml b/stack/stack_mac.yaml index 79b4f1ab..5d79fc89 100644 --- a/stack/stack_mac.yaml +++ b/stack/stack_mac.yaml @@ -344,13 +344,11 @@ miniconda3: version: 4.7.12 r2d2: - build: NO + build: YES version: 1.0.0 - requirements: r2d2.txt is_pyvenv: YES ufswm: build: NO version: 1.0.0 - requirements: ufswm.txt is_pyvenv: YES diff --git a/stack_helpers.sh b/stack_helpers.sh index e65c9480..5992d4fd 100755 --- a/stack_helpers.sh +++ b/stack_helpers.sh @@ -8,7 +8,11 @@ function update_modules { local py_version=${4:-} case $modpath in python ) - local tmpl_file=$HPC_STACK_ROOT/modulefiles/python/pythonName/pythonVersion/pyvenv/pyvenv.lua + if [[ "$VENVTYPE" == "pyvenv" ]]; then + local tmpl_file=$HPC_STACK_ROOT/modulefiles/python/pythonName/pythonVersion/pyvenv/pyvenv.lua + elif [[ "$VENVTYPE" == "condaenv" ]]; then + local tmpl_file=$HPC_STACK_ROOT/modulefiles/python/pythonName/pythonVersion/condaenv/condaenv.lua + fi local to_dir=$prefix/modulefiles/python/$HPC_PYTHON ;; core ) From e6ecdc1cb6a2c18a201f80197873b7e1ba7bcec4 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 15 Jul 2021 10:36:08 -0400 Subject: [PATCH 4/5] update configs and yamls. Default python venv is pyvenv using python+pip --- README.md | 2 +- config/config_hera.sh | 1 + config/config_mac.sh | 4 +--- config/config_s4.sh | 1 + stack/stack_custom.yaml | 2 -- stack/stack_jedi.yaml | 1 - stack/stack_mac.yaml | 4 ++-- stack/stack_noaa.yaml | 3 --- stack_helpers.sh | 11 ++++------- 9 files changed, 10 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 8d3ce4a7..eb4867bf 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ some of the parameter settings available. - **MAKE_VERBOSE:** Print out extra information to the log files during the build -- **VENVTYPE:** Set the type of python environment to build. Value depends on whether using `pip` or `conda`. Set `VENVTYPE=pyvenv` when using `pip` and `VENVTYPE=condaenv` when using `Miniconda` for creating virtual environments. +- **VENVTYPE:** Set the type of python environment to build. Value depends on whether using `pip` or `conda`. Set `VENVTYPE=pyvenv` when using `pip` and `VENVTYPE=condaenv` when using `Miniconda` for creating virtual environments. Default is `pyvenv` The next step is to choose what components of the stack you wish to build. This is done by editing the file `stack/stack_custom.yaml` diff --git a/config/config_hera.sh b/config/config_hera.sh index 8e9e7d38..65e598e0 100644 --- a/config/config_hera.sh +++ b/config/config_hera.sh @@ -17,6 +17,7 @@ export MAKE_CLEAN=N export DOWNLOAD_ONLY=N export STACK_EXIT_ON_FAIL=Y export WGET="wget -nv" +export VENVTYPE="condaenv" # Load these basic modules for Hera module purge diff --git a/config/config_mac.sh b/config/config_mac.sh index b5b83b5c..64d5f6cf 100644 --- a/config/config_mac.sh +++ b/config/config_mac.sh @@ -3,8 +3,7 @@ # Compiler/MPI combination export HPC_COMPILER="clang/12.0.5" export HPC_MPI="mpich/3.3.2" -#export HPC_PYTHON="python/3.9.6" -export HPC_PYTHON="miniconda3/4.7.12" +export HPC_PYTHON="python/3.9.6" # Build options export USE_SUDO=N @@ -18,7 +17,6 @@ export MAKE_CLEAN=N export DOWNLOAD_ONLY=N export STACK_EXIT_ON_FAIL=Y export WGET="wget -nv" -export VENVTYPE="condaenv" # Conda Env. OR Python Virtual Env. # gfortran-10 needs the following #export STACK_esmf_FFLAGS="-fallow-argument-mismatch -fallow-invalid-boz" diff --git a/config/config_s4.sh b/config/config_s4.sh index ad9dfb95..b0d51f67 100644 --- a/config/config_s4.sh +++ b/config/config_s4.sh @@ -17,6 +17,7 @@ export MAKE_CLEAN=N export DOWNLOAD_ONLY=N export STACK_EXIT_ON_FAIL=Y export WGET="wget -nv" +export VENVTYPE="condaenv" # Load these basic modules for S4 module purge diff --git a/stack/stack_custom.yaml b/stack/stack_custom.yaml index 6c38d3af..9308f65b 100644 --- a/stack/stack_custom.yaml +++ b/stack/stack_custom.yaml @@ -342,11 +342,9 @@ proj: r2d2: build: NO version: 1.0.0 - requirements: r2d2.txt is_pyvenv: YES ufswm: build: NO version: 1.0.0 - requirements: ufswm.txt is_pyvenv: YES diff --git a/stack/stack_jedi.yaml b/stack/stack_jedi.yaml index 761f7644..f0c76e99 100644 --- a/stack/stack_jedi.yaml +++ b/stack/stack_jedi.yaml @@ -297,5 +297,4 @@ atlas: r2d2: build: YES version: 1.0.0 - requirements: r2d2.txt is_pyvenv: YES diff --git a/stack/stack_mac.yaml b/stack/stack_mac.yaml index 5d79fc89..ad8e643e 100644 --- a/stack/stack_mac.yaml +++ b/stack/stack_mac.yaml @@ -340,11 +340,11 @@ proj: cmake_opts: "-DENABLE_CURL=OFF -DBUILD_PROJSYNC=OFF" miniconda3: - build: YES + build: NO version: 4.7.12 r2d2: - build: YES + build: NO version: 1.0.0 is_pyvenv: YES diff --git a/stack/stack_noaa.yaml b/stack/stack_noaa.yaml index c4420f61..d5f40235 100644 --- a/stack/stack_noaa.yaml +++ b/stack/stack_noaa.yaml @@ -338,17 +338,14 @@ proj: r2d2: build: NO version: 1.0.0 - requirements: r2d2.txt is_pyvenv: YES hofx: build: NO version: 1.0.0 - requirements: hofx.txt is_pyvenv: YES ufswm: build: YES version: 1.0.0 - requirements: ufswm.txt is_pyvenv: YES diff --git a/stack_helpers.sh b/stack_helpers.sh index 5992d4fd..7c565c92 100755 --- a/stack_helpers.sh +++ b/stack_helpers.sh @@ -8,9 +8,9 @@ function update_modules { local py_version=${4:-} case $modpath in python ) - if [[ "$VENVTYPE" == "pyvenv" ]]; then + if [[ "${VENVTYPE:-"pyvenv"}" == "pyvenv" ]]; then local tmpl_file=$HPC_STACK_ROOT/modulefiles/python/pythonName/pythonVersion/pyvenv/pyvenv.lua - elif [[ "$VENVTYPE" == "condaenv" ]]; then + elif [[ "${VENVTYPE:-"pyvenv"}" == "condaenv" ]]; then local tmpl_file=$HPC_STACK_ROOT/modulefiles/python/pythonName/pythonVersion/condaenv/condaenv.lua fi local to_dir=$prefix/modulefiles/python/$HPC_PYTHON @@ -205,13 +205,10 @@ function build_lib() { if [[ ${is_nceplib:-} =~ [yYtT] ]]; then ${HPC_STACK_ROOT}/libs/build_nceplibs.sh "$1" 2>&1 | tee "$log" elif [[ ${is_pyvenv:-} =~ [yYtT] ]]; then - if [[ "$VENVTYPE" == "pyvenv" ]]; then + if [[ "${VENVTYPE:-"pyvenv"}" == "pyvenv" ]]; then ${HPC_STACK_ROOT}/libs/build_pyvenv.sh "$1" 2>&1 | tee "$log" - elif [[ "$VENVTYPE" == "condaenv" ]]; then + elif [[ "${VENVTYPE:-"pyvenv"}" == "condaenv" ]]; then ${HPC_STACK_ROOT}/libs/build_condaenv.sh "$1" 2>&1 | tee "$log" - else - echo "BUILD FAIL! Lib: $1 Error: Unknown VENVTYPE = $VENVTYPE" - [[ ${STACK_EXIT_ON_FAIL} =~ [yYtT] ]] && exit 999 fi else ${HPC_STACK_ROOT}/libs/build_$1.sh 2>&1 | tee "$log" From cb48c28869575a73ba85a2de16416676cb5aab61 Mon Sep 17 00:00:00 2001 From: Kyle Gerheiser <3209794+kgerheiser@users.noreply.github.com> Date: Thu, 15 Jul 2021 15:16:36 -0400 Subject: [PATCH 5/5] Set PS1 to avoid unset variable error in conda.sh Co-authored-by: Rahul Mahajan --- libs/build_miniconda3.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/build_miniconda3.sh b/libs/build_miniconda3.sh index f9f46e28..2da854fb 100755 --- a/libs/build_miniconda3.sh +++ b/libs/build_miniconda3.sh @@ -44,6 +44,7 @@ URL="https://repo.anaconda.com/miniconda/$installer" $SUDO bash $software/$installer -b -p $prefix -s set +x echo "sourcing conda.sh" +PS1= source $prefix/etc/profile.d/conda.sh echo "setting conda default threads to 4" conda config --system --set default_threads 4