From 92ed9bccb603f24f341ec1aee49323e9b90a35dc Mon Sep 17 00:00:00 2001 From: noel Date: Wed, 9 Aug 2017 10:38:57 -0700 Subject: [PATCH 1/2] For all NERSC mmachines, add a `--compiler=intel18` option and continue making the machine settings more consistent. --- config/acme/machines/config_compilers.xml | 59 +++ config/acme/machines/config_machines.xml | 512 +++++++++++----------- 2 files changed, 326 insertions(+), 245 deletions(-) diff --git a/config/acme/machines/config_compilers.xml b/config/acme/machines/config_compilers.xml index b514dd6bc3c..62e55049e40 100644 --- a/config/acme/machines/config_compilers.xml +++ b/config/acme/machines/config_compilers.xml @@ -300,7 +300,34 @@ for mct, etc. -cxxlib TRUE TRUE + + + -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL + -qopenmp + -qopenmp + -qopenmp + -free + -fixed -132 + -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created + -O2 -debug minimal -qno-opt-dynamic-align + -O2 -debug minimal + -O0 -g + -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source + -O2 -fp-model precise -std=gnu99 + -O0 + -qopenmp + -r8 + ifort + icc + icpc + mpif90 + mpicc + mpicxx + FORTRAN + -cxxlib + TRUE + TRUE @@ -593,6 +620,38 @@ for mct, etc. $(PETSC_DIR) + + + --host=Linux + -L$(NETCDF_DIR) -lnetcdff -Wl,--as-needed,-L$(NETCDF_DIR)/lib -lnetcdff -lnetcdf + -mkl -lpthread + ifort + icc + icpc + $(PETSC_DIR) + + + + --host=Linux + -L$(NETCDF_DIR) -lnetcdff -Wl,--as-needed,-L$(NETCDF_DIR)/lib -lnetcdff -lnetcdf + -mkl -lpthread + ifort + icc + icpc + $(PETSC_DIR) + + + + --host=Linux + -L$(NETCDF_DIR) -lnetcdff -Wl,--as-needed,-L$(NETCDF_DIR)/lib -lnetcdff -lnetcdf + -mkl -lpthread + -DARCH_MIC_KNL + ifort + icc + icpc + $(PETSC_DIR) + + -O2 -g -traceback -O0 -fpe0 -check all -check noarg_temp_created -ftrapuv diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml index 87f5bf331fa..77831c4707c 100644 --- a/config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -55,12 +55,11 @@ This allows using a different mpirun command to launch unit tests --> - NERSC XC30, os is CNL, 24 pes/node, batch system is SLURM edison acme_developer - intel,gnu + intel,intel18,gnu mpt,mpi-serial $ENV{CSCRATCH}/acme_scratch/edison $CIME_OUTPUT_ROOT/$CASE/run @@ -128,12 +127,20 @@ intel/17.0.2.174 cray-libsci + + + PrgEnv-intel + intel + intel/2018.beta + cray-libsci + + + PrgEnv-intel PrgEnv-gnu gcc gcc/6.3.0 - cray-libsci - cray-libsci/17.06.1 + cray-libsci/16.09.1 @@ -170,276 +177,291 @@ spread threads yes + yes - Cori. XC40 Cray system at NERSC. Haswell partition. os is CNL, 32 pes/node, batch system is SLURM - cori - acme_developer - intel,gnu,cray - mpt,mpi-serial - $ENV{SCRATCH}/acme_scratch/cori-haswell - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - /project/projectdirs/acme/inputdata - /project/projectdirs/acme/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - csm/$CASE - /project/projectdirs/acme/baselines - /project/projectdirs/acme/tools/cprnc.cori/cprnc - /project/projectdirs/$PROJECT - CNL - slurm - acme - 8 - 32 - 32 - acme - TRUE - -D PIO_BUILD_TIMING:BOOL=ON - - srun - - --label - -n $TOTALPES - - -c 2 - --cpu_bind=cores - - - - /opt/modules/default/init/perl - /opt/modules/default/init/python - /opt/modules/default/init/sh - /opt/modules/default/init/csh - /opt/modules/default/bin/modulecmd perl - /opt/modules/default/bin/modulecmd python - module - module + Cori. XC40 Cray system at NERSC. Haswell partition. os is CNL, 32 pes/node, batch system is SLURM + cori + acme_developer + intel,intel18,gnu + mpt,mpi-serial + $ENV{SCRATCH}/acme_scratch/cori-haswell + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + /project/projectdirs/acme/inputdata + /project/projectdirs/acme/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + csm/$CASE + /project/projectdirs/acme/baselines + /project/projectdirs/acme/tools/cprnc.cori/cprnc + /project/projectdirs/$PROJECT + CNL + slurm + acme + 8 + 32 + 32 + acme + TRUE + -D PIO_BUILD_TIMING:BOOL=ON + + srun + + --label + -n $TOTALPES + + -c 2 + --cpu_bind=cores + + + + /opt/modules/default/init/perl + /opt/modules/default/init/python + /opt/modules/default/init/sh + /opt/modules/default/init/csh + /opt/modules/default/bin/modulecmd perl + /opt/modules/default/bin/modulecmd python + module + module - - PrgEnv-intel - PrgEnv-cray - PrgEnv-gnu - intel - cce - gcc - cray-parallel-netcdf - cray-parallel-hdf5 - pmi - cray-libsci - cray-mpich2 - cray-mpich - cray-netcdf - cray-hdf5 - cray-netcdf-hdf5parallel - craype-sandybridge - craype-ivybridge - craype - papi - cmake - cray-petsc - esmf - + + PrgEnv-intel + PrgEnv-cray + PrgEnv-gnu + intel + cce + gcc + cray-parallel-netcdf + cray-parallel-hdf5 + pmi + cray-libsci + cray-mpich2 + cray-mpich + cray-netcdf + cray-hdf5 + cray-netcdf-hdf5parallel + craype-sandybridge + craype-ivybridge + craype + papi + cmake + cray-petsc + esmf + - - craype - craype/2.5.7 - craype-haswell + + PrgEnv-intel + intel + intel/17.0.2.174 + craype + craype/2.5.7 + - - cray-mpich/7.5.3 - + + PrgEnv-intel + intel + intel/2018.beta + craype + craype/2.5.10 + - - PrgEnv-intel - intel - intel/17.0.2.174 - + + PrgEnv-intel + PrgEnv-gnu + gcc + gcc/6.3.0 + cray-libsci/16.09.1 + craype + craype/2.5.7 + - - PrgEnv-cray - cce cce/8.5.4 - - - PrgEnv-gnu - gcc gcc/6.3.0 - + + cray-mpich + cray-mpich/7.5.5 - - cray-libsci - cray-libsci/16.09.1 - + craype-mic-knl + craype-haswell + - - cray-netcdf-hdf5parallel - cray-hdf5-parallel - cray-parallel-netcdf - cray-hdf5/1.8.16 - cray-netcdf/4.4.0 - - - cray-netcdf-hdf5parallel - cray-netcdf-hdf5parallel/4.4.0 - cray-hdf5-parallel/1.8.16 - cray-parallel-netcdf/1.7.0 - - - git/2.9.1 - cmake/3.3.2 - pmi/5.0.10-1.0000.11069.183.8.ari - zlib/1.2.8 - - + + cray-netcdf-hdf5parallel + cray-hdf5-parallel + cray-parallel-netcdf + cray-hdf5/1.8.16 + cray-netcdf/4.4.0 + + + cray-netcdf-hdf5parallel + cray-netcdf-hdf5parallel/4.4.0 + cray-hdf5-parallel/1.8.16 + cray-parallel-netcdf/1.7.0 + - + + git/2.9.1 + cmake/3.3.2 + pmi/5.0.10-1.0000.11069.183.8.ari + zlib/1.2.8 + + - 1 - 1 - + - 128M - spread - threads - yes - + 1 + 1 + + + 128M + spread + threads + yes + yes + - Cori. XC40 Cray system at NERSC. KNL partition. os is CNL, 68 pes/node (for now only use 64), batch system is SLURM - cori-knl-haswell-is-default - acme_developer - intel,gnu,cray - mpt,mpi-serial - $ENV{SCRATCH}/acme_scratch/cori-knl - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - /project/projectdirs/acme/inputdata - /project/projectdirs/acme/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - csm/$CASE - /project/projectdirs/acme/baselines - /project/projectdirs/acme/tools/cprnc.cori/cprnc - /project/projectdirs/$PROJECT - CNL - slurm - acme - 8 - 256 - 64 - acme - TRUE - -D PIO_BUILD_TIMING:BOOL=ON - - srun - - --label - -n $TOTALPES - - -c 4 - --cpu_bind=cores - - - - /opt/modules/default/init/perl - /opt/modules/default/init/python - /opt/modules/default/init/sh - /opt/modules/default/init/csh - /opt/modules/default/bin/modulecmd perl - /opt/modules/default/bin/modulecmd python - module - module - - intel - cce - gcc - cray-parallel-netcdf - cray-parallel-hdf5 - pmi - cray-libsci - cray-mpich2 - cray-mpich - cray-netcdf - cray-hdf5 - cray-netcdf-hdf5parallel - craype-sandybridge - craype-ivybridge - craype - cray-libsci - papi - cmake - cray-petsc - esmf - + Cori. XC40 Cray system at NERSC. KNL partition. os is CNL, 68 pes/node (for now only use 64), batch system is SLURM + cori-knl-haswell-is-default + acme_developer + intel,intel18,gnu + mpt,mpi-serial + $ENV{SCRATCH}/acme_scratch/cori-knl + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + /project/projectdirs/acme/inputdata + /project/projectdirs/acme/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + csm/$CASE + /project/projectdirs/acme/baselines + /project/projectdirs/acme/tools/cprnc.cori/cprnc + /project/projectdirs/$PROJECT + CNL + slurm + acme + 8 + 256 + 64 + acme + TRUE + -D PIO_BUILD_TIMING:BOOL=ON + + srun + + --label + -n $TOTALPES + + -c 4 + --cpu_bind=cores + + + + /opt/modules/default/init/perl + /opt/modules/default/init/python + /opt/modules/default/init/sh + /opt/modules/default/init/csh + /opt/modules/default/bin/modulecmd perl + /opt/modules/default/bin/modulecmd python + module + module + + intel + cce + gcc + cray-parallel-netcdf + cray-parallel-hdf5 + pmi + cray-libsci + cray-mpich2 + cray-mpich + cray-netcdf + cray-hdf5 + cray-netcdf-hdf5parallel + craype-sandybridge + craype-ivybridge + craype + cray-libsci + papi + cmake + cray-petsc + esmf + - - PrgEnv-intel - intel - intel/17.0.2.174 - + + PrgEnv-intel + intel + intel/17.0.2.174 + craype + craype/2.5.7 + cray-mpich/7.4.4 + - - PrgEnv-intel - PrgEnv-cray - cce - cce/8.5.4 - - - PrgEnv-intel - PrgEnv-gnu - gcc - gcc/6.3.0 - - - cray-libsci/16.09.1 - + + PrgEnv-intel + intel + intel/2018.beta + craype + craype/2.5.10 + cray-mpich + cray-mpich/7.5.5 + - - craype - craype/2.5.7 + + PrgEnv-intel + PrgEnv-gnu + gcc + gcc/6.3.0 + cray-libsci/16.09.1 + craype + craype/2.5.7 + cray-mpich/7.4.4 + - craype-haswell - craype-mic-knl + + craype-haswell + craype-mic-knl + - cray-mpich/7.4.4 - + + cray-netcdf-hdf5parallel + cray-hdf5-parallel + cray-parallel-netcdf + cray-hdf5/1.8.16 + cray-netcdf/4.4.0 + + + cray-netcdf-hdf5parallel + cray-netcdf-hdf5parallel/4.4.0 + cray-hdf5-parallel/1.8.16 + cray-parallel-netcdf/1.7.0 + + + git/2.9.1 + cmake/3.3.2 + pmi/5.0.10-1.0000.11069.183.8.ari + zlib/1.2.8 + + - - cray-netcdf-hdf5parallel - cray-hdf5-parallel - cray-parallel-netcdf - cray-hdf5/1.8.16 - cray-netcdf/4.4.0 - - - cray-netcdf-hdf5parallel - cray-netcdf-hdf5parallel/4.4.0 - cray-hdf5-parallel/1.8.16 - cray-parallel-netcdf/1.7.0 - - - git/2.9.1 - cmake/3.3.2 - pmi/5.0.10-1.0000.11069.183.8.ari - zlib/1.2.8 - - + + 1 + 1 + - - 1 - 1 - + 128M + spread + threads - 128M - spread - threads + 1 - yes - + yes + yes + + From 1a1c6ce2f58f869102836e7a6779ce0998ef5f00 Mon Sep 17 00:00:00 2001 From: noel Date: Fri, 11 Aug 2017 08:13:19 -0700 Subject: [PATCH 2/2] After Cori maintenance in August 2017, some modules needed to be updated. Update craype, cray-mpich, pmi for cori-knl and cori-haswell. With cori-knl there are some apparent module bugs that I am working around to allow for the intel18 option. --- config/acme/machines/config_compilers.xml | 2 + config/acme/machines/config_machines.xml | 73 +++++++++++++---------- 2 files changed, 43 insertions(+), 32 deletions(-) diff --git a/config/acme/machines/config_compilers.xml b/config/acme/machines/config_compilers.xml index 62e55049e40..ac07257c90f 100644 --- a/config/acme/machines/config_compilers.xml +++ b/config/acme/machines/config_compilers.xml @@ -635,6 +635,8 @@ for mct, etc. --host=Linux -L$(NETCDF_DIR) -lnetcdff -Wl,--as-needed,-L$(NETCDF_DIR)/lib -lnetcdff -lnetcdf -mkl -lpthread + + ifort icc icpc diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml index 77831c4707c..a2bae974b7e 100644 --- a/config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -121,6 +121,16 @@ esmf + + craype + craype/2.5.12.3 + craype-ivybridge + pmi + pmi/5.0.12 + cray-mpich + cray-mpich/7.6.0 + + PrgEnv-intel intel @@ -140,17 +150,8 @@ PrgEnv-gnu gcc gcc/6.3.0 - cray-libsci/16.09.1 - - - - cray-mpich/7.6.0 - - craype - craype/2.5.12.3 - craype-ivybridge - pmi - pmi/5.0.12 + cray-libsci + cray-libsci/17.06.1 @@ -252,20 +253,25 @@ esmf + + craype + craype/2.5.12 + pmi/5.0.12 + + cray-mpich + cray-mpich/7.6.0 + + PrgEnv-intel intel intel/17.0.2.174 - craype - craype/2.5.7 PrgEnv-intel intel intel/2018.beta - craype - craype/2.5.10 @@ -273,20 +279,14 @@ PrgEnv-gnu gcc gcc/6.3.0 - cray-libsci/16.09.1 - craype - craype/2.5.7 + cray-libsci/17.06.1 - cray-mpich - cray-mpich/7.5.5 - craype-mic-knl craype-haswell - cray-netcdf-hdf5parallel cray-hdf5-parallel @@ -304,7 +304,6 @@ git/2.9.1 cmake/3.3.2 - pmi/5.0.10-1.0000.11069.183.8.ari zlib/1.2.8 @@ -396,18 +395,25 @@ intel intel/17.0.2.174 craype - craype/2.5.7 - cray-mpich/7.4.4 + craype/2.5.12 + cray-mpich + cray-mpich/7.6.0 + PrgEnv-intel intel - intel/2018.beta + intel/17.0.2.174 + craype - craype/2.5.10 + craype/2.5.12 cray-mpich - cray-mpich/7.5.5 + cray-mpich/7.6.0 + + PrgEnv-intel + intel + intel/2018.beta @@ -415,13 +421,17 @@ PrgEnv-gnu gcc gcc/6.3.0 - cray-libsci/16.09.1 + cray-libsci/17.06.1 + craype - craype/2.5.7 - cray-mpich/7.4.4 + craype/2.5.12 + cray-mpich + cray-mpich/7.6.0 + pmi + pmi/5.0.12 craype-haswell craype-mic-knl @@ -442,7 +452,6 @@ git/2.9.1 cmake/3.3.2 - pmi/5.0.10-1.0000.11069.183.8.ari zlib/1.2.8