Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

release/public-v2: remove WW3, update modulefiles and cmake configs, update .gitmodules #205

Merged
merged 25 commits into from
Oct 6, 2020
Merged

release/public-v2: remove WW3, update modulefiles and cmake configs, update .gitmodules #205

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
866577d
Remove WW3, update .gitmodules branches
climbfuji Sep 16, 2020
c9789b7
Update modulefiles for SRW App release
climbfuji Sep 21, 2020
e52bfee
Remove parallel netCDF from cmake config, add odin cmake config
climbfuji Sep 21, 2020
e5d81cf
Updaate regression test baseline directory for public release
climbfuji Sep 21, 2020
ba5bfbd
Update .gitmodules and submodule pointers for NEMS and fv3atm
climbfuji Sep 21, 2020
af04c5a
Updates for odin.intel from Yunheng Wang
climbfuji Sep 22, 2020
71d1034
Turn off inline post on all machines
climbfuji Sep 23, 2020
1877b4c
Replace GSL drag suite with CIRES UGWP/GFS GWD in tests/tests/fv3_ccp…
climbfuji Sep 23, 2020
7b0e0c0
Update submodule pointer for fv3atm / GFDL_atmos_cubed_sphere to poin…
climbfuji Sep 24, 2020
e250021
Turn off the dycore debug print statements in regression tests
climbfuji Sep 24, 2020
9558bd1
Update submodule pointer for fv3atm after dycore merge
climbfuji Sep 24, 2020
cde0c21
Revert change for NEMS in .gitmodules, update submodule pointer for NEMS
climbfuji Sep 25, 2020
7695cae
Bugfix in modulefiles/odin.intel/fv3, add missing 'setenv CMAKE_Platf…
climbfuji Sep 25, 2020
7d51a0f
First take at updating rt.conf for SRW App release
climbfuji Sep 25, 2020
f53d8ae
Update cmake/FindESMF.cmake and cmake/FindNetCDF.cmake from develop
climbfuji Sep 28, 2020
ff95e33
Update submodule pointer for fv3atm
climbfuji Sep 28, 2020
4c7c275
Minor update of modulefiles/odin.intel/fv3
climbfuji Oct 1, 2020
966aa1a
Update regression test baseline tag in tests/rt.sh
climbfuji Oct 1, 2020
7a0f3a9
Update to regression tests for SRW App public release
climbfuji Oct 2, 2020
006c63f
Regression test logs for cheyenne.gnu and cheyenne.intel
climbfuji Oct 2, 2020
1b6ed2d
Update submodule pointer for fv3atm; updates to modulefiles and regre…
climbfuji Oct 2, 2020
9d4edb8
Regression test logs for hera.gnu and hera.intel
climbfuji Oct 2, 2020
39f8fc5
Turn off tests fv3_ccpp_regional_c768 and fv3_ccpp_rrfs_v1beta_debug …
climbfuji Oct 5, 2020
c28ae35
Regression test logs for jet.intel
climbfuji Oct 5, 2020
44dff9a
Regression test logs for gaea.intel
climbfuji Oct 5, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
[submodule "FV3"]
path = FV3
url = https://github.com/NOAA-EMC/fv3atm
branch = develop
#url = https://github.com/NOAA-EMC/fv3atm
#branch = release/public-v2
url = https://github.com/climbfuji/fv3atm
branch = update_gitmodules
[submodule "NEMS"]
path = NEMS
url = https://github.com/NOAA-EMC/NEMS
branch = develop
#url = https://github.com/NOAA-EMC/NEMS
#branch = release/public-v2
url = https://github.com/climbfuji/NEMS
branch = update_gitmodules
[submodule "FMS"]
path = FMS
url = https://github.com/NOAA-GFDL/FMS
branch = master
[submodule "WW3"]
path = WW3
url = https://github.com/NOAA-EMC/WW3
branch = develop
[submodule "stochastic_physics"]
path = stochastic_physics
url = https://github.com/noaa-psd/stochastic_physics
branch = master
branch = release/public-v2
40 changes: 0 additions & 40 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ set(OPENMP ON CACHE BOOL "Enable OpenMP threading")
set(PARALLEL_NETCDF OFF CACHE BOOL "Enable parallel NetCDF")
set(QUAD_PRECISION ON CACHE BOOL "Enable QUAD_PRECISION (for certain grid metric terms in dycore)")
set(REPRO OFF CACHE BOOL "Enable REPRO mode")
set(WW3 OFF CACHE BOOL "Enable WW3")

if(CMAKE_Platform)
message("")
Expand All @@ -52,7 +51,6 @@ message("OPENMP ........... ${OPENMP}")
message("PARALLEL_NETCDF .. ${PARALLEL_NETCDF}")
message("QUAD_PRECISION ... ${QUAD_PRECISION}")
message("REPRO ............ ${REPRO}")
message("WW3 .............. ${WW3}")
message("")

get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME)
Expand Down Expand Up @@ -177,30 +175,6 @@ add_subdirectory(stochastic_physics)
###############################################################################
add_subdirectory(FV3)

###############################################################################
### WW3
###############################################################################
if(WW3)
set(WW3_COMP ${CMAKE_Platform})
if(${CMAKE_Platform} STREQUAL "hera.intel")
set(WW3_COMP "hera")
endif()
if(${CMAKE_Platform} STREQUAL "orion.intel")
set(WW3_COMP "orion")
endif()
message("Build WW3:")
message(" run: ${CMAKE_BUILD_TOOL} WW3_PARCOMPN=4 WW3_COMP=${WW3_COMP} ww3_nemslibonly")
message(" in: ${PROJECT_SOURCE_DIR}/WW3/model/esmf")
message("")

add_custom_target(ww3_nems
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/WW3/model/esmf
COMMAND ${CMAKE_BUILD_TOOL} WW3_PARCOMPN=4 WW3_COMP=${WW3_COMP} ww3_nemslibonly > ${PROJECT_BINARY_DIR}/ww3_make.log 2>&1
)

set(WW3_LIBS ${PROJECT_SOURCE_DIR}/WW3/model/obj/libww3_multi_esmf.a)
endif()

###############################################################################
### NEMS (ufs-weather-model-lib)
###############################################################################
Expand Down Expand Up @@ -230,13 +204,6 @@ target_include_directories(ufs-weather-model-lib PRIVATE ${PROJECT_BINARY_DIR}/N
target_include_directories(ufs-weather-model-lib PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/FV3/mod)
target_include_directories(ufs-weather-model-lib PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/FMS/mod)

if(WW3)
target_compile_definitions(ufs-weather-model-lib PRIVATE -DFRONT_WW3=WMESMFMD)
target_include_directories(ufs-weather-model-lib PRIVATE ${PROJECT_SOURCE_DIR}/WW3/model/mod)
set_target_properties(ufs-weather-model-lib PROPERTIES Fortran_MODULE_DIRECTORY ${PROJECT_SOURCE_DIR}/WW3/model/mod)
add_dependencies(ufs-weather-model-lib ww3_nems)
endif()

set_target_properties(ufs-weather-model-lib PROPERTIES Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/mod)
target_include_directories(ufs-weather-model-lib PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/FV3/mod>
Expand Down Expand Up @@ -268,15 +235,8 @@ target_include_directories(NEMS.exe PRIVATE ${PROJECT_BINARY_DIR}/NEMS/src)

set_target_properties(NEMS.exe PROPERTIES Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/mod)

if(WW3)
target_compile_definitions(NEMS.exe PRIVATE -DFRONT_WW3=WMESMFMD)
set_target_properties(NEMS.exe PROPERTIES Fortran_MODULE_DIRECTORY ${PROJECT_SOURCE_DIR}/WW3/model/mod)
add_dependencies(NEMS.exe ww3_nems)
endif()

target_link_libraries(NEMS.exe
ufs-weather-model-lib
${WW3_LIBS}
esmf NetCDF::NetCDF_Fortran ${MKL_LIB})

###############################################################################
Expand Down
2 changes: 1 addition & 1 deletion FV3
Open in desktop
Submodule FV3 updated 5 files
+10 −6 .gitmodules
+1 −1 atmos_cubed_sphere
+1 −1 ccpp/framework
+1 −1 ccpp/physics
+0 −1 gfsphysics/CMakeLists.txt
2 changes: 1 addition & 1 deletion NEMS
Open in desktop
Submodule NEMS updated 1 files
+1 −1 .gitmodules
1 change: 0 additions & 1 deletion WW3
Open in desktop
Submodule WW3 deleted from 19f312
1 change: 0 additions & 1 deletion cmake/configure_hera.intel.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
set(INLINE_POST ON CACHE BOOL "Enable inline post" FORCE)
set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE)
2 changes: 2 additions & 0 deletions cmake/configure_odin.intel.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
set(INLINE_POST ON CACHE BOOL "Enable inline post" FORCE)
set(AVX2 OFF CACHE BOOL "Enable AVX2 instruction set" FORCE)
1 change: 0 additions & 1 deletion cmake/configure_orion.intel.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
set(INLINE_POST ON CACHE BOOL "Enable inline post" FORCE)
climbfuji marked this conversation as resolved.
Show resolved Hide resolved
set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE)
1 change: 0 additions & 1 deletion cmake/configure_wcoss_dell_p3.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
set(INLINE_POST ON CACHE BOOL "Enable inline post" FORCE)
set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE)
101 changes: 65 additions & 36 deletions conf/configure.fv3.odin → conf/configure.fv3.odin.intel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,10 @@ DEBUG =
REPRO =
VERBOSE =
OPENMP = Y
AVX2 = Y
AVX2 = N
HYDRO = N
QUAD_PRECISION = Y
MULTI_GASES = N
CCPP = N
STATIC = N

include $(ESMFMKFILE)
ESMF_INC = $(ESMF_F90COMPILEPATHS)
Expand All @@ -68,50 +68,64 @@ ifneq ($(need),$(ok))
$(error Need at least make version $(need). Load module gmake/3.81)
endif

NETCDF_ROOT = $(NETCDF4)
NETCDF_ROOT = $(NETCDF_DIR)
INCLUDE = -I$(NETCDF_ROOT)/include
NETCDF_INC = -I$(NETCDF_ROOT)/include
ifneq ($(findstring netcdf/4,$(LOADEDMODULES)),)
NETCDF_LIB += -L$(NETCDF4)/lib -lnetcdff -lnetcdf
NETCDF_LIB += -L$(NETCDF_DIR)/lib -lnetcdff -lnetcdf
else
NETCDF_LIB += -L$(NETCDF4)/lib -lnetcdff -lnetcdf
NETCDF_LIB += -L$(NETCDF_DIR)/lib -lnetcdff -lnetcdf
endif

FPPFLAGS := -fpp -Wp,-w $(INCLUDE)
CFLAGS := $(INCLUDE)

FFLAGS := $(INCLUDE) -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -nowarn -sox -align array64byte
FFLAGS := $(INCLUDE) -fno-alias -auto -safe-cray-ptr -save-temps -ftz -assume byterecl -nowarn -sox -align array64byte

CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -DUSE_GFSL63 -DGFS_PHYS -Duse_WRTCOMP
CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML -DNO_INLINE_POST

ifeq ($(HYDRO),Y)
CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -DUSE_GFSL63 -DGFS_PHYS -Duse_WRTCOMP -DNO_INLINE_POST
CPPDEFS +=
else
CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -DUSE_GFSL63 -DGFS_PHYS -DMOIST_CAPPA -DUSE_COND -Duse_WRTCOMP -DNO_INLINE_POST
CPPDEFS += -DMOIST_CAPPA -DUSE_COND
endif

CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML
ifeq ($(NAM_phys),Y)
CPPDEFS += -DNAM_phys
endif

ifeq ($(32BIT),Y)
CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8
FFLAGS += -i4 -real-size 32
else
ifeq ($(REPRO),Y)
FFLAGS += -i4 -real-size 64
else
FFLAGS += -i4 -real-size 64 -no-prec-div -no-prec-sqrt
endif
endif

FFLAGS += -xCORE-AVX-I #-axavx
CFLAGS += -xCORE-AVX-I #-axavx
ifeq ($(REPRO),Y)
FFLAGS += -qno-opt-dynamic-align
CFLAGS += -qno-opt-dynamic-align
else
ifeq ($(AVX2),Y)
FFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align
CFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align
else
FFLAGS += -qno-opt-dynamic-align
CFLAGS += -qno-opt-dynamic-align
endif
endif

ifeq ($(MULTI_GASES),Y)
CPPDEFS += -DMULTI_GASES
endif

ifeq ($(QUAD_PRECISION),Y)
CPPDEFS += -DENABLE_QUAD_PRECISION
endif

FFLAGS_OPT = -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3
FFLAGS_REPRO = -O2 -debug minimal -fp-model consistent -qoverride-limits -g -traceback
#FFLAGS_DEBUG = -g -O0 -check bounds -check -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv
FFLAGS_DEBUG = -g -O0 -check bounds -traceback
FFLAGS_REPRO = -O2 -debug minimal -fp-model precise -qoverride-limits -g -traceback
FFLAGS_DEBUG = -g -O0 -check all -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv

TRANSCENDENTALS := -fast-transcendentals
FFLAGS_OPENMP = -qopenmp
Expand All @@ -136,36 +150,51 @@ LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M
# start with blank LIBS
LIBS :=

#ifneq ($(REPRO),)
#CFLAGS += $(CFLAGS_REPRO)
#FFLAGS += $(FFLAGS_REPRO)
#FAST :=
#else ifneq ($(DEBUG),)
ifeq ($(REPRO),Y)
CPPDEFS += -DREPRO
CFLAGS += $(CFLAGS_REPRO)
FFLAGS += $(FFLAGS_REPRO)
FAST :=
else ifeq ($(DEBUG),Y)
CPPDEFS += -DDEBUG
CFLAGS += $(CFLAGS_DEBUG)
FFLAGS += $(FFLAGS_DEBUG)
FAST :=
#else ifneq ($(TEST),)
#CFLAGS += $(CFLAGS_TEST)
#FFLAGS += $(FFLAGS_TEST)
#FAST :=
#else
#CFLAGS += $(CFLAGS_OPT)
#FFLAGS += $(FFLAGS_OPT)
#FAST := $(TRANSCENDENTALS)
#endif

ifneq ($(OPENMP),)
else ifeq ($(TEST),Y)
CFLAGS += $(CFLAGS_TEST)
FFLAGS += $(FFLAGS_TEST)
FAST :=
else
CFLAGS += $(CFLAGS_OPT)
FFLAGS += $(FFLAGS_OPT)
FAST := $(TRANSCENDENTALS)
endif

ifeq ($(OPENMP),Y)
CPPDEFS += -DOPENMP
CFLAGS += $(CFLAGS_OPENMP)
FFLAGS += $(FFLAGS_OPENMP)
LDFLAGS += $(LDFLAGS_OPENMP)
endif

ifneq ($(VERBOSE),)
ifeq ($(VERBOSE),Y)
CFLAGS += $(CFLAGS_VERBOSE)
FFLAGS += $(FFLAGS_VERBOSE)
LDFLAGS += $(LDFLAGS_VERBOSE)
endif

ifeq ($(CCPP),Y)
CPPDEFS += -DCCPP
CFLAGS += -I$(PATH_CCPP)/include
FFLAGS += -I$(PATH_CCPP)/include
ifeq ($(STATIC),Y)
CPPDEFS += -DSTATIC
LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -L/oldscratch/ywang/external/libxml2_static/lib -lxml2
else
LDFLAGS += -L$(PATH_CCPP)/lib -lccpp
endif
endif

LDFLAGS += $(LIBS)

ifdef InNemsMakefile
Expand Down
33 changes: 14 additions & 19 deletions modulefiles/cheyenne.gnu/fv3
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,30 +23,25 @@ module load ncarcompilers/0.5.0
##
## use pre-compiled NetCDF, ESMF and NCEP libraries for above compiler / MPI combination
##
module use -a /glade/p/ral/jntp/GMTB/tools/ufs-stack-20200728/gnu-9.1.0/mpt-2.19/modules
module use /glade/p/ral/jntp/GMTB/tools/NCEPLIBS-ufs-v2.0.0/gnu-9.1.0/mpt-2.19/modules
module load libpng/1.6.35
module load netcdf/4.7.4
module load esmf/8.0.0

module load bacio/2.4.0
module load bacio/2.4.1
module load crtm/2.3.0
module load g2/3.4.0
module load g2tmpl/1.9.0
module load ip/3.3.0
module load g2/3.4.1
module load g2tmpl/1.9.1
module load ip/3.3.3
module load nceppost/dceca26
module load nemsio/2.5.1
module load sp/2.3.0
module load w3emc/2.7.0
module load w3nco/2.4.0

module load gfsio/1.4.0
module load sfcio/1.4.0
module load sigio/2.3.0

##
## SIONlib library
##
module use -a /glade/p/ral/jntp/GMTB/tools/modulefiles/gnu-9.1.0/mpt-2.19
module load SIONlib/1.7.4
module load nemsio/2.5.2
module load sp/2.3.3
module load w3emc/2.7.3
module load w3nco/2.4.1

module load gfsio/1.4.1
module load sfcio/1.4.1
module load sigio/2.3.2

##
## load cmake
Expand Down
33 changes: 14 additions & 19 deletions modulefiles/cheyenne.intel/fv3
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,30 +23,25 @@ module load ncarcompilers/0.5.0
##
## use pre-compiled NetCDF, ESMF and NCEP libraries for above compiler / MPI combination
##
module use -a /glade/p/ral/jntp/GMTB/tools/ufs-stack-20200728/intel-19.1.1/mpt-2.19/modules
module use /glade/p/ral/jntp/GMTB/tools/NCEPLIBS-ufs-v2.0.0/intel-19.1.1/mpt-2.19/modules
module load libpng/1.6.35
module load netcdf/4.7.4
module load esmf/8.0.0

module load bacio/2.4.0
module load bacio/2.4.1
module load crtm/2.3.0
module load g2/3.4.0
module load g2tmpl/1.9.0
module load ip/3.3.0
module load g2/3.4.1
module load g2tmpl/1.9.1
module load ip/3.3.3
module load nceppost/dceca26
module load nemsio/2.5.1
module load sp/2.3.0
module load w3emc/2.7.0
module load w3nco/2.4.0

module load gfsio/1.4.0
module load sfcio/1.4.0
module load sigio/2.3.0

##
## SIONlib library
##
module use -a /glade/p/ral/jntp/GMTB/tools/modulefiles/intel-19.1.1/mpt-2.19
module load SIONlib/1.7.4
module load nemsio/2.5.2
module load sp/2.3.3
module load w3emc/2.7.3
module load w3nco/2.4.1

module load gfsio/1.4.1
module load sfcio/1.4.1
module load sigio/2.3.2

##
## load cmake
Expand Down
Loading