Skip to content

Commit

Permalink
Lilac protype working with a driver and CTSM. (ESCOMP#12)
Browse files Browse the repository at this point in the history
Lilac protype working with a driver  and CTSM.
  • Loading branch information
billsacks authored Nov 26, 2019
2 parents d4b6dcc + dc10dd2 commit 052758a
Show file tree
Hide file tree
Showing 24 changed files with 3,121 additions and 946 deletions.
1 change: 1 addition & 0 deletions lilac/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,4 @@ components/
*.pyc

build/
_build/
193 changes: 185 additions & 8 deletions lilac/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,15 +1,172 @@
cmake_minimum_required(VERSION 2.8.12.1)

project(LILAC Fortran)
enable_language(Fortran)
##include("/glade/work/negins/UFSCOMP/cime/tools/Macros.cmake")

set (CIME_ROOT "/glade/work/negins/UFSCOMP/cime")
message ("CIME_ROOT: ${CIME_ROOT}")

set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/CMakeModules")
message ("CMAKE_MODULE_PATH: ${CMAKE_MODULE_PATH}")


set (CIME_CMAKE_MODULE_DIRECTORY "/glade/work/negins/UFSCOMP/cime/src/CMake/")
message ("CIME_CMAKE_MODULE_DIRECTORY: ${CIME_CMAKE_MODULE_DIRECTORY}")


list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY})
message ("CMAKE_MODULE_PATH: ${CMAKE_MODULE_PATH}")

set (MACRO_ROOT "/glade/work/negins/UFSCOMP/cime/tools/")
include(${MACRO_ROOT}/Macros.cmake)

list(APPEND CMAKE_MODULE_PATH ${MACRO_ROOT})
message ("CMAKE_MODULE_PATH: ${CMAKE_MODULE_PATH}")


set (CLM_ROOT "/glade/work/negins/UFSCOMP/components/clm")

message("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")

include_directories (${CMAKE_SOURCE_DIR}/cmake/CMakeModules/)
include (${CMAKE_SOURCE_DIR}/cmake/CMakeModules/genf90_utils.cmake)
include (${CMAKE_SOURCE_DIR}/cmake/CMakeModules/Sourcelist_utils.cmake)
include (${CMAKE_SOURCE_DIR}/cmake/CMakeModules/pFUnit_utils.cmake)
include (${CMAKE_SOURCE_DIR}/cmake/CMakeModules/FindpFUnit.cmake)


#include (Macros.cmake)
#include(CIME_initial_setup)

message("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")

### -------------------------------------------------------------

# project name
project(LILAC Fortran C)
enable_language(Fortran)


# This definition is needed to avoid having ESMF depend on mpi
add_definitions(-DHIDE_MPI)


message("----------------------------------------------------")
message ("CMAKE_CURRENT_SOURCE_DIR: ${CMAKE_CURRENT_SOURCE_DIR}")
message ("CMAKE_MODULE_PATH: ${CMAKE_MODULE_PATH}")
message("----------------------------------------------------")



message("----------------------------------------------------")
# Add source directories from other share code (csm_share, etc.). This should be
# done first, so that in case of name collisions, the CLM versions take
# precedence (when there are two files with the same name, the one added later
# wins).
add_subdirectory(${CIME_ROOT}/src/share/util csm_share)
add_subdirectory(${CIME_ROOT}/src/share/unit_test_stubs/util csm_share_stubs)
add_subdirectory(${CIME_ROOT}/src/share/esmf_wrf_timemgr esmf_wrf_timemgr)
add_subdirectory(${CIME_ROOT}/src/drivers/mct/shr drv_share)
message("----------------------------------------------------")

# Extract just the files we need from drv_share
set (drv_sources_needed_base
glc_elevclass_mod.F90
)
extract_sources("${drv_sources_needed_base}" "${drv_sources}" drv_sources_needed)

message("~~~~~~~~~~~~~~~~~~~~~~CLM_ROOT~~~~~~~~~~~~~~~~~~~~~~")
# Add CLM source directories (these add their own test directories)
add_subdirectory(${CLM_ROOT}/src/utils clm_utils)
add_subdirectory(${CLM_ROOT}/src/biogeochem clm_biogeochem)
add_subdirectory(${CLM_ROOT}/src/soilbiogeochem clm_soilbiogeochem)
add_subdirectory(${CLM_ROOT}/src/biogeophys clm_biogeophys)
add_subdirectory(${CLM_ROOT}/src/dyn_subgrid clm_dyn_subgrid)
add_subdirectory(${CLM_ROOT}/src/main clm_main)
add_subdirectory(${CLM_ROOT}/src/init_interp clm_init_interp)
add_subdirectory(${CLM_ROOT}/src/fates/main fates_main)

# Add general unit test directories (stubbed out files, etc.)
add_subdirectory(unit_test_stubs)
add_subdirectory(unit_test_shr)


# Remove shr_mpi_mod from share_sources.
# This is needed because we want to use the mock shr_mpi_mod in place of the real one
#
# TODO: this should be moved into a general-purpose function in Sourcelist_utils.
# Then this block of code could be replaced with a single call, like:
# remove_source_file(${share_sources} "shr_mpi_mod.F90")}

foreach (sourcefile ${share_sources})
string(REGEX MATCH "shr_mpi_mod.F90" match_found ${sourcefile})
if(match_found)
list(REMOVE_ITEM share_sources ${sourcefile})
endif()
endforeach()


# We rely on pio for cmake utilities like findnetcdf.cmake, so that we don't
# need to duplicate this cmake code
message ("CMAKE_MODULE_PATH: ${CMAKE_MODULE_PATH}")
list (APPEND CMAKE_MODULE_PATH "${CIME_ROOT}/src/externals/pio2/cmake")
message ("CMAKE_MODULE_PATH: ${CMAKE_MODULE_PATH}")


add_subdirectory (${CIME_ROOT}/src/externals/pio2/test)

message("----------------------------------------------------")
option(ENABLE_PFUNIT "Enable pfUnit testing Framework" ON)
if (ENABLE_PFUNIT)
find_package(pfUnit)
include(pfUnit_utils)
include_directories("${PFUNIT_INCLUDE_DIRS}")
endif (ENABLE_PFUNIT)
message("----------------------------------------------------")


find_package(MPI REQUIRED)
# TODO: This should be found from the find_package call but its not working
set(CMAKE_Fortran_COMPILER "/usr/lib64/mpich/bin/mpif90")
#set(CMAKE_Fortran_COMPILER "/usr/lib64/mpich/bin/mpif90")
find_package(ESMF REQUIRED)


message("------------include (CIME_utils)--------------------")
include(CIME_utils)
message("----------------------------------------------------")

find_package(NetCDF COMPONENTS C Fortran)
include_directories(${NetCDF_C_INCLUDE_DIRS} ${NetCDF_Fortran_INCLUDE_DIRS})
message("NetCDF_C_INCLUDE_DIRS: ${NetCDF_C_INCLUDE_DIRS}")
message("----------------------------------------------------")

##=======##
#set(CESM_ROOT "/glade/work/negins/UFSCOMP/")
#set(CSM_SHR "/glade/work/negins/UFSCOMP/components/clm/src/unit_test_stubs/csm_share/")

#add_subdirectory(${CESM_ROOT}/models/csm_share/shr csm_share)
#add_subdirectory(${CSM_SHR} )

message("----------------------------------------------------")



# -lclm libclm.a
SET(NAMES libclm.a)

#find_library(LIB_TO_INCLUDE
# libclm.a
# PATHS /glade/scratch/negins/baghale6/bld/intel/mpt/nodebug/nothreads/nuopc/nuopc/esmf/lib/)
#find_library(LIB_TO_INCLUDE /glade/scratch/negins/baghale6/bld/intel/mpt/nodebug/nothreads/nuopc/nuopc/esmf/lib/)

#message(STATUS "include_directories for ${NAMES}: ${LIB_TO_INCLUDE}")
#include_directories(${LIB_TO_INCLUDE})
#link_directories(${LIB_TO_INCLUDE})
#message(STATUS "include_directories for ${NAMES}: ${LIB_TO_INCLUDE}")
#find_library(LIB_TO_INCLUDE /glade/scratch/negins/baghale6/bld/intel/mpt/nodebug/nothreads/nuopc/nuopc/esmf/lib/)
#message(STATUS "include_directories: ${LIB_TO_INCLUDE}")
#target_link_libraries (${LIB_TO_INCLUDE})


# Local CMake modules

if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
Expand All @@ -30,9 +187,29 @@ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${dialect}")

set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${ESMF_COMPILER_LINE}")
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${ESMF_LINK_LINE} -g -cpp")
# TODO: This should not be necessary but certain header files are missing from the build
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -I /usr/include/ -I/usr/src/lilac/external/esmf/src/Infrastructure/Util/include -I/usr/src/lilac/external/esmf/build_config/Linux.gfortran.default -I /usr/src/lilac/external/esmf/src/include")
message("CMAKE_Fortran_FLAGS:" ${CMAKE_Fortran_FLAGS})

add_subdirectory(lilac)
add_subdirectory(tests)

message(STATUS "==============================================================")
message(STATUS "Fortran Compiler : ${CMAKE_Fortran_COMPILER}")
message(STATUS "cmake Fortran Flags : ${CMAKE_Fortran_FLAGS}")
message(STATUS "==============================================================")
message(STATUS "==============================================================")


#add_executable("lilac.exe" ../lilac/*.F90)

#
# Compile.
#

file(GLOB_RECURSE SOURCES lilac/*.F90)
#add_subdirectory(lilac)
#add_executable(${PROJECT_NAME}.exe ../lilac/demo_driver.F90
# ../lilac/lilac_mod.F90 ../lilac/atmos_cap.F90 ../lilac/lilac_utils.F90
# ../lilac/lnd_cap.F90 ../lilac/cpl_mod.F90)

add_executable (${PROJECT_NAME}.exe ${SOURCES})
target_link_libraries(${PROJECT_NAME}.exe ${LIB_TO_INCLUDE})

#add_subdirectory(lilac)
#add_subdirectory(tests)
12 changes: 4 additions & 8 deletions lilac/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,30 +1,26 @@
FROM centos:latest
FROM jhamman/esmf:latest
LABEL description="LILAC development environment"

RUN yum install -y curl
RUN yum upgrade -y
RUN yum update -y
RUN yum clean all
RUN yum -y install wget bzip2 gcc gcc-c++ gcc-gfortran mpich-devel make git
ENV PATH="/usr/lib64/mpich/bin:${PATH}"
RUN yum -y install wget bzip2

WORKDIR /usr/src/lilac/

RUN mkdir -p external
RUN mkdir -p ci

COPY external/esmf external/esmf
COPY external/pfunit external/pfunit
COPY ci/* ci/

# Install some remaining dependencies
ENV PATH /usr/local/miniconda/bin:$PATH
RUN ./ci/install_python.sh

# Install ESMF
RUN ./ci/install_esmf.sh
ENV ESMF_CONFIG_FILE /usr/local/lib/esmf.mk

# Install PFUNIT
# RUN ./ci/install_pfunit.sh
# ENV PFUNIT_INSTALL /usr/pfunit
RUN ./ci/install_pfunit.sh
ENV PFUNIT_INSTALL /usr/pfunit
6 changes: 5 additions & 1 deletion lilac/ci/build_and_test_lilac.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,8 @@ make VERBOSE=1 # -j 4
echo "done building lilac, time to run the tests..."

# run test suite
ctest
ctest

# run system tests
# TODO: these should probably be run via ctest
/lilac/build/tests/rand_atm_rand_lnd/rand_atm_rand_lnd
1 change: 0 additions & 1 deletion lilac/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,3 @@ services:
volumes:
- .:/lilac
command: /lilac/ci/build_and_test_lilac.sh

5 changes: 5 additions & 0 deletions lilac/lilac/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
*.o
job_name*
PET*
*.exe
batch.sub
4 changes: 0 additions & 4 deletions lilac/lilac/CMakeLists.txt

This file was deleted.

99 changes: 99 additions & 0 deletions lilac/lilac/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@

#================================================================================
# Makefile to compile the lilac program
#================================================================================
## This is temporary Makefile for building lilac against CTSM pre-compiled library



#================================================================================
### Finding and including esmf.mk
#================================================================================

# Note: This fully portable Makefile template depends on finding environment
# # variable "ESMFMKFILE" set to point to the appropriate "esmf.mk" file,
# # as is discussed in the User's Guide.
# # However, you can still use this Makefile template even if the person
# # that installed ESMF on your system did not provide for a mechanism to
# # automatically set the environment variable "ESMFMKFILE". In this case
# # either manually set "ESMFMKFILE" in your environment or hard code the
# # location of "esmf.mk" into the include statement below.
# # Notice that the latter approach has negative impact on flexibility and
# # portability.


ifneq ($(origin ESMFMKFILE), environment)
$(error Environment variable ESMFMKFILE was not set.)
endif

include $(ESMFMKFILE)

#================================================================================
### Define directory paths
#================================================================================
# Temporarily hard-coded
# TODO: Please fix this part.
CASE_NAME = why01-g
#CASE_NAME = ctsm1.0.dev066_MCT_I2000Clm50SpRsGs_testing
#CASE_NAME = ctsm1.0.dev066_MCT_I2000Clm50Sp_03
#CASE_NAME = lilac_ctsm
CTSM_BLD_DIR = /glade/scratch/negins/$(CASE_NAME)/bld/intel/mpt/debug/nothreads/mct/mct/esmf
CTSM_INC = -I$(CTSM_BLD_DIR)/include
CTSM_LIB = -L$(CTSM_BLD_DIR)/lib -lclm
#TRACEBACK_FLAGS = -g -traceback -debug all -check all -O2 -r8
#TRACEBACK_FLAGS = -qno-opt-dynamic-align -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source -qopt-report -xCORE_AVX2 -no-fma -O2 -debug minimal -DLINUX -DCESMCOUPLED -DFORTRANUNDERSCORE -DCPRINTEL -DNDEBUG -DUSE_ESMF_LIB -DMCT_INTERFACE -DHAVE_MPI -DPIO1 -DHAVE_SLASHPROC -D_PNETCDF -DESMF_VERSION_MAJOR=8 -DESMF_VERSION_MINOR=0 -DATM_PRESENT -DICE_PRESENT -DLND_PRESENT -DOCN_PRESENT -DROF_PRESENT -DGLC_PRESENT -DWAV_PRESENT -DESP_PRESENT -free -DUSE_CONTIGUOUS=contiguous
#TRACEBACK_FLAGS = -qno-opt-dynamic-align -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source -qopt-report -xCORE_AVX2 -no-fma -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -DLINUX -DCESMCOUPLED -DFORTRANUNDERSCORE -DCPRINTEL -DDEBUG -DUSE_ESMF_LIB -DPIO1 -DHAVE_SLASHPROC -D_PNETCDF -DESMF_VERSION_MAJOR=7 -DESMF_VERSION_MINOR=1 -DATM_PRESENT -DICE_PRESENT -DLND_PRESENT -DOCN_PRESENT -DROF_PRESENT -DGLC_PRESENT -DWAV_PRESENT -DESP_PRESENT -free -DUSE_CONTIGUOUS=contiguous
TRACEBACK_FLAGS = -qno-opt-dynamic-align -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source -qopt-report -xCORE_AVX2 -no-fma -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -DLINUX -DCESMCOUPLED -DFORTRANUNDERSCORE -DCPRINTEL -DDEBUG -DUSE_ESMF_LIB -DMCT_INTERFACE -DHAVE_MPI -DPIO1 -DHAVE_SLASHPROC -D_PNETCDF -DESMF_VERSION_MAJOR=7 -DESMF_VERSION_MINOR=1 -DATM_PRESENT -DICE_PRESENT -DLND_PRESENT -DOCN_PRESENT -DROF_PRESENT -DGLC_PRESENT -DWAV_PRESENT -DESP_PRESENT -free -DUSE_CONTIGUOUS=contiguous

# -----------------------------------------------------------------------------
#EXTRA_LIBS = $(EXTRA_LIBS) -I/glade/scratch/negins/$(CASE_NAME)/bld/intel/mpt/debug/nothreads/nuopc/pio/pio2
EXTRA_LIBS = -L/glade/scratch/negins/$(CASE_NAME)/bld/intel/mpt/debug/nothreads/mct/mct/esmf/c1a1l1i1o1r1g1w1i1e1/lib -lcsm_share -L/glade/scratch/negins/$(CASE_NAME)/bld/intel/mpt/debug/nothreads/mct/lib -lpio -lgptl -lmct -lmpeu -mkl=cluster -L/glade/u/apps/ch/opt/pnetcdf/1.11.0/mpt/2.19/intel/19.0.2//lib -lpnetcdf -L/glade/u/home/dunlap/ESMF-INSTALL/intel19/8.0.0bs32/lib/libO/Linux.intel.64.mpt.default -L/glade/u/apps/ch/opt/netcdf-mpi/4.6.1/mpt/2.19/intel/19.0.2/lib -L/glade/u/home/dunlap/YAML-INSTALL/lib -Wl,-rpath,/glade/u/home/dunlap/ESMF-INSTALL/intel19/8.0.0bs32/lib/libO/Linux.intel.64.mpt.default -Wl,-rpath,/glade/u/apps/ch/opt/netcdf-mpi/4.6.1/mpt/2.19/intel/19.0.2/lib -Wl,-rpath,/glade/u/home/dunlap/YAML-INSTALL/lib -lesmf -cxxlib -lrt -ldl -lnetcdff -lnetcdf -lyaml-cpp -cxxlib
MORE_LIBS = -I/glade/scratch/negins/$(CASE_NAME)/bld/intel/mpt/debug/mct/mct/esmf/c1a1l1i1o1r1g1w1i1e1/csm_share/ -L/glade/scratch/negins/$(CASE_NAME)/bld/intel/mpt/debug/mct/mct/esmf/c1a1l1i1o1r1g1w1i1e1/csm_share/ -I/glade/scratch/negins/$(CASE_NAME)/bld/intel/mpt/debug/nothreads/mct/mct/esmf/clm/obj/ -I//glade/scratch/negins/$(CASE_NAME)/bld/intel/mpt/debug/nothreads/mct/mct/esmf/c1a1l1i1o1r1g1w1i1e1/csm_share/
# -----------------------------------------------------------------------------


#================================================================================
### Compiler and linker rules using ESMF_ variables supplied by esmf.mk
#================================================================================

.SUFFIXES: .f90 .F90 .c .C

%.o : %.f90
$(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) \
$(ESMF_F90COMPILEFREENOCPP) $<

%.o : %.F90
$(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) \
$(ESMF_F90COMPILEFREECPP) $(ESMF_F90COMPILECPPFLAGS) \
$(CTSM_INC) $(CTSM_LIB) $(TRACEBACK_FLAGS) \
$(EXTRA_LIBS) $(MORE_LIBS) $<

%.o : %.c
$(ESMF_CXXCOMPILER) -c $(ESMF_CXXCOMPILEOPTS) $(ESMF_CXXCOMPILEPATHSLOCAL) \
$(ESMF_CXXCOMPILEPATHS) $(ESMF_CXXCOMPILECPPFLAGS) $<

% : %.C
$(ESMF_CXXCOMPILER) -c $(ESMF_CXXCOMPILEOPTS) $(ESMF_CXXCOMPILEPATHSLOCAL) \
$(ESMF_CXXCOMPILEPATHS) $(ESMF_CXXCOMPILECPPFLAGS) $<

demo_driver: demo_driver.o atmos_cap.o lilac_mod.o lilac_utils.o cpl_mod.o lnd_cap.o
$(ESMF_F90LINKER) $(ESMF_F90LINKOPTS) $(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) -o $@ $^ $(ESMF_F90ESMFLINKLIBS) $(CTSM_INC) $(CTSM_LIB) $(EXTRA_LIBS) $(TRACEBACK_FLAGS) $(MORE_LIBS)
mv demo_driver demo_driver.exe
rm *.o *.mod

# module dependencies:
#demo_driver.o: lilac_mod.o atmos_cap.o lilac_utils.o cpl_mod.o shr_string_mod.o shr_kind_mod.o shr_sys_mod.o shr_log_mod.o shr_timer_mod.o
demo_driver.o: lilac_mod.o atmos_cap.o lilac_utils.o cpl_mod.o demo_utils.o demo_mod.o
lilac_mod.o: atmos_cap.o lilac_utils.o cpl_mod.o lnd_cap.o #shr_pio_mod.o
atmos_cap.o: lilac_utils.o
demo_mod.o:
# -----------------------------------------------------------------------------

.PHONY: clean berzerk remake
clean:
rm -f *.exe *.o
berzerk:
rm -f PET*.ESMF_LogFile job_name* *.o *.mod *.exe
remake:
rm lilac_mod.o demo_driver.o demo_driver.exe & make
# -----------------------------------------------------------------------------
Loading

0 comments on commit 052758a

Please sign in to comment.