Skip to content

Commit

Permalink
Merge branch 'develop' into feature/wjiang/GEOS_GigaTraj
Browse files Browse the repository at this point in the history
  • Loading branch information
@weiyuan-jiang
weiyuan-jiang committed Dec 6, 2024
2 parents f01034d + 6901c19 commit 4d453ca
Show file tree
Hide file tree
Showing 7 changed files with 164 additions and 199 deletions.
6 changes: 2 additions & 4 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
ecbuild_add_executable (
TARGET GEOSgcm.x
SOURCES GEOSgcm.F90
LIBS GEOSgcm_GridComp esmf OpenMP::OpenMP_Fortran
LINKER_LANGUAGE Fortran
LIBS GEOSgcm_GridComp ESMF::ESMF OpenMP::OpenMP_Fortran
)

ecbuild_add_executable (
TARGET idfupd.x
SOURCES idfupd.F90
LIBS GEOSgcm_GridComp esmf
LINKER_LANGUAGE Fortran
LIBS GEOSgcm_GridComp ESMF::ESMF
)

set (CMAKE_Fortran_FLAGS_RELEASE "${GEOS_Fortran_FLAGS_VECT}")
Expand Down
110 changes: 72 additions & 38 deletions HISTORY.rc.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,9 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
# 'inst3_2d_met_Nx+-'
# IAU increment for forecast
# 'tavg2_3d_aiau_Np+-'
# Constant file
# Constant files
# 'const_2d_asm_Nx'
# 'const_2d_lnd_Nx'
::

inst3_3d_asm_Np-.format: 'CFIO' ,
Expand Down Expand Up @@ -844,6 +845,7 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
'THAT' , 'SURFACE' , 'TSH' ,
'QHAT' , 'SURFACE' , 'QSH' ,
'FRACI' , 'SURFACE' , 'FRSEAICE' ,
'RUNOFF' , 'SURFACE' , 'RUNOFFTOT' ,
'AN_PRCP' , 'MOIST' , 'PRECANV' ,
'CN_PRCP' , 'MOIST' , 'PRECCON' ,
'LS_PRCP' , 'MOIST' , 'PRECLSC' ,
Expand Down Expand Up @@ -931,11 +933,12 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
'WCPR' , 'SURFACE' , 'PRMC' ,
'WCRZ' , 'SURFACE' , 'RZMC' ,
'WCSF' , 'SURFACE' , 'SFMC' ,
'TPSNOW' , 'SURFACE' ,
'TPUNST' , 'SURFACE' , 'TUNST' ,
'TPSAT' , 'SURFACE' , 'TSAT' ,
'TPWLT' , 'SURFACE' , 'TWLT' ,
'TPSURF' , 'SURFACE' , 'TSURF' ,
'CAPAC' , 'SURFACE' , 'INTRWATR' ,
'TPSNOW' , 'SURFACE' , 'TPSNOWLAND',
'TPUNST' , 'SURFACE' , 'TUNSTLAND' ,
'TPSAT' , 'SURFACE' , 'TSATLAND' ,
'TPWLT' , 'SURFACE' , 'TWLTLAND' ,
'TPSURF' , 'SURFACE' , 'TSURFLAND' ,
'TSOIL1' , 'SURFACE' ,
'TSOIL2' , 'SURFACE' ,
'TSOIL3' , 'SURFACE' ,
Expand All @@ -944,20 +947,20 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
'TSOIL6' , 'SURFACE' ,
'SNOLAND' , 'SURFACE' , 'PRECSNOLAND' ,
'PRLAND' , 'SURFACE' , 'PRECTOTLAND' ,
'TSLAND' , 'SURFACE' , 'SNOMAS' ,
'SNOWDP' , 'SURFACE' , 'SNODP' ,
'TSLAND' , 'SURFACE' , 'SNOMASLAND' ,
'SNOWDP' , 'SURFACE' , 'SNODPLAND' ,
'EVPSOI' , 'SURFACE' , 'LHLANDSOIL',
'EVPVEG' , 'SURFACE' , 'LHLANDTRNS',
'EVPINT' , 'SURFACE' , 'LHLANDINTR',
'EVPICE' , 'SURFACE' , 'LHLANDSBLN',
'RUNSURF' , 'SURFACE' , 'RUNOFF' ,
'BASEFLOW' , 'SURFACE' ,
'RUNSURF' , 'SURFACE' , 'RUNSURFLAND' ,
'BASEFLOW' , 'SURFACE' , 'BASEFLOWLAND' ,
'SMLAND' , 'SURFACE' ,
'QINFIL' , 'SURFACE' ,
'FRUST' , 'SURFACE' , 'FRUNST' ,
'FRSAT' , 'SURFACE' ,
'ASNOW' , 'SURFACE' , 'FRSNO' ,
'FRWLT' , 'SURFACE' ,
'QINFIL' , 'SURFACE' , 'QINFILLAND',
'FRUST' , 'SURFACE' , 'FRLANDUNST',
'FRSAT' , 'SURFACE' , 'FRLANDSAT' ,
'ASNOW' , 'SURFACE' , 'FRLANDSNO' ,
'FRWLT' , 'SURFACE' , 'FRLANDWLT' ,
'DFPARLAND' , 'SURFACE' , 'PARDFLAND' ,
'DRPARLAND' , 'SURFACE' , 'PARDRLAND' ,
'SHLAND' , 'SURFACE' ,
Expand All @@ -968,8 +971,8 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
'GHLAND' , 'SURFACE' ,
'TWLAND' , 'SURFACE' ,
'TELAND' , 'SURFACE' ,
'DWLAND' , 'SURFACE' , 'WCHANGE' ,
'DHLAND' , 'SURFACE' , 'ECHANGE' ,
'DWLAND' , 'SURFACE' , 'WCHANGELAND' ,
'DHLAND' , 'SURFACE' , 'ECHANGELAND' ,
'SPLAND' , 'SURFACE' , 'SPSHLAND' ,
'SPLH' , 'SURFACE' , 'SPLHLAND' ,
'SPWATR' , 'SURFACE' , 'SPEVLAND' ,
Expand All @@ -988,8 +991,8 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
tavg1_2d_lfo_Nx-.end_time: >>>IOETIME<<< ,
tavg1_2d_lfo_Nx-.fields: 'SLRSF' , 'SOLAR' , 'SWGDN' ,
'LWS' , 'IRRAD' , 'LWGAB' ,
'PCU' , 'SURFACE' , 'PRECCU' ,
'PLS' , 'SURFACE' , 'PRECLS' ,
'PCU' , 'SURFACE' , 'PRECRAINCU' ,
'PLS' , 'SURFACE' , 'PRECRAINLS' ,
'SNO' , 'SURFACE' , 'PRECSNO' ,
'DFPAR' , 'SOLAR' , 'PARDF' ,
'DRPAR' , 'SOLAR' , 'PARDR' ,
Expand Down Expand Up @@ -1993,11 +1996,11 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
tavg1_2d_lfo_Nx+-.ref_time: >>>IOBBKGT<<< ,
tavg1_2d_lfo_Nx+-.end_date: >>>IOEBKGD<<< ,
tavg1_2d_lfo_Nx+-.end_time: >>>IOEBKGT<<< ,
tavg1_2d_lfo_Nx.fields: 'SLRSF' , 'SOLAR' , 'SWGDN' ,
tavg1_2d_lfo_Nx+-.fields: 'SLRSF' , 'SOLAR' , 'SWGDN' ,
'SWLAND' , 'SURFACE' ,
'LWS' , 'IRRAD' , 'LWGAB' ,
'PCU' , 'SURFACE' , 'PRECCU' ,
'PLS' , 'SURFACE' , 'PRECLS' ,
'PCU' , 'SURFACE' , 'PRECRAINCU' ,
'PLS' , 'SURFACE' , 'PRECRAINLS' ,
'SNO' , 'SURFACE' , 'PRECSNO' ,
'DFPAR' , 'SOLAR' , 'PARDF' ,
'DRPAR' , 'SOLAR' , 'PARDR' ,
Expand Down Expand Up @@ -2218,6 +2221,7 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
'THAT' , 'SURFACE' , 'TSH' ,
'QHAT' , 'SURFACE' , 'QSH' ,
'FRACI' , 'SURFACE' , 'FRSEAICE' ,
'RUNOFF' , 'SURFACE' , 'RUNOFFTOT' ,
'AN_PRCP' , 'MOIST' , 'PRECANV' ,
'CN_PRCP' , 'MOIST' , 'PRECCON' ,
'LS_PRCP' , 'MOIST' , 'PRECLSC' ,
Expand Down Expand Up @@ -2248,11 +2252,12 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
'WCPR' , 'SURFACE' , 'PRMC' ,
'WCRZ' , 'SURFACE' , 'RZMC' ,
'WCSF' , 'SURFACE' , 'SFMC' ,
'TPSNOW' , 'SURFACE' ,
'TPUNST' , 'SURFACE' , 'TUNST' ,
'TPSAT' , 'SURFACE' , 'TSAT' ,
'TPWLT' , 'SURFACE' , 'TWLT' ,
'TPSURF' , 'SURFACE' , 'TSURF' ,
'CAPAC' , 'SURFACE' , 'INTRWATR' ,
'TPSNOW' , 'SURFACE' , 'TPSNOWLAND',
'TPUNST' , 'SURFACE' , 'TUNSTLAND' ,
'TPSAT' , 'SURFACE' , 'TSATLAND' ,
'TPWLT' , 'SURFACE' , 'TWLTLAND' ,
'TPSURF' , 'SURFACE' , 'TSURFLAND' ,
'TSOIL1' , 'SURFACE' ,
'TSOIL2' , 'SURFACE' ,
'TSOIL3' , 'SURFACE' ,
Expand All @@ -2261,20 +2266,20 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
'TSOIL6' , 'SURFACE' ,
'SNOLAND' , 'SURFACE' , 'PRECSNOLAND' ,
'PRLAND' , 'SURFACE' , 'PRECTOTLAND' ,
'TSLAND' , 'SURFACE' , 'SNOMAS' ,
'SNOWDP' , 'SURFACE' , 'SNODP' ,
'TSLAND' , 'SURFACE' , 'SNOMASLAND' ,
'SNOWDP' , 'SURFACE' , 'SNODPLAND' ,
'EVPSOI' , 'SURFACE' , 'LHLANDSOIL',
'EVPVEG' , 'SURFACE' , 'LHLANDTRNS',
'EVPINT' , 'SURFACE' , 'LHLANDINTR',
'EVPICE' , 'SURFACE' , 'LHLANDSBLN',
'RUNSURF' , 'SURFACE' , 'RUNOFF' ,
'BASEFLOW' , 'SURFACE' ,
'RUNSURF' , 'SURFACE' , 'RUNSURFLAND' ,
'BASEFLOW' , 'SURFACE' , 'BASEFLOWLAND' ,
'SMLAND' , 'SURFACE' ,
'QINFIL' , 'SURFACE' ,
'FRUST' , 'SURFACE' , 'FRUNST' ,
'FRSAT' , 'SURFACE' ,
'ASNOW' , 'SURFACE' , 'FRSNO' ,
'FRWLT' , 'SURFACE' ,
'QINFIL' , 'SURFACE' , 'QINFILLAND',
'FRUST' , 'SURFACE' , 'FRLANDUNST',
'FRSAT' , 'SURFACE' , 'FRLANDSAT' ,
'ASNOW' , 'SURFACE' , 'FRLANDSNO' ,
'FRWLT' , 'SURFACE' , 'FRLANDWLT' ,
'DFPARLAND' , 'SURFACE' , 'PARDFLAND' ,
'DRPARLAND' , 'SURFACE' , 'PARDRLAND' ,
'SHLAND' , 'SURFACE' ,
Expand All @@ -2285,8 +2290,8 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
'GHLAND' , 'SURFACE' ,
'TWLAND' , 'SURFACE' ,
'TELAND' , 'SURFACE' ,
'DWLAND' , 'SURFACE' , 'WCHANGE' ,
'DHLAND' , 'SURFACE' , 'ECHANGE' ,
'DWLAND' , 'SURFACE' , 'WCHANGELAND' ,
'DHLAND' , 'SURFACE' , 'ECHANGELAND' ,
'SPLAND' , 'SURFACE' , 'SPSHLAND' ,
'SPLH' , 'SURFACE' , 'SPLHLAND' ,
'SPWATR' , 'SURFACE' , 'SPEVLAND' ,
Expand Down Expand Up @@ -2732,6 +2737,35 @@ COLLECTIONS: 'inst3_3d_asm_Np-'
'AREA' , 'DYN' ,
::

const_2d_lnd_Nx.format: 'CFIO' ,
const_2d_lnd_Nx.descr: '2d,Constant,Time-invariant,Single-Level,Assimilation,Land Surface Model Parameters',
const_2d_lnd_Nx.template: '%y4%m2%d2_%h2%n2z.>>>NCSUFFIX<<<' ,
const_2d_lnd_Nx.mode: 'instantaneous' ,
const_2d_lnd_Nx.grid_label: PC@HIST_IMx@HIST_JM-DC ,
const_2d_lnd_Nx.frequency: 240000 ,
const_2d_lnd_Nx.duration: 240000 ,
const_2d_lnd_Nx.ref_time: >>>IOBBKGT<<< ,
const_2d_lnd_Nx.end_date: >>>IOEDATE<<< ,
const_2d_lnd_Nx.end_time: >>>IOETIME<<< ,
const_2d_lnd_Nx.fields: 'DZGT1' , 'SURFACE' ,
'DZGT2' , 'SURFACE' ,
'DZGT3' , 'SURFACE' ,
'DZGT4' , 'SURFACE' ,
'DZGT5' , 'SURFACE' ,
'DZGT6' , 'SURFACE' ,
'DZPR' , 'SURFACE' ,
'DZRZ' , 'SURFACE' ,
'DZSF' , 'SURFACE' ,
'DZTS' , 'SURFACE' ,
'WPWET' , 'SURFACE' ,
'WPEMW' , 'SURFACE' ,
'WPMC' , 'SURFACE' ,
'CDCR2' , 'SURFACE' ,
'POROS' , 'SURFACE' ,
::



#######################################################################
# Trajectory Files
#######################################################################
Expand Down
1 change: 1 addition & 0 deletions construct_extdata_yaml_list.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ def parse_args():
- WSUB_ExtData.yaml
- DataSea_ExtData.yaml
- UMWM_ExtData.yaml
- CORE_NYF_Data_AtmForcings_ExtData.yaml
""")
components = {}
for line in temp_text:
Expand Down
57 changes: 20 additions & 37 deletions gcm_setup
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,20 +37,17 @@ setenv ARCH `uname -s`
setenv NODE `uname -n`

if ($ARCH == Darwin) then
set FINDPATH = realpath
set PRELOAD_COMMAND = 'DYLD_INSERT_LIBRARIES'
set LD_LIBRARY_PATH_CMD = 'DYLD_LIBRARY_PATH'
# On macOS we seem to need to call mpirun directly and not use esma_mpirun
# For some reason SIP does not let the libraries be preloaded
set RUN_CMD = 'mpirun -np '
else
set FINDPATH = 'readlink -f'
set PRELOAD_COMMAND = 'LD_PRELOAD'
set LD_LIBRARY_PATH_CMD = 'LD_LIBRARY_PATH'
set RUN_CMD = '$GEOSBIN/esma_mpirun -np '
endif
set GCMSETUP = `$FINDPATH $0`
set BINDIR = `dirname $GCMSETUP`
set BINDIR = `pwd -L`
set GEOSDEF = `dirname $BINDIR`
set ETCDIR = ${GEOSDEF}/etc

Expand Down Expand Up @@ -1815,7 +1812,7 @@ else if( $SITE == 'AWS' | $SITE == 'Azure' ) then

setenv SSTDIR ${BOUNDARY_DIR}/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions
setenv CHMDIR ${BOUNDARY_DIR}/fvInput_nc3 # locations of Aerosol Chemistry BCs
setenv WRKDIR /home/$LOGNAME # user work directory
setenv WRKDIR $HOME # user work directory
setenv COUPLEDIR ${BOUNDARY_DIR}/bcs_shared/make_bcs_inputs/ocean # Coupled Ocean/Atmos Forcing
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files

Expand Down Expand Up @@ -1853,7 +1850,7 @@ else

setenv SSTDIR ${BOUNDARY_DIR}/@SSTNAME/${OGCM_IM}x${OGCM_JM} # location of SST Boundary Conditions
setenv CHMDIR ${BOUNDARY_DIR}/fvInput_nc3 # locations of Aerosol Chemistry BCs
setenv WRKDIR /home/$LOGNAME # user work directory
setenv WRKDIR $HOME # user work directory
setenv COUPLEDIR ${BOUNDARY_DIR}/bcs_shared/make_bcs_inputs/ocean # Coupled Ocean/Atmos Forcing
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files
set NX = 1
Expand Down Expand Up @@ -2188,9 +2185,6 @@ endif
else if( $MPI_STACK == intelmpi ) then

cat > $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3
# This flag prints out the Intel MPI state. Uncomment if needed
#setenv I_MPI_DEBUG 9
EOF
Expand All @@ -2200,7 +2194,24 @@ EOF
# specific compared to the above adjustments
if ( $SITE == 'NCCS' ) then

# Some flags we know work on SLES15 and Milan (see below)
# For safety, we keep the old SLES12 flags for that system
#
# NOTE: When Cascade Lake is on SLES15, the following flags
# might need to be Milan only

if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FABRICS shm:ofi
setenv I_MPI_OFI_PROVIDER psm3
EOF

else

cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3
setenv I_MPI_SHM_HEAP_VSIZE 512
setenv PSM2_MEMORY large
EOF
Expand All @@ -2216,34 +2227,6 @@ EOF

endif # if NOT Singularity

# Testing by Bill Putman found these to be
# useful flags with Intel MPI on SLES15 on the
# Milan nodes.
# Note 1: Testing by NCCS shows the PSM3 provider
# runs on the Infiniband fabric. Tests show it runs
# up to C720.
# Note 2: When the Cascade Lakes are moved to
# SLES15, these will need to be Milan-only flags
# as Intel MPI will probably work just fine with
# Intel chips.
if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FALLBACK 0
setenv I_MPI_FABRICS ofi
setenv I_MPI_OFI_PROVIDER psm3
setenv I_MPI_ADJUST_SCATTER 2
setenv I_MPI_ADJUST_SCATTERV 2
setenv I_MPI_ADJUST_GATHER 2
setenv I_MPI_ADJUST_GATHERV 3
setenv I_MPI_ADJUST_ALLGATHER 3
setenv I_MPI_ADJUST_ALLGATHERV 3
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_REDUCE 10
setenv I_MPI_ADJUST_BCAST 11
setenv I_MPI_ADJUST_REDUCE_SCATTER 4
setenv I_MPI_ADJUST_BARRIER 9
EOF

endif # if SLES15

endif # if NCCS
Expand Down
Loading

0 comments on commit 4d453ca

Please sign in to comment.