Merge pull request NOAA-EMC#2 from NOAA-EMC/develop

develop update

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "tests/produtil/NCEPLIBS-pyprodutil"]
 	path = tests/produtil/NCEPLIBS-pyprodutil
-	url = gerrit:NCEPLIBS-pyprodutil
+	url = https://github.com/NOAA-EMC/NCEPLIBS-pyprodutil
+	branch = port-2-hera

src/conf/module-setup.csh.inc

@@ -8,7 +8,13 @@ if ( { test -d /lfs3 } ) then
             source /apps/lmod/lmod/init/$__ms_shell
     endif
     module purge
-else if ( { test -d /scratch3 } ) then
+else if ( { test -d /scratch1 -a ! -d /scratch } ) then
+    # We are on NOAA Hera
+    if ( ! { module help >& /dev/null } ) then
+        source /apps/lmod/lmod/init/$__ms_shell
+    endif
+    module purge
+else if ( { test -d /scratch3 -a -d /scratch } ) then
     # We are on NOAA Theia
     if ( ! { module help >& /dev/null } ) then
         source /apps/lmod/lmod/init/$__ms_shell

src/conf/module-setup.sh.inc

@@ -22,7 +22,13 @@ if [[ -d /lfs3 ]] ; then
         source /apps/lmod/lmod/init/$__ms_shell
     fi
     module purge
-elif [[ -d /scratch3 ]] ; then
+elif [[ -d /scratch1 && ! -d /scratch ]] ; then
+    # We are on NOAA Hera
+    if ( ! eval module help > /dev/null 2>&1 ) ; then
+        source /apps/lmod/lmod/init/$__ms_shell
+    fi
+    module purge
+elif [[ -d /scratch3 && -d /scratch ]] ; then
     # We are on NOAA Theia
     if ( ! eval module help > /dev/null 2>&1 ) ; then
         source /apps/lmod/lmod/init/$__ms_shell

src/incmake/env/rdhpcs/detect.mk

@@ -4,17 +4,22 @@
 #
 ########################################################################
 
-ifneq (,$(and $(wildcard /scratch4),$(wildcard /scratch3)))
+ifneq (,$(and $(wildcard /scratch),$(wildcard /scratch3)))
   NEMS_COMPILER?=intel
   $(call add_build_env,theia.$(NEMS_COMPILER),env/rdhpcs/theia.$(NEMS_COMPILER).mk)
 else
-  ifneq (,$(and $(wildcard /lfs1),$(wildcard /lfs3)))
+  ifneq (,$(and $(wildcard /scratch1),$(wildcard /scratch2)))
     NEMS_COMPILER?=intel
-    $(call add_build_env,jet.$(NEMS_COMPILER),env/rdhpcs/jet.$(NEMS_COMPILER).mk)
+    $(call add_build_env,hera.$(NEMS_COMPILER),env/rdhpcs/hera.$(NEMS_COMPILER).mk)
   else
-    ifneq (,$(shell hostname | grep -i gaea))
+    ifneq (,$(and $(wildcard /lfs1),$(wildcard /lfs3)))
       NEMS_COMPILER?=intel
-      $(call add_build_env,gaea.$(NEMS_COMPILER),env/rdhpcs/gaea.$(NEMS_COMPILER).mk)
+      $(call add_build_env,jet.$(NEMS_COMPILER),env/rdhpcs/jet.$(NEMS_COMPILER).mk)
+    else
+      ifneq (,$(shell hostname | grep -i gaea))
+        NEMS_COMPILER?=intel
+        $(call add_build_env,gaea.$(NEMS_COMPILER),env/rdhpcs/gaea.$(NEMS_COMPILER).mk)
+      endif
     endif
   endif
 endif

src/incmake/env/rdhpcs/hera.intel.mk

@@ -0,0 +1,7 @@
+MACHINE_ID=hera
+FULL_MACHINE_ID=hera
+USE_MODULES=YES
+DEFAULT_MODULE=$(FULL_MACHINE_ID)/ESMF_NUOPC
+BUILD_TARGET=$(FULL_MACHINE_ID).$(NEMS_COMPILER)
+NEMS_COMPILER=intel
+MODULE_LOGIC=$(call ULIMIT_MODULE_LOGIC,200000)

src/module_EARTH_GRID_COMP.F90

@@ -587,6 +587,76 @@ SUBROUTINE EARTH_REGISTER(EARTH_GRID_COMP,RC_REG)
           file=__FILE__)) &
           return  ! bail out
       endif
+      !For MOM6 and WW3 variables to match:
+      if (.not.NUOPC_FieldDictionaryHasEntry( &
+        "eastward_partitioned_stokes_drift_1")) then
+        call NUOPC_FieldDictionaryAddEntry( &
+          standardName="eastward_partitioned_stokes_drift_1", &
+          canonicalUnits="m s-1", &
+          rc=rc)
+        if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+          line=__LINE__, &
+          file=__FILE__)) &
+          return  ! bail out
+      endif
+
+      if (.not.NUOPC_FieldDictionaryHasEntry( &
+        "northward_partitioned_stokes_drift_1")) then
+        call NUOPC_FieldDictionaryAddEntry( &
+          standardName="northward_partitioned_stokes_drift_1", &
+          canonicalUnits="m s-1", &
+          rc=rc)
+        if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+          line=__LINE__, &
+          file=__FILE__)) &
+          return  ! bail out
+      endif
+      if (.not.NUOPC_FieldDictionaryHasEntry( &
+        "eastward_partitioned_stokes_drift_2")) then
+        call NUOPC_FieldDictionaryAddEntry( &
+          standardName="eastward_partitioned_stokes_drift_2", &
+          canonicalUnits="m s-1", &
+          rc=rc)
+        if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+          line=__LINE__, &
+          file=__FILE__)) &
+          return  ! bail out
+      endif
+      if (.not.NUOPC_FieldDictionaryHasEntry( &
+        "northward_partitioned_stokes_drift_2")) then
+        call NUOPC_FieldDictionaryAddEntry( &
+          standardName="northward_partitioned_stokes_drift_2", &
+          canonicalUnits="m s-1", &
+          rc=rc)
+        if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+          line=__LINE__, &
+          file=__FILE__)) &
+          return  ! bail out
+      endif
+     if (.not.NUOPC_FieldDictionaryHasEntry( &
+        "eastward_partitioned_stokes_drift_3")) then
+        call NUOPC_FieldDictionaryAddEntry( &
+          standardName="eastward_partitioned_stokes_drift_3", &
+          canonicalUnits="m s-1", &
+          rc=rc)
+        if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+          line=__LINE__, &
+          file=__FILE__)) &
+          return  ! bail out
+      endif
+
+      if (.not.NUOPC_FieldDictionaryHasEntry( &
+        "northward_partitioned_stokes_drift_3")) then
+        call NUOPC_FieldDictionaryAddEntry( &
+          standardName="northward_partitioned_stokes_drift_3", &
+          canonicalUnits="m s-1", &
+          rc=rc)
+        if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+          line=__LINE__, &
+          file=__FILE__)) &
+          return  ! bail out
+      endif
+      ! end of MOM6 and WW3 variables to match
       if (.not.NUOPC_FieldDictionaryHasEntry( &
         "inst_temp_height_surface")) then
         call NUOPC_FieldDictionaryAddEntry( &

src/module_MEDIATOR.F90

@@ -922,7 +922,7 @@ subroutine InitializeP0(gcomp, importState, exportState, clock, rc)
     if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
       line=__LINE__, file=__FILE__)) return  ! bail out
     dbug_flag = ESMF_UtilString2Int(value, &
-      specialStringList=(/"off","low","high","max"/), &
+      specialStringList=(/character(4)::"off","low","high","max"/), &
       specialValueList=(/0,1,100,255/), rc=rc)
     if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
       line=__LINE__, file=__FILE__)) return  ! bail out
@@ -5502,6 +5502,29 @@ subroutine MedPhase_prep_ocn(gcomp, rc)
     if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
       line=__LINE__, file=__FILE__)) return  ! bail out
 
+    call fieldBundle_FieldMerge(is_local%wrap%FBforOcn, 'mean_prec_rate', &
+                                is_local%wrap%FBAtm_o , 'mean_prec_rate', atmwgt, &
+                                rc=rc)
+    if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+      line=__LINE__, file=__FILE__)) return  ! bail out
+
+    !-------------
+    ! field_for_ocn = field_from_ice * ice_fraction
+    !-------------
+
+    call fieldBundle_FieldMerge(is_local%wrap%FBforOcn, 'net_heat_flx_to_ocn', &
+                                is_local%wrap%FBIce_o , 'net_heat_flx_to_ocn', icewgt, &
+                                rc=rc)
+    if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+      line=__LINE__, file=__FILE__)) return  ! bail out
+
+
+    call fieldBundle_FieldMerge(is_local%wrap%FBforOcn, 'mean_fresh_water_to_ocean_rate', &
+                                is_local%wrap%FBIce_o , 'mean_fresh_water_to_ocean_rate', icewgt, &
+                                rc=rc)
+    if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+      line=__LINE__, file=__FILE__)) return  ! bail out
+
 ! not used by mom, mom uses net, also mean_down_lw_flx not connected to ocn
 !    call fieldBundle_FieldMerge(is_local%wrap%FBforOcn, 'mean_down_lw_flx', & 
 !                                is_local%wrap%FBAtm_o , 'mean_down_lw_flx', atmwgt, &
@@ -5518,6 +5541,13 @@ subroutine MedPhase_prep_ocn(gcomp, rc)
     if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
       line=__LINE__, file=__FILE__)) return  ! bail out
 
+    call fieldBundle_FieldMerge(is_local%wrap%FBforOcn     , 'mean_sensi_heat_flx'             , & 
+                                is_local%wrap%FBAccumAtmOcn, 'mean_sensi_heat_flx_atm_into_ocn', atmwgt1, &
+                                is_local%wrap%FBAtm_o      , 'mean_sensi_heat_flx'             , wgtm01, &
+                                rc=rc)
+    if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+      line=__LINE__, file=__FILE__)) return  ! bail out
+
     call fieldBundle_FieldMerge(is_local%wrap%FBforOcn     , 'mean_net_lw_flx'   , & 
                                 is_local%wrap%FBAtm_o      , 'mean_down_lw_flx'  , atmwgt1, &
                                 is_local%wrap%FBAccumAtmOcn, 'mean_up_lw_flx_ocn', atmwgt1, &
@@ -5535,23 +5565,18 @@ subroutine MedPhase_prep_ocn(gcomp, rc)
 !      line=__LINE__, file=__FILE__)) return  ! bail out
 
     !-------------
-    ! field_for_ocn = field_from_atm * (1-ice_fraction) + field_from_ice * (ice_fraction)
+    ! field_for_ocn = field_from_ice * ice_fraction
     !-------------
 
-    call fieldBundle_FieldMerge(is_local%wrap%FBforOcn, 'mean_prec_rate'                , & 
-                                is_local%wrap%FBAtm_o , 'mean_prec_rate'                , atmwgt, &
-                                is_local%wrap%FBIce_o , 'mean_fresh_water_to_ocean_rate', icewgt, &
+    call fieldBundle_FieldMerge(is_local%wrap%FBforOcn, 'mean_salt_rate', &
+                                is_local%wrap%FBIce_o , 'mean_salt_rate', icewgt, &
                                 rc=rc)
     if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
       line=__LINE__, file=__FILE__)) return  ! bail out
 
-    call fieldBundle_FieldMerge(is_local%wrap%FBforOcn     , 'mean_sensi_heat_flx'             , & 
-                                is_local%wrap%FBAccumAtmOcn, 'mean_sensi_heat_flx_atm_into_ocn', atmwgt1, &
-                                is_local%wrap%FBIce_o      , 'net_heat_flx_to_ocn'             , icewgt1, &
-                                is_local%wrap%FBAtm_o      , 'mean_sensi_heat_flx'             , wgtm01, &
-                                rc=rc)
-    if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
-      line=__LINE__, file=__FILE__)) return  ! bail out
+    !-------------
+    ! field_for_ocn = field_from_atm * (1-ice_fraction) + field_from_ice * (ice_fraction)
+    !-------------
 
     call fieldBundle_FieldMerge(is_local%wrap%FBforOcn     , 'mean_zonal_moment_flx'  , & 
                                 is_local%wrap%FBAccumAtmOcn, 'stress_on_air_ocn_zonal', atmwgt1, &

tests/apps.def

@@ -1,19 +1,12 @@
-# JET NOTE: We only list one jet generation (the oldest) because it
-# takes too long to test all three generations.
-
 # List of valid platforms and the human-readable names of each.
-#PLATFORM xjet              NAME xJet
-#PLATFORM svjet             NAME sJet and vJet
-#PLATFORM tujet             NAME uJet and tJet
-#PLATFORM ujet.slurm        NAME uJet SLURM Test
 PLATFORM theia             NAME Theia
-#PLATFORM theia.intel       NAME Theia (Intel compiler)
 PLATFORM theia.slurm.intel NAME Theia (SLURM test)
+PLATFORM hera              NAME Hera
+PLATFORM hera.intel        NAME Hera (SLURM test)
 PLATFORM wcoss1            NAME WCOSS Phase 1
 PLATFORM wcoss2            NAME WCOSS Phase 2
 PLATFORM wcoss_cray        NAME WCOSS Cray
 PLATFORM wcoss_dell_p3     NAME WCOSS Phase 3
-#PLATFORM gaea              NAME GAEA C3
 
 # List of known apps and the compsets to run for each app.
 APP NEMSfv3gfs      COMPSETS -f
@@ -29,39 +22,27 @@ APP FV3GFS-GSDCHEM  URL gerrit:EMC_FV3GFS-GSDCHEM
 
 # Shell expressions that generate scrub space for a given $username
 # on each platform.
-#ON xjet             SCRUB /lfs3/projects/hfv3gfs/$username/scrub/xjet
-#ON svjet            SCRUB /lfs3/projects/hfv3gfs/$username/scrub/svjet
-#ON tujet            SCRUB /lfs3/projects/hfv3gfs/$username/scrub/tujet
-#ON ujet.slurm       SCRUB /lfs3/projects/hfv3gfs/$username/scrub/ujet.slurm
-ON theia            SCRUB /scratch4/NCEPDEV/nems/scrub/$username
-#ON theia.intel      SCRUB /scratch4/NCEPDEV/nems/scrub/$username
+ON theia             SCRUB /scratch4/NCEPDEV/nems/scrub/$username
 ON theia.slurm.intel SCRUB /scratch4/NCEPDEV/nems/scrub/$username
-ON wcoss1           SCRUB /ptmpp1/$username
-ON wcoss2           SCRUB /ptmpd3/$username
-ON wcoss_cray       SCRUB /gpfs/hps2/ptmp/$username
-ON wcoss_dell_p3    SCRUB /gpfs/dell2/ptmp/$username
-#ON gaea             SCRUB $( ls -1d /lustre/f2/scratch/*/$username /lustre/f2/scratch/$username | head -1 )
+ON hera              SCRUB /scratch1/NCEPDEV/nems/$username
+ON hera.intel        SCRUB /scratch1/NCEPDEV/nems/$username
+ON wcoss1            SCRUB /ptmpp1/$username
+ON wcoss2            SCRUB /ptmpd3/$username
+ON wcoss_cray        SCRUB /gpfs/hps2/ptmp/$username
+ON wcoss_dell_p3     SCRUB /gpfs/dell2/ptmp/$username
 
 # List of apps to run on each platform.
-#ON xjet               APPS NEMSfv3gfs
-#ON svjet              APPS NEMSfv3gfs
-#ON tujet              APPS NEMSfv3gfs
-#ON ujet.slurm         APPS NEMSfv3gfs
 ON theia              APPS FV3-MOM6-CICE5
-#ON theia.intel        APPS NEMSfv3gfs FV3GFS-GSDCHEM
 ON theia.slurm.intel  APPS NEMSfv3gfs WW3-FV3
+ON hera               APPS FV3-MOM6-CICE5
+ON hera.intel         APPS NEMSfv3gfs WW3-FV3
 ON wcoss1             APPS NEMSfv3gfs
 ON wcoss2             APPS NEMSfv3gfs WW3-FV3
 ON wcoss_cray         APPS NEMSfv3gfs FV3GFS-GSDCHEM
 ON wcoss_dell_p3      APPS NEMSfv3gfs
-#ON gaea               APPS NEMSfv3gfs
 
 # Extra arguments to rt.sh for each platform
-#ON xjet              EXTRA_ARGS --temp-dir /lfs3/projects/hfv3gfs/$USER/scrub/xjet --project hfv3gfs --platform xjet
-#ON svjet             EXTRA_ARGS --temp-dir /lfs3/projects/hfv3gfs/$USER/scrub/svjet --project hfv3gfs --platform svjet
-#ON ujet.slurm        EXTRA_ARGS --temp-dir /lfs3/projects/hfv3gfs/$USER/scrub/ujet.slurm --project hfv3gfs --platform ujet.slurm
-#ON tujet             EXTRA_ARGS --temp-dir /lfs3/projects/hfv3gfs/$USER/scrub/tujet --project hfv3gfs --platform tujet
 ON theia             EXTRA_ARGS --platform theia
-#ON theia.intel       EXTRA_ARGS --platform theia.intel
 ON theia.slurm.intel EXTRA_ARGS --platform theia.slurm.intel
-#ON gaea              EXTRA_ARGS --platform gaea.c3
+ON hera              EXTRA_ARGS --platform hera
+ON hera.intel        EXTRA_ARGS --platform hera.intel

tests/commit.def

@@ -1,28 +1,26 @@
 # Directory to receive the webpage with regression test results
-#ON tujet          WEBPAGE /lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/combo/
-#ON ujet.slurm     WEBPAGE /lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/combo/
-ON theia          WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/combo/
-#ON theia.intel    WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/combo/
-ON theia.slurm.intel  WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/combo/
-ON wcoss1         WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/combo/
-ON wcoss2         WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/combo/
-ON wcoss_cray     WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/combo/
-ON wcoss_dell_p3  WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/combo/
-#ON gaea           WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/combo/
+ON theia              WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/NUOPC/
+ON theia.slurm.intel  WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/NUOPC/
+ON hera               WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/NUOPC/
+ON hera.intel         WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/NUOPC/
+ON wcoss1             WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/NUOPC/
+ON wcoss2             WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/NUOPC/
+ON wcoss_cray         WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/NUOPC/
+ON wcoss_dell_p3      WEBPAGE jetscp.rdhpcs.noaa.gov:/lfs3/projects/hfv3gfs/emc.nemspara/web/nems-commit/NUOPC/
 
 USER ACCOUNT IS Ratko.Vasic
 ROLE ACCOUNT IS emc.nemspara
 
 # nems branch - branch of NEMS that is being tested.  Set to "default"
 # to test the app's own NEMS
-NEMS   BRANCH IS combogsdww3nems
+NEMS   BRANCH IS commit_tmp
 
-# app branch - branch to use for committing logs to each 
-APP    BRANCH IS combo-commit
+# app branch - branch to use for committing logs to each
+#APP    BRANCH IS NUOPC-commit
 
 # If relevant, the starting branch that we copy to make the app
 # branch.  Default is the master branch.
-#APP NEMSfv3gfs       CHECKOUT combo
+#APP NEMSfv3gfs       CHECKOUT NUOPC
 
 # master branch - name of the git "master" branch.  Do not change.
 MASTER BRANCH IS master

tests/multi-app-test.sh

@@ -1560,7 +1560,7 @@ push_to_master() {
     done
 
     # Check out $master_branch of NEMS:
-    git clone gerrit:NEMS NEMS
+    git clone github:NEMS NEMS
     cd NEMS
     git checkout "$master_branch"
 

tests/nightly/run-from-wcoss.sh

@@ -13,6 +13,12 @@ if [[ "$hostchar" == "$prodchar" ]] ; then
     exit 1
 fi
 
+#WCOSS_Cray need to load newer git version
+if [[ `hostname | cut -c2-6` == "login" ]] ; then
+    source /opt/modules/default/init/sh
+    module use /usrx/local/dev/modulefiles
+    module load git/2.18.0
+fi
 
 cd $( dirname "$0" )
 cd ..