Merge branch 'add_dens_deriv_diag' of github.com:gmacgilchrist/MOM6 i…

…nto add_dens_deriv_diag

.gitlab-ci.yml

@@ -116,7 +116,7 @@ run:
     - time tar zxf $CACHE_DIR/build-intel-repro-$CI_PIPELINE_ID.tgz
     - time tar zxf $CACHE_DIR/build-pgi-repro-$CI_PIPELINE_ID.tgz
     # time tar zxf $CACHE_DIR/build-gnu-debug-$CI_PIPELINE_ID.tgz
-    - (echo '#!/bin/tcsh';echo 'make -f MRS/Makefile.tests all -B') > job.sh
+    - (echo '#!/bin/tcsh';echo 'make -f MRS/Makefile.tests all') > job.sh
     - sbatch --clusters=c3,c4 --nodes=29 --time=0:34:00 --account=gfdl_o --qos=debug --job-name=mom6_regressions --output=log.$CI_PIPELINE_ID --wait job.sh
     - cat log.$CI_PIPELINE_ID
     - test -f restart_results_gnu.tar.gz

.gitmodules

@@ -4,9 +4,3 @@
 [submodule "pkg/GSW-Fortran"]
 	path = pkg/GSW-Fortran
 	url = https://github.com/TEOS-10/GSW-Fortran.git
-[submodule "pkg/MOM6_DA_hooks"]
-	path = pkg/MOM6_DA_hooks
-	url = https://github.com/MJHarrison-GFDL/MOM6_DA_hooks.git
-[submodule "pkg/geoKdTree"]
-	path = pkg/geoKdTree
-	url = https://github.com/travissluka/geoKdTree.git

.testing/Makefile

@@ -20,7 +20,7 @@ MKMF := $(abspath $(DEPS)/mkmf/bin/mkmf)
 
 # FMS framework
 FMS_URL ?= https://github.com/NOAA-GFDL/FMS.git
-FMS_COMMIT ?= 2019.01.01
+FMS_COMMIT ?= 2019.01.02
 FMS := $(DEPS)/fms
 
 #---
@@ -268,18 +268,6 @@ $(eval $(call CMP_RULE,regression,symmetric target))
 #---
 # Test run output files
 
-# Generalized MPI environment variable support
-# XXX: Using `-env` in the MPICH test can erroneously producing an `nv` file.
-# $(1): Environment variables
-ifeq ($(shell $(MPIRUN) -x tmp=1 true 2> /dev/null ; echo $$?), 0)
-  MPIRUN_CMD=$(MPIRUN) $(if $(1),-x $(1),)
-else ifeq ($(shell $(MPIRUN) -env tmp=1 true 2> /dev/null ; echo $$? ; rm -f nv), 0)
-  MPIRUN_CMD=$(MPIRUN) $(if $(1),-env $(1),)
-else
-  MPIRUN_CMD=$(1) $(MPIRUN)
-endif
-
-
 # Rule to build work/<tc>/{ocean.stats,chksum_diag}.<tag>
 # $(1): Test configuration name <tag>
 # $(2): Executable type
@@ -297,20 +285,20 @@ work/%/$(1)/ocean.stats work/%/$(1)/chksum_diag: build/$(2)/MOM6
 	mkdir -p $$(@D)/RESTART
 	echo -e "$(4)" > $$(@D)/MOM_override
 	cd $$(@D) \
-	  && $$(call MPIRUN_CMD,$(5)) -n $(6) ../../../$$< 2> std.err > std.out \
+	  && $(5) $(MPIRUN) -n $(6) ../../../$$< 2> std.err > std.out \
 	  || !( \
 	    mkdir -p ../../../results/$$*/ ; \
 	    cat std.out | tee ../../../results/$$*/std.$(1).out | tail -20 ; \
-		cat std.err | tee ../../../results/$$*/std.$(1).err | tail -20 ; \
-		rm ocean.stats chksum_diag ; \
-		echo -e "${FAIL}: $$*.$(1) failed at runtime." \
+	    cat std.err | tee ../../../results/$$*/std.$(1).err | tail -20 ; \
+	    rm ocean.stats chksum_diag ; \
+	    echo -e "${FAIL}: $$*.$(1) failed at runtime." \
 	  )
 	@echo -e "${DONE}: $$*.$(1); no runtime errors."
 	if [ $(3) ]; then \
 	  mkdir -p results/$$* ; \
 	  bash <(curl -s https://codecov.io/bash) -n $$@ \
 	    > work/$$*/codecov.$(1).out \
-		2> work/$$*/codecov.$(1).err ; \
+	    2> work/$$*/codecov.$(1).err ; \
 	fi
 endef
 
@@ -354,8 +342,8 @@ work/%/restart/ocean.stats: build/symmetric/MOM6
 	cd $(@D) && $(MPIRUN) -n 1 ../../../$< 2> std1.err > std1.out \
 	  || !( \
 	    cat std1.out | tee ../../../results/$*/std.restart1.out | tail ; \
-		cat std1.err | tee ../../../results/$*/std.restart1.err | tail ; \
-		echo -e "${FAIL}: $*.restart failed at runtime." \
+	    cat std1.err | tee ../../../results/$*/std.restart1.err | tail ; \
+	    echo -e "${FAIL}: $*.restart failed at runtime." \
 	  )
 	# Setup the next inputs
 	cd $(@D) && rm -rf INPUT && mv RESTART INPUT
@@ -365,8 +353,8 @@ work/%/restart/ocean.stats: build/symmetric/MOM6
 	cd $(@D) && $(MPIRUN) -n 1 ../../../$< 2> std2.err > std2.out \
 	  || !( \
 	    cat std2.out | tee ../../../results/$*/std.restart2.out | tail ; \
-		cat std2.err | tee ../../../results/$*/std.restart2.err | tail ; \
-		echo -e "${FAIL}: $*.restart failed at runtime." \
+	    cat std2.err | tee ../../../results/$*/std.restart2.err | tail ; \
+	    echo -e "${FAIL}: $*.restart failed at runtime." \
 	  )
 
 # TODO: Restart checksum diagnostics

config_src/external/GFDL_ocean_BGC/README.md

@@ -3,4 +3,4 @@ GFDL_ocean_BGC
 
 These APIs reflect those for the GFDL ocean_BGC available at https://github.com/NOAA-GFDL/ocean_BGC.
 
-The modules in this directory do not do any computations. They simple reflect the APIs of the above package.
+The modules in this directory do not do any computations. They simply reflect the APIs of the above package.

config_src/external/ODA_hooks/README.md

@@ -0,0 +1,9 @@
+ODA_hooks
+=========
+
+These APIs reflect those for the ocean data assimilation hooks similar to https://github.com/MJHarrison-GFDL/MOM6_DA_hooks
+
+The modules in this directory do not do any computations. They simply reflect the APIs of the above package.
+
+- kdtree.f90 - would come from https://github.com/travissluka/geoKdTree
+- ocean_da_core.F90, ocean_da_types.F90, write_ocean_obs.F90 were copied from https://github.com/MJHarrison-GFDL/MOM6_DA_hooks

config_src/external/ODA_hooks/kdtree.f90

@@ -0,0 +1,12 @@
+!> A null version of K-d tree from geoKdTree
+module kdtree
+  implicit none
+  private
+
+  public :: kd_root
+
+  !> A K-d tree tpe
+  type kd_root
+    integer :: dummy !< To stop a compiler from doing nothing
+  end type kd_root
+end module kdtree

config_src/external/ODA_hooks/ocean_da_core.F90

@@ -0,0 +1,47 @@
+!> A set of dummy interfaces for compiling the MOM6 DA driver code.
+module ocean_da_core_mod
+  ! MOM modules
+  use MOM_domains, only : MOM_domain_type, domain2D
+  use MOM_time_manager, only : time_type, set_time, get_date
+  ! ODA_tools modules
+  use ocean_da_types_mod, only : ocean_profile_type, grid_type
+  use kdtree, only : kd_root
+
+  implicit none
+  private
+  public :: ocean_da_core_init
+  public :: get_profiles
+
+contains
+
+  !> Initializes the MOM6 DA driver code.
+  subroutine ocean_da_core_init(Domain, global_grid, Profiles, model_time)
+    type(domain2D), pointer, intent(in) :: Domain !< A MOM domain type
+    type(grid_type), pointer, intent(in) :: global_grid !< The global ODA horizontal grid type
+    type(ocean_profile_type), pointer :: Profiles  !< This is an unstructured recursive list of profiles
+                                          !! which are either within the localized domain corresponding
+                                          !! to the Domain argument, or the global profile list (type).
+    type(time_type), intent(in) :: model_time !< The current model time type.
+
+
+
+    Profiles=>NULL()
+    return
+  end subroutine ocean_da_core_init
+
+
+  !> Get profiles obs within the current analysis interval
+  subroutine get_profiles(model_time, Profiles, Current_profiles)
+    type(time_type), intent(in) :: model_time !< The current analysis time.
+    type(ocean_profile_type), pointer :: Profiles !< The full recursive list of profiles.
+    type(ocean_profile_type), pointer :: Current_profiles !< A returned list of profiles for the
+                                                          !! current analysis step.
+
+    Profiles=>NULL()
+    Current_Profiles=>NULL()
+
+    return
+  end subroutine get_profiles
+
+
+end module ocean_da_core_mod

config_src/external/ODA_hooks/ocean_da_types.F90

@@ -0,0 +1,85 @@
+!> Dummy aata structures and methods for ocean data assimilation.
+module ocean_da_types_mod
+
+  use MOM_time_manager, only : time_type
+
+  implicit none
+
+  private
+
+
+  !> Example type for ocean ensemble DA state
+  type, public :: OCEAN_CONTROL_STRUCT
+     integer :: ensemble_size
+     real, pointer, dimension(:,:,:)   :: SSH=>NULL() !<sea surface height (m) across ensembles
+     real, pointer, dimension(:,:,:,:) :: h=>NULL() !<layer thicknesses (m or kg) across ensembles
+     real, pointer, dimension(:,:,:,:) :: T=>NULL() !<layer potential temperature (degC) across ensembles
+     real, pointer, dimension(:,:,:,:) :: S=>NULL() !<layer salinity (psu or g kg-1) across ensembles
+     real, pointer, dimension(:,:,:,:) :: U=>NULL() !<layer zonal velocity (m s-1) across ensembles
+     real, pointer, dimension(:,:,:,:) :: V=>NULL() !<layer meridional velocity (m s-1) across ensembles
+  end type OCEAN_CONTROL_STRUCT
+
+  !> Example of a profile type
+  type, public :: ocean_profile_type
+     integer :: inst_type !< A numeric code indicating the type of instrument (e.g. ARGO drifter, CTD, ...)
+     logical :: initialized !< a True value indicates that this profile has been allocated for use
+     logical :: colocated !< a True value indicated that the measurements of (num_variables) data are
+                          !! co-located in space-time
+     integer :: ensemble_size !< size of the ensemble of model states used in association with this profile
+     integer :: num_variables !< number of measurement types associated with this profile.
+     integer, pointer, dimension(:) :: var_id !< variable ids are defined by the ocean_types module
+     integer :: platform !< platform types are defined by platform class (e.g. MOORING, DROP, etc.)
+                         !! and instrument type (XBT, CDT, etc.)
+     integer :: levels !< number of levels in the current profile
+     integer :: basin_mask !< 1:Southern Ocean, 2:Atlantic Ocean, 3:Pacific Ocean,
+                           !! 4:Arctic Ocean, 5:Indian Ocean, 6:Mediterranean Sea, 7:Black Sea,
+                           !! 8:Hudson Bay, 9:Baltic Sea, 10:Red Sea, 11:Persian Gulf
+     integer :: profile_flag !< an overall flag for the profile
+     real :: lat, lon !< latitude and longitude (degrees E and N)
+     logical :: accepted !< logical flag to disable a profile
+     type(time_type) :: time_window !< The time window associated with this profile [s]
+     real, pointer, dimension(:) :: obs_error  !< The observation error by variable
+     real  :: loc_dist   !< The impact radius of this observation (m)
+     type(ocean_profile_type), pointer :: next=>NULL() !< all profiles are stored as linked list.
+     type(ocean_profile_type), pointer :: prev=>NULL()
+     type(ocean_profile_type), pointer :: cnext=>NULL() ! current profiles are stored as linked list.
+     type(ocean_profile_type), pointer :: cprev=>NULL()
+     integer :: nbr_xi, nbr_yi ! nearest neighbor model gridpoint for the profile
+     real :: nbr_dist ! distance to nearest neighbor model gridpoint
+     logical :: compute !< profile is within current compute domain
+     real, dimension(:,:), pointer :: depth => NULL() !< depth of measurement [m]
+     real, dimension(:,:), pointer :: data => NULL() !< data by variable type
+     integer, dimension(:,:), pointer :: flag => NULL() !< flag by depth and variable type
+     real, dimension(:,:,:), pointer :: forecast => NULL() !< ensemble member first guess
+     real, dimension(:,:,:), pointer :: analysis => NULL() !< ensemble member analysis
+     type(forward_operator_type), pointer :: obs_def => NULL() !< observation forward operator
+     type(time_type) :: time !< profile time type
+     real :: i_index, j_index !< model longitude and latitude indices respectively
+     real, dimension(:,:), pointer :: k_index !< model depth indices
+     type(time_type) :: tdiff !< difference between model time and observation time
+     character(len=128) :: filename
+  end type ocean_profile_type
+
+  !>  Example forward operator type.
+  type, public :: forward_operator_type
+     integer :: num
+     integer, dimension(2) :: state_size !< for
+     integer, dimension(:), pointer :: state_var_index !< for flattened data
+     integer, dimension(:), pointer :: i_index !< i-dimension index
+     integer, dimension(:), pointer :: j_index !< j-dimension index
+     real, dimension(:), pointer :: coef
+  end type forward_operator_type
+
+  !> Grid type for DA
+  type, public :: grid_type
+     real, pointer, dimension(:,:) :: x=>NULL(), y=>NULL()
+     real, pointer, dimension(:,:,:) :: z=>NULL()
+     real, pointer, dimension(:,:,:) :: h=>NULL()
+     real, pointer, dimension(:,:) :: basin_mask => NULL()
+     real, pointer, dimension(:,:,:) :: mask => NULL()
+     real, pointer, dimension(:,:) :: bathyT => NULL()
+     logical :: tripolar_N
+     integer :: ni, nj, nk
+  end type grid_type
+
+end module ocean_da_types_mod

config_src/external/ODA_hooks/write_ocean_obs.F90

@@ -0,0 +1,50 @@
+!> Dummy interfaces for writing ODA data
+module write_ocean_obs_mod
+
+
+ use ocean_da_types_mod, only : ocean_profile_type
+ use MOM_time_manager, only : time_type, get_time, set_date
+
+ implicit none
+
+ private
+
+ public :: open_profile_file, write_profile, close_profile_file, &
+            write_ocean_obs_init
+
+contains
+
+!> Open a profile file
+integer function open_profile_file(name, nvar, grid_lon, grid_lat,thread,fset)
+  character(len=*), intent(in) :: name !< File name
+  integer, intent(in), optional :: nvar !< Number of variables
+  real, dimension(:), optional, intent(in) :: grid_lon !< Longitude [degreeE]
+  real, dimension(:), optional, intent(in) :: grid_lat !< Latitude [degreeN]
+  integer, intent(in), optional :: thread !< Thread
+  integer, intent(in), optional :: fset !< File set
+
+  open_profile_file=-1
+end function open_profile_file
+
+!> Write a profile
+subroutine write_profile(unit,profile)
+  integer, intent(in) :: unit !< File unit
+  type(ocean_profile_type), intent(in) :: profile !< Profile
+
+  return
+end subroutine write_profile
+
+!> Close a profile file
+subroutine close_profile_file(unit)
+  integer, intent(in) :: unit !< File unit
+
+  return
+end subroutine close_profile_file
+
+!> Initialize write_ocean_obs module
+subroutine write_ocean_obs_init()
+
+  return
+end subroutine write_ocean_obs_init
+
+end module write_ocean_obs_mod

docs/Doxyfile_nortd

@@ -794,7 +794,9 @@ INPUT                  = ../src \
                          front_page.md \
                          ../config_src/solo_driver \
                          ../config_src/dynamic_symmetric
-                         ../config_src/coupled_driver/ocean_model_MOM.F90
+                         ../config_src/external
+                         ../config_src/coupled_driver
+
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

docs/Doxyfile_rtd

@@ -783,8 +783,8 @@ WARN_LOGFILE           = doxygen.log
 INPUT                  = ../src \
                          ../config_src/solo_driver \
                          ../config_src/dynamic_symmetric \
-                         ../config_src/coupled_driver/coupler_util.F90 \
-                         ../config_src/coupled_driver/ocean_model_MOM.F90
+                         ../config_src/external \
+                         ../config_src/coupled_driver
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses