Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use more cesm style logging #231

Merged
merged 9 commits into from
Jan 18, 2023
Merged

use more cesm style logging #231

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a80b91d
use more cesm style logging
jedwards4b Dec 12, 2022
7824dce
switch to openmpi for github tests
jedwards4b Dec 12, 2022
12238ae
remove whitespace
jedwards4b Dec 12, 2022
cac67d1
revert change to github action
jedwards4b Dec 12, 2022
8bc69f6
further improvement to logunit setting
jedwards4b Dec 13, 2022
a789254
get the variable name correct
jedwards4b Dec 13, 2022
1dce6df
merge latest dev/ncar and resolve conflicts
alperaltuntas Jan 12, 2023
b693dd2
remove more direct calls to FMS in mom_cap.F90
alperaltuntas Jan 12, 2023
4959ee1
update more write units to stdout in mom6 nuopc cap and let only root…
alperaltuntas Jan 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 78 additions & 98 deletions config_src/drivers/nuopc_cap/mom_cap.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,18 @@

module MOM_cap_mod

use constants_mod, only: constants_init
use diag_manager_mod, only: diag_manager_init, diag_manager_end
use field_manager_mod, only: field_manager_init, field_manager_end
use fms_mod, only: fms_init, fms_end, open_namelist_file, check_nml_error
use fms_mod, only: close_file, file_exist, uppercase
use fms_io_mod, only: fms_io_exit
use mpp_domains_mod, only: domain2d, mpp_get_compute_domain, mpp_get_compute_domains
use MOM_domains, only: get_domain_extent
use MOM_io, only: stdout, io_infra_end
use mpp_domains_mod, only: mpp_get_compute_domains
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps this can be replaced with
use MOM_domains, only : get_domain_extent

use mpp_domains_mod, only: mpp_get_ntile_count, mpp_get_pelist, mpp_get_global_domain
use mpp_domains_mod, only: mpp_get_domain_npes
use mpp_io_mod, only: mpp_open, MPP_RDONLY, MPP_ASCII, MPP_OVERWR, MPP_APPEND, mpp_close, MPP_SINGLE
use mpp_mod, only: stdlog, stdout, mpp_root_pe, mpp_clock_id
use mpp_mod, only: mpp_clock_begin, mpp_clock_end, MPP_CLOCK_SYNC
use mpp_mod, only: MPP_CLOCK_DETAILED, CLOCK_COMPONENT, MAXPES
use time_manager_mod, only: set_calendar_type, time_type, increment_date
use time_manager_mod, only: set_time, set_date, get_time, get_date, month_name
use time_manager_mod, only: GREGORIAN, JULIAN, NOLEAP, THIRTY_DAY_MONTHS, NO_CALENDAR
use time_manager_mod, only: operator( <= ), operator( < ), operator( >= )
use time_manager_mod, only: operator( + ), operator( - ), operator( / )
use time_manager_mod, only: operator( * ), operator( /= ), operator( > )
use time_manager_mod, only: date_to_string
use time_manager_mod, only: fms_get_calendar_type => get_calendar_type
use MOM_domains, only: MOM_infra_init, num_pes, root_pe, pe_here

use MOM_time_manager, only: set_calendar_type, time_type, set_time, set_date, month_name
use MOM_time_manager, only: GREGORIAN, JULIAN, NOLEAP
use MOM_time_manager, only: operator( <= ), operator( < ), operator( >= )
use MOM_time_manager, only: operator( + ), operator( - ), operator( / )
use MOM_time_manager, only: operator( * ), operator( /= ), operator( > )
use MOM_domains, only: MOM_infra_init, MOM_infra_end, num_pes, root_pe, pe_here
use MOM_file_parser, only: get_param, log_version, param_file_type, close_param_file
use MOM_get_input, only: get_MOM_input, directories
use MOM_domains, only: pass_var
Expand All @@ -40,8 +30,7 @@ module MOM_cap_mod
use MOM_cap_methods, only: ChkErr

#ifdef CESMCOUPLED
use shr_file_mod, only: shr_file_setLogUnit, shr_file_getLogUnit
use shr_mpi_mod, only : shr_mpi_min, shr_mpi_max
use shr_log_mod, only: shr_log_setLogUnit
#endif
use time_utils_mod, only: esmf2fms_time

Expand Down Expand Up @@ -80,6 +69,7 @@ module MOM_cap_mod
use ESMF, only: ESMF_AlarmGet, ESMF_AlarmIsCreated, ESMF_ALARMLIST_ALL, ESMF_AlarmIsEnabled
use ESMF, only: ESMF_STATEITEM_NOTFOUND, ESMF_FieldWrite
use ESMF, only: ESMF_END_ABORT, ESMF_Finalize
use ESMF, only: ESMF_REDUCE_MAX, ESMF_REDUCE_MIN, ESMF_VMAllReduce
use ESMF, only: operator(==), operator(/=), operator(+), operator(-)

! TODO ESMF_GridCompGetInternalState does not have an explicit Fortran interface.
Expand Down Expand Up @@ -138,7 +128,6 @@ module MOM_cap_mod
logical :: write_diagnostics = .false.
logical :: overwrite_timeslice = .false.
character(len=32) :: runtype !< run type
integer :: logunit !< stdout logging unit number
logical :: profile_memory = .true.
logical :: grid_attach_area = .false.
logical :: use_coldstart = .true.
Expand Down Expand Up @@ -462,9 +451,32 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
CALL ESMF_TimeIntervalGet(TINT, S=DT_OCEAN, RC=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

call fms_init(mpi_comm_mom)
call constants_init
call field_manager_init
! reset shr logging to my log file
if (localPet==0) then
call NUOPC_CompAttributeGet(gcomp, name="diro", &
isPresent=isPresentDiro, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
call NUOPC_CompAttributeGet(gcomp, name="logfile", &
isPresent=isPresentLogfile, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
if (isPresentDiro .and. isPresentLogfile) then
call NUOPC_CompAttributeGet(gcomp, name="diro", value=diro, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
call NUOPC_CompAttributeGet(gcomp, name="logfile", value=logfile, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
open(newunit=stdout,file=trim(diro)//"/"//trim(logfile))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jedwards4b , here the newunit argument returns a negative number -10. Within FMS, unit numbers are used as array indices (see mpp_io_misc.inc), so when the unit is -10, i get the following error:

Fortran runtime error: Index '-10' of dimension 1 of array 'mpp_file' below lower bound of -1

You need to run a GNU debug test to reproduce the error.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes newunit returns a negative value - did you try with the latest FMS_Interface? I said it wasn't required, but perhaps it is.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I tried it with the latest FMS_interface and 2021.03.01

else
stdout = output_unit
endif
else
stdout = output_unit
endif
call shr_log_setLogUnit(stdout)
call NUOPC_CompAttributeAdd(gcomp, (/"logunit"/), rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
call NUOPC_CompAttributeSet(gcomp, "logunit", stdout, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
call MOM_infra_init(mpi_comm_mom)

! determine the calendar
if (cesm_coupled) then
Expand All @@ -491,15 +503,13 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
call set_calendar_type (JULIAN)
endif

call diag_manager_init

! this ocean connector will be driven at set interval
DT = set_time (DT_OCEAN, 0)
! get current time
time_start = set_date (YEAR,MONTH,DAY,HOUR,MINUTE,SECOND)

if (is_root_pe()) then
write(logunit,*) subname//'current time: y,m,d-',year,month,day,'h,m,s=',hour,minute,second
write(stdout,*) subname//'current time: y,m,d-',year,month,day,'h,m,s=',hour,minute,second
endif

! get start/reference time
Expand All @@ -511,33 +521,14 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)

time0 = set_date (YEAR,MONTH,DAY,HOUR,MINUTE,SECOND)

if (is_root_pe()) then
write(logunit,*) subname//'start time: y,m,d-',year,month,day,'h,m,s=',hour,minute,second
endif

! rsd need to figure out how to get this without share code
!call shr_nuopc_get_component_instance(gcomp, inst_suffix, inst_index)
!inst_name = "OCN"//trim(inst_suffix)

! reset shr logging to my log file

if (is_root_pe()) then
call NUOPC_CompAttributeGet(gcomp, name="diro", &
isPresent=isPresentDiro, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
call NUOPC_CompAttributeGet(gcomp, name="logfile", &
isPresent=isPresentLogfile, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
if (isPresentDiro .and. isPresentLogfile) then
call NUOPC_CompAttributeGet(gcomp, name="diro", value=diro, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
call NUOPC_CompAttributeGet(gcomp, name="logfile", value=logfile, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
open(newunit=logunit,file=trim(diro)//"/"//trim(logfile))
else
logunit = output_unit
endif
else
logunit = output_unit
write(stdout,*) subname//'start time: y,m,d-',year,month,day,'h,m,s=',hour,minute,second
endif

starttype = ""
Expand Down Expand Up @@ -632,7 +623,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)

call ocean_model_init_sfc(ocean_state, ocean_public)

call mpp_get_compute_domain(ocean_public%domain, isc, iec, jsc, jec)
call get_domain_extent(ocean_public%domain, isc, iec, jsc, jec)

allocate ( Ice_ocean_boundary% u_flux (isc:iec,jsc:jec), &
Ice_ocean_boundary% v_flux (isc:iec,jsc:jec), &
Expand Down Expand Up @@ -796,7 +787,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
call NUOPC_Advertise(exportState, standardName=fldsFrOcn(n)%stdname, name=fldsFrOcn(n)%shortname, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
enddo

if(is_root_pe()) write(stdout,*) 'InitializeAdvertise complete'
end subroutine InitializeAdvertise

!> Called by NUOPC to realize import and export fields. "Realizing" a field
Expand Down Expand Up @@ -873,20 +864,16 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
real(ESMF_KIND_R8), allocatable :: mesh_areas(:)
real(ESMF_KIND_R8), allocatable :: model_areas(:)
real(ESMF_KIND_R8), pointer :: dataPtr_mesh_areas(:)
real(ESMF_KIND_R8) :: max_mod2med_areacor
real(ESMF_KIND_R8) :: max_med2mod_areacor
real(ESMF_KIND_R8) :: min_mod2med_areacor
real(ESMF_KIND_R8) :: min_med2mod_areacor
real(ESMF_KIND_R8) :: max_mod2med_areacor_glob
real(ESMF_KIND_R8) :: max_med2mod_areacor_glob
real(ESMF_KIND_R8) :: min_mod2med_areacor_glob
real(ESMF_KIND_R8) :: min_med2mod_areacor_glob
real(ESMF_KIND_R8) :: min_areacor(2)
real(ESMF_KIND_R8) :: max_areacor(2)
real(ESMF_KIND_R8) :: min_areacor_glob(2)
real(ESMF_KIND_R8) :: max_areacor_glob(2)
character(len=*), parameter :: subname='(MOM_cap:InitializeRealize)'
!--------------------------------

rc = ESMF_SUCCESS

call shr_file_setLogUnit (logunit)
call shr_log_setLogUnit (stdout)

!----------------------------------------------------------------------------
! Get pointers to ocean internal state
Expand Down Expand Up @@ -983,7 +970,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

if (localPet == 0) then
write(logunit,*)'mesh file for mom6 domain is ',trim(cvalue)
write(stdout,*)'mesh file for mom6 domain is ',trim(cvalue)
endif

! recreate the mesh using the above distGrid
Expand Down Expand Up @@ -1011,7 +998,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
call ESMF_MeshGet(Emesh, elemMaskArray=elemMaskArray, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

call mpp_get_compute_domain(ocean_public%domain, isc, iec, jsc, jec)
call get_domain_extent(ocean_public%domain, isc, iec, jsc, jec)
n = 0
do j = jsc, jec
jg = j + ocean_grid%jsc - jsc
Expand Down Expand Up @@ -1099,19 +1086,20 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
deallocate(model_areas)

! Write diagnostic output for correction factors
min_mod2med_areacor = minval(mod2med_areacor)
max_mod2med_areacor = maxval(mod2med_areacor)
min_med2mod_areacor = minval(med2mod_areacor)
max_med2mod_areacor = maxval(med2mod_areacor)
call shr_mpi_max(max_mod2med_areacor, max_mod2med_areacor_glob, mpicom)
call shr_mpi_min(min_mod2med_areacor, min_mod2med_areacor_glob, mpicom)
call shr_mpi_max(max_med2mod_areacor, max_med2mod_areacor_glob, mpicom)
call shr_mpi_min(min_med2mod_areacor, min_med2mod_areacor_glob, mpicom)
min_areacor(1) = minval(mod2med_areacor)
max_areacor(1) = maxval(mod2med_areacor)
min_areacor(2) = minval(med2mod_areacor)
max_areacor(2) = maxval(med2mod_areacor)
call ESMF_VMAllReduce(vm, min_areacor, min_areacor_glob, 2, ESMF_REDUCE_MIN, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
call ESMF_VMAllReduce(vm, max_areacor, max_areacor_glob, 2, ESMF_REDUCE_MAX, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

if (localPet == 0) then
write(logunit,'(2A,2g23.15,A )') trim(subname),' : min_mod2med_areacor, max_mod2med_areacor ',&
min_mod2med_areacor_glob, max_mod2med_areacor_glob, 'MOM6'
write(logunit,'(2A,2g23.15,A )') trim(subname),' : min_med2mod_areacor, max_med2mod_areacor ',&
min_med2mod_areacor_glob, max_med2mod_areacor_glob, 'MOM6'
write(stdout,'(2A,2g23.15,A )') trim(subname),' : min_mod2med_areacor, max_mod2med_areacor ',&
min_areacor_glob(1), max_areacor_glob(1), 'MOM6'
write(stdout,'(2A,2g23.15,A )') trim(subname),' : min_med2mod_areacor, max_med2mod_areacor ',&
min_areacor_glob(2), max_areacor_glob(2), 'MOM6'
end if
#endif

Expand Down Expand Up @@ -1262,7 +1250,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
! values for j=0 and wrap-around in i. on tripole seam, decomposition
! domains are 1 larger in j; to load corner values need to loop one extra row

call mpp_get_compute_domain(ocean_public%domain, isc, iec, jsc, jec)
call get_domain_extent(ocean_public%domain, isc, iec, jsc, jec)

lbnd1 = lbound(dataPtr_mask,1)
ubnd1 = ubound(dataPtr_mask,1)
Expand Down Expand Up @@ -1519,7 +1507,7 @@ subroutine ModelAdvance(gcomp, rc)
rc = ESMF_SUCCESS
if(profile_memory) call ESMF_VMLogMemInfo("Entering MOM Model_ADVANCE: ")

call shr_file_setLogUnit (logunit)
call shr_log_setLogUnit (stdout)

! query the Component for its clock, importState and exportState
call ESMF_GridCompGet(gcomp, clock=clock, importState=importState, &
Expand Down Expand Up @@ -1731,7 +1719,7 @@ subroutine ModelAdvance(gcomp, rc)
endif

if (is_root_pe()) then
write(logunit,*) subname//' writing restart file ',trim(restartname)
write(stdout,*) subname//' writing restart file ',trim(restartname)
endif
endif
endif ! restart_mode
Expand Down Expand Up @@ -1960,7 +1948,9 @@ subroutine ocean_model_finalize(gcomp, rc)
logical :: write_restart
character(len=*),parameter :: subname='(MOM_cap:ocean_model_finalize)'

write(*,*) 'MOM: --- finalize called ---'
if (is_root_pe()) then
write(stdout,*) 'MOM: --- finalize called ---'
endif
rc = ESMF_SUCCESS

call ESMF_GridCompGetInternalState(gcomp, ocean_internalstate, rc)
Expand All @@ -1986,12 +1976,13 @@ subroutine ocean_model_finalize(gcomp, rc)
ESMF_LOGMSG_INFO)

call ocean_model_end(ocean_public, ocean_State, Time, write_restart=write_restart)
call field_manager_end()

call fms_io_exit()
call fms_end()
call io_infra_end()
call MOM_infra_end()

write(*,*) 'MOM: --- completed ---'
if (is_root_pe()) then
write(stdout,*) 'MOM: --- completed ---'
endif

end subroutine ocean_model_finalize

Expand Down Expand Up @@ -2202,17 +2193,11 @@ end subroutine fld_list_add


#ifndef CESMCOUPLED
subroutine shr_file_setLogUnit(nunit)
integer, intent(in) :: nunit
! do nothing for this stub - its just here to replace
! having cppdefs in the main program
end subroutine shr_file_setLogUnit

subroutine shr_file_getLogUnit(nunit)
subroutine shr_log_setLogUnit(nunit)
integer, intent(in) :: nunit
! do nothing for this stub - its just here to replace
! having cppdefs in the main program
end subroutine shr_file_getLogUnit
end subroutine shr_log_setLogUnit
#endif

!>
Expand Down Expand Up @@ -2342,8 +2327,7 @@ end subroutine shr_file_getLogUnit
!! @subsection Initialization Initialization
!!
!! During the [InitializeAdvertise] (@ref MOM_cap_mod::initializeadvertise) phase, calls are
!! made to MOM's native initialization subroutines, including `fms_init()`, `constants_init()`,
!! `field_manager_init()`, `diag_manager_init()`, and `set_calendar_type()`. The MPI communicator
!! made to MOM's native initialization subroutines. The MPI communicator
!! is pulled in through the ESMF VM object for the MOM component. The dt and start time are set
!! from parameters from the incoming ESMF clock with calls to `set_time()` and `set_date().`
!!
Expand Down Expand Up @@ -2418,10 +2402,6 @@ end subroutine shr_file_getLogUnit
!! procedures:
!!
!! call ocean_model_end (ocean_public, ocean_State, Time)
!! call diag_manager_end(Time )
!! call field_manager_end
!! call fms_io_exit
!! call fms_end
!!
!! @section ModelFields Model Fields
!!
Expand Down Expand Up @@ -2710,7 +2690,7 @@ end subroutine shr_file_getLogUnit
!! with incoming coupling fields from other components. These three derived types are allocated during the
!! [InitializeAdvertise] (@ref MOM_cap_mod::initializeadvertise) phase. Also during that
!! phase, the `ice_ocean_boundary` type members are all allocated using bounds retrieved
!! from `mpp_get_compute_domain()`.
!! from `get_domain_extent()`.
!!
!! During the [InitializeRealize] (@ref MOM_cap_mod::initializerealize) phase,
!! `ESMF_Field`s are created for each of the coupling fields in the `ice_ocean_boundary`
Expand Down
Loading