From 684c22691e623a32677410046ff8d2dab2e3ac0e Mon Sep 17 00:00:00 2001 From: Brian O'Neill Date: Fri, 11 Feb 2022 15:12:00 -0500 Subject: [PATCH 1/2] replace mesh pool with ocn_mesh module - replace mesh pool variables and arrays with counterparts in ocn_mesh module - remove meshPool from ocn_tend_thick subroutine calls in time integrator routines --- .../mpas_ocn_time_integration_rk4.F | 4 +-- .../mpas_ocn_time_integration_si.F | 2 +- .../mpas_ocn_time_integration_split.F | 2 +- .../src/shared/mpas_ocn_frazil_forcing.F | 11 ++----- .../shared/mpas_ocn_surface_bulk_forcing.F | 14 ++------ .../shared/mpas_ocn_surface_land_ice_fluxes.F | 14 ++------ .../mpas-ocean/src/shared/mpas_ocn_tendency.F | 23 +++++-------- .../src/shared/mpas_ocn_thick_hadv.F | 32 +++---------------- .../src/shared/mpas_ocn_thick_surface_flux.F | 19 +++-------- .../src/shared/mpas_ocn_thick_vadv.F | 21 +++--------- .../src/shared/mpas_ocn_tidal_forcing.F | 11 ++----- 11 files changed, 34 insertions(+), 119 deletions(-) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F index cbebcd97dadb..5b02c7c1c08d 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_rk4.F @@ -890,7 +890,7 @@ subroutine ocn_time_integrator_rk4_compute_thick_tends(block, dt, rkSubstepWeigh vertAleTransportTop, err) endif - call ocn_tend_thick(tendPool, forcingPool, meshPool) + call ocn_tend_thick(tendPool, forcingPool) end subroutine ocn_time_integrator_rk4_compute_thick_tends!}}} @@ -1035,7 +1035,7 @@ subroutine ocn_time_integrator_rk4_compute_tends(block, dt, rkWeight, err)!{{{ vertAleTransportTop, err) endif - call ocn_tend_thick(tendPool, forcingPool, meshPool) + call ocn_tend_thick(tendPool, forcingPool) if (config_filter_btr_mode) then call ocn_filter_btr_mode_tend_vel(tendPool, provisStatePool, meshPool, 1) diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F index 956f83360283..df4cc4175a7b 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F @@ -2612,7 +2612,7 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ #endif call mpas_timer_stop('thick vert trans vel top') - call ocn_tend_thick(tendPool, forcingPool, meshPool) + call ocn_tend_thick(tendPool, forcingPool) call mpas_timer_stop('si thick tend') diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F index 545e28a970e8..0d24a942c23a 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F @@ -1516,7 +1516,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ #endif call mpas_timer_stop('thick vert trans vel top') - call ocn_tend_thick(tendPool, forcingPool, meshPool) + call ocn_tend_thick(tendPool, forcingPool) call mpas_timer_stop('se thick tend') diff --git a/components/mpas-ocean/src/shared/mpas_ocn_frazil_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_frazil_forcing.F index ee42af596040..a8041ef9c083 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_frazil_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_frazil_forcing.F @@ -27,6 +27,7 @@ module ocn_frazil_forcing use mpas_timer use ocn_constants use ocn_config + use ocn_mesh use ocn_diagnostics_variables use ocn_equation_of_state @@ -134,14 +135,13 @@ end subroutine ocn_frazil_forcing_tracers!}}} ! !----------------------------------------------------------------------- - subroutine ocn_frazil_forcing_layer_thickness(meshPool, forcingPool, layerThicknessTend, err)!{{{ + subroutine ocn_frazil_forcing_layer_thickness(forcingPool, layerThicknessTend, err)!{{{ !----------------------------------------------------------------- ! ! input variables ! !----------------------------------------------------------------- - type (mpas_pool_type), intent(in) :: meshPool !< Input: mesh information type (mpas_pool_type), intent(in) :: forcingPool !< Input: Forcing information !----------------------------------------------------------------- @@ -166,8 +166,6 @@ subroutine ocn_frazil_forcing_layer_thickness(meshPool, forcingPool, layerThickn !----------------------------------------------------------------- integer :: iCell, k, nCells - integer, dimension(:), pointer :: nCellsArray - integer, dimension(:), pointer :: minLevelCell, maxLevelCell real (kind=RKIND), dimension(:,:), pointer :: frazilLayerThicknessTendency err = 0 @@ -176,13 +174,10 @@ subroutine ocn_frazil_forcing_layer_thickness(meshPool, forcingPool, layerThickn call mpas_timer_start("frazil thickness tendency") - call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) - call mpas_pool_get_array(meshPool, 'minLevelCell', minLevelCell) - call mpas_pool_get_array(meshPool, 'maxLevelCell', maxLevelCell) call mpas_pool_get_array(forcingPool, 'frazilLayerThicknessTendency', frazilLayerThicknessTendency) ! frazil fields are needed only over 0 and 1 halos - nCells = nCellsArray( 2 ) + nCells = nCellsHalo( 1 ) ! Build surface fluxes at cell centers !$omp parallel diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index da5995f12a1d..92fd397d01c4 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -279,14 +279,7 @@ end subroutine ocn_surface_bulk_forcing_vel!}}} ! !----------------------------------------------------------------------- - subroutine ocn_surface_bulk_forcing_thick(meshPool, forcingPool, surfaceThicknessFlux, surfaceThicknessFluxRunoff, err)!{{{ - - !----------------------------------------------------------------- - ! - ! input variables - ! - !----------------------------------------------------------------- - type (mpas_pool_type), intent(in) :: meshPool !< Input: mesh information + subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, surfaceThicknessFluxRunoff, err)!{{{ !----------------------------------------------------------------- ! @@ -313,7 +306,6 @@ subroutine ocn_surface_bulk_forcing_thick(meshPool, forcingPool, surfaceThicknes !----------------------------------------------------------------- integer :: iCell, nCells - integer, dimension(:), pointer :: nCellsArray real (kind=RKIND), dimension(:), pointer :: evaporationFlux, snowFlux real (kind=RKIND), dimension(:), pointer :: seaIceFreshWaterFlux, icebergFreshWaterFlux, riverRunoffFlux, iceRunoffFlux @@ -325,8 +317,6 @@ subroutine ocn_surface_bulk_forcing_thick(meshPool, forcingPool, surfaceThicknes call mpas_timer_start("bulk_thick", .false.) - call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) - call mpas_pool_get_array(forcingPool, 'evaporationFlux', evaporationFlux) call mpas_pool_get_array(forcingPool, 'snowFlux', snowFlux) call mpas_pool_get_array(forcingPool, 'seaIceFreshWaterFlux', seaIceFreshWaterFlux) @@ -335,7 +325,7 @@ subroutine ocn_surface_bulk_forcing_thick(meshPool, forcingPool, surfaceThicknes call mpas_pool_get_array(forcingPool, 'iceRunoffFlux', iceRunoffFlux) call mpas_pool_get_array(forcingPool, 'rainFlux', rainFlux) - nCells = nCellsArray( 3 ) + nCells = nCellsHalo( 2 ) ! Build surface fluxes at cell centers !$omp parallel diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F index e2436a13cc1f..96599cb7f2c1 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F @@ -245,14 +245,7 @@ end subroutine ocn_surface_land_ice_fluxes_vel!}}} ! !----------------------------------------------------------------------- - subroutine ocn_surface_land_ice_fluxes_thick(meshPool, forcingPool, surfaceThicknessFlux, err)!{{{ - - !----------------------------------------------------------------- - ! - ! input variables - ! - !----------------------------------------------------------------- - type (mpas_pool_type), intent(in) :: meshPool !< Input: mesh information + subroutine ocn_surface_land_ice_fluxes_thick(forcingPool, surfaceThicknessFlux, err)!{{{ !----------------------------------------------------------------- ! @@ -277,7 +270,6 @@ subroutine ocn_surface_land_ice_fluxes_thick(meshPool, forcingPool, surfaceThick !----------------------------------------------------------------- integer :: iCell - integer, pointer :: nCells real (kind=RKIND), dimension(:), pointer :: landIceFreshwaterFlux @@ -287,14 +279,12 @@ subroutine ocn_surface_land_ice_fluxes_thick(meshPool, forcingPool, surfaceThick call mpas_timer_start("land_ice_thick") - call mpas_pool_get_dimension(meshPool, 'nCells', nCells) - call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFlux) ! Build surface fluxes at cell centers !$omp parallel !$omp do schedule(runtime) - do iCell = 1, nCells + do iCell = 1, nCellsAll surfaceThicknessFlux(iCell) = surfaceThicknessFlux(iCell) + landIceFreshwaterFlux(iCell) / rho_sw end do !$omp end do diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index e7100ed556e5..b38b881b382c 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -111,14 +111,7 @@ module ocn_tendency ! !----------------------------------------------------------------------- - subroutine ocn_tend_thick(tendPool, forcingPool, meshPool)!{{{ - - !----------------------------------------------------------------- - ! input variables - !----------------------------------------------------------------- - - type (mpas_pool_type), intent(in) :: & - meshPool !< [in] Mesh information (for pass-thru) + subroutine ocn_tend_thick(tendPool, forcingPool)!{{{ !----------------------------------------------------------------- ! output variables @@ -185,12 +178,12 @@ subroutine ocn_tend_thick(tendPool, forcingPool, meshPool)!{{{ call mpas_timer_start("ocn_tend_thick") ! Compute surface mass flux array from bulk forcing - call ocn_surface_bulk_forcing_thick(meshPool, forcingPool, & + call ocn_surface_bulk_forcing_thick(forcingPool, & surfaceThicknessFlux, & surfaceThicknessFluxRunoff, err) ! Compute surface thickness flux from land ice - call ocn_surface_land_ice_fluxes_thick(meshPool, forcingPool, & + call ocn_surface_land_ice_fluxes_thick(forcingPool, & surfaceThicknessFlux, err) ! @@ -198,26 +191,26 @@ subroutine ocn_tend_thick(tendPool, forcingPool, meshPool)!{{{ ! See Ringler et al. (2010) jcp paper, eqn 19, 21, and fig. 3. ! for explanation of divergence operator. ! - call ocn_thick_hadv_tend(meshPool, normalTransportVelocity, & + call ocn_thick_hadv_tend(normalTransportVelocity, & layerThickEdge, tendThick, err) ! Compute vertical advection term -d/dz(hw) - call ocn_thick_vadv_tend(meshPool, vertAleTransportTop, & + call ocn_thick_vadv_tend(vertAleTransportTop, & tendThick, err) ! Compute surface flux tendency - call ocn_thick_surface_flux_tend(meshPool, fractionAbsorbed, & + call ocn_thick_surface_flux_tend(fractionAbsorbed, & fractionAbsorbedRunoff, & surfaceThicknessFlux, & surfaceThicknessFluxRunoff, & tendThick, err) ! Compute contribution from frazil ice formation - call ocn_frazil_forcing_layer_thickness(meshPool, forcingPool, & + call ocn_frazil_forcing_layer_thickness(forcingPool, & tendThick, err) ! Compute thickness change due to tidal forcing - call ocn_tidal_forcing_layer_thickness(meshPool, forcingPool, & + call ocn_tidal_forcing_layer_thickness(forcingPool, & tendThick, err) call mpas_timer_stop("ocn_tend_thick") diff --git a/components/mpas-ocean/src/shared/mpas_ocn_thick_hadv.F b/components/mpas-ocean/src/shared/mpas_ocn_thick_hadv.F index acccaa135a40..05c2b5bd4d30 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_thick_hadv.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_thick_hadv.F @@ -25,6 +25,7 @@ module ocn_thick_hadv use mpas_pool_routines use ocn_constants use ocn_config + use ocn_mesh implicit none private @@ -70,7 +71,7 @@ module ocn_thick_hadv ! !----------------------------------------------------------------------- - subroutine ocn_thick_hadv_tend(meshPool, normalVelocity, layerThicknessEdge, tend, err)!{{{ + subroutine ocn_thick_hadv_tend(normalVelocity, layerThicknessEdge, tend, err)!{{{ !----------------------------------------------------------------- ! @@ -84,9 +85,6 @@ subroutine ocn_thick_hadv_tend(meshPool, normalVelocity, layerThicknessEdge, ten real (kind=RKIND), dimension(:,:), intent(in) :: & layerThicknessEdge !< Input: thickness at edge - type (mpas_pool_type), intent(in) :: & - meshPool !< Input: mesh information - !----------------------------------------------------------------- ! ! input/output variables @@ -110,15 +108,9 @@ subroutine ocn_thick_hadv_tend(meshPool, normalVelocity, layerThicknessEdge, ten ! !----------------------------------------------------------------- - integer :: iEdge, k, i, iCell, nCells - integer, pointer :: nVertLevels - integer, dimension(:), pointer :: nCellsArray - - integer, dimension(:), pointer :: minLevelEdgeBot, maxLevelEdgeTop, MaxLevelCell, nEdgesOnCell - integer, dimension(:,:), pointer :: cellsOnEdge, edgesOnCell, edgeSignOnCell + integer :: iEdge, k, i, iCell real (kind=RKIND) :: flux, invAreaCell - real (kind=RKIND), dimension(:), pointer :: dvEdge, areaCell !----------------------------------------------------------------- ! @@ -134,25 +126,9 @@ subroutine ocn_thick_hadv_tend(meshPool, normalVelocity, layerThicknessEdge, ten call mpas_timer_start("thick hadv") - call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) - call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) - - call mpas_pool_get_array(meshPool, 'maxLevelCell', maxLevelCell) - call mpas_pool_get_array(meshPool, 'minLevelEdgeBot', minLevelEdgeBot) - call mpas_pool_get_array(meshPool, 'maxLevelEdgeTop', maxLevelEdgeTop) - call mpas_pool_get_array(meshPool, 'cellsOnEdge', cellsOnEdge) - call mpas_pool_get_array(meshPool, 'dvEdge', dvEdge) - call mpas_pool_get_array(meshPool, 'areaCell', areaCell) - - call mpas_pool_get_array(meshPool, 'nEdgesOnCell', nEdgesOnCell) - call mpas_pool_get_array(meshPool, 'edgesOnCell', edgesOnCell) - call mpas_pool_get_array(meshPool, 'edgeSignOnCell', edgeSignOnCell) - - nCells = nCellsArray( 1 ) - !$omp parallel !$omp do schedule(runtime) private(invAreaCell, i, iEdge, k, flux) - do iCell = 1, nCells + do iCell = 1, nCellsOwned invAreaCell = 1.0_RKIND / areaCell(iCell) do i = 1, nEdgesOnCell(iCell) iEdge = edgesOnCell(i, iCell) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F index d29313984555..60ff30e463d9 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F @@ -25,6 +25,7 @@ module ocn_thick_surface_flux use mpas_pool_routines use ocn_constants use ocn_config + use ocn_mesh use ocn_forcing @@ -72,7 +73,7 @@ module ocn_thick_surface_flux ! !----------------------------------------------------------------------- - subroutine ocn_thick_surface_flux_tend(meshPool, transmissionCoefficients, transmissionCoefficientsRunoff, & + subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoefficientsRunoff, & surfaceThicknessFlux, surfaceThicknessFluxRunoff, tend, err)!{{{ !----------------------------------------------------------------- ! @@ -80,9 +81,6 @@ subroutine ocn_thick_surface_flux_tend(meshPool, transmissionCoefficients, trans ! !----------------------------------------------------------------- - type (mpas_pool_type), intent(in) :: & - meshPool !< Input: mesh information - real (kind=RKIND), dimension(:,:), intent(in) :: & transmissionCoefficients, &!< Input: Coefficients for the transmission of surface fluxes transmissionCoefficientsRunoff !< Input: Coefficients for the transmission of surface fluxes due to river runoff @@ -115,9 +113,7 @@ subroutine ocn_thick_surface_flux_tend(meshPool, transmissionCoefficients, trans ! !----------------------------------------------------------------- - integer :: iCell, k, nCells - integer, dimension(:), pointer :: nCellsArray - integer, dimension(:), pointer :: minLevelCell, maxLevelCell + integer :: iCell, k real (kind=RKIND) :: remainingFlux, remainingFluxRunoff @@ -127,16 +123,9 @@ subroutine ocn_thick_surface_flux_tend(meshPool, transmissionCoefficients, trans call mpas_timer_start("thick surface flux") - call mpas_pool_get_array(meshPool, 'minLevelCell', minLevelCell) - call mpas_pool_get_array(meshPool, 'maxLevelCell', maxLevelCell) - - call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) - - nCells = nCellsArray( 1 ) - !$omp parallel !$omp do schedule(runtime) private(remainingFlux, remainingFluxRunoff, k) - do iCell = 1, nCells + do iCell = 1, nCellsOwned remainingFlux = 1.0_RKIND remainingFluxRunoff = 1.0_RKIND do k = minLevelCell(iCell), maxLevelCell(iCell) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_thick_vadv.F b/components/mpas-ocean/src/shared/mpas_ocn_thick_vadv.F index 233f0a4d4cdb..c789e4a3c3dc 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_thick_vadv.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_thick_vadv.F @@ -25,6 +25,7 @@ module ocn_thick_vadv use mpas_pool_routines use ocn_constants use ocn_config + use ocn_mesh implicit none private @@ -70,7 +71,7 @@ module ocn_thick_vadv ! !----------------------------------------------------------------------- - subroutine ocn_thick_vadv_tend(meshPool, vertAleTransportTop, tend, err)!{{{ + subroutine ocn_thick_vadv_tend(vertAleTransportTop, tend, err)!{{{ !----------------------------------------------------------------- ! @@ -81,9 +82,6 @@ subroutine ocn_thick_vadv_tend(meshPool, vertAleTransportTop, tend, err)!{{{ real (kind=RKIND), dimension(:,:), intent(in) :: & vertAleTransportTop !< Input: vertical velocity on top layer - type (mpas_pool_type), intent(in) :: & - meshPool !< Input: mesh information - !----------------------------------------------------------------- ! ! input/output variables @@ -107,10 +105,7 @@ subroutine ocn_thick_vadv_tend(meshPool, vertAleTransportTop, tend, err)!{{{ ! !----------------------------------------------------------------- - integer :: iCell, k, nCells - integer, pointer :: nVertLevels - integer, dimension(:), pointer :: nCellsArray - integer, dimension(:), pointer :: minLevelCell, maxLevelCell + integer :: iCell, k !----------------------------------------------------------------- ! @@ -126,17 +121,9 @@ subroutine ocn_thick_vadv_tend(meshPool, vertAleTransportTop, tend, err)!{{{ call mpas_timer_start("thick vadv") - call mpas_pool_get_array(meshPool, 'minLevelCell', minLevelCell) - call mpas_pool_get_array(meshPool, 'maxLevelCell', maxLevelCell) - - call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) - call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) - - nCells = nCellsArray( 1 ) - !$omp parallel !$omp do schedule(runtime) private(k) - do iCell = 1, nCells + do iCell = 1, nCellsOwned do k = minLevelCell(iCell), maxLevelCell(iCell) tend(k,iCell) = tend(k,iCell) + vertAleTransportTop(k+1,iCell) - vertAleTransportTop(k,iCell) end do diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F index 25e09dab00a4..926862155c6b 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F @@ -27,6 +27,7 @@ module ocn_tidal_forcing use mpas_timer use ocn_constants use ocn_config + use ocn_mesh use ocn_diagnostics_variables implicit none @@ -73,14 +74,13 @@ module ocn_tidal_forcing ! !----------------------------------------------------------------------- - subroutine ocn_tidal_forcing_layer_thickness(meshPool, forcingPool, layerThicknessTend, err)!{{{ + subroutine ocn_tidal_forcing_layer_thickness(forcingPool, layerThicknessTend, err)!{{{ !----------------------------------------------------------------- ! ! input variables ! !----------------------------------------------------------------- - type (mpas_pool_type), intent(in) :: meshPool !< Input: mesh information type (mpas_pool_type), intent(in) :: forcingPool !< Input: Forcing information !----------------------------------------------------------------- @@ -105,8 +105,6 @@ subroutine ocn_tidal_forcing_layer_thickness(meshPool, forcingPool, layerThickne !----------------------------------------------------------------- integer :: iCell, k, nCells - integer, dimension(:), pointer :: nCellsArray - integer, dimension(:), pointer :: minLevelCell, maxLevelCell real (kind=RKIND), dimension(:,:), pointer :: tidalLayerThicknessTendency err = 0 @@ -115,14 +113,11 @@ subroutine ocn_tidal_forcing_layer_thickness(meshPool, forcingPool, layerThickne call mpas_timer_start("tidal thickness tendency") - call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) - call mpas_pool_get_array(meshPool, 'minLevelCell', minLevelCell) - call mpas_pool_get_array(meshPool, 'maxLevelCell', maxLevelCell) call mpas_pool_get_array(forcingPool, 'tidalLayerThicknessTendency', & tidalLayerThicknessTendency) ! frazil fields are needed only over 0 and 1 halos - nCells = nCellsArray( 2 ) + nCells = nCellsHalo( 1 ) ! Build surface fluxes at cell centers !$omp parallel From e9088f2feabf6d15d39239ec3a744004f8d1540e Mon Sep 17 00:00:00 2001 From: Brian O'Neill Date: Fri, 11 Feb 2022 15:55:10 -0500 Subject: [PATCH 2/2] GPU port of ocn_tend_thick routines - resolve OpenACC error in mpas_ocn_diagnostics.F by copying needed arrays onto device in ocn_compute_land_ice_flux_input_fields routine - add OpenACC directives to loops in ocn_tend_thick and associated subroutines - copyout tendThick, surfaceThicknessFlux, and surfaceThicknessFluxRunoff back to host at end of ocn_tend_thick --- .../mpas-ocean/src/shared/mpas_ocn_diagnostics.F | 4 ++++ .../src/shared/mpas_ocn_frazil_forcing.F | 14 ++++++++++++++ .../src/shared/mpas_ocn_surface_bulk_forcing.F | 16 ++++++++++++++++ .../shared/mpas_ocn_surface_land_ice_fluxes.F | 11 +++++++++++ .../mpas-ocean/src/shared/mpas_ocn_tendency.F | 13 +++++++++++++ .../mpas-ocean/src/shared/mpas_ocn_thick_hadv.F | 9 +++++++++ .../src/shared/mpas_ocn_thick_surface_flux.F | 15 +++++++++++++++ .../mpas-ocean/src/shared/mpas_ocn_thick_vadv.F | 8 ++++++++ .../src/shared/mpas_ocn_tidal_forcing.F | 14 ++++++++++++++ 9 files changed, 104 insertions(+) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index f90d3aea170e..98004c85630b 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -2658,6 +2658,8 @@ subroutine ocn_compute_land_ice_flux_input_fields(layerThickness, normalVelocity ! Compute top drag #ifdef MPAS_OPENACC + !$acc enter data copyin(landIceFraction) + !$acc parallel loop present(cellsOnEdge, kineticEnergyCell, minLevelCell, minLevelEdgeBot) & !$acc present(landIceFraction, topDrag, normalVelocity) #else @@ -2808,6 +2810,8 @@ subroutine ocn_compute_land_ice_flux_input_fields(layerThickness, normalVelocity ! modify the spatially-varying attenuation coefficient where there is land ice #ifdef MPAS_OPENACC + !$acc enter data copyin(landIceMask) + !$acc parallel loop present(landIceMask, sfcFlxAttCoeff) #else !$omp parallel diff --git a/components/mpas-ocean/src/shared/mpas_ocn_frazil_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_frazil_forcing.F index a8041ef9c083..6a8c07b0fbd2 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_frazil_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_frazil_forcing.F @@ -180,15 +180,29 @@ subroutine ocn_frazil_forcing_layer_thickness(forcingPool, layerThicknessTend, e nCells = nCellsHalo( 1 ) ! Build surface fluxes at cell centers +#ifdef MPAS_OPENACC + !$acc enter data copyin(frazilLayerThicknessTendency) + + !$acc parallel loop & + !$acc present(layerThicknessTend, minLevelCell, maxLevelCell, frazilLayerThicknessTendency) & + !$acc private(k) +#else !$omp parallel !$omp do schedule(runtime) private(k) +#endif do iCell = 1, nCells do k = minLevelCell(iCell), maxLevelCell(iCell) layerThicknessTend(k,iCell) = layerThicknessTend(k,iCell) + frazilLayerThicknessTendency(k,iCell) end do end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif + +#ifdef MPAS_OPENACC + !$acc exit data delete(frazilLayerThicknessTendency) +#endif call mpas_timer_stop("frazil thickness tendency") diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F index 92fd397d01c4..7834d4509e5c 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_bulk_forcing.F @@ -328,15 +328,31 @@ subroutine ocn_surface_bulk_forcing_thick(forcingPool, surfaceThicknessFlux, sur nCells = nCellsHalo( 2 ) ! Build surface fluxes at cell centers +#ifdef MPAS_OPENACC + !$acc enter data copyin(evaporationFlux, snowFlux, seaIceFreshWaterFlux, icebergFreshWaterFlux, & + !$acc riverRunoffFlux, iceRunoffFlux, rainFlux) + + !$acc parallel loop & + !$acc present(surfaceThicknessFlux, surfaceThicknessFluxRunoff, evaporationFlux, snowFlux, & + !$acc seaIceFreshWaterFlux, icebergFreshWaterFlux, riverRunoffFlux, iceRunoffFlux, rainFlux) +#else !$omp parallel !$omp do schedule(runtime) +#endif do iCell = 1, nCells surfaceThicknessFlux(iCell) = surfaceThicknessFlux(iCell) + ( snowFlux(iCell) + rainFlux(iCell) + evaporationFlux(iCell) & + seaIceFreshWaterFlux(iCell) + icebergFreshWaterFlux(iCell) + iceRunoffFlux(iCell) ) / rho_sw surfaceThicknessFluxRunoff(iCell) = riverRunoffFlux(iCell) / rho_sw end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif + +#ifdef MPAS_OPENACC + !$acc exit data delete(evaporationFlux, snowFlux, seaIceFreshWaterFlux, icebergFreshWaterFlux, & + !$acc riverRunoffFlux, iceRunoffFlux, rainFlux) +#endif call mpas_timer_stop("bulk_thick") diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F index 96599cb7f2c1..95c86c688520 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F @@ -282,14 +282,25 @@ subroutine ocn_surface_land_ice_fluxes_thick(forcingPool, surfaceThicknessFlux, call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFlux) ! Build surface fluxes at cell centers +#ifdef MPAS_OPENACC + !$acc enter data copyin(landIceFreshwaterFlux) + + !$acc parallel loop present(surfaceThicknessFlux, landIceFreshwaterFlux) +#else !$omp parallel !$omp do schedule(runtime) +#endif do iCell = 1, nCellsAll surfaceThicknessFlux(iCell) = surfaceThicknessFlux(iCell) + landIceFreshwaterFlux(iCell) / rho_sw end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif +#ifdef MPAS_OPENACC + !$acc exit data delete(landIceFreshwaterFlux) +#endif call mpas_timer_stop("land_ice_thick") end subroutine ocn_surface_land_ice_fluxes_thick!}}} diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index b38b881b382c..3280de81ea10 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -159,9 +159,16 @@ subroutine ocn_tend_thick(tendPool, forcingPool)!{{{ ! layer thickness tendency: ! initialize to zero and start accumulating tendency terms ! +#ifdef MPAS_OPENACC + !$acc enter data create(tendThick, surfaceThicknessFlux, surfaceThicknessFluxRunoff) + !$acc parallel loop & + !$acc present(tendThick, surfaceThicknessFlux, surfaceThicknessFluxRunoff) & + !$acc private(k) +#else !$omp parallel !$omp do schedule(runtime) private(k) +#endif do iCell = 1, nCellsAll surfaceThicknessFlux(iCell) = 0.0_RKIND surfaceThicknessFluxRunoff(iCell) = 0.0_RKIND @@ -169,8 +176,10 @@ subroutine ocn_tend_thick(tendPool, forcingPool)!{{{ tendThick(k, iCell) = 0.0_RKIND end do end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif ! If turned off, return with zero fluxes, tendencies ! Otherwise, start time and call routines to accumulate @@ -213,6 +222,10 @@ subroutine ocn_tend_thick(tendPool, forcingPool)!{{{ call ocn_tidal_forcing_layer_thickness(forcingPool, & tendThick, err) +#ifdef MPAS_OPENACC + !$acc exit data copyout(tendThick, surfaceThicknessFlux, surfaceThicknessFluxRunoff) +#endif + call mpas_timer_stop("ocn_tend_thick") !-------------------------------------------------------------------- diff --git a/components/mpas-ocean/src/shared/mpas_ocn_thick_hadv.F b/components/mpas-ocean/src/shared/mpas_ocn_thick_hadv.F index 05c2b5bd4d30..4c9adf69a05d 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_thick_hadv.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_thick_hadv.F @@ -126,8 +126,15 @@ subroutine ocn_thick_hadv_tend(normalVelocity, layerThicknessEdge, tend, err)!{{ call mpas_timer_start("thick hadv") +#ifdef MPAS_OPENACC + !$acc parallel loop & + !$acc present(tend, normalVelocity, dvEdge, layerThicknessEdge, edgeSignOnCell, & + !$acc minLevelEdgeBot, maxLevelEdgeBot) & + !$acc private(i, k, invAreaCell, flux) +#else !$omp parallel !$omp do schedule(runtime) private(invAreaCell, i, iEdge, k, flux) +#endif do iCell = 1, nCellsOwned invAreaCell = 1.0_RKIND / areaCell(iCell) do i = 1, nEdgesOnCell(iCell) @@ -138,8 +145,10 @@ subroutine ocn_thick_hadv_tend(normalVelocity, layerThicknessEdge, tend, err)!{{ end do end do end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif call mpas_timer_stop("thick hadv") diff --git a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F index 60ff30e463d9..a5e3193d5147 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_thick_surface_flux.F @@ -123,8 +123,17 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe call mpas_timer_start("thick surface flux") +#ifdef MPAS_OPENACC + !$acc enter data copyin(transmissionCoefficients, transmissionCoefficientsRunoff) + + !$acc parallel loop & + !$acc present(tend, surfaceThicknessFlux, surfaceThicknessFluxRunoff, & + !$acc transmissionCoefficients, transmissionCoefficientsRunoff, minLevelCell, maxLevelCell) & + !$acc private(k, remainingFlux, remainingFluxRunoff) +#else !$omp parallel !$omp do schedule(runtime) private(remainingFlux, remainingFluxRunoff, k) +#endif do iCell = 1, nCellsOwned remainingFlux = 1.0_RKIND remainingFluxRunoff = 1.0_RKIND @@ -145,8 +154,14 @@ subroutine ocn_thick_surface_flux_tend(transmissionCoefficients, transmissionCoe + remainingFluxRunoff * surfaceThicknessFluxRunoff(iCell) end if end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif + +#ifdef MPAS_OPENACC + !$acc exit data delete(transmissionCoefficients, transmissionCoefficientsRunoff) +#endif call mpas_timer_stop("thick surface flux") diff --git a/components/mpas-ocean/src/shared/mpas_ocn_thick_vadv.F b/components/mpas-ocean/src/shared/mpas_ocn_thick_vadv.F index c789e4a3c3dc..067dcdeec6c3 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_thick_vadv.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_thick_vadv.F @@ -121,15 +121,23 @@ subroutine ocn_thick_vadv_tend(vertAleTransportTop, tend, err)!{{{ call mpas_timer_start("thick vadv") +#ifdef MPAS_OPENACC + !$acc parallel loop & + !$acc present(tend, vertAleTransportTop, minLevelCell, maxLevelCell) & + !$acc private(k) +#else !$omp parallel !$omp do schedule(runtime) private(k) +#endif do iCell = 1, nCellsOwned do k = minLevelCell(iCell), maxLevelCell(iCell) tend(k,iCell) = tend(k,iCell) + vertAleTransportTop(k+1,iCell) - vertAleTransportTop(k,iCell) end do end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif call mpas_timer_stop("thick vadv") diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F index 926862155c6b..bf9b98142fb4 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F @@ -120,8 +120,16 @@ subroutine ocn_tidal_forcing_layer_thickness(forcingPool, layerThicknessTend, er nCells = nCellsHalo( 1 ) ! Build surface fluxes at cell centers +#ifdef MPAS_OPENACC + !$acc enter data copyin(tidalLayerThicknessTendency) + + !$acc parallel loop present(layerThicknessTend, maxLevelCell, minLevelCell, & + !$acc tidalLayerThicknessTendency) & + !$acc private(k) +#else !$omp parallel !$omp do schedule(runtime) private(k) +#endif do iCell = 1, nCells do k = minLevelCell(iCell), maxLevelCell(iCell) layerThicknessTend(k,iCell) = layerThicknessTend(k,iCell) + & @@ -129,8 +137,14 @@ subroutine ocn_tidal_forcing_layer_thickness(forcingPool, layerThicknessTend, er end do end do +#ifndef MPAS_OPENACC !$omp end do !$omp end parallel +#endif + +#ifdef MPAS_OPENACC + !$acc exit data delete(tidalLayerThicknessTendency) +#endif call mpas_timer_stop("tidal thickness tendency")