Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update OMP #680

Merged
merged 15 commits into from
Feb 18, 2022
Merged

Update OMP #680

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
3003c67
Add narwhal intel, gnu, cray, aocc
apcraig Dec 30, 2021
3525b14
update narwhal_cray and perf_suite
apcraig Dec 31, 2021
80e8caa
Review and update OMP implementation
apcraig Jan 7, 2022
deda037
Merge branch 'ompB' of https://github.com/apcraig/cice into ompC
apcraig Jan 7, 2022
9c442bc
- Review diagnostics OMP implementation
apcraig Jan 10, 2022
3446cf6
Update EVP OMP implementation
apcraig Jan 11, 2022
f91f459
- Refactor puny/pi scalars in eap dynamics to improve performance
apcraig Jan 11, 2022
7905bf6
Clean up
apcraig Jan 12, 2022
667e998
Comment out temporary timers
apcraig Jan 12, 2022
6bb8e22
Update OMP env variables on Narwhal
apcraig Jan 13, 2022
0d4c2e3
Update gaffney OMP_STACKSIZE
apcraig Jan 14, 2022
d56737c
update OMP_STACKSIZE on cori
apcraig Jan 14, 2022
021e0f3
Update Onyx OMP_STACKSIZE
apcraig Jan 14, 2022
3a00b69
Update OMP_STACKSIZE on mustang
apcraig Jan 17, 2022
c8b9f17
- Update Tsfc values on land in various places in the code, was affec…
apcraig Jan 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions cicecore/cicedynB/dynamics/ice_dyn_eap.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ subroutine eap (dt)
! tcraig, tcx, turned off this threaded region, in evp, this block and
! the icepack_ice_strength call seems to not be thread safe. more
! debugging needed
!$TCXOMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block)
!$TCXOMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block,ij)
do iblk = 1, nblocks

!-----------------------------------------------------------------
Expand Down Expand Up @@ -429,7 +429,7 @@ subroutine eap (dt)
!$TCXOMP PARALLEL DO PRIVATE(iblk,strtmp)
do iblk = 1, nblocks

! call ice_timer_start(timer_tmp1) ! dynamics
! call ice_timer_start(timer_tmp1,iblk) ! dynamics
call stress_eap (nx_block, ny_block, &
ksub, ndte, &
icellt(iblk), &
Expand Down Expand Up @@ -462,7 +462,7 @@ subroutine eap (dt)
! rdg_conv (:,:,iblk), rdg_shear (:,:,iblk), &
rdg_conv (:,:,iblk), &
strtmp (:,:,:))
! call ice_timer_stop(timer_tmp1) ! dynamics
! call ice_timer_stop(timer_tmp1,iblk) ! dynamics

!-----------------------------------------------------------------
! momentum equation
Expand All @@ -488,7 +488,7 @@ subroutine eap (dt)
! evolution of structure tensor A
!-----------------------------------------------------------------

! call ice_timer_start(timer_tmp3) ! dynamics
! call ice_timer_start(timer_tmp3,iblk) ! dynamics
if (mod(ksub,10) == 1) then ! only called every 10th timestep
call stepa (nx_block, ny_block, &
dtei, icellt (iblk), &
Expand All @@ -505,7 +505,7 @@ subroutine eap (dt)
stress12_1(:,:,iblk), stress12_2(:,:,iblk), &
stress12_3(:,:,iblk), stress12_4(:,:,iblk))
endif
! call ice_timer_stop(timer_tmp3) ! dynamics
! call ice_timer_stop(timer_tmp3,iblk) ! dynamics
enddo
!$TCXOMP END PARALLEL DO

Expand Down
2 changes: 1 addition & 1 deletion cicecore/cicedynB/dynamics/ice_dyn_evp.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,7 @@ subroutine evp (dt)

! tcraig, tcx, threading here leads to some non-reproducbile results and failures in icepack_ice_strength
! need to do more debugging
!$TCXOMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block)
!$TCXOMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,ij,i,j)
do iblk = 1, nblocks

!-----------------------------------------------------------------
Expand Down
32 changes: 16 additions & 16 deletions cicecore/cicedynB/dynamics/ice_dyn_vp.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@ subroutine implicit_solver (dt)

! tcraig, tcx, threading here leads to some non-reproducbile results and failures in icepack_ice_strength
! need to do more debugging
!$TCXOMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block)
!$TCXOMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block,ij,i,j)
do iblk = 1, nblocks

!-----------------------------------------------------------------
Expand Down Expand Up @@ -814,7 +814,7 @@ subroutine anderson_solver (icellt , icellu, &
!-----------------------------------------------------------------
! Calc zetax2, etax2, dPr/dx, dPr/dy, Cb and vrel = f(uprev_k, vprev_k)
!-----------------------------------------------------------------
!$OMP PARALLEL DO PRIVATE(iblk)
!$OMP PARALLEL DO PRIVATE(iblk,stress_Pr)
do iblk = 1, nblocks

if (use_mean_vrel) then
Expand Down Expand Up @@ -912,7 +912,7 @@ subroutine anderson_solver (icellt , icellu, &

! Prepare diagonal for preconditioner
if (precond == 'diag' .or. precond == 'pgmres') then
!$OMP PARALLEL DO PRIVATE(iblk)
!$OMP PARALLEL DO PRIVATE(iblk,diag_rheo)
do iblk = 1, nblocks
! first compute diagonal contributions due to rheology term
call formDiag_step1 (nx_block , ny_block , &
Expand Down Expand Up @@ -2851,7 +2851,7 @@ subroutine fgmres (zetax2 , etax2 , &

! Normalize the first Arnoldi vector
inverse_norm = c1 / norm_residual
!$OMP PARALLEL DO PRIVATE(iblk)
!$OMP PARALLEL DO PRIVATE(iblk,ij,i,j)
do iblk = 1, nblocks
do ij = 1, icellu(iblk)
i = indxui(ij, iblk)
Expand Down Expand Up @@ -2947,7 +2947,7 @@ subroutine fgmres (zetax2 , etax2 , &
if (.not. almost_zero( hessenberg(nextit,initer) ) ) then
! Normalize next Arnoldi vector
inverse_norm = c1 / hessenberg(nextit,initer)
!$OMP PARALLEL DO PRIVATE(iblk)
!$OMP PARALLEL DO PRIVATE(iblk,ij,i,j)
do iblk = 1, nblocks
do ij = 1, icellu(iblk)
i = indxui(ij, iblk)
Expand Down Expand Up @@ -3013,7 +3013,7 @@ subroutine fgmres (zetax2 , etax2 , &
! Form linear combination to get new solution iterate
do it = 1, initer
t = rhs_hess(it)
!$OMP PARALLEL DO PRIVATE(iblk)
!$OMP PARALLEL DO PRIVATE(iblk,ij,i,j)
do iblk = 1, nblocks
do ij = 1, icellu(iblk)
i = indxui(ij, iblk)
Expand Down Expand Up @@ -3057,7 +3057,7 @@ subroutine fgmres (zetax2 , etax2 , &
workspace_x = c0
workspace_y = c0
do it = 1, nextit
!$OMP PARALLEL DO PRIVATE(iblk)
!$OMP PARALLEL DO PRIVATE(iblk,ij,i,j)
do iblk = 1, nblocks
do ij = 1, icellu(iblk)
i = indxui(ij, iblk)
Expand Down Expand Up @@ -3244,7 +3244,7 @@ subroutine pgmres (zetax2 , etax2 , &

! Normalize the first Arnoldi vector
inverse_norm = c1 / norm_residual
!$OMP PARALLEL DO PRIVATE(iblk)
!$OMP PARALLEL DO PRIVATE(iblk,ij,i,j)
do iblk = 1, nblocks
do ij = 1, icellu(iblk)
i = indxui(ij, iblk)
Expand Down Expand Up @@ -3329,7 +3329,7 @@ subroutine pgmres (zetax2 , etax2 , &
if (.not. almost_zero( hessenberg(nextit,initer) ) ) then
! Normalize next Arnoldi vector
inverse_norm = c1 / hessenberg(nextit,initer)
!$OMP PARALLEL DO PRIVATE(iblk)
!$OMP PARALLEL DO PRIVATE(iblk,ij,i,j)
do iblk = 1, nblocks
do ij = 1, icellu(iblk)
i = indxui(ij, iblk)
Expand Down Expand Up @@ -3397,7 +3397,7 @@ subroutine pgmres (zetax2 , etax2 , &
workspace_y = c0
do it = 1, initer
t = rhs_hess(it)
!$OMP PARALLEL DO PRIVATE(iblk)
!$OMP PARALLEL DO PRIVATE(iblk,ij,i,j)
do iblk = 1, nblocks
do ij = 1, icellu(iblk)
i = indxui(ij, iblk)
Expand Down Expand Up @@ -3453,7 +3453,7 @@ subroutine pgmres (zetax2 , etax2 , &
workspace_x = c0
workspace_y = c0
do it = 1, nextit
!$OMP PARALLEL DO PRIVATE(iblk)
!$OMP PARALLEL DO PRIVATE(iblk,ij,i,j)
do iblk = 1, nblocks
do ij = 1, icellu(iblk)
i = indxui(ij, iblk)
Expand Down Expand Up @@ -3534,7 +3534,7 @@ subroutine precondition(zetax2 , etax2, &
wx = vx
wy = vy
elseif (precond_type == 'diag') then ! Jacobi preconditioner (diagonal)
!$OMP PARALLEL DO PRIVATE(iblk)
!$OMP PARALLEL DO PRIVATE(iblk,ij,i,j)
do iblk = 1, nblocks
do ij = 1, icellu(iblk)
i = indxui(ij, iblk)
Expand Down Expand Up @@ -3617,7 +3617,7 @@ subroutine orthogonalize(ortho_type , initer , &
do it = 1, initer
local_dot = c0

!$OMP PARALLEL DO PRIVATE(iblk, ij, i, j)
!$OMP PARALLEL DO PRIVATE(iblk,ij,i,j)
do iblk = 1, nblocks
do ij = 1, icellu(iblk)
i = indxui(ij, iblk)
Expand All @@ -3637,7 +3637,7 @@ subroutine orthogonalize(ortho_type , initer , &

! Second loop of Gram-Schmidt (orthonormalize)
do it = 1, initer
!$OMP PARALLEL DO PRIVATE(iblk)
!$OMP PARALLEL DO PRIVATE(iblk,ij,i,j)
do iblk = 1, nblocks
do ij = 1, icellu(iblk)
i = indxui(ij, iblk)
Expand All @@ -3656,7 +3656,7 @@ subroutine orthogonalize(ortho_type , initer , &
do it = 1, initer
local_dot = c0

!$OMP PARALLEL DO PRIVATE(iblk, ij, i, j)
!$OMP PARALLEL DO PRIVATE(iblk,ij,i,j)
do iblk = 1, nblocks
do ij = 1, icellu(iblk)
i = indxui(ij, iblk)
Expand All @@ -3671,7 +3671,7 @@ subroutine orthogonalize(ortho_type , initer , &

hessenberg(it,initer) = global_sum(sum(local_dot), distrb_info)

!$OMP PARALLEL DO PRIVATE(iblk, ij, i, j)
!$OMP PARALLEL DO PRIVATE(iblk,ij,i,j)
do iblk = 1, nblocks
do ij = 1, icellu(iblk)
i = indxui(ij, iblk)
Expand Down
Loading