From b176bf546c43e167435cd55703c0798187b94e31 Mon Sep 17 00:00:00 2001 From: anton-climate Date: Tue, 30 Apr 2024 16:37:13 +1000 Subject: [PATCH 1/2] Deprecate nprocs, and better estimate for max_blocks if not set in ice_in Co-authored-by: minghangli-uni --- CICE/CMakeLists.txt | 5 +- CICE/patches/ice_distribution.F90.patch | 16 ++++++ CICE/patches/ice_domain.F90.patch | 71 +++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 CICE/patches/ice_distribution.F90.patch create mode 100644 CICE/patches/ice_domain.F90.patch diff --git a/CICE/CMakeLists.txt b/CICE/CMakeLists.txt index 25c67cf..637d520 100644 --- a/CICE/CMakeLists.txt +++ b/CICE/CMakeLists.txt @@ -26,7 +26,6 @@ target_sources(OM3_cice PRIVATE CICE/cicecore/shared/ice_arrays_column.F90 CICE/cicecore/shared/ice_calendar.F90 CICE/cicecore/shared/ice_constants.F90 - CICE/cicecore/shared/ice_distribution.F90 CICE/cicecore/shared/ice_domain_size.F90 CICE/cicecore/shared/ice_fileunits.F90 CICE/cicecore/shared/ice_init_column.F90 @@ -68,7 +67,6 @@ target_sources(OM3_cice PRIVATE # Infrastructure CICE/cicecore/cicedyn/infrastructure/ice_blocks.F90 - CICE/cicecore/cicedyn/infrastructure/ice_domain.F90 CICE/cicecore/cicedyn/infrastructure/ice_grid.F90 CICE/cicecore/cicedyn/infrastructure/ice_memusage.F90 CICE/cicecore/cicedyn/infrastructure/ice_memusage_gptl.c @@ -155,6 +153,9 @@ elseif(OM3_CICE_IO MATCHES "Binary") ) endif() +add_patched_source(OM3_cice CICE/cicecore/cicedyn/infrastructure/ice_domain.F90) +add_patched_source(OM3_cice CICE/cicecore/shared/ice_distribution.F90) + ### Install and Export ## Library diff --git a/CICE/patches/ice_distribution.F90.patch b/CICE/patches/ice_distribution.F90.patch new file mode 100644 index 0000000..fccefe2 --- /dev/null +++ b/CICE/patches/ice_distribution.F90.patch @@ -0,0 +1,16 @@ +diff --git a/cicecore/shared/ice_distribution.F90 b/cicecore/shared/ice_distribution.F90 +old mode 100644 +new mode 100755 +index 0f3f6c1..6e06069 +--- a/cicecore/shared/ice_distribution.F90 ++++ b/cicecore/shared/ice_distribution.F90 +@@ -41,7 +41,8 @@ module ice_distribution + ice_distributionGet, & + ice_distributionGetBlockLoc, & + ice_distributionGetBlockID, & +- create_local_block_ids ++ create_local_block_ids, & ++ proc_decomposition + + character (char_len), public :: & + processor_shape ! 'square-pop' (approx) POP default config diff --git a/CICE/patches/ice_domain.F90.patch b/CICE/patches/ice_domain.F90.patch new file mode 100644 index 0000000..34cda0e --- /dev/null +++ b/CICE/patches/ice_domain.F90.patch @@ -0,0 +1,71 @@ +diff --git a/cicecore/cicedyn/infrastructure/ice_domain.F90 b/cicecore/cicedyn/infrastructure/ice_domain.F90 +index 8b680f2..cf13fcc 100644 +--- a/cicecore/cicedyn/infrastructure/ice_domain.F90 ++++ b/cicecore/cicedyn/infrastructure/ice_domain.F90 +@@ -88,7 +88,7 @@ module ice_domain + distribution_wght_file ! file for distribution_wght=file + + integer (int_kind) :: & +- nprocs ! num of processors ++ nprocs, nprocs_x, nprocs_y ! num of processors + + !*********************************************************************** + +@@ -101,7 +101,7 @@ subroutine init_domain_blocks + ! This routine reads in domain information and calls the routine + ! to set up the block decomposition. + +- use ice_distribution, only: processor_shape ++ use ice_distribution, only: processor_shape, proc_decomposition + use ice_domain_size, only: ncat, nilyr, nslyr, max_blocks, & + nx_global, ny_global, block_size_x, block_size_y + use ice_fileunits, only: goto_nml +@@ -204,6 +204,14 @@ subroutine init_domain_blocks + + endif + ++ if (nprocs .ne. -1) then ++ if (my_task == master_task) then ++ write(nu_diag,*) subname//' WARNING: nprocs is deprecated, please remove from namelist' ++ endif ++ endif ++ ++ nprocs = get_num_procs() ++ + call broadcast_scalar(nprocs, master_task) + call broadcast_scalar(processor_shape, master_task) + call broadcast_scalar(distribution_type, master_task) +@@ -216,10 +224,14 @@ subroutine init_domain_blocks + call broadcast_scalar(maskhalo_bound, master_task) + call broadcast_scalar(add_mpi_barriers, master_task) + call broadcast_scalar(debug_blocks, master_task) ++ ++ ! update nprocs_x and nprocs_y ++ call proc_decomposition(nprocs, nprocs_x, nprocs_y) ++ ! set max_blocks + if (my_task == master_task) then + if (max_blocks < 1) then +- max_blocks=( ((nx_global-1)/block_size_x + 1) * & +- ((ny_global-1)/block_size_y + 1) - 1) / nprocs + 1 ++ max_blocks=((nx_global-1)/block_size_x/nprocs_x+1) * & ++ ((ny_global-1)/block_size_y/nprocs_y+1) + max_blocks=max(1,max_blocks) + write(nu_diag,'(/,a52,i6,/)') & + '(ice_domain): max_block < 1: max_block estimated to ',max_blocks +@@ -242,16 +254,6 @@ subroutine init_domain_blocks + !*** domain size zero or negative + !*** + call abort_ice(subname//' ERROR: Invalid domain: size < 1', file=__FILE__, line=__LINE__) ! no domain +- else if (nprocs /= get_num_procs()) then +- !*** +- !*** input nprocs does not match system (eg MPI) request +- !*** +-#if (defined CESMCOUPLED) +- nprocs = get_num_procs() +-#else +- write(nu_diag,*) subname,' ERROR: nprocs, get_num_procs = ',nprocs,get_num_procs() +- call abort_ice(subname//' ERROR: Input nprocs not same as system request', file=__FILE__, line=__LINE__) +-#endif + else if (nghost < 1) then + !*** + !*** must have at least 1 layer of ghost cells From 8122289d3fc8d6374dc4f65f08ce82cf2f9ba5f6 Mon Sep 17 00:00:00 2001 From: anton-climate Date: Fri, 3 May 2024 11:17:00 +1000 Subject: [PATCH 2/2] Review feedback --- CICE/patches/ice_domain.F90.patch | 82 ++++++++++++++++++------------- 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/CICE/patches/ice_domain.F90.patch b/CICE/patches/ice_domain.F90.patch index 34cda0e..f769ddd 100644 --- a/CICE/patches/ice_domain.F90.patch +++ b/CICE/patches/ice_domain.F90.patch @@ -1,16 +1,7 @@ diff --git a/cicecore/cicedyn/infrastructure/ice_domain.F90 b/cicecore/cicedyn/infrastructure/ice_domain.F90 -index 8b680f2..cf13fcc 100644 +index 8b680f2..47a0f00 100644 --- a/cicecore/cicedyn/infrastructure/ice_domain.F90 +++ b/cicecore/cicedyn/infrastructure/ice_domain.F90 -@@ -88,7 +88,7 @@ module ice_domain - distribution_wght_file ! file for distribution_wght=file - - integer (int_kind) :: & -- nprocs ! num of processors -+ nprocs, nprocs_x, nprocs_y ! num of processors - - !*********************************************************************** - @@ -101,7 +101,7 @@ subroutine init_domain_blocks ! This routine reads in domain information and calls the routine ! to set up the block decomposition. @@ -20,39 +11,60 @@ index 8b680f2..cf13fcc 100644 use ice_domain_size, only: ncat, nilyr, nslyr, max_blocks, & nx_global, ny_global, block_size_x, block_size_y use ice_fileunits, only: goto_nml -@@ -204,6 +204,14 @@ subroutine init_domain_blocks +@@ -112,7 +112,8 @@ subroutine init_domain_blocks + !---------------------------------------------------------------------- - endif + integer (int_kind) :: & +- nml_error ! namelist read error flag ++ nml_error, & ! namelist read error flag ++ nprocs_x, nprocs_y ! procs decomposed into blocks -+ if (nprocs .ne. -1) then -+ if (my_task == master_task) then -+ write(nu_diag,*) subname//' WARNING: nprocs is deprecated, please remove from namelist' -+ endif -+ endif -+ -+ nprocs = get_num_procs() -+ - call broadcast_scalar(nprocs, master_task) - call broadcast_scalar(processor_shape, master_task) - call broadcast_scalar(distribution_type, master_task) -@@ -216,10 +224,14 @@ subroutine init_domain_blocks + character(len=char_len) :: nml_name ! text namelist name + character(len=char_len_long) :: tmpstr2 ! for namelist check +@@ -216,21 +217,33 @@ subroutine init_domain_blocks call broadcast_scalar(maskhalo_bound, master_task) call broadcast_scalar(add_mpi_barriers, master_task) call broadcast_scalar(debug_blocks, master_task) -+ -+ ! update nprocs_x and nprocs_y -+ call proc_decomposition(nprocs, nprocs_x, nprocs_y) -+ ! set max_blocks - if (my_task == master_task) then - if (max_blocks < 1) then +- if (my_task == master_task) then +- if (max_blocks < 1) then - max_blocks=( ((nx_global-1)/block_size_x + 1) * & - ((ny_global-1)/block_size_y + 1) - 1) / nprocs + 1 -+ max_blocks=((nx_global-1)/block_size_x/nprocs_x+1) * & +- max_blocks=max(1,max_blocks) +- write(nu_diag,'(/,a52,i6,/)') & +- '(ice_domain): max_block < 1: max_block estimated to ',max_blocks +- endif +- endif + call broadcast_scalar(max_blocks, master_task) + call broadcast_scalar(block_size_x, master_task) + call broadcast_scalar(block_size_y, master_task) + call broadcast_scalar(nx_global, master_task) + call broadcast_scalar(ny_global, master_task) + ++ ! Set nprocs if not set in namelist ++ if (nprocs == -1) then ++ nprocs = get_num_procs() ++ else if (nprocs /= get_num_procs()) then ++ write(nu_diag,*) subname,' ERROR: nprocs, get_num_procs = ',nprocs,get_num_procs() ++ call abort_ice(subname//' ERROR: Input nprocs not same as system request', file=__FILE__, line=__LINE__) ++ endif ++ ++ ! Determine max_blocks if not set ++ if (max_blocks < 1) then ++ call proc_decomposition(nprocs, nprocs_x, nprocs_y) ++ max_blocks=((nx_global-1)/block_size_x/nprocs_x+1) * & + ((ny_global-1)/block_size_y/nprocs_y+1) - max_blocks=max(1,max_blocks) - write(nu_diag,'(/,a52,i6,/)') & - '(ice_domain): max_block < 1: max_block estimated to ',max_blocks -@@ -242,16 +254,6 @@ subroutine init_domain_blocks ++ max_blocks=max(1,max_blocks) ++ if (my_task == master_task) then ++ write(nu_diag,'(/,a52,i6,/)') & ++ '(ice_domain): max_block < 1: max_block estimated to ',max_blocks ++ endif ++ endif ++ ++ + !---------------------------------------------------------------------- + ! + ! perform some basic checks on domain +@@ -242,16 +255,6 @@ subroutine init_domain_blocks !*** domain size zero or negative !*** call abort_ice(subname//' ERROR: Invalid domain: size < 1', file=__FILE__, line=__LINE__) ! no domain