Skip to content

Commit

Permalink
Merge pull request #397 from ESMCI/jayeshkrishna/pio/more_pio_rearr_opts
Browse files Browse the repository at this point in the history 
Add more pio (pio1) rearranger options
  • Loading branch information
@jedwards4b
jedwards4b authored Aug 15, 2016
2 parents dab4528 + eed3ba1 commit 6be24a0
Show file tree
Hide file tree
Showing 8 changed files with 210 additions and 78 deletions.
9 changes: 6 additions & 3 deletions driver_cpl/bld/build-namelist
View file
Original file line number Diff line number Diff line change
Expand Up @@ -758,9 +758,12 @@ add_default($nl, 'pio_buffer_size_limit');
if($opts{pio_version} == 1){
add_default($nl, 'pio_rearr_comm_type', 'PIO_REARR_COMM_TYPE'=>"$xmlvars{'PIO_REARR_COMM_TYPE'}");
add_default($nl, 'pio_rearr_comm_fcd', 'PIO_REARR_COMM_FCD'=>"$xmlvars{'PIO_REARR_COMM_FCD'}");
add_default($nl, 'pio_rearr_comm_max_pend_req', 'PIO_REARR_COMM_MAX_PEND_REQ'=>"$xmlvars{'PIO_REARR_COMM_MAX_PEND_REQ'}");
add_default($nl, 'pio_rearr_comm_enable_hs', 'PIO_REARR_COMM_ENABLE_HS'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_HS'}");
add_default($nl, 'pio_rearr_comm_enable_isend', 'PIO_REARR_COMM_ENABLE_ISEND'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_ISEND'}");
add_default($nl, 'pio_rearr_comm_max_pend_req_comp2io', 'PIO_REARR_COMM_MAX_PEND_REQ_COMP2IO'=>"$xmlvars{'PIO_REARR_COMM_MAX_PEND_REQ_COMP2IO'}");
add_default($nl, 'pio_rearr_comm_enable_hs_comp2io', 'PIO_REARR_COMM_ENABLE_HS_COMP2IO'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_HS_COMP2IO'}");
add_default($nl, 'pio_rearr_comm_enable_isend_comp2io', 'PIO_REARR_COMM_ENABLE_ISEND_COMP2IO'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_ISEND_COMP2IO'}");
add_default($nl, 'pio_rearr_comm_max_pend_req_io2comp', 'PIO_REARR_COMM_MAX_PEND_REQ_IO2COMP'=>"$xmlvars{'PIO_REARR_COMM_MAX_PEND_REQ_IO2COMP'}");
add_default($nl, 'pio_rearr_comm_enable_hs_io2comp', 'PIO_REARR_COMM_ENABLE_HS_IO2COMP'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_HS_IO2COMP'}");
add_default($nl, 'pio_rearr_comm_enable_isend_io2comp', 'PIO_REARR_COMM_ENABLE_ISEND_IO2COMP'=>"$xmlvars{'PIO_REARR_COMM_ENABLE_ISEND_IO2COMP'}");
}
# Note: pio_async_interface=.true. is not yet supported
# If pio_async_interface is .true. or {component}_PIO_* variable is not set or set to -99
Expand Down
15 changes: 10 additions & 5 deletions driver_cpl/bld/namelist_files/namelist_defaults_drv.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,11 +180,16 @@
<pio_async_interface PIO_ASYNC_INTERFACE="FALSE">.false.</pio_async_interface>
<pio_rearr_comm_type >$PIO_REARR_COMM_TYPE</pio_rearr_comm_type>
<pio_rearr_comm_fcd >$PIO_REARR_COMM_FCD</pio_rearr_comm_fcd>
<pio_rearr_comm_max_pend_req>$PIO_REARR_COMM_MAX_PEND_REQ</pio_rearr_comm_max_pend_req>
<pio_rearr_comm_enable_hs PIO_REARR_COMM_ENABLE_HS="TRUE">.true.</pio_rearr_comm_enable_hs>
<pio_rearr_comm_enable_hs PIO_REARR_COMM_ENABLE_HS="FALSE">.false.</pio_rearr_comm_enable_hs>
<pio_rearr_comm_enable_isend PIO_REARR_COMM_ENABLE_ISEND="TRUE">.true.</pio_rearr_comm_enable_isend>
<pio_rearr_comm_enable_isend PIO_REARR_COMM_ENABLE_ISEND="FALSE">.false.</pio_rearr_comm_enable_isend>
<pio_rearr_comm_max_pend_req_comp2io>$PIO_REARR_COMM_MAX_PEND_REQ_COMP2IO</pio_rearr_comm_max_pend_req_comp2io>
<pio_rearr_comm_enable_hs_comp2io PIO_REARR_COMM_ENABLE_HS_COMP2IO="TRUE">.true.</pio_rearr_comm_enable_hs_comp2io>
<pio_rearr_comm_enable_hs_comp2io PIO_REARR_COMM_ENABLE_HS_COMP2IO="FALSE">.false.</pio_rearr_comm_enable_hs_comp2io>
<pio_rearr_comm_enable_isend_comp2io PIO_REARR_COMM_ENABLE_ISEND_COMP2IO="TRUE">.true.</pio_rearr_comm_enable_isend_comp2io>
<pio_rearr_comm_enable_isend_comp2io PIO_REARR_COMM_ENABLE_ISEND_COMP2IO="FALSE">.false.</pio_rearr_comm_enable_isend_comp2io>
<pio_rearr_comm_max_pend_req_io2comp>$PIO_REARR_COMM_MAX_PEND_REQ_IO2COMP</pio_rearr_comm_max_pend_req_io2comp>
<pio_rearr_comm_enable_hs_io2comp PIO_REARR_COMM_ENABLE_HS_IO2COMP="TRUE">.true.</pio_rearr_comm_enable_hs_io2comp>
<pio_rearr_comm_enable_hs_io2comp PIO_REARR_COMM_ENABLE_HS_IO2COMP="FALSE">.false.</pio_rearr_comm_enable_hs_io2comp>
<pio_rearr_comm_enable_isend_io2comp PIO_REARR_COMM_ENABLE_ISEND_IO2COMP="TRUE">.true.</pio_rearr_comm_enable_isend_io2comp>
<pio_rearr_comm_enable_isend_io2comp PIO_REARR_COMM_ENABLE_ISEND_IO2COMP="FALSE">.false.</pio_rearr_comm_enable_isend_io2comp>

<!-- ========================================= -->
<!--- seq_maps.rc -->
Expand Down
38 changes: 32 additions & 6 deletions driver_cpl/bld/namelist_files/namelist_definition_drv.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2116,28 +2116,54 @@ valid values: 2denable,io2comp,comp2io,disable,default
</entry>

<entry
id="pio_rearr_comm_max_pend_req"
id="pio_rearr_comm_max_pend_req_comp2io"
type="integer"
category="pio"
group="pio_default_inparm">
pio rearranger communication max pending req
pio rearranger communication max pending req (comp2io)
</entry>

<entry
id="pio_rearr_comm_enable_hs"
id="pio_rearr_comm_enable_hs_comp2io"
type="logical"
category="pio"
group="pio_default_inparm">
pio rearranger communication option: Enable handshake
pio rearranger communication option: Enable handshake (comp2io)
default: .true.
</entry>

<entry
id="pio_rearr_comm_enable_isend"
id="pio_rearr_comm_enable_isend_comp2io"
type="logical"
category="pio"
group="pio_default_inparm">
pio rearranger communication option: Enable isends
pio rearranger communication option: Enable isends (comp2io)
default: .false.
</entry>

<entry
id="pio_rearr_comm_max_pend_req_io2comp"
type="integer"
category="pio"
group="pio_default_inparm">
pio rearranger communication max pending req (io2comp)
</entry>

<entry
id="pio_rearr_comm_enable_hs_io2comp"
type="logical"
category="pio"
group="pio_default_inparm">
pio rearranger communication option: Enable handshake (io2comp)
default: .true.
</entry>

<entry
id="pio_rearr_comm_enable_isend_io2comp"
type="logical"
category="pio"
group="pio_default_inparm">
pio rearranger communication option: Enable isends (io2comp)
default: .false.
</entry>

Expand Down
43 changes: 36 additions & 7 deletions driver_cpl/cime_config/config_component.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2417,31 +2417,60 @@
<desc>pio rearranger communication flow control direction</desc>
</entry>

<entry id="PIO_REARR_COMM_MAX_PEND_REQ">
<entry id="PIO_REARR_COMM_MAX_PEND_REQ_COMP2IO">
<type>integer</type>
<valid_values></valid_values>
<default_value>64</default_value>
<default_value>0</default_value>
<group>run_pio</group>
<file>env_run.xml</file>
<desc>pio rearranger communication max pending requests</desc>
<desc>pio rearranger communication max pending requests (comp2io) : 0 implies that CIME internally calculates the value ( = max(64, 2 * PIO_NUMTASKS) ), -1 implies no bound on max pending requests </desc>
</entry>

<entry id="PIO_REARR_COMM_ENABLE_HS">
<entry id="PIO_REARR_COMM_ENABLE_HS_COMP2IO">
<type>logical</type>
<valid_values>TRUE,FALSE</valid_values>
<default_value>TRUE</default_value>
<group>run_pio</group>
<file>env_run.xml</file>
<desc>pio rearranger communiation options: TRUE implies enable handshake</desc>
<desc>pio rearranger communiation options (comp2io) : TRUE implies enable handshake </desc>
</entry>

<entry id="PIO_REARR_COMM_ENABLE_ISEND">
<entry id="PIO_REARR_COMM_ENABLE_ISEND_COMP2IO">
<type>logical</type>
<valid_values>TRUE,FALSE</valid_values>
<default_value>FALSE</default_value>
<group>run_pio</group>
<file>env_run.xml</file>
<desc>pio rearranger communiation options: TRUE implies enable isend</desc>
<desc>pio rearranger communiation options (comp2io) : TRUE implies enable isend</desc>
</entry>

<entry id="PIO_REARR_COMM_MAX_PEND_REQ_IO2COMP">
<type>integer</type>
<valid_values></valid_values>
<default_value>64</default_value>
<group>run_pio</group>
<file>env_run.xml</file>
<desc>pio rearranger communication max pending requests (io2comp) : -1 implies no bound on max pending requests </desc>
</entry>

<!-- Handshaking is a very bad idea for large process counts and for -->
<!-- io2comp (but important for comp2io -->
<entry id="PIO_REARR_COMM_ENABLE_HS_IO2COMP">
<type>logical</type>
<valid_values>TRUE,FALSE</valid_values>
<default_value>FALSE</default_value>
<group>run_pio</group>
<file>env_run.xml</file>
<desc>pio rearranger communiation options (io2comp) : TRUE implies enable handshake</desc>
</entry>

<entry id="PIO_REARR_COMM_ENABLE_ISEND_IO2COMP">
<type>logical</type>
<valid_values>TRUE,FALSE</valid_values>
<default_value>TRUE</default_value>
<group>run_pio</group>
<file>env_run.xml</file>
<desc>pio rearranger communiation options (io2comp) : TRUE implies enable isend</desc>
</entry>

<entry id="PIO_TYPENAME">
Expand Down
20 changes: 10 additions & 10 deletions externals/pio1/pio/box_rearrange.F90.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,8 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, &

! The rearranger options in IODESC overrides the defaults
if(IOsystem%rearr_opts%comm_type == PIO_rearr_comm_p2p) then
if( (IOsystem%rearr_opts%comm_fc_opts%fcd == PIO_rearr_comm_fc_2d_disable) .or.&
(IOsystem%rearr_opts%comm_fc_opts%fcd == PIO_rearr_comm_fc_1d_io2comp) ) then
if( (IOsystem%rearr_opts%fcd == PIO_rearr_comm_fc_2d_disable) .or.&
(IOsystem%rearr_opts%fcd == PIO_rearr_comm_fc_1d_io2comp) ) then
pio_option = POINT_TO_POINT
else
pio_option = FLOW_CONTROL
Expand All @@ -210,9 +210,9 @@ subroutine box_rearrange_comp2io_{TYPE} (IOsystem, ioDesc, s1, src, niodof, &
endif

if (pio_option == FLOW_CONTROL) then
pio_hs = IOsystem%rearr_opts%comm_fc_opts%enable_hs
pio_isend = IOsystem%rearr_opts%comm_fc_opts%enable_isend
pio_maxreq = IOsystem%rearr_opts%comm_fc_opts%max_pend_req
pio_hs = IOsystem%rearr_opts%comm_fc_opts_comp2io%enable_hs
pio_isend = IOsystem%rearr_opts%comm_fc_opts_comp2io%enable_isend
pio_maxreq = IOsystem%rearr_opts%comm_fc_opts_comp2io%max_pend_req

! The rearranger options passed to this function overrides
! both the default and the options in IODESC
Expand Down Expand Up @@ -488,8 +488,8 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf,

! The rearranger options in IODESC overrides the defaults
if(IOsystem%rearr_opts%comm_type == PIO_rearr_comm_p2p) then
if( (IOsystem%rearr_opts%comm_fc_opts%fcd == PIO_rearr_comm_fc_2d_disable) .or.&
(IOsystem%rearr_opts%comm_fc_opts%fcd == PIO_rearr_comm_fc_1d_comp2io) ) then
if( (IOsystem%rearr_opts%fcd == PIO_rearr_comm_fc_2d_disable) .or.&
(IOsystem%rearr_opts%fcd == PIO_rearr_comm_fc_1d_comp2io) ) then
pio_option = POINT_TO_POINT
else
pio_option = FLOW_CONTROL
Expand All @@ -509,9 +509,9 @@ subroutine box_rearrange_io2comp_{TYPE} (IOsystem,ioDesc,s1, iobuf,s2, compbuf,
endif

if (pio_option == FLOW_CONTROL) then
pio_hs = IOsystem%rearr_opts%comm_fc_opts%enable_hs
pio_isend = IOsystem%rearr_opts%comm_fc_opts%enable_isend
pio_maxreq = IOsystem%rearr_opts%comm_fc_opts%max_pend_req
pio_hs = IOsystem%rearr_opts%comm_fc_opts_io2comp%enable_hs
pio_isend = IOsystem%rearr_opts%comm_fc_opts_io2comp%enable_isend
pio_maxreq = IOsystem%rearr_opts%comm_fc_opts_io2comp%max_pend_req

! The rearranger options passed to this function overrides
! both the default and the options in IODESC
Expand Down
9 changes: 5 additions & 4 deletions externals/pio1/pio/pio_types.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,14 +79,12 @@ module pio_types
!! @defgroup PIO_rearr_comm_fc_options PIO_rearr_comm_fc_options
!! @brief Type that defines the PIO rearranger options
!! @details
!! - fcd : @copydoc PIO_rearr_comm_dir
!! - enable_hs : Enable handshake (true/false)
!! - enable_hs : Enable handshake (true/false)
!! - enable_isend : Enable Isends (true/false)
!! - max_pend_req : Maximum pending requests (To indicated unlimited
!! number of requests use PIO_REARR_COMM_UNLIMITED_PEND_REQ)
!>
type, public :: PIO_rearr_comm_fc_opt_t
integer :: fcd ! Flow control direction
logical :: enable_hs ! Enable handshake?
logical :: enable_isend ! Enable isends?
integer :: max_pend_req ! Maximum pending requests
Expand All @@ -98,11 +96,14 @@ module pio_types
!! @brief Type that defines the PIO rearranger options
!! @details
!! - comm_type : @copydoc PIO_rearr_comm_t
!! - fcd : @copydoc PIO_rearr_comm_dir
!! - comm_fc_opts : @copydoc PIO_rearr_comm_fc_options
!>
type, public :: PIO_rearr_opt_t
integer :: comm_type
type(PIO_rearr_comm_fc_opt_t) :: comm_fc_opts
integer :: fcd ! Flow control direction
type(PIO_rearr_comm_fc_opt_t) :: comm_fc_opts_comp2io
type(PIO_rearr_comm_fc_opt_t) :: comm_fc_opts_io2comp
end type PIO_rearr_opt_t

public :: PIO_rearr_comm_p2p, PIO_rearr_comm_coll,&
Expand Down
16 changes: 10 additions & 6 deletions externals/pio1/pio/piolib_mod.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1600,21 +1600,25 @@ subroutine init_iosystem_rearr_options(iosystem)
#endif

#ifdef _NO_FLOW_CONTROL
iosystem%rearr_opts%comm_fc_opts%fcd = PIO_rearr_comm_fc_2d_disable
iosystem%rearr_opts%fcd = PIO_rearr_comm_fc_2d_disable
#else
! We ignore the following flags
! We ignore the following flags
! 1) _MPISERIAL : The flow control code is never used when _MPISERIAL is set
! 2) _USE_COMP2IO_FC/_USE_IO2COMP_FC : These flags are not currently used
! (These were experimental flags). The user can explicitly control
! these options (comp2io and io2comp flow control) via rearranger
! options passed to pio_init()
iosystem%rearr_opts%comm_fc_opts%fcd = PIO_rearr_comm_fc_2d_enable
iosystem%rearr_opts%fcd = PIO_rearr_comm_fc_2d_enable
#endif

! the following will be ignored if not p2p with flow control
iosystem%rearr_opts%comm_fc_opts%enable_hs = DEF_P2P_HANDSHAKE
iosystem%rearr_opts%comm_fc_opts%enable_isend = DEF_P2P_ISEND
iosystem%rearr_opts%comm_fc_opts%max_pend_req = DEF_P2P_MAXREQ
iosystem%rearr_opts%comm_fc_opts_comp2io%enable_hs = DEF_P2P_HANDSHAKE
iosystem%rearr_opts%comm_fc_opts_comp2io%enable_isend = DEF_P2P_ISEND
iosystem%rearr_opts%comm_fc_opts_comp2io%max_pend_req = DEF_P2P_MAXREQ

iosystem%rearr_opts%comm_fc_opts_io2comp%enable_hs = DEF_P2P_HANDSHAKE
iosystem%rearr_opts%comm_fc_opts_io2comp%enable_isend = DEF_P2P_ISEND
iosystem%rearr_opts%comm_fc_opts_io2comp%max_pend_req = DEF_P2P_MAXREQ

end subroutine init_iosystem_rearr_options

Expand Down
Loading

0 comments on commit 6be24a0

Please sign in to comment.