From 02141276ee9ffa570737e749f77927c0ee4e7db9 Mon Sep 17 00:00:00 2001 From: Ralph Castain Date: Fri, 11 Sep 2020 14:42:00 -0700 Subject: [PATCH] Revise fabric support chapter Remove get_device_info/vertex functions as the data can be adequately covered by PMIx_Get. Add function to register resources not associated with a namespace. Cleanup location of some attribute definitions Reorder a few chapters Remove unused constants Per HPE request, replace "Cray" with "HPE" Add missing fabric attribute Minor typo corrections Signed-off-by: Ralph Castain --- App_Python.tex | 128 ++++++------- Chap_API_Fabric.tex | 377 +++++++++++-------------------------- Chap_API_Reserved_Keys.tex | 58 ++++++ Chap_API_Server.tex | 181 +++++++++++++----- Chap_API_Struct.tex | 12 ++ Chap_API_Sync_Access.tex | 13 +- Chap_Revisions.tex | 20 +- pmix-standard.tex | 27 +-- 8 files changed, 411 insertions(+), 405 deletions(-) diff --git a/App_Python.tex b/App_Python.tex index f1f4fd02..b4b6795b 100644 --- a/App_Python.tex +++ b/App_Python.tex @@ -40,9 +40,9 @@ \section{Datatype Definitions} \item provided values fall outside the range of the C-equivalent definition - e.g., if a value identified as \refconst{PMIX_UINT8} lies outside the \code{uint8_t}range \end{itemize} -Note that explicit labeling of \ac{PMIx} datatype, even when Python itself doesn’t care, is often required for the Python bindings to know how to properly interpret and label the provided value when passing it to the \ac{PMIx} library. +Note that explicit labeling of \ac{PMIx} data type, even when Python itself doesn’t care, is often required for the Python bindings to know how to properly interpret and label the provided value when passing it to the \ac{PMIx} library. -Table~\ref{app:python:ctopy} lists the correspondence between datatypes in the two languages. +Table~\ref{app:python:ctopy} lists the correspondence between data types in the two languages. \begin{landscape} \begin{small} @@ -84,12 +84,16 @@ \section{Datatype Definitions} \refstruct{pmix_app_t} & PMIX_APP & \pylabel{app}\{'cmd': cmd, 'argv': [argv], 'env': [env], 'maxprocs': maxprocs, 'info': [info]\} & \refarg{cmd} is a Python string; \refarg{argv} and \refarg{env} are Python \emph{lists} containing Python strings; \refarg{maxprocs} is an integer; and \refarg{info} is a Python \emph{list} of \refpy{info} values \\ \hline \refstruct{pmix_query_t} & PMIX_QUERY & \pylabel{query}\{'keys': [keys], 'qualifiers': [info]\} & \refarg{keys} is a Python \emph{list} of Python strings, and \refarg{qualifiers} is a Python \emph{list} of \refpy{info} values \\ \hline \refstruct{pmix_regattr_t} & PMIX_REGATTR & \pylabel{regattr}\{'name': name, 'key': key, 'type': type, 'info': [info], 'description': [desc]\} & \refarg{name} and \refarg{string} are Python strings; \refarg{type} is the \ac{PMIx} datatype for the attribute's value; \refarg{info} is a Python \emph{list} of \refpy{info} values; and \refarg{description} is a list of Python strings describing the attribute \\ \hline - \refstruct{pmix_coord_t} & PMIX_COORD & \pylabel{coord}\{'fabric': fabric, 'plane': plane, 'view': view, 'coord': [coords]\} & \refarg{fabric} and \refarg{plane} are Python strings; \refarg{view} is the \refstruct{pmix_coord_view_t} of the coordinate; and \refarg{coord} is a list of integer coordinates, one for each dimension of the fabric \\ \hline \refstruct{pmix_job_state_t} & PMIX_JOB_STATE & integer & value shall be limited to the \code{uint8_t} range \\ \hline \refstruct{pmix_link_state_t} & PMIX_LINK_STATE & integer & value shall be limited to the \code{uint8_t} range \\ \hline \refstruct{pmix_cpuset_t} & N/A & \pylabel{cpuset}\{'source': source, 'cpus': bitmap\} & \refarg{source} is a string name of the library that created the cpuset; and \refarg{cpus} is a bitarray containing the cpuset \\ \hline \refstruct{pmix_locality_t} & N/A & \pylabel{locality}bitarray & 16-bit array containing the relative locality of the specified local process \\ \hline \refstruct{pmix_fabric_t} & N/A & \pylabel{fabric}\{'name': name, 'index': idx, 'info': [info]\} & \refarg{name} is the string name assigned to the fabric; \refarg{index} is the integer ID assigned to the fabric; \refarg{info} is a list of \refpy{info} describing the fabric \\ \hline + \refstruct{pmix_endpoint_t} & N/A & \pylabel{endpoint}\{'uuid': uuid, 'endpt': endpt\} & \refarg{uuid} is the string system-unique identifier assigned to the device; \refarg{endpt} is a \refpy{byteobject} containing the endpoint information \\ \hline + \refstruct{pmix_device_distance_t} & PMIX_DEVICE_DIST & \pylabel{devdist}\{'uuid': uuid, 'mindist': mindist, 'maxdist': maxdist\} & \refarg{uuid} is the string system-unique identifier assigned to the device; and \refarg{mindist} and \refarg{maxdist} are Python integers \\ \hline + \refstruct{pmix_topology_t} & N/A & \pylabel{topology}\{'name': name, 'index': idx, 'info': [info]\} & \refarg{name} is the string name assigned to the fabric; \refarg{index} is the integer ID assigned to the fabric; \refarg{info} is a list of \refpy{info} describing the fabric \\ \hline + \refstruct{pmix_coord_t} & PMIX_COORD & \pylabel{coord}\{'view': view, 'coord': [coords]\} & \refarg{view} is the \refstruct{pmix_coord_view_t} of the coordinate; and \refarg{coord} is a list of integer coordinates, one for each dimension of the fabric \\ \hline + \refstruct{pmix_geometry_t} & PMIX_GEOMETRY & \pylabel{geometry}\{'fabric': idx, 'uuid': uuid, 'coordinates': [coords]\} & \refarg{fabric} is the Python integer index of the fabric; \refarg{uuid} is the string system-unique identifier assigned to the device; and \refarg{coordinates} is a list of \refpy{coord} containing the coordinates for the device across all views \\ \hline \end{longtable} \end{small} \end{landscape} @@ -2308,70 +2312,6 @@ \subsection{Client.fabric_deregister} See \refapi{PMIx_Fabric_deregister} for details. -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\subsection{Client.fabric_get_device_info} -\declareapibinding{PMIxClient.fabric_get_device_info}{PMIx_Fabric_get_device_info}{Python} - -\summary -Given a communication cost matrix index for a specified fabric, return an array of information describing the corresponding \ac{NIC}. - -\format - -\versionMarker{4.0} -\pyspecificstart -\begin{codepar} -rc,nicinfo = myclient.fabric_get_device_info(fabric:integer, - device:integer) -\end{codepar} -\pyspecificend - - -\begin{arglist} -\argin{fabric}{Index of the registered fabric (list)} -\argin{fabric}{Index of the device within the communication cost matrix for the fabric (list)} -\end{arglist} - -Returns: - -\begin{itemize} - \item \refarg{rc} - \refconst{PMIX_SUCCESS} or a negative value corresponding to a PMIx error constant (integer) - \item \refarg{nicinfo} - List of Python \refpy{info} describing the referenced \ac{NIC} (list) -\end{itemize} - -See \refapi{PMIx_Fabric_get_device_info} for details. - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\subsection{Client.fabric_get_device_index} -\declareapibinding{PMIxClient.fabric_get_device_index}{PMIx_Fabric_get_device_index}{Python} - -\summary -Given info describing a given device, return the corresponding communication cost matrix index. - -\format - -\versionMarker{4.0} -\pyspecificstart -\begin{codepar} -rc,index = myclient.fabric_get_device_index(fabric:integer, info:list) -\end{codepar} -\pyspecificend - -\begin{arglist} -\argin{fabric}{Index of the registered fabric (list)} -\argin{info}{List of Python \refpy{info} containing device description (list)} -\end{arglist} - -Returns: - -\begin{itemize} - \item \refarg{rc} - \refconst{PMIX_SUCCESS} or a negative value corresponding to a PMIx error constant (integer) - \item \refarg{index} - Index of corresponding device within the communication cost matrix of the fabric (integer) -\end{itemize} - -See \refapi{PMIx_Fabric_get_device_index} for details. - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \subsection{Client.load_topology} \declareapibinding{PMIxClient.load_topology}{PMIx_Load_topology}{Python} @@ -3017,6 +2957,60 @@ \subsection{Server.deregister_nspace} See \refapi{PMIx_server_deregister_nspace} for details. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\subsection{Server.register_resources} +\declareapibinding{PMIxServer.register_resources}{PMIx_server_register_resources}{Python} + +\summary + +Register non-namespace related information with the local \ac{PMIx} library + +\format + +\versionMarker{4.0} +\pyspecificstart +\begin{codepar} +myserver.register_resources(directives:list) +\end{codepar} +\pyspecificend + + +\begin{arglist} +\argin{directives}{List of Python \refpy{info} dictionaries (list)} +\end{arglist} + +Returns: None + +See \refapi{PMIx_server_register_resources} for details. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\subsection{Server.deregister_resources} +\declareapibinding{PMIxServer.deregister_resources}{PMIx_server_deregister_resources}{Python} + +\summary + +Remove non-namespace related information from the local \ac{PMIx} library + +\format + +\versionMarker{4.0} +\pyspecificstart +\begin{codepar} +myserver.deregister_resources(directives:list) +\end{codepar} +\pyspecificend + + +\begin{arglist} +\argin{directives}{List of Python \refpy{info} dictionaries (list)} +\end{arglist} + +Returns: None + +See \refapi{PMIx_server_deregister_resources} for details. + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \subsection{Server.register_client} \declareapibinding{PMIxServer.register_client}{PMIx_server_register_client}{Python} diff --git a/Chap_API_Fabric.tex b/Chap_API_Fabric.tex index df8b449b..366ab84f 100644 --- a/Chap_API_Fabric.tex +++ b/Chap_API_Fabric.tex @@ -28,10 +28,49 @@ \chapter{Fabric Support Definitions} \begin{enumerate}[label=\alph*)] \item A process can select a remote endpoint to use based on its own preferred device and reachability of the peer's remote devices. Once the initial connection has been made, the two processes can exchange information and mutually determine their desired communication path going forward. - \item The application can use knowledge of both the local and remote distance arrays to compute the best communication path and establish that connection. In some instances (e.g., a homogeneous system), the \ac{PMIx} system may provide distance information for both local and remote devices. Alternatively, when this isn't available, an application can opt to collect the information using the \refattr{PMIX_COLLECT_GENERATED_JOB_INFO} with the \refapi{PMIx_Fence} \ac{API}, or can obtain it on a one peer-at-a-time basis using the \refapi{PMIx_Get} \ac{API} on systems where the host environment supports the \refterm{Direct Modex} operation. + \item The application can use knowledge of both the local and remote distance arrays to compute the best communication path and establish that connection. In some instances (e.g., a homogeneous system), a \ac{PMIx} server may provide distance information for both local and remote devices. Alternatively, when this isn't available, an application can opt to collect the information using the \refattr{PMIX_COLLECT_GENERATED_JOB_INFO} with the \refapi{PMIx_Fence} \ac{API}, or can obtain it on a one peer-at-a-time basis using the \refapi{PMIx_Get} \ac{API} on systems where the host environment supports the \refterm{Direct Modex} operation. \end{enumerate} +Information on fabric coordinates, endpoints, and device distances are provided as \emph{reserved keys} as detailed in Chapter \ref{chap:api_rsvd_keys} - i.e., they are to be available at client start of execution and are subject to the retrieval rules of Section \ref{chap:rkeys:retrules}. Examples for retrieving fabric-related information include retrieval of: + +\begin{itemize} + \item An array of information on fabric devices for a node by passing \refattr{PMIX_FABRIC_DEVICES} as the key to \refapi{PMIx_Get} along with the \refattr{PMIX_HOSTNAME} of the node as a directive + + \item An array of information on a specific fabric device by passing \refattr{PMIX_FABRIC_DEVICE} as the key to \refapi{PMIx_Get} along with the \refattr{PMIX_FABRIC_DEVICE_ID} of the device as a directive + + \item An array of information on a specific fabric device by passing \refattr{PMIX_FABRIC_DEVICE} as the key to \refapi{PMIx_Get} along with both \refattr{PMIX_FABRIC_DEVICE_NAME} of the device and the \refattr{PMIX_HOSTNAME} of the node as directives +\end{itemize} + +When requesting data on a device, returned data must include at least the following attributes: + +\begin{itemize} + \item \pasteAttributeItemBegin{PMIX_HOSTNAME} The \refattr{PMIX_NODEID} may be returned in its place, or in addition to the hostname. + \pasteAttributeItemEnd + \item \pasteAttributeItem{PMIX_FABRIC_DEVICE_ID} + \item \pasteAttributeItem{PMIX_FABRIC_DEVICE_NAME} + \item \pasteAttributeItem{PMIX_FABRIC_DEVICE_VENDOR} + \item \pasteAttributeItem{PMIX_FABRIC_DEVICE_BUS_TYPE} + \item \pasteAttributeItemBegin{PMIX_FABRIC_DEVICE_PCI_DEVID} This item should be included if the device bus type is \ac{PCI} - the equivalent should be provided for any other bus type. + \pasteAttributeItemEnd +\end{itemize} + +The returned array may optionally contain one or more of the following in addition to the above list: + +\begin{itemize} + \item \pasteAttributeItem{PMIX_FABRIC_DEVICE_INDEX} + \item \pasteAttributeItem{PMIX_FABRIC_DEVICE_VENDORID} + \item \pasteAttributeItem{PMIX_FABRIC_DEVICE_DRIVER} + \item \pasteAttributeItem{PMIX_FABRIC_DEVICE_FIRMWARE} + \item \pasteAttributeItem{PMIX_FABRIC_DEVICE_ADDRESS} + \item \pasteAttributeItem{PMIX_FABRIC_DEVICE_COORDINATES} + \item \pasteAttributeItem{PMIX_FABRIC_DEVICE_MTU} + \item \pasteAttributeItem{PMIX_FABRIC_DEVICE_SPEED} + \item \pasteAttributeItem{PMIX_FABRIC_DEVICE_STATE} + \item \pasteAttributeItem{PMIX_FABRIC_DEVICE_TYPE} +\end{itemize} + + The remainder of this chapter details the events, data types, attributes, and \acp{API} associated with fabric-related operations. @@ -140,7 +179,7 @@ \subsection{Fabric endpoint support macros} %%%% \littleheader{Release an endpoint array} -\declaremacro{PMIX_ENDPOIONT_FREE} +\declaremacro{PMIX_ENDPOINT_FREE} Release an array of \refstruct{pmix_endpoint_t} structures. @@ -241,7 +280,7 @@ \subsection{Fabric device distance support macros} %%%% \littleheader{Release an device distance array} -\declaremacro{PMIX_ENDPOIONT_DIST_FREE} +\declaremacro{PMIX_DEVICE_DIST_FREE} Release an array of \refstruct{pmix_device_distance_t} structures. @@ -532,21 +571,11 @@ \subsection{Fabric Operation Constants} \begin{constantdesc} % \declareconstitemNEW{PMIX_FABRIC_REQUEST_INFO} -Request information on a specific fabric - if the fabric isn't specified as per \refapi{PMIx_Fabric_register}, then return information on the system default fabric. Information to be returned is described in \refstruct{pmix_fabric_t}. +Request information on a specific fabric - if the fabric isn't specified as per \refapi{PMIx_Fabric_register}, then return information on the default fabric of the overall system. Information to be returned is described in \refstruct{pmix_fabric_t}. % \declareconstitemNEW{PMIX_FABRIC_UPDATE_INFO} Update information on a specific fabric - the index of the fabric (\refattr{PMIX_FABRIC_INDEX}) to be updated must be provided. % -\declareconstitemNEW{PMIX_FABRIC_GET_DEVICE_INFO} -Request information on a specific device within the identified fabric - the index of the fabric (\refattr{PMIX_FABRIC_INDEX}) and the index of the device (\refattr{PMIX_FABRIC_DEVICE_INDEX}) within the associated communication cost matrix must be provided. If the device index is not specified, then return information on all devices in the fabric. Information to be included on each device is described in \refstruct{pmix_fabric_t}. -% -\adviceuserstart -Requesting information on every device in the fabric may be an expensive operation in terms of both memory footprint and time. -\adviceuserend -% -\declareconstitemNEW{PMIX_FABRIC_GET_DEVICE_INDEX} -Request the communication matrix index (returned as \refattr{PMIX_FABRIC_DEVICE_INDEX}) for a specific device within a given fabric based on the provided device information. A status of \refconst{PMIX_ERR_NOT_FOUND} must be returned if the specified device is not a part of the given fabric. -% \end{constantdesc} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -598,7 +627,7 @@ \subsection{Fabric registration structure} \pasteAttributeItem{PMIX_FABRIC_SHAPE} \pasteAttributeItem{PMIX_FABRIC_SHAPE_STRING} -While unusual due to scaling issues, implementations may include an array of \refattr{PMIX_FABRIC_DEVICE} elements describing the vertex information for each device in the system. Each element shall contain a \refstruct{pmix_data_array_t} of \refstruct{pmix_info_t} values describing the device. Each array may contain one or more of the following (ordering is arbitrary): +While unusual due to scaling issues, implementations may include an array of \refattr{PMIX_FABRIC_DEVICE} elements describing the device information for each device in the overall system. Each element shall contain a \refstruct{pmix_data_array_t} of \refstruct{pmix_info_t} values describing the device. Each array may contain one or more of the following (ordering is arbitrary): \pasteAttributeItem{PMIX_FABRIC_DEVICE_NAME} \pasteAttributeItem{PMIX_FABRIC_DEVICE_VENDOR} @@ -636,7 +665,7 @@ \subsubsection{Initialize the fabric structure} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \section{Fabric Support Attributes} -\label{api:sched:attrs} +\label{api:fabric:attrs} The following attribute is used by the \ac{PMIx} server library supporting the system's \ac{WLM} to indicate that it wants access to the fabric support functions: @@ -645,7 +674,7 @@ \section{Fabric Support Attributes} } \vspace{\baselineskip} -The following attributes may be returned in response to fabric-specific \acp{API} or queries (e.g., \refapi{PMIx_Get} or \refapi{PMIx_Query_info}) made by processes or tools. +The following attributes may be returned in response to fabric-specific \acp{API} or queries (e.g., \refapi{PMIx_Get} or \refapi{PMIx_Query_info}). These attributes are not related to a specific \refterm{data realm} (as described in Section \ref{api:struct:attributes:retrieval}) - the \refapi{PMIx_Get} function shall therefore ignore the value in its \refarg{proc} process identifier argument when retrieving these values. % \declareAttributeNEW{PMIX_FABRIC_COST_MATRIX}{"pmix.fab.cm"}{pointer}{ @@ -653,15 +682,27 @@ \section{Fabric Support Attributes} } % \declareAttributeNEW{PMIX_FABRIC_GROUPS}{"pmix.fab.grps"}{string}{ -A string delineating the group membership of nodes in the system, where each fabric group consists of the group number followed by a colon and a comma-delimited list of nodes in that group, with the groups delimited by semi-colons (e.g., \code{0:node000,node002,node004,node006;\allowbreak 1:node001,node003,\allowbreak node005,node007}) +A string delineating the group membership of nodes in the overall system, where each fabric group consists of the group number followed by a colon and a comma-delimited list of nodes in that group, with the groups delimited by semi-colons (e.g., \code{0:node000,node002,node004,node006;\allowbreak 1:node001,node003,\allowbreak node005,node007}) +} +% +\declareAttributeNEW{PMIX_FABRIC_PLANE}{"pmix.fab.plane"}{string}{ +ID string of a fabric plane (e.g., CIDR for Ethernet). When used as a modifier in a request for information, specifies the plane whose information is to be returned. When used directly as a key in a request, returns a \refstruct{pmix_data_array_t} of string identifiers for all fabric planes in the overall system. } +% +\declareAttributeNEW{PMIX_FABRIC_SWITCH}{"pmix.fab.switch"}{string}{ +ID string of a fabric switch. When used as a modifier in a request for information, specifies the switch whose information is to be returned. When used directly as a key in a request, returns a \refstruct{pmix_data_array_t} of string identifiers for all fabric switches in the overall system. +} +% +\vspace{\baselineskip} +The following attributes may be returned in response to queries (e.g., \refapi{PMIx_Get} or \refapi{PMIx_Query_info}). A qualifier (e.g., \refattr{PMIX_FABRIC_INDEX}) identifying the fabric whose value is being referenced must be provided for queries on systems supporting more than one fabric when values for the non-default fabric are requested. These attributes are not related to a specific \refterm{data realm} (as described in Section \ref{api:struct:attributes:retrieval}) - the \refapi{PMIx_Get} function shall therefore ignore the value in its \refarg{proc} process identifier argument when retrieving these values. + % \declareAttributeNEW{PMIX_FABRIC_VENDOR}{"pmix.fab.vndr"}{string}{ -Name of fabric vendor (e.g., Amazon, Mellanox, Cray, Intel). +Name of the vendor (e.g., Amazon, Mellanox, HPE, Intel) for the specified fabric. } % \declareAttributeNEW{PMIX_FABRIC_IDENTIFIER}{"pmix.fab.id"}{string}{ -An identifier for the fabric (e.g., MgmtEthernet, Slingshot-11, OmniPath-1). +An identifier for the specified fabric (e.g., MgmtEthernet, Slingshot-11, OmniPath-1). } % \declareAttributeNEW{PMIX_FABRIC_INDEX}{"pmix.fab.idx"}{size_t}{ @@ -669,60 +710,40 @@ \section{Fabric Support Attributes} } % \declareAttributeNEW{PMIX_FABRIC_NUM_DEVICES}{"pmix.fab.nverts"}{size_t}{ -Total number of fabric devices in the system - corresponds to the number of rows or columns in the cost matrix. +Total number of fabric devices in the overall system - corresponds to the number of rows or columns in the cost matrix. } % -\declareAttributeNEW{PMIX_FABRIC_DIMS}{"pmix.fab.dims"}{uint32_t}{ -Number of dimensions in the specified fabric plane/view. If no plane is specified in a request, then the dimensions of all planes in the system will be returned as a \refstruct{pmix_data_array_t} containing an array of \code{uint32_t} values. Default is to provide dimensions in \emph{logical} view. -} -% -\declareAttributeNEW{PMIX_FABRIC_COORDINATES}{"pmix.fab.coord"}{pmix_data_array_t}{ -Array of \refstruct{pmix_geometry_t} fabric coordinates for devices on the specified node. The array will contain the coordinates of all devices on the node, including values for all supported coordinate views. The information for devices on the local node shall be provided if the node is not specified in the request. -} -% -\declareAttributeNEW{PMIX_FABRIC_ENDPT}{"pmix.fab.endpt"}{pmix_data_array_t}{ -Fabric endpoints for a specified process. As multiple endpoints may be assigned to a given process (e.g., in the case where multiple devices are associated with a package to which the process is bound), the returned values will be provided in a \refstruct{pmix_data_array_t} of \refstruct{pmix_endpoint_t} elements. +\declareAttributeNEW{PMIX_FABRIC_VIEW}{"pmix.fab.view"}{pmix_coord_view_t}{ +Used purely as a qualifier to requests, specifies the view type (e.g., local vs. physical) for the requested information. } % -\declareAttributeNEW{PMIX_FABRIC_DEVICE_DIST}{"pmix.fab.endptdist"}{pmix_data_array_t}{ -Relative distance from the location of the calling process (either as sampled at the time of a \refapi{PMIx_Fabric_update_distances} request, or based on the initial binding location set at time of start of execution) to each local fabric device, returned as an array of \refstruct{pmix_device_distance_t} elements in no particular order. +\declareAttributeNEW{PMIX_FABRIC_DIMS}{"pmix.fab.dims"}{uint32_t}{ +Number of dimensions in the specified fabric plane/view. If no plane is specified in a request, then the dimensions of all planes in the overall system will be returned as a \refstruct{pmix_data_array_t} containing an array of \code{uint32_t} values. Default is to provide dimensions in \emph{logical} view. } % \declareAttributeNEW{PMIX_FABRIC_SHAPE}{"pmix.fab.shape"}{pmix_data_array_t*}{ -The size of each dimension in the specified fabric plane/view, returned in a \refstruct{pmix_data_array_t} containing an array of \code{uint32_t} values. The size is defined as the number of elements present in that dimension - e.g., the number of devices in one dimension of a physical view of a fabric plane. If no plane is specified, then the shape of each plane in the system will be returned in a \refstruct{pmix_data_array_t} array where each element is itself a two-element array containing the \refattr{PMIX_FABRIC_PLANE} followed by that plane's fabric shape. Default is to provide the shape in \emph{logical} view. +The size of each dimension in the specified fabric plane/view, returned in a \refstruct{pmix_data_array_t} containing an array of \code{uint32_t} values. The size is defined as the number of elements present in that dimension - e.g., the number of devices in one dimension of a physical view of a fabric plane. If no plane is specified, then the shape of each plane in the overall system will be returned in a \refstruct{pmix_data_array_t} array where each element is itself a two-element array containing the \refattr{PMIX_FABRIC_PLANE} followed by that plane's fabric shape. Default is to provide the shape in \emph{logical} view. } % \declareAttributeNEW{PMIX_FABRIC_SHAPE_STRING}{"pmix.fab.shapestr"}{string}{ -Network shape expressed as a string (e.g., \code{"10x12x2"}). If no plane is specified, then the shape of each plane in the system will be returned in a \refstruct{pmix_data_array_t} array where each element is itself a two-element array containing the \refattr{PMIX_FABRIC_PLANE} followed by that plane's fabric shape string. Default is to provide the shape in \emph{logical} view. +Network shape expressed as a string (e.g., \code{"10x12x2"}). If no plane is specified, then the shape of each plane in the overall system will be returned in a \refstruct{pmix_data_array_t} array where each element is itself a two-element array containing the \refattr{PMIX_FABRIC_PLANE} followed by that plane's fabric shape string. Default is to provide the shape in \emph{logical} view. } % -\declareAttributeNEW{PMIX_SWITCH_PEERS}{"pmix.speers"}{string}{ -Comma-delimited string of peer ranks that share the same switch as the process specified in the call to \refapi{PMIx_Get}. Single-device environments will return a string. Multi-device environments will return a \refstruct{pmix_data_array_t} array of results, each element consisting of a two-element array containing the \refattr{PMIX_FABRIC_DEVICE_INDEX} of the local fabric device and a comma-delimited string of peer ranks sharing the switch to which that device is connected. -} % - \vspace{\baselineskip} -The following attributes can be used either as a key (e.g., when requesting information via \refapi{PMIx_Get}) or as a modifier to such a request: +The following attributes are related to the \emph{node realm} (as described in Section \ref{chap:res:nrealm}) and are retrieved according to those rules. % -\declareAttributeNEW{PMIX_FABRIC_PLANE}{"pmix.fab.plane"}{string}{ -ID string of a fabric plane (e.g., CIDR for Ethernet). When used as a modifier in a request for information, specifies the plane whose information is to be returned. When used directly as a key in a request, returns a \refstruct{pmix_data_array_t} of string identifiers for all fabric planes in the system. +\declareAttributeNEW{PMIX_FABRIC_DEVICES}{"pmix.fab.devs"}{pmix_data_array_t}{ +Array of \refstruct{pmix_info_t} containing information for all devices on the specified node. Each element of the array will contain a \refattr{PMIX_FABRIC_DEVICE} entry, which in turn will contain an array of information on a given device. } % -\declareAttributeNEW{PMIX_FABRIC_SWITCH}{"pmix.fab.switch"}{string}{ -ID string of a fabric switch. When used as a modifier in a request for information, specifies the switch whose information is to be returned. When used directly as a key in a request, returns a \refstruct{pmix_data_array_t} of string identifiers for all fabric switches in the system. -} -% -\declareAttributeNEW{PMIX_FABRIC_VIEW}{"pmix.fab.view"}{pmix_coord_view_t}{ -Fabric coordinate view to be used for the requested coordinate - see \refstruct{pmix_coord_view_t} for the list of accepted values. This attribute is solely defined as a modifier/qualifier by which the caller can indicate the desired coordinate view for the information being requested and can not be used as a key to a query. +\declareAttributeNEW{PMIX_FABRIC_COORDINATES}{"pmix.fab.coords"}{pmix_data_array_t}{ +Array of \refstruct{pmix_geometry_t} fabric coordinates for devices on the specified node. The array will contain the coordinates of all devices on the node, including values for all supported coordinate views. The information for devices on the local node shall be provided if the node is not specified in the request. } % - -\vspace{\baselineskip} -The following attributes are used to describe \refterm{devices} attached to the fabric. - \declareAttributeNEW{PMIX_FABRIC_DEVICE}{"pmix.fabdev"}{\refstruct{pmix_data_array_t}}{ -An array of \refstruct{pmix_info_t} describing a particular fabric device using one or more of the attributes defined below. +An array of \refstruct{pmix_info_t} describing a particular fabric device using one or more of the attributes defined below. The first element in the array shall be the \refattr{PMIX_FABRIC_DEVICE_ID} of the device. } % \declareAttributeNEW{PMIX_FABRIC_DEVICE_ID}{"pmix.fabdev.id"}{string}{ @@ -761,6 +782,10 @@ \section{Fabric Support Attributes} The primary link-level address associated with the device, such as a \ac{MAC} address. If multiple addresses are available, only one will be reported. } % +\declareAttributeNEW{PMIX_FABRIC_DEVICE_COORDINATES}{"pmix.fab.coord"}{pmix_geometry_t}{ +The \refstruct{pmix_geometry_t} fabric coordinates for the device, including values for all supported coordinate views. +} +% \declareAttributeNEW{PMIX_FABRIC_DEVICE_MTU}{"pmix.fabdev.mtu"}{size_t}{ The maximum transfer unit of link level frames or packets, in bytes. } @@ -778,9 +803,30 @@ \section{Fabric Support Attributes} } % \declareAttributeNEW{PMIX_FABRIC_DEVICE_PCI_DEVID}{"pmix.fabdev.pcidevid"}{string}{ -A node-level unique identifier for a \ac{PCI} device. Provided only if the device is located on a \ac{PCI} bus. The identifier is constructed as a four-part tuple delimited by colons comprised of the \ac{PCI} 16-bit domain, 8-bit bus, 8-bit device, and 8-bit function IDs, each expressed in zero-extended hexadecimal form. Thus, an example identifier might be "abc1:0f:23:01". The combination of node identifier (\refattr{PMIX_HOSTNAME} or \refattr{PMIX_NODEID}) and \refattr{PMIX_FABRIC_DEVICE_PCI_DEVID} shall be unique within the system. +A node-level unique identifier for a \ac{PCI} device. Provided only if the device is located on a \ac{PCI} bus. The identifier is constructed as a four-part tuple delimited by colons comprised of the \ac{PCI} 16-bit domain, 8-bit bus, 8-bit device, and 8-bit function IDs, each expressed in zero-extended hexadecimal form. Thus, an example identifier might be "abc1:0f:23:01". The combination of node identifier (\refattr{PMIX_HOSTNAME} or \refattr{PMIX_NODEID}) and \refattr{PMIX_FABRIC_DEVICE_PCI_DEVID} shall be unique within the overall system. } +% +\vspace{\baselineskip} +The following attributes are related to the \emph{process realm} (as described in Section \ref{chap:res:prealm}) and are retrieved according to those rules. +% +\declareAttributeNEW{PMIX_FABRIC_ENDPT}{"pmix.fab.endpt"}{pmix_data_array_t}{ +Fabric endpoints for a specified process. As multiple endpoints may be assigned to a given process (e.g., in the case where multiple devices are associated with a package to which the process is bound), the returned values will be provided in a \refstruct{pmix_data_array_t} of \refstruct{pmix_endpoint_t} elements. +} +% +\declareAttributeNEW{PMIX_FABRIC_DEVICE_DIST}{"pmix.fab.endptdist"}{pmix_data_array_t}{ +Relative distance from the location of the calling process (either as sampled at the time of a \refapi{PMIx_Fabric_update_distances} request, or based on the initial binding location set at time of start of execution) to each local fabric device, returned as an array of \refstruct{pmix_device_distance_t} elements in no particular order. +} +% +\vspace{\baselineskip} +The following attributes are related to the \emph{job realm} (as described in Section \ref{chap:res:jrealm}) and are retrieved according to those rules. + +% +\declareAttributeNEW{PMIX_SWITCH_PEERS}{"pmix.speers"}{pmix_data_array_t}{ +Peer ranks that share the same switch as the process specified in the call to \refapi{PMIx_Get}. Returns a \refstruct{pmix_data_array_t} array of \refstruct{pmix_info_t} results, each element containing the \refattr{PMIX_SWITCH_PEERS} key with a three-element \refstruct{pmix_data_array_t} array of +\refstruct{pmix_info_t} containing the \refattr{PMIX_FABRIC_DEVICE_ID} of the local fabric device, the \refattr{PMIX_FABRIC_SWITCH} identifying the switch to which it is connected, and a comma-delimited string of peer ranks sharing the switch to which that device is connected. +} +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -815,7 +861,7 @@ \subsection{\code{PMIx_Fabric_register}} \cspecificend \begin{arglist} -\argin{fabric}{address of a \refstruct{pmix_fabric_t} (backed by storage). User may populate the "name" field at will - \ac{PMIx} does not utilize this field (handle)} +\arginout{fabric}{address of a \refstruct{pmix_fabric_t} (backed by storage). User may populate the "name" field at will - \ac{PMIx} does not utilize this field (handle)} \argin{directives}{an optional array of values indicating desired behaviors and/or fabric to be accessed. If \code{NULL}, then the highest priority available fabric will be used (array of handles)} \argin{ndirs}{Number of elements in the \refarg{directives} array (integer)} \end{arglist} @@ -834,7 +880,7 @@ \subsection{\code{PMIx_Fabric_register}} %%%% \descr -Register for access to fabric-related information, including the communication cost matrix. This call must be made prior to requesting information from a fabric. The caller may request access to a particular fabric using the vendor, type, or identifier, or to a specific \refterm{fabric plane} via the \refattr{PMIX_FABRIC_PLANE} attribute - otherwise, the default fabric will be returned. +Register for access to fabric-related information, including the communication cost matrix. This call must be made prior to requesting information from a fabric. The caller may request access to a particular fabric using the vendor, type, or identifier, or to a specific \refterm{fabric plane} via the \refattr{PMIX_FABRIC_PLANE} attribute - otherwise, information for the default fabric will be returned. Upon successful completion of the call, information will have been filled into the fields of the provided \refarg{fabric} structure. For performance reasons, the \ac{PMIx} library does not provide thread protection for accessing the information in the \refstruct{pmix_fabric_t} structure. Instead, the \ac{PMIx} implementation shall provide two methods for coordinating updates to the provided fabric information: @@ -872,7 +918,7 @@ \subsection{\code{PMIx_Fabric_register_nb}} \cspecificend \begin{arglist} -\argin{fabric}{address of a \refstruct{pmix_fabric_t} (backed by storage). User may populate the "name" field at will - \ac{PMIx} does not utilize this field (handle)} +\arginout{fabric}{address of a \refstruct{pmix_fabric_t} (backed by storage). User may populate the "name" field at will - \ac{PMIx} does not utilize this field (handle)} \argin{directives}{an optional array of values indicating desired behaviors and/or fabric to be accessed. If \code{NULL}, then the highest priority available fabric will be used (array of handles)} \argin{ndirs}{Number of elements in the \refarg{directives} array (integer)} \argin{cbfunc}{Callback function \refapi{pmix_op_cbfunc_t} (function reference)} @@ -913,7 +959,7 @@ \subsection{\code{PMIx_Fabric_update}} \cspecificend \begin{arglist} -\argin{fabric}{address of a \refstruct{pmix_fabric_t} (backed by storage) (handle)} +\arginout{fabric}{address of a \refstruct{pmix_fabric_t} (backed by storage) (handle)} \end{arglist} Returns \refconst{PMIX_SUCCESS} or a negative value corresponding to a \ac{PMIx} error constant. @@ -921,7 +967,7 @@ \subsection{\code{PMIx_Fabric_update}} %%%% \descr -Update fabric-related information. This call can be made at any time to request an update of the fabric information contained in the provided \refstruct{pmix_fabric_t} object. The caller is not allowed to access the provided \refstruct{pmix_fabric_t} until the call has returned. +Update fabric-related information. This call can be made at any time to request an update of the fabric information contained in the provided \refstruct{pmix_fabric_t} object. The caller is not allowed to access the provided \refstruct{pmix_fabric_t} until the call has returned. Upon successful return, the information fields in the \refarg{fabric} structure will have been updated. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -946,7 +992,7 @@ \subsection{\code{PMIx_Fabric_update_nb}} \cspecificend \begin{arglist} -\argin{fabric}{address of a \refstruct{pmix_fabric_t} (handle)} +\arginout{fabric}{address of a \refstruct{pmix_fabric_t} (handle)} \argin{cbfunc}{Callback function \refapi{pmix_op_cbfunc_t} (function reference)} \argin{cbdata}{Data to be passed to the callback function (memory reference)} \end{arglist} @@ -961,7 +1007,7 @@ \subsection{\code{PMIx_Fabric_update_nb}} %%%% \descr -Non-blocking form of \refapi{PMIx_Fabric_update}. The caller is not allowed to access the provided \refstruct{pmix_fabric_t} until the callback function has been executed. +Non-blocking form of \refapi{PMIx_Fabric_update}. The caller is not allowed to access the provided \refstruct{pmix_fabric_t} until the callback function has been executed, at which time the fields in the provided \refarg{fabric} structure will have been updated. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1036,213 +1082,6 @@ \subsection{\code{PMIx_Fabric_deregister_nb}} Non-blocking form of \refapi{PMIx_Fabric_deregister}. Provided \refarg{fabric} must not be accessed until after callback function has been executed. -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\subsection{\code{PMIx_Fabric_get_device_info}} -\declareapi{PMIx_Fabric_get_device_info} - -%%%% -\summary - -Given a communication cost matrix index for a specified fabric, return the corresponding device info. - -%%%% -\format - -\versionMarker{4.0} -\cspecificstart -\begin{codepar} -pmix_status_t -PMIx_Fabric_get_device_info(pmix_fabric_t *fabric, uint32_t index, - pmix_info_t **info, size_t *ninfo); -\end{codepar} -\cspecificend - -\begin{arglist} -\argin{fabric}{Address of a \refstruct{pmix_fabric_t} (handle)} -\argin{index}{Device index (i.e., communication cost matrix row or column number) (integer)} -\arginout{info}{Address where a pointer to an array of \refstruct{pmix_info_t} containing the results of the query can be returned (memory reference)} -\arginout{ninfo}{Address where the number of elements in \refarg{info} can be returned (handle)} -\end{arglist} - -Returns one of the following: - -\begin{itemize} - \item \refconst{PMIX_SUCCESS}, indicating return of a valid value. - \item \refconst{PMIX_ERR_BAD_PARAM}, indicating that the provided index is out of bounds. - \item a \ac{PMIx} error constant indicating either an error in the input or that the request failed. -\end{itemize} - -%%%% -\descr - -Query information about a specified fabric device in the system. The returned \refarg{status} indicates if requested data was found or not. The returned array of \refstruct{pmix_info_t} will contain information on the specified device - the exact contents will depend on the \ac{PMIx} implementation and the fabric vendor. At a minimum, it must contain sufficient information to uniquely identify the device within the system (ordering is arbitrary). - -\reqattrstart -\pasteAttributeItemBegin{PMIX_HOSTNAME} The \refattr{PMIX_NODEID} may be returned in its place, or in addition to the hostname. -\pasteAttributeItemEnd -\pasteAttributeItem{PMIX_FABRIC_DEVICE_ID} -\pasteAttributeItem{PMIX_FABRIC_DEVICE_NAME} -\pasteAttributeItem{PMIX_FABRIC_DEVICE_VENDOR} -\pasteAttributeItem{PMIX_FABRIC_DEVICE_BUS_TYPE} -\pasteAttributeItemBegin{PMIX_FABRIC_DEVICE_PCI_DEVID} This item should be included if the device bus type is \ac{PCI} - the equivalent should be provided for any other bus type. -\pasteAttributeItemEnd - -\reqattrend - -The returned array may optionally contain one or more of the following: - -\optattrstart -\pasteAttributeItem{PMIX_FABRIC_DEVICE_VENDORID} -\pasteAttributeItem{PMIX_FABRIC_DEVICE_DRIVER} -\pasteAttributeItem{PMIX_FABRIC_DEVICE_FIRMWARE} -\pasteAttributeItem{PMIX_FABRIC_DEVICE_ADDRESS} -\pasteAttributeItem{PMIX_FABRIC_DEVICE_MTU} -\pasteAttributeItem{PMIX_FABRIC_DEVICE_SPEED} -\pasteAttributeItem{PMIX_FABRIC_DEVICE_STATE} -\pasteAttributeItem{PMIX_FABRIC_DEVICE_TYPE} -\optattrend - -The caller is responsible for releasing the returned array. - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\subsection{\code{PMIx_Fabric_get_device_info_nb}} -\declareapi{PMIx_Fabric_get_device_info_nb} - -%%%% -\summary - -Given a communication cost matrix index for a specified fabric, return the corresponding device info. - -%%%% -\format - -\versionMarker{4.0} -\cspecificstart -\begin{codepar} -pmix_status_t -PMIx_Fabric_get_device_info_nb(pmix_fabric_t *fabric, - uint32_t index, - pmix_info_cbfunc_t cbfunc, - void *cbdata); -\end{codepar} -\cspecificend - -\begin{arglist} -\argin{fabric}{Address of a \refstruct{pmix_fabric_t} (handle)} -\argin{index}{Device index (i.e., communication cost matrix row or column number) (integer)} -\argin{cbfunc}{Callback function \refapi{pmix_info_cbfunc_t} (function reference)} -\argin{cbdata}{Data to be passed to the callback function (memory reference)} -\end{arglist} - -Returns one of the following: - -\begin{itemize} -\item \refconst{PMIX_SUCCESS} indicating that the request has been accepted for processing and the provided callback function will be executed upon completion of the operation. Note that the library must not invoke the callback function prior to returning from the \ac{API}. -\item a non-zero \ac{PMIx} error constant indicating a reason for the request to have been rejected. In this case, the provided callback function will not be executed -\end{itemize} - -%%%% -\descr - -Non-blocking form of the \refapi{PMIx_Fabric_get_device_info} \ac{API}. Data will be returned in the provided callback function. - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\subsection{\code{PMIx_Fabric_get_device_index}} -\declareapi{PMIx_Fabric_get_device_index} - -%%%% -\summary - -Given device info, return the corresponding communication cost matrix index. - -%%%% -\format - -\versionMarker{4.0} -\cspecificstart -\begin{codepar} -pmix_status_t -PMIx_Fabric_get_device_index(pmix_fabric_t *fabric, - const pmix_info_t device[], - size_t ninfo, - uint32_t *index); -\end{codepar} -\cspecificend - -\begin{arglist} -\argin{fabric}{Address of a \refstruct{pmix_fabric_t} (handle)} -\argin{device}{Array of \refstruct{pmix_info_t} containing info describing the device whose index is being queried (handle)} -\argin{ninfo} number of elements in \refarg{device} -\argout{index}{pointer to the location where the index is to be returned (memory reference (handle))} -\end{arglist} - -Returns one of the following: - -\begin{itemize} - \item \refconst{PMIX_SUCCESS}, indicating return of a valid value. - \item \refconst{PMIX_ERR_NOT_FOUND}, indicating that the specified device is not part of the given fabric - \item a \ac{PMIx} error constant indicating either an error in the input or that the request failed. -\end{itemize} - - -%%%% -\descr - -Query the index number of a device corresponding to the provided description. The description must provide adequate information to uniquely identify the target device. At a minimum, this must include either: - -\begin{enumerate}[label=\alph*)] - \item identification of the node hosting the device using either the \refattr{PMIX_HOSTNAME} or \refattr{PMIX_NODEID}, plus a node-level unique identifier for the device (e.g., the \refattr{PMIX_FABRIC_DEVICE_PCI_DEVID} for a \ac{PCI} device); or - \item the \refattr{PMIX_FABRIC_DEVICE_ID} of the device -\end{enumerate} - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\subsection{\code{PMIx_Fabric_get_device_index_nb}} -\declareapi{PMIx_Fabric_get_device_index_nb} - -%%%% -\summary - -Given device info, return the corresponding communication cost matrix index. - -%%%% -\format - -\versionMarker{4.0} -\cspecificstart -\begin{codepar} -pmix_status_t -PMIx_Fabric_get_device_index_nb(pmix_fabric_t *fabric, - const pmix_info_t device[], - size_t ninfo, - pmix_info_cbfunc_t cbfunc, - void *cbdata); -\end{codepar} -\cspecificend - -\begin{arglist} -\argin{fabric}{Address of a \refstruct{pmix_fabric_t} (handle)} -\argin{device}{Array of \refstruct{pmix_info_t} containing info describing the device whose index is being queried (handle)} -\argin{ninfo} Number of elements in \refarg{vertex} -\argin{cbfunc}{Callback function \refapi{pmix_info_cbfunc_t} (function reference)} -\argin{cbdata}{Data to be passed to the callback function (memory reference)} -\end{arglist} - -Returns one of the following: - -\begin{itemize} -\item \refconst{PMIX_SUCCESS} indicating that the request has been accepted for processing and the provided callback function will be executed upon completion of the operation. Note that the library must not invoke the callback function prior to returning from the \ac{API}. -\item a non-zero \ac{PMIx} error constant indicating a reason for the request to have been rejected. In this case, the provided callback function will not be executed -\end{itemize} - - -%%%% -\descr - -Non-blocking form of the \refapi{PMIx_Fabric_get_device_index} \ac{API}. Index will be returned in the provided callback function via the \refattr{PMIX_FABRIC_DEVICE_INDEX} attribute. - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \subsection{\code{PMIx_Fabric_update_distances}} \declareapi{PMIx_Fabric_update_distances} @@ -1280,7 +1119,7 @@ \subsection{\code{PMIx_Fabric_update_distances}} %%%% \descr -Both the minimum and maximum distance fields in the elements of the array shall be filled with the respective distances between the current process location and the respective fabric device. +Both the minimum and maximum distance fields in the elements of the array shall be filled with the respective distances between the current process location and the respective fabric device. Any distance information stored in the local \ac{PMIx} server's cache should also be updated so that subsequent queries return the updated values. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/Chap_API_Reserved_Keys.tex b/Chap_API_Reserved_Keys.tex index b07a8dc7..4b501692 100644 --- a/Chap_API_Reserved_Keys.tex +++ b/Chap_API_Reserved_Keys.tex @@ -166,6 +166,18 @@ \subsection{Session realm attributes} Number of nodes currently hosting processes in the specified realm. } +% +\declareAttribute{PMIX_NODE_MAP}{"pmix.nmap"}{char*}{ +Regular expression of nodes currently hosting processes in the specified realm - see \ref{cptr:api_server:noderegex} for an explanation of its generation. +} +% +\declareAttribute{PMIX_PROC_MAP}{"pmix.pmap"}{char*}{ +Regular expression describing processes on each node in the specified realm - see \ref{cptr:api_server:ppnregex} for an explanation of its generation. +} +% +\declareAttribute{PMIX_ANL_MAP}{"pmix.anlmap"}{char*}{ +Process mapping in Argonne National Laboratory's PMI-1/PMI-2 notation. +} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \subsection{Job realm attributes} @@ -215,9 +227,19 @@ \subsection{Job realm attributes} point in time. \pasteAttributeItemEnd{} % +\pasteAttributeItemBegin{PMIX_NODE_MAP}In this context, this is the regular expression of nodes currently hosting processes in the specified job. +\pasteAttributeItemEnd{} +% \pasteAttributeItemBegin{PMIX_NODE_LIST}In this context, this is the comma-delimited list of nodes currently hosting processes in the specified job. \pasteAttributeItemEnd{} % +\pasteAttributeItemBegin{PMIX_PROC_MAP}In this context, this is the regular expression describing processes on each node in the specified job. +\pasteAttributeItemEnd{} +% +\pasteAttributeItemBegin{PMIX_ANL_MAP}In this context, this is the +process mapping in Argonne National Laboratory's PMI-1/PMI-2 notation of the processes in the specified job. +\pasteAttributeItemEnd{} +% \declareAttributeNEW{PMIX_CMD_LINE}{"pmix.cmd.line"}{char*}{ Command line used to execute the specified job (e.g., "mpirun -n 2 --map-by foo ./myapp : -n 4 ./myapp2"). } @@ -276,6 +298,32 @@ \subsection{Application realm attributes} \declareAttributeNEW{PMIX_APP_ARGV}{"pmix.app.argv"}{char*}{ Consolidated argv passed to the spawn command for the given application (e.g., "./myapp arg1 arg2 arg3"). } +% +\pasteAttributeItemBegin{PMIX_MAX_PROCS}In this context, this is the maximum number of processes that can be executed in the specified application, which may be a subset of the number allocated to the overall session and job. +\pasteAttributeItemEnd{} +% +\pasteAttributeItemBegin{PMIX_NUM_SLOTS}In this context, this is the number of +slots assigned to the specified application, which may be a subset of the slots +allocated to the overall session and job. +\pasteAttributeItemEnd{} +% +% +\pasteAttributeItemBegin{PMIX_NODE_MAP}In this context, this is the regular expression of nodes currently hosting processes in the specified application. +\pasteAttributeItemEnd{} +% +\pasteAttributeItemBegin{PMIX_NODE_LIST}In this context, this is the comma-delimited list of nodes currently hosting processes in the specified application. +\pasteAttributeItemEnd{} +% +\pasteAttributeItemBegin{PMIX_PROC_MAP}In this context, this is the regular expression describing processes on each node in the specified application. +\pasteAttributeItemEnd{} +% +\declareAttribute{PMIX_APP_MAP_TYPE}{"pmix.apmap.type"}{char*}{ +Type of mapping used to layout the application (e.g., \code{cyclic}). +} +% +\declareAttribute{PMIX_APP_MAP_REGEX}{"pmix.apmap.regex"}{char*}{ +Regular expression describing the result of the process mapping. +} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -366,6 +414,9 @@ \subsection{Process realm attributes} Number of times this process has been re-instantiated - i.e, a value of zero indicates that the process has never been restarted. } +\vspace{\baselineskip} + +In addition, process-level information includes functional attributes directly associated with a process - for example, the process-related fabric attributes included in Section \ref{api:fabric:attrs}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -399,6 +450,10 @@ \subsection{Node realm keys} \declareAttribute{PMIX_NODE_SIZE}{"pmix.node.size"}{uint32_t}{ Number of processes across all jobs that are executing upon the node. } +% +\declareAttribute{PMIX_AVAIL_PHYS_MEMORY}{"pmix.pmem"}{uint64_t}{ +Total available physical memory on a node. +} \vspace{\baselineskip} @@ -421,6 +476,9 @@ \subsection{Node realm keys} Number of processes in the specified job or application realm on the caller's node. Defaults to job realm unless the \refattr{PMIX_APP_INFO} and the \refattr{PMIX_APPNUM} qualifiers are given. } +\vspace{\baselineskip} + +In addition, node-level information includes functional attributes directly associated with a node - for example, the node-related fabric attributes included in Section \ref{api:fabric:attrs}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/Chap_API_Server.tex b/Chap_API_Server.tex index db9f076f..6a6cb395 100644 --- a/Chap_API_Server.tex +++ b/Chap_API_Server.tex @@ -342,7 +342,7 @@ \subsection{\code{PMIx_server_register_nspace}} \argin{nlocalprocs}{number of local processes (integer)} \argin{info}{Array of info structures (array of handles)} \argin{ninfo}{Number of elements in the \refarg{info} array (integer)} -\argin{cbfunc}{Callback function \refapi{pmix_op_cbfunc_t} (function reference)} +\argin{cbfunc}{Callback function \refapi{pmix_op_cbfunc_t} to be executed upon completion of the operation. A \code{NULL} function reference indicates that the function is to be executed as a blocking operation (function reference)} \argin{cbdata}{Data to be passed to the callback function (memory reference)} \end{arglist} @@ -366,9 +366,9 @@ \subsection{\code{PMIx_server_register_nspace}} \divider -Host environments are required to provide a wide range of session-, job-, application-, node-, and process-level information, and may choose to provide a similarly wide range of optional information. The information is broadly separated into categories based on the \emph{level} definitions explained in \ref{api:struct:attributes:storage}. +Host environments are required to provide a wide range of session-, job-, application-, node-, and process-realm information, and may choose to provide a similarly wide range of optional information. The information is broadly separated into categories based on the \emph{data realm} definitions explained in Section \ref{api:struct:attributes:retrieval}, and retrieved according to the rules detailed in Section \ref{chap:rkeys:retrules}. -Session-level information may be passed as individual \refstruct{pmix_info_t} entries, or as part of a \refstruct{pmix_data_array_t} using the \refattr{PMIX_SESSION_INFO_ARRAY} attribute. Session-level information is always accessed using the namespace and the \refconst{PMIX_RANK_WILDCARD} rank, accompanied by the \refattr{PMIX_SESSION_INFO} attribute where ambiguity may exist. The list of data referenced in this way shall include: +Session-realm information may be passed as individual \refstruct{pmix_info_t} entries, or as part of a \refstruct{pmix_data_array_t} using the \refattr{PMIX_SESSION_INFO_ARRAY} attribute.The list of data referenced in this way shall include: \begin{itemize} \item \pasteAttributeItem{PMIX_UNIV_SIZE} @@ -380,11 +380,12 @@ \subsection{\code{PMIx_server_register_nspace}} plus the following optional information: \begin{itemize} - \item \pasteAttributeItem{PMIX_CLUSTER_ID} + \item \pasteAttributeItemBegin{PMIX_CLUSTER_ID}As this information is not related to the namespace, it is best passed using the \refapi{PMIx_server_register_resources} \ac{API}. + \pasteAttributeItemEnd \item \pasteAttributeItem{PMIX_ALLOCATED_NODELIST} \end{itemize} -Job-level information may be passed as individual \refstruct{pmix_info_t} entries, or as part of a \refstruct{pmix_data_array_t} using the \refattr{PMIX_JOB_INFO_ARRAY} attribute. The list of data referenced in this way shall include: +Job-realm information may be passed as individual \refstruct{pmix_info_t} entries, or as part of a \refstruct{pmix_data_array_t} using the \refattr{PMIX_JOB_INFO_ARRAY} attribute. The list of data referenced in this way shall include: \begin{itemize} \item \pasteAttributeItemBegin{PMIX_SERVER_NSPACE}Identifies the namespace of the \ac{PMIx} server itself @@ -414,7 +415,7 @@ \subsection{\code{PMIx_server_register_nspace}} \item \pasteAttributeItem{PMIX_ANL_MAP} \end{itemize} -Application-level information is accessed by providing the namespace of the job with the rank set to \refconst{PMIX_RANK_WILDCARD}. If application-level information is requested for an application other than the one the caller belongs to, then the \refattr{PMIX_APPNUM} attribute must be provided. If more than one application is included in the namespace, then the host environment is also required to supply data consisting of the following items for each application in the job, passed as a \refstruct{pmix_data_array_t} using the \refattr{PMIX_APP_INFO_ARRAY} attribute: +If more than one application is included in the namespace, then the host environment is also required to supply data consisting of the following items for each application in the job, passed as a \refstruct{pmix_data_array_t} using the \refattr{PMIX_APP_INFO_ARRAY} attribute: \begin{itemize} \item \pasteAttributeItemBegin{PMIX_APPNUM}This attribute must appear at the beginning of the array. @@ -444,12 +445,14 @@ \subsection{\code{PMIx_server_register_nspace}} \end{itemize} -Node-level information is accessed by providing the namespace of the job with the rank set to \refconst{PMIX_RANK_WILDCARD}. If node-level information is requested for a node other than the one the caller is executing on, then the \refattr{PMIX_NODEID} or the \refattr{PMIX_HOSTNAME} attribute of the target node must be provided. Registration shall include the following data for each node in the job, passed as a \refstruct{pmix_data_array_t} using the \refattr{PMIX_NODE_INFO_ARRAY} attribute: +Node-realm information may be passed as individual \refstruct{pmix_info_t} entries if only one node will host processes from the job being registered, or as part of a \refstruct{pmix_data_array_t} using the \refattr{PMIX_NODE_INFO_ARRAY} attribute when multiple nodes are involved in the job. The list of data referenced in this way shall include: \begin{itemize} \item \pasteAttributeItem{PMIX_NODEID} - \item \pasteAttributeItem{PMIX_HOSTNAME} - \item \pasteAttributeItem{PMIX_HOSTNAME_ALIASES} + \item \pasteAttributeItemBegin{PMIX_HOSTNAME}As this information is not related to the namespace, it can be passed using the \refapi{PMIx_server_register_resources} \ac{API}. However, either it or the \refattr{PMIX_NODEID} must be included in the array to properly identify the node. + \pasteAttributeItemEnd + \item \pasteAttributeItemBegin{PMIX_HOSTNAME_ALIASES}As this information is not related to the namespace, it is best passed using the \refapi{PMIx_server_register_resources} \ac{API}. + \pasteAttributeItemEnd \item \pasteAttributeItem{PMIX_LOCAL_SIZE} \item \pasteAttributeItem{PMIX_NODE_SIZE} \item \pasteAttributeItem{PMIX_LOCALLDR} @@ -468,17 +471,18 @@ \subsection{\code{PMIx_server_register_nspace}} \begin{itemize} \item \pasteAttributeItem{PMIX_LOCAL_CPUSETS} - \item \pasteAttributeItem{PMIX_AVAIL_PHYS_MEMORY} + \item \pasteAttributeItemBegin{PMIX_AVAIL_PHYS_MEMORY}As this information is not related to the namespace, it can be passed using the \refapi{PMIx_server_register_resources} \ac{API}. + \pasteAttributeItemEnd \end{itemize} -and the following optional information for arbitrary nodes: +and the following optional information for other nodes: \begin{itemize} \item \pasteAttributeItemBegin{PMIX_MAX_PROCS}Requires use of the \refattr{PMIX_NODE_INFO} attribute to avoid ambiguity when retrieving it. \pasteAttributeItemEnd \end{itemize} -Process-level information is accessed by providing the namespace of the job and the rank of the target process. Registration shall include the following data for each process in the job, passed as a \refstruct{pmix_data_array_t} using the \refattr{PMIX_PROC_INFO_ARRAY} attribute: +Process-realm information shall include the following data for each process in the job, passed as a \refstruct{pmix_data_array_t} using the \refattr{PMIX_PROC_INFO_ARRAY} attribute: \begin{itemize} \item \pasteAttributeItem{PMIX_RANK} @@ -527,6 +531,8 @@ \subsection{\code{PMIx_server_register_nspace}} The caller must also provide the number of local processes that will be launched within this namespace. This is required for the \ac{PMIx} server library to correctly handle collectives as a collective operation call can occur before all the local processes have been started. + +A \code{NULL} \refarg{cbfunc} reference indicates that the function is to be executed as a blocking operation. \advicermend \adviceuserstart @@ -535,30 +541,37 @@ \subsection{\code{PMIx_server_register_nspace}} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -\subsubsection{Information storage attributes} +\subsubsection{Namespace registration attributes} \label{api:struct:attributes:storage} -The following attributes are used to assemble information by its level (e.g., \refterm{session}, \refterm{job}, or \refterm{application}) for storage where ambiguity may exist - see \ref{chap:api_server:assemble} for examples of their use. +The following attributes are defined specifically for use with the \refapi{PMIx_server_register_nspace} \ac{API}: +% +\declareAttribute{PMIX_REGISTER_NODATA}{"pmix.reg.nodata"}{bool}{ +Registration is for this namespace only, do not copy job data. +} + +\vspace{\baselineskip} +The following attributes are used to assemble information according to its data realm (\refterm{session}, \refterm{job}, \refterm{application}, \refterm{node}, or \refterm{process} as defined in Section \ref{api:struct:attributes:retrieval}) for registration where ambiguity may exist - see \ref{chap:api_server:assemble} for examples of their use. % \declareAttribute{PMIX_SESSION_INFO_ARRAY}{"pmix.ssn.arr"}{pmix_data_array_t}{ -Provide an array of \refstruct{pmix_info_t} containing session-level information. The \refattr{PMIX_SESSION_ID} attribute is required to be included in the array. +Provide an array of \refstruct{pmix_info_t} containing session-realm information. The \refattr{PMIX_SESSION_ID} attribute is required to be included in the array. } % \declareAttributeNEW{PMIX_JOB_INFO_ARRAY}{"pmix.job.arr"}{pmix_data_array_t}{ -Provide an array of \refstruct{pmix_info_t} containing job-level information. The \refattr{PMIX_SESSION_ID} attribute of the \refterm{session} containing the \refterm{job} is required to be included in the array whenever the \ac{PMIx} server library may host multiple sessions (e.g., when executing with a host \ac{RM} daemon). As information is registered one job (aka namespace) at a time via the \refapi{PMIx_server_register_nspace} \ac{API}, there is no requirement that the array contain either the \refattr{PMIX_NSPACE} or \refattr{PMIX_JOBID} attributes when used in that context (though either or both of them may be included). At least one of the job identifiers must be provided in all other contexts where the job being referenced is ambiguous. +Provide an array of \refstruct{pmix_info_t} containing job-realm information. The \refattr{PMIX_SESSION_ID} attribute of the \refterm{session} containing the \refterm{job} is required to be included in the array whenever the \ac{PMIx} server library may host multiple sessions (e.g., when executing with a host \ac{RM} daemon). As information is registered one job (aka namespace) at a time via the \refapi{PMIx_server_register_nspace} \ac{API}, there is no requirement that the array contain either the \refattr{PMIX_NSPACE} or \refattr{PMIX_JOBID} attributes when used in that context (though either or both of them may be included). At least one of the job identifiers must be provided in all other contexts where the job being referenced is ambiguous. } % \declareAttributeNEW{PMIX_APP_INFO_ARRAY}{"pmix.app.arr"}{pmix_data_array_t}{ -Provide an array of \refstruct{pmix_info_t} containing app-level information. The \refattr{PMIX_NSPACE} or \refattr{PMIX_JOBID} attributes of the \refterm{job} containing the application, plus its \refattr{PMIX_APPNUM} attribute, must to be included in the array when the array is \textit{not} included as part of a call to \refapi{PMIx_server_register_nspace} - i.e., when the job containing the application is ambiguous. The job identification is otherwise optional. +Provide an array of \refstruct{pmix_info_t} containing application-realm information. The \refattr{PMIX_NSPACE} or \refattr{PMIX_JOBID} attributes of the \refterm{job} containing the application, plus its \refattr{PMIX_APPNUM} attribute, must to be included in the array when the array is \textit{not} included as part of a call to \refapi{PMIx_server_register_nspace} - i.e., when the job containing the application is ambiguous. The job identification is otherwise optional. } % \declareAttributeNEW{PMIX_PROC_INFO_ARRAY}{"pmix.pdata"}{pmix_data_array_t}{ -Provide an array of \refstruct{pmix_info_t} containing process-level information. The \refattr{PMIX_RANK} and \refattr{PMIX_NSPACE} attributes, or the \refattr{PMIX_PROCID} attribute, are required to be included in the array when the array is not included as part of a call to \refapi{PMIx_server_register_nspace} - i.e., when the job containing the process is ambiguous. All three may be included if desired. When the array is included in some broader structure that identifies the job, then only the \refattr{PMIX_RANK} or the \refattr{PMIX_PROCID} attribute must be included (the others are optional). +Provide an array of \refstruct{pmix_info_t} containing process-realm information. The \refattr{PMIX_RANK} and \refattr{PMIX_NSPACE} attributes, or the \refattr{PMIX_PROCID} attribute, are required to be included in the array when the array is not included as part of a call to \refapi{PMIx_server_register_nspace} - i.e., when the job containing the process is ambiguous. All three may be included if desired. When the array is included in some broader structure that identifies the job, then only the \refattr{PMIX_RANK} or the \refattr{PMIX_PROCID} attribute must be included (the others are optional). } % \declareAttributeNEW{PMIX_NODE_INFO_ARRAY}{"pmix.node.arr"}{pmix_data_array_t}{ -Provide an array of \refstruct{pmix_info_t} containing node-level information. At a minimum, either the \refattr{PMIX_NODEID} or \refattr{PMIX_HOSTNAME} attribute is required to be included in the array, though both may be included. +Provide an array of \refstruct{pmix_info_t} containing node-realm information. At a minimum, either the \refattr{PMIX_NODEID} or \refattr{PMIX_HOSTNAME} attribute is required to be included in the array, though both may be included. } Note that these assemblages can be used hierarchically: @@ -573,31 +586,6 @@ \subsubsection{Information storage attributes} \ac{PMIx} implementations must be capable of properly parsing and storing any hierarchical depth of information arrays. The resulting stored values are must to be accessible via both \refapi{PMIx_Get} and \refapi{PMIx_Query_info_nb} \acp{API}, assuming appropriate directives are provided by the caller. \adviceimplend -% -\declareAttribute{PMIX_REGISTER_NODATA}{"pmix.reg.nodata"}{bool}{ -Registration is for this namespace only, do not copy job data. -} -% -\declareAttribute{PMIX_NODE_MAP}{"pmix.nmap"}{char*}{ -Regular expression of nodes - see \ref{cptr:api_server:noderegex} for an explanation of its generation. -} -% -\declareAttribute{PMIX_PROC_MAP}{"pmix.pmap"}{char*}{ -Regular expression describing processes on each node - see \ref{cptr:api_server:ppnregex} for an explanation of its generation. -} -% -\declareAttribute{PMIX_ANL_MAP}{"pmix.anlmap"}{char*}{ -Process mapping in Argonne National Laboratory's PMI-1/PMI-2 notation. -} -% -\declareAttribute{PMIX_APP_MAP_TYPE}{"pmix.apmap.type"}{char*}{ -Type of mapping used to layout the application (e.g., \code{cyclic}). -} -% -\declareAttribute{PMIX_APP_MAP_REGEX}{"pmix.apmap.regex"}{char*}{ -Regular expression describing the result of the process mapping. -} - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \subsubsection{Assembling the registration information} \label{chap:api_server:assemble} @@ -869,7 +857,7 @@ \subsection{\code{PMIx_server_deregister_nspace}} \begin{arglist} \argin{nspace}{Namespace (string)} -\argin{cbfunc}{Callback function \refapi{pmix_op_cbfunc_t} (function reference)} +\argin{cbfunc}{Callback function \refapi{pmix_op_cbfunc_t}. A \code{NULL} function reference indicates that the function is to be executed as a blocking operation. (function reference)} \argin{cbdata}{Data to be passed to the callback function (memory reference)} \end{arglist} @@ -877,7 +865,89 @@ \subsection{\code{PMIx_server_deregister_nspace}} \descr Deregister the specified \refarg{nspace} and purge all objects relating to it, including any client information from that namespace. -This is intended to support persistent \ac{PMIx} servers by providing an opportunity for the host \ac{RM} to tell the \ac{PMIx} server library to release all memory for a completed job. Note that the library must not invoke the callback function prior to returning from the \ac{API}. +This is intended to support persistent \ac{PMIx} servers by providing an opportunity for the host \ac{RM} to tell the \ac{PMIx} server library to release all memory for a completed job. Note that the library must not invoke the callback function prior to returning from the \ac{API}, and that a \code{NULL} \refarg{cbfunc} reference indicates that the function is to be executed as a blocking operation. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\subsection{\code{PMIx_server_register_resources}} +\declareapi{PMIx_server_register_resources} + +%%%% +\summary + +Register non-namespace related information with the local \ac{PMIx} server library. + +%%%% +\format + +\versionMarker{4.0} +\cspecificstart +\begin{codepar} +pmix_status_t +PMIx_server_register_resources(pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, + void *cbdata); +\end{codepar} +\cspecificend + +\begin{arglist} +\argin{info}{Array of info structures (array of handles)} +\argin{ninfo}{Number of elements in the \refarg{info} array (integer)} +\argin{cbfunc}{Callback function \refapi{pmix_op_cbfunc_t}. A \code{NULL} function reference indicates that the function is to be executed as a blocking operation (function reference)} +\argin{cbdata}{Data to be passed to the callback function (memory reference)} +\end{arglist} + +%%%% +\descr + +Pass information about resources not associated with a given namespace to the \ac{PMIx} server library for distribution to local client processes. This includes information on fabric devices, \acp{GPU}, and other resources. All information provided through this \ac{API} shall be made available to each job as part of its job-level information. Duplicate information provided with the \refapi{PMIx_server_register_nspace} \ac{API} shall override any information provided by this function for that namespace, but only for that specific namespace. + +\advicermstart +Note that information passed in this manner could also have been included in a call to \refapi{PMIx_server_register_nspace} - e.g., as part of a \refattr{PMIX_NODE_INFO_ARRAY} array. This \ac{API} is provided as a logical alternative for code clarity, especially where multiple jobs may be supported by a single \ac{PMIx} server library instance, to avoid multiple registration of static resource information. + +A \code{NULL} \refarg{cbfunc} reference indicates that the function is to be executed as a blocking operation. +\advicermend + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\subsection{\code{PMIx_server_deregister_resources}} +\declareapi{PMIx_server_deregister_resources} + +%%%% +\summary + +Remove specified non-namespace related information from the local \ac{PMIx} server library. + +%%%% +\format + +\versionMarker{4.0} +\cspecificstart +\begin{codepar} +pmix_status_t +PMIx_server_deregister_resources(pmix_info_t info[], size_t ninfo, + pmix_op_cbfunc_t cbfunc, + void *cbdata); +\end{codepar} +\cspecificend + +\begin{arglist} +\argin{info}{Array of info structures (array of handles)} +\argin{ninfo}{Number of elements in the \refarg{info} array (integer)} +\argin{cbfunc}{Callback function \refapi{pmix_op_cbfunc_t}. A \code{NULL} function reference indicates that the function is to be executed as a blocking operation (function reference)} +\argin{cbdata}{Data to be passed to the callback function (memory reference)} +\end{arglist} + +%%%% +\descr + +Remove information about resources not associated with a given namespace from the \ac{PMIx} server library. Only the \refarg{key} fields of the provided \refarg{info} array shall be used for the operation - the associated values shall be ignored except where they serve as qualifiers to the request. For example, to remove a specific fabric device from a given node, the \refarg{info} array might include a \refattr{PMIX_NODE_INFO_ARRAY} containing the \refattr{PMIX_NODEID} or \refattr{PMIX_HOSTNAME} identifying the node hosting the device, and the \refattr{PMIX_FABRIC_DEVICE_NAME} specifying the device to be removed. Alternatively, the device could be removed using only the \refattr{PMIX_FABRIC_DEVICE_ID} as this is unique across the overall system. + +\advicermstart +As information not related to namespaces is considered \emph{static}, there is no requirement that the host environment deregister resources prior to finalizing the \ac{PMIx} server library. The server library shall properly cleanup as part of its normal finalize operations. Deregistration of resources is only required, therefore, when the host environment determines that client processes should no longer have access to that information. + +A \code{NULL} \refarg{cbfunc} reference indicates that the function is to be executed as a blocking operation. +\advicermend %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1773,6 +1843,25 @@ \subsubsection{Cpuset support macros} \end{arglist} + +\littleheader{Create a cpuset array} +\declaremacro{PMIX_CPUSET_CREATE} + +Allocate and initialize a \refstruct{pmix_cpuset_t} array. + +\versionMarker{4.0} +\cspecificstart +\begin{codepar} +PMIX_CPUSET_CREATE(m, n) +\end{codepar} +\cspecificend + +\begin{arglist} +\arginout{m}{Address where the pointer to the array of \refstruct{pmix_cpuset_t} structures shall be stored (handle)} +\argin{n}{Number of structures to be allocated (size_t)} +\end{arglist} + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \subsection{\code{PMIx_server_define_process_set}} \declareapi{PMIx_server_define_process_set} diff --git a/Chap_API_Struct.tex b/Chap_API_Struct.tex index 2a7ee694..d03f1f91 100644 --- a/Chap_API_Struct.tex +++ b/Chap_API_Struct.tex @@ -2092,6 +2092,18 @@ \section{Generalized Data Types Used for Packing/Unpacking} \declareconstitemNEW{PMIX_LINK_STATE} Link state (\refstruct{pmix_link_state_t}). % +\declareconstitemNEW{PMIX_PROC_CPUSET} +Structure containing the binding bitmap of a process (\refstruct{pmix_cpuset_t}). +% +\declareconstitemNEW{PMIX_GEOMETRY} +Geometry structure containing the fabric coordinates of a specified device.(\refstruct{pmix_geometry_t}). +% +\declareconstitemNEW{PMIX_DEVICE_DIST} +Structure containing the minimum and maximum relative distance from the caller to a given fabric device. (\refstruct{pmix_device_distance_t}). +% +\declareconstitemNEW{PMIX_ENDPOINT} +Structure containing an assigned endpoint for a given fabric device. (\refstruct{pmix_endpoint_t}). +% \declareconstitemNEW{PMIX_DATA_TYPE_MAX} A starting point for implementer-specific data types. Values above this are guaranteed not to conflict with \ac{PMIx} values. diff --git a/Chap_API_Sync_Access.tex b/Chap_API_Sync_Access.tex index 0b7a03b4..08423e1e 100644 --- a/Chap_API_Sync_Access.tex +++ b/Chap_API_Sync_Access.tex @@ -572,7 +572,7 @@ \subsection{\code{PMIx_Query_info}} This can include a list of active namespaces, fabric topology, etc. Also can be used to query node-specific info such as the list of peers executing on a given node. The host environment is responsible for exercising appropriate access control on the information. -The returned \refarg{status} indicates if requested data was found or not. The returned \refarg{info} array will contain an element for each key in each element of the \refarg{queries} array for which a value was found, in the order of the queries and the keys within each query. The element shall be referenced by the query \emph{key} and contain a \refstruct{pmix_data_array_t} of \refstruct{pmix_info_t} responses. If qualifiers were included in the query, then the first element of the array shall contain the \refattr{PMIX_QUERY_QUALIFIERS} key with a \refstruct{pmix_data_array_t} containing the qualifiers. The remaining \refstruct{pmix_info_t} shall contain the results of the query. Note that duplicate keys in the \refarg{queries} array shall result in duplicate responses within the constraints of the accompanying qualifiers. The caller is responsible for releasing the returned array. +The returned \refarg{status} indicates if requested data was found or not. The returned \refarg{info} array will contain a \refattr{PMIX_QUERY_RESULTS} element for each query of the \refarg{queries} array. If qualifiers were included in the query, then the first element of each results array shall contain the \refattr{PMIX_QUERY_QUALIFIERS} key with a \refstruct{pmix_data_array_t} containing the qualifiers. The remaining \refstruct{pmix_info_t} shall contain the results of the query, one entry for each key that was found. Note that duplicate keys in the \refarg{queries} array shall result in duplicate responses within the constraints of the accompanying qualifiers. The caller is responsible for releasing the returned array. \adviceimplstart Information returned from \refapi{PMIx_Query_info} shall be locally cached so that retrieval by subsequent calls to \refapi{PMIx_Get}, \refapi{PMIx_Query_info}, or \refapi{PMIx_Query_info_nb} can succeed with minimal overhead. The local cache shall be checked prior to querying the \ac{PMIx} server and/or the host environment. Queries that include the \refattr{PMIX_QUERY_REFRESH_CACHE} attribute shall bypass the local cache and retrieve a new value for the query, refreshing the values in the cache upon return. @@ -703,9 +703,13 @@ \subsection{Query attributes} Attributes used to direct behavior of the \refapi{PMIx_Query_info} \acp{API}. +% +\declareAttributeNEW{PMIX_QUERY_RESULTS}{"pmix.qry.res"}{pmix_data_array_t}{ +Contains an array of query results for a given \refstruct{pmix_query_t} passed to the \refapi{PMIx_Query_info} \acp{API}. If qualifiers were included in the query, then the first element of the array shall be the \refattr{PMIX_QUERY_QUALIFIERS} attribute containing those qualifiers. Each of the remaining elements of the array is a \refstruct{pmix_info_t} containing the query key and the corresponding value returned by the query. This attribute is solely for reporting purposes and cannot be used in \refapi{PMIx_Get} or other query operations. +} % \declareAttributeNEW{PMIX_QUERY_QUALIFIERS}{"pmix.qry.quals"}{pmix_data_array_t}{ -Contains an array of qualifiers that were included in the query that produced the provided results. NO QUALIFIERS. +Contains an array of qualifiers that were included in the query that produced the provided results. This attribute is solely for reporting purposes and cannot be used in \refapi{PMIx_Get} or other query operations. } % \declareAttributeNEW{PMIX_QUERY_SUPPORTED_KEYS}{"pmix.qry.keys"}{char*}{ @@ -801,9 +805,8 @@ \subsection{Query attributes} These attributes are used to query memory available and used in the system. % -\declareAttribute{PMIX_AVAIL_PHYS_MEMORY}{"pmix.pmem"}{uint64_t}{ -Total available physical memory on a node. OPTIONAL QUALIFERS: \refattr{PMIX_HOSTNAME} or \refattr{PMIX_NODEID} (defaults to caller's node). -} +\pasteAttributeItemBegin{PMIX_AVAIL_PHYS_MEMORY}OPTIONAL QUALIFERS: \refattr{PMIX_HOSTNAME} or \refattr{PMIX_NODEID} (defaults to caller's node). +\pasteAttributeItemEnd % \declareAttribute{PMIX_DAEMON_MEMORY}{"pmix.dmn.mem"}{float}{ Megabytes of memory currently used by the \ac{RM} daemon on the node. OPTIONAL QUALIFERS: \refattr{PMIX_HOSTNAME} or \refattr{PMIX_NODEID} (defaults to caller's node). diff --git a/Chap_Revisions.tex b/Chap_Revisions.tex index bfa5e2bf..0cce9a76 100644 --- a/Chap_Revisions.tex +++ b/Chap_Revisions.tex @@ -363,8 +363,6 @@ \section{Version 4.0: Sept 2020} \item \refapi{PMIx_Fabric_register}, \refapi{PMIx_Fabric_register_nb} \item \refapi{PMIx_Fabric_update}, \refapi{PMIx_Fabric_update_nb} \item \refapi{PMIx_Fabric_deregister}, \refapi{PMIx_Fabric_deregister_nb} - \item \refapi{PMIx_Fabric_get_device_info}, \refapi{PMIx_Fabric_get_device_info_nb} - \item \refapi{PMIx_Fabric_get_device_index}, \refapi{PMIx_Fabric_get_device_index_nb} \item \refapi{PMIx_Fabric_update_distances}, \refapi{PMIx_Fabric_update_distances_nb} \end{compactitemize} @@ -376,6 +374,7 @@ \section{Version 4.0: Sept 2020} \item \refapi{pmix_server_grp_fn_t}, \refapi{pmix_server_fabric_fn_t} \item \refapi{pmix_server_client_connected2_fn_t} \item \refapi{PMIx_server_generate_cpuset_string} + \item \refapi{PMIx_server_register_resources}, \refapi{PMIx_server_deregister_resources} \end{compactitemize} \item Tool \acp{API} @@ -406,6 +405,18 @@ \section{Version 4.0: Sept 2020} \subsection{Added Constants} +\littleheader{Data type constants} +\refconst{PMIX_COORD} +\refconst{PMIX_REGATTR} +\refconst{PMIX_REGEX} +\refconst{PMIX_JOB_STATE} +\refconst{PMIX_LINK_STATE} +\refconst{PMIX_PROC_CPUSET} +\refconst{PMIX_GEOMETRY} +\refconst{PMIX_DEVICE_DIST} +\refconst{PMIX_ENDPOINT} +\refconst{PMIX_DATA_TYPE_MAX} + \littleheader{Query constants} \refconst{PMIX_ERR_GET_MALLOC_REQD} % @@ -458,8 +469,6 @@ \subsection{Added Constants} \refconst{PMIX_LINK_UP} \\ \refconst{PMIX_FABRIC_REQUEST_INFO} \\ \refconst{PMIX_FABRIC_UPDATE_INFO} \\ -\refconst{PMIX_FABRIC_GET_DEVICE_INFO} \\ -\refconst{PMIX_FABRIC_GET_DEVICE_INDEX} \\ % % \littleheader{Sets-Groups constants} @@ -512,6 +521,7 @@ \subsection{Added Attributes} \pasteAttributeItem{PMIX_ALL_CLONES_PARTICIPATE} \pasteAttributeItem{PMIX_GET_STATIC_VALUES} \pasteAttributeItem{PMIX_GET_REFRESH_CACHE} +\pasteAttributeItem{PMIX_QUERY_RESULTS} \pasteAttributeItem{PMIX_QUERY_QUALIFIERS} \pasteAttributeItem{PMIX_QUERY_SUPPORTED_KEYS} \pasteAttributeItem{PMIX_QUERY_SUPPORTED_QUALIFIERS} @@ -605,7 +615,6 @@ \subsection{Added Attributes} \pasteAttributeItem{PMIX_SWITCH_PEERS} \pasteAttributeItem{PMIX_FABRIC_PLANE} \pasteAttributeItem{PMIX_FABRIC_SWITCH} -\pasteAttributeItem{PMIX_FABRIC_VIEW} \pasteAttributeItem{PMIX_FABRIC_DEVICE} \pasteAttributeItem{PMIX_FABRIC_DEVICE_ID} \pasteAttributeItem{PMIX_FABRIC_DEVICE_INDEX} @@ -616,6 +625,7 @@ \subsection{Added Attributes} \pasteAttributeItem{PMIX_FABRIC_DEVICE_DRIVER} \pasteAttributeItem{PMIX_FABRIC_DEVICE_FIRMWARE} \pasteAttributeItem{PMIX_FABRIC_DEVICE_ADDRESS} +\pasteAttributeItem{PMIX_FABRIC_DEVICE_COORDINATES} \pasteAttributeItem{PMIX_FABRIC_DEVICE_MTU} \pasteAttributeItem{PMIX_FABRIC_DEVICE_SPEED} \pasteAttributeItem{PMIX_FABRIC_DEVICE_STATE} diff --git a/pmix-standard.tex b/pmix-standard.tex index 18d3727c..fe0e2fc8 100644 --- a/pmix-standard.tex +++ b/pmix-standard.tex @@ -128,6 +128,7 @@ \acrodef{FQDN}{Fully Qualified Domain Name} \acrodef{NUMA}{Non-Uniform Memory Access} \acrodef{UUID}{Universally Unique IDentifier} +\acrodef{GPU}{Graphics Processing Unit} %%%%%%%%%%%%%%%%%%% @@ -184,6 +185,14 @@ \input{Chap_API_NonReserved_Keys.tex} \input{Chap_API_Publish.tex} + % Event Handling + % - (de)register_event, notify_event + \input{Chap_API_Event.tex} + + % Data Packing & Unpacking + % - (un)pack, copy + \input{Chap_API_Data_Mgmt.tex} + % Process Management % - spawn, (dis)connect, resolve_peers \input{Chap_API_Proc_Mgmt.tex} @@ -192,13 +201,12 @@ % - Allocation request, process monitoring \input{Chap_API_Job_Mgmt.tex} - % Event Handling - % - (de)register_event, notify_event - \input{Chap_API_Event.tex} + % PMIx Process Sets and Groups + \input{Chap_API_Sets_Groups.tex} - % Data Packing & Unpacking - % - (un)pack, copy - \input{Chap_API_Data_Mgmt.tex} + % PMIx fabric support interfaces + % - register/deregister fabric, get vertex/index + \input{Chap_API_Fabric.tex} % Security credentials \input{Chap_API_Security.tex} @@ -207,13 +215,6 @@ % - setup_fork, (de)register_nspace, pmix_server_module_t \input{Chap_API_Server.tex} - % PMIx fabric support interfaces - % - register/deregister fabric, get vertex/index - \input{Chap_API_Fabric.tex} - - % PMIx Process Sets and Groups - \input{Chap_API_Sets_Groups.tex} - % PMIx Tools and Debugger Support \input{Chap_API_Tools.tex}