From b6e2763d5c94e1308ca6fcf2a6e2571553e81cbb Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Fri, 12 Jan 2024 12:39:40 -0800 Subject: [PATCH] Document class and type descriptions. --- cpp/include/ucxx/address.h | 6 +++ cpp/include/ucxx/buffer.h | 34 ++++++++++++ cpp/include/ucxx/config.h | 6 +++ cpp/include/ucxx/constructors.h | 2 +- cpp/include/ucxx/context.h | 9 +++- cpp/include/ucxx/delayed_submission.h | 49 ++++++++++++++++++ cpp/include/ucxx/endpoint.h | 27 +++++++++- cpp/include/ucxx/future.h | 6 +++ cpp/include/ucxx/header.h | 10 +++- cpp/include/ucxx/inflight_requests.h | 36 ++++++++++++- cpp/include/ucxx/internal/request_am.h | 13 +++++ cpp/include/ucxx/listener.h | 6 +++ cpp/include/ucxx/log.h | 33 ++++++++++-- cpp/include/ucxx/notifier.h | 18 +++++++ cpp/include/ucxx/request.h | 8 +++ cpp/include/ucxx/request_am.h | 6 +++ cpp/include/ucxx/request_data.h | 56 ++++++++++++++++++-- cpp/include/ucxx/request_helper.h | 20 ++++++++ cpp/include/ucxx/request_stream.h | 6 +++ cpp/include/ucxx/request_tag.h | 6 +++ cpp/include/ucxx/request_tag_multi.h | 26 ++++++++++ cpp/include/ucxx/typedefs.h | 60 ++++++++++++++++++++-- cpp/include/ucxx/utils/callback_notifier.h | 6 +++ cpp/include/ucxx/worker.h | 6 +++ cpp/include/ucxx/worker_progress_thread.h | 33 ++++++++++++ 25 files changed, 472 insertions(+), 16 deletions(-) diff --git a/cpp/include/ucxx/address.h b/cpp/include/ucxx/address.h index 32147a0f..80fb6128 100644 --- a/cpp/include/ucxx/address.h +++ b/cpp/include/ucxx/address.h @@ -14,6 +14,12 @@ namespace ucxx { +/** + * @brief Component encapsulating the address of a UCP worker. + * + * A UCP worker has a unique address that can is contained in a `ucp_address_t*` object, + * this class encapsulates that object and provides methods to simplify its handling. + */ class Address : public Component { private: ucp_address_t* _handle{nullptr}; diff --git a/cpp/include/ucxx/buffer.h b/cpp/include/ucxx/buffer.h index 7328c3fc..3fa0f3b0 100644 --- a/cpp/include/ucxx/buffer.h +++ b/cpp/include/ucxx/buffer.h @@ -15,12 +15,25 @@ namespace ucxx { +/** + * @brief The type of a buffer. + * + * The type of a buffer that can be used to match among the different supported types. + */ enum class BufferType { Host = 0, RMM, Invalid, }; +/** + * @brief A simple object to simplify managing buffers. + * + * UCXX can work with raw pointers in most cases, but in some circumstances it's required + * to know more information about the buffer, such as with `ucxx::RequestTagMulti`. In such + * circumstances it may need to allocate buffers internally and UCXX can utilize objects of + * this type to describe the internally-allocated buffers. + */ class Buffer { protected: BufferType _bufferType{BufferType::Invalid}; ///< Buffer type @@ -82,6 +95,11 @@ class Buffer { virtual void* data() = 0; }; +/** + * @brief A simple object containing a host buffer. + * + * A buffer encapsulating a host buffer with its properties. + */ class HostBuffer : public Buffer { private: void* _buffer; ///< Pointer to the allocated buffer @@ -166,6 +184,11 @@ class HostBuffer : public Buffer { }; #if UCXX_ENABLE_RMM +/** + * @brief A simple object containing a RMM (CUDA) buffer. + * + * A buffer encapsulating an RMM (CUDA) buffer with its properties. + */ class RMMBuffer : public Buffer { private: std::unique_ptr _buffer; ///< RMM-allocated device buffer @@ -245,6 +268,17 @@ class RMMBuffer : public Buffer { }; #endif +/** + * @brief Allocate a buffer of specified type and size. + * + * Allocate a buffer of the specified type and size pair, returning the `ucxx::Buffer` + * object wrapped in a `std::shared_ptr`. + * + * @param[in] bufferType the type of buffer to allocate. + * @param[in] size the size (in bytes) of the buffer to allocate. + * + * @returns the `std::shared_ptr` to the allocated buffer. + */ std::shared_ptr allocateBuffer(BufferType bufferType, const size_t size); } // namespace ucxx diff --git a/cpp/include/ucxx/config.h b/cpp/include/ucxx/config.h index 962eef90..e6c7f542 100644 --- a/cpp/include/ucxx/config.h +++ b/cpp/include/ucxx/config.h @@ -10,6 +10,12 @@ namespace ucxx { +/** + * @brief Component encapsulating the UCP configuration. + * + * The UCP layer provides a handle to its configuration in form of `ucp_config_t*` object, + * this class encapsulates that object and provides methods to simplify its handling. + */ class Config { private: ucp_config_t* _handle{nullptr}; ///< Handle to the UCP config diff --git a/cpp/include/ucxx/constructors.h b/cpp/include/ucxx/constructors.h index bcda9583..ab8e7d17 100644 --- a/cpp/include/ucxx/constructors.h +++ b/cpp/include/ucxx/constructors.h @@ -27,7 +27,7 @@ class RequestTagMulti; class Worker; // Components -std::shared_ptr
createAddressFromWorker(std::shared_ptr worker); +std::shared_ptr
createAddressFromWorker(std::shared_ptr worker); std::shared_ptr
createAddressFromString(std::string addressString); diff --git a/cpp/include/ucxx/context.h b/cpp/include/ucxx/context.h index 00065362..272f7eab 100644 --- a/cpp/include/ucxx/context.h +++ b/cpp/include/ucxx/context.h @@ -19,6 +19,12 @@ namespace ucxx { class Worker; +/** + * @brief Component encapsulating the UCP context. + * + * The UCP layer provides a handle to access its context in form of `ucp_context_h` object, + * this class encapsulates that object and provides methods to simplify its handling. + */ class Context : public Component { private: ucp_context_h _handle{nullptr}; ///< The UCP context handle @@ -40,7 +46,8 @@ class Context : public Component { public: static constexpr uint64_t defaultFeatureFlags = - UCP_FEATURE_TAG | UCP_FEATURE_WAKEUP | UCP_FEATURE_STREAM | UCP_FEATURE_AM | UCP_FEATURE_RMA; + UCP_FEATURE_TAG | UCP_FEATURE_WAKEUP | UCP_FEATURE_STREAM | UCP_FEATURE_AM | + UCP_FEATURE_RMA; ///< Suggested default context feature flags to use. Context() = delete; Context(const Context&) = delete; diff --git a/cpp/include/ucxx/delayed_submission.h b/cpp/include/ucxx/delayed_submission.h index 002018fd..374a9287 100644 --- a/cpp/include/ucxx/delayed_submission.h +++ b/cpp/include/ucxx/delayed_submission.h @@ -22,8 +22,20 @@ namespace ucxx { +/** + * @brief A user-defined function to execute as part of delayed submission callback. + * + * A user-defined function to execute in the scope of a `ucxx::DelayedSubmission`, allowing + * execution of custom code upon the completion of the delayed submission. + */ typedef std::function DelayedSubmissionCallbackType; +/** + * @brief Base type for a collection of delayed submissions. + * + * Base type for a collection of delayed submission. Delayed submissions may have different + * purposes and this class encapsulates generic data for all derived types. + */ template class BaseDelayedSubmissionCollection { protected: @@ -126,6 +138,12 @@ class BaseDelayedSubmissionCollection { } }; +/** + * @brief A collection of delayed request submissions. + * + * A collection of delayed submissions used specifically for message transfer + * `ucxx::Request` submissions. + */ class RequestDelayedSubmissionCollection : public BaseDelayedSubmissionCollection< std::pair, DelayedSubmissionCallbackType>> { @@ -137,9 +155,25 @@ class RequestDelayedSubmissionCollection std::pair, DelayedSubmissionCallbackType> item) override; public: + /** + * @brief Constructor of a collection of delayed request submissions. + * + * Construct a collection of delayed submissions used specifically for message transfer + * `ucxx::Request` submissions. + * + * @param[in] name the human-readable name of the type of delayed submission for + * debugging purposes. + * @param[in] enabled whether delayed request submissions should be enabled. + */ explicit RequestDelayedSubmissionCollection(const std::string name, const bool enabled); }; +/** + * @brief A collection of delayed submissions of generic callbacks. + * + * A collection of delayed submissions used specifically for execution of generic callbacks + * at pre-defined stages of the progress loop. + */ class GenericDelayedSubmissionCollection : public BaseDelayedSubmissionCollection { protected: @@ -148,9 +182,24 @@ class GenericDelayedSubmissionCollection void processItem(DelayedSubmissionCallbackType callback) override; public: + /** + * @brief Constructor of a collection of delayed submissions of generic callbacks. + * + * Construct a collection of delayed submissions used specifically for execution of + * generic callbacks at pre-defined stages of the progress loop. + * + * @param[in] name the human-readable name of the type of delayed submission for + * debugging purposes. + */ explicit GenericDelayedSubmissionCollection(const std::string name); }; +/** + * @brief A collection of delayed submissions of multiple types. + * + * A collection of delayed submissions of multiple types used by the owner to manage each + * of the delayed submission types via specialized methods. + */ class DelayedSubmissionCollection { private: GenericDelayedSubmissionCollection _genericPre{ diff --git a/cpp/include/ucxx/endpoint.h b/cpp/include/ucxx/endpoint.h index 4d7e1d55..489e2aa5 100644 --- a/cpp/include/ucxx/endpoint.h +++ b/cpp/include/ucxx/endpoint.h @@ -24,10 +24,29 @@ namespace ucxx { +/** + * @brief Deleter for a endpoint parameters object. + * + * Deleter used during allocation of a `ucp_ep_params_t*` to handle automated deletion of + * the object when its reference count goes to zero. + */ struct EpParamsDeleter { + /** + * @brief Execute the deletion. + * + * Execute the deletion of the `ucp_ep_params_t*` object. + * + * param[in] ptr the point to the object to be deleted. + */ void operator()(ucp_ep_params_t* ptr); }; +/** + * @brief The endpoint data that is accessible by the error callback. + * + * The `ucxx::Endpoint` data that is accessible by the asynchronous UCP endpoint error + * callback to modify the `ucxx::Endpoint` with information relevant to the error occurred. + */ struct ErrorCallbackData { ucs_status_t status; ///< Endpoint status std::shared_ptr inflightRequests; ///< Endpoint inflight requests @@ -36,6 +55,12 @@ struct ErrorCallbackData { std::shared_ptr worker; ///< Worker the endpoint has been created from }; +/** + * @brief Component encapsulating a UCP endpoint. + * + * The UCP layer provides a handle to access endpoints in form of `ucp_ep_h` object, + * this class encapsulates that object and provides methods to simplify its handling. + */ class Endpoint : public Component { private: ucp_ep_h _handle{nullptr}; ///< Handle to the UCP endpoint @@ -260,8 +285,6 @@ class Endpoint : public Component { * @param[in] closeCallback `std::function` to a function definition return `void` and * receiving a single opaque pointer. * @param[in] closeCallbackArg pointer to optional user-allocated callback argument. - * - * @returns Number of requests that were canceled. */ void setCloseCallback(std::function closeCallback, void* closeCallbackArg); diff --git a/cpp/include/ucxx/future.h b/cpp/include/ucxx/future.h index 860f7794..398b9eba 100644 --- a/cpp/include/ucxx/future.h +++ b/cpp/include/ucxx/future.h @@ -12,6 +12,12 @@ namespace ucxx { +/** + * @brief Represent a future that may be notified by a specialized notifier. + * + * Represent a future object that may postpone notification of its status to a more + * appropriate stage by a specialize notifier, such as `ucxx::Notifier`. + */ class Future : public std::enable_shared_from_this { protected: std::shared_ptr _notifier{nullptr}; ///< The notifier object diff --git a/cpp/include/ucxx/header.h b/cpp/include/ucxx/header.h index 4db1b91b..ad967b99 100644 --- a/cpp/include/ucxx/header.h +++ b/cpp/include/ucxx/header.h @@ -10,8 +10,16 @@ namespace ucxx { -const size_t HeaderFramesSize = 100; +const size_t HeaderFramesSize = + 100; ///< The number of buffers contained in a single `ucxx::Header` object. +/** + * @brief A serializable object containing metadata of multiple buffers. + * + * A serializable object containing metadata of a pre-defined number of buffers used to + * inform the remote endpoint of multiple incoming messages from buffers of given + * properties. + */ class Header { private: /** diff --git a/cpp/include/ucxx/inflight_requests.h b/cpp/include/ucxx/inflight_requests.h index 1595fc3a..de27584c 100644 --- a/cpp/include/ucxx/inflight_requests.h +++ b/cpp/include/ucxx/inflight_requests.h @@ -13,11 +13,30 @@ namespace ucxx { class Request; +/** + * @brief An inflight request map. + * + * A map of inflight requests, where keys are a unique identifier of the request and + * value is the reference-counted `ucxx::Request`. + */ typedef std::map> InflightRequestsMap; + +/** + * @brief Pre-defined type for a pointer to an inflight request map. + * + * A pre-defined type for a pointer to an inflight request map, used as a convenience type. + */ typedef std::unique_ptr InflightRequestsMapPtr; + +/** + * @brief A container for the different types of tracked requests. + * + * A container encapsulating the different types of handled tracked requests, currently + * those still valid (inflight), and those scheduled for cancelation (canceling). + */ typedef struct TrackedRequests { - InflightRequestsMapPtr _inflight; - InflightRequestsMapPtr _canceling; + InflightRequestsMapPtr _inflight; ///< Valid requests awaiting completion. + InflightRequestsMapPtr _canceling; ///< Requests scheduled for cancelation. TrackedRequests() : _inflight(std::make_unique()), @@ -25,8 +44,21 @@ typedef struct TrackedRequests { { } } TrackedRequests; + +/** + * @brief Pre-defined type for a pointer to a container of tracked requests. + * + * A pre-defined type for a pointer to a container of tracked requests, used as a + * convenience type. + */ typedef std::unique_ptr TrackedRequestsPtr; +/** + * @brief Handle tracked requests. + * + * Handle tracked requests, providing functionality so that its owner can modify those + * requests, performing operations such as insertion, removal and cancelation. + */ class InflightRequests { private: TrackedRequestsPtr _trackedRequests{ diff --git a/cpp/include/ucxx/internal/request_am.h b/cpp/include/ucxx/internal/request_am.h index 9fa6d02b..494c8306 100644 --- a/cpp/include/ucxx/internal/request_am.h +++ b/cpp/include/ucxx/internal/request_am.h @@ -25,6 +25,12 @@ namespace internal { class AmData; +/** + * @brief Handle receiving of a `ucxx::RequestAm`. + * + * Handle receiving of a `ucxx::RequestAm`, delivering the message to the user and + * notifying of completion. + */ class RecvAmMessage { public: internal::AmData* _amData{nullptr}; ///< Active messages data @@ -80,6 +86,13 @@ class RecvAmMessage { typedef std::unordered_map>> AmPoolType; typedef std::unordered_map> RecvAmMessageMapType; +/** + * @brief Active Message data owned by a `ucxx::Worker`. + * + * Receiving Active Messages are handled directly by a `ucxx::Worker` without the user + * necessarily creating a `ucxx::RequestAm` for it. When there is an incoming message, the + * worker will populate the internal pool of received messages in an orderly-fashion. + */ class AmData { public: std::weak_ptr _worker{}; ///< The worker to which the Active Message callback belongs diff --git a/cpp/include/ucxx/listener.h b/cpp/include/ucxx/listener.h index 4705d317..7172f797 100644 --- a/cpp/include/ucxx/listener.h +++ b/cpp/include/ucxx/listener.h @@ -14,6 +14,12 @@ namespace ucxx { +/** + * @brief Component encapsulating a UCP listener. + * + * The UCP layer provides a handle to access listeners in form of `ucp_listener_h` object, + * this class encapsulates that object and provides methods to simplify its handling. + */ class Listener : public Component { private: ucp_listener_h _handle{nullptr}; ///< The UCP listener handle diff --git a/cpp/include/ucxx/log.h b/cpp/include/ucxx/log.h index 57e8984c..2aa838b3 100644 --- a/cpp/include/ucxx/log.h +++ b/cpp/include/ucxx/log.h @@ -13,6 +13,11 @@ namespace ucxx { +/** + * @brief The UCXX log level component configuration. + * + * The type with the UCXX log level component configuration. + */ extern ucs_log_component_config_t ucxx_log_component_config; // Macros @@ -60,7 +65,12 @@ extern ucs_log_component_config_t ucxx_log_component_config; ucxx_log(ucxx::UCXX_LOG_LEVEL_TRACE_FUNC, "%s(" _fmt ")", __FUNCTION__, ##__VA_ARGS__) #define ucxx_trace_poll(_fmt, ...) ucxx_log(ucxx::UCXX_LOG_LEVEL_TRACE_POLL, _fmt, ##__VA_ARGS__) -// Constants +/** + * @brief Map of log level names to their respective types. + * + * Map of log level names, used by the user to specify levels to enable, to their respective + * internal types. + */ const std::unordered_map logLevelNames = { {"FATAL", UCXX_LOG_LEVEL_FATAL}, {"ERROR", UCXX_LOG_LEVEL_ERROR}, @@ -77,10 +87,27 @@ const std::unordered_map logLevelNames = { {"", UCXX_LOG_LEVEL_LAST}, {"PRINT", UCXX_LOG_LEVEL_PRINT}}; -const char logLevelNameDefault[] = "WARN"; +/** + * @brief The name of the default log level. + * + * The name of default log level, must be one of the keys in `logLevelNames`. + */ +const char logLevelNameDefault[] = "WARN"; + +/** + * @brief The type of the default log level. + * + * The type of default log level, automatically set based on the value of + * `logLevelNameDefault`. + */ const ucs_log_level_t logLevelDefault = (ucs_log_level_t)logLevelNames.at(logLevelNameDefault); -// Functions +/** + * @brief Parse the active log level. + * + * Parse the active log level and set appropriate internal values to match the specified + * level. + */ void parseLogLevel(); } // namespace ucxx diff --git a/cpp/include/ucxx/notifier.h b/cpp/include/ucxx/notifier.h index 06dabba2..b1a29a25 100644 --- a/cpp/include/ucxx/notifier.h +++ b/cpp/include/ucxx/notifier.h @@ -11,12 +11,30 @@ namespace ucxx { +/** + * @brief The state of the notifier thread. + * + * The current state of the notifier thread. + */ enum class RequestNotifierThreadState { NotRunning = 0, Running, Stopping }; +/** + * @brief The state with which a wait operation completed. + * + * The state with which a blocking call to wait for the request notifier completed. + */ enum class RequestNotifierWaitState { Ready = 0, Timeout, Shutdown }; class Future; +/** + * @brief Notifier for status of futures. + * + * A notifier used to delay notification of futures to a more appropriate stage of the + * program execution, such as when it will be less resource intensive or free of risks of + * effects such as deadlocks, for example when notifying Python futures where the GIL is + * required. + */ class Notifier { protected: Notifier() = default; diff --git a/cpp/include/ucxx/request.h b/cpp/include/ucxx/request.h index ddcecd90..9e90d754 100644 --- a/cpp/include/ucxx/request.h +++ b/cpp/include/ucxx/request.h @@ -22,6 +22,14 @@ namespace ucxx { +/** + * @brief Base type for a UCXX transfer request. + * + * Base type for one of the multiple UCXX transfer requests. Encapsulates information such + * as the UCP request pointer, the current status, a future to notify and a callback to + * execute upon completion, as well operation-specific data and to maintain a reference to + * its parent until completion. + */ class Request : public Component { protected: ucs_status_t _status{UCS_INPROGRESS}; ///< Requests status diff --git a/cpp/include/ucxx/request_am.h b/cpp/include/ucxx/request_am.h index e952bfa6..8e0e1cfb 100644 --- a/cpp/include/ucxx/request_am.h +++ b/cpp/include/ucxx/request_am.h @@ -21,6 +21,12 @@ namespace internal { class RecvAmMessage; } // namespace internal +/** + * @brief Send or receive a message with the UCX Active Message API. + * + * Send or receive a message with the UCX Active Message API, using non-blocking UCP calls + * `ucp_am_send_nbx` or `ucp_am_recv_data_nbx`. + */ class RequestAm : public Request { private: friend class internal::RecvAmMessage; diff --git a/cpp/include/ucxx/request_data.h b/cpp/include/ucxx/request_data.h index d09e81bb..8143dad7 100644 --- a/cpp/include/ucxx/request_data.h +++ b/cpp/include/ucxx/request_data.h @@ -18,6 +18,12 @@ class Buffer; namespace data { +/** + * @brief Data for an Active Message send. + * + * Type identifying an Active Message send operation and containing data specific to this + * request type. + */ class AmSend { public: const void* _buffer{nullptr}; ///< The raw pointer where data to be sent is stored. @@ -29,6 +35,8 @@ class AmSend { * * Construct an object containing Active Message-specific send data. * + * @param[in] buffer a raw pointer to the data to be sent. + * @param[in] length the size in bytes of the message to be sent. * @param[in] memoryType the memory type of the buffer. */ explicit AmSend(const decltype(_buffer) buffer, @@ -38,6 +46,12 @@ class AmSend { AmSend() = delete; }; +/** + * @brief Data for an Active Message receive. + * + * Type identifying an Active Message receive operation and containing data specific to this + * request type. + */ class AmReceive { public: std::shared_ptr<::ucxx::Buffer> _buffer{nullptr}; ///< The AM received message buffer @@ -52,6 +66,12 @@ class AmReceive { AmReceive(); }; +/** + * @brief Data for a Stream send. + * + * Type identifying a Stream send operation and containing data specific to this request + * type. + */ class StreamSend { public: const void* _buffer{nullptr}; ///< The raw pointer where data to be sent is stored. @@ -70,6 +90,12 @@ class StreamSend { StreamSend() = delete; }; +/** + * @brief Data for an Stream receive. + * + * Type identifying an Stream receive operation and containing data specific to this + * request type. + */ class StreamReceive { public: void* _buffer{nullptr}; ///< The raw pointer where received data should be stored. @@ -89,6 +115,11 @@ class StreamReceive { StreamReceive() = delete; }; +/** + * @brief Data for a Tag send. + * + * Type identifying a Tag send operation and containing data specific to this request type. + */ class TagSend { public: const void* _buffer{nullptr}; ///< The raw pointer where data to be sent is stored. @@ -111,6 +142,12 @@ class TagSend { TagSend() = delete; }; +/** + * @brief Data for a Tag receive. + * + * Type identifying a Tag receive operation and containing data specific to this request + * type. + */ class TagReceive { public: void* _buffer{nullptr}; ///< The raw pointer where received data should be stored. @@ -136,6 +173,12 @@ class TagReceive { TagReceive() = delete; }; +/** + * @brief Data for a multi-buffer Tag send. + * + * Type identifying a multi-buffer Tag send operation and containing data specific to this + * request type. + */ class TagMultiSend { public: const std::vector _buffer{}; ///< Raw pointers where data to be sent is stored. @@ -148,9 +191,10 @@ class TagMultiSend { * * Construct an object containing tag/multi-buffer tag-specific data. * - * @param[in] buffer a raw pointer to the data to be sent. - * @param[in] length the size in bytes of the tag message to be sent. - * @param[in] tag the tag to match. + * @param[in] buffer a raw pointers to the data to be sent. + * @param[in] length the size in bytes of the tag messages to be sent. + * @param[in] isCUDA flags indicating whether buffers being sent are CUDA. + * @param[in] tag the tags to match. */ explicit TagMultiSend(const decltype(_buffer)& buffer, const decltype(_length)& length, @@ -160,6 +204,12 @@ class TagMultiSend { TagMultiSend() = delete; }; +/** + * @brief Data for a multi-buffer Tag receive. + * + * Type identifying a multi-buffer Tag receive operation and containing data specific to + * this request type. + */ class TagMultiReceive { public: const ::ucxx::Tag _tag{0}; ///< Tag to match diff --git a/cpp/include/ucxx/request_helper.h b/cpp/include/ucxx/request_helper.h index 1c8c564c..c1a2a0ee 100644 --- a/cpp/include/ucxx/request_helper.h +++ b/cpp/include/ucxx/request_helper.h @@ -12,8 +12,28 @@ namespace ucxx { +/** + * @brief Wait for a single request to complete. + * + * Block while waiting for a single request to complete. + * + * @throws ucxx::Error a specific error if the request failed. + * + * @param[in] worker the worker to progress until completion. + * @param[in] request the request to wait for. + */ void waitSingleRequest(std::shared_ptr worker, std::shared_ptr request); +/** + * @brief Wait for a multiple requests to complete. + * + * Block while waiting for all requests to complete. + * + * @throws ucxx::Error the specific error of the first request that failed. + * + * @param[in] worker the worker to progress until completion. + * @param[in] requests the requests to wait for. + */ void waitRequests(std::shared_ptr worker, std::vector> requests); } // namespace ucxx diff --git a/cpp/include/ucxx/request_stream.h b/cpp/include/ucxx/request_stream.h index 95743e77..5233682a 100644 --- a/cpp/include/ucxx/request_stream.h +++ b/cpp/include/ucxx/request_stream.h @@ -15,6 +15,12 @@ namespace ucxx { +/** + * @brief Send or receive a message with the UCX Stream API. + * + * Send or receive a message with the UCX Stream API, using non-blocking UCP calls + * `ucp_stream_send_nbx` or `ucp_stream_recv_nbx`. + */ class RequestStream : public Request { private: /** diff --git a/cpp/include/ucxx/request_tag.h b/cpp/include/ucxx/request_tag.h index e7c47449..6ac5fe0e 100644 --- a/cpp/include/ucxx/request_tag.h +++ b/cpp/include/ucxx/request_tag.h @@ -15,6 +15,12 @@ namespace ucxx { +/** + * @brief Send or receive a message with the UCX Tag API. + * + * Send or receive a message with the UCX Tag API, using non-blocking UCP calls + * `ucp_tag_send_nbx` or `ucp_tag_recv_nbx`. + */ class RequestTag : public Request { private: /** diff --git a/cpp/include/ucxx/request_tag_multi.h b/cpp/include/ucxx/request_tag_multi.h index 6780a1c6..991fca1b 100644 --- a/cpp/include/ucxx/request_tag_multi.h +++ b/cpp/include/ucxx/request_tag_multi.h @@ -20,6 +20,13 @@ namespace ucxx { class RequestTagMulti; +/** + * @brief Container for data required by a `ucxx::RequestTagMulti`. + * + * Container for the data required by a `ucxx::RequestTagMulti`, such as the + * `ucxx::RequestTag` that is doing the operation, as well as buffers to send from or + * receive at. + */ struct BufferRequest { std::shared_ptr request{nullptr}; ///< The `ucxx::RequestTag` of a header or frame std::shared_ptr stringBuffer{nullptr}; ///< Serialized `Header` @@ -34,8 +41,22 @@ struct BufferRequest { BufferRequest& operator=(BufferRequest&& o) = delete; }; +/** + * @brief Pre-defined type for a pointer to an `ucxx::BufferRequest`. + * + * A pre-defined type for a pointer to a `ucxx::BufferRequest`, used as a convenience type. + */ typedef std::shared_ptr BufferRequestPtr; +/** + * @brief Send or receive multiple messages with the UCX Tag API. + * + * Send or receive multiple messages with the UCX Tag API. This is done combining multiple + * messages with `ucxx::RequestTag`, first sending/receiving a header, followed by + * sending/receiving the user messages. Intended primarily for use with Python, such that + * the program can then only wait for the completion of one future and thus reduce + * potentially expensive iterations over multiple futures. + */ class RequestTagMulti : public Request { private: size_t _totalFrames{0}; ///< The total number of frames handled by this request @@ -204,6 +225,11 @@ class RequestTagMulti : public Request { void cancel() override; }; +/** + * @brief Pre-defined type for a pointer to an `ucxx::RequestTagMulti`. + * + * A pre-defined type for a pointer to a `ucxx::RequestTagMulti`, used as a convenience type. + */ typedef std::shared_ptr RequestTagMultiPtr; } // namespace ucxx diff --git a/cpp/include/ucxx/typedefs.h b/cpp/include/ucxx/typedefs.h index 03763017..c4ef29cd 100644 --- a/cpp/include/ucxx/typedefs.h +++ b/cpp/include/ucxx/typedefs.h @@ -17,7 +17,13 @@ namespace ucxx { class Buffer; class Request; -// Logging levels +/** + * @brief Available logging levels. + * + * Available logging levels that are used to enable specific log types based on user's + * configuration and also to define appropriate functions to be used in UCXX code to log + * only when the appropriate level is enabled. + */ typedef enum { UCXX_LOG_LEVEL_FATAL, /* Immediate termination */ UCXX_LOG_LEVEL_ERROR, /* Error is returned to the user */ @@ -31,22 +37,70 @@ typedef enum { UCXX_LOG_LEVEL_TRACE_ASYNC, /* Asynchronous progress engine */ UCXX_LOG_LEVEL_TRACE_FUNC, /* Function calls */ UCXX_LOG_LEVEL_TRACE_POLL, /* Polling functions */ - UCXX_LOG_LEVEL_LAST, - UCXX_LOG_LEVEL_PRINT /* Temporary output */ + UCXX_LOG_LEVEL_LAST, /* Last level barrier, not an actual level */ + UCXX_LOG_LEVEL_PRINT /* Temporary output */ } ucxx_log_level_t; +/** + * @brief The direction of a UCXX transfer. + * + * The direction of a UCXX transfer, can be either `Send` or `Receive`. + */ enum class TransferDirection { Send = 0, Receive }; +/** + * @brief Strong type for a UCP tag. + * + * Strong type for a UCP tag, preventing accidental mixing with wrong types, especially + * useful to prevent passing an argument in wrong order. + */ enum Tag : ucp_tag_t {}; + +/** + * @brief Strong type for a UCP tag mask. + * + * Strong type for a UCP tag mask, preventing accidental mixing with wrong types, especially + * useful to prevent passing an argument in wrong order. + */ enum TagMask : ucp_tag_t {}; +/** + * @brief A full UCP tag mask. + * + * A convenience constant providing a full UCP tag mask (all bits set). + */ static constexpr TagMask TagMaskFull{std::numeric_limits>::max()}; +/** + * @brief A UCP configuration map. + * + * A UCP configuration map, with keys being the configuration name and value being the + * actual value set. + */ typedef std::unordered_map ConfigMap; +/** + * @brief A user-defined function to execute as part of a `ucxx::Request` callback. + * + * A user-defined function to execute as part of a `ucxx::Request` callback, allowing + * execution of custom code upon request completion. + */ typedef std::function)> RequestCallbackUserFunction; + +/** + * @brief Data for the user-defined function provided to the `ucxx::Request` callback. + * + * Data passed to the user-defined function provided to the `ucxx::Request` callback, which + * the custom user-defined function may act upon. + */ typedef std::shared_ptr RequestCallbackUserData; +/** + * @brief Custom Active Message allocator type. + * + * Type for a custom Active Message allocator that can be registered by a user so that the + * Active Message receiver can allocate a buffer of such type upon receiving message. + */ typedef std::function(size_t)> AmAllocatorType; } // namespace ucxx diff --git a/cpp/include/ucxx/utils/callback_notifier.h b/cpp/include/ucxx/utils/callback_notifier.h index ea5bea61..35421082 100644 --- a/cpp/include/ucxx/utils/callback_notifier.h +++ b/cpp/include/ucxx/utils/callback_notifier.h @@ -9,6 +9,12 @@ namespace ucxx { namespace utils { +/** + * A thread-safe notification object. + * + * A thread-safe notification object which can signal release of some shared state while + * a single thread blocks until the shared state is released. + */ class CallbackNotifier { private: std::atomic_bool _flag{}; //< flag storing state diff --git a/cpp/include/ucxx/worker.h b/cpp/include/ucxx/worker.h index 55463992..40916580 100644 --- a/cpp/include/ucxx/worker.h +++ b/cpp/include/ucxx/worker.h @@ -34,6 +34,12 @@ namespace internal { class AmData; } // namespace internal +/** + * @brief Component encapsulating a UCP worker. + * + * The UCP layer provides a handle to access workers in form of `ucp_worker_h` object, + * this class encapsulates that object and provides methods to simplify its handling. + */ class Worker : public Component { private: ucp_worker_h _handle{nullptr}; ///< The UCP worker handle diff --git a/cpp/include/ucxx/worker_progress_thread.h b/cpp/include/ucxx/worker_progress_thread.h index 8f6a738c..1b6c320b 100644 --- a/cpp/include/ucxx/worker_progress_thread.h +++ b/cpp/include/ucxx/worker_progress_thread.h @@ -13,10 +13,38 @@ namespace ucxx { +/** + * @brief A user-defined function used to wake the worker. + * + * A user-defined function signaling worker to wake the progress event. + */ typedef std::function SignalWorkerFunction; + +/** + * @brief A user-defined function to execute at the start of the progress thread. + * + * A user-defined function to execute at the start of the progress thread, used for example + * to ensure resources are properly set for the thread, such as a CUDA context. + */ typedef std::function ProgressThreadStartCallback; + +/** + * @brief Data for the user-defined function provided to progress thread start callback. + * + * Data passed to the user-defined function provided to the callback executed when the + * progress thread starts, which the custom user-defined function may act upon. + */ typedef void* ProgressThreadStartCallbackArg; +/** + * @brief A thread to progress a `ucxx::Worker`. + * + * A thread to progress the `ucxx::Worker`, thus moving all such blocking operations out of + * the main program thread. It may also be used to execute submissions, such as from + * `ucxx::Request` objects, therefore also moving any blocking costs of those to this + * thread, as well as generic pre-progress and post-progress callbacks that can be used by + * the program to block until that stage is reached. + */ class WorkerProgressThread { private: std::thread _thread{}; ///< Thread object @@ -113,6 +141,11 @@ class WorkerProgressThread { */ bool pollingMode() const; + /** + * @brief Returns the ID of the progress thread. + * + * @returns the progress thread ID. + */ std::thread::id getId() const; };