From 139eaf452ab79294c6a4d682eba4100d00b02676 Mon Sep 17 00:00:00 2001 From: divyegala Date: Fri, 8 Jul 2022 11:14:19 -0700 Subject: [PATCH 01/19] working tests for add.cuh --- cpp/include/raft/core/cudart_utils.hpp | 10 +++ cpp/include/raft/core/mdarray.hpp | 88 ++++++++++++++++++++++---- cpp/include/raft/linalg/add.cuh | 31 +++++++++ cpp/include/raft/linalg/detail/add.cuh | 1 + cpp/test/linalg/add.cu | 7 +- 5 files changed, 124 insertions(+), 13 deletions(-) diff --git a/cpp/include/raft/core/cudart_utils.hpp b/cpp/include/raft/core/cudart_utils.hpp index 7fe6ecaf6b..f499c84c6d 100644 --- a/cpp/include/raft/core/cudart_utils.hpp +++ b/cpp/include/raft/core/cudart_utils.hpp @@ -28,6 +28,7 @@ #include #include +#include #include #include @@ -445,6 +446,15 @@ constexpr T upper_bound() return std::numeric_limits::max(); } +template +bool is_device_ptr(T* ptr) +{ + CUmemorytype mem_type; + auto res = cuPointerGetAttribute(&mem_type, CU_POINTER_ATTRIBUTE_MEMORY_TYPE, (CUdeviceptr)ptr); + + return mem_type == CU_MEMORYTYPE_DEVICE; +} + } // namespace raft #endif diff --git a/cpp/include/raft/core/mdarray.hpp b/cpp/include/raft/core/mdarray.hpp index a4f6ca67b1..b2523c2c49 100644 --- a/cpp/include/raft/core/mdarray.hpp +++ b/cpp/include/raft/core/mdarray.hpp @@ -104,23 +104,20 @@ template inline constexpr bool is_mdspan_v = is_mdspan_t::value; } // namespace detail -template -struct is_mdspan : std::true_type { -}; -template -struct is_mdspan : detail::is_mdspan_t { -}; -template -struct is_mdspan - : std::conditional_t, is_mdspan, std::false_type> { -}; - /** * @\brief Boolean to determine if variadic template types Tn are either * raft::host_mdspan/raft::device_mdspan or their derived types */ template -inline constexpr bool is_mdspan_v = is_mdspan::value; +inline constexpr bool is_mdspan_v = std::bool_constant<(... && detail::is_mdspan_v)>::value; + +/** + * @\brief Ensure all variadic template types are raft::mdspan + * Usage: as last template paramater of function + * `typename = raft::enable_if_mdspan` + */ +template +using enable_if_mdspan = std::enable_if_t, void>; /** * @brief stdex::mdspan with device tag to avoid accessing incorrect memory location. @@ -578,6 +575,13 @@ using host_matrix = host_mdarray; template using device_matrix = device_mdarray; +/** + * @brief Shorthand for 0-dim mdspan (scalar). + * @tparam ElementType the data type of the scalar element + */ +template +using scalar_view = mdspan; + /** * @brief Shorthand for 0-dim host mdspan (scalar). * @tparam ElementType the data type of the scalar element @@ -592,6 +596,13 @@ using host_scalar_view = host_mdspan; template using device_scalar_view = device_mdspan; +/** + * @brief Shorthand for 1-dim mdspan. + * @tparam ElementType the data type of the vector elements + */ +template +using vector_view = mdspan; + /** * @brief Shorthand for 1-dim host mdspan. * @tparam ElementType the data type of the vector elements @@ -606,6 +617,15 @@ using host_vector_view = host_mdspan; template using device_vector_view = device_mdspan; +/** + * @brief Shorthand for c-contiguous matrix view. + * @tparam ElementType the data type of the matrix elements + * @tparam LayoutPolicy policy for strides and layout ordering + * + */ +template +using matrix_view = mdspan; + /** * @brief Shorthand for c-contiguous host matrix view. * @tparam ElementType the data type of the matrix elements @@ -624,6 +644,19 @@ using host_matrix_view = host_mdspan; template using device_matrix_view = device_mdspan; +/** + * @brief Create a 0-dim (scalar) mdspan instance. + * + * @tparam ElementType the data type of the matrix elements + * @param[in] ptr on device to wrap + */ +template +auto make_scalar_view(ElementType* ptr) +{ + scalar_extent extents; + return scalar_view{ptr, extents}; +} + /** * @brief Create a 0-dim (scalar) mdspan instance for host value. * @@ -650,6 +683,23 @@ auto make_device_scalar_view(ElementType* ptr) return device_scalar_view{ptr, extents}; } +/** + * @brief Create a 2-dim c-contiguous mdspan instance. It's expected + * that the given layout policy match the layout of the underlying + * pointer. + * @tparam ElementType the data type of the matrix elements + * @tparam LayoutPolicy policy for strides and layout ordering + * @param[in] ptr on host to wrap + * @param[in] n_rows number of rows in pointer + * @param[in] n_cols number of columns in pointer + */ +template +auto make_matrix_view(ElementType* ptr, size_t n_rows, size_t n_cols) +{ + matrix_extent extents{n_rows, n_cols}; + return matrix_view{ptr, extents}; +} + /** * @brief Create a 2-dim c-contiguous mdspan instance for host pointer. It's * expected that the given layout policy match the layout of the underlying @@ -683,6 +733,20 @@ auto make_device_matrix_view(ElementType* ptr, size_t n_rows, size_t n_cols) return device_matrix_view{ptr, extents}; } +/** + * @brief Create a 1-dim mdspan instance. + * @tparam ElementType the data type of the vector elements + * @param[in] ptr on host to wrap + * @param[in] n number of elements in pointer + * @return raft::host_vector_view + */ +template +auto make_vector_view(ElementType* ptr, size_t n) +{ + vector_extent extents{n}; + return vector_view{ptr, extents}; +} + /** * @brief Create a 1-dim mdspan instance for host pointer. * @tparam ElementType the data type of the vector elements diff --git a/cpp/include/raft/linalg/add.cuh b/cpp/include/raft/linalg/add.cuh index e25c9df9ef..0362eef5f0 100644 --- a/cpp/include/raft/linalg/add.cuh +++ b/cpp/include/raft/linalg/add.cuh @@ -25,6 +25,8 @@ #include "detail/add.cuh" +#include + namespace raft { namespace linalg { @@ -70,6 +72,18 @@ void add(OutT* out, const InT* in1, const InT* in2, IdxType len, cudaStream_t st detail::add(out, in1, in2, len, stream); } +template > +void add(const raft::handle_t& handle, out_t out, const in_t in1, const in_t in2) +{ + RAFT_EXPECTS(out.is_contiguous(), "Output must be contiguous"); + RAFT_EXPECTS(in1.is_contiguous(), "Input 1 must be contiguous"); + RAFT_EXPECTS(in2.is_contiguous(), "Input 2 must be contiguous"); + RAFT_EXPECTS(out.size() == in1.size() && in1.size() == in2.size(), + "Size mismatch between Output and Inputs"); + + add(out.data(), in1.data(), in2.data(), out.size(), handle.get_stream()); +} + /** Substract single value pointed by singleScalarDev parameter in device memory from inDev[i] and * write result to outDev[i] * @tparam math_t data-type upon which the math operation will be performed @@ -90,6 +104,23 @@ void addDevScalar(math_t* outDev, detail::addDevScalar(outDev, inDev, singleScalarDev, len, stream); } +template > +void add_scalar(const raft::handle_t& handle, + out_t out, + const in_t in, + raft::scalar_view scalar) +{ + RAFT_EXPECTS(out.is_contiguous(), "Output must be contiguous"); + RAFT_EXPECTS(in.is_contiguous(), "Input must be contiguous"); + RAFT_EXPECTS(out.size() == in.size(), "Size mismatch between Output and Input"); + + if (raft::is_device_ptr(scalar.data())) { + addDevScalar(out.data(), in.data(), scalar.data(), out.size(), handle.get_stream()); + } else { + addScalar(out.data(), in.data(), *scalar.data(), out.size(), handle.get_stream()); + } +} + }; // end namespace linalg }; // end namespace raft diff --git a/cpp/include/raft/linalg/detail/add.cuh b/cpp/include/raft/linalg/detail/add.cuh index 288ac228c9..a6fb21aebf 100644 --- a/cpp/include/raft/linalg/detail/add.cuh +++ b/cpp/include/raft/linalg/detail/add.cuh @@ -18,6 +18,7 @@ #include "functional.cuh" +#include #include #include #include diff --git a/cpp/test/linalg/add.cu b/cpp/test/linalg/add.cu index ba9dac5ac2..790d0a314c 100644 --- a/cpp/test/linalg/add.cu +++ b/cpp/test/linalg/add.cu @@ -46,7 +46,12 @@ class AddTest : public ::testing::TestWithParam> { uniform(handle, r, in1.data(), len, InT(-1.0), InT(1.0)); uniform(handle, r, in2.data(), len, InT(-1.0), InT(1.0)); naiveAddElem(out_ref.data(), in1.data(), in2.data(), len, stream); - add(out.data(), in1.data(), in2.data(), len, stream); + + auto out_view = raft::make_vector_view(out.data(), out.size()); + auto in1_view = raft::make_vector_view(in1.data(), in1.size()); + auto in2_view = raft::make_vector_view(in2.data(), in2.size()); + + add(handle, out_view, in1_view, in2_view); handle.sync_stream(stream); } From ddfffc3b93b93160533e6620241eb69c6c3877ce Mon Sep 17 00:00:00 2001 From: divyegala Date: Wed, 13 Jul 2022 09:07:14 -0700 Subject: [PATCH 02/19] updates till eig --- cpp/include/raft/linalg/add.cuh | 57 ++++++--- cpp/include/raft/linalg/axpy.cuh | 38 ++++++ cpp/include/raft/linalg/binary_op.cuh | 42 ++++++ .../raft/linalg/cholesky_r1_update.cuh | 17 ++- .../raft/linalg/coalesced_reduction.cuh | 83 +++++++++++- cpp/include/raft/linalg/detail/divide.cuh | 6 +- cpp/include/raft/linalg/detail/eig.cuh | 42 +++--- cpp/include/raft/linalg/divide.cuh | 42 +++++- cpp/include/raft/linalg/eig.cuh | 121 +++++++++++++++++- cpp/test/linalg/binary_op.cu | 12 +- cpp/test/linalg/coalesced_reduction.cu | 12 +- cpp/test/linalg/divide.cu | 5 +- cpp/test/linalg/eig.cu | 62 +++++---- cpp/test/linalg/eig_sel.cu | 27 ++-- 14 files changed, 461 insertions(+), 105 deletions(-) diff --git a/cpp/include/raft/linalg/add.cuh b/cpp/include/raft/linalg/add.cuh index 0362eef5f0..530961300e 100644 --- a/cpp/include/raft/linalg/add.cuh +++ b/cpp/include/raft/linalg/add.cuh @@ -48,7 +48,7 @@ using detail::adds_scalar; * @param stream cuda stream where to launch work */ template -void addScalar(OutT* out, const InT* in, InT scalar, IdxType len, cudaStream_t stream) +void addScalar(OutT* out, const InT* in, const InT scalar, IdxType len, cudaStream_t stream) { detail::addScalar(out, in, scalar, len, stream); } @@ -72,18 +72,6 @@ void add(OutT* out, const InT* in1, const InT* in2, IdxType len, cudaStream_t st detail::add(out, in1, in2, len, stream); } -template > -void add(const raft::handle_t& handle, out_t out, const in_t in1, const in_t in2) -{ - RAFT_EXPECTS(out.is_contiguous(), "Output must be contiguous"); - RAFT_EXPECTS(in1.is_contiguous(), "Input 1 must be contiguous"); - RAFT_EXPECTS(in2.is_contiguous(), "Input 2 must be contiguous"); - RAFT_EXPECTS(out.size() == in1.size() && in1.size() == in2.size(), - "Size mismatch between Output and Inputs"); - - add(out.data(), in1.data(), in2.data(), out.size(), handle.get_stream()); -} - /** Substract single value pointed by singleScalarDev parameter in device memory from inDev[i] and * write result to outDev[i] * @tparam math_t data-type upon which the math operation will be performed @@ -104,11 +92,46 @@ void addDevScalar(math_t* outDev, detail::addDevScalar(outDev, inDev, singleScalarDev, len, stream); } +/** + * @defgroup add Addition Arithmetic + * @{ + */ + +/** + * @brief Elementwise add operation on the input buffers + * @tparam out_t Output Type raft::mdspan + * @tparam in_t Input Type raft::mdspan + * @param handle raft::handle_t + * @param out Output + * @param in1 First Input + * @param in2 Second Input + */ template > +void add(const raft::handle_t& handle, out_t out, const in_t in1, const in_t in2) +{ + RAFT_EXPECTS(out.is_contiguous(), "Output must be contiguous"); + RAFT_EXPECTS(in1.is_contiguous(), "Input 1 must be contiguous"); + RAFT_EXPECTS(in2.is_contiguous(), "Input 2 must be contiguous"); + RAFT_EXPECTS(out.size() == in1.size() && in1.size() == in2.size(), + "Size mismatch between Output and Inputs"); + + add(out.data(), in1.data(), in2.data(), out.size(), handle.get_stream()); +} + +/** + * @brief Elementwise addition of scalar to input + * @tparam OutType Output Type raft::mdspan + * @tparam InType Input Type raft::mdspan + * @param handle raft::handle_t + * @param out Output + * @param in Input + * @param scalar raft::scalar_view in either host or device memory + */ +template > void add_scalar(const raft::handle_t& handle, - out_t out, - const in_t in, - raft::scalar_view scalar) + OutType out, + const InType in, + const raft::scalar_view scalar) { RAFT_EXPECTS(out.is_contiguous(), "Output must be contiguous"); RAFT_EXPECTS(in.is_contiguous(), "Input must be contiguous"); @@ -121,6 +144,8 @@ void add_scalar(const raft::handle_t& handle, } } +/** @} */ // end of group add + }; // end namespace linalg }; // end namespace raft diff --git a/cpp/include/raft/linalg/axpy.cuh b/cpp/include/raft/linalg/axpy.cuh index 2e23047b5a..2c03d7cf50 100644 --- a/cpp/include/raft/linalg/axpy.cuh +++ b/cpp/include/raft/linalg/axpy.cuh @@ -50,6 +50,44 @@ void axpy(const raft::handle_t& handle, detail::axpy(handle, n, alpha, x, incx, y, incy, stream); } +/** + * @defgroup axpy cuBLAS axpy + * @{ + */ + +/** + * @brief the wrapper of cublas axpy function + * It computes the following equation: y = alpha * x + y + * + * @tparam MdspanType Type raft::mdspan + * @tparam DevicePointerMode whether pointers alpha, beta point to device memory + * @param [in] handle raft::handle_t + * @param [in] alpha raft::scalar_view in either host or device memory + * @param [in] x Input vector + * @param [in] incx stride between consecutive elements of x + * @param [inout] y Output vector + * @param [in] incy stride between consecutive elements of y + */ +template > +void axpy(const raft::handle_t& handle, + raft::scalar_view alpha, + const MdspanType x, + const int incx, + MdspanType y, + const int incy) +{ + RAFT_EXPECTS(y.is_contiguous(), "Output must be contiguous"); + RAFT_EXPECTS(x.is_contiguous(), "Input must be contiguous"); + RAFT_EXPECTS(y.size() == x.size(), "Size mismatch between Output and Input") + + axpy( + handle, y.size(), alpha, x.data(), incx, y.data(), incy, handle.get_stream()); +} + +/** @} */ // end of group axpy + } // namespace raft::linalg #endif \ No newline at end of file diff --git a/cpp/include/raft/linalg/binary_op.cuh b/cpp/include/raft/linalg/binary_op.cuh index a85bf698f7..a16f7b476e 100644 --- a/cpp/include/raft/linalg/binary_op.cuh +++ b/cpp/include/raft/linalg/binary_op.cuh @@ -20,6 +20,7 @@ #include "detail/binary_op.cuh" +#include #include namespace raft { @@ -52,6 +53,47 @@ void binaryOp( detail::binaryOp(out, in1, in2, len, op, stream); } +/** + * @defgroup binary_op Element-Wise Binary Operation + * @{ + */ + +/** + * @brief perform element-wise binary operation on the input arrays + * @tparam InType Input Type raft::mdspan + * @tparam Lambda the device-lambda performing the actual operation + * @tparam OutType Output Type raft::mdspan + * @tparam TPB threads-per-block in the final kernel launched + * @param handle raft::handle_t + * @param out Output + * @param in1 First input + * @param in2 Second input + * @param op the device-lambda + * @note Lambda must be a functor with the following signature: + * `OutType func(const InType& val1, const InType& val2);` + */ +template > +void binary_op( + const raft::handle_t& handle, OutType out, const InType in1, const InType in2, Lambda op) +{ + RAFT_EXPECTS(out.is_contiguous(), "Output must be contiguous"); + RAFT_EXPECTS(in1.is_contiguous(), "Input 1 must be contiguous"); + RAFT_EXPECTS(in2.is_contiguous(), "Input 2 must be contiguous"); + RAFT_EXPECTS(out.size() == in1.size() && in1.size() == in2.size(), + "Size mismatch between Output and Inputs"); + + using in_element_t = typename InType::element_type; + using out_element_t = typename OutType::element_type; + binaryOp( + out.data(), in1.data(), in2.data(), out.size(), op, handle.get_stream()); +} + +/** @} */ // end of group binary_op + }; // end namespace linalg }; // end namespace raft diff --git a/cpp/include/raft/linalg/cholesky_r1_update.cuh b/cpp/include/raft/linalg/cholesky_r1_update.cuh index d8e838a634..d4806dd8e1 100644 --- a/cpp/include/raft/linalg/cholesky_r1_update.cuh +++ b/cpp/include/raft/linalg/cholesky_r1_update.cuh @@ -109,7 +109,7 @@ namespace linalg { * @param L device array for to store the triangular matrix L, and the new * column of A in column major format, size [n*n] * @param n number of elements in the new row. - * @param ld stride of colums in L + * @param ld stride of columns in L * @param workspace device pointer to workspace shall be nullptr ar an array * of size [n_bytes]. * @param n_bytes size of workspace is returned here if workspace==nullptr. @@ -132,6 +132,21 @@ void choleskyRank1Update(const raft::handle_t& handle, { detail::choleskyRank1Update(handle, L, n, ld, workspace, n_bytes, uplo, stream, eps); } + +// template +// void choleskyRank1Update(const raft::handle_t& handle, +// raft::matrix_view L, +// int n, +// int ld, +// raft::vector_view workspace, +// int* n_bytes, +// cublasFillMode_t uplo, +// cudaStream_t stream, +// math_t eps = -1) +// { +// detail::choleskyRank1Update(handle, L.data(), n, ld, workspace.data(), n_bytes, uplo, stream, +// eps); +// } }; // namespace linalg }; // namespace raft diff --git a/cpp/include/raft/linalg/coalesced_reduction.cuh b/cpp/include/raft/linalg/coalesced_reduction.cuh index 03477f72d6..66d071f9a4 100644 --- a/cpp/include/raft/linalg/coalesced_reduction.cuh +++ b/cpp/include/raft/linalg/coalesced_reduction.cuh @@ -20,6 +20,9 @@ #include "detail/coalesced_reduction.cuh" +#include +#include + namespace raft { namespace linalg { @@ -58,8 +61,8 @@ template > void coalescedReduction(OutType* dots, const InType* data, - int D, - int N, + IdxType D, + IdxType N, OutType init, cudaStream_t stream, bool inplace = false, @@ -70,6 +73,82 @@ void coalescedReduction(OutType* dots, detail::coalescedReduction(dots, data, D, N, init, stream, inplace, main_op, reduce_op, final_op); } +/** + * @defgroup coalesced_reduction Coalesced Memory Access Reductions + * For reducing along rows for row-major and along columns for column-major + * @{ + */ + +/** + * @brief Compute reduction of the input matrix along the leading dimension + * + * @tparam InElementType the input data-type of underlying raft::matrix_view + * @tparam LayoutPolicy The layout of Input/Output (row or col major) + * @tparam OutElementType the output data-type of underlying raft::matrix_view and reduction + * @tparam MainLambda Unary lambda applied while acculumation (eg: L1 or L2 norm) + * It must be a 'callable' supporting the following input and output: + * 
OutType (*MainLambda)(InType, IdxType);
+ * @tparam ReduceLambda Binary lambda applied for reduction (eg: addition(+) for L2 norm) + * It must be a 'callable' supporting the following input and output: + * 
OutType (*ReduceLambda)(OutType);
+ * @tparam FinalLambda the final lambda applied before STG (eg: Sqrt for L2 norm) + * It must be a 'callable' supporting the following input and output: + * 
OutType (*FinalLambda)(OutType);
+ * @param handle raft::handle_t + * @param dots Output + * @param data Input + * @param init initial value to use for the reduction + * @param main_op elementwise operation to apply before reduction + * @param reduce_op binary reduction operation + * @param final_op elementwise operation to apply before storing results + * @param inplace reduction result added inplace or overwrites old values? + */ +template , + typename ReduceLambda = raft::Sum, + typename FinalLambda = raft::Nop> +void coalesced_reduction(const raft::handle_t& handle, + raft::matrix_view dots, + const raft::matrix_view data, + OutElementType init, + bool inplace = false, + MainLambda main_op = raft::Nop(), + ReduceLambda reduce_op = raft::Sum(), + FinalLambda final_op = raft::Nop()) +{ + RAFT_EXPECTS(dots.is_contiguous(), "Output must be contiguous"); + RAFT_EXPECTS(data.is_contiguous(), "Input must be contiguous"); + RAFT_EXPECTS(dots.size() == data.size(), "Size mismatch between Output and Input"); + + if constexpr (std::is_same_v) { + coalescedReduction(dots.data(), + data.data(), + data.extent(1), + data.extent(0), + init, + handle.get_stream(), + inplace, + main_op, + reduce_op, + final_op); + } else if constexpr (std::is_same_v) { + coalescedReduction(dots.data(), + data.data(), + data.extent(0), + data.extent(1), + init, + handle.get_stream(), + inplace, + main_op, + reduce_op, + final_op); + } +} + +/** @} */ // end of group coalesced_reduction + }; // end namespace linalg }; // end namespace raft diff --git a/cpp/include/raft/linalg/detail/divide.cuh b/cpp/include/raft/linalg/detail/divide.cuh index cb46ae76de..a594bf59b2 100644 --- a/cpp/include/raft/linalg/detail/divide.cuh +++ b/cpp/include/raft/linalg/detail/divide.cuh @@ -23,10 +23,10 @@ namespace raft { namespace linalg { namespace detail { -template -void divideScalar(math_t* out, const math_t* in, math_t scalar, IdxType len, cudaStream_t stream) +template +void divideScalar(OutT* out, const InT* in, InT scalar, IdxType len, cudaStream_t stream) { - raft::linalg::unaryOp(out, in, len, divides_scalar(scalar), stream); + raft::linalg::unaryOp(out, in, len, divides_scalar(scalar), stream); } }; // end namespace detail diff --git a/cpp/include/raft/linalg/detail/eig.cuh b/cpp/include/raft/linalg/detail/eig.cuh index 1d9a6bfa8f..7c8acf3a06 100644 --- a/cpp/include/raft/linalg/detail/eig.cuh +++ b/cpp/include/raft/linalg/detail/eig.cuh @@ -139,9 +139,9 @@ enum EigVecMemUsage { OVERWRITE_INPUT, COPY_INPUT }; template void eigSelDC(const raft::handle_t& handle, math_t* in, - int n_rows, - int n_cols, - int n_eig_vals, + std::size_t n_rows, + std::size_t n_cols, + std::size_t n_eig_vals, math_t* eig_vectors, math_t* eig_vals, EigVecMemUsage memUsage, @@ -156,13 +156,13 @@ void eigSelDC(const raft::handle_t& handle, CUSOLVER_EIG_MODE_VECTOR, CUSOLVER_EIG_RANGE_I, CUBLAS_FILL_MODE_UPPER, - n_rows, + static_cast(n_rows), in, - n_cols, + static_cast(n_cols), math_t(0.0), math_t(0.0), - n_cols - n_eig_vals + 1, - n_cols, + static_cast(n_cols - n_eig_vals + 1), + static_cast(n_cols), &h_meig, eig_vals, &lwork)); @@ -176,13 +176,13 @@ void eigSelDC(const raft::handle_t& handle, CUSOLVER_EIG_MODE_VECTOR, CUSOLVER_EIG_RANGE_I, CUBLAS_FILL_MODE_UPPER, - n_rows, + static_cast(n_rows), in, - n_cols, + static_cast(n_cols), math_t(0.0), math_t(0.0), - n_cols - n_eig_vals + 1, - n_cols, + static_cast(n_cols - n_eig_vals + 1), + static_cast(n_cols), &h_meig, eig_vals, d_work.data(), @@ -197,13 +197,13 @@ void eigSelDC(const raft::handle_t& handle, CUSOLVER_EIG_MODE_VECTOR, CUSOLVER_EIG_RANGE_I, CUBLAS_FILL_MODE_UPPER, - n_rows, + static_cast(n_rows), eig_vectors, - n_cols, + static_cast(n_cols), math_t(0.0), math_t(0.0), - n_cols - n_eig_vals + 1, - n_cols, + static_cast(n_cols - n_eig_vals + 1), + static_cast(n_cols), &h_meig, eig_vals, d_work.data(), @@ -230,8 +230,8 @@ void eigSelDC(const raft::handle_t& handle, template void eigJacobi(const raft::handle_t& handle, const math_t* in, - int n_rows, - int n_cols, + std::size_t n_rows, + std::size_t n_cols, math_t* eig_vectors, math_t* eig_vals, cudaStream_t stream, @@ -249,9 +249,9 @@ void eigJacobi(const raft::handle_t& handle, RAFT_CUSOLVER_TRY(cusolverDnsyevj_bufferSize(cusolverH, CUSOLVER_EIG_MODE_VECTOR, CUBLAS_FILL_MODE_UPPER, - n_rows, + static_cast(n_rows), eig_vectors, - n_cols, + static_cast(n_cols), eig_vals, &lwork, syevj_params)); @@ -264,9 +264,9 @@ void eigJacobi(const raft::handle_t& handle, RAFT_CUSOLVER_TRY(cusolverDnsyevj(cusolverH, CUSOLVER_EIG_MODE_VECTOR, CUBLAS_FILL_MODE_UPPER, - n_rows, + static_cast(n_rows), eig_vectors, - n_cols, + static_cast(n_cols), eig_vals, d_work.data(), lwork, diff --git a/cpp/include/raft/linalg/divide.cuh b/cpp/include/raft/linalg/divide.cuh index 820c42f0ea..26c252d2c4 100644 --- a/cpp/include/raft/linalg/divide.cuh +++ b/cpp/include/raft/linalg/divide.cuh @@ -20,6 +20,8 @@ #include "detail/divide.cuh" +#include + namespace raft { namespace linalg { @@ -27,7 +29,8 @@ using detail::divides_scalar; /** * @defgroup ScalarOps Scalar operations on the input buffer - * @tparam math_t data-type upon which the math operation will be performed + * @tparam OutT output data-type upon which the math operation will be performed + * @tparam InT input data-type upon which the math operation will be performed * @tparam IdxType Integer type used to for addressing * @param out the output buffer * @param in the input buffer @@ -36,13 +39,46 @@ using detail::divides_scalar; * @param stream cuda stream where to launch work * @{ */ -template -void divideScalar(math_t* out, const math_t* in, math_t scalar, IdxType len, cudaStream_t stream) +template +void divideScalar(OutT* out, const InT* in, InT scalar, IdxType len, cudaStream_t stream) { detail::divideScalar(out, in, scalar, len, stream); } /** @} */ +/** + * @defgroup divide Division Arithmetic + * @{ + */ + +/** + * @brief Elementwise addition of scalar to input + * @tparam OutType Output Type raft::mdspan + * @tparam InType Input Type raft::mdspan + * @param handle raft::handle_t + * @param out Output + * @param in Input + * @param scalar raft::scalar_view in host memory + */ +template > +void divide_scalar(const raft::handle_t& handle, + OutType out, + const InType in, + const raft::scalar_view scalar) +{ + RAFT_EXPECTS(out.is_contiguous(), "Output must be contiguous"); + RAFT_EXPECTS(in.is_contiguous(), "Input must be contiguous"); + RAFT_EXPECTS(out.size() == in.size(), "Size mismatch between Output and Input"); + + // if (raft::is_device_ptr(scalar.data())) { + // RAFT_FAIL("Scalar in device memory is not supported"); + // } else { + divideScalar(out.data(), in.data(), *scalar.data(), out.size(), handle.get_stream()); + // } +} + +/** @} */ // end of group add + }; // end namespace linalg }; // end namespace raft diff --git a/cpp/include/raft/linalg/eig.cuh b/cpp/include/raft/linalg/eig.cuh index f1f02dc13e..863e81039a 100644 --- a/cpp/include/raft/linalg/eig.cuh +++ b/cpp/include/raft/linalg/eig.cuh @@ -20,6 +20,8 @@ #include "detail/eig.cuh" +#include + namespace raft { namespace linalg { @@ -73,9 +75,9 @@ using detail::OVERWRITE_INPUT; template void eigSelDC(const raft::handle_t& handle, math_t* in, - int n_rows, - int n_cols, - int n_eig_vals, + std::size_t n_rows, + std::size_t n_cols, + std::size_t n_eig_vals, math_t* eig_vectors, math_t* eig_vals, EigVecMemUsage memUsage, @@ -102,8 +104,8 @@ void eigSelDC(const raft::handle_t& handle, template void eigJacobi(const raft::handle_t& handle, const math_t* in, - int n_rows, - int n_cols, + std::size_t n_rows, + std::size_t n_cols, math_t* eig_vectors, math_t* eig_vals, cudaStream_t stream, @@ -112,6 +114,115 @@ void eigJacobi(const raft::handle_t& handle, { detail::eigJacobi(handle, in, n_rows, n_cols, eig_vectors, eig_vals, stream, tol, sweeps); } + +/** + * @brief eig decomp with divide and conquer method for the column-major + * symmetric matrices + * @tparam ElementType the data-type of input and output + * @param handle raft::handle_t + * @param in input raft::matrix_view (symmetric matrix that has real eig values and + * vectors) + * @param eig_vectors: eigenvectors output of type raft::matrix_view + * @param eig_vals: eigen values output of type raft::vector_view + * @param memUsage: the memory selection for eig vector output + */ +template +void eig_dc(const raft::handle_t& handle, + const raft::matrix_view in, + raft::matrix_view eig_vectors, + raft::vector_view eig_vals) +{ + RAFT_EXPECTS(in.is_contiguous(), "Input must be contiguous"); + RAFT_EXPECTS(eig_vectors.is_contiguous(), "Eigen Vectors must be contiguous"); + RAFT_EXPECTS(eig_vals.is_contiguous(), "Eigen Values must be contiguous"); + RAFT_EXPECTS(in.size() == eig_vectors.size(), "Size mismatch between Input and Eigen Vectors"); + RAFT_EXPECTS(eig_vals.size() == in.extent(1), "Size mismatch between Input and Eigen Values"); + + eigDC(handle, + in.data(), + in.extent(0), + in.extent(1), + eig_vectors.data(), + eig_vals.data(), + handle.get_stream()); +} + +/** + * @brief eig decomp to select top-n eigen values with divide and conquer method + * for the column-major symmetric matrices + * @tparam ElementType the data-type of input and output + * @param handle raft::handle_t + * @param in input raft::matrix_view (symmetric matrix that has real eig values and + * vectors) + * @param n_eig_vals: number of eigenvectors to be generated + * @param eig_vectors: eigenvectors output of type raft::matrix_view + * @param eig_vals: eigen values output of type raft::vector_view + */ +template +void eig_dc_select(const raft::handle_t& handle, + const raft::matrix_view in, + std::size_t n_eig_vals, + raft::matrix_view eig_vectors, + raft::vector_view eig_vals, + EigVecMemUsage memUsage) +{ + RAFT_EXPECTS(in.is_contiguous(), "Input must be contiguous"); + RAFT_EXPECTS(eig_vectors.is_contiguous(), "Eigen Vectors must be contiguous"); + RAFT_EXPECTS(eig_vals.is_contiguous(), "Eigen Values must be contiguous"); + RAFT_EXPECTS(eig_vectors.size() == n_eig_vals * in.extent(0), + "Size mismatch between Input and Eigen Vectors"); + RAFT_EXPECTS(eig_vals.size() == n_eig_vals, "Size mismatch between Input and Eigen Values"); + + raft::linalg::eigSelDC(handle, + in.data(), + in.extent(0), + in.extent(1), + n_eig_vals, + eig_vectors.data(), + eig_vals.data(), + memUsage, + handle.get_stream()); +} + +/** + * @brief overloaded function for eig decomp with Jacobi method for the + * column-major symmetric matrices (in parameter) + * @tparam ElementType the data-type of input and output + * @param handle raft::handle_t + * @param in input raft::matrix_view (symmetric matrix that has real eig values and + * vectors) + * @param eig_vectors: eigenvectors output of type raft::matrix_view + * @param eig_vals: eigen values output of type raft::vector_view + * @param tol: error tolerance for the jacobi method. Algorithm stops when the + * error is below tol + * @param sweeps: number of sweeps in the Jacobi algorithm. The more the better + * accuracy. + */ +template +void eig_jacobi(const raft::handle_t& handle, + const raft::matrix_view in, + raft::matrix_view eig_vectors, + raft::vector_view eig_vals, + ElementType tol = 1.e-7, + int sweeps = 15) +{ + RAFT_EXPECTS(in.is_contiguous(), "Input must be contiguous"); + RAFT_EXPECTS(eig_vectors.is_contiguous(), "Eigen Vectors must be contiguous"); + RAFT_EXPECTS(eig_vals.is_contiguous(), "Eigen Values must be contiguous"); + RAFT_EXPECTS(in.size() == eig_vectors.size(), "Size mismatch between Input and Eigen Vectors"); + RAFT_EXPECTS(eig_vals.size() == in.extent(1), "Size mismatch between Input and Eigen Values"); + + eigJacobi(handle, + in.data(), + in.extent(0), + in.extent(1), + eig_vectors.data(), + eig_vals.data(), + handle.get_stream(), + tol, + sweeps); +} + /** @} */ // end of eig }; // end namespace linalg diff --git a/cpp/test/linalg/binary_op.cu b/cpp/test/linalg/binary_op.cu index cd4340f5cd..722fbae569 100644 --- a/cpp/test/linalg/binary_op.cu +++ b/cpp/test/linalg/binary_op.cu @@ -30,10 +30,14 @@ namespace linalg { // within its class template void binaryOpLaunch( - OutType* out, const InType* in1, const InType* in2, IdxType len, cudaStream_t stream) + const raft::handle_t& handle, OutType* out, const InType* in1, const InType* in2, IdxType len) { - binaryOp( - out, in1, in2, len, [] __device__(InType a, InType b) { return a + b; }, stream); + auto out_view = raft::make_vector_view(out, len); + auto in1_view = raft::make_vector_view(in1, len); + auto in2_view = raft::make_vector_view(in2, len); + + binary_op( + handle, out_view, in1_view, in2_view, [] __device__(InType a, InType b) { return a + b; }); } template @@ -57,7 +61,7 @@ class BinaryOpTest : public ::testing::TestWithParam // within its class template void coalescedReductionLaunch( - T* dots, const T* data, int cols, int rows, cudaStream_t stream, bool inplace = false) + const raft::handle_t& handle, T* dots, const T* data, int cols, int rows, bool inplace = false) { - coalescedReduction( - dots, data, cols, rows, (T)0, stream, inplace, [] __device__(T in, int i) { return in * in; }); + auto dots_view = raft::make_matrix_view(dots, rows, cols); + auto data_view = raft::make_matrix_view(data, rows, cols); + coalesced_reduction( + handle, dots_view, data_view, (T)0, inplace, [] __device__(T in, int i) { return in * in; }); } template @@ -71,9 +73,9 @@ class coalescedReductionTest : public ::testing::TestWithParam> { raft::update_device(eig_vectors_ref.data(), eig_vectors_ref_h, len, stream); raft::update_device(eig_vals_ref.data(), eig_vals_ref_h, params.n_col, stream); - eigDC(handle, - cov_matrix.data(), - params.n_row, - params.n_col, - eig_vectors.data(), - eig_vals.data(), - stream); + auto cov_matrix_view = + raft::make_matrix_view(cov_matrix.data(), params.n_row, params.n_col); + auto eig_vectors_view = + raft::make_matrix_view(eig_vectors.data(), params.n_row, params.n_col); + auto eig_vals_view = raft::make_vector_view(eig_vals.data(), params.n_row); + + auto eig_vectors_jacobi_view = raft::make_matrix_view( + eig_vectors_jacobi.data(), params.n_row, params.n_col); + auto eig_vals_jacobi_view = raft::make_vector_view(eig_vals_jacobi.data(), params.n_row); + + eig_dc(handle, cov_matrix_view, eig_vectors_view, eig_vals_view); T tol = 1.e-7; int sweeps = 15; - eigJacobi(handle, - cov_matrix.data(), - params.n_row, - params.n_col, - eig_vectors_jacobi.data(), - eig_vals_jacobi.data(), - stream, - tol, - sweeps); + eig_jacobi(handle, cov_matrix_view, eig_vectors_jacobi_view, eig_vals_jacobi_view, tol, sweeps); // test code for comparing two methods len = params.n * params.n; uniform(handle, r, cov_matrix_large.data(), len, T(-1.0), T(1.0)); - eigDC(handle, - cov_matrix_large.data(), - params.n, - params.n, - eig_vectors_large.data(), - eig_vals_large.data(), - stream); - eigJacobi(handle, - cov_matrix_large.data(), - params.n, - params.n, - eig_vectors_jacobi_large.data(), - eig_vals_jacobi_large.data(), - stream, - tol, - sweeps); + auto cov_matrix_large_view = + raft::make_matrix_view(cov_matrix_large.data(), params.n, params.n); + auto eig_vectors_large_view = + raft::make_matrix_view(eig_vectors_large.data(), params.n, params.n); + auto eig_vals_large_view = raft::make_vector_view(eig_vals_large.data(), params.n); + + auto eig_vectors_jacobi_large_view = raft::make_matrix_view( + eig_vectors_jacobi_large.data(), params.n, params.n); + auto eig_vals_jacobi_large_view = + raft::make_vector_view(eig_vals_jacobi_large.data(), params.n); + + eig_dc(handle, cov_matrix_large_view, eig_vectors_large_view, eig_vals_large_view); + eig_jacobi(handle, + cov_matrix_large_view, + eig_vectors_jacobi_large_view, + eig_vals_jacobi_large_view, + tol, + sweeps); handle.sync_stream(stream); } diff --git a/cpp/test/linalg/eig_sel.cu b/cpp/test/linalg/eig_sel.cu index 23ded35174..a0f430593b 100644 --- a/cpp/test/linalg/eig_sel.cu +++ b/cpp/test/linalg/eig_sel.cu @@ -80,18 +80,21 @@ class EigSelTest : public ::testing::TestWithParam> { raft::update_device( eig_vectors_ref.data(), eig_vectors_ref_h, params.n_eigen_vals * params.n, stream); - raft::update_device(eig_vals_ref.data(), eig_vals_ref_h, params.n, stream); - - raft::linalg::eigSelDC(handle, - cov_matrix.data(), - params.n, - params.n, - params.n_eigen_vals, - eig_vectors.data(), - eig_vals.data(), - EigVecMemUsage::OVERWRITE_INPUT, - stream); - handle.sync_stream(stream); + raft::update_device(eig_vals_ref.data(), eig_vals_ref_h, params.n_eigen_vals, stream); + + auto cov_matrix_view = + raft::make_matrix_view(cov_matrix.data(), params.n, params.n); + auto eig_vectors_view = + raft::make_matrix_view(eig_vectors.data(), params.n_eigen_vals, params.n); + auto eig_vals_view = raft::make_vector_view(eig_vals.data(), params.n_eigen_vals); + + raft::linalg::eig_dc_select(handle, + cov_matrix_view, + params.n_eigen_vals, + eig_vectors_view, + eig_vals_view, + EigVecMemUsage::OVERWRITE_INPUT); + handle.sync_stream(); } protected: From 362beefb8b78e54eeda1e8bd070890b98c1de9a0 Mon Sep 17 00:00:00 2001 From: divyegala Date: Wed, 13 Jul 2022 10:22:13 -0700 Subject: [PATCH 03/19] cholesky r1 --- .../raft/linalg/cholesky_r1_update.cuh | 134 ++++++++++++++++-- cpp/test/linalg/cholesky_r1.cu | 18 ++- 2 files changed, 128 insertions(+), 24 deletions(-) diff --git a/cpp/include/raft/linalg/cholesky_r1_update.cuh b/cpp/include/raft/linalg/cholesky_r1_update.cuh index d4806dd8e1..f38b85e910 100644 --- a/cpp/include/raft/linalg/cholesky_r1_update.cuh +++ b/cpp/include/raft/linalg/cholesky_r1_update.cuh @@ -133,20 +133,126 @@ void choleskyRank1Update(const raft::handle_t& handle, detail::choleskyRank1Update(handle, L, n, ld, workspace, n_bytes, uplo, stream, eps); } -// template -// void choleskyRank1Update(const raft::handle_t& handle, -// raft::matrix_view L, -// int n, -// int ld, -// raft::vector_view workspace, -// int* n_bytes, -// cublasFillMode_t uplo, -// cudaStream_t stream, -// math_t eps = -1) -// { -// detail::choleskyRank1Update(handle, L.data(), n, ld, workspace.data(), n_bytes, uplo, stream, -// eps); -// } +/** + * @defgroup cholesky_rank1_update Cholesky Rank1 Update + * @{ + */ + +/** + * @brief Rank 1 update of Cholesky decomposition. + * + * This method is useful if an algorithm iteratively builds up matrix A, and + * the Cholesky decomposition of A is required at each step. + * + * On entry, L is the Cholesky decomposition of matrix A, where both A and L + * have size n-1 x n-1. We are interested in the Cholesky decomposition of a new + * matrix A', which we get by adding a row and column to A. In Python notation: + * - A'[0:n-1, 0:n-1] = A; + * - A'[:,n-1] = A[n-1,:] = A_new + * + * On entry, the new column A_new, is stored as the n-th column of L if uplo == + * CUBLAS_FILL_MODE_UPPER, else A_new is stored as the n-th row of L. + * + * On exit L contains the Cholesky decomposition of A'. In practice the elements + * of A_new are overwritten with new row/column of the L matrix. + * + * The uplo paramater is used to select the matrix layout. + * If (uplo != CUBLAS_FILL_MODE_UPPER) then the input arg L stores the + * lower triangular matrix L, so that A = L * L.T. Otherwise the input arg L + * stores an upper triangular matrix U: A = U.T * U. + * + * On exit L will be updated to store the Cholesky decomposition of A'. + * + * If the matrix is not positive definit, or very ill conditioned then the new + * diagonal element of L would be NaN. In such a case an exception is thrown. + * The eps argument can be used to override this behavior: if eps >= 0 then + * the diagonal element is replaced by eps in case the diagonal is NaN or + * smaller than eps. Note: for an iterative solver it is probably better to + * stop early in case of error, rather than relying on the eps parameter. + * + * Examples: + * + * - Lower triangular factorization: + * @code{.cpp} + * // Initialize arrays + * int ld_L = n_rows; + * rmm::device_uvector L(ld_L * n_rows, stream); + * raft::linalg::choleskyRank1Update(handle, L, n_rows, ld_L, nullptr, + * &n_bytes, CUBLAS_FILL_MODE_LOWER, + * stream); + * rmm::device_uvector workspace(n_bytes, stream); + * + * for (n=1; n<=n_rows; rank++) { + * // Calculate a new row/column of matrix A into A_new + * // ... + * // Copy new row to L[rank-1,:] + * RAFT_CUBLAS_TRY(cublasCopy(handle.get_cublas_handle(), n - 1, A_new, 1, + * L + n - 1, ld_L, stream)); + * // Update Cholesky factorization + * raft::linalg::choleskyRank1Update( + * handle, L, rank, ld_L, workspace, &n_bytes, CUBLAS_FILL_MODE_LOWER, + * stream); + * } + * Now L stores the Cholesky decomposition of A: A = L * L.T + * @endcode + * + * - Upper triangular factorization: + * @code{.cpp} + * // Initialize arrays + * int ld_U = n_rows; + * rmm::device_uvector U(ld_U * n_rows, stream); + * raft::linalg::choleskyRank1Update(handle, L, n_rows, ld_U, nullptr, + * &n_bytes, CUBLAS_FILL_MODE_UPPER, + * stream); + * rmm::device_uvector workspace(stream, n_bytes, stream); + * + * for (n=1; n<=n_rows; n++) { + * // Calculate a new row/column of matrix A into array A_new + * // ... + * // Copy new row to U[:,n-1] (column major layout) + * raft::copy(U + ld_U * (n-1), A_new, n-1, stream); + * // + * // Update Cholesky factorization + * raft::linalg::choleskyRank1Update( + * handle, U, n, ld_U, workspace, &n_bytes, CUBLAS_FILL_MODE_UPPER, + * stream); + * } + * // Now U stores the Cholesky decomposition of A: A = U.T * U + * @endcode + * + * @param handle RAFT handle (used to retrive cuBLAS handles). + * @param L raft::device_matrix_view to store the + * triangular matrix L, and the new column of A in column major format, size [n*n] + * @param n number of elements in the new row. + * @param ld stride of columns in L + * @param workspace optional raft::device_vector shall be std::nullopt ar an array + * of size [n_bytes]. + * @param n_bytes size of workspace is returned here if workspace==std::nullopt. + * @param uplo indicates whether L is stored as an upper or lower triangular + * matrix (CUBLAS_FILL_MODE_UPPER or CUBLAS_FILL_MODE_LOWER) + * @param eps numerical parameter that can act as a regularizer for ill + * conditioned systems. Negative values mean no regularizaton. + */ +template +void cholesky_rank1_update(const raft::handle_t& handle, + raft::matrix_view L, + int n, + int ld, + std::optional> workspace, + int* n_bytes, + cublasFillMode_t uplo, + ElementType eps = -1) +{ + if (workspace) { + choleskyRank1Update( + handle, L.data(), n, ld, workspace.value().data(), n_bytes, uplo, handle.get_stream(), eps); + } else { + choleskyRank1Update(handle, L.data(), n, ld, nullptr, n_bytes, uplo, handle.get_stream(), eps); + } +} + +/** @} */ // end of group cholesky_rank1_update + }; // namespace linalg }; // namespace raft diff --git a/cpp/test/linalg/cholesky_r1.cu b/cpp/test/linalg/cholesky_r1.cu index c057c20403..fb9bdd78e2 100644 --- a/cpp/test/linalg/cholesky_r1.cu +++ b/cpp/test/linalg/cholesky_r1.cu @@ -48,14 +48,9 @@ class CholeskyR1Test : public ::testing::Test { int n_bytes = 0; // Initializing in CUBLAS_FILL_MODE_LOWER, because that has larger workspace // requirements. - raft::linalg::choleskyRank1Update(handle, - L.data(), - n_rows, - n_rows, - nullptr, - &n_bytes, - CUBLAS_FILL_MODE_LOWER, - handle.get_stream()); + auto L_view = raft::make_matrix_view(L.data(), n_rows, n_rows); + raft::linalg::cholesky_rank1_update( + handle, L_view, n_rows, n_rows, std::nullopt, &n_bytes, CUBLAS_FILL_MODE_LOWER); Lwork = std::max(Lwork * sizeof(math_t), (size_t)n_bytes); workspace.resize(Lwork, handle.get_stream()); } @@ -85,8 +80,11 @@ class CholeskyR1Test : public ::testing::Test { handle.get_stream())); // Incremental Cholesky factorization using rank one updates. - raft::linalg::choleskyRank1Update( - handle, L.data(), rank, n_rows, workspace.data(), &Lwork, uplo, handle.get_stream()); + auto L_view = raft::make_matrix_view(L.data(), n_rows, n_rows); + auto workspace_view = raft::make_vector_view(workspace.data(), Lwork); + + raft::linalg::cholesky_rank1_update( + handle, L_view, rank, n_rows, workspace_view, &Lwork, uplo); ASSERT_TRUE(raft::devArrMatch(L_exp.data(), L.data(), From 5ef4391d8615924a6d00c6747ff65de4db896c09 Mon Sep 17 00:00:00 2001 From: divyegala Date: Wed, 13 Jul 2022 13:39:14 -0700 Subject: [PATCH 04/19] matrix_vector --- cpp/include/raft/linalg/apply.hpp | 28 ++++ cpp/include/raft/linalg/matrix_vector_op.cuh | 135 +++++++++++++++++++ cpp/test/linalg/matrix_vector_op.cu | 74 ++++++---- 3 files changed, 209 insertions(+), 28 deletions(-) create mode 100644 cpp/include/raft/linalg/apply.hpp diff --git a/cpp/include/raft/linalg/apply.hpp b/cpp/include/raft/linalg/apply.hpp new file mode 100644 index 0000000000..a6d3858d5e --- /dev/null +++ b/cpp/include/raft/linalg/apply.hpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2022, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace raft::linalg { + +/** + * @brief Enum for reduction/broadcast where an operation is to be performed along + * a matrix's rows or columns + * + */ +enum class Apply { ALONG_ROWS, ALONG_COLUMNS }; + +} // end namespace raft::linalg \ No newline at end of file diff --git a/cpp/include/raft/linalg/matrix_vector_op.cuh b/cpp/include/raft/linalg/matrix_vector_op.cuh index 56437313e3..020a5e8566 100644 --- a/cpp/include/raft/linalg/matrix_vector_op.cuh +++ b/cpp/include/raft/linalg/matrix_vector_op.cuh @@ -18,8 +18,11 @@ #pragma once +#include "apply.hpp" #include "detail/matrix_vector_op.cuh" +#include + namespace raft { namespace linalg { @@ -99,6 +102,138 @@ void matrixVectorOp(Type* out, detail::matrixVectorOp(out, matrix, vec1, vec2, D, N, rowMajor, bcastAlongRows, op, stream); } +/** + * @defgroup matrix_vector_op Matrix Vector Operations + * @{ + */ + +/** + * @brief Operations for all the columns or rows with a given vector. + * Caution : Threads process multiple elements to speed up processing. These + * are loaded in a single read thanks to type promotion. Faster processing + * would thus only be enabled when adresses are optimally aligned for it. + * Note : the function will also check that the size of the window of accesses + * is a multiple of the number of elements processed by a thread in order to + * enable faster processing + * @tparam InElementType the data-type of the input matrices and vectors + * @tparam LayoutPolicy the layout of input and output (raft::row_major or raft::col_major) + * @tparam Lambda a device function which represents a binary operator + * @tparam OutElementType the data-type of the output raft::matrix_view + * @tparam TPB threads per block of the cuda kernel launched + * @param handle raft::handle_t + * @param out output raft::matrix_view + * @param matrix input raft::matrix_view + * @param vec vector raft::vector_view + * @param apply whether the broadcast of vector needs to happen along + * the rows of the matrix or columns using enum class raft::linalg::Apply + * @param op the mathematical operation + */ +template +void matrix_vector_op(const raft::handle_t& handle, + raft::matrix_view out, + const raft::matrix_view matrix, + const raft::vector_view vec, + Apply apply, + Lambda op) +{ + static_assert( + std::is_same_v, + "Layout mismatch between Input and Output"); + RAFT_EXPECTS(out.is_contiguous(), "Output must be contiguous"); + RAFT_EXPECTS(matrix.is_contiguous(), "Input must be contiguous"); + RAFT_EXPECTS(out.size() == matrix.size(), "Size mismatch between Output and Input"); + + auto constexpr rowMajor = std::is_same_v; + auto bcastAlongRows = apply == Apply::ALONG_ROWS; + + if (bcastAlongRows) { + RAFT_EXPECTS(out.extent(1) == vec.size(), "Size mismatch between matrix and vector"); + } else { + RAFT_EXPECTS(out.extent(0) == vec.size(), "Size mismatch between matrix and vector"); + } + + matrixVectorOp(out.data(), + matrix.data(), + vec.data(), + out.extent(1), + out.extent(0), + rowMajor, + bcastAlongRows, + op, + handle.get_stream()); +} + +/** + * @brief Operations for all the columns or rows with the given vectors. + * Caution : Threads process multiple elements to speed up processing. These + * are loaded in a single read thanks to type promotion. Faster processing + * would thus only be enabled when adresses are optimally aligned for it. + * Note : the function will also check that the size of the window of accesses + * is a multiple of the number of elements processed by a thread in order to + * enable faster processing + * @tparam InElementType the data-type of the input matrices and vectors + * @tparam LayoutPolicy the layout of input and output (raft::row_major or raft::col_major) + * @tparam Lambda a device function which represents a binary operator + * @tparam OutElementType the data-type of the output raft::matrix_view + * @tparam TPB threads per block of the cuda kernel launched + * @param handle raft::handle_t + * @param out output raft::matrix_view + * @param matrix input raft::matrix_view + * @param vec1 the first vector raft::vector_view + * @param vec2 the second vector raft::vector_view + * @param apply whether the broadcast of vector needs to happen along + * the rows of the matrix or columns using enum class raft::linalg::Apply + * @param op the mathematical operation + */ +template +void matrix_vector_op(const raft::handle_t& handle, + raft::matrix_view out, + const raft::matrix_view matrix, + const raft::vector_view vec1, + const raft::vector_view vec2, + Apply apply, + Lambda op) +{ + static_assert( + std::is_same_v, + "Layout mismatch between Input and Output"); + RAFT_EXPECTS(out.is_contiguous(), "Output must be contiguous"); + RAFT_EXPECTS(matrix.is_contiguous(), "Input must be contiguous"); + RAFT_EXPECTS(out.size() == matrix.size(), "Size mismatch between Output and Input"); + + auto constexpr rowMajor = std::is_same_v; + auto bcastAlongRows = apply == Apply::ALONG_ROWS; + + if (bcastAlongRows) { + RAFT_EXPECTS(out.extent(1) == vec1.size(), "Size mismatch between matrix and vector"); + RAFT_EXPECTS(out.extent(1) == vec2.size(), "Size mismatch between matrix and vector"); + } else { + RAFT_EXPECTS(out.extent(0) == vec1.size(), "Size mismatch between matrix and vector"); + RAFT_EXPECTS(out.extent(0) == vec2.size(), "Size mismatch between matrix and vector"); + } + + matrixVectorOp(out.data(), + matrix.data(), + vec1.data(), + vec2.data(), + out.extent(1), + out.extent(0), + rowMajor, + bcastAlongRows, + op, + handle.get_stream()); +} + +/** @} */ // end of group matrix_vector_op + }; // end namespace linalg }; // end namespace raft diff --git a/cpp/test/linalg/matrix_vector_op.cu b/cpp/test/linalg/matrix_vector_op.cu index b01b3a1ca1..304df8b6df 100644 --- a/cpp/test/linalg/matrix_vector_op.cu +++ b/cpp/test/linalg/matrix_vector_op.cu @@ -41,7 +41,8 @@ template // for an extended __device__ lambda cannot have private or protected access // within its class template -void matrixVectorOpLaunch(T* out, +void matrixVectorOpLaunch(const raft::handle_t& handle, + T* out, const T* in, const T* vec1, const T* vec2, @@ -49,32 +50,49 @@ void matrixVectorOpLaunch(T* out, IdxType N, bool rowMajor, bool bcastAlongRows, - bool useTwoVectors, - cudaStream_t stream) + bool useTwoVectors) { + auto out_row_major = raft::make_matrix_view(out, N, D); + auto in_row_major = raft::make_matrix_view(in, N, D); + + auto out_col_major = raft::make_matrix_view(out, N, D); + auto in_col_major = raft::make_matrix_view(in, N, D); + + auto apply = bcastAlongRows ? Apply::ALONG_ROWS : Apply::ALONG_COLUMNS; + auto len = bcastAlongRows ? D : N; + auto vec1_view = raft::make_vector_view(vec1, len); + auto vec2_view = raft::make_vector_view(vec2, len); + if (useTwoVectors) { - matrixVectorOp( - out, - in, - vec1, - vec2, - D, - N, - rowMajor, - bcastAlongRows, - [] __device__(T a, T b, T c) { return a + b + c; }, - stream); + if (rowMajor) { + matrix_vector_op(handle, + out_row_major, + in_row_major, + vec1_view, + vec2_view, + apply, + [] __device__(T a, T b, T c) { return a + b + c; }); + } else { + matrix_vector_op(handle, + out_col_major, + in_col_major, + vec1_view, + vec2_view, + apply, + [] __device__(T a, T b, T c) { return a + b + c; }); + } } else { - matrixVectorOp( - out, - in, - vec1, - D, - N, - rowMajor, - bcastAlongRows, - [] __device__(T a, T b) { return a + b; }, - stream); + if (rowMajor) { + matrix_vector_op( + handle, out_row_major, in_row_major, vec1_view, apply, [] __device__(T a, T b) { + return a + b; + }); + } else { + matrix_vector_op( + handle, out_col_major, in_col_major, vec1_view, apply, [] __device__(T a, T b) { + return a + b; + }); + } } } @@ -124,7 +142,8 @@ class MatVecOpTest : public ::testing::TestWithParam> (T)1.0, stream); } - matrixVectorOpLaunch(out.data(), + matrixVectorOpLaunch(handle, + out.data(), in.data(), vec1.data(), vec2.data(), @@ -132,9 +151,8 @@ class MatVecOpTest : public ::testing::TestWithParam> N, params.rowMajor, params.bcastAlongRows, - params.useTwoVectors, - stream); - handle.sync_stream(stream); + params.useTwoVectors); + handle.sync_stream(); } protected: From be85cac91c5854e97c59a6cfa237fdcb3f5bb0ee Mon Sep 17 00:00:00 2001 From: divyegala Date: Tue, 19 Jul 2022 14:52:41 -0700 Subject: [PATCH 05/19] updating mdspan for index_type extents --- .../mdspan/.github/workflows/cmake.yml | 4 +- .../.github/workflows/single-header.yml | 2 +- .../raft/thirdparty/mdspan/CMakeLists.txt | 36 +- cpp/include/raft/thirdparty/mdspan/README.md | 25 +- .../benchmarks/copy/copy_layout_stride.cpp | 169 ++-- .../thirdparty/mdspan/benchmarks/fill.hpp | 16 +- .../benchmarks/matvec/cuda/matvec_cuda.cu | 6 +- .../matvec/openmp/matvec_openmp.cpp | 75 +- .../benchmarks/stencil/cuda/CMakeLists.txt | 4 + .../stencil/cuda/stencil_3d_cuda.cu | 88 +- .../stencil/openmp/stencil_3d_openmp.cpp | 130 +-- .../mdspan/benchmarks/stencil/stencil_3d.cpp | 25 +- .../benchmarks/sum/openmp/sum_3d_openmp.cpp | 30 +- .../mdspan/benchmarks/sum/sum_3d_common.hpp | 34 +- .../mdspan/benchmarks/sum/sum_3d_left.cpp | 13 +- .../mdspan/benchmarks/sum/sum_3d_right.cpp | 14 +- .../benchmarks/sum/sum_submdspan_right.cpp | 37 +- .../openmp/tiny_matrix_add_openmp.cpp | 106 +-- .../tiny_matrix_add/tiny_matrix_add.cpp | 36 +- .../mdspan/compilation_tests/CMakeLists.txt | 11 +- .../ctest_constexpr_dereference.cpp | 22 +- .../ctest_constexpr_layouts.cpp | 4 +- .../ctest_constexpr_submdspan.cpp | 44 +- .../ctest_constructor_sfinae.cpp | 44 +- .../compilation_tests/ctest_extents_ctors.cpp | 58 +- .../ctest_extents_type_check.cpp | 87 ++ .../ctest_layout_convertible.cpp | 121 +++ .../ctest_mdspan_convertible.cpp | 8 +- .../ctest_no_unique_address.cpp | 20 +- .../ctest_standard_layout.cpp | 40 +- .../ctest_trivially_copyable.cpp | 40 +- .../examples/dot_product/dot_product.cpp | 8 +- .../tiled_layout/simple_tiled_layout.cpp | 2 +- .../experimental/__p0009_bits/config.hpp | 8 +- .../__p0009_bits/default_accessor.hpp | 12 +- .../experimental/__p0009_bits/extents.hpp | 235 ++++-- .../experimental/__p0009_bits/layout_left.hpp | 139 ++-- .../__p0009_bits/layout_right.hpp | 147 ++-- .../__p0009_bits/layout_stride.hpp | 342 +++++--- .../experimental/__p0009_bits/macros.hpp | 81 +- .../__p0009_bits/maybe_static_value.hpp | 36 +- .../experimental/__p0009_bits/mdspan.hpp | 232 ++++-- .../standard_layout_static_array.hpp | 185 +++-- .../__p0009_bits/static_array.hpp | 48 +- .../experimental/__p0009_bits/submdspan.hpp | 108 +-- .../experimental/__p0009_bits/type_list.hpp | 3 +- .../experimental/__p1684_bits/mdarray.hpp | 463 +++++++++++ .../mdspan/include/experimental/mdarray | 48 ++ .../thirdparty/mdspan/tests/CMakeLists.txt | 58 +- .../thirdparty/mdspan/tests/offload_utils.hpp | 9 +- .../mdspan/tests/test_contiguous_layouts.cpp | 373 --------- .../mdspan/tests/test_element_access.cpp | 2 +- .../mdspan/tests/test_exhaustive_layouts.cpp | 427 ++++++++++ .../thirdparty/mdspan/tests/test_extents.cpp | 75 +- .../mdspan/tests/test_layout_ctors.cpp | 40 +- .../mdspan/tests/test_layout_stride.cpp | 37 +- .../mdspan/tests/test_mdarray_ctors.cpp | 770 ++++++++++++++++++ .../mdspan/tests/test_mdspan_conversion.cpp | 6 +- .../mdspan/tests/test_mdspan_ctors.cpp | 209 +++-- .../mdspan/tests/test_submdspan.cpp | 95 +-- 60 files changed, 3873 insertions(+), 1674 deletions(-) create mode 100644 cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_extents_type_check.cpp create mode 100644 cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_layout_convertible.cpp create mode 100644 cpp/include/raft/thirdparty/mdspan/include/experimental/__p1684_bits/mdarray.hpp create mode 100644 cpp/include/raft/thirdparty/mdspan/include/experimental/mdarray delete mode 100644 cpp/include/raft/thirdparty/mdspan/tests/test_contiguous_layouts.cpp create mode 100644 cpp/include/raft/thirdparty/mdspan/tests/test_exhaustive_layouts.cpp create mode 100644 cpp/include/raft/thirdparty/mdspan/tests/test_mdarray_ctors.cpp diff --git a/cpp/include/raft/thirdparty/mdspan/.github/workflows/cmake.yml b/cpp/include/raft/thirdparty/mdspan/.github/workflows/cmake.yml index 482e267b13..a5411082af 100644 --- a/cpp/include/raft/thirdparty/mdspan/.github/workflows/cmake.yml +++ b/cpp/include/raft/thirdparty/mdspan/.github/workflows/cmake.yml @@ -16,7 +16,7 @@ jobs: compiler_prefix: /usr/bin - compiler_driver: icpx compiler_prefix: /opt/intel/oneapi/compiler/latest/linux/bin - compiler_url: https://registrationcenter-download.intel.com/akdlm/irc_nas/18209/l_dpcpp-cpp-compiler_p_2021.4.0.3201_offline.sh + compiler_url: https://registrationcenter-download.intel.com/akdlm/irc_nas/18478/l_dpcpp-cpp-compiler_p_2022.0.2.84_offline.sh name: ${{ matrix.compiler_driver }} steps: @@ -26,7 +26,7 @@ jobs: uses: actions/cache@v2 with: path: /opt/intel/oneapi - key: oneapi-${{ matrix.compile_url}} + key: oneapi-${{ matrix.compiler_url}} - name: Install icpx if: ${{ matrix.compiler_driver == 'icpx' && steps.cache-icpx.outputs.cache-hit != 'true' }} run: | diff --git a/cpp/include/raft/thirdparty/mdspan/.github/workflows/single-header.yml b/cpp/include/raft/thirdparty/mdspan/.github/workflows/single-header.yml index 1f37f0f781..a8fc8911d9 100644 --- a/cpp/include/raft/thirdparty/mdspan/.github/workflows/single-header.yml +++ b/cpp/include/raft/thirdparty/mdspan/.github/workflows/single-header.yml @@ -11,7 +11,7 @@ jobs: steps: - uses: actions/checkout@v2 - run: sudo apt-get install python3 - - run: ./make_single_header.py ./include/experimental/mdspan > /tmp/mdspan.hpp + - run: ./make_single_header.py ./include/experimental/mdarray > /tmp/mdspan.hpp - uses: actions/checkout@v2 with: ref: single-header diff --git a/cpp/include/raft/thirdparty/mdspan/CMakeLists.txt b/cpp/include/raft/thirdparty/mdspan/CMakeLists.txt index 3e81ca9297..0ff692820a 100644 --- a/cpp/include/raft/thirdparty/mdspan/CMakeLists.txt +++ b/cpp/include/raft/thirdparty/mdspan/CMakeLists.txt @@ -1,10 +1,12 @@ -cmake_minimum_required(VERSION 3.13) +cmake_minimum_required(VERSION 3.14) project(MDSpan - VERSION 0.2.0 + VERSION 0.4.0 LANGUAGES CXX ) +include(GNUInstallDirs) + ################################################################################ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") @@ -17,10 +19,11 @@ option(MDSPAN_ENABLE_BENCHMARKS "Enable benchmarks." Off) option(MDSPAN_ENABLE_COMP_BENCH "Enable compilation benchmarks." Off) option(MDSPAN_ENABLE_CUDA "Enable Cuda tests/benchmarks/examples if tests/benchmarks/examples are enabled." Off) option(MDSPAN_ENABLE_OPENMP "Enable OpenMP benchmarks if benchmarks are enabled." On) +option(MDSPAN_USE_SYSTEM_GTEST "Use system-installed GoogleTest library for tests." Off) # Option to override which C++ standard to use set(MDSPAN_CXX_STANDARD DETECT CACHE STRING "Override the default CXX_STANDARD to compile with.") -set_property(CACHE MDSPAN_CXX_STANDARD PROPERTY STRINGS DETECT 14 17 20) +set_property(CACHE MDSPAN_CXX_STANDARD PROPERTY STRINGS DETECT 14 17 20 23) option(MDSPAN_ENABLE_CONCEPTS "Try to enable concepts support by giving extra flags." On) @@ -48,8 +51,18 @@ elseif(MDSPAN_CXX_STANDARD STREQUAL "20") else() message(FATAL_ERROR "Requested MDSPAN_CXX_STANDARD \"20\" not supported by provided C++ compiler") endif() +elseif(MDSPAN_CXX_STANDARD STREQUAL "23") + if("cxx_std_23" IN_LIST CMAKE_CXX_COMPILE_FEATURES) + message(STATUS "Using C++23 standard") + set(CMAKE_CXX_STANDARD 23) + else() + message(FATAL_ERROR "Requested MDSPAN_CXX_STANDARD \"23\" not supported by provided C++ compiler") + endif() else() - if("cxx_std_20" IN_LIST CMAKE_CXX_COMPILE_FEATURES) + if("cxx_std_23" IN_LIST CMAKE_CXX_COMPILE_FEATURES) + set(CMAKE_CXX_STANDARD 23) + message(STATUS "Detected support for C++23 standard") + elseif("cxx_std_20" IN_LIST CMAKE_CXX_COMPILE_FEATURES) set(CMAKE_CXX_STANDARD 20) message(STATUS "Detected support for C++20 standard") elseif("cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES) @@ -66,7 +79,7 @@ endif() ################################################################################ if(MDSPAN_ENABLE_CONCEPTS) - if(CMAKE_CXX_STANDARD STREQUAL "20") + if(CMAKE_CXX_STANDARD GREATER_EQUAL 20) include(CheckCXXCompilerFlag) check_cxx_compiler_flag("-fconcepts" COMPILER_SUPPORTS_FCONCEPTS) if(COMPILER_SUPPORTS_FCONCEPTS) @@ -109,19 +122,19 @@ add_library(std::mdspan ALIAS mdspan) target_include_directories(mdspan INTERFACE $ - $ + $ ) ################################################################################ install(TARGETS mdspan EXPORT mdspanTargets - INCLUDES DESTINATION include + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) install(EXPORT mdspanTargets FILE mdspanTargets.cmake NAMESPACE std:: - DESTINATION lib/cmake/mdspan + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/mdspan ) export(TARGETS mdspan @@ -129,19 +142,20 @@ export(TARGETS mdspan FILE mdspanTargets.cmake ) -install(DIRECTORY include/experimental DESTINATION include) +install(DIRECTORY include/experimental DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) include(CMakePackageConfigHelpers) configure_package_config_file(cmake/mdspanConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/mdspanConfig.cmake - INSTALL_DESTINATION lib/cmake/mdspan + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/mdspan ) write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/mdspanConfigVersion.cmake COMPATIBILITY SameMajorVersion + ARCH_INDEPENDENT ) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/mdspanConfig.cmake ${CMAKE_CURRENT_BINARY_DIR}/mdspanConfigVersion.cmake - DESTINATION lib/cmake/mdspan + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/mdspan ) ################################################################################ diff --git a/cpp/include/raft/thirdparty/mdspan/README.md b/cpp/include/raft/thirdparty/mdspan/README.md index af4de7966a..1fee071212 100644 --- a/cpp/include/raft/thirdparty/mdspan/README.md +++ b/cpp/include/raft/thirdparty/mdspan/README.md @@ -3,7 +3,15 @@ Reference `mdspan` implementation The ISO-C++ proposal [P0009](https://wg21.link/p0009) will add support for non-owning multi-dimensional array references to the C++ standard library. This repository aims to provide a production-quality implementation of the proposal as written (with a few caveats, see below) in preparation for the addition of `mdspan` to the standard. Please feel free to use this, file bugs when it breaks, and let us know how it works for you :-) -[Try it out on Godbolt](https://godbolt.org/z/KMT3G9Ese){: .btn } +[Try it out on Godbolt](https://godbolt.org/z/Mxa7cej1a){: .btn } + +During the final leg of the ISO C++ committee review process a number of breaking changes were proposed and accepted (issue #136). These are now merged into the stable branch. + +Note: There is a tag mdspan-0.3.0 which reflects the status of P0009 before R17 - i.e. it does not have the integral type template parameter for `extents`. +Note: There is a tag mdspan-0.4.0 which reflects the status of P0009 before + +* renaming `pointer`, `data`, `is_contiguous` and `is_always_contiguous`; and before +* renaming `size_type` to `index_type` and introducing a new `size_type = make_unsigned_t` alias. Using `mdspan` -------------- @@ -25,7 +33,14 @@ This implementation is header-only, with compiler features detected using featur ### Running tests -TODO write this +#### Configurations + +- clang-15 / cmake 3.23 + - Warning free with `-Wall -Wextra -pedantic` for C++23/20. In C++17 pedantic will give a few warnings, in C++14 Wextra will also give some. + - `cmake -DMDSPAN_ENABLE_TESTS=ON -DMDSPAN_ENABLE_BENCHMARKS=ON -DCMAKE_CXX_FLAGS="-Werror -Wall -Wextra -pedantic" -DCMAKE_CXX_STANDARD=23 -DMDSPAN_CXX_STANDARD=23 -DCMAKE_CXX_COMPILER=clang++` +- gcc-11 / cmake 3.23 + - Warning free with `-Wall -Wextra -pedantic` for C++23/20. In C++17 and C++14 pedantic will give a warning (note only with `CMAKE_CXX_EXTENSION=OFF`). + - `cmake -DMDSPAN_ENABLE_TESTS=ON -DMDSPAN_ENABLE_BENCHMARKS=ON -DCMAKE_CXX_FLAGS="-Werror -Wall -Wextra -pedantic" -DCMAKE_CXX_STANDARD=17 -DMDSPAN_CXX_STANDARD=17 -DCMAKE_CXX_COMPILER=g++ -DCMAKE_CXX_EXTENSIONS=OFF` ### Running benchmarks @@ -34,19 +49,23 @@ TODO write this Caveats ------- -This implementation is fully conforming with a few exceptions (most of which are extensions): +This implementation is fully conforming with revision 14 of P0009 with a few exceptions (most of which are extensions): ### C++20 - implements `operator()` not `operator[]` - note you can control which operator is available with defining `MDSPAN_USE_BRACKET_OPERATOR=[0,1]` and `MDSPAN_USE_PAREN_OPERATOR=[0,1]` irrespective of whether multi dimensional subscript support is detected. ### C++17 and C++14 +- mdspan has a default constructor even in cases where it shouldn't (i.e. all static extents, and default constructible mapping/accessor) +- the `layout_stride::mapping::strides` function returns `array` not `span`. - the conditional explicit markup is missing, making certain constructors implicit - most notably you can implicitly convert from dynamic extent to static extent, which you can't in C++20 mode ### C++14 - deduction guides don't exist + + Acknowledgements ================ diff --git a/cpp/include/raft/thirdparty/mdspan/benchmarks/copy/copy_layout_stride.cpp b/cpp/include/raft/thirdparty/mdspan/benchmarks/copy/copy_layout_stride.cpp index 553c8110d7..18f9284fbe 100644 --- a/cpp/include/raft/thirdparty/mdspan/benchmarks/copy/copy_layout_stride.cpp +++ b/cpp/include/raft/thirdparty/mdspan/benchmarks/copy/copy_layout_stride.cpp @@ -47,6 +47,8 @@ #include "fill.hpp" +using index_type = int; + namespace stdex = std::experimental; _MDSPAN_INLINE_VARIABLE constexpr auto dyn = stdex::dynamic_extent; @@ -63,25 +65,25 @@ void BM_MDSpan_Copy_2D_right(benchmark::State& state, MDSpan, DynSizes... dyn) { mdspan_benchmark::fill_random(s); auto dest = MDSpan{buffer2.get(), dyn...}; for (auto _ : state) { - for(size_t i = 0; i < s.extent(0); ++i) { - for (size_t j = 0; j < s.extent(1); ++j) { + for(index_type i = 0; i < s.extent(0); ++i) { + for (index_type j = 0; j < s.extent(1); ++j) { dest(i, j) = s(i, j); } } - benchmark::DoNotOptimize(s.data()); - benchmark::DoNotOptimize(dest.data()); + benchmark::DoNotOptimize(s.data_handle()); + benchmark::DoNotOptimize(dest.data_handle()); } state.SetBytesProcessed(s.size() * sizeof(value_type) * state.iterations()); } BENCHMARK_CAPTURE( - BM_MDSpan_Copy_2D_right, size_100_100, stdex::mdspan>() + BM_MDSpan_Copy_2D_right, size_100_100, stdex::mdspan>(nullptr) ); BENCHMARK_CAPTURE( - BM_MDSpan_Copy_2D_right, size_100_dyn, stdex::mdspan>(), 100 + BM_MDSpan_Copy_2D_right, size_100_dyn, stdex::mdspan>(), 100 ); BENCHMARK_CAPTURE( - BM_MDSpan_Copy_2D_right, size_dyn_dyn, stdex::mdspan>(), 100, 100 + BM_MDSpan_Copy_2D_right, size_dyn_dyn, stdex::mdspan>(), 100, 100 ); //================================================================================ @@ -100,133 +102,60 @@ void BM_MDSpan_Copy_2D_stride(benchmark::State& state, MDSpan, LayoutMapping map mdspan_benchmark::fill_random(s); auto dest = MDSpan{buffer2.get(), map}; for (auto _ : state) { - for(size_t i = 0; i < s.extent(0); ++i) { - for (size_t j = 0; j < s.extent(1); ++j) { + for(index_type i = 0; i < s.extent(0); ++i) { + for (index_type j = 0; j < s.extent(1); ++j) { dest(i, j) = s(i, j); } } - benchmark::DoNotOptimize(s.data()); - benchmark::DoNotOptimize(dest.data()); + benchmark::DoNotOptimize(s.data_handle()); + benchmark::DoNotOptimize(dest.data_handle()); } state.SetBytesProcessed(s.size() * sizeof(value_type) * state.iterations()); } BENCHMARK_CAPTURE( BM_MDSpan_Copy_2D_stride, size_100_100, - stdex::mdspan, stdex::layout_stride>(), - stdex::layout_stride::template mapping>( - stdex::extents<100, 100>{}, + stdex::mdspan, stdex::layout_stride>(nullptr, + stdex::layout_stride::mapping>()), + stdex::layout_stride::template mapping>( + stdex::extents{}, // layout right std::array{100, 1} ) ); BENCHMARK_CAPTURE( BM_MDSpan_Copy_2D_stride, size_100_100d, - stdex::mdspan, stdex::layout_stride>(), - stdex::layout_stride::template mapping>( - stdex::extents<100, dyn>{100}, + stdex::mdspan, stdex::layout_stride>(), + stdex::layout_stride::template mapping>( + stdex::extents{100}, // layout right std::array{100, 1} ) ); BENCHMARK_CAPTURE( BM_MDSpan_Copy_2D_stride, size_100d_100, - stdex::mdspan, stdex::layout_stride>(), - stdex::layout_stride::template mapping>( - stdex::extents{100}, + stdex::mdspan, stdex::layout_stride>(), + stdex::layout_stride::template mapping>( + stdex::extents{100}, // layout right std::array{100, 1} ) ); BENCHMARK_CAPTURE( BM_MDSpan_Copy_2D_stride, size_100d_100d, - stdex::mdspan, stdex::layout_stride>(), - stdex::layout_stride::template mapping>( - stdex::extents{100, 100}, + stdex::mdspan, stdex::layout_stride>(), + stdex::layout_stride::template mapping>( + stdex::extents{100, 100}, // layout right std::array{100, 1} ) ); -BENCHMARK_CAPTURE( - BM_MDSpan_Copy_2D_stride, size_fixed_dyn_100_100, - stdex::mdspan, stdex::layout_stride<100, dyn>>(), - stdex::layout_stride<100, dyn>::template mapping>( - stdex::extents<100, 100>{}, - // layout right - std::array{1} - ) -); -BENCHMARK_CAPTURE( - BM_MDSpan_Copy_2D_stride, size_fixed_dyn_100_100d, - stdex::mdspan, stdex::layout_stride<100, dyn>>(), - stdex::layout_stride<100, dyn>::template mapping>( - stdex::extents<100, dyn>{100}, - // layout right - std::array{1} - ) -); -BENCHMARK_CAPTURE( - BM_MDSpan_Copy_2D_stride, size_fixed_dyn_100d_100, - stdex::mdspan, stdex::layout_stride<100, dyn>>(), - stdex::layout_stride<100, dyn>::template mapping>( - stdex::extents{100}, - // layout right - std::array{1} - ) -); -BENCHMARK_CAPTURE( - BM_MDSpan_Copy_2D_stride, size_dyn_fixed_100d_100d, - stdex::mdspan, stdex::layout_stride<100, dyn>>(), - stdex::layout_stride<100, dyn>::template mapping>( - stdex::extents{100, 100}, - // layout right - std::array{1} - ) -); - -BENCHMARK_CAPTURE( - BM_MDSpan_Copy_2D_stride, size_dyn_fixed_100_100, - stdex::mdspan, stdex::layout_stride>(), - stdex::layout_stride::template mapping>( - stdex::extents<100, 100>{}, - // layout right - std::array{100} - ) -); -BENCHMARK_CAPTURE( - BM_MDSpan_Copy_2D_stride, size_dyn_fixed_100_100d, - stdex::mdspan, stdex::layout_stride>(), - stdex::layout_stride::template mapping>( - stdex::extents<100, dyn>{100}, - // layout right - std::array{100} - ) -); -BENCHMARK_CAPTURE( - BM_MDSpan_Copy_2D_stride, size_dyn_fixed_100d_100, - stdex::mdspan, stdex::layout_stride>(), - stdex::layout_stride::template mapping>( - stdex::extents{100}, - // layout right - std::array{100} - ) -); -BENCHMARK_CAPTURE( - BM_MDSpan_Copy_2D_stride, size_dyn_fixed_100d_100d, - stdex::mdspan, stdex::layout_stride>(), - stdex::layout_stride::template mapping>( - stdex::extents{100, 100}, - // layout right - std::array{100} - ) -); - //================================================================================ template void BM_MDSpan_Copy_2D_stride_diff_map(benchmark::State& state, - T, Extents exts, MapSrc map_src, MapDst map_dest + T, Extents, MapSrc map_src, MapDst map_dest ) { using value_type = T; auto buff_src = std::make_unique( @@ -235,33 +164,33 @@ void BM_MDSpan_Copy_2D_stride_diff_map(benchmark::State& state, auto buff_dest = std::make_unique( map_dest.required_span_size() ); - using map_stride_dyn = stdex::layout_stride; + using map_stride_dyn = stdex::layout_stride; using mdspan_type = stdex::mdspan; auto src = mdspan_type{buff_src.get(), map_src}; mdspan_benchmark::fill_random(src); auto dest = mdspan_type{buff_dest.get(), map_dest}; for (auto _ : state) { - for(size_t i = 0; i < src.extent(0); ++i) { - for (size_t j = 0; j < src.extent(1); ++j) { + for(index_type i = 0; i < src.extent(0); ++i) { + for (index_type j = 0; j < src.extent(1); ++j) { dest(i, j) = src(i, j); } } - benchmark::DoNotOptimize(src.data()); - benchmark::DoNotOptimize(dest.data()); + benchmark::DoNotOptimize(src.data_handle()); + benchmark::DoNotOptimize(dest.data_handle()); } state.SetBytesProcessed(src.extent(0) * src.extent(1) * sizeof(value_type) * state.iterations()); } BENCHMARK_CAPTURE( BM_MDSpan_Copy_2D_stride_diff_map, size_100d_100d_bcast_0, int(), - stdex::extents{100, 100}, - stdex::layout_stride::template mapping>( - stdex::extents{100, 100}, + stdex::extents{100, 100}, + stdex::layout_stride::template mapping>( + stdex::extents{100, 100}, // layout right std::array{0, 1} ), - stdex::layout_stride::template mapping>( - stdex::extents{100, 100}, + stdex::layout_stride::template mapping>( + stdex::extents{100, 100}, // layout right std::array{100, 1} ) @@ -269,14 +198,14 @@ BENCHMARK_CAPTURE( BENCHMARK_CAPTURE( BM_MDSpan_Copy_2D_stride_diff_map, size_100d_100d_bcast_1, int(), - stdex::extents{100, 100}, - stdex::layout_stride::template mapping>( - stdex::extents{100, 100}, + stdex::extents{100, 100}, + stdex::layout_stride::template mapping>( + stdex::extents{100, 100}, // layout right std::array{1, 0} ), - stdex::layout_stride::template mapping>( - stdex::extents{100, 100}, + stdex::layout_stride::template mapping>( + stdex::extents{100, 100}, // layout right std::array{100, 1} ) @@ -284,14 +213,14 @@ BENCHMARK_CAPTURE( BENCHMARK_CAPTURE( BM_MDSpan_Copy_2D_stride_diff_map, size_100d_100d_bcast_both, int(), - stdex::extents{100, 100}, - stdex::layout_stride::template mapping>( - stdex::extents{100, 100}, + stdex::extents{100, 100}, + stdex::layout_stride::template mapping>( + stdex::extents{100, 100}, // layout right std::array{0, 0} ), - stdex::layout_stride::template mapping>( - stdex::extents{100, 100}, + stdex::layout_stride::template mapping>( + stdex::extents{100, 100}, // layout right std::array{100, 1} ) @@ -307,7 +236,7 @@ void BM_Raw_Copy_1D(benchmark::State& state, T, size_t size) { auto buffer = std::make_unique(size); { // just for setup... - auto wrapped = stdex::mdspan>{buffer.get(), size}; + auto wrapped = stdex::mdspan>{buffer.get(), size}; mdspan_benchmark::fill_random(wrapped); } value_type* src = buffer.get(); @@ -335,7 +264,7 @@ void BM_Raw_Copy_2D(benchmark::State& state, T, size_t x, size_t y) { auto buffer = std::make_unique(x * y); { // just for setup... - auto wrapped = stdex::mdspan>{buffer.get(), x * y}; + auto wrapped = stdex::mdspan>{buffer.get(), x * y}; mdspan_benchmark::fill_random(wrapped); } value_type* src = buffer.get(); diff --git a/cpp/include/raft/thirdparty/mdspan/benchmarks/fill.hpp b/cpp/include/raft/thirdparty/mdspan/benchmarks/fill.hpp index d45f1f76e6..8b38cb2b82 100644 --- a/cpp/include/raft/thirdparty/mdspan/benchmarks/fill.hpp +++ b/cpp/include/raft/thirdparty/mdspan/benchmarks/fill.hpp @@ -86,9 +86,9 @@ namespace _impl { template T&& _repeated_with(T&& v) noexcept { return std::forward(v); } -template +template void _do_fill_random( - std::experimental::mdspan, Rest...> s, + std::experimental::mdspan, Rest...> s, RNG& gen, Dist& dist ) @@ -96,14 +96,14 @@ void _do_fill_random( s() = dist(gen); } -template +template void _do_fill_random( - std::experimental::mdspan, Rest...> s, + std::experimental::mdspan, Rest...> s, RNG& gen, Dist& dist ) { - for(size_t i = 0; i < s.extent(0); ++i) { + for(SizeT i = 0; i < s.extent(0); ++i) { _do_fill_random(std::experimental::submdspan(s, i, _repeated_with(std::experimental::full_extent)...), gen, dist); } } @@ -124,7 +124,7 @@ void fill_random(std::experimental::mdspan s, long long seed = 12 #define MDSPAN_BENCHMARK_ALL_3D(bench_template, prefix, md_template, X, Y, Z) \ BENCHMARK_CAPTURE( \ - bench_template, prefix##fixed_##X##_##Y##_##Z, md_template{} \ + bench_template, prefix##fixed_##X##_##Y##_##Z, md_template{nullptr} \ ); \ BENCHMARK_CAPTURE( \ bench_template, prefix##dyn_##X##_##Y##_d##Z, md_template{}, Z \ @@ -150,7 +150,7 @@ BENCHMARK_CAPTURE( \ #define MDSPAN_BENCHMARK_ALL_3D_MANUAL(bench_template, prefix, md_template, X, Y, Z) \ BENCHMARK_CAPTURE( \ - bench_template, prefix##fixed_##X##_##Y##_##Z, md_template{} \ + bench_template, prefix##fixed_##X##_##Y##_##Z, md_template{nullptr} \ )->UseManualTime(); \ BENCHMARK_CAPTURE( \ bench_template, prefix##dyn_##X##_##Y##_d##Z, md_template{}, Z \ @@ -176,7 +176,7 @@ BENCHMARK_CAPTURE( \ #define MDSPAN_BENCHMARK_ALL_3D_REAL_TIME(bench_template, prefix, md_template, X, Y, Z) \ BENCHMARK_CAPTURE( \ - bench_template, prefix##fixed_##X##_##Y##_##Z, md_template{} \ + bench_template, prefix##fixed_##X##_##Y##_##Z, md_template{nullptr} \ )->UseRealTime(); \ BENCHMARK_CAPTURE( \ bench_template, prefix##dyn_##X##_##Y##_d##Z, md_template{}, Z \ diff --git a/cpp/include/raft/thirdparty/mdspan/benchmarks/matvec/cuda/matvec_cuda.cu b/cpp/include/raft/thirdparty/mdspan/benchmarks/matvec/cuda/matvec_cuda.cu index 17748c2414..8833cff2bd 100644 --- a/cpp/include/raft/thirdparty/mdspan/benchmarks/matvec/cuda/matvec_cuda.cu +++ b/cpp/include/raft/thirdparty/mdspan/benchmarks/matvec/cuda/matvec_cuda.cu @@ -58,10 +58,12 @@ static constexpr int global_repeat = 16; //================================================================================ +using size_type = int; + template -using lmdspan = stdex::mdspan, stdex::layout_left>; +using lmdspan = stdex::mdspan, stdex::layout_left>; template -using rmdspan = stdex::mdspan, stdex::layout_right>; +using rmdspan = stdex::mdspan, stdex::layout_right>; void throw_runtime_exception(const std::string &msg) { std::ostringstream o; diff --git a/cpp/include/raft/thirdparty/mdspan/benchmarks/matvec/openmp/matvec_openmp.cpp b/cpp/include/raft/thirdparty/mdspan/benchmarks/matvec/openmp/matvec_openmp.cpp index aa34f1c4c7..429aa54aa3 100644 --- a/cpp/include/raft/thirdparty/mdspan/benchmarks/matvec/openmp/matvec_openmp.cpp +++ b/cpp/include/raft/thirdparty/mdspan/benchmarks/matvec/openmp/matvec_openmp.cpp @@ -52,16 +52,15 @@ //================================================================================ -static constexpr int warpsPerBlock = 4; -static constexpr int global_delta = 1; static constexpr int global_repeat = 1; //================================================================================ +using index_type = int; template -using lmdspan = stdex::mdspan, stdex::layout_left>; +using lmdspan = stdex::mdspan, stdex::layout_left>; template -using rmdspan = stdex::mdspan, stdex::layout_right>; +using rmdspan = stdex::mdspan, stdex::layout_right>; void throw_runtime_exception(const std::string &msg) { std::ostringstream o; @@ -72,8 +71,8 @@ void throw_runtime_exception(const std::string &msg) { template void OpenMP_first_touch_2D(MDSpan s) { #pragma omp parallel for - for(size_t i = 0; i < s.extent(0); i ++) { - for(size_t j = 0; j < s.extent(1); j ++) { + for(index_type i = 0; i < s.extent(0); i ++) { + for(index_type j = 0; j < s.extent(1); j ++) { s(i,j) = 0; } } @@ -82,7 +81,7 @@ void OpenMP_first_touch_2D(MDSpan s) { template void OpenMP_first_touch_1D(MDSpan s) { #pragma omp parallel for - for(size_t i = 0; i < s.extent(0); i ++) { + for(index_type i = 0; i < s.extent(0); i ++) { s(i) = 0; } } @@ -114,9 +113,9 @@ void BM_MDSpan_OpenMP_MatVec(benchmark::State& state, MDSpanMatrix, DynSizes... mdspan_benchmark::fill_random(y); #pragma omp parallel for - for(size_t i = 0; i < A.extent(0); i ++) { + for(index_type i = 0; i < A.extent(0); i ++) { value_type y_i = 0; - for(size_t j = 0; j < A.extent(1); j ++) { + for(index_type j = 0; j < A.extent(1); j ++) { y_i += A(i,j) * x(j); } y(i) = y_i; @@ -124,14 +123,14 @@ void BM_MDSpan_OpenMP_MatVec(benchmark::State& state, MDSpanMatrix, DynSizes... int R = 10; for (auto _ : state) { - benchmark::DoNotOptimize(A.data()); - benchmark::DoNotOptimize(y.data()); - benchmark::DoNotOptimize(x.data()); + benchmark::DoNotOptimize(A.data_handle()); + benchmark::DoNotOptimize(y.data_handle()); + benchmark::DoNotOptimize(x.data_handle()); for(int r=0; r diff --git a/cpp/include/raft/thirdparty/mdspan/benchmarks/stencil/cuda/stencil_3d_cuda.cu b/cpp/include/raft/thirdparty/mdspan/benchmarks/stencil/cuda/stencil_3d_cuda.cu index 0dda433086..81b9f83702 100644 --- a/cpp/include/raft/thirdparty/mdspan/benchmarks/stencil/cuda/stencil_3d_cuda.cu +++ b/cpp/include/raft/thirdparty/mdspan/benchmarks/stencil/cuda/stencil_3d_cuda.cu @@ -41,9 +41,6 @@ //@HEADER */ -#include "fill.hpp" - -#include #include #include @@ -51,6 +48,16 @@ #include #include +// Whether to let mapping convert index calculation to the type used +// to index into the mdspan +//#define _MDSPAN_USE_MAPPING_ARG_CAST +// Overwrite what extents.extent() returns and what the actual storage type is +//#define _MDSPAN_OVERWRITE_EXTENTS_SIZE_TYPE int +// Choose the index type used by the code +using idx_t = size_t; + +#include "fill.hpp" +#include //================================================================================ static constexpr int global_delta = 1; @@ -63,6 +70,7 @@ using lmdspan = stdex::mdspan, stdex::layout_left>; template using rmdspan = stdex::mdspan, stdex::layout_right>; + void throw_runtime_exception(const std::string &msg) { std::ostringstream o; o << msg; @@ -155,22 +163,24 @@ void BM_MDSpan_Cuda_Stencil_3D(benchmark::State& state, MDSpan, DynSizes... dyn) auto s = fill_device_mdspan(MDSpan{}, dyn...); auto o = fill_device_mdspan(MDSpan{}, dyn...); - int d = global_delta; + idx_t d = static_cast(global_delta); int repeats = global_repeat==0? (s.extent(0)*s.extent(1)*s.extent(2) > (100*100*100) ? 50 : 1000) : global_repeat; auto lambda = [=] __device__ { for(int r = 0; r < repeats; ++r) { - for(size_t i = blockIdx.x+d; i < s.extent(0)-d; i += gridDim.x) { - for(size_t j = threadIdx.z+d; j < s.extent(1)-d; j += blockDim.z) { - for(size_t k = threadIdx.y+d; k < s.extent(2)-d; k += blockDim.y) { - value_type sum_local = 0; - for(size_t di = i-d; di < i+d+1; di++) { - for(size_t dj = j-d; dj < j+d+1; dj++) { - for(size_t dk = k-d; dk < k+d+1; dk++) { + for(idx_t i = blockIdx.x+d; i < static_cast(s.extent(0))-d; i += gridDim.x) { + for(idx_t j = threadIdx.z+d; j < static_cast(s.extent(1))-d; j += blockDim.z) { + for(idx_t k = threadIdx.y+d; k < static_cast(s.extent(2))-d; k += blockDim.y) { + for(int q=0; q<128; q++) { + value_type sum_local = o(i,j,k); + for(idx_t di = i-d; di < i+d+1; di++) { + for(idx_t dj = j-d; dj < j+d+1; dj++) { + for(idx_t dk = k-d; dk < k+d+1; dk++) { sum_local += s(di, dj, dk); }}} o(i,j,k) = sum_local; + } } } } @@ -192,14 +202,19 @@ void BM_MDSpan_Cuda_Stencil_3D(benchmark::State& state, MDSpan, DynSizes... dyn) CUDA_SAFE_CALL(cudaFree(s.data())); } MDSPAN_BENCHMARK_ALL_3D_MANUAL(BM_MDSpan_Cuda_Stencil_3D, right_, rmdspan, 80, 80, 80); -MDSPAN_BENCHMARK_ALL_3D_MANUAL(BM_MDSpan_Cuda_Stencil_3D, left_, lmdspan, 80, 80, 80); -MDSPAN_BENCHMARK_ALL_3D_MANUAL(BM_MDSpan_Cuda_Stencil_3D, right_, rmdspan, 400, 400, 400); -MDSPAN_BENCHMARK_ALL_3D_MANUAL(BM_MDSpan_Cuda_Stencil_3D, left_, lmdspan, 400, 400, 400); +//MDSPAN_BENCHMARK_ALL_3D_MANUAL(BM_MDSpan_Cuda_Stencil_3D, left_, lmdspan, 80, 80, 80); +//MDSPAN_BENCHMARK_ALL_3D_MANUAL(BM_MDSpan_Cuda_Stencil_3D, right_, rmdspan, 400, 400, 400); +//MDSPAN_BENCHMARK_ALL_3D_MANUAL(BM_MDSpan_Cuda_Stencil_3D, left_, lmdspan, 400, 400, 400); //================================================================================ template -void BM_Raw_Cuda_Stencil_3D_right(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) { +void BM_Raw_Cuda_Stencil_3D_right(benchmark::State& state, T, SizeX x_, SizeY y_, SizeZ z_) { + + idx_t d = static_cast(global_delta); + idx_t x = static_cast(x_); + idx_t y = static_cast(y_); + idx_t z = static_cast(z_); using value_type = T; value_type* data = nullptr; @@ -213,22 +228,23 @@ void BM_Raw_Cuda_Stencil_3D_right(benchmark::State& state, T, SizeX x, SizeY y, data_o = o.data(); } - int d = global_delta; int repeats = global_repeat==0? (x*y*z > (100*100*100) ? 50 : 1000) : global_repeat; auto lambda = [=] __device__ { for(int r = 0; r < repeats; ++r) { - for(size_t i = blockIdx.x+d; i < x-d; i += gridDim.x) { - for(size_t j = threadIdx.z+d; j < y-d; j += blockDim.z) { - for(size_t k = threadIdx.y+d; k < z-d; k += blockDim.y) { - value_type sum_local = 0; - for(size_t di = i-d; di < i+d+1; di++) { - for(size_t dj = j-d; dj < j+d+1; dj++) { - for(size_t dk = k-d; dk < k+d+1; dk++) { + for(idx_t i = blockIdx.x+d; i < x-d; i += gridDim.x) { + for(idx_t j = threadIdx.z+d; j < y-d; j += blockDim.z) { + for(idx_t k = threadIdx.y+d; k < z-d; k += blockDim.y) { + for(int q=0; q<128; q++) { + value_type sum_local = data_o[k + j*z + i*z*y]; + for(idx_t di = i-d; di < i+d+1; di++) { + for(idx_t dj = j-d; dj < j+d+1; dj++) { + for(idx_t dk = k-d; dk < k+d+1; dk++) { sum_local += data[dk + dj*z + di*z*y]; }}} data_o[k + j*z + i*z*y] = sum_local; + } } } } @@ -255,7 +271,12 @@ BENCHMARK_CAPTURE(BM_Raw_Cuda_Stencil_3D_right, size_400_400_400, int(), 400, 40 //================================================================================ template -void BM_Raw_Cuda_Stencil_3D_left(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) { +void BM_Raw_Cuda_Stencil_3D_left(benchmark::State& state, T, SizeX x_, SizeY y_, SizeZ z_) { + + idx_t d = static_cast(global_delta); + idx_t x = static_cast(x_); + idx_t y = static_cast(y_); + idx_t z = static_cast(z_); using value_type = T; value_type* data = nullptr; @@ -269,21 +290,22 @@ void BM_Raw_Cuda_Stencil_3D_left(benchmark::State& state, T, SizeX x, SizeY y, S data_o = o.data(); } - int d = global_delta; int repeats = global_repeat==0? (x*y*z > (100*100*100) ? 50 : 1000) : global_repeat; auto lambda = [=] __device__ { for(int r = 0; r < repeats; ++r) { - for(size_t i = blockIdx.x+d; i < x-d; i += gridDim.x) { - for(size_t j = threadIdx.z+d; j < y-d; j += blockDim.z) { - for(size_t k = threadIdx.y+d; k < z-d; k += blockDim.y) { - value_type sum_local = 0; - for(size_t di = i-d; di < i+d+1; di++) { - for(size_t dj = j-d; dj < j+d+1; dj++) { - for(size_t dk = k-d; dk < k+d+1; dk++) { + for(idx_t i = blockIdx.x+d; i < x-d; i += gridDim.x) { + for(idx_t j = threadIdx.z+d; j < y-d; j += blockDim.z) { + for(idx_t k = threadIdx.y+d; k < z-d; k += blockDim.y) { + for(int q=0; q<128; q++) { + value_type sum_local = data_o[k*x*y + j*x + i]; + for(idx_t di = i-d; di < i+d+1; di++) { + for(idx_t dj = j-d; dj < j+d+1; dj++) { + for(idx_t dk = k-d; dk < k+d+1; dk++) { sum_local += data[dk*x*y + dj*x + di]; }}} data_o[k*x*y + j*x + i] = sum_local; + } } } } @@ -306,7 +328,7 @@ void BM_Raw_Cuda_Stencil_3D_left(benchmark::State& state, T, SizeX x, SizeY y, S CUDA_SAFE_CALL(cudaFree(data)); } BENCHMARK_CAPTURE(BM_Raw_Cuda_Stencil_3D_left, size_80_80_80, int(), 80, 80, 80); -BENCHMARK_CAPTURE(BM_Raw_Cuda_Stencil_3D_left, size_400_400_400, int(), 400, 400, 400); +//BENCHMARK_CAPTURE(BM_Raw_Cuda_Stencil_3D_left, size_400_400_400, int(), 400, 400, 400); //================================================================================ diff --git a/cpp/include/raft/thirdparty/mdspan/benchmarks/stencil/openmp/stencil_3d_openmp.cpp b/cpp/include/raft/thirdparty/mdspan/benchmarks/stencil/openmp/stencil_3d_openmp.cpp index b38259ef6e..bed2c43a78 100644 --- a/cpp/include/raft/thirdparty/mdspan/benchmarks/stencil/openmp/stencil_3d_openmp.cpp +++ b/cpp/include/raft/thirdparty/mdspan/benchmarks/stencil/openmp/stencil_3d_openmp.cpp @@ -56,16 +56,16 @@ //================================================================================ -static constexpr int warpsPerBlock = 4; static constexpr int global_delta = 1; -static constexpr int global_repeat = 16; //================================================================================ +using index_type = int; + template -using lmdspan = stdex::mdspan, stdex::layout_left>; +using lmdspan = stdex::mdspan, stdex::layout_left>; template -using rmdspan = stdex::mdspan, stdex::layout_right>; +using rmdspan = stdex::mdspan, stdex::layout_right>; void throw_runtime_exception(const std::string &msg) { std::ostringstream o; @@ -76,9 +76,9 @@ void throw_runtime_exception(const std::string &msg) { template void OpenMP_first_touch_3D(MDSpan s) { #pragma omp parallel for - for(size_t i = 0; i < s.extent(0); i ++) { - for(size_t j = 0; j < s.extent(1); j ++) { - for(size_t k = 0; k < s.extent(2); k ++) { + for(index_type i = 0; i < s.extent(0); i ++) { + for(index_type j = 0; j < s.extent(1); j ++) { + for(index_type k = 0; k < s.extent(2); k ++) { s(i,j,k) = 0; } } @@ -106,13 +106,13 @@ void BM_MDSpan_OpenMP_Stencil_3D(benchmark::State& state, MDSpan, DynSizes... dy int d = global_delta; #pragma omp parallel for - for(size_t i = d; i < s.extent(0)-d; i ++) { - for(size_t j = d; j < s.extent(1)-d; j ++) { - for(size_t k = d; k < s.extent(2)-d; k ++) { + for(index_type i = d; i < s.extent(0)-d; i ++) { + for(index_type j = d; j < s.extent(1)-d; j ++) { + for(index_type k = d; k < s.extent(2)-d; k ++) { value_type sum_local = 0; - for(size_t di = i-d; di < i+d+1; di++) { - for(size_t dj = j-d; dj < j+d+1; dj++) { - for(size_t dk = k-d; dk < k+d+1; dk++) { + for(index_type di = i-d; di < i+d+1; di++) { + for(index_type dj = j-d; dj < j+d+1; dj++) { + for(index_type dk = k-d; dk < k+d+1; dk++) { sum_local += s(di, dj, dk); }}} o(i,j,k) = sum_local; @@ -122,13 +122,13 @@ void BM_MDSpan_OpenMP_Stencil_3D(benchmark::State& state, MDSpan, DynSizes... dy for (auto _ : state) { #pragma omp parallel for - for(size_t i = d; i < s.extent(0)-d; i ++) { - for(size_t j = d; j < s.extent(1)-d; j ++) { - for(size_t k = d; k < s.extent(2)-d; k ++) { + for(index_type i = d; i < s.extent(0)-d; i ++) { + for(index_type j = d; j < s.extent(1)-d; j ++) { + for(index_type k = d; k < s.extent(2)-d; k ++) { value_type sum_local = 0; - for(size_t di = i-d; di < i+d+1; di++) { - for(size_t dj = j-d; dj < j+d+1; dj++) { - for(size_t dk = k-d; dk < k+d+1; dk++) { + for(index_type di = i-d; di < i+d+1; di++) { + for(index_type dj = j-d; dj < j+d+1; dj++) { + for(index_type dk = k-d; dk < k+d+1; dk++) { sum_local += s(di, dj, dk); }}} o(i,j,k) = sum_local; @@ -150,7 +150,7 @@ MDSPAN_BENCHMARK_ALL_3D(BM_MDSpan_OpenMP_Stencil_3D, left_, lmdspan, 400, 400, 4 template void BM_Raw_OpenMP_Stencil_3D_right(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) { - using MDSpan = stdex::mdspan>; + using MDSpan = stdex::mdspan>; using value_type = typename MDSpan::value_type; auto buffer_size = MDSpan{nullptr, x,y,z}.mapping().required_span_size(); @@ -158,24 +158,24 @@ void BM_Raw_OpenMP_Stencil_3D_right(benchmark::State& state, T, SizeX x, SizeY y auto s = MDSpan{buffer_s.get(), x,y,z}; OpenMP_first_touch_3D(s); mdspan_benchmark::fill_random(s); - T* s_ptr = s.data(); + T* s_ptr = s.data_handle(); auto buffer_o = std::make_unique(buffer_size); auto o = MDSpan{buffer_o.get(), x,y,z}; OpenMP_first_touch_3D(o); mdspan_benchmark::fill_random(o); - T* o_ptr = o.data(); + T* o_ptr = o.data_handle(); int d = global_delta; #pragma omp parallel for - for(size_t i = d; i < x-d; i ++) { - for(size_t j = d; j < y-d; j ++) { - for(size_t k = d; k < z-d; k ++) { + for(index_type i = d; i < x-d; i ++) { + for(index_type j = d; j < y-d; j ++) { + for(index_type k = d; k < z-d; k ++) { value_type sum_local = 0; - for(size_t di = i-d; di < i+d+1; di++) { - for(size_t dj = j-d; dj < j+d+1; dj++) { - for(size_t dk = k-d; dk < k+d+1; dk++) { + for(index_type di = i-d; di < i+d+1; di++) { + for(index_type dj = j-d; dj < j+d+1; dj++) { + for(index_type dk = k-d; dk < k+d+1; dk++) { sum_local += s_ptr[dk + dj*z + di*z*y]; }}} o_ptr[k + j*z + i*z*y] = sum_local; @@ -185,13 +185,13 @@ void BM_Raw_OpenMP_Stencil_3D_right(benchmark::State& state, T, SizeX x, SizeY y for (auto _ : state) { #pragma omp parallel for - for(size_t i = d; i < x-d; i ++) { - for(size_t j = d; j < y-d; j ++) { - for(size_t k = d; k < z-d; k ++) { + for(index_type i = d; i < x-d; i ++) { + for(index_type j = d; j < y-d; j ++) { + for(index_type k = d; k < z-d; k ++) { value_type sum_local = 0; - for(size_t di = i-d; di < i+d+1; di++) { - for(size_t dj = j-d; dj < j+d+1; dj++) { - for(size_t dk = k-d; dk < k+d+1; dk++) { + for(index_type di = i-d; di < i+d+1; di++) { + for(index_type dj = j-d; dj < j+d+1; dj++) { + for(index_type dk = k-d; dk < k+d+1; dk++) { sum_local += s_ptr[dk + dj*z + di*z*y]; }}} o_ptr[k + j*z + i*z*y] = sum_local; @@ -211,7 +211,7 @@ BENCHMARK_CAPTURE(BM_Raw_OpenMP_Stencil_3D_right, size_400_400_400, int(), 400, template void BM_Raw_OpenMP_Stencil_3D_left(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) { - using MDSpan = stdex::mdspan>; + using MDSpan = stdex::mdspan>; using value_type = typename MDSpan::value_type; auto buffer_size = MDSpan{nullptr, x,y,z}.mapping().required_span_size(); @@ -219,24 +219,24 @@ void BM_Raw_OpenMP_Stencil_3D_left(benchmark::State& state, T, SizeX x, SizeY y, auto s = MDSpan{buffer_s.get(), x,y,z}; OpenMP_first_touch_3D(s); mdspan_benchmark::fill_random(s); - T* s_ptr = s.data(); + T* s_ptr = s.data_handle(); auto buffer_o = std::make_unique(buffer_size); auto o = MDSpan{buffer_o.get(), x,y,z}; OpenMP_first_touch_3D(o); mdspan_benchmark::fill_random(o); - T* o_ptr = o.data(); + T* o_ptr = o.data_handle(); int d = global_delta; #pragma omp parallel for - for(size_t i = d; i < x-d; i ++) { - for(size_t j = d; j < y-d; j ++) { - for(size_t k = d; k < z-d; k ++) { + for(index_type i = d; i < x-d; i ++) { + for(index_type j = d; j < y-d; j ++) { + for(index_type k = d; k < z-d; k ++) { value_type sum_local = 0; - for(size_t di = i-d; di < i+d+1; di++) { - for(size_t dj = j-d; dj < j+d+1; dj++) { - for(size_t dk = k-d; dk < k+d+1; dk++) { + for(index_type di = i-d; di < i+d+1; di++) { + for(index_type dj = j-d; dj < j+d+1; dj++) { + for(index_type dk = k-d; dk < k+d+1; dk++) { sum_local += s_ptr[dk*x*y + dj*x + di]; }}} o_ptr[k*x*y + j*x + i] = sum_local; @@ -246,13 +246,13 @@ void BM_Raw_OpenMP_Stencil_3D_left(benchmark::State& state, T, SizeX x, SizeY y, for (auto _ : state) { #pragma omp parallel for - for(size_t i = d; i < x-d; i ++) { - for(size_t j = d; j < y-d; j ++) { - for(size_t k = d; k < z-d; k ++) { + for(index_type i = d; i < x-d; i ++) { + for(index_type j = d; j < y-d; j ++) { + for(index_type k = d; k < z-d; k ++) { value_type sum_local = 0; - for(size_t di = i-d; di < i+d+1; di++) { - for(size_t dj = j-d; dj < j+d+1; dj++) { - for(size_t dk = k-d; dk < k+d+1; dk++) { + for(index_type di = i-d; di < i+d+1; di++) { + for(index_type dj = j-d; dj < j+d+1; dj++) { + for(index_type dk = k-d; dk < k+d+1; dk++) { sum_local += s_ptr[dk*x*y + dj*x + di]; }}} o_ptr[k*x*y + j*x + i] = sum_local; @@ -272,9 +272,9 @@ typename MDSpan::value_type*** make_3d_ptr_array(MDSpan s) { static_assert(std::is_same::value,"Creating MD Ptr only works from mdspan with layout_right"); using value_type = typename MDSpan::value_type; value_type*** ptr= new value_type**[s.extent(0)]; - for(size_t i = 0; i void free_3d_ptr_array(T*** ptr, size_t extent_0) { - for(int i=0; i void BM_RawMDPtr_OpenMP_Stencil_3D_right(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) { - using MDSpan = stdex::mdspan>; + using MDSpan = stdex::mdspan>; using value_type = typename MDSpan::value_type; auto buffer_size = MDSpan{nullptr, x,y,z}.mapping().required_span_size(); @@ -309,13 +309,13 @@ void BM_RawMDPtr_OpenMP_Stencil_3D_right(benchmark::State& state, T, SizeX x, Si int d = global_delta; #pragma omp parallel for - for(size_t i = d; i < x-d; i ++) { - for(size_t j = d; j < y-d; j ++) { - for(size_t k = d; k < z-d; k ++) { + for(index_type i = d; i < x-d; i ++) { + for(index_type j = d; j < y-d; j ++) { + for(index_type k = d; k < z-d; k ++) { value_type sum_local = 0; - for(size_t di = i-d; di < i+d+1; di++) { - for(size_t dj = j-d; dj < j+d+1; dj++) { - for(size_t dk = k-d; dk < k+d+1; dk++) { + for(index_type di = i-d; di < i+d+1; di++) { + for(index_type dj = j-d; dj < j+d+1; dj++) { + for(index_type dk = k-d; dk < k+d+1; dk++) { sum_local += s_ptr[0][0][0]; }}} o_ptr[i][j][k] = sum_local; @@ -325,13 +325,13 @@ void BM_RawMDPtr_OpenMP_Stencil_3D_right(benchmark::State& state, T, SizeX x, Si for (auto _ : state) { #pragma omp parallel for - for(size_t i = d; i < x-d; i ++) { - for(size_t j = d; j < y-d; j ++) { - for(size_t k = d; k < z-d; k ++) { + for(index_type i = d; i < x-d; i ++) { + for(index_type j = d; j < y-d; j ++) { + for(index_type k = d; k < z-d; k ++) { value_type sum_local = 0; - for(size_t di = i-d; di < i+d+1; di++) { - for(size_t dj = j-d; dj < j+d+1; dj++) { - for(size_t dk = k-d; dk < k+d+1; dk++) { + for(index_type di = i-d; di < i+d+1; di++) { + for(index_type dj = j-d; dj < j+d+1; dj++) { + for(index_type dk = k-d; dk < k+d+1; dk++) { sum_local += s_ptr[di][dj][dk]; }}} o_ptr[i][j][k] = sum_local; diff --git a/cpp/include/raft/thirdparty/mdspan/benchmarks/stencil/stencil_3d.cpp b/cpp/include/raft/thirdparty/mdspan/benchmarks/stencil/stencil_3d.cpp index 97b9ee6fd2..25f5a502e4 100644 --- a/cpp/include/raft/thirdparty/mdspan/benchmarks/stencil/stencil_3d.cpp +++ b/cpp/include/raft/thirdparty/mdspan/benchmarks/stencil/stencil_3d.cpp @@ -58,10 +58,12 @@ static constexpr int global_delta = 1; +using index_type = int; + template -using lmdspan = stdex::mdspan, stdex::layout_left>; +using lmdspan = stdex::mdspan, stdex::layout_left>; template -using rmdspan = stdex::mdspan, stdex::layout_right>; +using rmdspan = stdex::mdspan, stdex::layout_right>; //================================================================================ @@ -81,15 +83,16 @@ void BM_MDSpan_Stencil_3D(benchmark::State& state, MDSpan, DynSizes... dyn) { int d = global_delta; + using index_type = typename MDSpan::index_type; for (auto _ : state) { benchmark::DoNotOptimize(o); - for(size_t i = d; i < s.extent(0)-d; i ++) { - for(size_t j = d; j < s.extent(1)-d; j ++) { - for(size_t k = d; k < s.extent(2)-d; k ++) { + for(index_type i = d; i < s.extent(0)-d; i ++) { + for(index_type j = d; j < s.extent(1)-d; j ++) { + for(index_type k = d; k < s.extent(2)-d; k ++) { value_type sum_local = 0; - for(size_t di = i-d; di < i+d+1; di++) { - for(size_t dj = j-d; dj < j+d+1; dj++) { - for(size_t dk = k-d; dk < k+d+1; dk++) { + for(index_type di = i-d; di < i+d+1; di++) { + for(index_type dj = j-d; dj < j+d+1; dj++) { + for(index_type dk = k-d; dk < k+d+1; dk++) { sum_local += s(di, dj, dk); }}} o(i,j,k) = sum_local; @@ -112,7 +115,7 @@ MDSPAN_BENCHMARK_ALL_3D(BM_MDSpan_Stencil_3D, left_, lmdspan, 400, 400, 400); template void BM_Raw_Stencil_3D_right(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) { - using MDSpan = stdex::mdspan>; + using MDSpan = stdex::mdspan>; using value_type = typename MDSpan::value_type; auto buffer_size = MDSpan{nullptr, x,y,z}.mapping().required_span_size(); @@ -121,7 +124,7 @@ void BM_Raw_Stencil_3D_right(benchmark::State& state, T, SizeX x, SizeY y, SizeZ { auto s = MDSpan{buffer_s.get(), x, y, z}; mdspan_benchmark::fill_random(s); - s_ptr = s.data(); + s_ptr = s.data_handle(); } T* o_ptr = nullptr; @@ -129,7 +132,7 @@ void BM_Raw_Stencil_3D_right(benchmark::State& state, T, SizeX x, SizeY y, SizeZ { auto o = MDSpan{buffer_o.get(), x, y, z}; mdspan_benchmark::fill_random(o); - o_ptr = o.data(); + o_ptr = o.data_handle(); } int d = global_delta; diff --git a/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/openmp/sum_3d_openmp.cpp b/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/openmp/sum_3d_openmp.cpp index eea0ff659c..9ab6a0ddf4 100644 --- a/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/openmp/sum_3d_openmp.cpp +++ b/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/openmp/sum_3d_openmp.cpp @@ -52,10 +52,12 @@ //================================================================================ +using index_type = int; + template -using lmdspan = stdex::mdspan, stdex::layout_left>; +using lmdspan = stdex::mdspan, stdex::layout_left>; template -using rmdspan = stdex::mdspan, stdex::layout_right>; +using rmdspan = stdex::mdspan, stdex::layout_right>; //================================================================================ @@ -75,15 +77,15 @@ void BM_MDSpan_Sum_3D_OpenMP(benchmark::State& state, MDSpan, DynSizes... dyn) { { for (int r = 0; r < repeats; ++r) { value_type sum = 0; - for (size_t i = omp_get_thread_num(); i < s.extent(0); i += omp_get_num_threads()) { - for (size_t j = 0; j < s.extent(1); ++j) { - for (size_t k = 0; k < s.extent(2); ++k) { + for (index_type i = omp_get_thread_num(); i < s.extent(0); i += omp_get_num_threads()) { + for (index_type j = 0; j < s.extent(1); ++j) { + for (index_type k = 0; k < s.extent(2); ++k) { sum += s(i, j, k); } } } benchmark::DoNotOptimize(sum); - benchmark::DoNotOptimize(s.data()); + benchmark::DoNotOptimize(s.data_handle()); } } } @@ -110,11 +112,11 @@ void BM_MDSpan_Sum_3D_loop_OpenMP(benchmark::State& state, MDSpan, DynSizes... d auto* sum = sums_buffer.get(); for (auto _ : state) { benchmark::DoNotOptimize(sums_buffer.get()); - benchmark::DoNotOptimize(s.data()); + benchmark::DoNotOptimize(s.data_handle()); #pragma omp parallel for default(none) shared(s, sum) - for (size_t i = 0; i < s.extent(0); ++i) { - for (size_t j = 0; j < s.extent(1); ++j) { - for (size_t k = 0; k < s.extent(2); ++k) { + for (index_type i = 0; i < s.extent(0); ++i) { + for (index_type j = 0; j < s.extent(1); ++j) { + for (index_type k = 0; k < s.extent(2); ++k) { sum[omp_get_thread_num()] += s(i, j, k); } } @@ -133,7 +135,7 @@ void BM_Raw_Sum_3D_OpenMP(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) auto buffer = std::make_unique(x * y * z); { // just for setup... - auto wrapped = stdex::mdspan>{buffer.get(), x*y*z}; + auto wrapped = stdex::mdspan>{buffer.get(), x*y*z}; mdspan_benchmark::fill_random(wrapped); } @@ -147,9 +149,9 @@ void BM_Raw_Sum_3D_OpenMP(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) T sum = 0; benchmark::DoNotOptimize(sum); benchmark::DoNotOptimize(data); - for (size_t i = omp_get_thread_num(); i < x; i += omp_get_num_threads()) { - for (size_t j = 0; j < y; ++j) { - for (size_t k = 0; k < z; ++k) { + for (index_type i = omp_get_thread_num(); i < x; i += omp_get_num_threads()) { + for (index_type j = 0; j < y; ++j) { + for (index_type k = 0; k < z; ++k) { sum += data[k + j*z + i*z*y]; } } diff --git a/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_3d_common.hpp b/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_3d_common.hpp index 23b3d4c38d..50dd9eb79c 100644 --- a/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_3d_common.hpp +++ b/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_3d_common.hpp @@ -55,13 +55,13 @@ void BM_Raw_Sum_1D(benchmark::State& state, T, Size size) { auto buffer = std::make_unique(size); { // just for setup... - auto wrapped = stdex::mdspan>{buffer.get(), size}; + auto wrapped = stdex::mdspan>{buffer.get(), size}; mdspan_benchmark::fill_random(wrapped); } T* data = buffer.get(); for (auto _ : state) { T sum = 0; - for(size_t i = 0; i < size; ++i) { + for(Size i = 0; i < size; ++i) { sum += data[i]; } benchmark::DoNotOptimize(sum); @@ -82,7 +82,7 @@ void BM_Raw_Sum_3D_right(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) auto buffer = std::make_unique(x * y * z); { // just for setup... - auto wrapped = stdex::mdspan>{buffer.get(), x*y*z}; + auto wrapped = stdex::mdspan>{buffer.get(), x*y*z}; mdspan_benchmark::fill_random(wrapped); } T* data = buffer.get(); @@ -91,9 +91,9 @@ void BM_Raw_Sum_3D_right(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) for (auto _ : state) { benchmark::DoNotOptimize(data); T sum = 0; - for(size_t i = 0; i < x; ++i) { - for(size_t j = 0; j < y; ++j) { - for(size_t k = 0; k < z; ++k) { + for(SizeX i = 0; i < x; ++i) { + for(SizeY j = 0; j < y; ++j) { + for(SizeZ k = 0; k < z; ++k) { sum += data[k + j*z + i*z*y]; } } @@ -111,16 +111,16 @@ void BM_Raw_Sum_3D_left(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) { auto buffer = std::make_unique(x * y * z); { // just for setup... - auto wrapped = stdex::mdspan>{buffer.get(), x*y*z}; + auto wrapped = stdex::mdspan>{buffer.get(), x*y*z}; mdspan_benchmark::fill_random(wrapped); } T* data = buffer.get(); for (auto _ : state) { benchmark::DoNotOptimize(data); T sum = 0; - for(size_t k = 0; k < z; ++k) { - for(size_t j = 0; j < y; ++j) { - for(size_t i = 0; i < x; ++i) { + for(SizeZ k = 0; k < z; ++k) { + for(SizeY j = 0; j < y; ++j) { + for(SizeX i = 0; i < x; ++i) { sum += data[i + j*x + k*x*y]; } } @@ -138,7 +138,7 @@ void BM_Raw_Sum_3D_right_iter_left(benchmark::State& state, T, SizeX x, SizeY y, auto buffer = std::make_unique(x * y * z); { // just for setup... - auto wrapped = stdex::mdspan>{buffer.get(), x*y*z}; + auto wrapped = stdex::mdspan>{buffer.get(), x*y*z}; mdspan_benchmark::fill_random(wrapped); } @@ -151,9 +151,9 @@ void BM_Raw_Sum_3D_right_iter_left(benchmark::State& state, T, SizeX x, SizeY y, for (auto _ : state) { benchmark::DoNotOptimize(data); T sum = 0; - for(size_t k = 0; k < z; ++k) { - for(size_t j = 0; j < y; ++j) { - for(size_t i = 0; i < x; ++i) { + for(SizeZ k = 0; k < z; ++k) { + for(SizeY j = 0; j < y; ++j) { + for(SizeX i = 0; i < x; ++i) { sum += data[k + j*z + i*z*y]; } } @@ -177,7 +177,7 @@ void BM_Raw_Static_Sum_3D_right(benchmark::State& state, T, auto buffer = std::make_unique(x * y * z); { // just for setup... - auto wrapped = stdex::mdspan>{buffer.get(), x*y*z}; + auto wrapped = stdex::mdspan>{buffer.get(), x*y*z}; mdspan_benchmark::fill_random(wrapped); } T* data = buffer.get(); @@ -208,7 +208,7 @@ void BM_Raw_Static_Sum_3D_left(benchmark::State& state, T, auto buffer = std::make_unique(x * y * z); { // just for setup... - auto wrapped = stdex::mdspan>{buffer.get(), x*y*z}; + auto wrapped = stdex::mdspan>{buffer.get(), x*y*z}; mdspan_benchmark::fill_random(wrapped); } T* data = buffer.get(); @@ -239,7 +239,7 @@ void BM_Raw_Static_Sum_3D_right_iter_left(benchmark::State& state, T, auto buffer = std::make_unique(x * y * z); { // just for setup... - auto wrapped = stdex::mdspan>{buffer.get(), x*y*z}; + auto wrapped = stdex::mdspan>{buffer.get(), x*y*z}; mdspan_benchmark::fill_random(wrapped); } T* data = buffer.get(); diff --git a/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_3d_left.cpp b/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_3d_left.cpp index d343ce49dd..ac30cc9ca2 100644 --- a/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_3d_left.cpp +++ b/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_3d_left.cpp @@ -49,12 +49,13 @@ #include "sum_3d_common.hpp" #include "../fill.hpp" +using index_type = int; //================================================================================ template -using lmdspan = stdex::mdspan, stdex::layout_left>; +using lmdspan = stdex::mdspan, stdex::layout_left>; template -using rmdspan = stdex::mdspan, stdex::layout_right>; +using rmdspan = stdex::mdspan, stdex::layout_right>; //================================================================================ @@ -68,15 +69,15 @@ void BM_MDSpan_Sum_3D_left(benchmark::State& state, MDSpan, DynSizes... dyn) { mdspan_benchmark::fill_random(s); for (auto _ : state) { value_type sum = 0; - for (size_t k = 0; k < s.extent(2); ++k) { - for (size_t j = 0; j < s.extent(1); ++j) { - for(size_t i = 0; i < s.extent(0); ++i) { + for (index_type k = 0; k < s.extent(2); ++k) { + for (index_type j = 0; j < s.extent(1); ++j) { + for(index_type i = 0; i < s.extent(0); ++i) { sum += s(i, j, k); } } } benchmark::DoNotOptimize(sum); - benchmark::DoNotOptimize(s.data()); + benchmark::DoNotOptimize(s.data_handle()); } state.SetBytesProcessed(s.size() * sizeof(value_type) * state.iterations()); } diff --git a/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_3d_right.cpp b/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_3d_right.cpp index 396ae9ff2d..90b9e475ca 100644 --- a/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_3d_right.cpp +++ b/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_3d_right.cpp @@ -51,10 +51,12 @@ //================================================================================ +using index_type = int; + template -using lmdspan = stdex::mdspan, stdex::layout_left>; +using lmdspan = stdex::mdspan, stdex::layout_left>; template -using rmdspan = stdex::mdspan, stdex::layout_right>; +using rmdspan = stdex::mdspan, stdex::layout_right>; //================================================================================ @@ -71,11 +73,11 @@ void BM_MDSpan_Sum_3D_right(benchmark::State& state, MDSpan, DynSizes... dyn) { for (auto _ : state) { benchmark::DoNotOptimize(s); - benchmark::DoNotOptimize(s.data()); + benchmark::DoNotOptimize(s.data_handle()); value_type sum = 0; - for(size_t i = 0; i < s.extent(0); ++i) { - for (size_t j = 0; j < s.extent(1); ++j) { - for (size_t k = 0; k < s.extent(2); ++k) { + for(index_type i = 0; i < s.extent(0); ++i) { + for (index_type j = 0; j < s.extent(1); ++j) { + for (index_type k = 0; k < s.extent(2); ++k) { sum += s(i, j, k); } } diff --git a/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_submdspan_right.cpp b/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_submdspan_right.cpp index 43127ba1ba..f106e2f5ff 100644 --- a/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_submdspan_right.cpp +++ b/cpp/include/raft/thirdparty/mdspan/benchmarks/sum/sum_submdspan_right.cpp @@ -51,8 +51,12 @@ //================================================================================ +using index_type = int; + +template +using lmdspan = stdex::mdspan, stdex::layout_left>; template -using lmdspan = stdex::mdspan, stdex::layout_left>; +using rmdspan = stdex::mdspan, stdex::layout_right>; //================================================================================ @@ -69,19 +73,20 @@ void BM_MDSpan_Sum_Subspan_3D_right(benchmark::State& state, MDSpan, DynSizes... for (auto _ : state) { benchmark::DoNotOptimize(s); - benchmark::DoNotOptimize(s.data()); + benchmark::DoNotOptimize(s.data_handle()); value_type sum = 0; - for(size_t i = 0; i < s.extent(0); ++i) { + using index_type = typename MDSpan::index_type; + for(index_type i = 0; i < s.extent(0); ++i) { auto sub_i = stdex::submdspan(s, i, stdex::full_extent, stdex::full_extent); - for (size_t j = 0; j < s.extent(1); ++j) { + for (index_type j = 0; j < s.extent(1); ++j) { auto sub_i_j = stdex::submdspan(sub_i, j, stdex::full_extent); - for (size_t k = 0; k < s.extent(2); ++k) { + for (index_type k = 0; k < s.extent(2); ++k) { sum += sub_i_j(k); } } } benchmark::DoNotOptimize(sum); - benchmark::DoNotOptimize(s.data()); + benchmark::DoNotOptimize(s.data_handle()); } state.SetBytesProcessed(s.size() * sizeof(value_type) * state.iterations()); } @@ -143,7 +148,7 @@ template MDSPAN_FORCE_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 void _do_sum_submdspan( T& sum, - stdex::mdspan, Rest...> s + stdex::mdspan, Rest...> s ) { sum += s(); @@ -153,10 +158,10 @@ template MDSPAN_FORCE_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 void _do_sum_submdspan( T& sum, - stdex::mdspan, Rest...> s + stdex::mdspan, Rest...> s ) { - for(size_t i = 0; i < s.extent(0); ++i) { + for(index_type i = 0; i < s.extent(0); ++i) { _impl::_do_sum_submdspan(sum, stdex::submdspan( s, i, _repeated_with(stdex::full_extent)...) ); @@ -177,38 +182,38 @@ void BM_MDSpan_Sum_Subspan_MD_right(benchmark::State& state, MDSpan, DynSizes... value_type sum = 0; _impl::_do_sum_submdspan(sum, s); benchmark::DoNotOptimize(sum); - benchmark::DoNotOptimize(s.data()); + benchmark::DoNotOptimize(s.data_handle()); } state.SetBytesProcessed(s.size() * sizeof(value_type) * state.iterations()); } BENCHMARK_CAPTURE( BM_MDSpan_Sum_Subspan_MD_right, fixed_10D_size_1024, - stdex::mdspan>{} + stdex::mdspan>{nullptr} ); BENCHMARK_CAPTURE( BM_MDSpan_Sum_Subspan_MD_right, dyn_10D_size_1024, - stdex::mdspan>{}, + stdex::mdspan>{}, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 ); BENCHMARK_CAPTURE( BM_MDSpan_Sum_Subspan_MD_right, fixed5_dyn5_10D_alternate_size_1024, - stdex::mdspan>{}, + stdex::mdspan>{}, 2, 2, 2, 2, 2 ); BENCHMARK_CAPTURE( BM_MDSpan_Sum_Subspan_MD_right, fixed8_dyn2_10D_alternate_size_1024, - stdex::mdspan>{}, + stdex::mdspan>{}, 2, 2 ); BENCHMARK_CAPTURE( BM_MDSpan_Sum_Subspan_MD_right, fixed_5D_size_1024, - stdex::mdspan>{} + stdex::mdspan>{nullptr} ); BENCHMARK_CAPTURE( BM_MDSpan_Sum_Subspan_MD_right, dyn_5D_size_1024, - stdex::mdspan>{}, + stdex::mdspan>{}, 4, 4, 4, 4, 4 ); BENCHMARK_CAPTURE( diff --git a/cpp/include/raft/thirdparty/mdspan/benchmarks/tiny_matrix_add/openmp/tiny_matrix_add_openmp.cpp b/cpp/include/raft/thirdparty/mdspan/benchmarks/tiny_matrix_add/openmp/tiny_matrix_add_openmp.cpp index 0bc5a0eea0..224fe5294f 100644 --- a/cpp/include/raft/thirdparty/mdspan/benchmarks/tiny_matrix_add/openmp/tiny_matrix_add_openmp.cpp +++ b/cpp/include/raft/thirdparty/mdspan/benchmarks/tiny_matrix_add/openmp/tiny_matrix_add_openmp.cpp @@ -54,16 +54,15 @@ //================================================================================ -static constexpr int warpsPerBlock = 4; -static constexpr int global_delta = 1; static constexpr int global_repeat = 1; //================================================================================ +using index_type = int; template -using lmdspan = stdex::mdspan, stdex::layout_left>; +using lmdspan = stdex::mdspan, stdex::layout_left>; template -using rmdspan = stdex::mdspan, stdex::layout_right>; +using rmdspan = stdex::mdspan, stdex::layout_right>; void throw_runtime_exception(const std::string &msg) { std::ostringstream o; @@ -74,9 +73,9 @@ void throw_runtime_exception(const std::string &msg) { template void OpenMP_first_touch_3D(MDSpan s) { #pragma omp parallel for - for(size_t i = 0; i < s.extent(0); i ++) { - for(size_t j = 0; j < s.extent(1); j ++) { - for(size_t k = 0; k < s.extent(2); k ++) { + for(index_type i = 0; i < s.extent(0); i ++) { + for(index_type j = 0; j < s.extent(1); j ++) { + for(index_type k = 0; k < s.extent(2); k ++) { s(i,j,k) = 0; } } @@ -103,8 +102,8 @@ void BM_MDSpan_OpenMP_noloop_TinyMatrixSum(benchmark::State& state, MDSpan, DynS for (auto _ : state) { - benchmark::DoNotOptimize(o.data()); - benchmark::DoNotOptimize(s.data()); + benchmark::DoNotOptimize(o.data_handle()); + benchmark::DoNotOptimize(s.data_handle()); #pragma omp parallel { auto chunk_size = s.extent(0) / omp_get_num_threads(); @@ -117,10 +116,10 @@ void BM_MDSpan_OpenMP_noloop_TinyMatrixSum(benchmark::State& state, MDSpan, DynS chunk_start += extra; } auto chunk_end = chunk_start + chunk_size; - for (size_t i = chunk_start; i < chunk_end; ++i) { + for (index_type i = chunk_start; i < chunk_end; ++i) { for (int r = 0; r < global_repeat; r++) { - for (size_t j = 0; j < s.extent(1); j++) { - for (size_t k = 0; k < s.extent(2); k++) { + for (index_type j = 0; j < s.extent(1); j++) { + for (index_type k = 0; k < s.extent(2); k++) { o(i, j, k) += s(i, j, k); } } @@ -142,6 +141,7 @@ template void BM_MDSpan_OpenMP_TinyMatrixSum(benchmark::State& state, MDSpan, DynSizes... dyn) { using value_type = typename MDSpan::value_type; + using index_type = typename MDSpan::index_type; auto buffer_size = MDSpan{nullptr, dyn...}.mapping().required_span_size(); auto buffer_s = std::make_unique(buffer_size); @@ -155,10 +155,10 @@ void BM_MDSpan_OpenMP_TinyMatrixSum(benchmark::State& state, MDSpan, DynSizes... mdspan_benchmark::fill_random(o); #pragma omp parallel for - for(size_t i = 0; i < s.extent(0); i ++) { + for(index_type i = 0; i < s.extent(0); i ++) { for(int r = 0; r void BM_Raw_Static_OpenMP_TinyMatrixSum_right(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) { - using MDSpan = stdex::mdspan>; + using MDSpan = stdex::mdspan>; using value_type = typename MDSpan::value_type; auto buffer_size = MDSpan{nullptr, x,y,z}.mapping().required_span_size(); @@ -200,16 +200,16 @@ void BM_Raw_Static_OpenMP_TinyMatrixSum_right(benchmark::State& state, T, SizeX auto s = MDSpan{buffer_s.get(), x,y,z}; OpenMP_first_touch_3D(s); mdspan_benchmark::fill_random(s); - T* s_ptr = s.data(); + T* s_ptr = s.data_handle(); auto buffer_o = std::make_unique(buffer_size); auto o = MDSpan{buffer_o.get(), x,y,z}; OpenMP_first_touch_3D(o); mdspan_benchmark::fill_random(o); - T* o_ptr = o.data(); + T* o_ptr = o.data_handle(); #pragma omp parallel for - for(size_t i = 0; i < x; i ++) { + for(SizeX i = 0; i < x; i ++) { for(int r = 0; r void BM_Raw_OpenMP_TinyMatrixSum_right(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) { - using MDSpan = stdex::mdspan>; + using MDSpan = stdex::mdspan>; using value_type = typename MDSpan::value_type; auto buffer_size = MDSpan{nullptr, x,y,z}.mapping().required_span_size(); @@ -251,19 +251,19 @@ void BM_Raw_OpenMP_TinyMatrixSum_right(benchmark::State& state, T, SizeX x, Size auto s = MDSpan{buffer_s.get(), x,y,z}; OpenMP_first_touch_3D(s); mdspan_benchmark::fill_random(s); - T* s_ptr = s.data(); + T* s_ptr = s.data_handle(); auto buffer_o = std::make_unique(buffer_size); auto o = MDSpan{buffer_o.get(), x,y,z}; OpenMP_first_touch_3D(o); mdspan_benchmark::fill_random(o); - T* o_ptr = o.data(); + T* o_ptr = o.data_handle(); #pragma omp parallel for - for(size_t i = 0; i < x; i ++) { + for(SizeX i = 0; i < x; i ++) { for(int r = 0; r void BM_Raw_Static_OpenMP_TinyMatrixSum_left(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) { - using MDSpan = stdex::mdspan>; + using MDSpan = stdex::mdspan>; using value_type = typename MDSpan::value_type; auto buffer_size = MDSpan{nullptr, x}.mapping().required_span_size(); @@ -304,16 +304,16 @@ void BM_Raw_Static_OpenMP_TinyMatrixSum_left(benchmark::State& state, T, SizeX x auto s = MDSpan{buffer_s.get(), x}; OpenMP_first_touch_3D(s); mdspan_benchmark::fill_random(s); - T* s_ptr = s.data(); + T* s_ptr = s.data_handle(); auto buffer_o = std::make_unique(buffer_size); auto o = MDSpan{buffer_o.get(), x}; OpenMP_first_touch_3D(o); mdspan_benchmark::fill_random(o); - T* o_ptr = o.data(); + T* o_ptr = o.data_handle(); #pragma omp parallel for - for(size_t i = 0; i < x; i ++) { + for(SizeX i = 0; i < x; i ++) { for(int r = 0; r void BM_Raw_OpenMP_TinyMatrixSum_left(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) { - using MDSpan = stdex::mdspan>; + using MDSpan = stdex::mdspan>; using value_type = typename MDSpan::value_type; auto buffer_size = MDSpan{nullptr, x}.mapping().required_span_size(); @@ -357,19 +357,19 @@ void BM_Raw_OpenMP_TinyMatrixSum_left(benchmark::State& state, T, SizeX x, SizeY auto s = MDSpan{buffer_s.get(), x}; OpenMP_first_touch_3D(s); mdspan_benchmark::fill_random(s); - T* s_ptr = s.data(); + T* s_ptr = s.data_handle(); auto buffer_o = std::make_unique(buffer_size); auto o = MDSpan{buffer_o.get(), x}; OpenMP_first_touch_3D(o); mdspan_benchmark::fill_random(o); - T* o_ptr = o.data(); + T* o_ptr = o.data_handle(); #pragma omp parallel for - for(size_t i = 0; i < x; i ++) { + for(SizeX i = 0; i < x; i ++) { for(int r = 0; r typename MDSpan::value_type*** make_3d_ptr_array(MDSpan s) { static_assert(std::is_same::value,"Creating MD Ptr only works from mdspan with layout_right"); using value_type = typename MDSpan::value_type; + using index_type = typename MDSpan::index_type; + value_type*** ptr= new value_type**[s.extent(0)]; - for(size_t i = 0; i void free_3d_ptr_array(T*** ptr, size_t extent_0) { - for(int i=0; i void BM_RawMDPtr_OpenMP_TinyMatrixSum_right(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) { - using MDSpan = stdex::mdspan>; + using MDSpan = stdex::mdspan>; using value_type = typename MDSpan::value_type; auto buffer_size = MDSpan{nullptr, x}.mapping().required_span_size(); @@ -437,7 +439,7 @@ void BM_RawMDPtr_OpenMP_TinyMatrixSum_right(benchmark::State& state, T, SizeX x, T*** o_ptr = make_3d_ptr_array(o); #pragma omp parallel for - for(size_t i = 0; i < x; i ++) { + for(SizeX i = 0; i < x; i ++) { for(int r = 0; r -using lmdspan = stdex::mdspan, stdex::layout_left>; +using lmdspan = stdex::mdspan, stdex::layout_left>; template -using rmdspan = stdex::mdspan, stdex::layout_right>; +using rmdspan = stdex::mdspan, stdex::layout_right>; //================================================================================ @@ -73,12 +75,12 @@ void BM_MDSpan_TinyMatrixSum_right(benchmark::State& state, MDSpan, DynSizes... for (auto _ : state) { benchmark::DoNotOptimize(o); - benchmark::DoNotOptimize(o.data()); + benchmark::DoNotOptimize(o.data_handle()); benchmark::DoNotOptimize(s); - benchmark::DoNotOptimize(s.data()); - for(size_t i = 0; i < s.extent(0); i ++) { - for(size_t j = 0; j < s.extent(1); j ++) { - for(size_t k = 0; k < s.extent(2); k ++) { + benchmark::DoNotOptimize(s.data_handle()); + for(index_type i = 0; i < s.extent(0); i ++) { + for(index_type j = 0; j < s.extent(1); j ++) { + for(index_type k = 0; k < s.extent(2); k ++) { o(i,j,k) += s(i,j,k); } } @@ -96,26 +98,26 @@ MDSPAN_BENCHMARK_ALL_3D(BM_MDSpan_TinyMatrixSum_right, left_, lmdspan, 1000000, template void BM_Raw_Static_TinyMatrixSum_right(benchmark::State& state, T, SizeX x, SizeY y, SizeZ z) { - using MDSpan = stdex::mdspan>; + using MDSpan = stdex::mdspan>; using value_type = typename MDSpan::value_type; auto buffer_size = MDSpan{nullptr, x,y,z}.mapping().required_span_size(); auto buffer_s = std::make_unique(buffer_size); auto s = MDSpan{buffer_s.get(), x,y,z}; mdspan_benchmark::fill_random(s); - T* s_ptr = s.data(); + T* s_ptr = s.data_handle(); auto buffer_o = std::make_unique(buffer_size); auto o = MDSpan{buffer_o.get(), x,y,z}; mdspan_benchmark::fill_random(o); - T* o_ptr = o.data(); + T* o_ptr = o.data_handle(); for (auto _ : state) { benchmark::DoNotOptimize(o_ptr); benchmark::DoNotOptimize(s_ptr); - for(size_t i = 0; i < 1000000; i ++) { - for(size_t j = 0; j < 3; j ++) { - for(size_t k = 0; k < 3; k ++) { + for(index_type i = 0; i < 1000000; i ++) { + for(index_type j = 0; j < 3; j ++) { + for(index_type k = 0; k < 3; k ++) { o_ptr[k + j*3 + i*3*3] += s_ptr[k + j*3 + i*3*3]; } } @@ -136,7 +138,7 @@ void BM_Raw_TinyMatrixSum_right(benchmark::State& state, T, SizeX x, SizeY y, Si benchmark::DoNotOptimize(y); benchmark::DoNotOptimize(z); - using MDSpan = stdex::mdspan>; + using MDSpan = stdex::mdspan>; using value_type = typename MDSpan::value_type; auto buffer_s = std::make_unique(x * y * z); @@ -156,9 +158,9 @@ void BM_Raw_TinyMatrixSum_right(benchmark::State& state, T, SizeX x, SizeY y, Si for (auto _ : state) { benchmark::DoNotOptimize(o_ptr); benchmark::DoNotOptimize(s_ptr); - for(size_t i = 0; i < x; i ++) { - for(size_t j = 0; j < y; j ++) { - for(size_t k = 0; k < z; k ++) { + for(SizeX i = 0; i < x; i ++) { + for(SizeY j = 0; j < y; j ++) { + for(SizeZ k = 0; k < z; k ++) { o_ptr[k + j*z + i*z*y] += s_ptr[k + j*z + i*z*y]; } } diff --git a/cpp/include/raft/thirdparty/mdspan/compilation_tests/CMakeLists.txt b/cpp/include/raft/thirdparty/mdspan/compilation_tests/CMakeLists.txt index f72afba5e2..2afd6611df 100644 --- a/cpp/include/raft/thirdparty/mdspan/compilation_tests/CMakeLists.txt +++ b/cpp/include/raft/thirdparty/mdspan/compilation_tests/CMakeLists.txt @@ -9,17 +9,12 @@ endmacro() add_compilation_test(ctest_constructor_sfinae) add_compilation_test(ctest_extents_ctors) +add_compilation_test(ctest_extents_type_check) +add_compilation_test(ctest_layout_convertible) add_compilation_test(ctest_mdspan_convertible) add_compilation_test(ctest_standard_layout) add_compilation_test(ctest_trivially_copyable) -if(MDSPAN_ENABLE_CUDA) - # For NVCC 11.5 and older we do not do no_unique_address optimization - # thus all these strict size tests here will fail - if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER "11.5") - add_compilation_test(ctest_no_unique_address) - add_compilation_test(ctest_compressed_pair_layout) - endif() -else() +if(NOT MDSPAN_ENABLE_CUDA) add_compilation_test(ctest_no_unique_address) add_compilation_test(ctest_compressed_pair_layout) endif() diff --git a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constexpr_dereference.cpp b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constexpr_dereference.cpp index fb97ea923a..f28bf75efa 100644 --- a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constexpr_dereference.cpp +++ b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constexpr_dereference.cpp @@ -59,7 +59,7 @@ simple_static_sum_test_1(int add_to_row) { 4, 5, 6, 7, 8, 9 }; - auto s = stdex::mdspan>(data); + auto s = stdex::mdspan>(data); int result = 0; for(int col = 0; col < 3; ++col) { for(int row = 0; row < 3; ++row) { @@ -82,7 +82,7 @@ MDSPAN_STATIC_TEST( #if !defined(__INTEL_COMPILER) || (__INTEL_COMPILER>=1800) MDSPAN_STATIC_TEST( // -1 - 2 - 3 + 7 + 8 + 9 = 18 - stdex::mdspan>{nullptr}.extent(0) == 18 + stdex::mdspan>{nullptr}.extent(0) == 18 ); #endif @@ -94,7 +94,7 @@ simple_test_1d_constexpr_in_type() { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18 }; - auto s = stdex::mdspan>(data); + auto s = stdex::mdspan>(data); // 4 + 14 + 18 + 1 = 37 return s[3] + s[13] + s[17] + s[0]; } @@ -112,7 +112,7 @@ simple_dynamic_sum_test_2(int add_to_row) { 4, 5, 6, 0, 7, 8, 9, 0 }; - auto s = stdex::mdspan>(data, 3, 4); + auto s = stdex::mdspan>(data, 3, 4); int result = 0; for(int col = 0; col < 3; ++col) { for(int row = 0; row < 3; ++row) { @@ -143,7 +143,7 @@ simple_mixed_layout_left_sum_test_3(int add_to_row) { 0, 0, 0 }; auto s = stdex::mdspan< - int, stdex::extents, + int, stdex::extents, stdex::layout_left >(data, 3, 4); int result = 0; @@ -173,9 +173,9 @@ multidimensional_single_element_stress_test_impl_2( std::integer_sequence ) { using mdspan_t = stdex::mdspan< - int, stdex::extents<_repeated_ptrdiff_t<1, Idxs>...>, Layout>; + int, stdex::extents...>, Layout>; using dyn_mdspan_t = stdex::mdspan< - int, stdex::extents<_repeated_ptrdiff_t...>, Layout>; + int, stdex::extents...>, Layout>; int data[] = { 42 }; auto s = mdspan_t(data); auto s_dyn = dyn_mdspan_t(data, _repeated_ptrdiff_t<1, Idxs>...); @@ -220,13 +220,13 @@ stress_test_2d_single_element_stress_test_impl_2( std::integral_constant ) { using mdspan_t = stdex::mdspan< - int, stdex::extents, Layout>; + int, stdex::extents, Layout>; using dyn_mdspan_1_t = stdex::mdspan< - int, stdex::extents, Layout>; + int, stdex::extents, Layout>; using dyn_mdspan_2_t = stdex::mdspan< - int, stdex::extents, Layout>; + int, stdex::extents, Layout>; using dyn_mdspan_t = stdex::mdspan< - int, stdex::extents, Layout>; + int, stdex::extents, Layout>; int data[Idx1*Idx2] = { }; auto s = mdspan_t(data); auto s1 = dyn_mdspan_1_t(data, Idx1); diff --git a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constexpr_layouts.cpp b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constexpr_layouts.cpp index 323fb438b1..adc2027692 100644 --- a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constexpr_layouts.cpp +++ b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constexpr_layouts.cpp @@ -53,9 +53,9 @@ namespace stdex = std::experimental; constexpr std::ptrdiff_t layout_stride_simple(int i) { using map_t = stdex::layout_stride::template mapping< - stdex::extents<3> + stdex::extents >; - return map_t(stdex::extents<3>{}, std::array{1})(i); + return map_t(stdex::extents{}, std::array{1})(i); } MDSPAN_STATIC_TEST( diff --git a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constexpr_submdspan.cpp b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constexpr_submdspan.cpp index dce5549934..0db1cd8d3d 100644 --- a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constexpr_submdspan.cpp +++ b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constexpr_submdspan.cpp @@ -57,9 +57,9 @@ template constexpr bool dynamic_extent_1d() { int data[] = {1, 2, 3, 4, 5}; - auto s = stdex::mdspan, Layout>(data, 5); + auto s = stdex::mdspan, Layout>(data, 5); int result = 0; - for (int i = 0; i < s.extent(0); ++i) { + for (size_t i = 0; i < s.extent(0); ++i) { auto ss = stdex::submdspan(s, i); result += __MDSPAN_OP0(ss); } @@ -83,10 +83,10 @@ constexpr bool dynamic_extent_1d_all_slice() { int data[] = {1, 2, 3, 4, 5}; auto s = stdex::mdspan< - int, stdex::extents, Layout>(data, 5); + int, stdex::extents, Layout>(data, 5); int result = 0; auto ss = stdex::submdspan(s, stdex::full_extent); - for (int i = 0; i < s.extent(0); ++i) { + for (size_t i = 0; i < s.extent(0); ++i) { result += __MDSPAN_OP(ss, i); } // 1 + 2 + 3 + 4 + 5 @@ -108,10 +108,10 @@ constexpr bool dynamic_extent_1d_pair_full() { int data[] = {1, 2, 3, 4, 5}; auto s = stdex::mdspan< - int, stdex::extents, Layout>(data, 5); + int, stdex::extents, Layout>(data, 5); int result = 0; auto ss = stdex::submdspan(s, std::pair{0, 5}); - for (int i = 0; i < s.extent(0); ++i) { + for (size_t i = 0; i < s.extent(0); ++i) { result += __MDSPAN_OP(ss, i); } constexpr_assert_equal(15, result); @@ -126,9 +126,9 @@ constexpr bool dynamic_extent_1d_pair_each() { int data[] = {1, 2, 3, 4, 5}; auto s = stdex::mdspan< - int, stdex::extents, Layout>(data, 5); + int, stdex::extents, Layout>(data, 5); int result = 0; - for (int i = 0; i < s.extent(0); ++i) { + for (size_t i = 0; i < s.extent(0); ++i) { auto ss = stdex::submdspan(s, std::pair{i, i+1}); result += __MDSPAN_OP(ss, 0); @@ -154,11 +154,11 @@ constexpr bool dynamic_extent_1d_all_three() { int data[] = {1, 2, 3, 4, 5}; auto s = stdex::mdspan< - int, stdex::extents, Layout>(data, 5); + int, stdex::extents, Layout>(data, 5); auto s1 = stdex::submdspan(s, std::pair{0, 5}); auto s2 = stdex::submdspan(s1, stdex::full_extent); int result = 0; - for (int i = 0; i < s.extent(0); ++i) { + for (size_t i = 0; i < s.extent(0); ++i) { auto ss = stdex::submdspan(s2, i); result += __MDSPAN_OP0(ss); } @@ -180,11 +180,11 @@ constexpr bool dynamic_extent_2d_idx_idx() { int data[] = { 1, 2, 3, 4, 5, 6 }; auto s = stdex::mdspan< - int, stdex::extents, Layout>( + int, stdex::extents, Layout>( data, 2, 3); int result = 0; - for(int row = 0; row < s.extent(0); ++row) { - for(int col = 0; col < s.extent(1); ++col) { + for(size_t row = 0; row < s.extent(0); ++row) { + for(size_t col = 0; col < s.extent(1); ++col) { auto ss = stdex::submdspan(s, row, col); result += __MDSPAN_OP0(ss); } @@ -200,12 +200,12 @@ constexpr bool dynamic_extent_2d_idx_all_idx() { int data[] = { 1, 2, 3, 4, 5, 6 }; auto s = stdex::mdspan< - int, stdex::extents, Layout>( + int, stdex::extents, Layout>( data, 2, 3); int result = 0; - for(int row = 0; row < s.extent(0); ++row) { + for(size_t row = 0; row < s.extent(0); ++row) { auto srow = stdex::submdspan(s, row, stdex::full_extent); - for(int col = 0; col < s.extent(1); ++col) { + for(size_t col = 0; col < s.extent(1); ++col) { auto scol = stdex::submdspan(srow, col); constexpr_assert_equal(__MDSPAN_OP0(scol), __MDSPAN_OP(srow, col)); result += __MDSPAN_OP0(scol); @@ -230,7 +230,7 @@ simple_static_submdspan_test_1(int add_to_row) { 4, 5, 6, 7, 8, 9 }; - auto s = stdex::mdspan>(data); + auto s = stdex::mdspan>(data); int result = 0; for(int col = 0; col < 3; ++col) { auto scol = stdex::submdspan(s, stdex::full_extent, col); @@ -256,7 +256,7 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( // -1 - 2 - 3 + 7 + 8 + 9 = 18 - stdex::mdspan>{nullptr}.extent(0) == 18 + stdex::mdspan>{nullptr}.extent(0) == 18 ); #endif @@ -272,7 +272,7 @@ mixed_submdspan_left_test_2() { 0, 0, 0 }; auto s = stdex::mdspan, stdex::layout_left>(data, 5); + stdex::extents, stdex::layout_left>(data, 5); int result = 0; for(int col = 0; col < 5; ++col) { auto scol = stdex::submdspan(s, stdex::full_extent, col); @@ -315,7 +315,7 @@ mixed_submdspan_test_3() { 0, 0, 0, 0, 0 }; auto s = stdex::mdspan< - int, stdex::extents<3, stdex::dynamic_extent>, Layout>(data, 5); + int, stdex::extents, Layout>(data, 5); int result = 0; for(int col = 0; col < 5; ++col) { auto scol = stdex::submdspan(s, stdex::full_extent, col); @@ -360,9 +360,9 @@ submdspan_single_element_stress_test_impl_2( std::integer_sequence ) { using mdspan_t = stdex::mdspan< - int, stdex::extents<_repeated_ptrdiff_t<1, Idxs>...>, Layout>; + int, stdex::extents...>, Layout>; using dyn_mdspan_t = stdex::mdspan< - int, stdex::extents<_repeated_ptrdiff_t...>, Layout>; + int, stdex::extents...>, Layout>; int data[] = { 42 }; auto s = mdspan_t(data); auto s_dyn = dyn_mdspan_t(data, _repeated_ptrdiff_t<1, Idxs>...); diff --git a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constructor_sfinae.cpp b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constructor_sfinae.cpp index 8d6f27ddd2..e39c7e8f52 100644 --- a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constructor_sfinae.cpp +++ b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_constructor_sfinae.cpp @@ -54,28 +54,28 @@ namespace stdex = std::experimental; MDSPAN_STATIC_TEST( std::is_constructible< - stdex::extents<2, stdex::dynamic_extent>, + stdex::extents, std::array >::value ); MDSPAN_STATIC_TEST( std::is_constructible< - stdex::extents<2, stdex::dynamic_extent>, + stdex::extents, std::array >::value ); MDSPAN_STATIC_TEST( std::is_constructible< - stdex::extents<2, stdex::dynamic_extent>, + stdex::extents, int >::value ); MDSPAN_STATIC_TEST( std::is_constructible< - stdex::extents<2, stdex::dynamic_extent>, + stdex::extents, int, int64_t >::value ); @@ -83,63 +83,63 @@ MDSPAN_STATIC_TEST( // TODO @proposal-bug: not sure we really intended this??? MDSPAN_STATIC_TEST( std::is_constructible< - stdex::extents<2, stdex::dynamic_extent>, + stdex::extents, std::array >::value ); MDSPAN_STATIC_TEST( std::is_constructible< - stdex::extents<2, stdex::dynamic_extent>, + stdex::extents, float, double >::value ); MDSPAN_STATIC_TEST( std::is_constructible< - stdex::mdspan>, + stdex::mdspan>, int* >::value ); MDSPAN_STATIC_TEST( std::is_constructible< - stdex::mdspan>, + stdex::mdspan>, int* >::value ); MDSPAN_STATIC_TEST( std::is_constructible< - stdex::mdspan>, + stdex::mdspan>, int*, int >::value ); MDSPAN_STATIC_TEST( std::is_constructible< - stdex::mdspan>, + stdex::mdspan>, double*, unsigned, int >::value ); MDSPAN_STATIC_TEST( std::is_constructible< - stdex::mdspan>, + stdex::mdspan>, int*, int, int >::value ); MDSPAN_STATIC_TEST( std::is_constructible< - stdex::mdspan>, + stdex::mdspan>, int*, std::array >::value ); MDSPAN_STATIC_TEST( std::is_constructible< - stdex::mdspan>, + stdex::mdspan>, int*, std::array >::value ); @@ -152,28 +152,28 @@ MDSPAN_STATIC_TEST( // {{{1 MDSPAN_STATIC_TEST( !std::is_constructible< - stdex::extents<2, stdex::dynamic_extent>, + stdex::extents, std::array >::value ); MDSPAN_STATIC_TEST( !std::is_constructible< - stdex::extents<2, stdex::dynamic_extent>, + stdex::extents, int, int, int >::value ); MDSPAN_STATIC_TEST( !std::is_constructible< - stdex::mdspan>, + stdex::mdspan>, int*, std::array >::value ); MDSPAN_STATIC_TEST( !std::is_constructible< - stdex::mdspan>, + stdex::mdspan>, double*, int >::value ); @@ -181,29 +181,29 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( !std::is_constructible< - stdex::mdspan>, + stdex::mdspan>, int*, int, int, int >::value ); MDSPAN_STATIC_TEST( !std::is_constructible< - stdex::mdspan, stdex::layout_stride>, + stdex::mdspan, stdex::layout_stride>, int*, int, int >::value ); MDSPAN_STATIC_TEST( !std::is_constructible< - stdex::mdspan, stdex::layout_stride>, + stdex::mdspan, stdex::layout_stride>, int*, std::array >::value ); MDSPAN_STATIC_TEST( !std::is_constructible< - stdex::mdspan, stdex::layout_stride>, - int*, stdex::dextents<2> + stdex::mdspan, stdex::layout_stride>, + int*, stdex::dextents >::value ); diff --git a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_extents_ctors.cpp b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_extents_ctors.cpp index 4d3a3cd156..00126691aa 100644 --- a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_extents_ctors.cpp +++ b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_extents_ctors.cpp @@ -49,131 +49,131 @@ namespace stdex = std::experimental; MDSPAN_STATIC_TEST( std::is_constructible< - stdex::extents<1, 2, stdex::dynamic_extent>, + stdex::extents, int >::value ); MDSPAN_STATIC_TEST( std::is_copy_constructible< - stdex::extents<1, 2, stdex::dynamic_extent> + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_copy_constructible< - stdex::extents<1, 2> + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_copy_constructible< - stdex::extents + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_move_constructible< - stdex::extents<1, 2, stdex::dynamic_extent> + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_default_constructible< - stdex::extents<1, 2, 3> + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_constructible< - stdex::extents, + stdex::extents, int, int, int >::value ); MDSPAN_STATIC_TEST( !std::is_constructible< - stdex::extents, + stdex::extents, int, int >::value ); MDSPAN_STATIC_TEST( !std::is_constructible< - stdex::extents, + stdex::extents, int >::value ); MDSPAN_STATIC_TEST( std::is_constructible< - stdex::extents, - stdex::extents + stdex::extents, + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_convertible< - stdex::extents<2, 3>, - stdex::extents<2, stdex::dynamic_extent> + stdex::extents, + stdex::extents >::value ); MDSPAN_STATIC_TEST( !std::is_convertible< - stdex::extents<3, 2>, - stdex::extents<2, stdex::dynamic_extent> + stdex::extents, + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_constructible< - stdex::extents<2, stdex::dynamic_extent>, - stdex::extents<2, 3> + stdex::extents, + stdex::extents >::value ); MDSPAN_STATIC_TEST( !std::is_constructible< - stdex::extents<3, stdex::dynamic_extent>, - stdex::extents<2, 3> + stdex::extents, + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_assignable< - stdex::extents<2, stdex::dynamic_extent>, - stdex::extents<2, 3> + stdex::extents, + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_same< - stdex::dextents<0>, - stdex::extents<> + stdex::dextents, + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_same< - stdex::dextents<1>, - stdex::extents + stdex::dextents, + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_same< - stdex::dextents<2>, - stdex::extents + stdex::dextents, + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_same< - stdex::dextents<3>, - stdex::extents + stdex::dextents, + stdex::extents >::value ); diff --git a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_extents_type_check.cpp b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_extents_type_check.cpp new file mode 100644 index 0000000000..aefc7c818e --- /dev/null +++ b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_extents_type_check.cpp @@ -0,0 +1,87 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos v. 2.0 +// Copyright (2019) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Christian R. Trott (crtrott@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include "ctest_common.hpp" + +#include + +namespace stdex = std::experimental; + +using E1 = stdex::extents; + +MDSPAN_STATIC_TEST( + std::is_same::value && + std::is_same::value && + std::is_same::value && + std::is_same::value && + std::is_same::value && + std::is_same::value && + std::is_same::value && + std::is_same().extent(0)), typename E1::index_type>::value && + std::is_same().extent(1)), typename E1::index_type>::value && + (E1::rank()==2) && + (E1::rank_dynamic()==1) && + (E1::static_extent(0) == stdex::dynamic_extent) && + (E1::static_extent(1) == 3) +); + +using E2 = stdex::extents; + +MDSPAN_STATIC_TEST( + std::is_same::value && + std::is_same::value && + std::is_same::value && + std::is_same::value && + std::is_same::value && + std::is_same::value && + std::is_same::value && + std::is_same::value && + std::is_same().extent(0)), typename E2::index_type>::value && + std::is_same().extent(1)), typename E2::index_type>::value && + std::is_same().extent(2)), typename E2::index_type>::value && + (E2::rank()==3) && + (E2::rank_dynamic()==2) && + (E2::static_extent(0) == stdex::dynamic_extent) && + (E2::static_extent(1) == 3) && + (E2::static_extent(2) == stdex::dynamic_extent) +); diff --git a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_layout_convertible.cpp b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_layout_convertible.cpp new file mode 100644 index 0000000000..e293734444 --- /dev/null +++ b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_layout_convertible.cpp @@ -0,0 +1,121 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos v. 2.0 +// Copyright (2019) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Christian R. Trott (crtrott@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include "ctest_common.hpp" + +#include + +namespace stdex = std::experimental; + +struct NotARealLayout { + template + struct mapping { + using extents_type = Extents; + using rank_type = typename extents_type::rank_type; + using index_type = typename extents_type::index_type; + using layout_type = NotARealLayout; + + constexpr extents_type& extents() const { return ext; } + + template + index_type operator()(Idx ...) const { return 0; } + + index_type required_span_size() const { return 0; } + + index_type stride(rank_type) const { return 1; } + + private: + extents_type ext; + }; +}; + +template +struct AStridedLayout { + template + struct mapping { + using extents_type = Extents; + using rank_type = typename extents_type::rank_type; + using index_type = typename extents_type::index_type; + using layout_type = AStridedLayout; + + constexpr extents_type& extents() const { return ext; } + + template + index_type operator()(Idx ...) const { return 0; } + + index_type required_span_size() const { return 0; } + + index_type stride(rank_type) const { return 1; } + + constexpr static bool is_always_strided() { return true; } + constexpr static bool is_always_unique() { return unique; } + constexpr static bool is_always_exhaustive() { return true; } + constexpr bool is_strided() { return true; } + constexpr bool is_unique() { return unique; } + constexpr bool is_exhaustive() { return true; } + + private: + extents_type ext; + }; +}; + +using E1 = stdex::extents; +using E2 = stdex::extents; +using LS1 = stdex::layout_stride::mapping; +using LS2 = stdex::layout_stride::mapping; + +MDSPAN_STATIC_TEST( + !std::is_constructible::mapping>::value && + !std::is_convertible::mapping, LS1>::value +); + +MDSPAN_STATIC_TEST( + std::is_constructible::mapping>::value && + std::is_convertible::mapping, LS2>::value +); + +MDSPAN_STATIC_TEST( + !std::is_constructible>::value +); + + diff --git a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_mdspan_convertible.cpp b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_mdspan_convertible.cpp index 059b5bd8ab..fa1136b9d6 100644 --- a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_mdspan_convertible.cpp +++ b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_mdspan_convertible.cpp @@ -54,15 +54,15 @@ namespace stdex = std::experimental; MDSPAN_STATIC_TEST( std::is_convertible< - stdex::mdspan>, - stdex::mdspan> + stdex::mdspan>, + stdex::mdspan> >::value ); MDSPAN_STATIC_TEST( !std::is_convertible< - stdex::mdspan>, - stdex::mdspan> + stdex::mdspan>, + stdex::mdspan> >::value ); diff --git a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_no_unique_address.cpp b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_no_unique_address.cpp index 289baa64f5..9f7c6c052d 100644 --- a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_no_unique_address.cpp +++ b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_no_unique_address.cpp @@ -53,31 +53,31 @@ namespace stdex = std::experimental; // {{{1 MDSPAN_STATIC_TEST( - sizeof(stdex::extents<1, 2, stdex::dynamic_extent>) == sizeof(ptrdiff_t) + sizeof(stdex::extents) == sizeof(ptrdiff_t) ); MDSPAN_STATIC_TEST( - sizeof(stdex::extents) == sizeof(ptrdiff_t) + sizeof(stdex::extents) == sizeof(ptrdiff_t) ); MDSPAN_STATIC_TEST( - sizeof(stdex::extents) == 2 * sizeof(ptrdiff_t) + sizeof(stdex::extents) == 2 * sizeof(ptrdiff_t) ); MDSPAN_STATIC_TEST( - sizeof(stdex::extents) == sizeof(ptrdiff_t) + sizeof(stdex::extents) == sizeof(ptrdiff_t) ); MDSPAN_STATIC_TEST( - sizeof(stdex::extents<45, stdex::dynamic_extent, 1>) == sizeof(ptrdiff_t) + sizeof(stdex::extents) == sizeof(ptrdiff_t) ); MDSPAN_STATIC_TEST( - std::is_empty>::value + std::is_empty>::value ); MDSPAN_STATIC_TEST( - std::is_empty>::value + std::is_empty>::value ); // end extents }}}1 @@ -88,21 +88,21 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( sizeof(stdex::layout_left::template mapping< - stdex::extents<42, stdex::dynamic_extent, 73> + stdex::extents >) == sizeof(size_t) ); #if defined(__GNUC__) && (__GNUC__>8) MDSPAN_STATIC_TEST( std::is_empty + stdex::extents >>::value ); #endif MDSPAN_STATIC_TEST( sizeof(stdex::layout_stride::template mapping< - stdex::extents<42, stdex::dynamic_extent, 73> + stdex::extents >) == 4 * sizeof(size_t) ); diff --git a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_standard_layout.cpp b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_standard_layout.cpp index 486702a9da..5f0f929e5e 100644 --- a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_standard_layout.cpp +++ b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_standard_layout.cpp @@ -53,16 +53,16 @@ namespace stdex = std::experimental; // {{{1 MDSPAN_STATIC_TEST( - !std::is_base_of, stdex::detail::__partially_static_sizes<1, 2, 3>>::value + !std::is_base_of, stdex::detail::__partially_static_sizes>::value ); MDSPAN_STATIC_TEST( - !std::is_base_of, stdex::extents<1, 2, 3>>::value + !std::is_base_of, stdex::extents>::value ); MDSPAN_STATIC_TEST( std::is_standard_layout< - stdex::detail::__partially_static_sizes<1, 2, 3> + stdex::detail::__partially_static_sizes >::value ); @@ -74,43 +74,43 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_standard_layout< - stdex::extents<1, 2, stdex::dynamic_extent> + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_standard_layout< - stdex::extents + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_standard_layout< - stdex::extents + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_standard_layout< - stdex::extents + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_standard_layout< - stdex::extents<45, stdex::dynamic_extent, 1> + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_standard_layout< - stdex::extents<1, 2, 3> + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_standard_layout< - stdex::extents<42> + stdex::extents >::value ); @@ -123,7 +123,7 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_standard_layout< stdex::layout_left::template mapping< - stdex::extents<42, stdex::dynamic_extent, 73> + stdex::extents > >::value ); @@ -131,7 +131,7 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_standard_layout< stdex::layout_right::template mapping< - stdex::extents<42, stdex::dynamic_extent, 73> + stdex::extents > >::value ); @@ -139,7 +139,7 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_standard_layout< stdex::layout_right::template mapping< - stdex::extents + stdex::extents > >::value ); @@ -147,7 +147,7 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_standard_layout< stdex::layout_stride::template mapping< - stdex::extents<42, stdex::dynamic_extent, 73> + stdex::extents > >::value ); @@ -155,7 +155,7 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_standard_layout< stdex::layout_stride::template mapping< - stdex::extents<42, 27, 73> + stdex::extents > >::value ); @@ -163,7 +163,7 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_standard_layout< stdex::layout_stride::template mapping< - stdex::extents + stdex::extents > >::value ); @@ -174,7 +174,7 @@ MDSPAN_STATIC_TEST( #if 0 struct layout_stride_as_member_should_be_standard_layout : stdex::layout_stride::template mapping< - stdex::extents<1, 2, 3>> + stdex::extents> { int foo; }; @@ -195,20 +195,20 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_standard_layout< - stdex::mdspan> + stdex::mdspan> >::value ); MDSPAN_STATIC_TEST( std::is_standard_layout< - stdex::mdspan> + stdex::mdspan> >::value ); MDSPAN_STATIC_TEST( std::is_standard_layout< stdex::mdspan< - double, stdex::dextents<2>, + double, stdex::dextents, stdex::layout_left, stdex::default_accessor > >::value diff --git a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_trivially_copyable.cpp b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_trivially_copyable.cpp index 53daed6077..73ab426afa 100644 --- a/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_trivially_copyable.cpp +++ b/cpp/include/raft/thirdparty/mdspan/compilation_tests/ctest_trivially_copyable.cpp @@ -53,16 +53,16 @@ namespace stdex = std::experimental; // {{{1 MDSPAN_STATIC_TEST( - !std::is_base_of, stdex::detail::__partially_static_sizes<1, 2, 3>>::value + !std::is_base_of, stdex::detail::__partially_static_sizes>::value ); MDSPAN_STATIC_TEST( - !std::is_base_of, stdex::extents<1, 2, 3>>::value + !std::is_base_of, stdex::extents>::value ); MDSPAN_STATIC_TEST( std::is_trivially_copyable< - stdex::detail::__partially_static_sizes<1, 2, 3> + stdex::detail::__partially_static_sizes >::value ); @@ -74,43 +74,43 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_trivially_copyable< - stdex::extents<1, 2, stdex::dynamic_extent> + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_trivially_copyable< - stdex::extents + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_trivially_copyable< - stdex::extents + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_trivially_copyable< - stdex::extents + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_trivially_copyable< - stdex::extents<45, stdex::dynamic_extent, 1> + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_trivially_copyable< - stdex::extents<1, 2, 3> + stdex::extents >::value ); MDSPAN_STATIC_TEST( std::is_trivially_copyable< - stdex::extents<42> + stdex::extents >::value ); @@ -123,7 +123,7 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_trivially_copyable< stdex::layout_left::template mapping< - stdex::extents<42, stdex::dynamic_extent, 73> + stdex::extents > >::value ); @@ -131,7 +131,7 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_trivially_copyable< stdex::layout_right::template mapping< - stdex::extents<42, stdex::dynamic_extent, 73> + stdex::extents > >::value ); @@ -139,7 +139,7 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_trivially_copyable< stdex::layout_right::template mapping< - stdex::extents + stdex::extents > >::value ); @@ -147,7 +147,7 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_trivially_copyable< stdex::layout_stride::template mapping< - stdex::extents<42, stdex::dynamic_extent, 73> + stdex::extents > >::value ); @@ -155,7 +155,7 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_trivially_copyable< stdex::layout_stride::template mapping< - stdex::extents<42, 27, 73> + stdex::extents > >::value ); @@ -163,14 +163,14 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_trivially_copyable< stdex::layout_stride::template mapping< - stdex::extents + stdex::extents > >::value ); struct layout_stride_as_member_should_be_standard_layout : stdex::layout_stride::template mapping< - stdex::extents<1, 2, 3> + stdex::extents > { int foo; @@ -191,20 +191,20 @@ MDSPAN_STATIC_TEST( MDSPAN_STATIC_TEST( std::is_trivially_copyable< - stdex::mdspan> + stdex::mdspan> >::value ); MDSPAN_STATIC_TEST( std::is_trivially_copyable< - stdex::mdspan> + stdex::mdspan> >::value ); MDSPAN_STATIC_TEST( std::is_trivially_copyable< stdex::mdspan< - double, stdex::extents, + double, stdex::extents, stdex::layout_left, stdex::default_accessor > >::value diff --git a/cpp/include/raft/thirdparty/mdspan/examples/dot_product/dot_product.cpp b/cpp/include/raft/thirdparty/mdspan/examples/dot_product/dot_product.cpp index a10aeccfad..1cb01daa30 100644 --- a/cpp/include/raft/thirdparty/mdspan/examples/dot_product/dot_product.cpp +++ b/cpp/include/raft/thirdparty/mdspan/examples/dot_product/dot_product.cpp @@ -120,8 +120,8 @@ constexpr int cols = 3; int main() { { - using span_2d_dynamic = stdex::mdspan, stdex::layout_right>; - using span_2d_dynamic_left = stdex::mdspan, stdex::layout_left>; + using span_2d_dynamic = stdex::mdspan, stdex::layout_right>; + using span_2d_dynamic_left = stdex::mdspan, stdex::layout_left>; auto data_a = std::make_unique(rows * cols); auto data_b = std::make_unique(rows * cols); @@ -135,8 +135,8 @@ int main() { } { - using span_2d_10_10 = stdex::mdspan, stdex::layout_right>; - using span_2d_10_10_left = stdex::mdspan, stdex::layout_right>; + using span_2d_10_10 = stdex::mdspan, stdex::layout_right>; + using span_2d_10_10_left = stdex::mdspan, stdex::layout_right>; auto data_a = std::make_unique(100); auto data_b = std::make_unique(100); diff --git a/cpp/include/raft/thirdparty/mdspan/examples/tiled_layout/simple_tiled_layout.cpp b/cpp/include/raft/thirdparty/mdspan/examples/tiled_layout/simple_tiled_layout.cpp index d02be32066..b8740d5227 100644 --- a/cpp/include/raft/thirdparty/mdspan/examples/tiled_layout/simple_tiled_layout.cpp +++ b/cpp/include/raft/thirdparty/mdspan/examples/tiled_layout/simple_tiled_layout.cpp @@ -180,7 +180,7 @@ int main() { }; //---------------------------------------- // Just use dynamic extents for the purposes of demonstration - using extents_type = stdex::extents; + using extents_type = stdex::dextents; using tiled_mdspan = stdex::mdspan; using tiled_layout_type = typename SimpleTileLayout2D::template mapping; using row_major_mdspan = stdex::mdspan; diff --git a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/config.hpp b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/config.hpp index def30ac398..9d220eeaaf 100644 --- a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/config.hpp +++ b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/config.hpp @@ -104,6 +104,12 @@ static_assert(_MDSPAN_CPLUSPLUS >= MDSPAN_CXX_STD_14, "mdspan requires C++14 or # endif #endif +#ifndef _MDSPAN_HAS_HIP +# if defined(__HIPCC__) +# define _MDSPAN_HAS_HIP __HIPCC__ +# endif +#endif + #ifndef __has_cpp_attribute # define __has_cpp_attribute(x) 0 #endif @@ -273,7 +279,7 @@ static_assert(_MDSPAN_CPLUSPLUS >= MDSPAN_CXX_STD_14, "mdspan requires C++14 or // Corentins demo compiler for subscript chokes on empty [] call, // though I believe the proposal supports it? #ifdef MDSPAN_NO_EMPTY_BRACKET_OPERATOR -# define __MDSPAN_OP0(mds) mds.accessor().access(mds.data(),0) +# define __MDSPAN_OP0(mds) mds.accessor().access(mds.data_handle(),0) #else # define __MDSPAN_OP0(mds) mds[] #endif diff --git a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/default_accessor.hpp b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/default_accessor.hpp index 62a36f50bd..d26a7e6dad 100644 --- a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/default_accessor.hpp +++ b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/default_accessor.hpp @@ -56,27 +56,27 @@ struct default_accessor { using offset_policy = default_accessor; using element_type = ElementType; using reference = ElementType&; - using pointer = ElementType*; + using data_handle_type = ElementType*; - constexpr default_accessor() noexcept = default; + MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr default_accessor() noexcept = default; MDSPAN_TEMPLATE_REQUIRES( class OtherElementType, /* requires */ ( - _MDSPAN_TRAIT(is_convertible, typename default_accessor::element_type(*)[], element_type(*)[]) + _MDSPAN_TRAIT(is_convertible, OtherElementType(*)[], element_type(*)[]) ) ) MDSPAN_INLINE_FUNCTION constexpr default_accessor(default_accessor) noexcept {} MDSPAN_INLINE_FUNCTION - constexpr pointer - offset(pointer p, size_t i) const noexcept { + constexpr data_handle_type + offset(data_handle_type p, size_t i) const noexcept { return p + i; } MDSPAN_FORCE_INLINE_FUNCTION - constexpr reference access(pointer p, size_t i) const noexcept { + constexpr reference access(data_handle_type p, size_t i) const noexcept { return p[i]; } diff --git a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/extents.hpp b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/extents.hpp index a509ce2921..6be71b432c 100644 --- a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/extents.hpp +++ b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/extents.hpp @@ -97,18 +97,21 @@ struct __extents_tag { }; } // end namespace detail -template +template class extents #if !defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) : private detail::__no_unique_address_emulation< - detail::__partially_static_sizes_tagged> + detail::__partially_static_sizes_tagged> #endif { public: - using size_type = size_t; + using rank_type = size_t; + using index_type = ThisIndexType; + using size_type = make_unsigned_t; - using __storage_t = detail::__partially_static_sizes_tagged; +// internal typedefs which for technical reasons are public + using __storage_t = detail::__partially_static_sizes_tagged; #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) _MDSPAN_NO_UNIQUE_ADDRESS __storage_t __storage_; @@ -116,6 +119,7 @@ class extents using __base_t = detail::__no_unique_address_emulation<__storage_t>; #endif +// private members dealing with the way we internally store dynamic extents private: MDSPAN_FORCE_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 @@ -138,20 +142,20 @@ class extents template MDSPAN_FORCE_INLINE_FUNCTION static constexpr - size_type _static_extent_impl(size_t n, std::integer_sequence) noexcept { + index_type _static_extent_impl(size_t n, std::integer_sequence) noexcept { return _MDSPAN_FOLD_PLUS_RIGHT(((Idxs == n) ? Extents : 0), /* + ... + */ 0); } - template + template friend class extents; - template + template MDSPAN_INLINE_FUNCTION - constexpr bool _eq_impl(std::experimental::extents, false_type, index_sequence) const noexcept { return false; } - template + constexpr bool _eq_impl(std::experimental::extents, false_type, index_sequence) const noexcept { return false; } + template MDSPAN_INLINE_FUNCTION constexpr bool _eq_impl( - std::experimental::extents other, + std::experimental::extents other, true_type, index_sequence ) const noexcept { return _MDSPAN_FOLD_AND( @@ -159,13 +163,13 @@ class extents ); } - template + template MDSPAN_INLINE_FUNCTION - constexpr bool _not_eq_impl(std::experimental::extents, false_type, index_sequence) const noexcept { return true; } - template + constexpr bool _not_eq_impl(std::experimental::extents, false_type, index_sequence) const noexcept { return true; } + template MDSPAN_INLINE_FUNCTION constexpr bool _not_eq_impl( - std::experimental::extents other, + std::experimental::extents other, true_type, index_sequence ) const noexcept { return _MDSPAN_FOLD_OR( @@ -180,25 +184,28 @@ class extents { } #endif + +// public interface: public: + /* Defined above for use in the private code + using rank_type = size_t; + using index_type = ThisIndexType; + */ MDSPAN_INLINE_FUNCTION - static constexpr size_t rank() noexcept { return sizeof...(Extents); } + static constexpr rank_type rank() noexcept { return sizeof...(Extents); } MDSPAN_INLINE_FUNCTION - static constexpr size_t rank_dynamic() noexcept { return _MDSPAN_FOLD_PLUS_RIGHT((int(Extents == dynamic_extent)), /* + ... + */ 0); } + static constexpr rank_type rank_dynamic() noexcept { return _MDSPAN_FOLD_PLUS_RIGHT((rank_type(Extents == dynamic_extent)), /* + ... + */ 0); } //-------------------------------------------------------------------------------- // Constructors, Destructors, and Assignment + // Default constructor MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr extents() noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr extents(extents const&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr extents(extents&&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED _MDSPAN_CONSTEXPR_14_DEFAULTED extents& operator=(extents const&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED _MDSPAN_CONSTEXPR_14_DEFAULTED extents& operator=(extents&&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED ~extents() noexcept = default; + // Converting constructor MDSPAN_TEMPLATE_REQUIRES( - size_t... OtherExtents, + class OtherIndexType, size_t... OtherExtents, /* requires */ ( /* multi-stage check to protect from invalid pack expansion when sizes don't match? */ decltype(detail::_check_compatible_extents( @@ -209,8 +216,10 @@ class extents ) ) MDSPAN_INLINE_FUNCTION - MDSPAN_CONDITIONAL_EXPLICIT((((Extents != dynamic_extent) && (OtherExtents == dynamic_extent)) || ...)) - constexpr extents(const extents& __other) + MDSPAN_CONDITIONAL_EXPLICIT( + (((Extents != dynamic_extent) && (OtherExtents == dynamic_extent)) || ...) || + (std::numeric_limits::max() < std::numeric_limits::max())) + constexpr extents(const extents& __other) noexcept #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) : __storage_{ @@ -223,7 +232,14 @@ class extents #else }}) #endif - { } + { + /* TODO: precondition check + * other.extent(r) equals Er for each r for which Er is a static extent, and + * either + * - sizeof...(OtherExtents) is zero, or + * - other.extent(r) is a representable value of type index_type for all rank index r of other + */ + } #ifdef __NVCC__ MDSPAN_TEMPLATE_REQUIRES( @@ -231,19 +247,20 @@ class extents /* requires */ ( // TODO: check whether the other version works with newest NVCC, doesn't with 11.4 // NVCC seems to pick up rank_dynamic from the wrong extents type??? - _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, Integral, size_type) /* && ... */) + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, Integral, index_type) /* && ... */) && + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_nothrow_constructible, index_type, Integral) /* && ... */) && // NVCC chokes on the fold thingy here so wrote the workaround - && ((sizeof...(Integral) == detail::_count_dynamic_extents::val) || - (sizeof...(Integral) == sizeof...(Extents))) + ((sizeof...(Integral) == detail::_count_dynamic_extents::val) || + (sizeof...(Integral) == sizeof...(Extents))) ) ) #else MDSPAN_TEMPLATE_REQUIRES( class... Integral, /* requires */ ( - _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, Integral, size_type) /* && ... */) - && ((sizeof...(Integral) == rank_dynamic()) || - (sizeof...(Integral) == rank())) + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, Integral, index_type) /* && ... */) && + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_nothrow_constructible, index_type, Integral) /* && ... */) && + ((sizeof...(Integral) == rank_dynamic()) || (sizeof...(Integral) == rank())) ) ) #endif @@ -257,39 +274,48 @@ class extents std::conditional_t(), - static_cast(exts)... + static_cast(exts)... #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) } #else }}) #endif - { } + { + /* TODO: precondition check + * If sizeof...(IndexTypes) != rank_dynamic() is true, exts_arr[r] equals Er for each r for which Er is a static extent, and + * either + * - sizeof...(exts) == 0 is true, or + * - each element of exts is nonnegative and is a representable value of type index_type. + */ + } // TODO: check whether this works with newest NVCC, doesn't with 11.4 #ifdef __NVCC__ // NVCC seems to pick up rank_dynamic from the wrong extents type??? // NVCC chokes on the fold thingy here so wrote the workaround MDSPAN_TEMPLATE_REQUIRES( - class SizeType, size_t N, + class IndexType, size_t N, /* requires */ ( - _MDSPAN_TRAIT(is_convertible, SizeType, size_type) - && ((N == detail::_count_dynamic_extents::val) || - (N == sizeof...(Extents))) + _MDSPAN_TRAIT(is_convertible, IndexType, index_type) && + _MDSPAN_TRAIT(is_nothrow_constructible, index_type, IndexType) && + ((N == detail::_count_dynamic_extents::val) || + (N == sizeof...(Extents))) ) ) #else MDSPAN_TEMPLATE_REQUIRES( - class SizeType, size_t N, + class IndexType, size_t N, /* requires */ ( - _MDSPAN_TRAIT(is_convertible, SizeType, size_type) - && (N == rank() || N == rank_dynamic()) + _MDSPAN_TRAIT(is_convertible, IndexType, index_type) && + _MDSPAN_TRAIT(is_nothrow_constructible, index_type, IndexType) && + (N == rank() || N == rank_dynamic()) ) ) #endif MDSPAN_CONDITIONAL_EXPLICIT(N != rank_dynamic()) MDSPAN_INLINE_FUNCTION constexpr - extents(std::array const& exts) noexcept + extents(std::array const& exts) noexcept #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) : __storage_{ #else @@ -298,13 +324,72 @@ class extents std::conditional_t, detail::__construct_psa_from_all_exts_array_tag_t>(), - std::array{exts} + std::array{exts} #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) } #else }}) #endif - { } + { + /* TODO: precondition check + * If N != rank_dynamic() is true, exts[r] equals Er for each r for which Er is a static extent, and + * either + * - N is zero, or + * - exts[r] is nonnegative and is a representable value of type index_type for all rank index r + */ + } + +#ifdef __cpp_lib_span + // TODO: check whether the below works with newest NVCC, doesn't with 11.4 +#ifdef __NVCC__ + // NVCC seems to pick up rank_dynamic from the wrong extents type??? + // NVCC chokes on the fold thingy here so wrote the workaround + MDSPAN_TEMPLATE_REQUIRES( + class IndexType, size_t N, + /* requires */ ( + _MDSPAN_TRAIT(is_convertible, IndexType, index_type) && + _MDSPAN_TRAIT(is_nothrow_constructible, index_type, IndexType) && + ((N == detail::_count_dynamic_extents::val) || + (N == sizeof...(Extents))) + ) + ) +#else + MDSPAN_TEMPLATE_REQUIRES( + class IndexType, size_t N, + /* requires */ ( + _MDSPAN_TRAIT(is_convertible, IndexType, index_type) && + _MDSPAN_TRAIT(is_nothrow_constructible, index_type, IndexType) && + (N == rank() || N == rank_dynamic()) + ) + ) +#endif + MDSPAN_CONDITIONAL_EXPLICIT(N != rank_dynamic()) + MDSPAN_INLINE_FUNCTION + constexpr + extents(std::span exts) noexcept +#if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) + : __storage_{ +#else + : __base_t(__base_t{typename __base_t::__stored_type{ +#endif + std::conditional_t, + detail::__construct_psa_from_all_exts_array_tag_t>(), + exts +#if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) + } +#else + }}) +#endif + { + /* TODO: precondition check + * If N != rank_dynamic() is true, exts[r] equals Er for each r for which Er is a static extent, and + * either + * - N is zero, or + * - exts[r] is nonnegative and is a representable value of type index_type for all rank index r + */ + } +#endif // Need this constructor for some submdspan implementation stuff // for the layout_stride case where I use an extents object for strides @@ -328,31 +413,35 @@ class extents MDSPAN_INLINE_FUNCTION static constexpr - size_type static_extent(size_t n) noexcept { + size_t static_extent(size_t n) noexcept { + // Can't do assert here since that breaks true constexpr ness + // assert(n{}); } MDSPAN_INLINE_FUNCTION constexpr - size_type extent(size_t n) const noexcept { + index_type extent(size_t n) const noexcept { + // Can't do assert here since that breaks true constexpr ness + // assert(n + template MDSPAN_INLINE_FUNCTION - friend constexpr bool operator==(extents const& lhs, extents const& rhs) noexcept { + friend constexpr bool operator==(extents const& lhs, extents const& rhs) noexcept { return lhs._eq_impl( rhs, std::integral_constant{}, make_index_sequence{} ); } -#ifndef MDSPAN_HAS_CXX_20 - template +#if !(MDSPAN_HAS_CXX_20) + template MDSPAN_INLINE_FUNCTION - friend constexpr bool operator!=(extents const& lhs, extents const& rhs) noexcept { + friend constexpr bool operator!=(extents const& lhs, extents const& rhs) noexcept { return lhs._not_eq_impl( rhs, std::integral_constant{}, make_index_sequence{} @@ -360,10 +449,12 @@ class extents } #endif + // End of public interface + public: // (but not really) MDSPAN_INLINE_FUNCTION static constexpr - extents __make_extents_impl(detail::__partially_static_sizes&& __bs) noexcept { + extents __make_extents_impl(detail::__partially_static_sizes&& __bs) noexcept { // This effectively amounts to a sideways cast that can be done in a constexpr // context, but we have to do it to handle the case where the extents and the // strides could accidentally end up with the same types in their hierarchies @@ -382,14 +473,14 @@ class extents template MDSPAN_FORCE_INLINE_FUNCTION constexpr - size_type __extent() const noexcept { + index_type __extent() const noexcept { return __storage().template __get_n(); } template MDSPAN_INLINE_FUNCTION static constexpr - size_type __static_extent() noexcept { + index_type __static_extent() noexcept { return __storage_t::template __get_static_n(); } @@ -397,29 +488,29 @@ class extents namespace detail { -template > +template > struct __make_dextents; -template -struct __make_dextents> { - using type = typename __make_dextents>::type; +template +struct __make_dextents> { + using type = typename __make_dextents>::type; }; -template -struct __make_dextents<0, ::std::experimental::extents> { - using type = ::std::experimental::extents; +template +struct __make_dextents> { + using type = ::std::experimental::extents; }; } // end namespace detail -template -using dextents = typename detail::__make_dextents::type; +template +using dextents = typename detail::__make_dextents::type; #if defined(_MDSPAN_USE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION) -template -extents(SizeTypes...) - -> extents()...>; +template +extents(IndexTypes...) + -> extents()...>; #endif namespace detail { @@ -427,8 +518,8 @@ namespace detail { template struct __is_extents : ::std::false_type {}; -template -struct __is_extents<::std::experimental::extents> : ::std::true_type {}; +template +struct __is_extents<::std::experimental::extents> : ::std::true_type {}; template static constexpr bool __is_extents_v = __is_extents::value; @@ -437,9 +528,11 @@ static constexpr bool __is_extents_v = __is_extents::value; template struct __extents_to_partially_static_sizes; -template -struct __extents_to_partially_static_sizes<::std::experimental::extents> { - using type = detail::__partially_static_sizes; +template +struct __extents_to_partially_static_sizes<::std::experimental::extents> { + using type = detail::__partially_static_sizes< + typename ::std::experimental::extents::index_type, size_t, + ExtentsPack...>; }; template diff --git a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/layout_left.hpp b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/layout_left.hpp index 8498818e6d..8cd2e14fb8 100644 --- a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/layout_left.hpp +++ b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/layout_left.hpp @@ -51,15 +51,18 @@ namespace std { namespace experimental { //============================================================================== -struct layout_right; -struct layout_stride; -struct layout_left { - template - class mapping { +template +class layout_left::mapping { + public: + using extents_type = Extents; + using index_type = typename extents_type::index_type; + using size_type = typename extents_type::size_type; + using rank_type = typename extents_type::rank_type; + using layout_type = layout_left; private: - static_assert(detail::__is_extents_v, "std::experimental::layout_left::mapping must be instantiated with a specialization of std::experimental::extents."); + static_assert(detail::__is_extents_v, "std::experimental::layout_left::mapping must be instantiated with a specialization of std::experimental::extents."); template friend class mapping; @@ -69,19 +72,19 @@ struct layout_left { struct __rank_count {}; template - constexpr size_t __compute_offset( + constexpr index_type __compute_offset( __rank_count, const I& i, Indices... idx) const { return __compute_offset(__rank_count(), idx...) * __extents.template __extent() + i; } template - constexpr size_t __compute_offset( - __rank_count, const I& i) const { + constexpr index_type __compute_offset( + __rank_count, const I& i) const { return i; } - constexpr size_t __compute_offset(__rank_count<0,0>) const { return 0; } + constexpr index_type __compute_offset(__rank_count<0,0>) const { return 0; } public: @@ -89,63 +92,64 @@ struct layout_left { MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mapping() noexcept = default; MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mapping(mapping const&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mapping(mapping&&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED _MDSPAN_CONSTEXPR_14_DEFAULTED mapping& operator=(mapping const&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED _MDSPAN_CONSTEXPR_14_DEFAULTED mapping& operator=(mapping&&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED ~mapping() noexcept = default; - - using layout_type = layout_left; - using extents_type = Extents; - using size_type = typename Extents::size_type; - - constexpr mapping(Extents const& __exts) noexcept + constexpr mapping(extents_type const& __exts) noexcept :__extents(__exts) { } MDSPAN_TEMPLATE_REQUIRES( class OtherExtents, /* requires */ ( - _MDSPAN_TRAIT(is_constructible, Extents, OtherExtents) + _MDSPAN_TRAIT(is_constructible, extents_type, OtherExtents) ) ) - MDSPAN_CONDITIONAL_EXPLICIT((!is_convertible::value)) // needs two () due to comma + MDSPAN_CONDITIONAL_EXPLICIT((!is_convertible::value)) // needs two () due to comma MDSPAN_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 mapping(mapping const& other) noexcept // NOLINT(google-explicit-constructor) :__extents(other.extents()) - { } + { + /* + * TODO: check precondition + * other.required_span_size() is a representable value of type index_type + */ + } MDSPAN_TEMPLATE_REQUIRES( - class OtherMapping, + class OtherExtents, /* requires */ ( - _MDSPAN_TRAIT(is_constructible, Extents, typename OtherMapping::extents_type) && - _MDSPAN_TRAIT(is_same, typename OtherMapping::layout_type, layout_right) && - _MDSPAN_TRAIT(is_same, typename OtherMapping::layout_type::template mapping, OtherMapping) && - (Extents::rank() <= 1) + _MDSPAN_TRAIT(is_constructible, extents_type, OtherExtents) && + (extents_type::rank() <= 1) ) ) - MDSPAN_CONDITIONAL_EXPLICIT((!is_convertible::value)) // needs two () due to comma + MDSPAN_CONDITIONAL_EXPLICIT((!is_convertible::value)) // needs two () due to comma MDSPAN_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 - mapping(OtherMapping const& other) noexcept // NOLINT(google-explicit-constructor) + mapping(layout_right::mapping const& other) noexcept // NOLINT(google-explicit-constructor) :__extents(other.extents()) - { } + { + /* + * TODO: check precondition + * other.required_span_size() is a representable value of type index_type + */ + } MDSPAN_TEMPLATE_REQUIRES( - class OtherMapping, + class OtherExtents, /* requires */ ( - _MDSPAN_TRAIT(is_constructible, Extents, typename OtherMapping::extents_type) && - _MDSPAN_TRAIT(is_same, typename OtherMapping::layout_type, layout_stride) && - _MDSPAN_TRAIT(is_same, typename OtherMapping::layout_type::template mapping, OtherMapping) + _MDSPAN_TRAIT(is_constructible, extents_type, OtherExtents) ) ) - MDSPAN_CONDITIONAL_EXPLICIT((Extents::rank()!=0)) + MDSPAN_CONDITIONAL_EXPLICIT((extents_type::rank() > 0)) MDSPAN_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 - mapping(OtherMapping const& other) // NOLINT(google-explicit-constructor) + mapping(layout_stride::mapping const& other) // NOLINT(google-explicit-constructor) :__extents(other.extents()) { + /* + * TODO: check precondition + * other.required_span_size() is a representable value of type index_type + */ #ifndef __CUDA_ARCH__ size_t stride = 1; - for(size_type r=0; r<__extents.rank(); r++) { + for(rank_type r=0; r<__extents.rank(); r++) { if(stride != other.stride(r)) throw std::runtime_error("Assigning layout_stride to layout_left with invalid strides."); stride *= __extents.extent(r); @@ -153,37 +157,53 @@ struct layout_left { #endif } - //-------------------------------------------------------------------------------- - - template - constexpr size_type operator()(Indices... idxs) const noexcept { - return __compute_offset(__rank_count<0, Extents::rank()>(), idxs...); - } + MDSPAN_INLINE_FUNCTION_DEFAULTED _MDSPAN_CONSTEXPR_14_DEFAULTED mapping& operator=(mapping const&) noexcept = default; - constexpr Extents extents() const noexcept { + MDSPAN_INLINE_FUNCTION + constexpr const extents_type& extents() const noexcept { return __extents; } - constexpr size_type stride(size_t i) const noexcept { - size_type value = 1; - for(size_type r=0; r(), idxs...); } + + MDSPAN_INLINE_FUNCTION static constexpr bool is_always_unique() noexcept { return true; } - MDSPAN_INLINE_FUNCTION static constexpr bool is_always_contiguous() noexcept { return true; } + MDSPAN_INLINE_FUNCTION static constexpr bool is_always_exhaustive() noexcept { return true; } MDSPAN_INLINE_FUNCTION static constexpr bool is_always_strided() noexcept { return true; } MDSPAN_INLINE_FUNCTION constexpr bool is_unique() const noexcept { return true; } - MDSPAN_INLINE_FUNCTION constexpr bool is_contiguous() const noexcept { return true; } + MDSPAN_INLINE_FUNCTION constexpr bool is_exhaustive() const noexcept { return true; } MDSPAN_INLINE_FUNCTION constexpr bool is_strided() const noexcept { return true; } + MDSPAN_INLINE_FUNCTION + constexpr index_type stride(rank_type i) const noexcept { + index_type value = 1; + for(rank_type r=0; r MDSPAN_INLINE_FUNCTION friend constexpr bool operator==(mapping const& lhs, mapping const& rhs) noexcept { @@ -200,20 +220,21 @@ struct layout_left { #endif // Not really public, but currently needed to implement fully constexpr useable submdspan: - template - constexpr size_type __get_stride(std::experimental::extents,integer_sequence) const { + template + constexpr index_type __get_stride(std::experimental::extents,integer_sequence) const { return _MDSPAN_FOLD_TIMES_RIGHT((Idx():1),1); } template - constexpr size_type __stride() const noexcept { + constexpr index_type __stride() const noexcept { return __get_stride(__extents, make_index_sequence()); } private: - _MDSPAN_NO_UNIQUE_ADDRESS Extents __extents{}; + _MDSPAN_NO_UNIQUE_ADDRESS extents_type __extents{}; - }; }; + } // end namespace experimental } // end namespace std + diff --git a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/layout_right.hpp b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/layout_right.hpp index 933ebfa914..118f3632c0 100644 --- a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/layout_right.hpp +++ b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/layout_right.hpp @@ -47,20 +47,23 @@ #include "trait_backports.hpp" #include "extents.hpp" #include +#include "layout_stride.hpp" namespace std { namespace experimental { //============================================================================== -struct layout_left; -struct layout_stride; - -struct layout_right { - template - class mapping { +template +class layout_right::mapping { + public: + using extents_type = Extents; + using index_type = typename extents_type::index_type; + using size_type = typename extents_type::size_type; + using rank_type = typename extents_type::rank_type; + using layout_type = layout_right; private: - static_assert(detail::__is_extents_v, "std::experimental::layout_right::mapping must be instantiated with a specialization of std::experimental::extents."); + static_assert(detail::__is_extents_v, "std::experimental::layout_right::mapping must be instantiated with a specialization of std::experimental::extents."); template friend class mapping; @@ -70,22 +73,22 @@ struct layout_right { struct __rank_count {}; template - constexpr size_t __compute_offset( - size_t offset, __rank_count, const I& i, Indices... idx) const { + constexpr index_type __compute_offset( + index_type offset, __rank_count, const I& i, Indices... idx) const { return __compute_offset(offset * __extents.template __extent() + i,__rank_count(), idx...); } template - constexpr size_t __compute_offset( - __rank_count<0,Extents::rank()>, const I& i, Indices... idx) const { - return __compute_offset(static_cast(i),__rank_count<1,Extents::rank()>(),idx...); + constexpr index_type __compute_offset( + __rank_count<0,extents_type::rank()>, const I& i, Indices... idx) const { + return __compute_offset(i,__rank_count<1,extents_type::rank()>(),idx...); } - constexpr size_t __compute_offset(size_t offset, __rank_count) const { - return offset; + constexpr index_type __compute_offset(size_t offset, __rank_count) const { + return static_cast(offset); } - constexpr size_t __compute_offset(__rank_count<0,0>) const { return 0; } + constexpr index_type __compute_offset(__rank_count<0,0>) const { return 0; } public: @@ -93,62 +96,64 @@ struct layout_right { MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mapping() noexcept = default; MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mapping(mapping const&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mapping(mapping&&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED _MDSPAN_CONSTEXPR_14_DEFAULTED mapping& operator=(mapping const&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED _MDSPAN_CONSTEXPR_14_DEFAULTED mapping& operator=(mapping&&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED ~mapping() noexcept = default; - - - using layout_type = layout_right; - using extents_type = Extents; - using size_type = typename Extents::size_type; - constexpr mapping(Extents const& __exts) noexcept + constexpr mapping(extents_type const& __exts) noexcept :__extents(__exts) { } MDSPAN_TEMPLATE_REQUIRES( class OtherExtents, /* requires */ ( - _MDSPAN_TRAIT(is_constructible, Extents, OtherExtents) + _MDSPAN_TRAIT(is_constructible, extents_type, OtherExtents) ) ) - MDSPAN_CONDITIONAL_EXPLICIT((!is_convertible::value)) // needs two () due to comma + MDSPAN_CONDITIONAL_EXPLICIT((!is_convertible::value)) // needs two () due to comma MDSPAN_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 mapping(mapping const& other) noexcept // NOLINT(google-explicit-constructor) :__extents(other.extents()) - { } + { + /* + * TODO: check precondition + * other.required_span_size() is a representable value of type index_type + */ + } MDSPAN_TEMPLATE_REQUIRES( - class OtherMapping, + class OtherExtents, /* requires */ ( - _MDSPAN_TRAIT(is_constructible, Extents, typename OtherMapping::extents_type) && - _MDSPAN_TRAIT(is_same, typename OtherMapping::layout_type, layout_left) && - _MDSPAN_TRAIT(is_same, typename OtherMapping::layout_type::template mapping, OtherMapping) && - (Extents::rank() <= 1) + _MDSPAN_TRAIT(is_constructible, extents_type, OtherExtents) && + (extents_type::rank() <= 1) ) ) - MDSPAN_CONDITIONAL_EXPLICIT((!is_convertible::value)) // needs two () due to comma + MDSPAN_CONDITIONAL_EXPLICIT((!is_convertible::value)) // needs two () due to comma MDSPAN_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 - mapping(OtherMapping const& other) noexcept // NOLINT(google-explicit-constructor) + mapping(layout_left::mapping const& other) noexcept // NOLINT(google-explicit-constructor) :__extents(other.extents()) - { } + { + /* + * TODO: check precondition + * other.required_span_size() is a representable value of type index_type + */ + } + MDSPAN_TEMPLATE_REQUIRES( - class OtherMapping, + class OtherExtents, /* requires */ ( - _MDSPAN_TRAIT(is_constructible, Extents, typename OtherMapping::extents_type) && - _MDSPAN_TRAIT(is_same, typename OtherMapping::layout_type, layout_stride) && - _MDSPAN_TRAIT(is_same, typename OtherMapping::layout_type::template mapping, OtherMapping) + _MDSPAN_TRAIT(is_constructible, extents_type, OtherExtents) ) ) - MDSPAN_CONDITIONAL_EXPLICIT((Extents::rank()!=0)) + MDSPAN_CONDITIONAL_EXPLICIT((extents_type::rank() > 0)) MDSPAN_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 - mapping(OtherMapping const& other) // NOLINT(google-explicit-constructor) + mapping(layout_stride::mapping const& other) // NOLINT(google-explicit-constructor) :__extents(other.extents()) { + /* + * TODO: check precondition + * other.required_span_size() is a representable value of type index_type + */ #ifndef __CUDA_ARCH__ size_t stride = 1; - for(size_type r=__extents.rank(); r>0; r--) { + for(rank_type r=__extents.rank(); r>0; r--) { if(stride != other.stride(r-1)) throw std::runtime_error("Assigning layout_stride to layout_right with invalid strides."); stride *= __extents.extent(r-1); @@ -156,36 +161,50 @@ struct layout_right { #endif } - //-------------------------------------------------------------------------------- - - template - constexpr size_type operator()(Indices... idxs) const noexcept { - return __compute_offset(__rank_count<0, Extents::rank()>(), idxs...); - } + MDSPAN_INLINE_FUNCTION_DEFAULTED _MDSPAN_CONSTEXPR_14_DEFAULTED mapping& operator=(mapping const&) noexcept = default; - constexpr Extents extents() const noexcept { + MDSPAN_INLINE_FUNCTION + constexpr const extents_type& extents() const noexcept { return __extents; } - constexpr size_type stride(size_t i) const noexcept { - size_type value = 1; - for(size_type r=Extents::rank()-1; r>i; r--) value*=__extents.extent(r); + MDSPAN_INLINE_FUNCTION + constexpr index_type required_span_size() const noexcept { + index_type value = 1; + for(rank_type r=0; r != extents_type::rank(); ++r) value*=__extents.extent(r); return value; } - constexpr size_type required_span_size() const noexcept { - size_type value = 1; - for(size_type r=0; r(), idxs...); } MDSPAN_INLINE_FUNCTION static constexpr bool is_always_unique() noexcept { return true; } - MDSPAN_INLINE_FUNCTION static constexpr bool is_always_contiguous() noexcept { return true; } + MDSPAN_INLINE_FUNCTION static constexpr bool is_always_exhaustive() noexcept { return true; } MDSPAN_INLINE_FUNCTION static constexpr bool is_always_strided() noexcept { return true; } MDSPAN_INLINE_FUNCTION constexpr bool is_unique() const noexcept { return true; } - MDSPAN_INLINE_FUNCTION constexpr bool is_contiguous() const noexcept { return true; } + MDSPAN_INLINE_FUNCTION constexpr bool is_exhaustive() const noexcept { return true; } MDSPAN_INLINE_FUNCTION constexpr bool is_strided() const noexcept { return true; } + MDSPAN_INLINE_FUNCTION + constexpr index_type stride(rank_type i) const noexcept { + index_type value = 1; + for(rank_type r=extents_type::rank()-1; r>i; r--) value*=__extents.extent(r); + return value; + } + template MDSPAN_INLINE_FUNCTION friend constexpr bool operator==(mapping const& lhs, mapping const& rhs) noexcept { @@ -202,20 +221,20 @@ struct layout_right { #endif // Not really public, but currently needed to implement fully constexpr useable submdspan: - template - constexpr size_type __get_stride(std::experimental::extents,integer_sequence) const { + template + constexpr index_type __get_stride(std::experimental::extents,integer_sequence) const { return _MDSPAN_FOLD_TIMES_RIGHT((Idx>N? __extents.template __extent():1),1); } template - constexpr size_type __stride() const noexcept { + constexpr index_type __stride() const noexcept { return __get_stride(__extents, make_index_sequence()); } private: - _MDSPAN_NO_UNIQUE_ADDRESS Extents __extents{}; + _MDSPAN_NO_UNIQUE_ADDRESS extents_type __extents{}; - }; }; } // end namespace experimental } // end namespace std + diff --git a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/layout_stride.hpp b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/layout_stride.hpp index e41cb84882..c04c0c45ae 100644 --- a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/layout_stride.hpp +++ b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/layout_stride.hpp @@ -56,10 +56,41 @@ #include #include #include +#if _MDSPAN_USE_CONCEPTS && MDSPAN_HAS_CXX_20 +#include +#endif namespace std { namespace experimental { +struct layout_left { + template + class mapping; +}; +struct layout_right { + template + class mapping; +}; + +namespace detail { + template + constexpr bool __is_mapping_of = + is_same, Mapping>::value; + +#if _MDSPAN_USE_CONCEPTS && MDSPAN_HAS_CXX_20 + template + concept __layout_mapping_alike = requires { + requires __is_extents::value; + { M::is_always_strided() } -> same_as; + { M::is_always_exhaustive() } -> same_as; + { M::is_always_unique() } -> same_as; + bool_constant::value; + bool_constant::value; + bool_constant::value; + }; +#endif +} // namespace detail + struct layout_stride { template class mapping @@ -67,25 +98,27 @@ struct layout_stride { : private detail::__no_unique_address_emulation< detail::__compressed_pair< Extents, - ::std::experimental::dextents + std::array > > #endif { public: + using extents_type = Extents; + using index_type = typename extents_type::index_type; + using size_type = typename extents_type::size_type; + using rank_type = typename extents_type::rank_type; + using layout_type = layout_stride; + // This could be a `requires`, but I think it's better and clearer as a `static_assert`. static_assert(detail::__is_extents_v, "std::experimental::layout_stride::mapping must be instantiated with a specialization of std::experimental::extents."); - using size_type = typename Extents::size_type; - using extents_type = Extents; - - using layout_type = layout_stride; private: //---------------------------------------------------------------------------- - using __strides_storage_t = ::std::experimental::dextents; + using __strides_storage_t = array;//::std::experimental::dextents; using __member_pair_t = detail::__compressed_pair; #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) @@ -128,18 +161,18 @@ struct layout_stride { template MDSPAN_INLINE_FUNCTION static constexpr bool _eq_impl(mapping const& self, mapping const& other) noexcept { - return _MDSPAN_FOLD_AND((self.template __stride() == other.template __stride()) /* && ... */); + return _MDSPAN_FOLD_AND((self.stride(Idxs) == other.stride(Idxs)) /* && ... */); } template MDSPAN_INLINE_FUNCTION static constexpr bool _not_eq_impl(mapping const& self, mapping const& other) noexcept { - return _MDSPAN_FOLD_OR((self.template __stride() != other.template __stride()) /* || ... */); + return _MDSPAN_FOLD_OR((self.stride(Idxs) != other.stride(Idxs)) /* || ... */); } template MDSPAN_FORCE_INLINE_FUNCTION static constexpr size_t _call_op_impl(mapping const& self, Integral... idxs) noexcept { - return _MDSPAN_FOLD_PLUS_RIGHT((idxs * self.template __stride()), /* + ... + */ 0); + return _MDSPAN_FOLD_PLUS_RIGHT((idxs * self.stride(Idxs)), /* + ... + */ 0); } MDSPAN_INLINE_FUNCTION @@ -149,9 +182,37 @@ struct layout_stride { } template - static constexpr __strides_storage_t fill_strides(const OtherMapping& map) { - return __strides_storage_t(map.stride(Idxs)...); + MDSPAN_INLINE_FUNCTION + static constexpr const __strides_storage_t fill_strides(const OtherMapping& map) { + return __strides_storage_t{static_cast(map.stride(Idxs))...}; + } + + MDSPAN_INLINE_FUNCTION + static constexpr const __strides_storage_t& fill_strides(const __strides_storage_t& s) { + return s; + } + + template + MDSPAN_INLINE_FUNCTION + static constexpr const __strides_storage_t fill_strides(const array& s) { + return __strides_storage_t{static_cast(s[Idxs])...}; } + + MDSPAN_INLINE_FUNCTION + static constexpr const __strides_storage_t fill_strides( + detail::__extents_to_partially_static_sizes_t< + ::std::experimental::dextents>&& s) { + return __strides_storage_t{static_cast(s.template __get_n())...}; + } + + template + MDSPAN_INLINE_FUNCTION + static constexpr size_t __return_zero() { return 0; } + + template + MDSPAN_INLINE_FUNCTION + static constexpr typename Mapping::index_type + __OFFSET(const Mapping& m) { return m(__return_zero()...); } }; // Can't use defaulted parameter in the __deduction_workaround template because of a bug in MSVC warning C4348. @@ -168,28 +229,13 @@ struct layout_stride { mapping(__base_t&& __b) : __base_t(::std::move(__b)) {} #endif - //---------------------------------------------------------------------------- - - public: // (but not really) - - template - MDSPAN_INLINE_FUNCTION - constexpr size_t __stride() const noexcept { - return __strides_storage().extent(R); - } - - template - MDSPAN_INLINE_FUNCTION - constexpr array< size_t, Extents::rank() > __strides(std::index_sequence) const noexcept { - return {__strides_storage().template __extent()...}; - } - + public: // but not really MDSPAN_INLINE_FUNCTION static constexpr mapping __make_mapping( detail::__extents_to_partially_static_sizes_t&& __exts, detail::__extents_to_partially_static_sizes_t< - ::std::experimental::dextents>&& __strs + ::std::experimental::dextents>&& __strs ) noexcept { // call the private constructor we created for this purpose return mapping( @@ -198,13 +244,15 @@ struct layout_stride { #endif __member_pair_t( extents_type::__make_extents_impl(::std::move(__exts)), - __strides_storage_t{::std::move(__strs)} + __strides_storage_t{__impl::fill_strides(::std::move(__strs))} ) #if !defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) } #endif ); } + //---------------------------------------------------------------------------- + public: @@ -212,66 +260,110 @@ struct layout_stride { MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mapping() noexcept = default; MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mapping(mapping const&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mapping(mapping&&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED _MDSPAN_CONSTEXPR_14_DEFAULTED - mapping& operator=(mapping const&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED _MDSPAN_CONSTEXPR_14_DEFAULTED - mapping& operator=(mapping&&) noexcept = default; - MDSPAN_INLINE_FUNCTION_DEFAULTED ~mapping() noexcept = default; - template + MDSPAN_TEMPLATE_REQUIRES( + class IntegralTypes, + /* requires */ ( + // MSVC 19.32 does not like using index_type here, requires the typename Extents::index_type + // error C2641: cannot deduce template arguments for 'std::experimental::layout_stride::mapping' + _MDSPAN_TRAIT(is_convertible, const remove_const_t&, typename Extents::index_type) && + _MDSPAN_TRAIT(is_nothrow_constructible, typename Extents::index_type, const remove_const_t&) + ) + ) MDSPAN_INLINE_FUNCTION constexpr mapping( - Extents const& e, - ::std::array const& strides + extents_type const& e, + array const& s ) noexcept #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) : __members{ #else : __base_t(__base_t{__member_pair_t( #endif - e, __strides_storage_t{strides} + e, __strides_storage_t(__impl::fill_strides(s)) #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) } #else )}) #endif - { } + { + /* + * TODO: check preconditions + * - s[i] > 0 is true for all i in the range [0, rank_ ). + * - REQUIRED-SPAN-SIZE(e, s) is a representable value of type index_type ([basic.fundamental]). + * - If rank_ is greater than 0, then there exists a permutation P of the integers in the + * range [0, rank_), such that s[ pi ] >= s[ pi − 1 ] * e.extent( pi − 1 ) is true for + * all i in the range [1, rank_ ), where pi is the ith element of P. + */ + } - template - MDSPAN_CONDITIONAL_EXPLICIT((!is_convertible::value)) // needs two () due to comma +#ifdef __cpp_lib_span + MDSPAN_TEMPLATE_REQUIRES( + class IntegralTypes, + /* requires */ ( + // MSVC 19.32 does not like using index_type here, requires the typename Extents::index_type + // error C2641: cannot deduce template arguments for 'std::experimental::layout_stride::mapping' + _MDSPAN_TRAIT(is_convertible, const remove_const_t&, typename Extents::index_type) && + _MDSPAN_TRAIT(is_nothrow_constructible, typename Extents::index_type, const remove_const_t&) + ) + ) MDSPAN_INLINE_FUNCTION constexpr mapping( - const mapping& rhs + extents_type const& e, + span const& s ) noexcept #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) : __members{ #else : __base_t(__base_t{__member_pair_t( #endif - rhs.extents(), __strides_storage_t{rhs.__strides_storage()} + e, __strides_storage_t(__impl::fill_strides(s)) #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) } #else )}) #endif - { } - + { + /* + * TODO: check preconditions + * - s[i] > 0 is true for all i in the range [0, rank_ ). + * - REQUIRED-SPAN-SIZE(e, s) is a representable value of type index_type ([basic.fundamental]). + * - If rank_ is greater than 0, then there exists a permutation P of the integers in the + * range [0, rank_), such that s[ pi ] >= s[ pi − 1 ] * e.extent( pi − 1 ) is true for + * all i in the range [1, rank_ ), where pi is the ith element of P. + */ + } +#endif // __cpp_lib_span +#if !(_MDSPAN_USE_CONCEPTS && MDSPAN_HAS_CXX_20) MDSPAN_TEMPLATE_REQUIRES( - class OtherMapping, + class StridedLayoutMapping, /* requires */ ( - _MDSPAN_TRAIT(is_constructible, Extents, typename OtherMapping::extents_type) && - _MDSPAN_TRAIT(is_same, typename OtherMapping::layout_type::template mapping, OtherMapping) && - OtherMapping::is_always_unique() && - OtherMapping::is_always_strided() + _MDSPAN_TRAIT(is_constructible, extents_type, typename StridedLayoutMapping::extents_type) && + detail::__is_mapping_of && + StridedLayoutMapping::is_always_unique() && + StridedLayoutMapping::is_always_strided() ) ) - MDSPAN_CONDITIONAL_EXPLICIT((!is_convertible::value)) // needs two () due to comma +#else + template + requires( + detail::__layout_mapping_alike && + _MDSPAN_TRAIT(is_constructible, extents_type, typename StridedLayoutMapping::extents_type) && + StridedLayoutMapping::is_always_unique() && + StridedLayoutMapping::is_always_strided() + ) +#endif + MDSPAN_CONDITIONAL_EXPLICIT( + (!is_convertible::value) && + (detail::__is_mapping_of || + detail::__is_mapping_of || + detail::__is_mapping_of) + ) // needs two () due to comma MDSPAN_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 - mapping(OtherMapping const& other) noexcept // NOLINT(google-explicit-constructor) + mapping(StridedLayoutMapping const& other) noexcept // NOLINT(google-explicit-constructor) #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) : __members{ #else @@ -283,11 +375,21 @@ struct layout_stride { #else )}) #endif - {} + { + /* + * TODO: check preconditions + * - other.stride(i) > 0 is true for all i in the range [0, rank_ ). + * - other.required_span_size() is a representable value of type index_type ([basic.fundamental]). + * - OFFSET(other) == 0 + */ + } //-------------------------------------------------------------------------------- - MDSPAN_INLINE_FUNCTION constexpr extents_type extents() const noexcept { + MDSPAN_INLINE_FUNCTION_DEFAULTED _MDSPAN_CONSTEXPR_14_DEFAULTED + mapping& operator=(mapping const&) noexcept = default; + + MDSPAN_INLINE_FUNCTION constexpr const extents_type& extents() const noexcept { #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) return __members.__first(); #else @@ -295,9 +397,45 @@ struct layout_stride { #endif }; - MDSPAN_INLINE_FUNCTION constexpr bool is_unique() const noexcept { return true; } - MDSPAN_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 bool is_contiguous() const noexcept { - // TODO @testing test layout_stride is_contiguous() + MDSPAN_INLINE_FUNCTION + constexpr array< index_type, extents_type::rank() > strides() const noexcept { + return __strides_storage(); + } + + MDSPAN_INLINE_FUNCTION + constexpr index_type required_span_size() const noexcept { + index_type span_size = 1; + for(unsigned r = 0; r < extents_type::rank(); r++) { + // Return early if any of the extents are zero + if(extents().extent(r)==0) return 0; + span_size = std::max(span_size, static_cast(extents().extent(r) * __strides_storage()[r])); + } + return span_size; + } + + + MDSPAN_TEMPLATE_REQUIRES( + class... Indices, + /* requires */ ( + sizeof...(Indices) == Extents::rank() && + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, Indices, index_type) /*&& ...*/ ) && + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_nothrow_constructible, index_type, Indices) /*&& ...*/) + ) + ) + MDSPAN_FORCE_INLINE_FUNCTION + constexpr size_t operator()(Indices... idxs) const noexcept { + return __impl::_call_op_impl(*this, static_cast(idxs)...); + } + + MDSPAN_INLINE_FUNCTION static constexpr bool is_always_unique() noexcept { return true; } + MDSPAN_INLINE_FUNCTION static constexpr bool is_always_exhaustive() noexcept { + return false; + } + MDSPAN_INLINE_FUNCTION static constexpr bool is_always_strided() noexcept { return true; } + + MDSPAN_INLINE_FUNCTION static constexpr bool is_unique() noexcept { return true; } + MDSPAN_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 bool is_exhaustive() const noexcept { +// TODO @testing test layout_stride is_exhaustive() // FIXME CUDA #ifdef __CUDA_ARCH__ return false; @@ -314,13 +452,13 @@ struct layout_stride { // "remove" the index constexpr auto removed_index_sentinel = static_cast(-1); *next_idx_iter = removed_index_sentinel; - int found_count = 1; + size_t found_count = 1; while (found_count != Extents::rank()) { next_idx_iter = std::find_if( rem.begin(), rem.end(), [&](size_t i) { return i != removed_index_sentinel - && this->extents().extent(i) == prev_stride_times_prev_extent; + && static_cast(this->extents().extent(i)) == prev_stride_times_prev_extent; } ); if (next_idx_iter != rem.end()) { @@ -335,55 +473,73 @@ struct layout_stride { return false; #endif } - MDSPAN_INLINE_FUNCTION constexpr bool is_strided() const noexcept { return true; } + MDSPAN_INLINE_FUNCTION static constexpr bool is_strided() noexcept { return true; } - MDSPAN_INLINE_FUNCTION static constexpr bool is_always_unique() noexcept { return true; } - MDSPAN_INLINE_FUNCTION static constexpr bool is_always_contiguous() noexcept { - return false; + + MDSPAN_INLINE_FUNCTION + constexpr index_type stride(rank_type r) const noexcept { + return __strides_storage()[r]; } - MDSPAN_INLINE_FUNCTION static constexpr bool is_always_strided() noexcept { return true; } +#if !(_MDSPAN_USE_CONCEPTS && MDSPAN_HAS_CXX_20) MDSPAN_TEMPLATE_REQUIRES( - class... Indices, + class StridedLayoutMapping, /* requires */ ( - sizeof...(Indices) == Extents::rank() && - _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_constructible, Indices, size_t) /*&& ...*/) + detail::__is_mapping_of && + (extents_type::rank() == StridedLayoutMapping::extents_type::rank()) && + StridedLayoutMapping::is_always_strided() ) ) - MDSPAN_FORCE_INLINE_FUNCTION - constexpr size_t operator()(Indices... idxs) const noexcept { - return __impl::_call_op_impl(*this, idxs...); - } - - MDSPAN_INLINE_FUNCTION - constexpr size_t stride(size_t r) const noexcept { - return __strides_storage().extent(r); - } - +#else + template + requires( + detail::__layout_mapping_alike && + (extents_type::rank() == StridedLayoutMapping::extents_type::rank()) && + StridedLayoutMapping::is_always_strided() + ) +#endif MDSPAN_INLINE_FUNCTION - constexpr array< size_t, Extents::rank() > strides() const noexcept { - return __strides(std::make_index_sequence()); + friend constexpr bool operator==(const mapping& x, const StridedLayoutMapping& y) noexcept { + bool strides_match = true; + for(rank_type r = 0; r < extents_type::rank(); r++) + strides_match = strides_match && (x.stride(r) == y.stride(r)); + return (x.extents() == y.extents()) && + (__impl::__OFFSET(y)== static_cast(0)) && + strides_match; } + // This one is not technically part of the proposal. Just here to make implementation a bit more optimal hopefully + MDSPAN_TEMPLATE_REQUIRES( + class OtherExtents, + /* requires */ ( + (extents_type::rank() == OtherExtents::rank()) + ) + ) MDSPAN_INLINE_FUNCTION - constexpr size_t required_span_size() const noexcept { - size_t span_size = 1; - for(unsigned r = 0; r < Extents::rank(); r++) { - // Return early if any of the extents are zero - if(extents().extent(r)==0) return 0; - span_size = std::max(span_size, extents().extent(r) * __strides_storage().extent(r)); - } - return span_size; + friend constexpr bool operator==(mapping const& lhs, mapping const& rhs) noexcept { + return __impl::_eq_impl(lhs, rhs); } - template +#if !MDSPAN_HAS_CXX_20 + MDSPAN_TEMPLATE_REQUIRES( + class StridedLayoutMapping, + /* requires */ ( + detail::__is_mapping_of && + (extents_type::rank() == StridedLayoutMapping::extents_type::rank()) && + StridedLayoutMapping::is_always_strided() + ) + ) MDSPAN_INLINE_FUNCTION - friend constexpr bool operator==(mapping const& lhs, mapping const& rhs) noexcept { - return __impl::_eq_impl(lhs, rhs); + friend constexpr bool operator!=(const mapping& x, const StridedLayoutMapping& y) noexcept { + return not (x == y); } -#if MDSPAN_HAS_CXX_20 - template + MDSPAN_TEMPLATE_REQUIRES( + class OtherExtents, + /* requires */ ( + (extents_type::rank() == OtherExtents::rank()) + ) + ) MDSPAN_INLINE_FUNCTION friend constexpr bool operator!=(mapping const& lhs, mapping const& rhs) noexcept { return __impl::_not_eq_impl(lhs, rhs); diff --git a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/macros.hpp b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/macros.hpp index 1d1c08b439..848dcf91a7 100644 --- a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/macros.hpp +++ b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/macros.hpp @@ -76,8 +76,6 @@ //============================================================================== // {{{1 -#if defined(_MDSPAN_COMPILER_MSVC) // Microsoft compilers - #define MDSPAN_PP_COUNT(...) \ _MDSPAN_PP_INTERNAL_EXPAND_ARGS_PRIVATE( \ _MDSPAN_PP_INTERNAL_ARGS_AUGMENTER(__VA_ARGS__) \ @@ -107,29 +105,6 @@ _70, count, ...) count \ /**/ -#else // Non-Microsoft compilers - -# define MDSPAN_PP_COUNT(...) \ - _MDSPAN_PP_INTERNAL_COUNT_PRIVATE( \ - 0, ## __VA_ARGS__, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, \ - 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, \ - 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, \ - 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, \ - 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) \ - /**/ -# define _MDSPAN_PP_INTERNAL_COUNT_PRIVATE( \ - _0_, _1_, _2_, _3_, _4_, _5_, _6_, _7_, _8_, _9_, \ - _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, \ - _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, \ - _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, \ - _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, \ - _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, \ - _60, _61, _62, _63, _64, _65, _66, _67, _68, _69, \ - _70, count, ...) count \ - /**/ - -#endif - #define MDSPAN_PP_STRINGIFY_IMPL(x) #x #define MDSPAN_PP_STRINGIFY(x) MDSPAN_PP_STRINGIFY_IMPL(x) @@ -201,6 +176,62 @@ template struct __maybe_static_value { - static constexpr _T __static_value = __v; - MDSPAN_FORCE_INLINE_FUNCTION constexpr _T __value() const noexcept { - return __v; + static constexpr _static_t __static_value = __v; + MDSPAN_FORCE_INLINE_FUNCTION constexpr _dynamic_t __value() const noexcept { + return static_cast<_dynamic_t>(__v); } template MDSPAN_FORCE_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 __mdspan_enable_fold_comma - __set_value(_U&& __rhs) noexcept { + __set_value(_U&& /*__rhs*/) noexcept { // Should we assert that the value matches the static value here? return {}; } @@ -93,7 +93,7 @@ struct __maybe_static_value { ~__maybe_static_value() noexcept = default; MDSPAN_INLINE_FUNCTION - constexpr explicit __maybe_static_value(_T const&) noexcept { + constexpr explicit __maybe_static_value(_dynamic_t const&) noexcept { // Should we assert that the value matches the static value here? } @@ -102,20 +102,20 @@ struct __maybe_static_value { }; // dynamic case -template -struct __maybe_static_value<_T, __is_dynamic_sentinal, __is_dynamic_sentinal, +template +struct __maybe_static_value<_dynamic_t, _static_t, __is_dynamic_sentinal, __is_dynamic_sentinal, __array_entry_index> #if !defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) : __no_unique_address_emulation<_T> #endif { - static constexpr _T __static_value = __is_dynamic_sentinal; + static constexpr _static_t __static_value = __is_dynamic_sentinal; #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) - _MDSPAN_NO_UNIQUE_ADDRESS _T __v = {}; - MDSPAN_FORCE_INLINE_FUNCTION constexpr _T __value() const noexcept { + _MDSPAN_NO_UNIQUE_ADDRESS _dynamic_t __v = {}; + MDSPAN_FORCE_INLINE_FUNCTION constexpr _dynamic_t __value() const noexcept { return __v; } - MDSPAN_FORCE_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 _T &__ref() noexcept { + MDSPAN_FORCE_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 _dynamic_t &__ref() noexcept { return __v; } template @@ -126,17 +126,17 @@ struct __maybe_static_value<_T, __is_dynamic_sentinal, __is_dynamic_sentinal, return {}; } #else - MDSPAN_FORCE_INLINE_FUNCTION constexpr _T __value() const noexcept { - return this->__no_unique_address_emulation<_T>::__ref(); + MDSPAN_FORCE_INLINE_FUNCTION constexpr _dynamic_t __value() const noexcept { + return this->__no_unique_address_emulation<_dynamic_t>::__ref(); } - MDSPAN_FORCE_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 _T &__ref() noexcept { - return this->__no_unique_address_emulation<_T>::__ref(); + MDSPAN_FORCE_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 _dynamic_t &__ref() noexcept { + return this->__no_unique_address_emulation<_dynamic_t>::__ref(); } template MDSPAN_FORCE_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 __mdspan_enable_fold_comma __set_value(_U&& __rhs) noexcept { - this->__no_unique_address_emulation<_T>::__ref() = (_U &&)__rhs; + this->__no_unique_address_emulation<_dynamic_t>::__ref() = (_U &&)__rhs; return {}; } #endif diff --git a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/mdspan.hpp b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/mdspan.hpp index 116d3d989a..214a52e642 100644 --- a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/mdspan.hpp +++ b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/mdspan.hpp @@ -93,11 +93,17 @@ class mdspan using mapping_type = typename layout_type::template mapping; using element_type = ElementType; using value_type = remove_cv_t; - using size_type = size_t; - using difference_type = ptrdiff_t; - using pointer = typename accessor_type::pointer; + using index_type = typename extents_type::index_type; + using size_type = typename extents_type::size_type; + using rank_type = typename extents_type::rank_type; + using data_handle_type = typename accessor_type::data_handle_type; using reference = typename accessor_type::reference; + MDSPAN_INLINE_FUNCTION static constexpr size_t rank() noexcept { return extents_type::rank(); } + MDSPAN_INLINE_FUNCTION static constexpr size_t rank_dynamic() noexcept { return extents_type::rank_dynamic(); } + MDSPAN_INLINE_FUNCTION static constexpr size_t static_extent(size_t r) noexcept { return extents_type::static_extent(r); } + MDSPAN_INLINE_FUNCTION constexpr index_type extent(size_t r) const noexcept { return __mapping_ref().extents().extent(r); }; + private: // Can't use defaulted parameter in the __deduction_workaround template because of a bug in MSVC warning C4348. @@ -110,76 +116,113 @@ class mdspan //-------------------------------------------------------------------------------- // [mdspan.basic.cons], mdspan constructors, assignment, and destructor +#if !MDSPAN_HAS_CXX_20 MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mdspan() = default; +#else + MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mdspan() + requires( + (rank_dynamic() > 0) && + _MDSPAN_TRAIT(is_default_constructible, data_handle_type) && + _MDSPAN_TRAIT(is_default_constructible, mapping_type) && + _MDSPAN_TRAIT(is_default_constructible, accessor_type) + ) = default; +#endif MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mdspan(const mdspan&) = default; MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mdspan(mdspan&&) = default; MDSPAN_TEMPLATE_REQUIRES( class... SizeTypes, /* requires */ ( - _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, SizeTypes, size_type) /* && ... */) && - _MDSPAN_TRAIT(is_constructible, extents_type, SizeTypes...) && + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, SizeTypes, index_type) /* && ... */) && + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_nothrow_constructible, index_type, SizeTypes) /* && ... */) && + ((sizeof...(SizeTypes) == rank()) || (sizeof...(SizeTypes) == rank_dynamic())) && _MDSPAN_TRAIT(is_constructible, mapping_type, extents_type) && _MDSPAN_TRAIT(is_default_constructible, accessor_type) ) ) MDSPAN_INLINE_FUNCTION - explicit constexpr mdspan(pointer p, SizeTypes... dynamic_extents) + explicit constexpr mdspan(data_handle_type p, SizeTypes... dynamic_extents) // TODO @proposal-bug shouldn't I be allowed to do `move(p)` here? - : __members(p, __map_acc_pair_t(mapping_type(extents_type(dynamic_extents...)), accessor_type())) + : __members(std::move(p), __map_acc_pair_t(mapping_type(extents_type(static_cast(std::move(dynamic_extents))...)), accessor_type())) + { } + + MDSPAN_TEMPLATE_REQUIRES( + class SizeType, size_t N, + /* requires */ ( + _MDSPAN_TRAIT(is_convertible, SizeType, index_type) && + _MDSPAN_TRAIT(is_nothrow_constructible, index_type, SizeType) && + ((N == rank()) || (N == rank_dynamic())) && + _MDSPAN_TRAIT(is_constructible, mapping_type, extents_type) && + _MDSPAN_TRAIT(is_default_constructible, accessor_type) + ) + ) + MDSPAN_CONDITIONAL_EXPLICIT(N != rank_dynamic()) + MDSPAN_INLINE_FUNCTION + constexpr mdspan(data_handle_type p, const array& dynamic_extents) + : __members(std::move(p), __map_acc_pair_t(mapping_type(extents_type(dynamic_extents)), accessor_type())) { } +#ifdef __cpp_lib_span MDSPAN_TEMPLATE_REQUIRES( class SizeType, size_t N, /* requires */ ( - _MDSPAN_TRAIT(is_convertible, SizeType, size_type) && - _MDSPAN_TRAIT(is_constructible, extents_type, array) && + _MDSPAN_TRAIT(is_convertible, SizeType, index_type) && + _MDSPAN_TRAIT(is_nothrow_constructible, index_type, SizeType) && + ((N == rank()) || (N == rank_dynamic())) && _MDSPAN_TRAIT(is_constructible, mapping_type, extents_type) && _MDSPAN_TRAIT(is_default_constructible, accessor_type) ) ) - MDSPAN_CONDITIONAL_EXPLICIT(N != extents_type::rank_dynamic()) + MDSPAN_CONDITIONAL_EXPLICIT(N != rank_dynamic()) MDSPAN_INLINE_FUNCTION - constexpr mdspan(pointer p, const array& dynamic_extents) - : __members(p, __map_acc_pair_t(mapping_type(extents_type(dynamic_extents)), accessor_type())) + constexpr mdspan(data_handle_type p, span dynamic_extents) + : __members(std::move(p), __map_acc_pair_t(mapping_type(extents_type(as_const(dynamic_extents))), accessor_type())) { } +#endif MDSPAN_FUNCTION_REQUIRES( (MDSPAN_INLINE_FUNCTION constexpr), - mdspan, (pointer p, const extents_type& exts), , + mdspan, (data_handle_type p, const extents_type& exts), , /* requires */ (_MDSPAN_TRAIT(is_default_constructible, accessor_type) && _MDSPAN_TRAIT(is_constructible, mapping_type, extents_type)) - ) : __members(p, __map_acc_pair_t(mapping_type(exts), accessor_type())) + ) : __members(std::move(p), __map_acc_pair_t(mapping_type(exts), accessor_type())) { } MDSPAN_FUNCTION_REQUIRES( (MDSPAN_INLINE_FUNCTION constexpr), - mdspan, (pointer p, const mapping_type& m), , + mdspan, (data_handle_type p, const mapping_type& m), , /* requires */ (_MDSPAN_TRAIT(is_default_constructible, accessor_type)) - ) : __members(p, __map_acc_pair_t(m, accessor_type())) + ) : __members(std::move(p), __map_acc_pair_t(m, accessor_type())) { } MDSPAN_INLINE_FUNCTION - constexpr mdspan(pointer p, const mapping_type& m, const accessor_type& a) - : __members(p, __map_acc_pair_t(m, a)) + constexpr mdspan(data_handle_type p, const mapping_type& m, const accessor_type& a) + : __members(std::move(p), __map_acc_pair_t(m, a)) { } MDSPAN_TEMPLATE_REQUIRES( class OtherElementType, class OtherExtents, class OtherLayoutPolicy, class OtherAccessor, /* requires */ ( _MDSPAN_TRAIT(is_constructible, mapping_type, typename OtherLayoutPolicy::template mapping) && - _MDSPAN_TRAIT(is_constructible, accessor_type, OtherAccessor) && - _MDSPAN_TRAIT(is_constructible, pointer, typename OtherAccessor::pointer) && - _MDSPAN_TRAIT(is_constructible, extents_type, OtherExtents) + _MDSPAN_TRAIT(is_constructible, accessor_type, OtherAccessor) ) ) MDSPAN_INLINE_FUNCTION constexpr mdspan(const mdspan& other) : __members(other.__ptr_ref(), __map_acc_pair_t(other.__mapping_ref(), other.__accessor_ref())) - { } + { + static_assert(_MDSPAN_TRAIT(is_constructible, data_handle_type, typename OtherAccessor::data_handle_type),"Incompatible data_handle_type for mdspan construction"); + static_assert(_MDSPAN_TRAIT(is_constructible, extents_type, OtherExtents),"Incompatible extents for mdspan construction"); + /* + * TODO: Check precondition + * For each rank index r of extents_type, static_extent(r) == dynamic_extent || static_extent(r) == other.extent(r) is true. + */ + } + /* Might need this on NVIDIA? MDSPAN_INLINE_FUNCTION_DEFAULTED ~mdspan() = default; + */ MDSPAN_INLINE_FUNCTION_DEFAULTED _MDSPAN_CONSTEXPR_14_DEFAULTED mdspan& operator=(const mdspan&) = default; MDSPAN_INLINE_FUNCTION_DEFAULTED _MDSPAN_CONSTEXPR_14_DEFAULTED mdspan& operator=(mdspan&&) = default; @@ -192,42 +235,59 @@ class mdspan MDSPAN_TEMPLATE_REQUIRES( class... SizeTypes, /* requires */ ( - _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, SizeTypes, size_type) /* && ... */) && - extents_type::rank() == sizeof...(SizeTypes) + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, SizeTypes, index_type) /* && ... */) && + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_nothrow_constructible, index_type, SizeTypes) /* && ... */) && + (rank() == sizeof...(SizeTypes)) ) ) MDSPAN_FORCE_INLINE_FUNCTION - constexpr reference operator[](SizeTypes... indices) const noexcept + constexpr reference operator[](SizeTypes... indices) const { - return __accessor_ref().access(__ptr_ref(), __mapping_ref()(size_type(indices)...)); + return __accessor_ref().access(__ptr_ref(), __mapping_ref()(index_type(indices)...)); } #endif MDSPAN_TEMPLATE_REQUIRES( - class SizeType, size_t N, + class SizeType, + /* requires */ ( + _MDSPAN_TRAIT(is_convertible, SizeType, index_type) && + _MDSPAN_TRAIT(is_nothrow_constructible, index_type, SizeType) + ) + ) + MDSPAN_FORCE_INLINE_FUNCTION + constexpr reference operator[](const array& indices) const + { + return __impl::template __callop(*this, indices); + } + + #ifdef __cpp_lib_span + MDSPAN_TEMPLATE_REQUIRES( + class SizeType, /* requires */ ( - _MDSPAN_TRAIT(is_convertible, SizeType, size_type) && - N == extents_type::rank() + _MDSPAN_TRAIT(is_convertible, SizeType, index_type) && + _MDSPAN_TRAIT(is_nothrow_constructible, index_type, SizeType) ) ) MDSPAN_FORCE_INLINE_FUNCTION - constexpr reference operator[](const array& indices) const noexcept + constexpr reference operator[](span indices) const { return __impl::template __callop(*this, indices); } + #endif // __cpp_lib_span #if !MDSPAN_USE_BRACKET_OPERATOR MDSPAN_TEMPLATE_REQUIRES( class Index, /* requires */ ( - _MDSPAN_TRAIT(is_convertible, Index, size_type) && + _MDSPAN_TRAIT(is_convertible, Index, index_type) && + _MDSPAN_TRAIT(is_nothrow_constructible, index_type, Index) && extents_type::rank() == 1 ) ) MDSPAN_FORCE_INLINE_FUNCTION - constexpr reference operator[](Index idx) const noexcept + constexpr reference operator[](Index idx) const { - return __accessor_ref().access(__ptr_ref(), __mapping_ref()(size_type(idx))); + return __accessor_ref().access(__ptr_ref(), __mapping_ref()(index_type(idx))); } #endif @@ -235,67 +295,78 @@ class mdspan MDSPAN_TEMPLATE_REQUIRES( class... SizeTypes, /* requires */ ( - _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, SizeTypes, size_type) /* && ... */) && - extents_type::rank() == sizeof...(SizeTypes) + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, SizeTypes, index_type) /* && ... */) && + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_nothrow_constructible, index_type, SizeTypes) /* && ... */) && + rank() == sizeof...(SizeTypes) ) ) MDSPAN_FORCE_INLINE_FUNCTION - constexpr reference operator()(SizeTypes... indices) const noexcept + constexpr reference operator()(SizeTypes... indices) const { - return __accessor_ref().access(__ptr_ref(), __mapping_ref()(size_type(indices)...)); + return __accessor_ref().access(__ptr_ref(), __mapping_ref()(indices...)); } MDSPAN_TEMPLATE_REQUIRES( - class SizeType, size_t N, + class SizeType, /* requires */ ( - _MDSPAN_TRAIT(is_convertible, SizeType, size_type) && - N == extents_type::rank() + _MDSPAN_TRAIT(is_convertible, SizeType, index_type) && + _MDSPAN_TRAIT(is_nothrow_constructible, index_type, SizeType) ) ) MDSPAN_FORCE_INLINE_FUNCTION - constexpr reference operator()(const array& indices) const noexcept + constexpr reference operator()(const array& indices) const { return __impl::template __callop(*this, indices); } - #endif - MDSPAN_INLINE_FUNCTION constexpr - accessor_type accessor() const { return __accessor_ref(); }; + #ifdef __cpp_lib_span + MDSPAN_TEMPLATE_REQUIRES( + class SizeType, + /* requires */ ( + _MDSPAN_TRAIT(is_convertible, SizeType, index_type) && + _MDSPAN_TRAIT(is_nothrow_constructible, index_type, SizeType) + ) + ) + MDSPAN_FORCE_INLINE_FUNCTION + constexpr reference operator()(span indices) const + { + return __impl::template __callop(*this, indices); + } + #endif // __cpp_lib_span + #endif // MDSPAN_USE_PAREN_OPERATOR + + MDSPAN_INLINE_FUNCTION constexpr size_type size() const noexcept { + return static_cast(__impl::__size(*this)); + }; + //-------------------------------------------------------------------------------- // [mdspan.basic.domobs], mdspan observers of the domain multidimensional index space - MDSPAN_INLINE_FUNCTION static constexpr size_t rank() noexcept { return extents_type::rank(); } - MDSPAN_INLINE_FUNCTION static constexpr size_t rank_dynamic() noexcept { return extents_type::rank_dynamic(); } - MDSPAN_INLINE_FUNCTION static constexpr size_type static_extent(size_t r) noexcept { return extents_type::static_extent(r); } - MDSPAN_INLINE_FUNCTION constexpr extents_type extents() const noexcept { return __mapping_ref().extents(); }; - MDSPAN_INLINE_FUNCTION constexpr size_type extent(size_t r) const noexcept { return __mapping_ref().extents().extent(r); }; - MDSPAN_INLINE_FUNCTION constexpr size_type size() const noexcept { - return __impl::__size(*this); - }; - - MDSPAN_INLINE_FUNCTION constexpr pointer data() const noexcept { return __ptr_ref(); }; + MDSPAN_INLINE_FUNCTION constexpr const extents_type& extents() const noexcept { return __mapping_ref().extents(); }; + MDSPAN_INLINE_FUNCTION constexpr const data_handle_type& data_handle() const noexcept { return __ptr_ref(); }; + MDSPAN_INLINE_FUNCTION constexpr const mapping_type& mapping() const noexcept { return __mapping_ref(); }; + MDSPAN_INLINE_FUNCTION constexpr const accessor_type& accessor() const noexcept { return __accessor_ref(); }; //-------------------------------------------------------------------------------- // [mdspan.basic.obs], mdspan observers of the mapping MDSPAN_INLINE_FUNCTION static constexpr bool is_always_unique() noexcept { return mapping_type::is_always_unique(); }; - MDSPAN_INLINE_FUNCTION static constexpr bool is_always_contiguous() noexcept { return mapping_type::is_always_contiguous(); }; + MDSPAN_INLINE_FUNCTION static constexpr bool is_always_exhaustive() noexcept { return mapping_type::is_always_exhaustive(); }; MDSPAN_INLINE_FUNCTION static constexpr bool is_always_strided() noexcept { return mapping_type::is_always_strided(); }; - MDSPAN_INLINE_FUNCTION constexpr mapping_type mapping() const noexcept { return __mapping_ref(); }; MDSPAN_INLINE_FUNCTION constexpr bool is_unique() const noexcept { return __mapping_ref().is_unique(); }; - MDSPAN_INLINE_FUNCTION constexpr bool is_contiguous() const noexcept { return __mapping_ref().is_contiguous(); }; + MDSPAN_INLINE_FUNCTION constexpr bool is_exhaustive() const noexcept { return __mapping_ref().is_exhaustive(); }; MDSPAN_INLINE_FUNCTION constexpr bool is_strided() const noexcept { return __mapping_ref().is_strided(); }; - MDSPAN_INLINE_FUNCTION constexpr size_type stride(size_t r) const { return __mapping_ref().stride(r); }; + MDSPAN_INLINE_FUNCTION constexpr index_type stride(size_t r) const { return __mapping_ref().stride(r); }; private: - detail::__compressed_pair __members{}; + detail::__compressed_pair __members{}; - MDSPAN_FORCE_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 pointer& __ptr_ref() noexcept { return __members.__first(); } - MDSPAN_FORCE_INLINE_FUNCTION constexpr pointer const& __ptr_ref() const noexcept { return __members.__first(); } + MDSPAN_FORCE_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 data_handle_type& __ptr_ref() noexcept { return __members.__first(); } + MDSPAN_FORCE_INLINE_FUNCTION constexpr data_handle_type const& __ptr_ref() const noexcept { return __members.__first(); } MDSPAN_FORCE_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 mapping_type& __mapping_ref() noexcept { return __members.__second().__first(); } MDSPAN_FORCE_INLINE_FUNCTION constexpr mapping_type const& __mapping_ref() const noexcept { return __members.__second().__first(); } MDSPAN_FORCE_INLINE_FUNCTION _MDSPAN_CONSTEXPR_14 accessor_type& __accessor_ref() noexcept { return __members.__second().__second(); } @@ -309,28 +380,47 @@ class mdspan #if defined(_MDSPAN_USE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION) MDSPAN_TEMPLATE_REQUIRES( class ElementType, class... SizeTypes, - /* requires */ _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_integral, SizeTypes) /* && ... */) + /* requires */ _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_integral, SizeTypes) /* && ... */) && + (sizeof...(SizeTypes) > 0) +) +explicit mdspan(ElementType*, SizeTypes...) + -> mdspan>; + +MDSPAN_TEMPLATE_REQUIRES( + class Pointer, + (_MDSPAN_TRAIT(is_pointer, std::remove_reference_t)) +) +mdspan(Pointer&&) -> mdspan>, extents>; + +MDSPAN_TEMPLATE_REQUIRES( + class CArray, + (_MDSPAN_TRAIT(is_array, CArray) && (rank_v == 1)) ) -mdspan(ElementType*, SizeTypes...) - -> mdspan>; +mdspan(CArray&) -> mdspan, extents>>; template mdspan(ElementType*, const ::std::array&) - -> mdspan>; + -> mdspan>; + +#ifdef __cpp_lib_span +template +mdspan(ElementType*, ::std::span) + -> mdspan>; +#endif -// This one is necessary because all the constructors take `pointer`s, not -// `ElementType*`s, and `pointer` is taken from `accessor_type::pointer`, which +// This one is necessary because all the constructors take `data_handle_type`s, not +// `ElementType*`s, and `data_handle_type` is taken from `accessor_type::data_handle_type`, which // seems to throw off automatic deduction guides. -template -mdspan(ElementType*, const extents&) - -> mdspan>; +template +mdspan(ElementType*, const extents&) + -> mdspan>; template mdspan(ElementType*, const MappingType&) -> mdspan; template -mdspan(const typename AccessorType::pointer, const MappingType&, const AccessorType&) +mdspan(const typename AccessorType::data_handle_type, const MappingType&, const AccessorType&) -> mdspan; #endif diff --git a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/standard_layout_static_array.hpp b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/standard_layout_static_array.hpp index d0b2e6d0e4..1c543b0805 100644 --- a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/standard_layout_static_array.hpp +++ b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/standard_layout_static_array.hpp @@ -53,6 +53,9 @@ #endif #include +#ifdef __cpp_lib_span +#include +#endif #include // integer_sequence #include @@ -88,20 +91,20 @@ template using __repeated_with_idxs = _T; * @tparam _ValsSeq * @tparam __sentinal */ -template (dynamic_extent), class _IdxsSeq = make_index_sequence<_ValsSeq::size()>> struct __standard_layout_psa; //============================================================================== // Static case -template +template struct __standard_layout_psa< - _Tag, _T, integer_sequence<_T, __value, __values_or_sentinals...>, + _Tag, _T, _static_t, integer_sequence<_static_t, __value, __values_or_sentinals...>, __sentinal, integer_sequence> #if !defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) : private __no_unique_address_emulation<__standard_layout_psa< - _Tag, _T, integer_sequence<_T, __values_or_sentinals...>, __sentinal, + _Tag, _T, _static_t, integer_sequence<_static_t, __values_or_sentinals...>, __sentinal, integer_sequence>> #endif { @@ -109,8 +112,8 @@ struct __standard_layout_psa< //-------------------------------------------------------------------------- using __next_t = - __standard_layout_psa<_Tag, _T, - integer_sequence<_T, __values_or_sentinals...>, + __standard_layout_psa<_Tag, _T, _static_t, + integer_sequence<_static_t, __values_or_sentinals...>, __sentinal, integer_sequence>; #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) @@ -135,20 +138,7 @@ struct __standard_layout_psa< } static constexpr auto __size = sizeof...(_Idxs) + 1; -#ifdef _MDSPAN_COMPILER_MSVC - // MSVC doesn't like the fact that __next_t happens to be a base - // class that's private, even though __size_synamic is public in - // it's definition. - struct __msvc_workaround_tag {}; - using __msvc_workaround_next_t = __standard_layout_psa< - __msvc_workaround_tag, _T, - integer_sequence<_T, __values_or_sentinals...>, __sentinal, - integer_sequence>; - static constexpr auto __size_dynamic = - __msvc_workaround_next_t::__size_dynamic; -#else static constexpr auto __size_dynamic = __next_t::__size_dynamic; -#endif //-------------------------------------------------------------------------- @@ -254,10 +244,62 @@ struct __standard_layout_psa< #endif { } - template + MDSPAN_INLINE_FUNCTION constexpr explicit __standard_layout_psa( + span<_U, _N> const &__vals) noexcept +#if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) + : __next_{ +#else + : __base_t(__base_t{__next_t( +#endif + __vals +#if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) + } +#else + )}) +#endif + { } + + template + MDSPAN_INLINE_FUNCTION constexpr explicit __standard_layout_psa( + __construct_psa_from_all_exts_array_tag_t const & __tag, + span<_U, _NStatic> const &__vals) noexcept +#if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) + : __next_{ +#else + : __base_t(__base_t{__next_t( +#endif + __tag, __vals +#if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) + } +#else + )}) +#endif + { } + + template + MDSPAN_INLINE_FUNCTION constexpr explicit __standard_layout_psa( + __construct_psa_from_dynamic_exts_array_tag_t<_IDynamic> __tag, + span<_U, _NDynamic> const &__vals) noexcept +#if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) + : __next_{ +#else + : __base_t(__base_t{__next_t( +#endif + __tag, __vals +#if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) + } +#else + )}) +#endif + { } +#endif + + template MDSPAN_INLINE_FUNCTION constexpr __standard_layout_psa( - __standard_layout_psa<_UTag, _U, _UValsSeq, __u_sentinal, _IdxsSeq> const + __standard_layout_psa<_UTag, _U, _static_U, _UValsSeq, __u_sentinal, _IdxsSeq> const &__rhs) noexcept #if defined(_MDSPAN_USE_ATTRIBUTE_NO_UNIQUE_ADDRESS) : __next_{ @@ -303,12 +345,12 @@ struct __standard_layout_psa< // Don't assert here because that would break constexpr. This better // not change anything, though } - template = __sentinal> - MDSPAN_FORCE_INLINE_FUNCTION static constexpr _T __get_static_n() noexcept { + template = __sentinal> + MDSPAN_FORCE_INLINE_FUNCTION static constexpr _static_t __get_static_n() noexcept { return __value; } - template __default = __sentinal> - MDSPAN_FORCE_INLINE_FUNCTION static constexpr _T __get_static_n() noexcept { + template __default = __sentinal> + MDSPAN_FORCE_INLINE_FUNCTION static constexpr _static_t __get_static_n() noexcept { return __next_t::template __get_static_n<_I, __default>(); } MDSPAN_FORCE_INLINE_FUNCTION constexpr _T __get(size_t __n) const noexcept { @@ -321,16 +363,16 @@ struct __standard_layout_psa< //============================================================================== // Dynamic case, __next_t may or may not be empty -template struct __standard_layout_psa< - _Tag, _T, integer_sequence<_T, __sentinal, __values_or_sentinals...>, + _Tag, _T, _static_t, integer_sequence<_static_t, __sentinal, __values_or_sentinals...>, __sentinal, integer_sequence> { //-------------------------------------------------------------------------- using __next_t = - __standard_layout_psa<_Tag, _T, - integer_sequence<_T, __values_or_sentinals...>, + __standard_layout_psa<_Tag, _T, _static_t, + integer_sequence<_static_t, __values_or_sentinals...>, __sentinal, integer_sequence>; using __value_pair_t = __compressed_pair<_T, __next_t>; @@ -404,10 +446,35 @@ struct __standard_layout_psa< __next_t(__construct_psa_from_dynamic_exts_array_tag_t<_IDynamic + 1>{}, __vals)) {} - template + MDSPAN_INLINE_FUNCTION constexpr explicit __standard_layout_psa( + span<_U, _N> const &__vals) noexcept + : __value_pair(__vals[_Idx], __vals) {} + + template + MDSPAN_INLINE_FUNCTION constexpr explicit __standard_layout_psa( + __construct_psa_from_all_exts_array_tag_t __tag, + span<_U, _NStatic> const &__vals) noexcept + : __value_pair( + __vals[_Idx], + __next_t(__tag, + __vals)) {} + + template + MDSPAN_INLINE_FUNCTION constexpr explicit __standard_layout_psa( + __construct_psa_from_dynamic_exts_array_tag_t<_IDynamic>, + span<_U, _NDynamic> const &__vals) noexcept + : __value_pair( + __vals[_IDynamic], + __next_t(__construct_psa_from_dynamic_exts_array_tag_t<_IDynamic + 1>{}, + __vals)) {} +#endif + + template MDSPAN_INLINE_FUNCTION constexpr __standard_layout_psa( - __standard_layout_psa<_UTag, _U, _UValsSeq, __u_sentinal, _UIdxsSeq> const + __standard_layout_psa<_UTag, _U, _static_U, _UValsSeq, __u_sentinal, _UIdxsSeq> const &__rhs) noexcept : __value_pair(__rhs.template __get_n<_Idx>(), __rhs.__next()) {} @@ -439,12 +506,12 @@ struct __standard_layout_psa< __set_n(_T const &__rhs) noexcept { __value_pair.__first() = __rhs; } - template __default = __sentinal> - MDSPAN_FORCE_INLINE_FUNCTION static constexpr _T __get_static_n() noexcept { + template __default = __sentinal> + MDSPAN_FORCE_INLINE_FUNCTION static constexpr _static_t __get_static_n() noexcept { return __default; } - template __default = __sentinal> - MDSPAN_FORCE_INLINE_FUNCTION static constexpr _T __get_static_n() noexcept { + template __default = __sentinal> + MDSPAN_FORCE_INLINE_FUNCTION static constexpr _static_t __get_static_n() noexcept { return __next_t::template __get_static_n<_I, __default>(); } MDSPAN_FORCE_INLINE_FUNCTION constexpr _T __get(size_t __n) const noexcept { @@ -455,8 +522,8 @@ struct __standard_layout_psa< }; // empty/terminal case -template -struct __standard_layout_psa<_Tag, _T, integer_sequence<_T>, __sentinal, +template +struct __standard_layout_psa<_Tag, _T, _static_t, integer_sequence<_static_t>, __sentinal, integer_sequence> { //-------------------------------------------------------------------------- @@ -510,10 +577,26 @@ struct __standard_layout_psa<_Tag, _T, integer_sequence<_T>, __sentinal, __construct_psa_from_dynamic_exts_array_tag_t<_IDynamic>, array<_U, _NDynamic> const &) noexcept {} - template + MDSPAN_INLINE_FUNCTION constexpr explicit __standard_layout_psa( + span<_U, _N> const &) noexcept {} + + template + MDSPAN_INLINE_FUNCTION constexpr explicit __standard_layout_psa( + __construct_psa_from_all_exts_array_tag_t, + span<_U, _NStatic> const &) noexcept {} + + template + MDSPAN_INLINE_FUNCTION constexpr explicit __standard_layout_psa( + __construct_psa_from_dynamic_exts_array_tag_t<_IDynamic>, + span<_U, _NDynamic> const &) noexcept {} +#endif + + template MDSPAN_INLINE_FUNCTION constexpr __standard_layout_psa( - __standard_layout_psa<_UTag, _U, _UValsSeq, __u_sentinal, _UIdxsSeq> const&) noexcept {} + __standard_layout_psa<_UTag, _U, _static_U, _UValsSeq, __u_sentinal, _UIdxsSeq> const&) noexcept {} // See comment in the previous partial specialization for why this is // necessary. Or just trust me that it's messy. @@ -530,14 +613,14 @@ struct __standard_layout_psa<_Tag, _T, integer_sequence<_T>, __sentinal, // Same thing, but with a disambiguator so that same-base issues doesn't cause // a loss of standard-layout-ness. -template +template struct __partially_static_sizes_tagged : __standard_layout_psa< - _Tag, size_t, - integer_sequence> { + _Tag, T, _static_t, + integer_sequence<_static_t, __values_or_sentinals...>> { using __tag_t = _Tag; using __psa_impl_t = __standard_layout_psa< - _Tag, size_t, integer_sequence>; + _Tag, T, _static_t, integer_sequence<_static_t, __values_or_sentinals...>>; using __psa_impl_t::__psa_impl_t; #ifdef _MDSPAN_DEFAULTED_CONSTRUCTORS_INHERITANCE_WORKAROUND MDSPAN_INLINE_FUNCTION @@ -565,20 +648,20 @@ struct __partially_static_sizes_tagged template MDSPAN_FORCE_INLINE_FUNCTION constexpr explicit __partially_static_sizes_tagged( - __partially_static_sizes_tagged<_UTag, __values_or_sentinals...> const& __vals + __partially_static_sizes_tagged<_UTag, T, _static_t, __values_or_sentinals...> const& __vals ) noexcept : __psa_impl_t(__vals.__enable_psa_conversion()) { } }; struct __no_tag {}; -template +template struct __partially_static_sizes - : __partially_static_sizes_tagged<__no_tag, __values_or_sentinals...> { + : __partially_static_sizes_tagged<__no_tag, T, _static_t, __values_or_sentinals...> { private: using __base_t = - __partially_static_sizes_tagged<__no_tag, __values_or_sentinals...>; + __partially_static_sizes_tagged<__no_tag, T, _static_t, __values_or_sentinals...>; template MDSPAN_FORCE_INLINE_FUNCTION constexpr __partially_static_sizes( - __partially_static_sizes_tagged<_UTag, __values_or_sentinals...>&& __vals + __partially_static_sizes_tagged<_UTag, T, _static_t, __values_or_sentinals...>&& __vals ) noexcept : __base_t(::std::move(__vals)) { } public: using __base_t::__base_t; @@ -589,9 +672,9 @@ struct __partially_static_sizes #endif template MDSPAN_FORCE_INLINE_FUNCTION constexpr __partially_static_sizes_tagged< - _UTag, __values_or_sentinals...> + _UTag, T, _static_t, __values_or_sentinals...> __with_tag() const noexcept { - return __partially_static_sizes_tagged<_UTag, __values_or_sentinals...>(*this); + return __partially_static_sizes_tagged<_UTag, T, _static_t, __values_or_sentinals...>(*this); } }; diff --git a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/static_array.hpp b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/static_array.hpp index d3c6317765..548bd8c8a7 100644 --- a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/static_array.hpp +++ b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/static_array.hpp @@ -91,31 +91,33 @@ struct __mask_sequence, integer_sequence class __partially_static_array_impl; template < - class _T, _T... __values_or_sentinals, _T __sentinal, + class _T, class _static_t, + _static_t... __values_or_sentinals, _static_t __sentinal, size_t... _Idxs, size_t... _IdxsDynamic, size_t... _IdxsDynamicIdxs > class __partially_static_array_impl< _T, - integer_sequence<_T, __values_or_sentinals...>, + _static_t, + integer_sequence<_static_t, __values_or_sentinals...>, __sentinal, integer_sequence, integer_sequence, integer_sequence > - : private __maybe_static_value<_T, __values_or_sentinals, __sentinal, + : private __maybe_static_value<_T, _static_t, __values_or_sentinals, __sentinal, _Idxs>... { private: template using __base_n = typename __type_at<_N, - __type_list<__maybe_static_value<_T, __values_or_sentinals, __sentinal, _Idxs>...> + __type_list<__maybe_static_value<_T, _static_t, __values_or_sentinals, __sentinal, _Idxs>...> >::type; public: @@ -173,11 +175,11 @@ class __partially_static_array_impl< ::std::get<_IdxsDynamicIdxs>(__vals)...) {} // clang-format on - template MDSPAN_INLINE_FUNCTION constexpr __partially_static_array_impl( __partially_static_array_impl< - _U, _UValsSeq, __u_sentinal, _UIdxsSeq, + _U, _static_u, _UValsSeq, __u_sentinal, _UIdxsSeq, _UIdxsDynamicSeq, _UIdxsDynamicIdxsSeq> const &__rhs) noexcept : __partially_static_array_impl( __construct_psa_from_all_exts_values_tag, @@ -203,8 +205,8 @@ class __partially_static_array_impl< static_cast<__base_n<_I>*>(this)->__set_value((_U&&)__rhs); } - template - MDSPAN_FORCE_INLINE_FUNCTION static constexpr _T + template + MDSPAN_FORCE_INLINE_FUNCTION static constexpr _static_t __get_static_n() noexcept { return __base_n<_I>::__static_value == __sentinal ? __default : __base_n<_I>::__static_value; @@ -221,14 +223,14 @@ class __partially_static_array_impl< //============================================================================== -template > +template > struct __partially_static_array_impl_maker; template < - class _T, _T... _Vals, _T __sentinal, size_t... _Idxs + class _T, class _static_t, _static_t... _Vals, _static_t __sentinal, size_t... _Idxs > struct __partially_static_array_impl_maker< - _T, integer_sequence<_T, _Vals...>, __sentinal, integer_sequence + _T, _static_t, integer_sequence<_static_t, _Vals...>, __sentinal, integer_sequence > { using __dynamic_idxs = typename __mask_sequence< @@ -236,46 +238,46 @@ struct __partially_static_array_impl_maker< integer_sequence >::type; using __impl_base = - __partially_static_array_impl<_T, - integer_sequence<_T, _Vals...>, + __partially_static_array_impl<_T, _static_t, + integer_sequence<_static_t, _Vals...>, __sentinal, integer_sequence, __dynamic_idxs, make_index_sequence<__dynamic_idxs::size()> >; }; -template +template class __partially_static_array_with_sentinal - : public __partially_static_array_impl_maker<_T, _ValsSeq, __sentinal>::__impl_base + : public __partially_static_array_impl_maker<_T, _static_t, _ValsSeq, __sentinal>::__impl_base { private: - using __base_t = typename __partially_static_array_impl_maker<_T, _ValsSeq, __sentinal>::__impl_base; + using __base_t = typename __partially_static_array_impl_maker<_T, _static_t, _ValsSeq, __sentinal>::__impl_base; public: using __base_t::__base_t; }; //============================================================================== -template +template struct __partially_static_sizes : __partially_static_array_with_sentinal< - size_t, ::std::integer_sequence> + T, _static_t, ::std::integer_sequence<_static_t, __values_or_sentinals...>> { private: using __base_t = __partially_static_array_with_sentinal< - size_t, ::std::integer_sequence>; + T, _static_t, ::std::integer_sequence<_static_t, __values_or_sentinals...>>; public: using __base_t::__base_t; template - MDSPAN_FORCE_INLINE_FUNCTION constexpr __partially_static_sizes<__values_or_sentinals...> + MDSPAN_FORCE_INLINE_FUNCTION constexpr __partially_static_sizes __with_tag() const noexcept { return *this; } }; // Tags are needed for the standard layout version, but not here -template -using __partially_static_sizes_tagged = __partially_static_sizes<__values_or_sentinals...>; +template +using __partially_static_sizes_tagged = __partially_static_sizes; } // end namespace detail } // end namespace experimental diff --git a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/submdspan.hpp b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/submdspan.hpp index 3b2b99099c..0467485c1c 100644 --- a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/submdspan.hpp +++ b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/submdspan.hpp @@ -174,18 +174,20 @@ struct preserve_layout_analysis //-------------------------------------------------------------------------------- template < + class _IndexT, class _PreserveLayoutAnalysis, - class _OffsetsArray=__partially_static_sizes<>, - class _ExtsArray=__partially_static_sizes<>, - class _StridesArray=__partially_static_sizes<>, - class=make_index_sequence<_OffsetsArray::__size>, - class=make_index_sequence<_ExtsArray::__size>, - class=make_index_sequence<_StridesArray::__size> + class _OffsetsArray=__partially_static_sizes<_IndexT, size_t>, + class _ExtsArray=__partially_static_sizes<_IndexT, size_t>, + class _StridesArray=__partially_static_sizes<_IndexT, size_t>, + class = make_index_sequence<_OffsetsArray::__size>, + class = make_index_sequence<_ExtsArray::__size>, + class = make_index_sequence<_StridesArray::__size> > struct __assign_op_slice_handler; /* clang-format: off */ template < + class _IndexT, class _PreserveLayoutAnalysis, size_t... _Offsets, size_t... _Exts, @@ -194,10 +196,11 @@ template < size_t... _ExtIdxs, size_t... _StrideIdxs> struct __assign_op_slice_handler< + _IndexT, _PreserveLayoutAnalysis, - __partially_static_sizes<_Offsets...>, - __partially_static_sizes<_Exts...>, - __partially_static_sizes<_Strides...>, + __partially_static_sizes<_IndexT, size_t, _Offsets...>, + __partially_static_sizes<_IndexT, size_t, _Exts...>, + __partially_static_sizes<_IndexT, size_t, _Strides...>, integer_sequence, integer_sequence, integer_sequence> @@ -212,9 +215,9 @@ struct __assign_op_slice_handler< " " ); - using __offsets_storage_t = __partially_static_sizes<_Offsets...>; - using __extents_storage_t = __partially_static_sizes<_Exts...>; - using __strides_storage_t = __partially_static_sizes<_Strides...>; + using __offsets_storage_t = __partially_static_sizes<_IndexT, size_t, _Offsets...>; + using __extents_storage_t = __partially_static_sizes<_IndexT, size_t, _Exts...>; + using __strides_storage_t = __partially_static_sizes<_IndexT, size_t, _Strides...>; __offsets_storage_t __offsets; __extents_storage_t __exts; __strides_storage_t __strides; @@ -236,7 +239,7 @@ struct __assign_op_slice_handler< // Don't define this unless we need it; they have a cost to compile #ifndef _MDSPAN_USE_RETURN_TYPE_DEDUCTION - using __extents_type = ::std::experimental::extents<_Exts...>; + using __extents_type = ::std::experimental::extents<_IndexT, _Exts...>; #endif // For size_t slice, skip the extent and stride, but add an offset corresponding to the value @@ -245,12 +248,13 @@ struct __assign_op_slice_handler< _MDSPAN_CONSTEXPR_14 auto operator=(__slice_wrap<_OldStaticExtent, _OldStaticStride, size_t>&& __slice) noexcept -> __assign_op_slice_handler< + _IndexT, typename _PreserveLayoutAnalysis::encounter_scalar, - __partially_static_sizes<_Offsets..., dynamic_extent>, - __partially_static_sizes<_Exts...>, - __partially_static_sizes<_Strides...>/* intentional space here to work around ICC bug*/> { + __partially_static_sizes<_IndexT, size_t, _Offsets..., dynamic_extent>, + __partially_static_sizes<_IndexT, size_t, _Exts...>, + __partially_static_sizes<_IndexT, size_t, _Strides...>/* intentional space here to work around ICC bug*/> { return { - __partially_static_sizes<_Offsets..., dynamic_extent>( + __partially_static_sizes<_IndexT, size_t, _Offsets..., dynamic_extent>( __construct_psa_from_all_exts_values_tag, __offsets.template __get_n<_OffsetIdxs>()..., __slice.slice), ::std::move(__exts), @@ -264,18 +268,19 @@ struct __assign_op_slice_handler< _MDSPAN_CONSTEXPR_14 auto operator=(__slice_wrap<_OldStaticExtent, _OldStaticStride, full_extent_t>&& __slice) noexcept -> __assign_op_slice_handler< + _IndexT, typename _PreserveLayoutAnalysis::encounter_all, - __partially_static_sizes<_Offsets..., 0>, - __partially_static_sizes<_Exts..., _OldStaticExtent>, - __partially_static_sizes<_Strides..., _OldStaticStride>/* intentional space here to work around ICC bug*/> { + __partially_static_sizes<_IndexT, size_t, _Offsets..., 0>, + __partially_static_sizes<_IndexT, size_t, _Exts..., _OldStaticExtent>, + __partially_static_sizes<_IndexT, size_t, _Strides..., _OldStaticStride>/* intentional space here to work around ICC bug*/> { return { - __partially_static_sizes<_Offsets..., 0>( + __partially_static_sizes<_IndexT, size_t, _Offsets..., 0>( __construct_psa_from_all_exts_values_tag, __offsets.template __get_n<_OffsetIdxs>()..., size_t(0)), - __partially_static_sizes<_Exts..., _OldStaticExtent>( + __partially_static_sizes<_IndexT, size_t, _Exts..., _OldStaticExtent>( __construct_psa_from_all_exts_values_tag, __exts.template __get_n<_ExtIdxs>()..., __slice.old_extent), - __partially_static_sizes<_Strides..., _OldStaticStride>( + __partially_static_sizes<_IndexT, size_t, _Strides..., _OldStaticStride>( __construct_psa_from_all_exts_values_tag, __strides.template __get_n<_StrideIdxs>()..., __slice.old_stride) }; @@ -287,18 +292,19 @@ struct __assign_op_slice_handler< _MDSPAN_CONSTEXPR_14 auto operator=(__slice_wrap<_OldStaticExtent, _OldStaticStride, tuple>&& __slice) noexcept -> __assign_op_slice_handler< + _IndexT, typename _PreserveLayoutAnalysis::encounter_pair, - __partially_static_sizes<_Offsets..., dynamic_extent>, - __partially_static_sizes<_Exts..., dynamic_extent>, - __partially_static_sizes<_Strides..., _OldStaticStride>/* intentional space here to work around ICC bug*/> { + __partially_static_sizes<_IndexT, size_t, _Offsets..., dynamic_extent>, + __partially_static_sizes<_IndexT, size_t, _Exts..., dynamic_extent>, + __partially_static_sizes<_IndexT, size_t, _Strides..., _OldStaticStride>/* intentional space here to work around ICC bug*/> { return { - __partially_static_sizes<_Offsets..., dynamic_extent>( + __partially_static_sizes<_IndexT, size_t, _Offsets..., dynamic_extent>( __construct_psa_from_all_exts_values_tag, __offsets.template __get_n<_OffsetIdxs>()..., ::std::get<0>(__slice.slice)), - __partially_static_sizes<_Exts..., dynamic_extent>( + __partially_static_sizes<_IndexT, size_t, _Exts..., dynamic_extent>( __construct_psa_from_all_exts_values_tag, __exts.template __get_n<_ExtIdxs>()..., ::std::get<1>(__slice.slice) - ::std::get<0>(__slice.slice)), - __partially_static_sizes<_Strides..., _OldStaticStride>( + __partially_static_sizes<_IndexT, size_t, _Strides..., _OldStaticStride>( __construct_psa_from_all_exts_values_tag, __strides.template __get_n<_StrideIdxs>()..., __slice.old_stride) }; @@ -321,8 +327,8 @@ struct __assign_op_slice_handler< ), ( /* not layout stride, so don't pass dynamic_strides */ - /* return */ typename NewLayout::template mapping<::std::experimental::extents<_Exts...>>( - experimental::extents<_Exts...>::__make_extents_impl(::std::move(__exts)) + /* return */ typename NewLayout::template mapping<::std::experimental::extents<_IndexT, _Exts...>>( + experimental::extents<_IndexT, _Exts...>::__make_extents_impl(::std::move(__exts)) ) /* ; */ ) ) @@ -334,7 +340,7 @@ struct __assign_op_slice_handler< _make_layout_mapping_impl(layout_stride) noexcept ), ( - /* return */ layout_stride::template mapping<::std::experimental::extents<_Exts...>> + /* return */ layout_stride::template mapping<::std::experimental::extents<_IndexT, _Exts...>> ::__make_mapping(::std::move(__exts), ::std::move(__strides)) /* ; */ ) ) @@ -356,23 +362,25 @@ struct __assign_op_slice_handler< #if _MDSPAN_USE_RETURN_TYPE_DEDUCTION // Forking this because the C++11 version will be *completely* unreadable -template +template MDSPAN_INLINE_FUNCTION constexpr auto _submdspan_impl( integer_sequence, - mdspan, LP, AP> const& src, + mdspan, LP, AP> const& src, SliceSpecs&&... slices ) noexcept { + using _IndexT = ST; auto _handled = _MDSPAN_FOLD_ASSIGN_LEFT( ( detail::__assign_op_slice_handler< + _IndexT, detail::preserve_layout_analysis >{ - __partially_static_sizes<>{}, - __partially_static_sizes<>{}, - __partially_static_sizes<>{} + __partially_static_sizes<_IndexT, size_t>{}, + __partially_static_sizes<_IndexT, size_t>{}, + __partially_static_sizes<_IndexT, size_t>{} } ), /* = ... = */ @@ -380,16 +388,17 @@ constexpr auto _submdspan_impl( Exts, dynamic_extent >( slices, src.extents().template __extent(), - src.mapping().template __stride() + src.mapping().stride(Idxs) ) ); size_t offset_size = src.mapping()(_handled.__offsets.template __get_n()...); - auto offset_ptr = src.accessor().offset(src.data(), offset_size); + auto offset_ptr = src.accessor().offset(src.data_handle(), offset_size); auto map = _handled.make_layout_mapping(src.mapping()); auto acc_pol = typename AP::offset_policy(src.accessor()); return mdspan< - ET, decltype(map.extents()), typename decltype(_handled)::layout_type, decltype(acc_pol) + ET, remove_const_t>, + typename decltype(_handled)::layout_type, remove_const_t> >( std::move(offset_ptr), std::move(map), std::move(acc_pol) ); @@ -403,19 +412,19 @@ auto _submdspan_impl_helper(Src&& src, Handled&& h, std::integer_sequence { return { - src.accessor().offset(src.data(), src.mapping()(h.__offsets.template __get_n()...)), + src.accessor().offset(src.data_handle(), src.mapping()(h.__offsets.template __get_n()...)), h.make_layout_mapping(src.mapping()), typename AP::offset_policy(src.accessor()) }; } -template +template MDSPAN_INLINE_FUNCTION _MDSPAN_DEDUCE_RETURN_TYPE_SINGLE_LINE( ( constexpr /* auto */ _submdspan_impl( std::integer_sequence seq, - mdspan, LP, AP> const& src, + mdspan, LP, AP> const& src, SliceSpecs&&... slices ) noexcept ), @@ -425,11 +434,12 @@ _MDSPAN_DEDUCE_RETURN_TYPE_SINGLE_LINE( _MDSPAN_FOLD_ASSIGN_LEFT( ( detail::__assign_op_slice_handler< + size_t, detail::preserve_layout_analysis >{ - __partially_static_sizes<>{}, - __partially_static_sizes<>{}, - __partially_static_sizes<>{} + __partially_static_sizes{}, + __partially_static_sizes{}, + __partially_static_sizes{} } ), /* = ... = */ @@ -458,7 +468,7 @@ struct _is_layout_stride< //============================================================================== MDSPAN_TEMPLATE_REQUIRES( - class ET, size_t... Exts, class LP, class AP, class... SliceSpecs, + class ET, class EXT, class LP, class AP, class... SliceSpecs, /* requires */ ( ( _MDSPAN_TRAIT(is_same, LP, layout_left) @@ -470,14 +480,14 @@ MDSPAN_TEMPLATE_REQUIRES( || _MDSPAN_TRAIT(is_convertible, SliceSpecs, tuple) || _MDSPAN_TRAIT(is_convertible, SliceSpecs, full_extent_t) ) /* && ... */) && - sizeof...(SliceSpecs) == sizeof...(Exts) + sizeof...(SliceSpecs) == EXT::rank() ) ) MDSPAN_INLINE_FUNCTION _MDSPAN_DEDUCE_RETURN_TYPE_SINGLE_LINE( ( constexpr submdspan( - mdspan, LP, AP> const& src, SliceSpecs... slices + mdspan const& src, SliceSpecs... slices ) noexcept ), ( diff --git a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/type_list.hpp b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/type_list.hpp index 6683a22bf9..7de72e6537 100644 --- a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/type_list.hpp +++ b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p0009_bits/type_list.hpp @@ -113,4 +113,5 @@ struct __type_at<3, __type_list<_T0, _T1, _T2, _T3, _Ts...>> { //============================================================================== } // end namespace experimental -} // end namespace std \ No newline at end of file +} // end namespace std + diff --git a/cpp/include/raft/thirdparty/mdspan/include/experimental/__p1684_bits/mdarray.hpp b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p1684_bits/mdarray.hpp new file mode 100644 index 0000000000..b27a25c9fe --- /dev/null +++ b/cpp/include/raft/thirdparty/mdspan/include/experimental/__p1684_bits/mdarray.hpp @@ -0,0 +1,463 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos v. 2.0 +// Copyright (2019) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Christian R. Trott (crtrott@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + + +#pragma once + +#include "../mdspan" +#include +#include + +namespace std { +namespace experimental { + +namespace { + template + struct size_of_extents; + + template + struct size_of_extents> { + constexpr static size_t value() { + size_t size = 1; + for(size_t r=0; r::rank(); r++) + size *= extents::static_extent(r); + return size; + } + }; +} + +namespace { + template + struct container_is_array : false_type { + template + static constexpr C construct(const M& m) { return C(m.required_span_size()); } + }; + template + struct container_is_array> : true_type { + template + static constexpr array construct(const M&) { return array(); } + }; +} + +template < + class ElementType, + class Extents, + class LayoutPolicy = layout_right, + class Container = vector +> +class mdarray { +private: + static_assert(detail::__is_extents_v, "std::experimental::mdspan's Extents template parameter must be a specialization of std::experimental::extents."); + + +public: + + //-------------------------------------------------------------------------------- + // Domain and codomain types + + using extents_type = Extents; + using layout_type = LayoutPolicy; + using container_type = Container; + using mapping_type = typename layout_type::template mapping; + using element_type = ElementType; + using value_type = remove_cv_t; + using index_type = typename Extents::index_type; + using pointer = typename container_type::pointer; + using reference = typename container_type::reference; + using const_pointer = typename container_type::const_pointer; + using const_reference = typename container_type::const_reference; + +public: + + //-------------------------------------------------------------------------------- + // [mdspan.basic.cons], mdspan constructors, assignment, and destructor + +#if !(MDSPAN_HAS_CXX_20) + MDSPAN_FUNCTION_REQUIRES( + (MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr), + mdarray, (), , + /* requires */ (extents_type::rank_dynamic()!=0)) {} +#else + MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mdarray() requires(extents_type::rank_dynamic()!=0) = default; +#endif + MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mdarray(const mdarray&) = default; + MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mdarray(mdarray&&) = default; + + // Constructors for container types constructible from a size + MDSPAN_TEMPLATE_REQUIRES( + class... SizeTypes, + /* requires */ ( + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, SizeTypes, index_type) /* && ... */) && + _MDSPAN_TRAIT(is_constructible, extents_type, SizeTypes...) && + _MDSPAN_TRAIT(is_constructible, mapping_type, extents_type) && + (_MDSPAN_TRAIT(is_constructible, container_type, size_t) || + container_is_array::value) && + (extents_type::rank()>0 || extents_type::rank_dynamic()==0) + ) + ) + MDSPAN_INLINE_FUNCTION + explicit constexpr mdarray(SizeTypes... dynamic_extents) + : map_(extents_type(dynamic_extents...)), ctr_(container_is_array::construct(map_)) + { } + + MDSPAN_FUNCTION_REQUIRES( + (MDSPAN_INLINE_FUNCTION constexpr), + mdarray, (const extents_type& exts), , + /* requires */ ((_MDSPAN_TRAIT(is_constructible, container_type, size_t) || + container_is_array::value) && + _MDSPAN_TRAIT(is_constructible, mapping_type, extents_type)) + ) : map_(exts), ctr_(container_is_array::construct(map_)) + { } + + MDSPAN_FUNCTION_REQUIRES( + (MDSPAN_INLINE_FUNCTION constexpr), + mdarray, (const mapping_type& m), , + /* requires */ (_MDSPAN_TRAIT(is_constructible, container_type, size_t) || + container_is_array::value) + ) : map_(m), ctr_(container_is_array::construct(map_)) + { } + + // Constructors from container + MDSPAN_TEMPLATE_REQUIRES( + class... SizeTypes, + /* requires */ ( + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, SizeTypes, index_type) /* && ... */) && + _MDSPAN_TRAIT(is_constructible, extents_type, SizeTypes...) && + _MDSPAN_TRAIT(is_constructible, mapping_type, extents_type) + ) + ) + MDSPAN_INLINE_FUNCTION + explicit constexpr mdarray(const container_type& ctr, SizeTypes... dynamic_extents) + : map_(extents_type(dynamic_extents...)), ctr_(ctr) + { assert(ctr.size() >= static_cast(map_.required_span_size())); } + + + MDSPAN_FUNCTION_REQUIRES( + (MDSPAN_INLINE_FUNCTION constexpr), + mdarray, (const container_type& ctr, const extents_type& exts), , + /* requires */ (_MDSPAN_TRAIT(is_constructible, mapping_type, extents_type)) + ) : map_(exts), ctr_(ctr) + { assert(ctr.size() >= static_cast(map_.required_span_size())); } + + constexpr mdarray(const container_type& ctr, const mapping_type& m) + : map_(m), ctr_(ctr) + { assert(ctr.size() >= static_cast(map_.required_span_size())); } + + + // Constructors from container + MDSPAN_TEMPLATE_REQUIRES( + class... SizeTypes, + /* requires */ ( + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, SizeTypes, index_type) /* && ... */) && + _MDSPAN_TRAIT(is_constructible, extents_type, SizeTypes...) && + _MDSPAN_TRAIT(is_constructible, mapping_type, extents_type) + ) + ) + MDSPAN_INLINE_FUNCTION + explicit constexpr mdarray(container_type&& ctr, SizeTypes... dynamic_extents) + : map_(extents_type(dynamic_extents...)), ctr_(std::move(ctr)) + { assert(ctr_.size() >= static_cast(map_.required_span_size())); } + + + MDSPAN_FUNCTION_REQUIRES( + (MDSPAN_INLINE_FUNCTION constexpr), + mdarray, (container_type&& ctr, const extents_type& exts), , + /* requires */ (_MDSPAN_TRAIT(is_constructible, mapping_type, extents_type)) + ) : map_(exts), ctr_(std::move(ctr)) + { assert(ctr_.size() >= static_cast(map_.required_span_size())); } + + constexpr mdarray(container_type&& ctr, const mapping_type& m) + : map_(m), ctr_(std::move(ctr)) + { assert(ctr_.size() >= static_cast(map_.required_span_size())); } + + + + MDSPAN_TEMPLATE_REQUIRES( + class OtherElementType, class OtherExtents, class OtherLayoutPolicy, class OtherContainer, + /* requires */ ( + _MDSPAN_TRAIT(is_constructible, mapping_type, typename OtherLayoutPolicy::template mapping) && + _MDSPAN_TRAIT(is_constructible, container_type, OtherContainer) + ) + ) + MDSPAN_INLINE_FUNCTION + constexpr mdarray(const mdarray& other) + : map_(other.mapping()), ctr_(other.container()) + { + static_assert(is_constructible::value, ""); + } + + // Constructors for container types constructible from a size and allocator + MDSPAN_TEMPLATE_REQUIRES( + class Alloc, + /* requires */ (_MDSPAN_TRAIT(is_constructible, container_type, size_t, Alloc) && + _MDSPAN_TRAIT(is_constructible, mapping_type, extents_type)) + ) + MDSPAN_INLINE_FUNCTION + constexpr mdarray(const extents_type& exts, const Alloc& a) + : map_(exts), ctr_(map_.required_span_size(), a) + { } + + MDSPAN_TEMPLATE_REQUIRES( + class Alloc, + /* requires */ (_MDSPAN_TRAIT(is_constructible, container_type, size_t, Alloc)) + ) + MDSPAN_INLINE_FUNCTION + constexpr mdarray(const mapping_type& map, const Alloc& a) + : map_(map), ctr_(map_.required_span_size(), a) + { } + + // Constructors for container types constructible from a container and allocator + MDSPAN_TEMPLATE_REQUIRES( + class Alloc, + /* requires */ (_MDSPAN_TRAIT(is_constructible, container_type, container_type, Alloc) && + _MDSPAN_TRAIT(is_constructible, mapping_type, extents_type)) + ) + MDSPAN_INLINE_FUNCTION + constexpr mdarray(const container_type& ctr, const extents_type& exts, const Alloc& a) + : map_(exts), ctr_(ctr, a) + { assert(ctr_.size() >= static_cast(map_.required_span_size())); } + + MDSPAN_TEMPLATE_REQUIRES( + class Alloc, + /* requires */ (_MDSPAN_TRAIT(is_constructible, container_type, size_t, Alloc)) + ) + MDSPAN_INLINE_FUNCTION + constexpr mdarray(const container_type& ctr, const mapping_type& map, const Alloc& a) + : map_(map), ctr_(ctr, a) + { assert(ctr_.size() >= static_cast(map_.required_span_size())); } + + MDSPAN_TEMPLATE_REQUIRES( + class Alloc, + /* requires */ (_MDSPAN_TRAIT(is_constructible, container_type, container_type, Alloc) && + _MDSPAN_TRAIT(is_constructible, mapping_type, extents_type)) + ) + MDSPAN_INLINE_FUNCTION + constexpr mdarray(container_type&& ctr, const extents_type& exts, const Alloc& a) + : map_(exts), ctr_(std::move(ctr), a) + { assert(ctr_.size() >= static_cast(map_.required_span_size())); } + + MDSPAN_TEMPLATE_REQUIRES( + class Alloc, + /* requires */ (_MDSPAN_TRAIT(is_constructible, container_type, size_t, Alloc)) + ) + MDSPAN_INLINE_FUNCTION + constexpr mdarray(container_type&& ctr, const mapping_type& map, const Alloc& a) + : map_(map), ctr_(std::move(ctr), a) + { assert(ctr_.size() >= map_.required_span_size()); } + + MDSPAN_TEMPLATE_REQUIRES( + class OtherElementType, class OtherExtents, class OtherLayoutPolicy, class OtherContainer, class Alloc, + /* requires */ ( + _MDSPAN_TRAIT(is_constructible, mapping_type, typename OtherLayoutPolicy::template mapping) && + _MDSPAN_TRAIT(is_constructible, container_type, OtherContainer, Alloc) + ) + ) + MDSPAN_INLINE_FUNCTION + constexpr mdarray(const mdarray& other, const Alloc& a) + : map_(other.mapping()), ctr_(other.container(), a) + { + static_assert(is_constructible::value, ""); + } + + MDSPAN_INLINE_FUNCTION_DEFAULTED + ~mdarray() = default; + + //-------------------------------------------------------------------------------- + // [mdspan.basic.mapping], mdspan mapping domain multidimensional index to access codomain element + + #if MDSPAN_USE_BRACKET_OPERATOR + MDSPAN_TEMPLATE_REQUIRES( + class... SizeTypes, + /* requires */ ( + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, SizeTypes, index_type) /* && ... */) && + extents_type::rank() == sizeof...(SizeTypes) + ) + ) + MDSPAN_FORCE_INLINE_FUNCTION + constexpr const_reference operator[](SizeTypes... indices) const noexcept + { + return ctr_[map_(index_type(indices)...)]; + } + + MDSPAN_TEMPLATE_REQUIRES( + class... SizeTypes, + /* requires */ ( + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, SizeTypes, index_type) /* && ... */) && + extents_type::rank() == sizeof...(SizeTypes) + ) + ) + MDSPAN_FORCE_INLINE_FUNCTION + constexpr reference operator[](SizeTypes... indices) noexcept + { + return ctr_[map_(index_type(indices)...)]; + } + #endif + +#if 0 + MDSPAN_TEMPLATE_REQUIRES( + class SizeType, size_t N, + /* requires */ ( + _MDSPAN_TRAIT(is_convertible, SizeType, index_type) && + N == extents_type::rank() + ) + ) + MDSPAN_FORCE_INLINE_FUNCTION + constexpr const_reference operator[](const array& indices) const noexcept + { + return __impl::template __callop(*this, indices); + } + + MDSPAN_TEMPLATE_REQUIRES( + class SizeType, size_t N, + /* requires */ ( + _MDSPAN_TRAIT(is_convertible, SizeType, index_type) && + N == extents_type::rank() + ) + ) + MDSPAN_FORCE_INLINE_FUNCTION + constexpr reference operator[](const array& indices) noexcept + { + return __impl::template __callop(*this, indices); + } +#endif + + + #if MDSPAN_USE_PAREN_OPERATOR + MDSPAN_TEMPLATE_REQUIRES( + class... SizeTypes, + /* requires */ ( + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, SizeTypes, index_type) /* && ... */) && + extents_type::rank() == sizeof...(SizeTypes) + ) + ) + MDSPAN_FORCE_INLINE_FUNCTION + constexpr const_reference operator()(SizeTypes... indices) const noexcept + { + return ctr_[map_(index_type(indices)...)]; + } + MDSPAN_TEMPLATE_REQUIRES( + class... SizeTypes, + /* requires */ ( + _MDSPAN_FOLD_AND(_MDSPAN_TRAIT(is_convertible, SizeTypes, index_type) /* && ... */) && + extents_type::rank() == sizeof...(SizeTypes) + ) + ) + MDSPAN_FORCE_INLINE_FUNCTION + constexpr reference operator()(SizeTypes... indices) noexcept + { + return ctr_[map_(index_type(indices)...)]; + } + +#if 0 + MDSPAN_TEMPLATE_REQUIRES( + class SizeType, size_t N, + /* requires */ ( + _MDSPAN_TRAIT(is_convertible, SizeType, index_type) && + N == extents_type::rank() + ) + ) + MDSPAN_FORCE_INLINE_FUNCTION + constexpr const_reference operator()(const array& indices) const noexcept + { + return __impl::template __callop(*this, indices); + } + + MDSPAN_TEMPLATE_REQUIRES( + class SizeType, size_t N, + /* requires */ ( + _MDSPAN_TRAIT(is_convertible, SizeType, index_type) && + N == extents_type::rank() + ) + ) + MDSPAN_FORCE_INLINE_FUNCTION + constexpr reference operator()(const array& indices) noexcept + { + return __impl::template __callop(*this, indices); + } +#endif + #endif + + MDSPAN_INLINE_FUNCTION constexpr pointer data() noexcept { return ctr_.data(); }; + MDSPAN_INLINE_FUNCTION constexpr const_pointer data() const noexcept { return ctr_.data(); }; + MDSPAN_INLINE_FUNCTION constexpr container_type& container() noexcept { return ctr_; }; + MDSPAN_INLINE_FUNCTION constexpr const container_type& container() const noexcept { return ctr_; }; + + //-------------------------------------------------------------------------------- + // [mdspan.basic.domobs], mdspan observers of the domain multidimensional index space + + MDSPAN_INLINE_FUNCTION static constexpr size_t rank() noexcept { return extents_type::rank(); } + MDSPAN_INLINE_FUNCTION static constexpr size_t rank_dynamic() noexcept { return extents_type::rank_dynamic(); } + MDSPAN_INLINE_FUNCTION static constexpr index_type static_extent(size_t r) noexcept { return extents_type::static_extent(r); } + + MDSPAN_INLINE_FUNCTION constexpr extents_type extents() const noexcept { return map_.extents(); }; + MDSPAN_INLINE_FUNCTION constexpr index_type extent(size_t r) const noexcept { return map_.extents().extent(r); }; + MDSPAN_INLINE_FUNCTION constexpr index_type size() const noexcept { +// return __impl::__size(*this); + return ctr_.size(); + }; + + + //-------------------------------------------------------------------------------- + // [mdspan.basic.obs], mdspan observers of the mapping + + MDSPAN_INLINE_FUNCTION static constexpr bool is_always_unique() noexcept { return mapping_type::is_always_unique(); }; + MDSPAN_INLINE_FUNCTION static constexpr bool is_always_exhaustive() noexcept { return mapping_type::is_always_exhaustive(); }; + MDSPAN_INLINE_FUNCTION static constexpr bool is_always_strided() noexcept { return mapping_type::is_always_strided(); }; + + MDSPAN_INLINE_FUNCTION constexpr mapping_type mapping() const noexcept { return map_; }; + MDSPAN_INLINE_FUNCTION constexpr bool is_unique() const noexcept { return map_.is_unique(); }; + MDSPAN_INLINE_FUNCTION constexpr bool is_exhaustive() const noexcept { return map_.is_exhaustive(); }; + MDSPAN_INLINE_FUNCTION constexpr bool is_strided() const noexcept { return map_.is_strided(); }; + MDSPAN_INLINE_FUNCTION constexpr index_type stride(size_t r) const { return map_.stride(r); }; + +private: + mapping_type map_; + container_type ctr_; + + template + friend class mdarray; +}; + + +} // end namespace experimental +} // end namespace std diff --git a/cpp/include/raft/thirdparty/mdspan/include/experimental/mdarray b/cpp/include/raft/thirdparty/mdspan/include/experimental/mdarray new file mode 100644 index 0000000000..fa710a59b6 --- /dev/null +++ b/cpp/include/raft/thirdparty/mdspan/include/experimental/mdarray @@ -0,0 +1,48 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos v. 2.0 +// Copyright (2019) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Christian R. Trott (crtrott@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#pragma once + +#include "mdspan" +#include "__p1684_bits/mdarray.hpp" + diff --git a/cpp/include/raft/thirdparty/mdspan/tests/CMakeLists.txt b/cpp/include/raft/thirdparty/mdspan/tests/CMakeLists.txt index 7cf7009edb..d92834beb7 100644 --- a/cpp/include/raft/thirdparty/mdspan/tests/CMakeLists.txt +++ b/cpp/include/raft/thirdparty/mdspan/tests/CMakeLists.txt @@ -15,40 +15,46 @@ macro(mdspan_add_test name) add_test(${name} ${name}) endmacro() -# adapted from https://github.com/google/googletest/blob/master/googletest/README.md -configure_file(${PROJECT_SOURCE_DIR}/cmake/googletest/CMakeLists.txt.in googletest-download/CMakeLists.txt) -execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . - RESULT_VARIABLE result - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download -) -if(result) - message(FATAL_ERROR "CMake step for googletest failed: ${result}") -endif() -execute_process(COMMAND ${CMAKE_COMMAND} --build . - RESULT_VARIABLE result - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download -) -if(result) - message(FATAL_ERROR "Build step for googletest failed: ${result}") -endif() +if(MDSPAN_USE_SYSTEM_GTEST) + find_package(GTest CONFIG REQUIRED) + add_library(gtest_main ALIAS GTest::gtest_main) +else() + # adapted from https://github.com/google/googletest/blob/master/googletest/README.md + configure_file(${PROJECT_SOURCE_DIR}/cmake/googletest/CMakeLists.txt.in googletest-download/CMakeLists.txt) + execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . + RESULT_VARIABLE result + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download + ) + if(result) + message(FATAL_ERROR "CMake step for googletest failed: ${result}") + endif() + execute_process(COMMAND ${CMAKE_COMMAND} --build . + RESULT_VARIABLE result + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download + ) + if(result) + message(FATAL_ERROR "Build step for googletest failed: ${result}") + endif() -# Prevent overriding the parent project's compiler/linker -# settings on Windows -set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + # Prevent overriding the parent project's compiler/linker + # settings on Windows + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) -# Add googletest directly to our build. This defines -# the gtest and gtest_main targets. -add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src - ${CMAKE_CURRENT_BINARY_DIR}/googletest-build - EXCLUDE_FROM_ALL -) + # Add googletest directly to our build. This defines + # the gtest and gtest_main targets. + add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src + ${CMAKE_CURRENT_BINARY_DIR}/googletest-build + EXCLUDE_FROM_ALL + ) +endif() mdspan_add_test(test_extents) mdspan_add_test(test_mdspan_ctors) mdspan_add_test(test_mdspan_conversion) mdspan_add_test(test_element_access) -mdspan_add_test(test_contiguous_layouts) +mdspan_add_test(test_exhaustive_layouts) mdspan_add_test(test_layout_ctors) mdspan_add_test(test_layout_stride) mdspan_add_test(test_submdspan) +mdspan_add_test(test_mdarray_ctors) diff --git a/cpp/include/raft/thirdparty/mdspan/tests/offload_utils.hpp b/cpp/include/raft/thirdparty/mdspan/tests/offload_utils.hpp index 645eb8e2cc..7c746337c6 100644 --- a/cpp/include/raft/thirdparty/mdspan/tests/offload_utils.hpp +++ b/cpp/include/raft/thirdparty/mdspan/tests/offload_utils.hpp @@ -1,6 +1,13 @@ namespace { bool dispatch_host = true; + +#define __MDSPAN_DEVICE_ASSERT_EQ(LHS, RHS) \ +if (!(LHS == RHS)) { \ + printf("expected equality of %s and %s\n", #LHS, #RHS); \ + errors[0]++; \ +} + #ifdef _MDSPAN_HAS_CUDA template @@ -43,7 +50,7 @@ void free_array(T* ptr) { A; #define __MDSPAN_TESTS_DISPATCH_DEFINED -#endif +#endif // _MDSPAN_HAS_CUDA #ifndef __MDSPAN_TESTS_DISPATCH_DEFINED template diff --git a/cpp/include/raft/thirdparty/mdspan/tests/test_contiguous_layouts.cpp b/cpp/include/raft/thirdparty/mdspan/tests/test_contiguous_layouts.cpp deleted file mode 100644 index de39f8dc51..0000000000 --- a/cpp/include/raft/thirdparty/mdspan/tests/test_contiguous_layouts.cpp +++ /dev/null @@ -1,373 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 2.0 -// Copyright (2019) Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#include - -#include - -#include -#include - -namespace stdex = std::experimental; -_MDSPAN_INLINE_VARIABLE constexpr auto dyn = stdex::dynamic_extent; - -template -size_t get_expected_mapping( - typename stdex::layout_left::template mapping const& map, - size_t i, size_t j, size_t k -) -{ - auto const& extents = map.extents(); - return i + j*extents.extent(0) + k*extents.extent(0)*extents.extent(1); -} - -template -size_t get_expected_mapping( - typename stdex::layout_right::template mapping const& map, - size_t i, size_t j, size_t k -) -{ - auto const& extents = map.extents(); - return k + j*extents.extent(2) + i*extents.extent(1)*extents.extent(2); -} - -template struct TestLayout; - -template -struct TestLayout ->> : public ::testing::Test -{ - Mapping map; - - void initialize_mapping() { - using extents_type = decltype(map.extents()); - map = Mapping(extents_type(DynamicSizes...)); - } - - void SetUp() override { - initialize_mapping(); - } - - template - size_t expected_mapping(Index... idxs) const { - return get_expected_mapping(map, idxs...); - } -}; - -template -using test_3d_left_types = std::tuple< - typename stdex::layout_left::template mapping, - std::integer_sequence ->; - -template -using test_3d_right_types = std::tuple< - typename stdex::layout_right::template mapping, - std::integer_sequence ->; - -using layout_test_types_3d = - ::testing::Types< - test_3d_left_types>, - test_3d_left_types>, - test_3d_left_types, 5>, - test_3d_left_types, 3>, - test_3d_left_types, 4>, - test_3d_left_types, 4>, - test_3d_left_types, 3>, - test_3d_left_types, 5>, - test_3d_left_types, 3, 4>, - test_3d_left_types, 5, 4>, - test_3d_left_types, 3, 5>, - test_3d_left_types, 5, 3>, - test_3d_left_types, 4, 5>, - test_3d_left_types, 4, 3>, - test_3d_left_types, 3, 4, 5>, - test_3d_left_types, 5, 4, 3>, - test_3d_right_types>, - test_3d_right_types>, - test_3d_right_types, 5>, - test_3d_right_types, 3>, - test_3d_right_types, 4>, - test_3d_right_types, 4>, - test_3d_right_types, 3>, - test_3d_right_types, 5>, - test_3d_right_types, 3, 4>, - test_3d_right_types, 5, 4>, - test_3d_right_types, 3, 5>, - test_3d_right_types, 5, 3>, - test_3d_right_types, 4, 5>, - test_3d_right_types, 4, 3>, - test_3d_right_types, 3, 4, 5>, - test_3d_right_types, 5, 4, 3> - >; - -template struct TestLayout3D : TestLayout { }; - -TYPED_TEST_SUITE(TestLayout3D, layout_test_types_3d); - -TYPED_TEST(TestLayout3D, mapping_works) { - for(size_t i = 0; i < this->map.extents().extent(0); ++i) { - for(size_t j = 0; j < this->map.extents().extent(1); ++j) { - for(size_t k = 0; k < this->map.extents().extent(2); ++k) { - EXPECT_EQ(this->map(i, j, k), this->expected_mapping(i, j, k)) - << "Incorrect mapping for index " << i << ", " << j << ", " << k - << " with extents " - << this->map.extents().extent(0) << ", " - << this->map.extents().extent(1) << ", " - << this->map.extents().extent(2) << "."; - } - } - } -} - -TYPED_TEST(TestLayout3D, required_span_size_works) { - ASSERT_EQ(this->map.required_span_size(), 3*4*5); -} - -template -using test_layout_conversion = - std::tuple, - std::integral_constant>; - - -using layout_conversion_test_types = - ::testing::Types< - // Left <=> Left conversion - test_layout_conversion, stdex::extents<5>, true, 5> - ,test_layout_conversion, stdex::extents<>, true> - ,test_layout_conversion, stdex::extents, false, 5> - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,dyn>, true, 5,0> // intentional 0 here to test degenerated - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,dyn>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, true, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,15,20,25>, true, 5, 10, 15, 20, 25> - - // Right <=> Right conversion - ,test_layout_conversion, stdex::extents<5>, true, 5> - ,test_layout_conversion, stdex::extents<>, true> - ,test_layout_conversion, stdex::extents, false, 5> - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,dyn>, true, 5,0> //intentional 0 to test degenerate - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,dyn>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, true, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,15,20,25>, true, 5, 10, 15, 20, 25> - - // Stride <=> Stride conversion - ,test_layout_conversion, stdex::extents<5>, true, 5> - ,test_layout_conversion, stdex::extents<>, true> - ,test_layout_conversion, stdex::extents, false, 5> - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,dyn>, true, 5,0> //intentional 0 to test degenerate - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,dyn>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, true, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,15,20,25>, true, 5, 10, 15, 20, 25> - - // Right => Stride conversion - ,test_layout_conversion, stdex::extents<5>, true, 5> - ,test_layout_conversion, stdex::extents<>, true> - ,test_layout_conversion, stdex::extents, false, 5> - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,dyn>, true, 5,0> //intentional 0 to test degenerate - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,dyn>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, true, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,15,20,25>, true, 5, 10, 15, 20, 25> - - // Left => Stride conversion - ,test_layout_conversion, stdex::extents<5>, true, 5> - ,test_layout_conversion, stdex::extents<>, true> - ,test_layout_conversion, stdex::extents, false, 5> - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,dyn>, true, 5,0> //intentional 0 to test degenerate - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,10>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,dyn>, true, 5,10> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, true, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,15,20,25>, true, 5, 10, 15, 20, 25> - - // Stride => Left conversion - ,test_layout_conversion, stdex::extents<5>, false, 5> - ,test_layout_conversion, stdex::extents<>, true> - ,test_layout_conversion, stdex::extents, false, 5> - ,test_layout_conversion, stdex::extents<5,10>, false, 5,10> - ,test_layout_conversion, stdex::extents<5,dyn>, false, 5,0> //intentional 0 to test degenerate - ,test_layout_conversion, stdex::extents<5,10>, false, 5,10> - ,test_layout_conversion, stdex::extents<5,10>, false, 5,10> - ,test_layout_conversion, stdex::extents<5,dyn>, false, 5,10> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,15,20,25>, false, 5, 10, 15, 20, 25> - - // Stride => Right conversion - ,test_layout_conversion, stdex::extents<5>, false, 5> - ,test_layout_conversion, stdex::extents<>, true> - ,test_layout_conversion, stdex::extents, false, 5> - ,test_layout_conversion, stdex::extents<5,10>, false, 5,10> - ,test_layout_conversion, stdex::extents<5,dyn>, false, 5,0> //intentional 0 to test degenerate - ,test_layout_conversion, stdex::extents<5,10>, false, 5,10> - ,test_layout_conversion, stdex::extents<5,10>, false, 5,10> - ,test_layout_conversion, stdex::extents<5,dyn>, false, 5,10> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,dyn,20,dyn>, false, 5, 10, 15, 20, 25> - ,test_layout_conversion, stdex::extents<5,10,15,20,25>, false, 5, 10, 15, 20, 25> - - // Left <=> Right conversion - ,test_layout_conversion, stdex::extents<5>, true, 5> - ,test_layout_conversion, stdex::extents, false, 5> - ,test_layout_conversion, stdex::extents<5>, true, 5> - ,test_layout_conversion, stdex::extents, false, 5> - ,test_layout_conversion, stdex::extents<>, true> - ,test_layout_conversion, stdex::extents<>, true> - - >; - -template struct TestLayoutConversion; - -template -struct TestLayoutConversion< - std::tuple, - std::integral_constant>> : public ::testing::Test { - static constexpr bool implicit = Implicit; - using layout_1_t = Layout1; - using exts_1_t = Extents1; - using map_1_t = typename Layout1::template mapping; - using layout_2_t = Layout2; - using exts_2_t = Extents2; - using map_2_t = typename Layout2::template mapping; - exts_1_t exts1 { AllSizes... }; - exts_2_t exts2 { AllSizes... }; - - map_1_t implicit_conv(map_1_t map) const { - return map; - } - - template - static constexpr std::enable_if_t< - std::is_same::value && - !std::is_same::value && - std::is_same::value, - map_2_t> create_map2(Extents exts) { - return map_2_t(map_1_t(Extents1(exts))); - } - - template - static constexpr std::enable_if_t< - std::is_same::value && - std::is_same::value && - std::is_same::value, - map_2_t> create_map2(Extents exts) { - return map_2_t(typename stdex::layout_left::template mapping(exts)); - } - - template - static constexpr std::enable_if_t< - std::is_same::value && - !std::is_same::value, - map_2_t> create_map2(Extents exts) { - return map_2_t(exts); - } -}; - -TYPED_TEST_SUITE(TestLayoutConversion, layout_conversion_test_types); - -TYPED_TEST(TestLayoutConversion, implicit_conversion) { - typename TestFixture::map_2_t map2 = this->create_map2(this->exts2); - typename TestFixture::map_1_t map1; - #if MDSPAN_HAS_CXX_20 && !defined(_MDSPAN_COMPILER_MSVC) - static_assert(TestFixture::implicit == - ( - ( - !std::is_same::value && - std::is_convertible::value - ) || ( - std::is_same::value && - std::is_same::value && - std::is_convertible::value - )|| ( - std::is_same::value && - !std::is_same::value && - TestFixture::exts_2_t::rank()==0 - ) - ) - ); - static_assert(std::is_convertible::value == - TestFixture::implicit); - if constexpr(TestFixture::implicit) - map1 = this->implicit_conv(map2); - else - #endif - map1 = typename TestFixture::map_1_t(map2); - - for(int r=0; rexts1.rank(); r++) { - ASSERT_EQ(map1.extents().extent(r), map2.extents().extent(r)); - ASSERT_EQ(map1.stride(r), map2.stride(r)); - } -} - diff --git a/cpp/include/raft/thirdparty/mdspan/tests/test_element_access.cpp b/cpp/include/raft/thirdparty/mdspan/tests/test_element_access.cpp index 400f64c4f1..d1642e83e0 100644 --- a/cpp/include/raft/thirdparty/mdspan/tests/test_element_access.cpp +++ b/cpp/include/raft/thirdparty/mdspan/tests/test_element_access.cpp @@ -50,7 +50,7 @@ namespace stdex = std::experimental; TEST(TestElementAccess, element_access_with_std_array) { std::array a{}; - stdex::mdspan> s(a.data()); + stdex::mdspan> s(a.data()); ASSERT_EQ(__MDSPAN_OP(s, (std::array{1, 2})), 0); __MDSPAN_OP(s, (std::array{0, 1})) = 3.14; ASSERT_EQ(__MDSPAN_OP(s, (std::array{0, 1})), 3.14); diff --git a/cpp/include/raft/thirdparty/mdspan/tests/test_exhaustive_layouts.cpp b/cpp/include/raft/thirdparty/mdspan/tests/test_exhaustive_layouts.cpp new file mode 100644 index 0000000000..f09b799684 --- /dev/null +++ b/cpp/include/raft/thirdparty/mdspan/tests/test_exhaustive_layouts.cpp @@ -0,0 +1,427 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos v. 2.0 +// Copyright (2019) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Christian R. Trott (crtrott@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include + +#include + +#include +#include + +namespace stdex = std::experimental; +_MDSPAN_INLINE_VARIABLE constexpr auto dyn = stdex::dynamic_extent; + +template +size_t get_expected_mapping( + typename stdex::layout_left::template mapping const& map, + size_t i, size_t j, size_t k +) +{ + auto const& extents = map.extents(); + return i + j*extents.extent(0) + k*extents.extent(0)*extents.extent(1); +} + +template +size_t get_expected_mapping( + typename stdex::layout_right::template mapping const& map, + size_t i, size_t j, size_t k +) +{ + auto const& extents = map.extents(); + return k + j*extents.extent(2) + i*extents.extent(1)*extents.extent(2); +} + +template struct TestLayout; + +template +struct TestLayout +>> : public ::testing::Test +{ + Mapping map; + + void initialize_mapping() { + using extents_type = std::remove_cv_t>; + map = Mapping(extents_type(DynamicSizes...)); + } + + void SetUp() override { + initialize_mapping(); + } + + template + size_t expected_mapping(Index... idxs) const { + return get_expected_mapping(map, idxs...); + } +}; + +template +using test_3d_left_types = std::tuple< + typename stdex::layout_left::template mapping, + std::integer_sequence +>; + +template +using test_3d_right_types = std::tuple< + typename stdex::layout_right::template mapping, + std::integer_sequence +>; + +using layout_test_types_3d = + ::testing::Types< + test_3d_left_types>, + test_3d_left_types>, + test_3d_left_types, 5>, + test_3d_left_types, 3>, + test_3d_left_types, 4>, + test_3d_left_types, 4>, + test_3d_left_types, 3>, + test_3d_left_types, 5>, + test_3d_left_types, 3, 4>, + test_3d_left_types, 5, 4>, + test_3d_left_types, 3, 5>, + test_3d_left_types, 5, 3>, + test_3d_left_types, 4, 5>, + test_3d_left_types, 4, 3>, + test_3d_left_types, 3, 4, 5>, + test_3d_left_types, 5, 4, 3>, + test_3d_right_types>, + test_3d_right_types>, + test_3d_right_types, 5>, + test_3d_right_types, 3>, + test_3d_right_types, 4>, + test_3d_right_types, 4>, + test_3d_right_types, 3>, + test_3d_right_types, 5>, + test_3d_right_types, 3, 4>, + test_3d_right_types, 5, 4>, + test_3d_right_types, 3, 5>, + test_3d_right_types, 5, 3>, + test_3d_right_types, 4, 5>, + test_3d_right_types, 4, 3>, + test_3d_right_types, 3, 4, 5>, + test_3d_right_types, 5, 4, 3> + >; + +template struct TestLayout3D : TestLayout { }; + +TYPED_TEST_SUITE(TestLayout3D, layout_test_types_3d); + +TYPED_TEST(TestLayout3D, mapping_works) { + for(size_t i = 0; i < this->map.extents().extent(0); ++i) { + for(size_t j = 0; j < this->map.extents().extent(1); ++j) { + for(size_t k = 0; k < this->map.extents().extent(2); ++k) { + EXPECT_EQ(this->map(i, j, k), this->expected_mapping(i, j, k)) + << "Incorrect mapping for index " << i << ", " << j << ", " << k + << " with extents " + << this->map.extents().extent(0) << ", " + << this->map.extents().extent(1) << ", " + << this->map.extents().extent(2) << "."; + } + } + } +} + +TYPED_TEST(TestLayout3D, required_span_size_works) { + ASSERT_EQ(this->map.required_span_size(), 3*4*5); +} + +template +using test_layout_conversion = + std::tuple, + std::integral_constant>; + +// We separate layout_conversion_test_types into multiple type lists +// in order to avoid an nvc++ compiler error (recursion too deep). + +// Left <=> Left conversion +using layout_conversion_test_types_left_to_left = + ::testing::Types< + test_layout_conversion, stdex::extents, true, 5> + ,test_layout_conversion, stdex::extents, true> + ,test_layout_conversion, stdex::extents, false, 5> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,0> // intentional 0 here to test degenerated + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, true, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, true, 5, 10, 15, 20, 25> + >; + +// Right <=> Right conversion +using layout_conversion_test_types_right_to_right = + ::testing::Types< + test_layout_conversion, stdex::extents, true, 5> + ,test_layout_conversion, stdex::extents, true> + ,test_layout_conversion, stdex::extents, false, 5> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,0> //intentional 0 to test degenerate + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, true, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, true, 5, 10, 15, 20, 25> + >; + +// Stride <=> Stride conversion +using layout_conversion_test_types_stride_to_stride = + ::testing::Types< + test_layout_conversion, stdex::extents, true, 5> + ,test_layout_conversion, stdex::extents, true> + ,test_layout_conversion, stdex::extents, false, 5> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,0> //intentional 0 to test degenerate + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, true, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, true, 5, 10, 15, 20, 25> + >; + +// Right => Stride conversion +using layout_conversion_test_types_right_to_stride = + ::testing::Types< + test_layout_conversion, stdex::extents, true, 5> + ,test_layout_conversion, stdex::extents, true> + ,test_layout_conversion, stdex::extents, false, 5> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,0> //intentional 0 to test degenerate + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, true, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, true, 5, 10, 15, 20, 25> + >; + +// Left => Stride conversion +using layout_conversion_test_types_left_to_stride = + ::testing::Types< + test_layout_conversion, stdex::extents, true, 5> + ,test_layout_conversion, stdex::extents, true> + ,test_layout_conversion, stdex::extents, false, 5> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,0> //intentional 0 to test degenerate + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, true, 5,10> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, true, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, true, 5, 10, 15, 20, 25> + >; + +// Stride => Left conversion +using layout_conversion_test_types_stride_to_left = + ::testing::Types< + test_layout_conversion, stdex::extents, false, 5> + ,test_layout_conversion, stdex::extents, true> + ,test_layout_conversion, stdex::extents, false, 5> + ,test_layout_conversion, stdex::extents, false, 5,10> + ,test_layout_conversion, stdex::extents, false, 5,0> //intentional 0 to test degenerate + ,test_layout_conversion, stdex::extents, false, 5,10> + ,test_layout_conversion, stdex::extents, false, 5,10> + ,test_layout_conversion, stdex::extents, false, 5,10> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + >; + +// Stride => Right conversion +using layout_conversion_test_types_stride_to_right = + ::testing::Types< + test_layout_conversion, stdex::extents, false, 5> + ,test_layout_conversion, stdex::extents, true> + ,test_layout_conversion, stdex::extents, false, 5> + ,test_layout_conversion, stdex::extents, false, 5,10> + ,test_layout_conversion, stdex::extents, false, 5,0> //intentional 0 to test degenerate + ,test_layout_conversion, stdex::extents, false, 5,10> + ,test_layout_conversion, stdex::extents, false, 5,10> + ,test_layout_conversion, stdex::extents, false, 5,10> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + ,test_layout_conversion, stdex::extents, false, 5, 10, 15, 20, 25> + >; + +// Left <=> Right conversion +using layout_conversion_test_types_left_to_right = + ::testing::Types< + test_layout_conversion, stdex::extents, true, 5> + ,test_layout_conversion, stdex::extents, false, 5> + ,test_layout_conversion, stdex::extents, true, 5> + ,test_layout_conversion, stdex::extents, false, 5> + ,test_layout_conversion, stdex::extents, true> + ,test_layout_conversion, stdex::extents, true> + >; + +template struct TestLayoutConversion; + +template +struct TestLayoutConversion< + std::tuple, + std::integral_constant>> : public ::testing::Test { + static constexpr bool implicit = Implicit; + using layout_1_t = Layout1; + using exts_1_t = Extents1; + using map_1_t = typename Layout1::template mapping; + using layout_2_t = Layout2; + using exts_2_t = Extents2; + using map_2_t = typename Layout2::template mapping; + exts_1_t exts1 { AllSizes... }; + exts_2_t exts2 { AllSizes... }; + + map_1_t implicit_conv(map_1_t map) const { + return map; + } + + template + static constexpr std::enable_if_t< + std::is_same::value && + !std::is_same::value && + std::is_same::value, + map_2_t> create_map2(Extents exts) { + return map_2_t(map_1_t(Extents1(exts))); + } + + template + static constexpr std::enable_if_t< + std::is_same::value && + std::is_same::value && + std::is_same::value, + map_2_t> create_map2(Extents exts) { + return map_2_t(typename stdex::layout_left::template mapping(exts)); + } + + template + static constexpr std::enable_if_t< + std::is_same::value && + !std::is_same::value, + map_2_t> create_map2(Extents exts) { + return map_2_t(exts); + } +}; + +// We can't reuse TestLayoutConversion directly for multiple suites, +// because this results in duplicate test names. +// However, creating an alias for each suite works around this. + +TYPED_TEST_SUITE(TestLayoutConversion, layout_conversion_test_types_left_to_left); + +template +using TestLayoutConversion_R2R = TestLayoutConversion; +TYPED_TEST_SUITE(TestLayoutConversion_R2R, layout_conversion_test_types_right_to_right); + +template +using TestLayoutConversion_S2S = TestLayoutConversion; +TYPED_TEST_SUITE(TestLayoutConversion_S2S, layout_conversion_test_types_stride_to_stride); + +template +using TestLayoutConversion_R2S = TestLayoutConversion; +TYPED_TEST_SUITE(TestLayoutConversion_R2S, layout_conversion_test_types_right_to_stride); + +template +using TestLayoutConversion_L2S = TestLayoutConversion; +TYPED_TEST_SUITE(TestLayoutConversion_L2S, layout_conversion_test_types_left_to_stride); + +template +using TestLayoutConversion_S2L = TestLayoutConversion; +TYPED_TEST_SUITE(TestLayoutConversion_S2L, layout_conversion_test_types_stride_to_left); + +template +using TestLayoutConversion_S2R = TestLayoutConversion; +TYPED_TEST_SUITE(TestLayoutConversion_S2R, layout_conversion_test_types_stride_to_right); + +template +using TestLayoutConversion_L2R = TestLayoutConversion; +TYPED_TEST_SUITE(TestLayoutConversion_L2R, layout_conversion_test_types_left_to_right); + +TYPED_TEST(TestLayoutConversion, implicit_conversion) { + typename TestFixture::map_2_t map2 = this->create_map2(this->exts2); + typename TestFixture::map_1_t map1; + #if MDSPAN_HAS_CXX_20 && !defined(_MDSPAN_COMPILER_MSVC) + static_assert(TestFixture::implicit == + ( + ( + !std::is_same::value && + std::is_convertible::value + ) || ( + std::is_same::value && + std::is_same::value && + std::is_convertible::value + )|| ( + std::is_same::value && + !std::is_same::value && + TestFixture::exts_2_t::rank()==0 + ) + ) + ); + static_assert(std::is_convertible::value == + TestFixture::implicit); + if constexpr(TestFixture::implicit) + map1 = this->implicit_conv(map2); + else + #endif + map1 = typename TestFixture::map_1_t(map2); + + for(size_t r=0; r != this->exts1.rank(); ++r) { + ASSERT_EQ(map1.extents().extent(r), map2.extents().extent(r)); + ASSERT_EQ(map1.stride(r), map2.stride(r)); + } +} + diff --git a/cpp/include/raft/thirdparty/mdspan/tests/test_extents.cpp b/cpp/include/raft/thirdparty/mdspan/tests/test_extents.cpp index f3ec724559..212b68adad 100644 --- a/cpp/include/raft/thirdparty/mdspan/tests/test_extents.cpp +++ b/cpp/include/raft/thirdparty/mdspan/tests/test_extents.cpp @@ -55,18 +55,18 @@ template struct TestExtents; template struct TestExtents< std::tuple< - stdex::extents, + stdex::extents, std::integer_sequence > > : public ::testing::Test { - using extents_type = stdex::extents; + using extents_type = stdex::extents; // Double Braces here to make it work with GCC 5 // Otherwise: "error: array must be initialized with a brace-enclosed initializer" const std::array static_sizes {{ Extents... }}; const std::array dyn_sizes {{ DynamicSizes... }}; extents_type exts { DynamicSizes... }; using Fixture = TestExtents< std::tuple< - stdex::extents, + stdex::extents, std::integer_sequence >>; @@ -75,7 +75,9 @@ struct TestExtents< dispatch([=] _MDSPAN_HOST_DEVICE () { extents_type _exts(DynamicSizes...); - result[0] = _exts.rank(); + // Silencing an unused warning in nvc++ the condition will never be true + size_t dyn_val = exts.rank()>0?static_cast(_exts.extent(0)):1; + result[0] = dyn_val > 1e9 ? dyn_val : _exts.rank(); result[1] = _exts.rank_dynamic(); // Some compilers warn about unused _exts since the functions are all static constexpr (void) _exts; @@ -91,12 +93,15 @@ struct TestExtents< dispatch([=] _MDSPAN_HOST_DEVICE () { extents_type _exts(DynamicSizes...); - for(int r=0; r<_exts.rank(); r++) - result[r] = _exts.static_extent(r); + for(size_t r=0; r<_exts.rank(); r++) { + // Silencing an unused warning in nvc++ the condition will never be true + size_t dyn_val = static_cast(_exts.extent(r)); + result[r] = dyn_val > 1e9 ? dyn_val : _exts.static_extent(r); + } // Some compilers warn about unused _exts since the functions are all static constexpr (void) _exts; }); - for(int r=0; r using _sizes = std::integer_sequence; template -using _exts = stdex::extents; +using _exts = stdex::extents; using extents_test_types = ::testing::Types< @@ -191,14 +196,14 @@ struct _BoolPairDeducer { template struct TestExtentsCompatCtors; template struct TestExtentsCompatCtors, + stdex::extents, std::integer_sequence, - stdex::extents, + stdex::extents, std::integer_sequence, _BoolPairDeducer >> : public ::testing::Test { - using extents_type1 = stdex::extents; - using extents_type2 = stdex::extents; + using extents_type1 = stdex::extents; + using extents_type2 = stdex::extents; extents_type1 exts1 { DynamicSizes... }; extents_type2 exts2 { DynamicSizes2... }; static constexpr bool implicit_exts1_to_exts2 = ImplicitExts1ToExts2; @@ -355,36 +360,48 @@ TYPED_TEST(TestExtentsCompatCtors, implicit_construct_1) { TEST(TestExtentsCtorStdArrayConvertibleToSizeT, test_extents_ctor_std_array_convertible_to_size_t) { std::array i{2, 2}; - stdex::dextents<2> e{i}; + stdex::dextents e{i}; ASSERT_EQ(e.rank(), 2); ASSERT_EQ(e.rank_dynamic(), 2); ASSERT_EQ(e.extent(0), 2); ASSERT_EQ(e.extent(1), 2); } +#ifdef __cpp_lib_span +TEST(TestExtentsCtorStdArrayConvertibleToSizeT, test_extents_ctor_std_span_convertible_to_size_t) { + std::array i{2, 2}; + std::span s(i.data(),2); + stdex::dextents e{s}; + ASSERT_EQ(e.rank(), 2); + ASSERT_EQ(e.rank_dynamic(), 2); + ASSERT_EQ(e.extent(0), 2); + ASSERT_EQ(e.extent(1), 2); +} +#endif + TYPED_TEST(TestExtentsCompatCtors, construct_from_dynamic_sizes) { using e1_t = typename TestFixture::extents_type1; constexpr bool e1_last_ext_static = e1_t::rank()>0?e1_t::static_extent(e1_t::rank()-1)!=stdex::dynamic_extent:true; e1_t e1 = TestFixture::template make_extents::construct(); - for(int r=0; r0?e2_t::static_extent(e2_t::rank()-1)!=stdex::dynamic_extent:true; e2_t e2 = TestFixture::template make_extents::construct(); - for(int r=0; r::construct(); - for(int r=0; r::construct(); - for(int r=0; r ext1_array_dynamic; int dyn_idx = 0; - for(int r=0; r((r+1)*5); dyn_idx++; } } e1_t e1(ext1_array_dynamic); - for(int r=0; r ext2_array_dynamic; dyn_idx = 0; - for(int r=0; r((r+1)*5); dyn_idx++; } } e2_t e2(ext2_array_dynamic); - for(int r=0; r ext1_array_all; - for(int r=0; r((r+1)*5); e1_t e1(ext1_array_all); - for(int r=0; r ext2_array_all; - for(int r=0; r((r+1)*5); e2_t e2(ext2_array_all); - for(int r=0; r >> : public ::testing::Test { using mapping_type = Mapping; - using extents_type = decltype(std::declval().extents()); + using extents_type = typename mapping_type::extents_type; Mapping map = { extents_type{ DynamicSizes... } }; }; @@ -73,18 +73,18 @@ using test_right_type = std::tuple< using layout_test_types = ::testing::Types< - test_left_type>, - test_right_type>, + test_left_type>, + test_right_type>, //---------- - test_left_type, 10>, - test_right_type, 10>, + test_left_type, 10>, + test_right_type, 10>, //---------- - test_left_type, 5>, - test_left_type, 10>, - test_left_type>, - test_right_type, 5>, - test_right_type, 10>, - test_right_type> + test_left_type, 5>, + test_left_type, 10>, + test_left_type>, + test_right_type, 5>, + test_right_type, 10>, + test_right_type> >; TYPED_TEST_SUITE(TestLayoutCtors, layout_test_types); @@ -108,8 +108,8 @@ struct TestLayoutCompatCtors> : public ::testing::Test { using mapping_type1 = Mapping; using mapping_type2 = Mapping2; - using extents_type1 = decltype(std::declval().extents()); - using extents_type2 = decltype(std::declval().extents()); + using extents_type1 = std::remove_reference_t().extents())>; + using extents_type2 = std::remove_reference_t().extents())>; Mapping map1 = { extents_type1{ DynamicSizes... } }; Mapping2 map2 = { extents_type2{ DynamicSizes2... } }; }; @@ -127,7 +127,7 @@ using test_right_type_compatible = std::tuple< template using _sizes = std::integer_sequence; template -using _exts = stdex::extents; +using _exts = stdex::extents; template