Venado optimizations

scripts/build_bml_cray.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Make sure all the paths are correct
+
+rm -rf build
+rm -rf install_cray
+
+MY_PATH=$(pwd)
+
+
+export CC=${CC:=cc}
+#export FC=${FC:=/opt/cray/pe/cce/17.0.0/bin/crayftn}
+export FC=${FC:=ftn}
+export CXX=${CXX:=CC}
+#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-ef -DCRAY_SDK"}
+#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-DINTEL_SDK"}
+#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-DCRAY_SDK"}
+#export BLAS_VENDOR=${BLAS_VENDOR:=OpenBlas}
+#export BLAS_VENDOR=${BLAS_VENDOR:=Intel10_64lp}
+#export BLAS_LIBRARIES="-L${MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl"
+#export BLAS_LIBRARIES="-L/projects/darwin-nv/rhel9/aarch64/packages/nvpl/23.11/lib -lnvpl_blas_lp64_gomp"
+#export LAPACK_LIBRARIES="-L/projects/darwin-nv/rhel9/aarch64/packages/nvpl/23.11/lib -lnvpl_lapack_lp64_gomp"
+export LAPACK_LIBRARIES="-L${CRAY_PE_LIBSCI_PREFIX}/lib"
+export BML_OPENMP=${BML_OPENMP:=yes}
+export BML_COMPLEX=${BML_COMPLEX:=no}
+#export CUDA_ROOT=${CUDA_PATH}
+#export CUDA_DIR=${CUDA_PATH}
+#export CUDA_INCLUDE=${CUDA_INCLUDES}
+#CUDA_INCLUDESexport CUDA_LIB=${CUDA_LIBS}
+#export CUDA_TOOLKIT_ROOT_DIR=${CUDATOOLKIT_ROOT}
+#export BML_CUDA=${BML_CUDA:=yes}
+export BML_CUSOLVER=${BML_CUSOLVER:=yes}
+export INSTALL_DIR=${INSTALL_DIR:="${MY_PATH}/install_cray"}
+#export BML_TESTING=${BML_TESTING:=yes}
+export CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:=Release}
+export MAGMA_ROOT=${MAGMA_ROOT:="/usr/projects/icapt/mewall/venado/packages/magma-2.7.2/install_cray"}
+export BML_MAGMA=${BML_MAGMA:=yes}
+export CMAKE_PREFIX_PATH="$CUDATOOLKIT_HOME/../../math_libs/lib64"
+#export EXTRA_FFLAGS=${EXTRA_FFLAGS:="-DNVHPC_SDK"}
+export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-DCRAY_SDK -ef"}
+export EXTRA_CFLAGS=${EXTRA_CFLAGS:=""}
+export BML_POSIX_MEMALIGN=${BML_POSIX_MEMALIGN:=no}
+#export EXTRA_CFLAGS=${EXTRA_CFLAGS:="-DCRAY_SDK -I${CUDA_ROOT}/include -lcudart"}
+#export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:="-DCRAY_SDK -L${CUDA_ROOT}/lib64 -lcudart"}
+
+./build.sh configure
+
+

scripts/build_bml_cray_nvpl.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Make sure all the paths are correct
+
+rm -rf build
+rm -rf install_cray_nvpl
+
+MY_PATH=$(pwd)
+
+
+export CC=${CC:=cc}
+#export FC=${FC:=/opt/cray/pe/cce/17.0.0/bin/crayftn}
+export FC=${FC:=ftn}
+export CXX=${CXX:=CC}
+#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-ef -DCRAY_SDK"}
+#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-DINTEL_SDK"}
+#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-DCRAY_SDK"}
+#export BLAS_VENDOR=${BLAS_VENDOR:=OpenBlas}
+#export BLAS_VENDOR=${BLAS_VENDOR:=Intel10_64lp}
+#export BLAS_LIBRARIES="-L${MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl"
+export BLAS_LIBRARIES="-L$NVPL_HOME/lib -lnvpl_blas_lp64_gomp"
+export LAPACK_LIBRARIES="-L$NVPL_HOME/lib -lnvpl_lapack_lp64_gomp"
+#export LAPACK_LIBRARIES="-L${CRAY_PE_LIBSCI_PREFIX}/lib"
+export BML_OPENMP=${BML_OPENMP:=yes}
+export BML_COMPLEX=${BML_COMPLEX:=no}
+#export CUDA_ROOT=${CUDA_PATH}
+#export CUDA_DIR=${CUDA_PATH}
+#export CUDA_INCLUDE=${CUDA_INCLUDES}
+#CUDA_INCLUDESexport CUDA_LIB=${CUDA_LIBS}
+#export CUDA_TOOLKIT_ROOT_DIR=${CUDATOOLKIT_ROOT}
+#export BML_CUDA=${BML_CUDA:=yes}
+export BML_CUSOLVER=${BML_CUSOLVER:=yes}
+export INSTALL_DIR=${INSTALL_DIR:="${MY_PATH}/install_cray_nvpl"}
+#export BML_TESTING=${BML_TESTING:=yes}
+export CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:=Release}
+export MAGMA_ROOT=${MAGMA_ROOT:="/usr/projects/icapt/mewall/venado/packages/magma-2.7.2/install_cray_nvpl"}
+export BML_MAGMA=${BML_MAGMA:=yes}
+export CMAKE_PREFIX_PATH="$CUDATOOLKIT_HOME/../../math_libs/lib64"
+#export EXTRA_FFLAGS=${EXTRA_FFLAGS:="-DNVHPC_SDK"}
+#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-DCRAY_SDK -ef"}
+export EXTRA_CFLAGS=${EXTRA_CFLAGS:="-I/usr/include -I/usr/include/c++/12 -I/usr/include/c++/12/aarch64-suse-linux"}
+#export CMAKE_C_FLAGS=${CMAKE_CXX_FLAGS:="-I/usr/include -I/usr/include/c++/12 -I/usr/include/c++/12/aarch64-suse-linux"}
+#export CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS:="-I/usr/include -I/usr/include/c++/12 -I/usr/include/c++/12/aarch64-suse-linux"}
+export BML_POSIX_MEMALIGN=${BML_POSIX_MEMALIGN:=no}
+export CMAKE_C_FLAGS="-homp -g -L/usr/lib64/gcc/aarch64-suse-linux/12 -DCRAY_SDK"
+export CMAKE_CXX_FLAGS="-homp -g -L/usr/lib64/gcc/aarch64-suse-linux/12"
+export CMAKE_Fortran_FLAGS="-homp -g -ef -L/usr/lib64/gcc/aarch64-suse-linux/12 -DCRAY_SDK"
+#export EXTRA_CFLAGS=${EXTRA_CFLAGS:="-DCRAY_SDK -I${CUDA_ROOT}/include -lcudart"}
+#export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:="-DCRAY_SDK -L${CUDA_ROOT}/lib64 -lcudart"}
+#EXTRA_LINK_FLAGS="-L/usr/lib64/gcc/aarch64-suse-linux/12"
+
+./build.sh configure
+
+

scripts/build_bml_nvhpc.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Make sure all the paths are correct
+
+rm -rf build
+rm -rf install_nvhpc
+
+MY_PATH=$(pwd)
+
+
+export CC=${CC:=nvc}
+export FC=${FC:=nvfortran}
+export CXX=${CXX:=nvc++}
+#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-ef -DCRAY_SDK"}
+#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-DINTEL_SDK"}
+#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-DCRAY_SDK"}
+#export BLAS_VENDOR=${BLAS_VENDOR:=OpenBlas}
+#export BLAS_VENDOR=${BLAS_VENDOR:=Intel10_64lp}
+#export BLAS_LIBRARIES="-L${MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl"
+#export BLAS_LIBRARIES="-L/projects/darwin-nv/rhel9/aarch64/packages/nvpl/23.11/lib -lnvpl_blas_lp64_gomp"
+#export LAPACK_LIBRARIES="-L/projects/darwin-nv/rhel9/aarch64/packages/nvpl/23.11/lib -lnvpl_lapack_lp64_gomp"
+#export LAPACK_LIBRARIES="-L${CRAY_PE_LIBSCI_PREFIX}/lib"
+export BLAS_LIBRARIES="-L$NVPL_HOME/lib -lnvpl_blas_lp64_gomp"
+export LAPACK_LIBRARIES="-L$NVPL_HOME/lib -lnvpl_lapack_lp64_gomp"
+export BML_OPENMP=${BML_OPENMP:=yes}
+export BML_COMPLEX=${BML_COMPLEX:=yes}
+#export CUDA_ROOT=${CUDA_PATH}
+export CUDA_DIR=${CUDA_PATH}
+#export CUDA_INCLUDE=${CUDA_INCLUDES}
+#CUDA_INCLUDESexport CUDA_LIB=${CUDA_LIBS}
+#export CUDA_TOOLKIT_ROOT_DIR=${CUDATOOLKIT_ROOT}
+#export BML_CUDA=${BML_CUDA:=yes}
+export BML_CUSOLVER=${BML_CUSOLVER:=yes}
+export INSTALL_DIR=${INSTALL_DIR:="${MY_PATH}/install_nvhpc"}
+#export BML_TESTING=${BML_TESTING:=yes}
+export CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:=Release}
+export MAGMA_ROOT=${MAGMA_ROOT:="/usr/projects/icapt/mewall/venado/packages/magma-2.7.2/install_nvhpc"}
+export BML_MAGMA=${BML_MAGMA:=yes}
+export CMAKE_PREFIX_PATH="${NVHPC_ROOT}/math_libs/lib64"
+#export EXTRA_FFLAGS=${EXTRA_FFLAGS:="-DNVHPC_SDK"}
+export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-DNVHPC_SDK -O2"}
+export CMAKE_C_FLAGS=${EXTRA_CFLAGS:="-DNVHPC_SDK -O2"}
+#export EXTRA_CFLAGS=${EXTRA_CFLAGS:="-DNVHPC_SDK"}
+#export EXTRA_CFLAGS=${EXTRA_CFLAGS:="-DCRAY_SDK -I${CUDA_ROOT}/include -lcudart"}
+#export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:="-DCRAY_SDK -L${CUDA_ROOT}/lib64 -lcudart"}
+
+./build.sh configure
+
+

scripts/build_chicoma_hackathon.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Make sure all the paths are correct
+
+rm -r build
+#rm -r install_magma_2.7.2
+rm -r install_hackathon
+
+MY_PATH=$(pwd)
+
+
+export CC=${CC:=cc}
+export FC=${FC:=ftn}
+export CXX=${CXX:=CC}
+#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-ef -DCRAY_SDK"}
+#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-DINTEL_SDK"}
+#export CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS:="-DCRAY_SDK"}
+export BLAS_VENDOR=${BLAS_VENDOR:=Intel10_64lp}
+#export BLAS_VENDOR=${BLAS_VENDOR:=OpenBLAS}
+#export LAPACK_LIBRARIES="-L${MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl"
+#export BLAS_LIBRARIES="-L${MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl"
+export BML_OPENMP=${BML_OPENMP:=yes}
+export BML_COMPLEX=${BML_COMPLEX:=yes}
+#export CUDA_ROOT=${CUDA_PATH}
+#export CUDA_DIR=${CUDA_PATH}
+#export CUDA_INCLUDE=${CUDA_INCLUDES}
+#CUDA_INCLUDESexport CUDA_LIB=${CUDA_LIBS}
+#export CUDA_TOOLKIT_ROOT_DIR=${CUDATOOLKIT_ROOT}
+#export BML_CUDA=${BML_CUDA:=yes}
+export BML_CUSOLVER=${BML_CUSOLVER:=yes}
+export INSTALL_DIR=${INSTALL_DIR:="${MY_PATH}/install_hackathon"}
+#export BML_TESTING=${BML_TESTING:=yes}
+export CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:=Release}
+#export MAGMA_ROOT=${MAGMA_ROOT:="/usr/projects/icapt/mewall/packages/gpmd/magma/install"}
+export MAGMA_ROOT=${MAGMA_ROOT:="/usr/projects/icapt/mewall/packages/gpmd/magma-2.7.2/install"}
+export BML_MAGMA=${BML_MAGMA:=yes}
+#export BML_POSIX_MEMALIGN=${BML_POSIX_MEMALIGN:=no}
+#export CMAKE_PREFIX_PATH="$MKLROOT/lib/intel64"
+#export EXTRA_FFLAGS=${EXTRA_FFLAGS:="-fdefault-integer-8"}
+#export EXTRA_CFLAGS=${EXTRA_CFLAGS:=""}
+#export EXTRA_CFLAGS=${EXTRA_CFLAGS:="-DCRAY_SDK -I${CUDA_ROOT}/include -lcudart"}
+#export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:="-DCRAY_SDK -L${CUDA_ROOT}/lib64 -lcudart"}
+
+./build.sh configure
+
+

scripts/build_venado_hackathon.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Make sure all the paths are correct
+
+rm -r build
+#rm -r install_magma_2.7.2
+rm -r install_hackathon
+
+MY_PATH=$(pwd)
+
+
+export CC=${CC:=cc}
+export FC=${FC:=ftn}
+export CXX=${CXX:=CC}
+#export BLAS_VENDOR=${BLAS_VENDOR:=OpenBLAS}
+#export LAPACK_LIBRARIES="-L${MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl"
+#export BLAS_LIBRARIES="-L${MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl"
+export LAPACK_LIBRARIES="-L/opt/cray/pe/libsci/23.12.5/GNU/12.3/aarch64/lib"
+export BML_OPENMP=${BML_OPENMP:=yes}
+export BML_COMPLEX=${BML_COMPLEX:=yes}
+export BML_CUSOLVER=${BML_CUSOLVER:=yes}
+export INSTALL_DIR=${INSTALL_DIR:="${MY_PATH}/install_hackathon"}
+#export BML_TESTING=${BML_TESTING:=yes}
+export CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:=Release}
+export MAGMA_ROOT=${MAGMA_ROOT:="/usr/projects/icapt/mewall/venado/packages/magma-2.7.2/install"}
+export BML_MAGMA=${BML_MAGMA:=yes}
+#export BML_POSIX_MEMALIGN=${BML_POSIX_MEMALIGN:=no}
+export CMAKE_PREFIX_PATH=$CUDA_HOME/../../math_libs/lib64:$CMAKE_PREFIX_PATH
+export EXTRA_FFLAGS=${EXTRA_CFLAGS:="-mcpu=neoverse-v2"}
+export EXTRA_CFLAGS=${EXTRA_CFLAGS:="-mcpu=neoverse-v2"}
+#export EXTRA_CFLAGS=${EXTRA_CFLAGS:="-DCRAY_SDK -I${CUDA_ROOT}/include -lcudart"}
+#export EXTRA_LINK_FLAGS=${EXTRA_LINK_FLAGS:="-DCRAY_SDK -L${CUDA_ROOT}/lib64 -lcudart"}
+
+./build.sh configure
+
+

src/C-interface/dense/bml_introspection_dense.c

@@ -200,3 +200,22 @@ bml_get_data_ptr_dense(
 {
     return A->matrix;
 }
+
+/** Return the dense matrix ld parameter.
+ *
+ * \param A The dense matrix.
+ * \return The matrix ld parameter.
+ */
+int
+bml_get_ld_dense(
+    bml_matrix_dense_t * A)
+{
+    if (A != NULL)
+    {
+        return A->ld;
+    }
+    else
+    {
+        return -1;
+    }
+}

src/C-interface/dense/bml_setters_dense.c

@@ -3,6 +3,22 @@
 #include "bml_setters_dense.h"
 #include "bml_types_dense.h"
 
+#ifdef BML_USE_MAGMA
+#include "magma_v2.h"
+#endif
+
+void bml_set_N_dense(
+    bml_matrix_dense_t * A,
+    int N)
+{
+  A->N = N;
+#ifdef BML_USE_MAGMA
+  A->ld = magma_roundup(A->N, 32);
+#else
+  A->ld = A->N;
+#endif
+}
+
 void
 bml_set_element_dense(
     bml_matrix_dense_t * A,

src/C-interface/dense/bml_setters_dense.h

@@ -7,6 +7,10 @@
 
 #include <complex.h>
 
+void bml_set_N_dense(
+    bml_matrix_dense_t * A,
+    int N);
+
 void bml_set_element_dense(
     bml_matrix_dense_t * A,
     int i,

src/Fortran-interface/bml_c_interface_m.F90

@@ -370,6 +370,18 @@ function bml_get_bandwidth_C(a) bind(C, name="bml_get_bandwidth")
       type(C_PTR), value, intent(in) :: a
       integer(C_INT) :: bml_get_bandwidth_C
     end function bml_get_bandwidth_C
+
+    function bml_get_data_ptr_dense_C(a) bind(C, name="bml_get_data_ptr_dense")
+      import :: C_PTR
+      type(C_PTR), value, intent(in) :: a
+      type(C_PTR) :: bml_get_data_ptr_dense_C
+    end function bml_get_data_ptr_dense_C
+
+    function bml_get_ld_dense_C(a) bind(C, name="bml_get_ld_dense")
+      import :: C_PTR, C_INT
+      type(C_PTR), value, intent(in) :: a
+      integer(C_INT) :: bml_get_ld_dense_C
+    end function bml_get_ld_dense_C
 
     function bml_get_sparsity_C(a, threshold) bind(C, name="bml_get_sparsity")
       import :: C_PTR, C_DOUBLE, C_INT
@@ -448,6 +460,12 @@ subroutine bml_scale_inplace_C(alpha, a) bind(C, name="bml_scale_inplace")
       type(C_PTR), value :: a
     end subroutine bml_scale_inplace_C
 
+    subroutine bml_set_N_dense_C(a,n) bind(C, name="bml_set_N_dense")
+      import :: C_PTR, C_INT
+      type(C_PTR), value, intent(in) :: a
+      integer(C_INT), value, intent(in) :: n
+    end subroutine bml_set_N_dense_C
+
     subroutine bml_set_row_C(a, i, row, threshold) bind(C, name="bml_set_row")
       import :: C_PTR, C_INT, C_DOUBLE
       type(C_PTR), value, intent(in) :: a
@@ -616,6 +634,11 @@ function bml_transpose_new_C(a) bind(C, name="bml_transpose_new")
       type(C_PTR) :: bml_transpose_new_C
     end function bml_transpose_new_C
 
+    subroutine bml_transpose_C(a) bind(C, name="bml_transpose")
+      import :: C_PTR
+      type(C_PTR), value, intent(in) :: a
+    end subroutine bml_transpose_C
+
     subroutine bml_write_bml_matrix_C(a, filename) &
          & bind(C, name="bml_write_bml_matrix")
       import :: C_PTR, C_CHAR

src/Fortran-interface/bml_introspection_m.F90

@@ -18,6 +18,8 @@ module bml_introspection_m
   public :: bml_get_bandwidth
   public :: bml_get_distribution_mode
   public :: bml_get_sparsity
+  public :: bml_get_data_ptr_dense
+  public :: bml_get_ld_dense
 
 contains
 
@@ -247,4 +249,25 @@ function bml_get_sparsity(a, threshold) result(sparsity)
 
   end function bml_get_sparsity
 
+  function bml_get_data_ptr_dense(a)
+    type(bml_matrix_t), intent(inout) :: a
+    type(C_PTR) :: bml_get_data_ptr_dense
+
+    bml_get_data_ptr_dense = bml_get_data_ptr_dense_C(a%ptr)
+
+  end function bml_get_data_ptr_dense
+
+    !> Return the dense matrix ld parameter.
+  !!
+  !!\param a The matrix.
+  !!\return The matrix ld parameter.
+  function bml_get_ld_dense(a)
+
+    type(bml_matrix_t), intent(in) :: a
+    integer :: bml_get_ld_dense
+
+    bml_get_ld_dense = bml_get_ld_dense_C(a%ptr)
+
+  end function bml_get_ld_dense
+
 end module bml_introspection_m

src/Fortran-interface/bml_setters_m.F90

@@ -52,9 +52,18 @@ module bml_setters_m
   public :: bml_set_diagonal
   public :: bml_set_element
   public :: bml_set_element_new
-
+  public :: bml_set_N_dense
+
 contains
 
+  subroutine bml_set_N_dense(a,n)
+    type(bml_matrix_t), intent(inout) :: a
+    integer(C_INT), intent(in) :: n
+
+    call bml_set_N_dense_C(a%ptr,n)
+
+  end subroutine bml_set_N_dense
+
   !Setters for element new
   subroutine bml_set_element_new_single_real(a,i,j,element)