Merge pull request #26 from xrq-phys/pfaffine-blocked+cmake

CMake Support for Blocked Pfaffian
issp-center-dev · Sep 23, 2021 · 716b5e3 · 716b5e3
2 parents f1b27d1 + 1a4597e
commit 716b5e3
Show file tree

Hide file tree

Showing 14 changed files with 174 additions and 34 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 2.8.0 FATAL_ERROR)
 project(mVMC NONE)
 
 option(USE_SCALAPACK "Use Scalapack" OFF)
+option(PFAFFIAN_BLOCKED "Use blocked-update Pfaffian to speed up." OFF)
 
 add_definitions(-D_mVMC)
 if(CONFIG)
@@ -15,6 +16,10 @@ message(STATUS "Build type: " ${CMAKE_BUILD_TYPE})
 
 option(BUILD_SHARED_LIBS "Build shared libraries" ON)
 enable_language(C Fortran)
+if(PFAFFIAN_BLOCKED)
+  enable_language(CXX)
+  set(CMAKE_CXX_STANDARD 11)
+endif(PFAFFIAN_BLOCKED)
 
 set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
 set(CMAKE_SKIP_BUILD_RPATH FALSE)
@@ -57,6 +62,14 @@ if(USE_SCALAPACK MATCHES OFF)
   endif(LAPACK_FOUND)
 endif()
 
+if(PFAFFIAN_BLOCKED)
+  add_definitions(-D_pf_block_update)
+  include("download_blis_artifact.cmake")
+  # Must set BLIS artifact BEFORE adding pfupdates target.
+  add_subdirectory(src/pfupdates)
+  add_dependencies(pfupdates blis_include)
+endif(PFAFFIAN_BLOCKED)
+
 add_subdirectory(src/ComplexUHF)
 add_subdirectory(src/pfapack/fortran)
 add_subdirectory(src/mVMC)

diff --git a/config/aocc.cmake b/config/aocc.cmake
@@ -0,0 +1,16 @@
+# for AOCC
+
+set(CMAKE_C_COMPILER "clang" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "clang++" CACHE STRING "" FORCE)
+set(CMAKE_C_FLAGS_DEBUG "-g -O0 -Wall -Wformat -Werror=format-security")
+set(CMAKE_C_FLAGS_RELEASE "-Wno-unknown-pragmas -O3 -DNDEBUG -DHAVE_SSE2" CACHE STRING "" FORCE)
+
+set(CMAKE_Fortran_COMPILER "flang" CACHE STRING "" FORCE)
+set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG -DHAVE_SSE2" CACHE STRING "" FORCE)
+
+# for AOCL
+set(BLA_VENDOR "FLAME" CACHE STRING "" FORCE)
+
+# for BLIS & block pfaffian
+set(BLIS_ARTIFACT_CONFIG "amd64")
+set(PFAFFIAN_BLOCKED ON)
diff --git a/config/apple.cmake b/config/apple.cmake
@@ -2,6 +2,7 @@
 # additional libomp and gfortran installation required
 # mac computers are suggested to use this configuration for better performance
 set(CMAKE_C_COMPILER "clang" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "clang++" CACHE STRING "" FORCE)
 set(CMAKE_C_FLAGS_DEBUG "-g -O0 -Wall  -Wformat -Werror=format-security")
 set(CMAKE_C_FLAGS_RELEASE "-O3 -Wno-unknown-pragmas -Wno-logical-not-parentheses")
 set(CMAKE_Fortran_COMPILER "gfortran" CACHE STRING "" FORCE)

diff --git a/config/fugaku.cmake b/config/fugaku.cmake
@@ -0,0 +1,20 @@
+# for Fujitsu Compiler in Clang-mode
+set(CMAKE_C_COMPILER "mpifccpx" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "mpiFCCpx" CACHE STRING "" FORCE)
+set(CMAKE_C_FLAGS_RELEASE "-Nclang -Ofast -DNDEBUG" CACHE STRING "" FORCE)
+set(CMAKE_CXX_FLAGS_RELEASE "-Nclang -Ofast -DNDEBUG" CACHE STRING "" FORCE)
+
+set(CMAKE_Fortran_COMPILER "mpifrtpx" CACHE STRING "" FORCE)
+set(CMAKE_Fortran_FLAGS_RELEASE "-Kfast,parallel -DNDEBUG -DFUJITSU" CACHE STRING "" FORCE)
+
+set(OpenMP_C_FLAGS "-fopenmp" CACHE STRING "" FORCE)
+set(OpenMP_Fortran_FLAGS "-Kopenmp" CACHE STRING "" FORCE)
+set(CMAKE_EXE_LINKER_FLAGS "-fopenmp" CACHE STRING "" FORCE)
+
+# for SSL2
+set(BLAS_LIBRARIES "-SSL2" CACHE STRING "" FORCE)
+set(LAPACK_LIBRARIES ${BLAS_LIBRARIES} CACHE STRING "" FORCE)
+
+# for BLIS & block pfaffian
+set(BLIS_ARTIFACT_CONFIG "a64fx")
+set(PFAFFIAN_BLOCKED ON)
diff --git a/config/fujitsu.cmake b/config/fujitsu.cmake
@@ -1,11 +1,13 @@
 # for Fujitsu Compiler
 set(CMAKE_C_COMPILER "mpifccpx" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "mpiFCCpx" CACHE STRING "" FORCE)
 set(CMAKE_C_FLAGS_RELEASE "-DNDEBUG -Kfast,parallel -Kmemalias,alias_const" CACHE STRING "" FORCE)
 set(OpenMP_C_FLAGS "-Kopenmp" CACHE STRING "" FORCE)
 set(CMAKE_Fortran_COMPILER "mpifrtpx" CACHE STRING "" FORCE)
 set(CMAKE_Fortran_FLAGS_RELEASE "-DNDEBUG  -DFUJITSU -Kfast,parallel" CACHE STRING "" FORCE)
 set(OpenMP_Fortran_FLAGS "-Kopenmp" CACHE STRING "" FORCE)
 set(MPI_EXTRA_LIBRARY "-lmpi_f90 -lmpi_f77" CACHE STRING "" FORCE)
+set(CMAKE_EXE_LINKER_FLAGS "-Kopenmp" CACHE STRING "" FORCE)
 
 # for SSL2
 set(BLAS_LIBRARIES "-SSL2" CACHE STRING "" FORCE)

diff --git a/config/gcc.cmake b/config/gcc.cmake
@@ -1,5 +1,6 @@
 # for GCC Compiler
 set(CMAKE_C_COMPILER "gcc" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "g++" CACHE STRING "" FORCE)
 set(CMAKE_C_FLAGS_DEBUG "-g -O0 -Wall  -Wformat -Werror=format-security")
 set(CMAKE_C_FLAGS_RELEASE "-O3 -Wno-unknown-pragmas ")
-Set(CMAKE_Fortran_COMPILER "gfortran" CACHE STRING "" FORCE)
+set(CMAKE_Fortran_COMPILER "gfortran" CACHE STRING "" FORCE)
diff --git a/config/intel.cmake b/config/intel.cmake
@@ -1,6 +1,7 @@
 # for Intel Compiler
 
 set(CMAKE_C_COMPILER "icc" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "icpc" CACHE STRING "" FORCE)
 set(CMAKE_C_FLAGS_DEBUG "-O0 -g -Wall -Wformat -Werror=format-security")
 set(CMAKE_C_FLAGS_RELEASE "-Wno-unknown-pragmas -O3 -DNDEBUG -DHAVE_SSE2" CACHE STRING "" FORCE)
 
@@ -9,3 +10,7 @@ set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG -DHAVE_SSE2" CACHE STRING "" FORCE
 
 # for Intel MKL
 set(BLA_VENDOR "Intel10_64lp" CACHE STRING "" FORCE)
+
+# for BLIS & block pfaffian
+set(BLIS_ARTIFACT_CONFIG "intel64")
+set(PFAFFIAN_BLOCKED ON)
diff --git a/config/sekirei.cmake b/config/sekirei.cmake
diff --git a/download_blis_artifact.cmake b/download_blis_artifact.cmake
@@ -0,0 +1,62 @@
+if(NOT DEFINED BLIS_ARTIFACT_CONFIG)
+  execute_process(
+    COMMAND uname -m
+    COMMAND tr -d '\n'
+    OUTPUT_VARIABLE ARCHITECTURE)
+  if(${ARCHITECTURE} STREQUAL "x86_64")
+    execute_process(
+      COMMAND cat /proc/cpuinfo
+      COMMAND grep Intel
+      COMMAND head -1
+      COMMAND tr -d '\n'
+      OUTPUT_VARIABLE UARCH_INTEL)
+    if(NOT "${UARCH_INTEL} " STREQUAL " ")
+      set(BLIS_ARTIFACT_CONFIG "intel64")
+    else()
+      set(BLIS_ARTIFACT_CONFIG "amd64")
+    endif()
+  elseif(${ARCHITECTURE} STREQUAL "aarch64" OR
+      ${ARCHITECTURE} STREQUAL "arm64")
+      # Currently SVE requires being set manually.
+      set(BLIS_ARTIFACT_CONFIG "arm64")
+  else()
+    message(FATAL_ERROR "Failed to recognize architecture ${ARCHITECTURE}.")
+  endif()
+endif(NOT DEFINED BLIS_ARTIFACT_CONFIG)
+
+message(STATUS "Downloading BLIS artifact...")
+
+set(BLIS_ARCHIVE ${CMAKE_CURRENT_BINARY_DIR}/libblis_artifact.tar.gz)
+if(${BLIS_ARTIFACT_CONFIG} STREQUAL "intel64")
+  set(BLIS_ARTIFACT_URL https://github.com/xrq-phys/blis/releases/download/sv0.8.1+arm/libblis_intel64_gcc.tar.gz)
+elseif(${BLIS_ARTIFACT_CONFIG} STREQUAL "amd64")
+  set(BLIS_ARTIFACT_URL https://github.com/xrq-phys/blis/releases/download/sv0.8.1+arm/libblis_amd64_gcc.tar.gz)
+elseif(${BLIS_ARTIFACT_CONFIG} STREQUAL "arm64")
+  set(BLIS_ARTIFACT_URL https://github.com/xrq-phys/blis/releases/download/sv0.8.1+arm/libblis_cortexa57_aarch64-linux-gnu-gcc.tar.gz)
+elseif(${BLIS_ARTIFACT_CONFIG} STREQUAL "armsve")
+  set(BLIS_ARTIFACT_URL https://github.com/xrq-phys/blis/releases/download/sv0.8.1+arm/libblis_armsve_aarch64-linux-gnu-gcc-10.tar.gz)
+elseif(${BLIS_ARTIFACT_CONFIG} STREQUAL "a64fx")
+  set(BLIS_ARTIFACT_URL https://github.com/xrq-phys/blis/releases/download/sv0.8.1+arm/libblis_a64fx_aarch64-linux-gnu-gcc-10.tar.gz)
+else()
+  message(FATAL_ERROR "Not BLIS artifact available.")
+endif()
+
+message("${BLIS_ARTIFACT_URL} ${BLIS_ARCHIVE}")
+file(DOWNLOAD "${BLIS_ARTIFACT_URL}" "${BLIS_ARCHIVE}")
+
+add_custom_command(
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/lib/libblis.a
+    ${CMAKE_CURRENT_BINARY_DIR}/include/blis/blis.h
+  COMMAND ${CMAKE_COMMAND} -E tar -zxf ${BLIS_ARCHIVE}
+  DEPENDS ${BLIS_ARCHIVE})
+add_custom_target(blis_target
+  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/lib/libblis.a)
+add_custom_target(blis_include
+  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/include/blis/blis.h)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/include/blis)
+
+add_library(blis STATIC IMPORTED GLOBAL)
+set_target_properties(blis PROPERTIES
+  IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/lib/libblis.a)
+add_dependencies(blis blis_target)
+
diff --git a/src/mVMC/CMakeLists.txt b/src/mVMC/CMakeLists.txt
@@ -27,7 +27,11 @@ add_executable(vmcdry.out vmcdry.c)
 target_link_libraries(vmcdry.out StdFace m)
 add_executable(vmc.out ${SOURCES_vmcmain} ${SOURCES_sfmt})
 target_link_libraries(vmc.out StdFace)
-target_link_libraries(vmc.out pfapack StdFace ${LAPACK_LIBRARIES} m)
+target_link_libraries(vmc.out pfapack)
+if(PFAFFIAN_BLOCKED)
+  target_link_libraries(vmc.out pfupdates blis pthread)
+endif(PFAFFIAN_BLOCKED)
+target_link_libraries(vmc.out ${LAPACK_LIBRARIES} m)
 
 if(USE_SCALAPACK)
   string(REGEX REPLACE "-L[ ]+" "-L" sc_libs ${SCALAPACK_LIBRARIES})

diff --git a/src/mVMC/vmcmake.c b/src/mVMC/vmcmake.c
@@ -74,11 +74,16 @@ void VMCMakeSample(MPI_Comm comm) {
   // TODO: Compute from qpStart to qpEnd to support loop splitting.
   void *pfOrbital[NQPFull];
   void *pfUpdator[NQPFull];
-  // TODO: Make it input parameter.
-  if (NExUpdatePath == 0)
-    NBlockUpdateSize = 4;
-  else
-    NBlockUpdateSize = 20;
+  // Read block size from input.
+  const char *optBlockSize = getenv("VMC_BLOCK_UPDATE_SIZE");
+  if (optBlockSize)
+    NBlockUpdateSize = atoi(optBlockSize);
+  // Fall back to default if input is invalid.
+  if (NBlockUpdateSize < 1 || NBlockUpdateSize > 100)
+    if (NExUpdatePath == 0)
+      NBlockUpdateSize = 4;
+    else
+      NBlockUpdateSize = 20;
 
   // Set one universal EleSpn.
   for (mi=0; mi<Ne;  mi++) EleSpn[mi] = 0;

diff --git a/src/mVMC/vmcmake_fsz.c b/src/mVMC/vmcmake_fsz.c
@@ -83,11 +83,16 @@ void VMCMakeSample_fsz(MPI_Comm comm) {
   // TODO: Compute from qpStart to qpEnd to support loop splitting.
   void *pfOrbital[NQPFull];
   void *pfUpdator[NQPFull];
-  // TODO: Make it input parameter.
-  if (NExUpdatePath == 0)
-    NBlockUpdateSize = 4;
-  else
-    NBlockUpdateSize = 20;
+  // Read block size from input.
+  const char *optBlockSize = getenv("VMC_BLOCK_UPDATE_SIZE");
+  if (optBlockSize)
+    NBlockUpdateSize = atoi(optBlockSize);
+  // Fall back to default if input is invalid.
+  if (NBlockUpdateSize < 1 || NBlockUpdateSize > 100)
+    if (NExUpdatePath == 0)
+      NBlockUpdateSize = 4;
+    else
+      NBlockUpdateSize = 20;
 
   // Initialize with free spin configuration.
   updated_tdi_v_init_z(NQPFull, Nsite, Nsite2, Nsize,

diff --git a/src/mVMC/vmcmake_real.c b/src/mVMC/vmcmake_real.c
@@ -76,11 +76,16 @@ void VMCMakeSample_real(MPI_Comm comm) {
   // TODO: Compute from qpStart to qpEnd to support loop splitting.
   void *pfOrbital[NQPFull];
   void *pfUpdator[NQPFull];
-  // TODO: Make it input parameter.
-  if (NExUpdatePath == 0)
-    NBlockUpdateSize = 2;
-  else
-    NBlockUpdateSize = 20;
+  // Read block size from input.
+  const char *optBlockSize = getenv("VMC_BLOCK_UPDATE_SIZE");
+  if (optBlockSize)
+    NBlockUpdateSize = atoi(optBlockSize);
+  // Fall back to default if input is invalid.
+  if (NBlockUpdateSize < 1 || NBlockUpdateSize > 100)
+    if (NExUpdatePath == 0)
+      NBlockUpdateSize = 2;
+    else
+      NBlockUpdateSize = 20;
 
   // Set one universal EleSpn.
   for (mi=0; mi<Ne;  mi++) EleSpn[mi] = 0;

diff --git a/src/pfupdates/CMakeLists.txt b/src/pfupdates/CMakeLists.txt
@@ -0,0 +1,18 @@
+# include guard
+cmake_minimum_required(VERSION 2.8.0 )
+
+if(${CMAKE_PROJECT_NAME} STREQUAL "Project")
+  message(FATAL_ERROR "cmake should be executed not for 'src' subdirectory, but for the top directory of mVMC.")
+endif(${CMAKE_PROJECT_NAME} STREQUAL "Project")
+
+add_definitions(-DBLAS_EXTERNAL)
+add_definitions(-DF77_COMPLEX_RET_INTEL)
+include_directories(../pfaffine/src)
+
+add_library(pfupdates STATIC
+  pf_interface.cc
+  ../pfaffine/src/skpfa.cc
+  ../pfaffine/src/sktdf.cc
+  ../pfaffine/src/sktdi.cc)
+target_compile_definitions(pfupdates PRIVATE -D_CC_IMPL)
+