From 9ae5988f2e76fc3b9b38e9a2146d9191170a740b Mon Sep 17 00:00:00 2001
From: RuQing Xu <r-xu@g.ecc.u-tokyo.ac.jp>
Date: Sun, 12 Sep 2021 21:39:36 +0900
Subject: [PATCH 1/8] Allow Changing Block Size w/ Env. Var.

Use `VMC_BLOCK_UPDATE_SIZE` to change block size upon each run.
---
 src/mVMC/vmcmake.c      | 15 ++++++++++-----
 src/mVMC/vmcmake_fsz.c  | 15 ++++++++++-----
 src/mVMC/vmcmake_real.c | 15 ++++++++++-----
 3 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/src/mVMC/vmcmake.c b/src/mVMC/vmcmake.c
index e7293405..57ab2dd2 100644
--- a/src/mVMC/vmcmake.c
+++ b/src/mVMC/vmcmake.c
@@ -74,11 +74,16 @@ void VMCMakeSample(MPI_Comm comm) {
   // TODO: Compute from qpStart to qpEnd to support loop splitting.
   void *pfOrbital[NQPFull];
   void *pfUpdator[NQPFull];
-  // TODO: Make it input parameter.
-  if (NExUpdatePath == 0)
-    NBlockUpdateSize = 4;
-  else
-    NBlockUpdateSize = 20;
+  // Read block size from input.
+  const char *optBlockSize = getenv("VMC_BLOCK_UPDATE_SIZE");
+  if (optBlockSize)
+    NBlockUpdateSize = atoi(optBlockSize);
+  // Fall back to default if input is invalid.
+  if (NBlockUpdateSize < 1 || NBlockUpdateSize > 100)
+    if (NExUpdatePath == 0)
+      NBlockUpdateSize = 4;
+    else
+      NBlockUpdateSize = 20;
 
   // Set one universal EleSpn.
   for (mi=0; mi<Ne;  mi++) EleSpn[mi] = 0;
diff --git a/src/mVMC/vmcmake_fsz.c b/src/mVMC/vmcmake_fsz.c
index bbf57a15..404318e4 100644
--- a/src/mVMC/vmcmake_fsz.c
+++ b/src/mVMC/vmcmake_fsz.c
@@ -83,11 +83,16 @@ void VMCMakeSample_fsz(MPI_Comm comm) {
   // TODO: Compute from qpStart to qpEnd to support loop splitting.
   void *pfOrbital[NQPFull];
   void *pfUpdator[NQPFull];
-  // TODO: Make it input parameter.
-  if (NExUpdatePath == 0)
-    NBlockUpdateSize = 4;
-  else
-    NBlockUpdateSize = 20;
+  // Read block size from input.
+  const char *optBlockSize = getenv("VMC_BLOCK_UPDATE_SIZE");
+  if (optBlockSize)
+    NBlockUpdateSize = atoi(optBlockSize);
+  // Fall back to default if input is invalid.
+  if (NBlockUpdateSize < 1 || NBlockUpdateSize > 100)
+    if (NExUpdatePath == 0)
+      NBlockUpdateSize = 4;
+    else
+      NBlockUpdateSize = 20;
 
   // Initialize with free spin configuration.
   updated_tdi_v_init_z(NQPFull, Nsite, Nsite2, Nsize,
diff --git a/src/mVMC/vmcmake_real.c b/src/mVMC/vmcmake_real.c
index 41923ea5..6c9bde7d 100644
--- a/src/mVMC/vmcmake_real.c
+++ b/src/mVMC/vmcmake_real.c
@@ -76,11 +76,16 @@ void VMCMakeSample_real(MPI_Comm comm) {
   // TODO: Compute from qpStart to qpEnd to support loop splitting.
   void *pfOrbital[NQPFull];
   void *pfUpdator[NQPFull];
-  // TODO: Make it input parameter.
-  if (NExUpdatePath == 0)
-    NBlockUpdateSize = 2;
-  else
-    NBlockUpdateSize = 20;
+  // Read block size from input.
+  const char *optBlockSize = getenv("VMC_BLOCK_UPDATE_SIZE");
+  if (optBlockSize)
+    NBlockUpdateSize = atoi(optBlockSize);
+  // Fall back to default if input is invalid.
+  if (NBlockUpdateSize < 1 || NBlockUpdateSize > 100)
+    if (NExUpdatePath == 0)
+      NBlockUpdateSize = 2;
+    else
+      NBlockUpdateSize = 20;
 
   // Set one universal EleSpn.
   for (mi=0; mi<Ne;  mi++) EleSpn[mi] = 0;

From ac0e07fc6ceee82dc2f959555ed5ff69a9855b07 Mon Sep 17 00:00:00 2001
From: RuQing Xu <r-xu@g.ecc.u-tokyo.ac.jp>
Date: Thu, 23 Sep 2021 02:25:02 +0900
Subject: [PATCH 2/8] CMake Build Support for Block-Pfa w/ BLIS Artifact

NOTE: With CMake, BLIS is no longer built from submodule.
An artifact is downloaded from GitHub instead.
---
 CMakeLists.txt               | 12 +++++++
 download_blis_artifact.cmake | 61 ++++++++++++++++++++++++++++++++++++
 src/mVMC/CMakeLists.txt      |  6 +++-
 src/pfupdates/CMakeLists.txt | 18 +++++++++++
 4 files changed, 96 insertions(+), 1 deletion(-)
 create mode 100644 download_blis_artifact.cmake
 create mode 100644 src/pfupdates/CMakeLists.txt

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 39546470..99009504 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 2.8.0 FATAL_ERROR)
 project(mVMC NONE)
 
 option(USE_SCALAPACK "Use Scalapack" OFF)
+option(PFAFFIAN_BLOCKED "Use blocked-update Pfaffian to speed up." OFF)
 
 add_definitions(-D_mVMC)
 if(CONFIG)
@@ -15,6 +16,9 @@ message(STATUS "Build type: " ${CMAKE_BUILD_TYPE})
 
 option(BUILD_SHARED_LIBS "Build shared libraries" ON)
 enable_language(C Fortran)
+if(PFAFFIAN_BLOCKED)
+  enable_language(CXX)
+endif(PFAFFIAN_BLOCKED)
 
 set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
 set(CMAKE_SKIP_BUILD_RPATH FALSE)
@@ -57,6 +61,14 @@ if(USE_SCALAPACK MATCHES OFF)
   endif(LAPACK_FOUND)
 endif()
 
+if(PFAFFIAN_BLOCKED)
+  add_definitions(-D_pf_block_update)
+  include("download_blis_artifact.cmake")
+  # Must set BLIS artifact BEFORE adding pfupdates target.
+  add_subdirectory(src/pfupdates)
+  add_dependencies(pfupdates blis_include)
+endif(PFAFFIAN_BLOCKED)
+
 add_subdirectory(src/ComplexUHF)
 add_subdirectory(src/pfapack/fortran)
 add_subdirectory(src/mVMC)
diff --git a/download_blis_artifact.cmake b/download_blis_artifact.cmake
new file mode 100644
index 00000000..a6b6272b
--- /dev/null
+++ b/download_blis_artifact.cmake
@@ -0,0 +1,61 @@
+if(NOT DEFINED BLIS_ARTIFACT_CONFIG)
+  execute_process(
+    COMMAND uname -m
+    COMMAND tr -d '\n'
+    OUTPUT_VARIABLE ARCHITECTURE)
+  if(${ARCHITECTURE} STREQUAL "x86_64")
+    execute_process(
+      COMMAND cat /proc/cpuinfo
+      COMMAND grep Intel
+      COMMAND head -1
+      COMMAND tr -d '\n'
+      OUTPUT_VARIABLE UARCH_INTEL)
+    if(NOT "${UARCH_INTEL} " STREQUAL " ")
+      set(BLIS_ARTIFACT_CONFIG "intel64")
+    else("${UARCH_INTEL} " STREQUAL " ")
+      set(BLIS_ARTIFACT_CONFIG "amd64")
+    endif("${UARCH_INTEL} " STREQUAL " ")
+  elseif(${ARCHITECTURE} STREQUAL "aarch64" OR
+      ${ARCHITECTURE} STREQUAL "arm64")
+      # Currently SVE requires being set manually.
+      set(BLIS_ARTIFACT_CONFIG "arm64")
+  else()
+    message(FATAL_ERROR "Failed to recognize architecture ${ARCHITECTURE}.")
+  endif()
+endif(NOT DEFINED BLIS_ARTIFACT_CONFIG)
+
+message(STATUS "Downloading BLIS artifact...")
+
+set(BLIS_ARCHIVE libblis_artifact.tar.gz)
+if(${BLIS_ARTIFACT_CONFIG} STREQUAL "intel64")
+  set(BLIS_ARTIFACT_URL https://github.com/xrq-phys/blis/releases/download/sv0.8.1+arm/libblis_intel64_gcc.tar.gz)
+elseif(${BLIS_ARTIFACT_CONFIG} STREQUAL "amd64")
+  set(BLIS_ARTIFACT_URL https://github.com/xrq-phys/blis/releases/download/sv0.8.1+arm/libblis_amd64_gcc.tar.gz)
+elseif(${BLIS_ARTIFACT_CONFIG} STREQUAL "arm64")
+  set(BLIS_ARTIFACT_URL https://github.com/xrq-phys/blis/releases/download/sv0.8.1+arm/libblis_cortexa57_aarch64-linux-gnu-gcc.tar.gz)
+elseif(${BLIS_ARTIFACT_CONFIG} STREQUAL "armsve")
+  set(BLIS_ARTIFACT_URL https://github.com/xrq-phys/blis/releases/download/sv0.8.1+arm/libblis_armsve_aarch64-linux-gnu-gcc-10.tar.gz)
+elseif(${BLIS_ARTIFACT_CONFIG} STREQUAL "a64fx")
+  set(BLIS_ARTIFACT_URL https://github.com/xrq-phys/blis/releases/download/sv0.8.1+arm/libblis_a64fx_aarch64-linux-gnu-gcc-10.tar.gz)
+else()
+  message(FATAL_ERROR "Not BLIS artifact available.")
+endif()
+
+file(DOWNLOAD "${BLIS_ARTIFACT_URL}" "${BLIS_ARCHIVE}")
+
+add_custom_command(
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/lib/libblis.a
+    ${CMAKE_CURRENT_BINARY_DIR}/include/blis/blis.h
+  COMMAND ${CMAKE_COMMAND} -E tar -zxf ${BLIS_ARCHIVE}
+  DEPENDS ${BLIS_ARCHIVE})
+add_custom_target(blis_target
+  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/lib/libblis.a)
+add_custom_target(blis_include
+  DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/include/blis/blis.h)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/include/blis)
+
+add_library(blis STATIC IMPORTED GLOBAL)
+set_target_properties(blis PROPERTIES
+  IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/lib/libblis.a)
+add_dependencies(blis blis_target)
+
diff --git a/src/mVMC/CMakeLists.txt b/src/mVMC/CMakeLists.txt
index 6f545bb6..0a0347a4 100644
--- a/src/mVMC/CMakeLists.txt
+++ b/src/mVMC/CMakeLists.txt
@@ -27,7 +27,11 @@ add_executable(vmcdry.out vmcdry.c)
 target_link_libraries(vmcdry.out StdFace m)
 add_executable(vmc.out ${SOURCES_vmcmain} ${SOURCES_sfmt})
 target_link_libraries(vmc.out StdFace)
-target_link_libraries(vmc.out pfapack StdFace ${LAPACK_LIBRARIES} m)
+target_link_libraries(vmc.out pfapack)
+if(PFAFFIAN_BLOCKED)
+  target_link_libraries(vmc.out pfupdates blis pthread)
+endif(PFAFFIAN_BLOCKED)
+target_link_libraries(vmc.out ${LAPACK_LIBRARIES} m)
 
 if(USE_SCALAPACK)
   string(REGEX REPLACE "-L[ ]+" "-L" sc_libs ${SCALAPACK_LIBRARIES})
diff --git a/src/pfupdates/CMakeLists.txt b/src/pfupdates/CMakeLists.txt
new file mode 100644
index 00000000..5f313232
--- /dev/null
+++ b/src/pfupdates/CMakeLists.txt
@@ -0,0 +1,18 @@
+# include guard
+cmake_minimum_required(VERSION 2.8.0 )
+
+if(${CMAKE_PROJECT_NAME} STREQUAL "Project")
+  message(FATAL_ERROR "cmake should be executed not for 'src' subdirectory, but for the top directory of mVMC.")
+endif(${CMAKE_PROJECT_NAME} STREQUAL "Project")
+
+add_definitions(-DBLAS_EXTERNAL)
+add_definitions(-DF77_COMPLEX_RET_INTEL)
+include_directories(../pfaffine/src)
+
+add_library(pfupdates STATIC
+  pf_interface.cc
+  ../pfaffine/src/skpfa.cc
+  ../pfaffine/src/sktdf.cc
+  ../pfaffine/src/sktdi.cc)
+target_compile_definitions(pfupdates PRIVATE -D_CC_IMPL)
+

From 128391a5a6c57340664359bddcc889a4f578737e Mon Sep 17 00:00:00 2001
From: RuQing Xu <r-xu@g.ecc.u-tokyo.ac.jp>
Date: Thu, 23 Sep 2021 02:48:48 +0900
Subject: [PATCH 3/8] CMake Set C++ 11 Std

---
 CMakeLists.txt               | 1 +
 download_blis_artifact.cmake | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 99009504..2d1f70c4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -18,6 +18,7 @@ option(BUILD_SHARED_LIBS "Build shared libraries" ON)
 enable_language(C Fortran)
 if(PFAFFIAN_BLOCKED)
   enable_language(CXX)
+  set(CMAKE_CXX_STANDARD 11)
 endif(PFAFFIAN_BLOCKED)
 
 set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
diff --git a/download_blis_artifact.cmake b/download_blis_artifact.cmake
index a6b6272b..0789d14d 100644
--- a/download_blis_artifact.cmake
+++ b/download_blis_artifact.cmake
@@ -12,9 +12,9 @@ if(NOT DEFINED BLIS_ARTIFACT_CONFIG)
       OUTPUT_VARIABLE UARCH_INTEL)
     if(NOT "${UARCH_INTEL} " STREQUAL " ")
       set(BLIS_ARTIFACT_CONFIG "intel64")
-    else("${UARCH_INTEL} " STREQUAL " ")
+    else()
       set(BLIS_ARTIFACT_CONFIG "amd64")
-    endif("${UARCH_INTEL} " STREQUAL " ")
+    endif()
   elseif(${ARCHITECTURE} STREQUAL "aarch64" OR
       ${ARCHITECTURE} STREQUAL "arm64")
       # Currently SVE requires being set manually.

From ab32a3743a5ece510304495396116e9505e5747b Mon Sep 17 00:00:00 2001
From: RuQing Xu <r-xu@g.ecc.u-tokyo.ac.jp>
Date: Thu, 23 Sep 2021 02:59:59 +0900
Subject: [PATCH 4/8] Update Configs

---
 config/aocc.cmake    | 16 ++++++++++++++++
 config/apple.cmake   |  1 +
 config/fugaku.cmake  | 19 +++++++++++++++++++
 config/fujitsu.cmake |  1 +
 config/gcc.cmake     |  3 ++-
 config/intel.cmake   |  5 +++++
 config/sekirei.cmake | 17 -----------------
 7 files changed, 44 insertions(+), 18 deletions(-)
 create mode 100644 config/aocc.cmake
 create mode 100644 config/fugaku.cmake
 delete mode 100644 config/sekirei.cmake

diff --git a/config/aocc.cmake b/config/aocc.cmake
new file mode 100644
index 00000000..8f055004
--- /dev/null
+++ b/config/aocc.cmake
@@ -0,0 +1,16 @@
+# for AOCC
+
+set(CMAKE_C_COMPILER "clang" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "clang++" CACHE STRING "" FORCE)
+set(CMAKE_C_FLAGS_DEBUG "-g -O0 -Wall -Wformat -Werror=format-security")
+set(CMAKE_C_FLAGS_RELEASE "-Wno-unknown-pragmas -O3 -DNDEBUG -DHAVE_SSE2" CACHE STRING "" FORCE)
+
+set(CMAKE_Fortran_COMPILER "flang" CACHE STRING "" FORCE)
+set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG -DHAVE_SSE2" CACHE STRING "" FORCE)
+
+# for AOCL
+set(BLA_VENDOR "FLAME" CACHE STRING "" FORCE)
+
+# for BLIS & block pfaffian
+set(BLIS_ARTIFACT_CONFIG "amd64")
+set(PFAFFIAN_BLOCKED ON)
diff --git a/config/apple.cmake b/config/apple.cmake
index 9da8493e..9b197129 100644
--- a/config/apple.cmake
+++ b/config/apple.cmake
@@ -2,6 +2,7 @@
 # additional libomp and gfortran installation required
 # mac computers are suggested to use this configuration for better performance
 set(CMAKE_C_COMPILER "clang" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "clang++" CACHE STRING "" FORCE)
 set(CMAKE_C_FLAGS_DEBUG "-g -O0 -Wall  -Wformat -Werror=format-security")
 set(CMAKE_C_FLAGS_RELEASE "-O3 -Wno-unknown-pragmas -Wno-logical-not-parentheses")
 set(CMAKE_Fortran_COMPILER "gfortran" CACHE STRING "" FORCE)
diff --git a/config/fugaku.cmake b/config/fugaku.cmake
new file mode 100644
index 00000000..a1158bb7
--- /dev/null
+++ b/config/fugaku.cmake
@@ -0,0 +1,19 @@
+# for Fujitsu Compiler in Clang-mode
+set(CMAKE_C_COMPILER "mpifccpx -Nclang" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "mpifCCpx -Nclang" CACHE STRING "" FORCE)
+set(CMAKE_C_FLAGS_RELEASE "-Ofast -DNDEBUG" CACHE STRING "" FORCE)
+set(CMAKE_CXX_FLAGS_RELEASE "-Ofast -DNDEBUG" CACHE STRING "" FORCE)
+
+set(CMAKE_Fortran_COMPILER "mpifrtpx" CACHE STRING "" FORCE)
+set(CMAKE_Fortran_FLAGS_RELEASE "-Kfast,parallel -DNDEBUG -DFUJITSU" CACHE STRING "" FORCE)
+
+set(OpenMP_C_FLAGS "-fopenmp" CACHE STRING "" FORCE)
+set(OpenMP_Fortran_FLAGS "-Kopenmp" CACHE STRING "" FORCE)
+
+# for SSL2
+set(BLAS_LIBRARIES "-SSL2" CACHE STRING "" FORCE)
+set(LAPACK_LIBRARIES ${BLAS_LIBRARIES} CACHE STRING "" FORCE)
+
+# for BLIS & block pfaffian
+set(BLIS_ARTIFACT_CONFIG "a64fx")
+set(PFAFFIAN_BLOCKED ON)
diff --git a/config/fujitsu.cmake b/config/fujitsu.cmake
index e4b01a98..1e6a9a9d 100644
--- a/config/fujitsu.cmake
+++ b/config/fujitsu.cmake
@@ -1,5 +1,6 @@
 # for Fujitsu Compiler
 set(CMAKE_C_COMPILER "mpifccpx" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "mpifCCpx" CACHE STRING "" FORCE)
 set(CMAKE_C_FLAGS_RELEASE "-DNDEBUG -Kfast,parallel -Kmemalias,alias_const" CACHE STRING "" FORCE)
 set(OpenMP_C_FLAGS "-Kopenmp" CACHE STRING "" FORCE)
 set(CMAKE_Fortran_COMPILER "mpifrtpx" CACHE STRING "" FORCE)
diff --git a/config/gcc.cmake b/config/gcc.cmake
index d14c82bb..751912d7 100644
--- a/config/gcc.cmake
+++ b/config/gcc.cmake
@@ -1,5 +1,6 @@
 # for GCC Compiler
 set(CMAKE_C_COMPILER "gcc" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "g++" CACHE STRING "" FORCE)
 set(CMAKE_C_FLAGS_DEBUG "-g -O0 -Wall  -Wformat -Werror=format-security")
 set(CMAKE_C_FLAGS_RELEASE "-O3 -Wno-unknown-pragmas ")
-Set(CMAKE_Fortran_COMPILER "gfortran" CACHE STRING "" FORCE)
+set(CMAKE_Fortran_COMPILER "gfortran" CACHE STRING "" FORCE)
diff --git a/config/intel.cmake b/config/intel.cmake
index 4194aad1..6e19049b 100644
--- a/config/intel.cmake
+++ b/config/intel.cmake
@@ -1,6 +1,7 @@
 # for Intel Compiler
 
 set(CMAKE_C_COMPILER "icc" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "icpc" CACHE STRING "" FORCE)
 set(CMAKE_C_FLAGS_DEBUG "-O0 -g -Wall -Wformat -Werror=format-security")
 set(CMAKE_C_FLAGS_RELEASE "-Wno-unknown-pragmas -O3 -DNDEBUG -DHAVE_SSE2" CACHE STRING "" FORCE)
 
@@ -9,3 +10,7 @@ set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG -DHAVE_SSE2" CACHE STRING "" FORCE
 
 # for Intel MKL
 set(BLA_VENDOR "Intel10_64lp" CACHE STRING "" FORCE)
+
+# for BLIS & block pfaffian
+set(BLIS_ARTIFACT_CONFIG "intel64")
+set(PFAFFIAN_BLOCKED ON)
diff --git a/config/sekirei.cmake b/config/sekirei.cmake
deleted file mode 100644
index 7f2cf557..00000000
--- a/config/sekirei.cmake
+++ /dev/null
@@ -1,17 +0,0 @@
-# for Intel Compiler
-set(CMAKE_C_COMPILER "icc" CACHE STRING "" FORCE)
-set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG -xCORE-AVX2 -mcmodel=large -shared-intel -Wno-unknown-pragmas" CACHE STRING "" FORCE)
-set(CMAKE_C_FLAGS_DEBUG "-g -O0 -Wall  -Wformat -Werror=format-security")
-
-set(CMAKE_Fortran_COMPILER "ifort" CACHE STRING "" FORCE)
-set(CMAKE_Fortran_FLAGS "-O3 -DNDEBUG -xCORE-AVX2 -mcmodel=large -shared-intel" CACHE STRING "" FORCE)
-
-set(USE_SCALAPACK ON)
-if(SCALAPACK_LIBRARIES MATCHES "")
-  set(SCALAPACK_LIBRARIES "-L$ENV{MKLROOT}/lib/intel64 -lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_sgimpt_lp64")
-endif(SCALAPACK_LIBRARIES MATCHES "")
-
-message(STATUS "SCALAPACK_LIBRARIES is ${SCALAPACK_LIBRARIES}")
-
-# for Intel MKL
-set(BLA_VENDOR "Intel10_64lp" CACHE STRING "" FORCE)

From b954eb5a3e7cdbf6c9849becceaf718652d207b4 Mon Sep 17 00:00:00 2001
From: RuQing Xu <r-xu@g.ecc.u-tokyo.ac.jp>
Date: Thu, 23 Sep 2021 03:09:29 +0900
Subject: [PATCH 5/8] Fix Fugaku Config Typo

---
 config/fugaku.cmake | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/config/fugaku.cmake b/config/fugaku.cmake
index a1158bb7..2e33c79b 100644
--- a/config/fugaku.cmake
+++ b/config/fugaku.cmake
@@ -1,8 +1,8 @@
 # for Fujitsu Compiler in Clang-mode
-set(CMAKE_C_COMPILER "mpifccpx -Nclang" CACHE STRING "" FORCE)
-set(CMAKE_CXX_COMPILER "mpifCCpx -Nclang" CACHE STRING "" FORCE)
-set(CMAKE_C_FLAGS_RELEASE "-Ofast -DNDEBUG" CACHE STRING "" FORCE)
-set(CMAKE_CXX_FLAGS_RELEASE "-Ofast -DNDEBUG" CACHE STRING "" FORCE)
+set(CMAKE_C_COMPILER "mpifccpx" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "mpifCCpx" CACHE STRING "" FORCE)
+set(CMAKE_C_FLAGS_RELEASE "-Nclang -Ofast -DNDEBUG" CACHE STRING "" FORCE)
+set(CMAKE_CXX_FLAGS_RELEASE "-Nclang -Ofast -DNDEBUG" CACHE STRING "" FORCE)
 
 set(CMAKE_Fortran_COMPILER "mpifrtpx" CACHE STRING "" FORCE)
 set(CMAKE_Fortran_FLAGS_RELEASE "-Kfast,parallel -DNDEBUG -DFUJITSU" CACHE STRING "" FORCE)

From f1d8d62dbe2987d2c5e5917f9f77ddf65f33629a Mon Sep 17 00:00:00 2001
From: RuQing Xu <r-xu@g.ecc.u-tokyo.ac.jp>
Date: Thu, 23 Sep 2021 03:10:09 +0900
Subject: [PATCH 6/8] Fix Fujitsu Compiler Config Typo

---
 config/fugaku.cmake  | 2 +-
 config/fujitsu.cmake | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/fugaku.cmake b/config/fugaku.cmake
index 2e33c79b..ad814bde 100644
--- a/config/fugaku.cmake
+++ b/config/fugaku.cmake
@@ -1,6 +1,6 @@
 # for Fujitsu Compiler in Clang-mode
 set(CMAKE_C_COMPILER "mpifccpx" CACHE STRING "" FORCE)
-set(CMAKE_CXX_COMPILER "mpifCCpx" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "mpiFCCpx" CACHE STRING "" FORCE)
 set(CMAKE_C_FLAGS_RELEASE "-Nclang -Ofast -DNDEBUG" CACHE STRING "" FORCE)
 set(CMAKE_CXX_FLAGS_RELEASE "-Nclang -Ofast -DNDEBUG" CACHE STRING "" FORCE)
 
diff --git a/config/fujitsu.cmake b/config/fujitsu.cmake
index 1e6a9a9d..c04f9742 100644
--- a/config/fujitsu.cmake
+++ b/config/fujitsu.cmake
@@ -1,6 +1,6 @@
 # for Fujitsu Compiler
 set(CMAKE_C_COMPILER "mpifccpx" CACHE STRING "" FORCE)
-set(CMAKE_CXX_COMPILER "mpifCCpx" CACHE STRING "" FORCE)
+set(CMAKE_CXX_COMPILER "mpiFCCpx" CACHE STRING "" FORCE)
 set(CMAKE_C_FLAGS_RELEASE "-DNDEBUG -Kfast,parallel -Kmemalias,alias_const" CACHE STRING "" FORCE)
 set(OpenMP_C_FLAGS "-Kopenmp" CACHE STRING "" FORCE)
 set(CMAKE_Fortran_COMPILER "mpifrtpx" CACHE STRING "" FORCE)

From 1e24b1e54fc839e5e163c736060d276041daaf7e Mon Sep 17 00:00:00 2001
From: RuQing Xu <r-xu@g.ecc.u-tokyo.ac.jp>
Date: Thu, 23 Sep 2021 03:24:52 +0900
Subject: [PATCH 7/8] Fujitsu Compiler Needs OMP Optn. Even at Link

---
 config/fugaku.cmake  | 1 +
 config/fujitsu.cmake | 1 +
 2 files changed, 2 insertions(+)

diff --git a/config/fugaku.cmake b/config/fugaku.cmake
index ad814bde..bac55b78 100644
--- a/config/fugaku.cmake
+++ b/config/fugaku.cmake
@@ -9,6 +9,7 @@ set(CMAKE_Fortran_FLAGS_RELEASE "-Kfast,parallel -DNDEBUG -DFUJITSU" CACHE STRIN
 
 set(OpenMP_C_FLAGS "-fopenmp" CACHE STRING "" FORCE)
 set(OpenMP_Fortran_FLAGS "-Kopenmp" CACHE STRING "" FORCE)
+set(CMAKE_EXE_LINKER_FLAGS "-fopenmp" CACHE STRING "" FORCE)
 
 # for SSL2
 set(BLAS_LIBRARIES "-SSL2" CACHE STRING "" FORCE)
diff --git a/config/fujitsu.cmake b/config/fujitsu.cmake
index c04f9742..559e0e1f 100644
--- a/config/fujitsu.cmake
+++ b/config/fujitsu.cmake
@@ -7,6 +7,7 @@ set(CMAKE_Fortran_COMPILER "mpifrtpx" CACHE STRING "" FORCE)
 set(CMAKE_Fortran_FLAGS_RELEASE "-DNDEBUG  -DFUJITSU -Kfast,parallel" CACHE STRING "" FORCE)
 set(OpenMP_Fortran_FLAGS "-Kopenmp" CACHE STRING "" FORCE)
 set(MPI_EXTRA_LIBRARY "-lmpi_f90 -lmpi_f77" CACHE STRING "" FORCE)
+set(CMAKE_EXE_LINKER_FLAGS "-Kopenmp" CACHE STRING "" FORCE)
 
 # for SSL2
 set(BLAS_LIBRARIES "-SSL2" CACHE STRING "" FORCE)

From 1a4597e5a02915a748245615d631ea640fc3a88f Mon Sep 17 00:00:00 2001
From: RuQing Xu <r-xu@g.ecc.u-tokyo.ac.jp>
Date: Thu, 23 Sep 2021 03:25:41 +0900
Subject: [PATCH 8/8] Some Ver. CMake Doesn't Recognize Filename as Downlaod
 Target

---
 download_blis_artifact.cmake | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/download_blis_artifact.cmake b/download_blis_artifact.cmake
index 0789d14d..f6274bcc 100644
--- a/download_blis_artifact.cmake
+++ b/download_blis_artifact.cmake
@@ -26,7 +26,7 @@ endif(NOT DEFINED BLIS_ARTIFACT_CONFIG)
 
 message(STATUS "Downloading BLIS artifact...")
 
-set(BLIS_ARCHIVE libblis_artifact.tar.gz)
+set(BLIS_ARCHIVE ${CMAKE_CURRENT_BINARY_DIR}/libblis_artifact.tar.gz)
 if(${BLIS_ARTIFACT_CONFIG} STREQUAL "intel64")
   set(BLIS_ARTIFACT_URL https://github.com/xrq-phys/blis/releases/download/sv0.8.1+arm/libblis_intel64_gcc.tar.gz)
 elseif(${BLIS_ARTIFACT_CONFIG} STREQUAL "amd64")
@@ -41,6 +41,7 @@ else()
   message(FATAL_ERROR "Not BLIS artifact available.")
 endif()
 
+message("${BLIS_ARTIFACT_URL} ${BLIS_ARCHIVE}")
 file(DOWNLOAD "${BLIS_ARTIFACT_URL}" "${BLIS_ARCHIVE}")
 
 add_custom_command(