added MPI rank to the trace (#169)

.gitignore

@@ -37,7 +37,7 @@
 .cache
 
 # Environment and Dependency
-venv*
+*venv*
 .venv*
 dependency/.spack-env
 dependency/spack.lock

.vscode/settings.json

@@ -90,6 +90,7 @@
         "typeindex": "cpp",
         "typeinfo": "cpp",
         "valarray": "cpp",
-        "variant": "cpp"
+        "variant": "cpp",
+        "ranges": "cpp"
     }
 }

CMakeLists.txt

@@ -111,6 +111,11 @@ option (DFTRACER_DISABLE_HWLOC "Disable HWLOC" On)
 if (NOT DFTRACER_DISABLE_HWLOC) 
     set(DFTRACER_HWLOC_ENABLE 1)
 endif()
+
+option (DFTRACER_ENABLE_MPI "Enable MPI" On)
+if (DFTRACER_ENABLE_MPI) 
+    set(DFTRACER_MPI_ENABLE 1)
+endif()
 option (DFTRACER_BUILD_PYTHON_BINDINGS "Build python bindings." ON)
 set(DFTRACER_PYTHON_EXE "python3" CACHE STRING "Python executable to use for building.")
 set(DFTRACER_PYTHON_SITE $ENV{DFTRACER_PYTHON_SITE} CACHE STRING "Python site packages to use for building.")
@@ -218,6 +223,18 @@ if (DFTRACER_BUILD_PYTHON_BINDINGS)
     find_package(pybind11 REQUIRED)
     #add_subdirectory(dependency/py11bind)
 endif()
+
+if (DFTRACER_ENABLE_MPI) 
+  find_package(MPI COMPONENTS CXX REQUIRED)
+  if (MPI_FOUND)
+      message(STATUS "[DFTRACER] found mpi.h at ${MPI_CXX_INCLUDE_DIRS}")
+      include_directories(${MPI_CXX_INCLUDE_DIRS})
+      set(DEPENDENCY_LIB ${DEPENDENCY_LIB} ${MPI_CXX_LIBRARIES})
+  else ()
+      message(FATAL_ERROR "-- [DFTRACER] mpi is needed for ${PROJECT_NAME} build")
+  endif ()
+endif()
+
 if (NOT DFTRACER_DISABLE_HWLOC)
     find_package(HWLOC REQUIRED)
     if (${HWLOC_FOUND})
@@ -393,14 +410,10 @@ install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules/dftracer-utils.cmake"
 #-----------------------------------------------------------------------------
 # Configure the ${PROJECT_NAME}-config-version .cmake file for the install directory
 #-----------------------------------------------------------------------------
-configure_file(
-        ${CMAKE_CURRENT_SOURCE_DIR}/cmake/configure_files/${PROJECT_NAME}-config-version.cmake.in
-        ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cmake/${PROJECT_NAME}/${PROJECT_NAME}-config-version.cmake @ONLY
-)
 
 install(
         FILES
-        ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cmake/${PROJECT_NAME}/${PROJECT_NAME}-config-version.cmake
+        ${CMAKE_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake
         DESTINATION
         ${DFTRACER_LIBDIR}/cmake/${PROJECT_NAME}
 )
@@ -421,7 +434,7 @@ include(CMakePackageConfigHelpers)
 
 # Write the version file. This is independent of build/install tree.
 write_basic_package_version_file(
-  DFTRACERConfigVersion.cmake
+  dftracer-config-version.cmake
   VERSION "${DFTRACER_PACKAGE_VERSION}"
   COMPATIBILITY SameMajorVersion)
 

cmake/configure_files/dftracer-config.cmake.build.in

@@ -75,6 +75,18 @@ else ()
     message(FATAL_ERROR "-- [DFTRACER] yaml-cpp is needed for ${PROJECT_NAME} build")
 endif ()
 
+set(DFTRACER_INCLUDE_MPI @DFTRACER_ENABLE_MPI@)
+if (DFTRACER_INCLUDE_MPI)
+  find_package(MPI COMPONENTS CXX REQUIRED)
+  if (MPI_FOUND)
+        message(STATUS "[DFTRACER] found mpi.h at ${MPI_CXX_INCLUDE_DIRS}")
+        include_directories(${MPI_CXX_INCLUDE_DIRS})
+        target_link_libraries(dftracer INTERFACE ${MPI_CXX_LIBRARIES})
+    else ()
+        message(FATAL_ERROR "-- [DFTRACER] mpi is needed for ${PROJECT_NAME} build")
+    endif ()
+endif()
+
 check_required_components(dftracer)
 
 set(DFTRACER_LIBRARIES dftracer)

cmake/configure_files/dftracer-config.cmake.install.in

@@ -74,6 +74,18 @@ else ()
     message(FATAL_ERROR "-- [DFTRACER] yaml-cpp is needed for ${PROJECT_NAME} build")
 endif ()
 
+set(DFTRACER_INCLUDE_MPI @DFTRACER_ENABLE_MPI@)
+if (DFTRACER_INCLUDE_MPI)
+  find_package(MPI COMPONENTS CXX REQUIRED)
+  if (MPI_FOUND)
+        message(STATUS "[DFTRACER] found mpi.h at ${MPI_CXX_INCLUDE_DIRS}")
+        include_directories(${MPI_CXX_INCLUDE_DIRS})
+        target_link_libraries(dftracer INTERFACE ${MPI_CXX_LIBRARIES})
+    else ()
+        message(FATAL_ERROR "-- [DFTRACER] mpi is needed for ${PROJECT_NAME} build")
+    endif ()
+endif()
+
 check_required_components(dftracer)
 
 set(DFTRACER_LIBRARIES dftracer)

cmake/configure_files/dftracer_config.hpp.in

@@ -1,36 +1,36 @@
 #ifndef DFTRACER_CONFIG_HPP
 #define DFTRACER_CONFIG_HPP
-
+// clang-format off
 /* Version string for DFTRACER */
 #define DFTRACER_PACKAGE_VERSION @DFTRACER_PACKAGE_VERSION@
 #cmakedefine DFTRACER_GIT_VERSION @DFTRACER_GIT_VERSION@
-
 #define DFTRACER_GET_VERSION(MAJOR, MINOR, PATCH) (MAJOR * 100000 + MINOR * 100 + PATCH)
 #define DFTRACER_VERSION (DFTRACER_GET_VERSION @DFTRACER_VERSION@)
 #define DFTRACER_VERSION_MAJOR (DFTRACER_VERSION / 100000)
 #define DFTRACER_VERSION_MINOR ((DFTRACER_VERSION / 100) % 1000)
 #define DFTRACER_VERSION_PATCH (DFTRACER_VERSION % 100)
 
 /* Compiler used */
-#cmakedefine CMAKE_BUILD_TYPE "@CMAKE_BUILD_TYPE@"
+#cmakedefine DFTRACER_CMAKE_BUILD_TYPE "@CMAKE_BUILD_TYPE@"
 
-#cmakedefine CMAKE_C_COMPILER "@CMAKE_C_COMPILER@"
-#cmakedefine CMAKE_C_FLAGS "@CMAKE_C_FLAGS@"
-#cmakedefine CMAKE_C_FLAGS_DEBUG "@CMAKE_C_FLAGS_DEBUG@"
-#cmakedefine CMAKE_C_FLAGS_RELWITHDEBINFO "@CMAKE_C_FLAGS_RELWITHDEBINFO@"
-#cmakedefine CMAKE_C_FLAGS_RELEASE "@CMAKE_C_FLAGS@_RELEASE"
+#cmakedefine DFTRACER_CMAKE_C_COMPILER "@CMAKE_C_COMPILER@"
+#cmakedefine DFTRACER_CMAKE_C_FLAGS "@CMAKE_C_FLAGS@"
+#cmakedefine DFTRACER_CMAKE_C_FLAGS_DEBUG "@CMAKE_C_FLAGS_DEBUG@"
+#cmakedefine DFTRACER_CMAKE_C_FLAGS_RELWITHDEBINFO "@CMAKE_C_FLAGS_RELWITHDEBINFO@"
+#cmakedefine DFTRACER_CMAKE_C_FLAGS_RELEASE "@CMAKE_C_FLAGS@_RELEASE"
 
-#cmakedefine CMAKE_CXX_COMPILER "@CMAKE_CXX_COMPILER@"
-#cmakedefine CMAKE_CXX_FLAGS "@CMAKE_CXX_FLAGS@"
-#cmakedefine CMAKE_CXX_FLAGS_DEBUG "@CMAKE_CXX_FLAGS_DEBUG@"
-#cmakedefine CMAKE_CXX_FLAGS_RELWITHDEBINFO "@CMAKE_CXX_FLAGS_RELWITHDEBINFO@"
-#cmakedefine CMAKE_CXX_FLAGS_RELEASE "@CMAKE_CXX_FLAGS_RELEASE@"
+#cmakedefine DFTRACER_CMAKE_CXX_COMPILER "@CMAKE_CXX_COMPILER@"
+#cmakedefine DFTRACER_CMAKE_CXX_FLAGS "@CMAKE_CXX_FLAGS@"
+#cmakedefine DFTRACER_CMAKE_CXX_FLAGS_DEBUG "@CMAKE_CXX_FLAGS_DEBUG@"
+#cmakedefine DFTRACER_CMAKE_CXX_FLAGS_RELWITHDEBINFO "@CMAKE_CXX_FLAGS_RELWITHDEBINFO@"
+#cmakedefine DFTRACER_CMAKE_CXX_FLAGS_RELEASE "@CMAKE_CXX_FLAGS_RELEASE@"
 
-#cmakedefine CMAKE_C_SHARED_LIBRARY_FLAGS "@CMAKE_C_SHARED_LIBRARY_FLAGS@"
-#cmakedefine CMAKE_CXX_SHARED_LIBRARY_FLAGS "@CMAKE_CXX_SHARED_LIBRARY_FLAGS@"
+#cmakedefine DFTRACER_CMAKE_C_SHARED_LIBRARY_FLAGS "@CMAKE_C_SHARED_LIBRARY_FLAGS@"
+#cmakedefine DFTRACER_CMAKE_CXX_SHARED_LIBRARY_FLAGS "@CMAKE_CXX_SHARED_LIBRARY_FLAGS@"
 
 /* Macro flags */
 #cmakedefine DFTRACER_GNU_LINUX 1
+#cmakedefine DFTRACER_MPI_ENABLE 1
 
 //==========================
 // Common macro definitions
@@ -55,5 +55,5 @@
 #if !defined(DFTRACER_HASH_SEED) || (DFTRACER_HASH_SEED <= 0)
 #define DFTRACER_SEED 104723u
 #endif
-
+// clang-format on
 #endif /* DFTRACER_CONFIG_H */

setup.py

@@ -72,6 +72,8 @@ def build_extension(self, ext: CMakeExtension) -> None:
         # auxiliary "native" libs
         build_type = os.environ.get("DFTRACER_BUILD_TYPE", "Release")
         cmake_args += [f"-DCMAKE_BUILD_TYPE={build_type}"]
+        enable_mpi = os.environ.get("DFTRACER_ENABLE_MPI", "ON")
+        cmake_args += [f"-DDFTRACER_ENABLE_MPI={enable_mpi}"]
         disable_hwloc = os.environ.get("DFTRACER_DISABLE_HWLOC", "ON")
         cmake_args += [f"-DDFTRACER_DISABLE_HWLOC={disable_hwloc}"]
         cmake_args += [f"-DDFTRACER_PYTHON_EXE={sys.executable}"]

src/dftracer/df_logger.h

@@ -22,6 +22,10 @@
 #include <dftracer/dftracer_config.hpp>
 #include <unordered_map>
 
+#ifdef DFTRACER_MPI_ENABLE
+#include <mpi.h>
+#endif
+
 typedef std::chrono::high_resolution_clock chrono;
 
 class DFTLogger {
@@ -34,7 +38,9 @@ class DFTLogger {
   std::vector<int> index_stack;
   std::atomic_int index;
   bool has_entry;
-
+#ifdef DFTRACER_MPI_ENABLE
+  bool mpi_event;
+#endif
  public:
   bool include_metadata;
   DFTLogger(bool init_log = false)
@@ -44,6 +50,9 @@ class DFTLogger {
         index_stack(),
         index(0),
         has_entry(false),
+#ifdef DFTRACER_MPI_ENABLE
+        mpi_event(false),
+#endif
         include_metadata(false) {
     DFTRACER_LOGDEBUG("DFTLogger.DFTLogger", "");
     auto conf =
@@ -64,18 +73,24 @@ class DFTLogger {
     this->writer = dftracer::Singleton<dftracer::ChromeWriter>::get_instance();
     if (this->writer != nullptr) {
       this->writer->initialize(log_file.data(), this->throw_error);
-      auto meta = std::unordered_map<std::string, std::any>();
-      meta.insert_or_assign("version", DFTRACER_VERSION);
-      meta.insert_or_assign("exec", exec_name);
-      meta.insert_or_assign("cmd", cmd);
-      time_t ltime;       /* calendar time */
-      ltime = time(NULL); /* get current cal time */
-      char timestamp[1024];
-      auto size = sprintf(timestamp, "%s", asctime(localtime(&ltime)));
-      timestamp[size - 1] = '\0';
-      meta.insert_or_assign("date", std::string(timestamp));
+      std::unordered_map<std::string, std::any> *meta = nullptr;
+      if (include_metadata) {
+        meta = new std::unordered_map<std::string, std::any>();
+        meta->insert_or_assign("version", DFTRACER_VERSION);
+        meta->insert_or_assign("exec", exec_name);
+        meta->insert_or_assign("cmd", cmd);
+        time_t ltime;       /* calendar time */
+        ltime = time(NULL); /* get current cal time */
+        char timestamp[1024];
+        auto size = sprintf(timestamp, "%s", asctime(localtime(&ltime)));
+        timestamp[size - 1] = '\0';
+        meta->insert_or_assign("date", std::string(timestamp));
+      }
       this->enter_event();
-      this->log("start", "dftracer", this->get_time(), 0, &meta);
+      this->log("start", "dftracer", this->get_time(), 0, meta);
+      if (include_metadata) {
+        delete (meta);
+      }
       this->exit_event();
     }
     this->is_init = true;
@@ -117,6 +132,28 @@ class DFTLogger {
       }
       metadata->insert_or_assign("p_idx", parent_index_value);
     }
+#ifdef DFTRACER_MPI_ENABLE
+    if (!mpi_event && include_metadata) {
+      int initialized;
+      int status = MPI_Initialized(&initialized);
+      if (status == MPI_SUCCESS && initialized == true) {
+        int rank;
+        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+        auto meta = std::unordered_map<std::string, std::any>();
+        meta.insert_or_assign("rank", rank);
+        this->enter_event();
+        if (this->writer != nullptr) {
+          auto start = this->get_time();
+          TimeResolution dur = 0;
+          this->writer->log(index_stack[level - 1], "mpi", "dftracer", start,
+                            dur, &meta, this->process_id, tid);
+        }
+        this->exit_event();
+        mpi_event = true;
+      }
+    }
+#endif
+
     if (this->writer != nullptr) {
       this->writer->log(index_stack[level - 1], event_name, category,
                         start_time, duration, metadata, this->process_id, tid);

test/paper/CMakeLists.txt

@@ -25,7 +25,7 @@ if (${MPI_FOUND})
 else ()
     message(FATAL_ERROR "-- [DFTRACER] MPI is needed for ${PROJECT_NAME} paper test build")
 endif ()
-find_package(cpp-logger 0.0.4 REQUIRED)
+find_package(cpp-logger 1.0.0 REQUIRED)
 if (${cpp-logger_FOUND})
     include_directories(${CPP_LOGGER_INCLUDE_DIRS})
     set(TEST_DEPENDENCY_LIB ${TEST_DEPENDENCY_LIB} ${CPP_LOGGER_LIBRARIES})
@@ -70,14 +70,14 @@ function(add_df_test machine PPN NODES example example_name)
     list(SUBLIST ARGV 5 10 TEST_ARGS)
     add_base_test(${machine} ${PPN} ${NODES} ${example} ${example_name} ${TEST_ARGS})
     set(test_name test_paper_${example_name}_${machine}_${NODES}_${PPN})
-    set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT LD_PRELOAD=${CMAKE_BINARY_DIR}/lib/libdftracer_preload.so)
+    set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT LD_PRELOAD=${CMAKE_BINARY_DIR}/${DFTRACER_LIBDIR}/libdftracer_preload.so)
     set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_DATA_DIR=${DFTRACER_DATA_DIR})
     set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_LOG_FILE=${CMAKE_CURRENT_BINARY_DIR}/${test_name})
     set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_INIT=PRELOAD)
     set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_ENABLE=1)
     #set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_LOG_LEVEL=INFO)
     set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_TRACE_COMPRESSION=1)
-    set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_INC_METADATA=0)
+    set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_INC_METADATA=1)
     set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_TRACE_ALL_FILES=1)
     set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_TRACE_TIDS=0)
     set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_BIND_SIGNALS=0)