Merge branch 'develop' of https://github.com/hariharan-devarajan/dftr…

…acer into develop
LLNL · Oct 14, 2024 · 65deb0e · 65deb0e
2 parents d1774de + fcca51b
commit 65deb0e
Show file tree

Hide file tree

Showing 32 changed files with 1,292 additions and 1,107 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,8 +1,8 @@
-name: DFTracer Build and Test
+name: Build and Test
 
 on:
   pull_request:
-    branches: [ main, develop ]
+    branches: [main, develop]
   push:
 jobs:
   build-and-test:

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -27,6 +27,6 @@ sphinx:
 # Optional but recommended, declare the Python requirements required
 # to build your documentation
 # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-# python:
-#    install:
-#    - requirements: docs/requirements.txt
+python:
+  install:
+  - requirements: docs/requirements.txt
diff --git a/.trace-0-of-2.pfw b/.trace-0-of-2.pfw
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -8,9 +8,10 @@
             "name": "Cmake Debug Target",
             "type": "lldb",
             "request": "launch",
-            "program": "${cmake.testProgram}",
+            "program": "/usr/workspace/haridev/dftracer/build/bin/test_cpp",
             "args": [
-                "${cmake.testArgs}"
+                "/usr/workspace/haridev/dftracer/build/test/data",
+                "1"
             ],
             "cwd": "${workspaceFolder}/build",
             "env": {

diff --git a/CITATION.cff b/CITATION.cff
@@ -0,0 +1,91 @@
+cff-version: 1.2.0
+message: If you use this software, please cite the software and the paper.
+title: "DFTracer: An Analysis-Friendly Data Flow Tracer for AI-Driven Workflows"
+version: 1.0.6
+doi: 10.5281/zenodo.13917384
+abstract: A multi-level dataflow tracer designed for capturing I/O calls from AI-driven workflows.
+license: MIT
+url: https://github.com/hariharan-devarajan/dftracer
+repository-code: https://github.com/hariharan-devarajan/dftracer
+contact:
+  - name: Hariharan Devarajan
+    email: [email protected]
+authors:
+  - family-names: Devarajan
+    given-names: Hariharan
+    orcid: https://orcid.org/0000-0001-5625-3494
+  - family-names: Pottier
+    given-names: Loic
+    orcid: https://orcid.org/0000-0002-7681-3521
+  - family-names: Velusamy
+    given-names: Kaushik
+    orcid: https://orcid.org/0000-0003-4902-8760
+  - family-names: Zheng
+    given-names: Huihuo
+    orcid: https://orcid.org/0000-0001-9008-9552
+  - family-names: Yildirim
+    given-names: Izzet
+    orcid: https://orcid.org/0000-0003-3513-0764
+  - family-names: Kogiou
+    given-names: Olga
+    orcid: https://orcid.org/0009-0009-6043-4437
+  - family-names: Yu
+    given-names: Weikuan
+    orcid: https://orcid.org/0000-0002-8754-0311
+  - family-names: Kougkas
+    given-names: Anthony
+    orcid: https://orcid.org/0000-0003-3943-663X
+  - family-names: Sun
+    given-names: Xian-He
+    orcid: https://orcid.org/0000-0002-1093-0792
+  - family-names: Yeom
+    given-names: Jae Seung
+    orcid: https://orcid.org/0000-0001-5464-6040
+  - family-names: Mohror
+    given-names: Kathryn
+    orcid: https://orcid.org/0000-0002-1366-1655
+preferred-citation:
+  type: conference-paper
+  title: "DFTracer: An Analysis-Friendly Data Flow Tracer for AI-Driven Workflows"
+  year: 2024
+  authors:
+    - family-names: Devarajan
+      given-names: Hariharan
+      orcid: https://orcid.org/0000-0001-5625-3494
+    - family-names: Pottier
+      given-names: Loic
+      orcid: https://orcid.org/0000-0002-7681-3521
+    - family-names: Velusamy
+      given-names: Kaushik
+      orcid: https://orcid.org/0000-0003-4902-8760
+    - family-names: Zheng
+      given-names: Huihuo
+      orcid: https://orcid.org/0000-0001-9008-9552
+    - family-names: Yildirim
+      given-names: Izzet
+      orcid: https://orcid.org/0000-0003-3513-0764
+    - family-names: Kogiou
+      given-names: Olga
+      orcid: https://orcid.org/0009-0009-6043-4437
+    - family-names: Yu
+      given-names: Weikuan
+      orcid: https://orcid.org/0000-0002-8754-0311
+    - family-names: Kougkas
+      given-names: Anthony
+      orcid: https://orcid.org/0000-0003-3943-663X
+    - family-names: Sun
+      given-names: Xian-He
+      orcid: https://orcid.org/0000-0002-1093-0792
+    - family-names: Yeom
+      given-names: Jae Seung
+      orcid: https://orcid.org/0000-0001-5464-6040
+    - family-names: Mohror
+      given-names: Kathryn
+      orcid: https://orcid.org/0000-0002-1366-1655
+  conference:
+    name: "SC24: International Conference for High Performance Computing, Networking, Storage and Analysis"
+    city: Atlanta
+    region: GA
+    country: USA
+    date-start: 2024-11-17
+    date-end: 2024-11-22
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -14,15 +14,14 @@ set(DFTRACER_PACKAGE_VERSION_MINOR "${DFTRACER_VERSION_PATCH}")
 set(DFTRACER_PACKAGE_STRING "${DFTRACER_PACKAGE_NAME} ${DFTRACER_PACKAGE_VERSION}")
 set(DFTRACER_PACKAGE_TARNAME "${DFTRACER_PACKAGE}")
 
-set(DFTRACER_VERSION "(1, 0, 5)")
+set(DFTRACER_VERSION "(1, 0, 6)")
 
 project(dftracer LANGUAGES C CXX)
 
-
 # Convenience defines
 string(TOUPPER "${PROJECT_NAME}" UPPER_PROJECT_NAME)
 string(TOLOWER "${PROJECT_NAME}" LOWER_PROJECT_NAME)
-
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-empty-body -Wno-format-extra-args")
 #------------------------------------------------------------------------------
 # Internal Includes for header and libraries
 #------------------------------------------------------------------------------
@@ -85,7 +84,6 @@ if (NOT DFTRACER_EXPORTED_TARGETS)
     set(DFTRACER_EXPORTED_TARGETS "dftracer-targets")
 endif ()
 
-
 include(dftracer-utils)
 #------------------------------------------------------------------------------
 # Build options
@@ -108,28 +106,27 @@ endif (DFTRACER_USE_CLANG_LIBCXX)
 
 # Options
 option (DFTRACER_DISABLE_HWLOC "Disable HWLOC" On)
-if (NOT DFTRACER_DISABLE_HWLOC) 
+if (NOT DFTRACER_DISABLE_HWLOC)
     set(DFTRACER_HWLOC_ENABLE 1)
 endif()
 
 # Options
 option (DFTRACER_ENABLE_FTRACING "Enable Function Tracing" OFF)
-if (DFTRACER_ENABLE_FTRACING) 
+if (DFTRACER_ENABLE_FTRACING)
     set(DFTRACER_FTRACING_ENABLE 1)
     set(DFTRACER_FUNCTION_FLAGS "-g" "-finstrument-functions" "-Wl,-E" "-fvisibility=default")
 else()
     set(DFTRACER_FUNCTION_FLAGS )
 endif()
 
 option (DFTRACER_ENABLE_MPI "Enable MPI" OFF)
-if (DFTRACER_ENABLE_MPI) 
+if (DFTRACER_ENABLE_MPI)
     set(DFTRACER_MPI_ENABLE 1)
 endif()
 option (DFTRACER_BUILD_PYTHON_BINDINGS "Build python bindings." ON)
 set(DFTRACER_PYTHON_EXE "python3" CACHE STRING "Python executable to use for building.")
 set(DFTRACER_PYTHON_SITE $ENV{DFTRACER_PYTHON_SITE} CACHE STRING "Python site packages to use for building.")
 
-
 if (DFTRACER_BUILD_PYTHON_BINDINGS)
     if (DFTRACER_PYTHON_SITE)
         set(PYTHON_SITE_PACKAGES "${DFTRACER_PYTHON_SITE}")
@@ -192,7 +189,7 @@ endif ()
 # Dependencies
 #------------------------------------------------------------------------------
 
-if (DFTRACER_INSTALL_DEPENDENCIES) 
+if (DFTRACER_INSTALL_DEPENDENCIES)
     message(STATUS "[${PROJECT_NAME}] downloading dependencies. Please run make for downloading depedencies and then do reconfigure without dependency flag.")
     message(STATUS "[${PROJECT_NAME}] Installing depedencies at ${CMAKE_INSTALL_PREFIX}")
     add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/dependency)
@@ -233,7 +230,7 @@ if (DFTRACER_BUILD_PYTHON_BINDINGS)
     #add_subdirectory(dependency/py11bind)
 endif()
 
-if (DFTRACER_ENABLE_MPI) 
+if (DFTRACER_ENABLE_MPI)
   find_package(MPI COMPONENTS CXX REQUIRED)
   if (MPI_FOUND)
       message(STATUS "[DFTRACER] found mpi.h at ${MPI_CXX_INCLUDE_DIRS}")
@@ -255,9 +252,6 @@ if (NOT DFTRACER_DISABLE_HWLOC)
     endif ()
 endif()
 
-
-
-
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
 set(DFTRACER_PRELOAD_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/dftracer/dftracer_preload.cpp)
@@ -416,7 +410,6 @@ if (DFTRACER_BUILD_PYTHON_BINDINGS)
                 . ${CMAKE_BINARY_DIR}/symlink.sh
                 \")")
 
-
     install(TARGETS py${PROJECT_NAME}_dbg
             LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
             PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
@@ -467,6 +460,30 @@ install(
         bin
 )
 
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/script/dftracer_split ${EXECUTABLE_OUTPUT_PATH}/dftracer_split COPYONLY)
+install(
+        FILES
+        ${EXECUTABLE_OUTPUT_PATH}/dftracer_split
+        DESTINATION
+        bin
+)
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/script/dftracer_event_count ${EXECUTABLE_OUTPUT_PATH}/dftracer_event_count COPYONLY)
+install(
+        FILES
+        ${EXECUTABLE_OUTPUT_PATH}/dftracer_event_count
+        DESTINATION
+        bin
+)
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/script/dftracer_validate ${EXECUTABLE_OUTPUT_PATH}/dftracer_validate COPYONLY)
+install(
+        FILES
+        ${EXECUTABLE_OUTPUT_PATH}/dftracer_validate
+        DESTINATION
+        bin
+)
+
 #cmake_policy(SET CMP0079 NEW) # In case that we need more control over the target building order
 
 if(DFTRACER_ENABLE_TESTS)
@@ -492,7 +509,7 @@ include(CMakePackageConfigHelpers)
 configure_package_config_file(
         ${CMAKE_CURRENT_SOURCE_DIR}/cmake/configure_files/${PROJECT_NAME}-config.cmake.build.in
         "${CMAKE_BINARY_DIR}/${PROJECT_NAME}-config.cmake"
-        INSTALL_DESTINATION  ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cmake/${PROJECT_NAME}/${PROJECT_NAME}-config.cmake 
+        INSTALL_DESTINATION  ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cmake/${PROJECT_NAME}/${PROJECT_NAME}-config.cmake
         PATH_VARS CMAKE_BINARY_DIR
 )
 
@@ -528,7 +545,6 @@ export(EXPORT ${DFTRACER_EXPORTED_TARGETS}
 configure_file("${CMAKE_SOURCE_DIR}/cmake/configure_files/dftracer_config.hpp.in"
                "${CMAKE_INCLUDE_OUTPUT_DIRECTORY}/dftracer/dftracer_config.hpp" @ONLY)
 
-
 ################################################################
 # Install DFTRACER
 ################################################################
@@ -561,7 +577,6 @@ install(FILES
         DESTINATION
         ${DFTRACER_INSTALL_DOCDIR})
 
-
 ###############################################################################
 #                      Print out configuration summary
 ###############################################################################
@@ -636,7 +651,6 @@ string(APPEND _str
 execute_process(COMMAND ${CMAKE_COMMAND} -E echo "${_str}")
 set(_str)
 
-
 ###############################################################################
 # Write a basic modulefile
 ###############################################################################