From c8c6533e2426e530a9bcc976f337b5e92db8af79 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Fri, 4 Oct 2024 17:54:58 -0700 Subject: [PATCH 01/15] Enhancement: add hostname to metadata events (#215) * add hostnames to metadata. * added pid tid and hhash column to metadata --- dfanalyzer/main.py | 28 +++++++++++++++------------ src/dftracer/writer/chrome_writer.cpp | 4 ++-- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/dfanalyzer/main.py b/dfanalyzer/main.py index e62e600..8e69cee 100644 --- a/dfanalyzer/main.py +++ b/dfanalyzer/main.py @@ -170,6 +170,12 @@ def load_objects(line, fn, time_granularity, time_approximate, condition_fn, loa d["name"] = val["name"] if "cat" in val: d["cat"] = val["cat"] + if "pid" in val: + d["pid"] = val["pid"] + if "tid" in val: + d["tid"] = val["tid"] + if "args" in val and "hhash" in val["args"]: + d["hhash"] = val["args"]["hhash"] if "M" == val["ph"]: if d["name"] == "FH": d["type"] = 1 # 1-> file hash @@ -186,6 +192,11 @@ def load_objects(line, fn, time_granularity, time_approximate, condition_fn, loa if "args" in val and "name" in val["args"] and "value" in val["args"]: d["name"] = val["args"]["name"] d["hash"] = val["args"]["value"] + elif d["name"] == "PR": + d["type"] = 5 # 5-> process metadata + if "args" in val and "name" in val["args"] and "value" in val["args"]: + d["name"] = val["args"]["name"] + d["hash"] = val["args"]["value"] else: d["type"] = 4 # 4-> others if "args" in val and "name" in val["args"] and "value" in val["args"]: @@ -193,10 +204,6 @@ def load_objects(line, fn, time_granularity, time_approximate, condition_fn, loa d["value"] = str(val["args"]["value"]) else: d["type"] = 0 # 0->regular event - if "pid" in val: - d["pid"] = val["pid"] - if "tid" in val: - d["tid"] = val["tid"] if "dur" in val: val["dur"] = int(val["dur"]) val["ts"] = int(val["ts"]) @@ -261,8 +268,6 @@ def io_function(json_object, current_dict, time_approximate,condition_fn): if "args" in json_object: if "fhash" in json_object["args"]: d["fhash"] = json_object["args"]["fhash"] - if "hhash" in json_object["args"]: - d["hhash"] = json_object["args"]["hhash"] if "POSIX" == json_object["cat"] and "ret" in json_object["args"]: size = int(json_object["args"]["ret"]) @@ -281,7 +286,6 @@ def io_function(json_object, current_dict, time_approximate,condition_fn): def io_columns(): conf = get_dft_configuration() return { - 'hhash': "uint64[pyarrow]", 'compute_time': "string[pyarrow]" if not conf.time_approximate else "uint64[pyarrow]", 'io_time': "string[pyarrow]" if not conf.time_approximate else "uint64[pyarrow]", 'app_io_time': "string[pyarrow]" if not conf.time_approximate else "uint64[pyarrow]", @@ -453,10 +457,10 @@ def __init__(self, file_pattern, load_fn=None, load_cols={}, load_data = {}, met 'tinterval': "string[pyarrow]" if not self.conf.time_approximate else "uint64[pyarrow]", 'trange': "uint64[pyarrow]"} columns.update(io_columns()) columns.update(load_cols) - file_hash_columns = {'name': "string[pyarrow]", 'hash':"uint64[pyarrow]"} - hostname_hash_columns = {'name': "string[pyarrow]", 'hash':"uint64[pyarrow]"} - string_hash_columns = {'name': "string[pyarrow]", 'hash':"uint64[pyarrow]"} - other_metadata_columns = { 'name':"string[pyarrow]" ,'value':"string[pyarrow]" } + file_hash_columns = {'name': "string[pyarrow]", 'hash':"uint64[pyarrow]",'pid': "uint64[pyarrow]", 'tid': "uint64[pyarrow]", 'hhash': "uint64[pyarrow]"} + hostname_hash_columns = {'name': "string[pyarrow]", 'hash':"uint64[pyarrow]",'pid': "uint64[pyarrow]", 'tid': "uint64[pyarrow]", 'hhash': "uint64[pyarrow]"} + string_hash_columns = {'name': "string[pyarrow]", 'hash':"uint64[pyarrow]",'pid': "uint64[pyarrow]", 'tid': "uint64[pyarrow]", 'hhash': "uint64[pyarrow]"} + other_metadata_columns = { 'name':"string[pyarrow]" ,'value':"string[pyarrow]",'pid': "uint64[pyarrow]", 'tid': "uint64[pyarrow]", 'hhash': "uint64[pyarrow]"} if "FH" in metadata_cols: file_hash_columns.update(metadata_cols["FH"]) if "HH" in metadata_cols: @@ -475,7 +479,7 @@ def __init__(self, file_pattern, load_fn=None, load_cols={}, load_data = {}, met self.file_hash = self.all_events.query("type == 1")[list(file_hash_columns.keys())].groupby('hash').first().persist() self.host_hash = self.all_events.query("type == 2")[list(hostname_hash_columns.keys())].groupby('hash').first().persist() self.string_hash = self.all_events.query("type == 3")[list(string_hash_columns.keys())].groupby('hash').first().persist() - self.metadata = self.all_events.query("type == 4")[list(other_metadata_columns.keys())].persist() + self.metadata = self.all_events.query("type == 4")[list(other_metadata_columns.keys())].persist() self.n_partition = math.ceil(total_size.compute() / (128 * 1024 ** 2)) logging.debug(f"Number of partitions used are {self.n_partition}") self.events = events.repartition(npartitions=self.n_partition).persist() diff --git a/src/dftracer/writer/chrome_writer.cpp b/src/dftracer/writer/chrome_writer.cpp index 72f3f01..9be3a65 100644 --- a/src/dftracer/writer/chrome_writer.cpp +++ b/src/dftracer/writer/chrome_writer.cpp @@ -255,12 +255,12 @@ void dftracer::ChromeWriter::convert_json_metadata( if (is_string) { written_size = sprintf( buffer.data() + current_index, - R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"name":"%s","value":"%s"}})", + R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"hhash":%d,"name":"%s","value":"%s"}})", is_first_char, index, ph, process_id, thread_id, name, value); } else { written_size = sprintf( buffer.data() + current_index, - R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"name":"%s","value":%s}})", + R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"hhash":%d,"name":"%s","value":%s}})", is_first_char, index, ph, process_id, thread_id, name, value); } current_index += written_size; From de325157c657c7a57f01752bfdf9f0add73817c2 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Sat, 5 Oct 2024 02:14:55 -0700 Subject: [PATCH 02/15] Update dftracer_sanitize --- script/dftracer_sanitize | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/dftracer_sanitize b/script/dftracer_sanitize index 587296f..6c12745 100755 --- a/script/dftracer_sanitize +++ b/script/dftracer_sanitize @@ -99,7 +99,7 @@ for file in "$LOG_DIR"/*.pfw*; do name=${filename%.pfw.gz} echo "extracted name $name" # replace non utf characters - gunzip -c $OUTPUT_DIR/.tmp.$filename | LANG=C sed 's/[\d128-\d255]//g' > $OUTPUT_DIR/$name.pfw + gunzip -c $OUTPUT_DIR/.tmp.$filename | LC_ALL=C sed 's/[^[:print:]\r\t]//g' > $OUTPUT_DIR/$name.pfw # compress if [ $compressed == 1 ]; then gzip $OUTPUT_DIR/$name.pfw @@ -112,7 +112,7 @@ for file in "$LOG_DIR"/*.pfw*; do name=${filename%.pfw} echo "extracted name $name" # replace non utf characters - LANG=C sed 's/[\d128-\d255]//g' $OUTPUT_DIR/.tmp.$filename > $OUTPUT_DIR/$name.pfw + LANG=C sed LC_ALL=C sed 's/[^[:print:]\r\t]//g' $OUTPUT_DIR/.tmp.$filename > $OUTPUT_DIR/$name.pfw if [ $compressed == 1 ]; then gzip $OUTPUT_DIR/$name.pfw echo "Processed $OUTPUT_DIR/$name.pfw.gz" From e7d800b962bb95f9c4b2a6c9bc877269f1fa4435 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Sat, 5 Oct 2024 02:28:52 -0700 Subject: [PATCH 03/15] Update dftracer_sanitize --- script/dftracer_sanitize | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/script/dftracer_sanitize b/script/dftracer_sanitize index 6c12745..237d4bd 100755 --- a/script/dftracer_sanitize +++ b/script/dftracer_sanitize @@ -98,8 +98,10 @@ for file in "$LOG_DIR"/*.pfw*; do # if file is gz get the name name=${filename%.pfw.gz} echo "extracted name $name" + echo "[" > $OUTPUT_DIR/$name.pfw # replace non utf characters - gunzip -c $OUTPUT_DIR/.tmp.$filename | LC_ALL=C sed 's/[^[:print:]\r\t]//g' > $OUTPUT_DIR/$name.pfw + gunzip -c $OUTPUT_DIR/.tmp.$filename | LC_ALL=C sed 's/[^[:print:]\r\t]//g' | jq -R -c "fromjson?" >> $OUTPUT_DIR/$name.pfw + echo "]" >> $OUTPUT_DIR/$name.pfw # compress if [ $compressed == 1 ]; then gzip $OUTPUT_DIR/$name.pfw @@ -111,8 +113,10 @@ for file in "$LOG_DIR"/*.pfw*; do # if file is pfw get the name name=${filename%.pfw} echo "extracted name $name" - # replace non utf characters - LANG=C sed LC_ALL=C sed 's/[^[:print:]\r\t]//g' $OUTPUT_DIR/.tmp.$filename > $OUTPUT_DIR/$name.pfw + # replace non utf characters and valid json lines + echo "[" > $OUTPUT_DIR/$name.pfw + LC_ALL=C sed 's/[^[:print:]\r\t]//g' $OUTPUT_DIR/.tmp.$filename | jq -R -c "fromjson?" >> $OUTPUT_DIR/$name.pfw + echo "]" >> $OUTPUT_DIR/$name.pfw if [ $compressed == 1 ]; then gzip $OUTPUT_DIR/$name.pfw echo "Processed $OUTPUT_DIR/$name.pfw.gz" From 8b1774a5d5218eaa0d26e7986275da25b3ce2658 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Sat, 5 Oct 2024 02:33:30 -0700 Subject: [PATCH 04/15] Set defaults for log file and data dir. --- src/dftracer/utils/configuration_manager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dftracer/utils/configuration_manager.cpp b/src/dftracer/utils/configuration_manager.cpp index d36af31..3b80728 100644 --- a/src/dftracer/utils/configuration_manager.cpp +++ b/src/dftracer/utils/configuration_manager.cpp @@ -43,8 +43,8 @@ bool dftracer::Singleton< dftracer::ConfigurationManager::ConfigurationManager() : enable(false), init_type(PROFILER_INIT_FUNCTION), - log_file(), - data_dirs(), + log_file(./trace), + data_dirs("all"), metadata(false), core_affinity(false), gotcha_priority(1), From 3ec3d4738fd35694476bcb6f80ba5c2e931f368b Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Sat, 5 Oct 2024 10:00:13 -0700 Subject: [PATCH 05/15] Update configuration_manager.cpp --- src/dftracer/utils/configuration_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dftracer/utils/configuration_manager.cpp b/src/dftracer/utils/configuration_manager.cpp index 3b80728..00b0ab5 100644 --- a/src/dftracer/utils/configuration_manager.cpp +++ b/src/dftracer/utils/configuration_manager.cpp @@ -43,7 +43,7 @@ bool dftracer::Singleton< dftracer::ConfigurationManager::ConfigurationManager() : enable(false), init_type(PROFILER_INIT_FUNCTION), - log_file(./trace), + log_file("./trace"), data_dirs("all"), metadata(false), core_affinity(false), From 60e502c26fc2efeebb2c0fdc2c63148fa3313fb8 Mon Sep 17 00:00:00 2001 From: Ray Andrew <4437323+rayandrew@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:11:49 -0500 Subject: [PATCH 06/15] add support for splitting trace (#214) * add support for splitting trace - bump version of dftracer - implement split trace based on user input * remove unused import * added installation parts and docs. * changes to split script * fix corner case of small traces * add sanity check checking number of lines between split files and original files * fuxed naming * add `dftracer_event_count` - use sqlite from python to calculate valid events inside traces - reuse `dftracer_event_count` for sanity checking inside `dftracer_split` * fix space in docs * make indexing optional * fixing remove when there are too many files * update branch and add sqlite3 * change dftracer_event_count params * remove logging from pushd and popd * added changes for testing * update for multi-threading and printing * more optimizations. * added pipelining. * final commit for checkpointing chunks --------- Co-authored-by: hariharandev1 --- CMakeLists.txt | 38 ++-- docs/utilities.rst | 45 ++++- script/dftracer_create_index | 94 +++++---- script/dftracer_event_count | 140 ++++++++++++++ script/dftracer_split | 356 +++++++++++++++++++++++++++++++++++ setup.py | 7 +- 6 files changed, 617 insertions(+), 63 deletions(-) create mode 100755 script/dftracer_event_count create mode 100755 script/dftracer_split diff --git a/CMakeLists.txt b/CMakeLists.txt index 08951fb..7487491 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,6 @@ set(DFTRACER_VERSION "(1, 0, 5)") project(dftracer LANGUAGES C CXX) - # Convenience defines string(TOUPPER "${PROJECT_NAME}" UPPER_PROJECT_NAME) string(TOLOWER "${PROJECT_NAME}" LOWER_PROJECT_NAME) @@ -85,7 +84,6 @@ if (NOT DFTRACER_EXPORTED_TARGETS) set(DFTRACER_EXPORTED_TARGETS "dftracer-targets") endif () - include(dftracer-utils) #------------------------------------------------------------------------------ # Build options @@ -108,13 +106,13 @@ endif (DFTRACER_USE_CLANG_LIBCXX) # Options option (DFTRACER_DISABLE_HWLOC "Disable HWLOC" On) -if (NOT DFTRACER_DISABLE_HWLOC) +if (NOT DFTRACER_DISABLE_HWLOC) set(DFTRACER_HWLOC_ENABLE 1) endif() # Options option (DFTRACER_ENABLE_FTRACING "Enable Function Tracing" OFF) -if (DFTRACER_ENABLE_FTRACING) +if (DFTRACER_ENABLE_FTRACING) set(DFTRACER_FTRACING_ENABLE 1) set(DFTRACER_FUNCTION_FLAGS "-g" "-finstrument-functions" "-Wl,-E" "-fvisibility=default") else() @@ -122,14 +120,13 @@ else() endif() option (DFTRACER_ENABLE_MPI "Enable MPI" OFF) -if (DFTRACER_ENABLE_MPI) +if (DFTRACER_ENABLE_MPI) set(DFTRACER_MPI_ENABLE 1) endif() option (DFTRACER_BUILD_PYTHON_BINDINGS "Build python bindings." ON) set(DFTRACER_PYTHON_EXE "python3" CACHE STRING "Python executable to use for building.") set(DFTRACER_PYTHON_SITE $ENV{DFTRACER_PYTHON_SITE} CACHE STRING "Python site packages to use for building.") - if (DFTRACER_BUILD_PYTHON_BINDINGS) if (DFTRACER_PYTHON_SITE) set(PYTHON_SITE_PACKAGES "${DFTRACER_PYTHON_SITE}") @@ -192,7 +189,7 @@ endif () # Dependencies #------------------------------------------------------------------------------ -if (DFTRACER_INSTALL_DEPENDENCIES) +if (DFTRACER_INSTALL_DEPENDENCIES) message(STATUS "[${PROJECT_NAME}] downloading dependencies. Please run make for downloading depedencies and then do reconfigure without dependency flag.") message(STATUS "[${PROJECT_NAME}] Installing depedencies at ${CMAKE_INSTALL_PREFIX}") add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/dependency) @@ -233,7 +230,7 @@ if (DFTRACER_BUILD_PYTHON_BINDINGS) #add_subdirectory(dependency/py11bind) endif() -if (DFTRACER_ENABLE_MPI) +if (DFTRACER_ENABLE_MPI) find_package(MPI COMPONENTS CXX REQUIRED) if (MPI_FOUND) message(STATUS "[DFTRACER] found mpi.h at ${MPI_CXX_INCLUDE_DIRS}") @@ -255,9 +252,6 @@ if (NOT DFTRACER_DISABLE_HWLOC) endif () endif() - - - include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) set(DFTRACER_PRELOAD_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/dftracer/dftracer_preload.cpp) @@ -416,7 +410,6 @@ if (DFTRACER_BUILD_PYTHON_BINDINGS) . ${CMAKE_BINARY_DIR}/symlink.sh \")") - install(TARGETS py${PROJECT_NAME}_dbg LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) @@ -467,6 +460,22 @@ install( bin ) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/script/dftracer_split ${EXECUTABLE_OUTPUT_PATH}/dftracer_split COPYONLY) +install( + FILES + ${EXECUTABLE_OUTPUT_PATH}/dftracer_split + DESTINATION + bin +) + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/script/dftracer_event_count ${EXECUTABLE_OUTPUT_PATH}/dftracer_event_count COPYONLY) +install( + FILES + ${EXECUTABLE_OUTPUT_PATH}/dftracer_event_count + DESTINATION + bin +) + #cmake_policy(SET CMP0079 NEW) # In case that we need more control over the target building order if(DFTRACER_ENABLE_TESTS) @@ -492,7 +501,7 @@ include(CMakePackageConfigHelpers) configure_package_config_file( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/configure_files/${PROJECT_NAME}-config.cmake.build.in "${CMAKE_BINARY_DIR}/${PROJECT_NAME}-config.cmake" - INSTALL_DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cmake/${PROJECT_NAME}/${PROJECT_NAME}-config.cmake + INSTALL_DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cmake/${PROJECT_NAME}/${PROJECT_NAME}-config.cmake PATH_VARS CMAKE_BINARY_DIR ) @@ -528,7 +537,6 @@ export(EXPORT ${DFTRACER_EXPORTED_TARGETS} configure_file("${CMAKE_SOURCE_DIR}/cmake/configure_files/dftracer_config.hpp.in" "${CMAKE_INCLUDE_OUTPUT_DIRECTORY}/dftracer/dftracer_config.hpp" @ONLY) - ################################################################ # Install DFTRACER ################################################################ @@ -561,7 +569,6 @@ install(FILES DESTINATION ${DFTRACER_INSTALL_DOCDIR}) - ############################################################################### # Print out configuration summary ############################################################################### @@ -636,7 +643,6 @@ string(APPEND _str execute_process(COMMAND ${CMAKE_COMMAND} -E echo "${_str}") set(_str) - ############################################################################### # Write a basic modulefile ############################################################################### diff --git a/docs/utilities.rst b/docs/utilities.rst index 369c103..2cb8e37 100644 --- a/docs/utilities.rst +++ b/docs/utilities.rst @@ -1,5 +1,5 @@ ======================== -DFTracer Utility scripts +DFTracer Utility scripts ======================== This section describes the utilities provided by DFTracer to assist users with logs. @@ -12,14 +12,14 @@ All scripts are installed with DFTracer in the installation's directories bin fo Merge Trace script ------------------ -This script allows users to combine all pfw format into one. +This script allows users to combine all pfw format into one. This has the following signature. .. code-block:: bash /bin/dftracer_merge [-fcv] [-d input_directory] [-o output_file] -Arguments for this script are +Arguments for this script are 1. **-d input_directory** folder containing all trace files. Default `PWD`. 2. **-o output_file** file for storing merged file. Default `combined.pfw`. @@ -38,7 +38,7 @@ The script compacts all trace file and then divides the trace into equal file pi /bin/dftracer_compact [-fcv] [-d input_directory] [-o output_directory] [-l num_lines] [-p prefix] -Arguments for this script are +Arguments for this script are 1. **-d input_directory** specify input directories. Should contain .pfw or .pfw.gz files. Default `PWD`. 2. **-o output_file** specify output directory. Default `combined.pfw`. @@ -49,7 +49,6 @@ Arguments for this script are 7. **-v** enable verbose mode. 8. **-h** display help - ------------------ Sanitize script ------------------ @@ -105,3 +104,39 @@ Arguments for this script are: 3. **-v** enable verbose mode 4. **-h** display help 5. **-d input_directory** specify input directories. should contain .pfw or .pfw.gz files. + +------------------ +Splitting DFTracer traces +------------------ + +The script splits the traces into equal sized chunk optimized for analysis + +.. code-block:: bash + + /bin/usage: dftracer_split [-fv] [-n app_name] [-d input_directory] [-o output_directory] [-s chunk_size] + +Arguments for this script are: + +1. **-n app_name** specify app name. +2. **-f** override indices. +3. **-s size** chunk size (in MB) +4. **-v** enable verbose mode +5. **-h** display help +6. **-d input_directory** specify input directories. should contain .pfw or .pfw.gz files. +7. **-o output_directory** specify output directory. + +------------------ +Counting DFTracer traces' events +------------------ + +The script will count number of valid events of traces + +.. code-block:: bash + + /bin/usage: dftracer_event_count [-f] [-d input_directory] + +Arguments for this script are: + +1. **-d input_directory** specify input directories. should contain .pfw or .pfw.gz files. +2. **-f** force index creation +3. **-h** display help diff --git a/script/dftracer_create_index b/script/dftracer_create_index index 373fe1f..9448ffd 100755 --- a/script/dftracer_create_index +++ b/script/dftracer_create_index @@ -1,6 +1,6 @@ #!/bin/bash -# The script creates indices for the dftracer traces +# The script creates indices for the dftracer traces # This has the following signature. # # usage: dftracer_create_index [-fcv] [-d input_directory] @@ -15,6 +15,11 @@ date_echo() { echo "$dt $@" } +progress_date_echo() { + dt=$(date '+%d/%m/%Y %H:%M:%S') + echo -ne "$dt $@"\\r +} + LOG_DIR=$PWD override=0 compressed=0 @@ -89,56 +94,67 @@ fi if [ "$override" == "1" ]; then date_echo "Removing existing indices as override is passed." - rm $LOG_DIR/*.zindex + files=("$LOG_DIR"/*.zindex) + for file_index in "${!files[@]}"; do + file=${files[$file_index]} + if [ -f "$file" ]; then + rm $file + fi + done fi -pushd $LOG_DIR +pushd $LOG_DIR > /dev/null JOBS_LIMIT=$(nproc --all) files=("$LOG_DIR"/*.pfw*) total=${#files[@]} # loop over logs for file_index in "${!files[@]}"; do file=${files[$file_index]} - running_jobs=$(jobs -rp | wc -l) - if [ $running_jobs -ge $JOBS_LIMIT ]; then - date_echo "waiting for Running $running_jobs jobs to be less than $JOBS_LIMIT" - while [ $running_jobs -ge $JOBS_LIMIT ] - do - sleep 1 - running_jobs=$(jobs -rp | wc -l) - done - date_echo "Running $running_jobs jobs are now less than $JOBS_LIMIT" - fi - # only look at files - if [ -f "$file" ]; then - # calculate basename and copy files - filename=$(basename -- "$file") - ext="${filename##*.}" - if [ "$ext" == "gz" ]; then - { - # if file is gz get the name - name=${filename%.pfw.gz} - if [ ! -f "$file.zindex" ]; then - $zindex_exec $file --index-file file:$file.zindex --regex 'id:([0-9]+)' --numeric --unique - fi - date_echo "Created index for file $file.gz $file_index of $total" - } & - else - { - # if file is pfw get the name - name=${filename%.pfw} - if [ ! -f "$file.gz.zindex" ]; then - if [ $compressed == 1 ]; then - date_echo "Compressing file $name" - gzip $file - $zindex_exec $file.gz --index-file file:$file.gz.zindex --regex 'id:([0-9]+)' --numeric --unique + filename=$(basename -- "$file") + ext="${filename##*.}" + if [ "$ext" != "zindex" ]; then + running_jobs=$(jobs -rp | wc -l) + if [ $running_jobs -ge $JOBS_LIMIT ]; then + date_echo "waiting for Running $running_jobs jobs to be less than $JOBS_LIMIT" + while [ $running_jobs -ge $JOBS_LIMIT ] + do + sleep 1 + running_jobs=$(jobs -rp | wc -l) + done + date_echo "Running $running_jobs jobs are now less than $JOBS_LIMIT" + fi + # only look at files + if [ -f "$file" ]; then + # calculate basename and copy files + filename=$(basename -- "$file") + ext="${filename##*.}" + if [ "$ext" == "gz" ]; then + { + # if file is gz get the name + name=${filename%.pfw.gz} + if [ ! -f "$file.zindex" ]; then + $zindex_exec $file --index-file file:$file.zindex --regex 'id:([0-9]+)' --numeric --unique + fi + progress_date_echo "Created index for file $file $((file_index+1)) of $total" + } & + else + { + # if file is pfw get the name + name=${filename%.pfw} + if [ ! -f "$file.gz.zindex" ]; then + if [ $compressed == 1 ]; then + date_echo "Compressing file $name" + gzip $file + $zindex_exec $file.gz --index-file file:$file.gz.zindex --regex 'id:([0-9]+)' --numeric --unique + fi fi + progress_date_echo "Created index for file $file.gz $((file_index+1)) of $total" + } & fi - date_echo "Created index for file $file.gz $file_index of $total" - } & fi fi done -popd +popd > /dev/null wait +echo "" date_echo Creation of index finished diff --git a/script/dftracer_event_count b/script/dftracer_event_count new file mode 100755 index 0000000..e4f3aa4 --- /dev/null +++ b/script/dftracer_event_count @@ -0,0 +1,140 @@ +#!/bin/bash + +# The script will count number of valid events of traces +# This has the following signature. +# +# usage: dftracer_split [-f] [-d input_directory] +# -d input_directory specify input directories. should contain .pfw or .pfw.gz files. +# -f force index creation +# -h display help +date_echo() { + dt=$(date '+%d/%m/%Y %H:%M:%S') + echo "$dt $@" +} + +progress_date_echo() { + dt=$(date '+%d/%m/%Y %H:%M:%S') + echo -ne "$dt $@ "\\r +} +LOG_DIR=$PWD +run_create_index=1 + +function usage { + echo "usage: $(basename $0) [-f] [-d input_directory]" + echo " -h display help" + echo " -f force index creation" + echo " -d input_directory specify input directories. should contain .pfw or .pfw.gz files." + exit 1 +} +while getopts ':fd:h' opt; do + case "$opt" in + f) + run_create_index=0 + ;; + d) + LOG_DIR="${OPTARG}" + ;; + h) + usage + exit 0 + ;; + :) + echo -e "option requires an argument.\n" + usage + exit 1 + ;; + + ?) + echo -e "Invalid command option.\n" + usage + exit 1 + ;; + esac +done +shift "$(($OPTIND -1))" + +function get_lines_count { + local dir=$1 + + # if dir empty use current dir + if [ -z "$dir" ]; then + dir=$PWD + fi + + pushd $dir > /dev/null + + # check if sqlite3 exists + sqlite_exists=0 + if command -v sqlite3 &> /dev/null; then + # echo "sqlite3 exists" + sqlite_exists=1 + fi + + files=("$dir"/*.zindex) + JOBS_LIMIT=$(nproc --all) + declare -A lines_counts + # for i in $(seq 1 $JOBS_LIMIT); do + # lines_counts+=("0") + # done + for file_index in "${!files[@]}"; do + running_jobs=$(jobs -rp | wc -l) + if [ $running_jobs -ge $JOBS_LIMIT ]; then + date_echo "waiting for Running $running_jobs jobs to be less than $JOBS_LIMIT" + while [ $running_jobs -ge $JOBS_LIMIT ] + do + sleep 1 + running_jobs=$(jobs -rp | wc -l) + done + date_echo "Running $running_jobs jobs are now less than $JOBS_LIMIT" + fi + + file_name=${files[$file_index]} + # echo "Processing $file_name" + if [ $sqlite_exists -eq 1 ]; then + lines_counts[$file_index]=$( + { + sqlite3 $file_name "select count(line) as a from LineOffsets where length > 8;" + progress_date_echo "Finished counting $file_index of $total" >&2 + } &) + else + lines_counts[$file_index]=$( + { +python3 <<-EOF +import zindex_py as zindex +import glob +import sqlite3 + +conn = sqlite3.connect("${file_name}") +res = conn.execute("select count(line) as a from LineOffsets where length > 8;") +lines_count = res.fetchone()[0] +print(lines_count) +EOF +} &) + fi + done + wait + + popd > /dev/null + + lines_count=0 + for line_count in "${lines_counts[@]}"; do + lines_count=$((lines_count + line_count)) + done + + echo $lines_count +} + +files=("$LOG_DIR"/*.pfw.gz) +for file in $files; do + if [ ! -f "$file.zindex" ]; then + run_create_index=1 + break + fi +done + +if [ "$run_create_index" == "1" ]; then + SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" + $SCRIPT_DIR/dftracer_create_index -f -d $LOG_DIR -f >/dev/null +fi + +get_lines_count $LOG_DIR diff --git a/script/dftracer_split b/script/dftracer_split new file mode 100755 index 0000000..6756eb5 --- /dev/null +++ b/script/dftracer_split @@ -0,0 +1,356 @@ +#!/bin/bash + +# The script splits the traces into equal sized chunk optimized for analysis +# This has the following signature. +# +# usage: dftracer_split [-fv] [-n app_name] [-d input_directory] [-o output_directory] [-s chunk_size] +# -n app_name specify app name +# -f override generated files +# -s size chunk size (in MB) +# -v enable verbose mode +# -h display help +# -d input_directory specify input directories. should contain .pfw or .pfw.gz files. +# -o output_directory specify output directory + +# PARALLEL populate metadata using count (bash parallel + sqlite query) +# - check whether we have sqlite3 +# - if not using python +# - put result as map of filename tuple[filename, size, min, max] +# SERIAL iterate to create chunks mapping (bash serial) +# - put result as array of array of tuple[filename, start, end] +# - the idea is that we can have multiple file data in different chunks +# PARALLEL extract chunks (bash parallel + zq) +# - for every chunk serially run zq +# - get data for start and end + +date_echo() { + dt=$(date '+%d/%m/%Y %H:%M:%S') + echo "$dt $@" +} + +progress_date_echo() { + dt=$(date '+%d/%m/%Y %H:%M:%S') + echo -ne "$dt $@ "\\r +} + +LOG_DIR=$PWD +override=0 +chunk_size=1024 # 1GB default +dest=$PWD/split +verbose=0 +app_name="app" + +function usage { + echo "usage: $(basename $0) [-fv] [-n app_name] [-d input_directory] [-o output_directory] [-s chunk_size]" + echo " -n app_name specify app name" + echo " -f override generated files" + echo " -s size chunk size (in MB)" + echo " -v enable verbose mode" + echo " -h display help" + echo " -d input_directory specify input directories. should contain .pfw or .pfw.gz files." + echo " -o output_directory specify output directory" + exit 1 +} +while getopts ':fvn:d:o:s:h' opt; do + case "$opt" in + n) + app_name="${OPTARG}" + ;; + d) + LOG_DIR="${OPTARG}" + ;; + s) + chunk_size="${OPTARG}" + ;; + o) + dest="${OPTARG}" + ;; + f) + override=1 + ;; + v) + verbose=0 + ;; + h) + usage + exit 0 + ;; + :) + echo -e "option requires an argument.\n" + usage + exit 1 + ;; + + ?) + echo -e "Invalid command option.\n" + usage + exit 1 + ;; + esac +done +shift "$(($OPTIND -1))" + +pfw_total=0 +pfw_gz_total=0 +zindex_total=0 +for file in *.pfw*; do pfw_total=1; break; done +for file in *.pfw.gz*; do pfw_gz_total=1; break; done +for file in *.zindex; do zindex_total=1; break; done + +printf "============================================\n" +printf "Arguments:\n" +printf " App name: %s\n" $app_name +printf " Override: %s\n" $override +printf " Data dir: %s\n" $LOG_DIR +printf " Output dir: %s\n" $dest +printf " Chunk size: %s\n" $chunk_size +printf "============================================\n" + +mkdir -p $dest + +if [ $pfw_total == 0 ] || [ $pfw_gz_total == 0 ]; then + date_echo "The folder does not contain any pfw or pfw.gz files." + exit 1 +fi + +python -c "import zindex_py;" +if [[ $? != 0 ]]; then + date_echo "failure: $?: zindex not found. Please install zindex with: pip install zindex_py" + exit 1 +fi +zindex_exec=$(python -c 'import zindex_py;import site; sp=site.getsitepackages()[0]; print(f"{sp}/zindex_py/bin/zindex")') +if [ ! -f "${zindex_exec}" ]; then + date_echo "failure: $?: zindex not found. Please install zindex with: pip install zindex_py" + exit 1 +else + date_echo "Found zindex executable at ${zindex_exec}" +fi + +zq_exec=$(python -c 'import zindex_py;import site; sp=site.getsitepackages()[0]; print(f"{sp}/zindex_py/bin/zq")') +if [ ! -f "${zq_exec}" ]; then + date_echo "failure: $?: zq not found. Please install zindex with: pip install zindex_py (zq is included in zindex_py)" + exit 1 +else + date_echo "Found zq executable at ${zq_exec}" +fi + +sqlite_exists=0 +if command -v sqlite3 &> /dev/null; then + date_echo "sqlite3 exists" + sqlite_exists=1 +else + date_echo "sqlite3 does not exist, will use python for querying" +fi + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +if [ $zindex_total == 0 ] || [ $override == 1 ]; then + $SCRIPT_DIR/dftracer_create_index -c -d $LOG_DIR -f +else + date_echo "Indices are present skipping it." +fi + +JOBS_LIMIT=1024 +pushd $LOG_DIR > /dev/null + +files=("$LOG_DIR"/*.zindex) +total=${#files[@]} +declare -A count +counting_file="counting.bak" +if [ ! -f $counting_file ] || [ $override == 1 ]; then + rm ${counting_file} + touch ${counting_file} + for file_index in "${!files[@]}"; do + running_jobs=$(jobs -rp | wc -l) + if [ $running_jobs -ge $JOBS_LIMIT ]; then + date_echo "waiting for Running $running_jobs jobs to be less than $JOBS_LIMIT" + while [ $running_jobs -ge $JOBS_LIMIT ] + do + sleep 1 + running_jobs=$(jobs -rp | wc -l) + done + date_echo "Running $running_jobs jobs are now less than $JOBS_LIMIT" + fi + file_name=${files[$file_index]} + filename_without_ext=$(basename $file_name .pfw.gz.zindex) + ( + IFS='|' read -r size start end <<< $(sqlite3 $file_name "select sum(length), min(line), max(line) as a from LineOffsets where length > 8;") + size_mb=$(bc -l <<< "scale=8; $size / (1024 * 1024)") + echo "${file_index}|${filename_without_ext}|${size_mb}|${start}|${end}" >> ${counting_file} + progress_date_echo Completed collecting size $size_mb $file_index of $total + ) & + done +else + date_echo "Previous counting present" +fi +wait + +counts=$(wc -l ${counting_file} | awk {'print $1'}) + +# for count_index in "${!count[@]}"; do +# echo "$count_index ${count[$count_index]}" +# done + +echo "" +if [ "$counts" != "${#files[@]}" ]; then + date_echo "Did not collect all files ${#count[@]} of ${#files[@]}" + exit 1 +else + date_echo "Finished collecting data from ${#files[@]} tasks" +fi + + +chunk_index=1 +file_processed=0 +CHUNKS_LINES=() +while true; do + chunk_file=".chunk-${chunk_index}.bak" + if [ -f $chunk_file ]; then + files_in_chunk=$(wc -l ".chunk-${chunk_index}.bak" | awk {'print $1'}) + chunk_index=$((chunk_index + 1)) + file_processed=$((file_processed + files_in_chunk)) + CHUNKS_LINES[$chunk_index]=$file_processed + else + break + fi +done +existing_chunk_index=$chunk_index + +total=${#files[@]} +CHUNKS=() +accumulated_size=0 +temp_chunk="" +scheduled_chunks=0 + + +chunk_index=1 +chunk_index_file=".chunk-${chunk_index}.bak" +line_number=0 +while IFS='|' read -r index file_name size start end; do + line_number=$((line_number + 1)) + if [ $line_number -lt $file_processed ]; then + if [[ $line_number -eq ${CHUNKS_LINES[$chunk_index]} ]]; then + chunk_index=$((chunk_index + 1)) + chunk_index_file=".chunk-${chunk_index}.bak" + fi + continue + fi + size_per_line=$(bc -l <<< "scale=8; $size / ($end - $start + 1)") + + while [ "$(bc -l <<< "scale=8; $size > 0")" -eq 1 ]; do + if [ "$(bc -l <<< "scale=8; $size + $accumulated_size > $chunk_size")" -eq 1 ]; then + + diff=$(bc -l <<< "scale=8; $chunk_size - $accumulated_size") + lines=$(bc -l <<< "scale=0; ($diff / $size_per_line) / 1") + size_chunk=$(bc -l <<< "scale=8; $lines * $size_per_line") + + if [ $start -gt $end ]; then + break + fi + + if [ $(bc -l <<< "scale=0; $lines > 0") -eq 1 ]; then + echo "$file_name|$size_chunk|$start|$((start + lines))" >> ${chunk_index_file} + + start=$((start + lines + 1)) + accumulated_size=$(bc -l <<< "scale=8; $accumulated_size + $size_chunk") + size=$(bc -l <<< "scale=8; $size - $size_chunk") + else + progress_date_echo "Scheduling $index of $total to chunk $chunk_index with size $accumulated_size" + i=$chunk_index + scheduled_chunks=$chunk_index + { + if [ "$verbose" == "1" ]; then + date_echo "Processing chunk $i with size $size MB" + fi + chunk_file=$dest/${app_name}-$i.pfw + rm -f $chunk_file + touch $chunk_file + echo '[' > $chunk_file + while IFS='|' read -r file_name size start end; do + if [ "$verbose" == "1" ]; then + date_echo "[CHUNK $i] Extracting from $file_name from $start to $end with size $size" + fi + $zq_exec ${file_name}.pfw.gz --index-file ${file_name}.pfw.gz.zindex --raw "select a.line from LineOffsets a where a.line >= $start AND a.line <= $end AND a.length > 8;" | grep -v '^[[]\|^[]]' >> $chunk_file + total_size=$(bc -l <<< "scale=2; $total_size + $size") + done < ${chunk_index_file} + progress_date_echo "Chunk $i out of ${#CHUNKS_LINES[@]} done with size $total_size MB, path = $chunk_file" + } & + chunk_index=$((chunk_index + 1)) + accumulated_size=0 + chunk_index_file=".chunk-${chunk_index}.bak" + fi + else + accumulated_size=$(bc -l <<< "scale=8; $accumulated_size + $size") + echo "$file_name|$size|$start|$end" >> ${chunk_index_file} + size=0 + fi + done +done < ${counting_file} + +total_chunks=$chunk_index; +date_echo "Scheduled chunks: $scheduled_chunks" +date_echo "Total chunks: $total_chunks" +date_echo "Start processing chunks" +start=$((scheduled_chunks + 1)) +total_size=0 +for i in $(seq $start $total_chunks); do + if [ "$verbose" == "1" ]; then + date_echo "Processing chunk $i" + fi + running_jobs=$(jobs -rp | wc -l) + if [ $running_jobs -ge $JOBS_LIMIT ]; then + # date_echo "waiting for Running $running_jobs jobs to be less than $JOBS_LIMIT" + while [ $running_jobs -ge $JOBS_LIMIT ] + do + sleep 1 + running_jobs=$(jobs -rp | wc -l) + done + date_echo "Running $running_jobs jobs are now less than $JOBS_LIMIT" + fi + #{ + if [ "$verbose" == "1" ]; then + date_echo "Processing chunk $i with size $size MB" + fi + chunk_file=$dest/${app_name}-$i.pfw + rm -f $chunk_file + touch $chunk_file + echo '[' > $chunk_file + while IFS='|' read -r file_name size start end; do + if [ "$verbose" == "1" ]; then + date_echo "[CHUNK $i] Extracting from $file_name from $start to $end with size $size" + fi + $zq_exec ${file_name}.pfw.gz --index-file ${file_name}.pfw.gz.zindex --raw "select a.line from LineOffsets a where a.line >= $start AND a.line <= $end AND a.length > 8;" | grep -v '^[[]\|^[]]' >> $chunk_file + total_size=$(bc -l <<< "scale=2; $total_size + $size") + done < ${chunk_index_file} + echo ']' >> $chunk_file + progress_date_echo "Chunk $i out of ${total_chunks} done with size $total_size MB, path = $chunk_file" + #} & +done +wait + +date_echo "All chunks processed" + +date_echo Reindexing split files +pushd $dest > /dev/null +rm -f *.pfw.gz +$SCRIPT_DIR/dftracer_create_index -c -d $dest -f +rm -f *.pfw + +LINES_COUNT=$( + { + $SCRIPT_DIR/dftracer_event_count -d $LOG_DIR + } &) +SPLIT_LINES_COUNT=$( + { + $SCRIPT_DIR/dftracer_event_count -d $dest + } &) +wait + +if [ $LINES_COUNT -ne $SPLIT_LINES_COUNT ]; then + date_echo "Error: Original lines count $LINES_COUNT does not match split lines count $SPLIT_LINES_COUNT" + exit 1 +else + date_echo "Original lines count $LINES_COUNT matches split lines count $SPLIT_LINES_COUNT" +fi +popd > /dev/null +rm -f .chunk* counting.bak +date_echo Done reindexing split files diff --git a/setup.py b/setup.py index 61d8c5e..fcfec35 100644 --- a/setup.py +++ b/setup.py @@ -184,7 +184,7 @@ def build_extension(self, ext: CMakeExtension) -> None: "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3 :: Only", ], - install_requires=["pybind11"], + install_requires=["pybind11", "zindex_py==0.0.4"], requires=["pybind11","setuptools"], keywords="profiler, deep learning, I/O, benchmark, NPZ, pytorch benchmark, tensorflow benchmark", project_urls={ # Optional @@ -196,7 +196,9 @@ def build_extension(self, ext: CMakeExtension) -> None: 'script/dftracer_merge', 'script/dftracer_sanitize', 'script/dftracer_anonymize', - 'script/dftracer_create_index',], + 'script/dftracer_split', + 'script/dftracer_create_index', + 'script/dftracer_event_count', ], package_dir={"dftracer": "dftracer", "dftracer_dbg": "dftracer_dbg", "dfanalyzer": "dfanalyzer"}, @@ -209,7 +211,6 @@ def build_extension(self, ext: CMakeExtension) -> None: "seaborn>=0.13.2", "bokeh>=2.4.2", "pybind11", - "zindex_py==0.0.2", "pandas>=2.0.3", "dask>=2023.5.0", "distributed", From b2fc688277c9f81def0aae638cf06165c21197ce Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Mon, 7 Oct 2024 15:31:04 -0700 Subject: [PATCH 07/15] BugFix: use hex string for hash as they will collide otherwise. (#216) * use hex string for hash as they will collide otherwise. * fixed counts. --- .vscode/launch.json | 5 +- dfanalyzer/main.py | 28 +- examples/dfanalyzer/dfanalyzer.ipynb | 356 +++++++++++++------------- include/dftracer/core/constants.h | 1 + src/dftracer/brahma/posix.h | 22 +- src/dftracer/brahma/stdio.h | 16 +- src/dftracer/df_logger.h | 69 +++-- src/dftracer/utils/md5.cpp | 9 +- src/dftracer/utils/md5.h | 27 +- src/dftracer/writer/chrome_writer.cpp | 17 +- src/dftracer/writer/chrome_writer.h | 6 +- test/CMakeLists.txt | 10 +- 12 files changed, 306 insertions(+), 260 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 890eebe..b9e1f77 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,9 +8,10 @@ "name": "Cmake Debug Target", "type": "lldb", "request": "launch", - "program": "${cmake.testProgram}", + "program": "/usr/workspace/haridev/dftracer/build/bin/test_cpp", "args": [ - "${cmake.testArgs}" + "/usr/workspace/haridev/dftracer/build/test/data", + "1" ], "cwd": "${workspaceFolder}/build", "env": { diff --git a/dfanalyzer/main.py b/dfanalyzer/main.py index 8e69cee..4504977 100644 --- a/dfanalyzer/main.py +++ b/dfanalyzer/main.py @@ -175,23 +175,35 @@ def load_objects(line, fn, time_granularity, time_approximate, condition_fn, loa if "tid" in val: d["tid"] = val["tid"] if "args" in val and "hhash" in val["args"]: - d["hhash"] = val["args"]["hhash"] + if type(val["args"]["hhash"]) is str: + d["hhash"] = int(val["args"]["hhash"],16) + else: + d["hhash"] = val["args"]["hhash"] if "M" == val["ph"]: if d["name"] == "FH": d["type"] = 1 # 1-> file hash if "args" in val and "name" in val["args"] and "value" in val["args"]: d["name"] = val["args"]["name"] - d["hash"] = val["args"]["value"] + if type(val["args"]["value"]) is str: + d["hash"] = int(val["args"]["value"],16) + else: + d["hash"] = val["args"]["value"] elif d["name"] == "HH": d["type"] = 2 # 2-> hostname hash if "args" in val and "name" in val["args"] and "value" in val["args"]: d["name"] = val["args"]["name"] - d["hash"] = val["args"]["value"] + if type(val["args"]["value"]) is str: + d["hash"] = int(val["args"]["value"],16) + else: + d["hash"] = val["args"]["value"] elif d["name"] == "SH": d["type"] = 3 # 3-> string hash if "args" in val and "name" in val["args"] and "value" in val["args"]: d["name"] = val["args"]["name"] - d["hash"] = val["args"]["value"] + if type(val["args"]["value"]) is str: + d["hash"] = int(val["args"]["value"],16) + else: + d["hash"] = val["args"]["value"] elif d["name"] == "PR": d["type"] = 5 # 5-> process metadata if "args" in val and "name" in val["args"] and "value" in val["args"]: @@ -267,8 +279,10 @@ def io_function(json_object, current_dict, time_approximate,condition_fn): d["io_time"] = I.to_string(I.empty()) if "args" in json_object: if "fhash" in json_object["args"]: - d["fhash"] = json_object["args"]["fhash"] - + if type(json_object["args"]["fhash"]) is str: + d["fhash"] = int(json_object["args"]["fhash"],16) + else: + d["fhash"] = json_object["args"]["fhash"] if "POSIX" == json_object["cat"] and "ret" in json_object["args"]: size = int(json_object["args"]["ret"]) if size > 0: @@ -290,7 +304,7 @@ def io_columns(): 'io_time': "string[pyarrow]" if not conf.time_approximate else "uint64[pyarrow]", 'app_io_time': "string[pyarrow]" if not conf.time_approximate else "uint64[pyarrow]", 'total_time': "string[pyarrow]" if not conf.time_approximate else "uint64[pyarrow]", - 'fhash': "uint32[pyarrow]", + 'fhash': "uint64[pyarrow]", 'phase': "uint16[pyarrow]", 'size': "uint64[pyarrow]" } diff --git a/examples/dfanalyzer/dfanalyzer.ipynb b/examples/dfanalyzer/dfanalyzer.ipynb index 1dc2973..4fdb5a1 100644 --- a/examples/dfanalyzer/dfanalyzer.ipynb +++ b/examples/dfanalyzer/dfanalyzer.ipynb @@ -156,7 +156,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[INFO] [19:32:21] Initialized Client with 4 workers and link http://127.0.0.1:45919/status [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:754]\n" + "[INFO] [14:58:55] Initialized Client with 4 workers and link http://127.0.0.1:8787/status [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:770]\n" ] } ], @@ -184,7 +184,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 15, "id": "236e50a4-03b6-4895-a0a5-d473f5e391b5", "metadata": { "scrolled": true @@ -194,17 +194,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-02 19:32:22,114 - distributed.nanny - WARNING - Restarting worker\n", - "2024-10-02 19:32:22,143 - distributed.nanny - WARNING - Restarting worker\n", - "2024-10-02 19:32:22,164 - distributed.nanny - WARNING - Restarting worker\n", - "2024-10-02 19:32:22,169 - distributed.nanny - WARNING - Restarting worker\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[INFO] [19:32:23] Restarting all workers [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:742]\n" + "[INFO] [15:01:44] Restarting all workers [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:760]\n" ] } ], @@ -224,7 +214,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 16, "id": "b37727ee-a221-43ab-81e1-cdf98c2cf314", "metadata": { "scrolled": true @@ -234,16 +224,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "[INFO] [19:32:25] Created index for 48 files [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:410]\n", - "[INFO] [19:32:25] Total size of all files are bytes [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:412]\n", - "[INFO] [19:32:25] Loading 48 batches out of 48 files and has 316798 lines overall [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:425]\n", - "[INFO] [19:32:36] Loaded events [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:492]\n", - "[INFO] [19:32:36] Loaded plots with slope threshold: 45 [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:498]\n" + "[INFO] [15:01:46] Created index for 1 files [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:428]\n", + "[INFO] [15:01:46] Total size of all files are bytes [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:430]\n", + "[INFO] [15:01:48] Loading 1 batches out of 1 files and has 30 lines overall [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:443]\n", + "[INFO] [15:01:49] Loaded events [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:510]\n", + "[INFO] [15:01:49] Loaded plots with slope threshold: 45 [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:516]\n" ] } ], "source": [ - "analyzer = DFAnalyzer(f\"{app_root}/examples/dfanalyzer/test-trace-distributed/cm1_1_48_20240926/*.gz\")" + "analyzer = DFAnalyzer(f\"/usr/workspace/haridev/dftracer/build/test/*.gz\")" ] }, { @@ -259,7 +249,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 17, "id": "4c0ffe10", "metadata": {}, "outputs": [ @@ -294,7 +284,6 @@ " dur\n", " tinterval\n", " trange\n", - " hhash\n", " compute_time\n", " io_time\n", " app_io_time\n", @@ -303,6 +292,7 @@ " phase\n", " size\n", " hash\n", + " hhash\n", " value\n", " \n", " \n", @@ -312,14 +302,13 @@ " start\n", " dftracer\n", " 0\n", - " 3537829\n", - " 3537829\n", - " 67327\n", - " 67327\n", + " 3562915\n", + " 3562915\n", + " 0\n", + " 0\n", " 0\n", " <NA>\n", " 0\n", - " 5692\n", " <NA>\n", " <NA>\n", " <NA>\n", @@ -328,98 +317,99 @@ " 0\n", " <NA>\n", " <NA>\n", + " 8770596893269328381\n", " <NA>\n", " \n", " \n", " 6\n", - " fopen\n", - " STDIO\n", + " CUSTOM_BLOCK\n", + " CPP_APP\n", " 0\n", - " 3537829\n", - " 3537829\n", - " 71265\n", - " 71499\n", - " 234\n", + " 3562915\n", + " 3562915\n", + " 13117\n", + " 14186\n", + " 1069\n", " <NA>\n", " 0\n", - " 5692\n", " <NA>\n", " <NA>\n", " <NA>\n", " 0\n", - " 33084\n", + " <NA>\n", " 0\n", " <NA>\n", " <NA>\n", + " 8770596893269328381\n", " <NA>\n", " \n", " \n", " 7\n", - " fread\n", - " STDIO\n", + " CUSTOM\n", + " CPP_APP\n", " 0\n", - " 3537829\n", - " 3537829\n", - " 71525\n", - " 71534\n", - " 9\n", + " 3562915\n", + " 3562915\n", + " 12035\n", + " 14232\n", + " 2197\n", " <NA>\n", " 0\n", - " 5692\n", " <NA>\n", " <NA>\n", " <NA>\n", " 0\n", - " 33084\n", + " <NA>\n", " 0\n", " <NA>\n", " <NA>\n", + " 8770596893269328381\n", " <NA>\n", " \n", " \n", " 8\n", - " fread\n", - " STDIO\n", + " foo\n", + " CPP_APP\n", " 0\n", - " 3537829\n", - " 3537829\n", - " 71569\n", - " 71571\n", - " 2\n", + " 3562915\n", + " 3562915\n", + " 10960\n", + " 14251\n", + " 3291\n", " <NA>\n", " 0\n", - " 5692\n", " <NA>\n", " <NA>\n", " <NA>\n", " 0\n", - " 33084\n", + " <NA>\n", " 0\n", " <NA>\n", " <NA>\n", + " 8770596893269328381\n", " <NA>\n", " \n", " \n", - " 9\n", - " fclose\n", + " 10\n", + " fopen\n", " STDIO\n", " 0\n", - " 3537829\n", - " 3537829\n", - " 71580\n", - " 71583\n", - " 3\n", + " 3562915\n", + " 3562915\n", + " 17774\n", + " 19748\n", + " 1974\n", " <NA>\n", " 0\n", - " 5692\n", " <NA>\n", " <NA>\n", " <NA>\n", " 0\n", - " 33084\n", + " 13134042296031641600\n", " 0\n", " <NA>\n", " <NA>\n", + " 8770596893269328381\n", " <NA>\n", " \n", " \n", @@ -427,29 +417,29 @@ "" ], "text/plain": [ - " name cat type pid tid ts te dur tinterval \\\n", - "4 start dftracer 0 3537829 3537829 67327 67327 0 \n", - "6 fopen STDIO 0 3537829 3537829 71265 71499 234 \n", - "7 fread STDIO 0 3537829 3537829 71525 71534 9 \n", - "8 fread STDIO 0 3537829 3537829 71569 71571 2 \n", - "9 fclose STDIO 0 3537829 3537829 71580 71583 3 \n", + " name cat type pid tid ts te dur \\\n", + "4 start dftracer 0 3562915 3562915 0 0 0 \n", + "6 CUSTOM_BLOCK CPP_APP 0 3562915 3562915 13117 14186 1069 \n", + "7 CUSTOM CPP_APP 0 3562915 3562915 12035 14232 2197 \n", + "8 foo CPP_APP 0 3562915 3562915 10960 14251 3291 \n", + "10 fopen STDIO 0 3562915 3562915 17774 19748 1974 \n", "\n", - " trange hhash compute_time io_time app_io_time total_time fhash \\\n", - "4 0 5692 0 \n", - "6 0 5692 0 33084 \n", - "7 0 5692 0 33084 \n", - "8 0 5692 0 33084 \n", - "9 0 5692 0 33084 \n", + " tinterval trange compute_time io_time app_io_time total_time \\\n", + "4 0 0 \n", + "6 0 0 \n", + "7 0 0 \n", + "8 0 0 \n", + "10 0 0 \n", "\n", - " phase size hash value \n", - "4 0 \n", - "6 0 \n", - "7 0 \n", - "8 0 \n", - "9 0 " + " fhash phase size hash hhash value \n", + "4 0 8770596893269328381 \n", + "6 0 8770596893269328381 \n", + "7 0 8770596893269328381 \n", + "8 0 8770596893269328381 \n", + "10 13134042296031641600 0 8770596893269328381 " ] }, - "execution_count": 10, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -460,7 +450,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 18, "id": "b5e9d273", "metadata": {}, "outputs": [ @@ -486,48 +476,68 @@ " \n", " \n", " name\n", + " pid\n", + " tid\n", + " hhash\n", " \n", " \n", " hash\n", " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " 23\n", - " /usr/tce/backend/installations/linux-rhel8-x86...\n", - " \n", - " \n", - " 191\n", - " /sys/class/drm/card5/device/serial_number\n", + " 4934138233606702080\n", + " /usr/workspace/haridev/dftracer/build/test/data\n", + " 3562915\n", + " 3562915\n", + " 8770596893269328381\n", " \n", " \n", - " 278\n", - " /proc/3537825/fd/39\n", + " 11997177448727658496\n", + " /usr/workspace/haridev/dftracer/build/test/dat...\n", + " 3562915\n", + " 3562915\n", + " 8770596893269328381\n", " \n", " \n", - " 289\n", - " /usr/tce/backend/installations/linux-rhel8-x86...\n", + " 13134042296031641600\n", + " /usr/workspace/haridev/dftracer/build/test/dat...\n", + " 3562915\n", + " 3562915\n", + " 8770596893269328381\n", " \n", " \n", - " 569\n", - " /usr/lib64/hwloc/..\n", + " 16963829776176506880\n", + " /usr/workspace/haridev/dftracer/build/test/dat...\n", + " 3562915\n", + " 3562915\n", + " 8770596893269328381\n", " \n", " \n", "\n", "" ], "text/plain": [ - " name\n", - "hash \n", - "23 /usr/tce/backend/installations/linux-rhel8-x86...\n", - "191 /sys/class/drm/card5/device/serial_number\n", - "278 /proc/3537825/fd/39\n", - "289 /usr/tce/backend/installations/linux-rhel8-x86...\n", - "569 /usr/lib64/hwloc/.." + " name \\\n", + "hash \n", + "4934138233606702080 /usr/workspace/haridev/dftracer/build/test/data \n", + "11997177448727658496 /usr/workspace/haridev/dftracer/build/test/dat... \n", + "13134042296031641600 /usr/workspace/haridev/dftracer/build/test/dat... \n", + "16963829776176506880 /usr/workspace/haridev/dftracer/build/test/dat... \n", + "\n", + " pid tid hhash \n", + "hash \n", + "4934138233606702080 3562915 3562915 8770596893269328381 \n", + "11997177448727658496 3562915 3562915 8770596893269328381 \n", + "13134042296031641600 3562915 3562915 8770596893269328381 \n", + "16963829776176506880 3562915 3562915 8770596893269328381 " ] }, - "execution_count": 11, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -538,7 +548,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 19, "id": "e46df3b8", "metadata": {}, "outputs": [ @@ -564,28 +574,37 @@ " \n", " \n", " name\n", + " pid\n", + " tid\n", + " hhash\n", " \n", " \n", " hash\n", " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " 5692\n", - " corona173\n", + " 8770596893269327872\n", + " corona82\n", + " 3562915\n", + " 3562915\n", + " 8770596893269328381\n", " \n", " \n", "\n", "" ], "text/plain": [ - " name\n", - "hash \n", - "5692 corona173" + " name pid tid hhash\n", + "hash \n", + "8770596893269327872 corona82 3562915 3562915 8770596893269328381" ] }, - "execution_count": 12, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -606,7 +625,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 20, "id": "543dfa22", "metadata": {}, "outputs": [ @@ -633,36 +652,39 @@ " \n", " name\n", " value\n", + " pid\n", + " tid\n", + " hhash\n", " \n", " \n", " \n", " \n", " 1\n", - " 3537829\n", + " 3562915\n", " thread_name\n", + " 3562915\n", + " 3562915\n", + " 8770596893269328381\n", " \n", " \n", - " 3744\n", - " rank\n", - " 45\n", - " \n", - " \n", - " 3745\n", - " Rank 45\n", - " process_name\n", + " 5\n", + " key\n", + " value\n", + " 3562915\n", + " 3562915\n", + " 8770596893269328381\n", " \n", " \n", "\n", "" ], "text/plain": [ - " name value\n", - "1 3537829 thread_name\n", - "3744 rank 45\n", - "3745 Rank 45 process_name" + " name value pid tid hhash\n", + "1 3562915 thread_name 3562915 3562915 8770596893269328381\n", + "5 key value 3562915 3562915 8770596893269328381" ] }, - "execution_count": 13, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -673,7 +695,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 21, "id": "9350218f", "metadata": {}, "outputs": [ @@ -681,7 +703,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[INFO] [19:32:36] Total number of events in the workload are 284041 [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:604]\n" + "[INFO] [15:02:00] Total number of events in the workload are 19 [/usr/WS2/haridev/dftracer/dfanalyzer/main.py:622]\n" ] }, { @@ -690,36 +712,32 @@ "
╭──────────────────────────────────────────────────── Summary ────────────────────────────────────────────────────╮\n",
                             "│  Allocation    Scheduler Allocation Details                                                                     │\n",
                             "│                ├── Nodes: 1                                                                                     │\n",
-                            "│                ├── Processes: 48                                                                                │\n",
+                            "│                ├── Processes: 1                                                                                 │\n",
                             "│                ├── Thread allocations across nodes (includes dynamically created threads)                       │\n",
                             "│                │   ├── Compute: 0                                                                               │\n",
-                            "│                │   └── I/O: 48                                                                                  │\n",
-                            "│                └── Events Recorded: 284K                                                                        │\n",
+                            "│                │   └── I/O: 1                                                                                   │\n",
+                            "│                └── Events Recorded: 19                                                                          │\n",
                             "│  Dataset       Description of Dataset Used                                                                      │\n",
-                            "│                └── Files: 2583                                                                                  │\n",
+                            "│                └── Files: 4                                                                                     │\n",
                             "│  I/O Behavior  Behavior of Application                                                                          │\n",
                             "│                ├── Split of Time in application                                                                 │\n",
-                            "│                │   ├── Total Time: 145.364 sec                                                                  │\n",
-                            "│                │   └── Overall I/O: 1.393 sec                                                                   │\n",
+                            "│                │   ├── Total Time: 0.129 sec                                                                    │\n",
+                            "│                │   └── Overall I/O: 0.092 sec                                                                   │\n",
                             "│                └── Metrics by function                                                                          │\n",
                             "│                    ├── Function       |count |                  size                   |                        │\n",
                             "│                    ├──                |      |min   |25    |mean  |median|75    |max   |                        │\n",
-                            "│                    ├── access         |672   |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
-                            "│                    ├── opendir        |9K    |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
-                            "│                    ├── __xstat        |47K   |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
-                            "│                    ├── open           |22K   |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
-                            "│                    ├── unlink         |192   |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
-                            "│                    ├── read           |14K   |1     |7     |137   |14    |310   |7KB   |                        │\n",
-                            "│                    ├── close          |22K   |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
-                            "│                    ├── mkdir          |76    |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
-                            "│                    ├── ftruncate      |643   |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
-                            "│                    ├── mmap           |576   |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
-                            "│                    ├── readlink       |432   |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
-                            "│                    ├── __fxstat       |3K    |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
-                            "│                    ├── write          |112K  |1     |5KB   |6KB   |5KB   |5KB   |64KB  |                        │\n",
-                            "│                    ├── lseek          |224   |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
-                            "│                    ├── fcntl          |96    |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
-                            "│                    └── rmdir          |58    |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
+                            "│                    ├── link           |1     |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
+                            "│                    ├── unlink         |1     |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
+                            "│                    ├── symlink        |1     |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
+                            "│                    ├── chmod          |1     |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
+                            "│                    ├── chown          |1     |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
+                            "│                    ├── lchown         |1     |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
+                            "│                    ├── utime          |1     |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
+                            "│                    ├── mkfifo         |1     |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
+                            "│                    ├── symlinkat      |1     |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
+                            "│                    ├── chdir          |1     |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
+                            "│                    ├── open           |1     |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
+                            "│                    └── remove         |2     |NA    |nan   |nan   |NA    |nan   |NA    |                        │\n",
                             "╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
                             "
\n" ], @@ -727,36 +745,32 @@ "╭──────────────────────────────────────────────────── Summary ────────────────────────────────────────────────────╮\n", "│ \u001b[36m \u001b[0m\u001b[36mAllocation \u001b[0m\u001b[36m \u001b[0m Scheduler Allocation Details │\n", "│ \u001b[36m \u001b[0m ├── Nodes: 1 │\n", - "│ \u001b[36m \u001b[0m ├── Processes: 48 │\n", + "│ \u001b[36m \u001b[0m ├── Processes: 1 │\n", "│ \u001b[36m \u001b[0m ├── Thread allocations across nodes (includes dynamically created threads) │\n", "│ \u001b[36m \u001b[0m │ ├── Compute: 0 │\n", - "│ \u001b[36m \u001b[0m │ └── I/O: 48 │\n", - "│ \u001b[36m \u001b[0m └── Events Recorded: 284K │\n", + "│ \u001b[36m \u001b[0m │ └── I/O: 1 │\n", + "│ \u001b[36m \u001b[0m └── Events Recorded: 19 │\n", "│ \u001b[36m \u001b[0m\u001b[36mDataset \u001b[0m\u001b[36m \u001b[0m Description of Dataset Used │\n", - "│ \u001b[36m \u001b[0m └── Files: 2583 │\n", + "│ \u001b[36m \u001b[0m └── Files: 4 │\n", "│ \u001b[36m \u001b[0m\u001b[36mI/O Behavior\u001b[0m\u001b[36m \u001b[0m Behavior of Application │\n", "│ \u001b[36m \u001b[0m ├── Split of Time in application │\n", - "│ \u001b[36m \u001b[0m │ ├── Total Time: 145.364 sec │\n", - "│ \u001b[36m \u001b[0m │ └── Overall I/O: 1.393 sec │\n", + "│ \u001b[36m \u001b[0m │ ├── Total Time: 0.129 sec │\n", + "│ \u001b[36m \u001b[0m │ └── Overall I/O: 0.092 sec │\n", "│ \u001b[36m \u001b[0m └── Metrics by function │\n", "│ \u001b[36m \u001b[0m ├── Function |count | size | │\n", "│ \u001b[36m \u001b[0m ├── | |min |25 |mean |median|75 |max | │\n", - "│ \u001b[36m \u001b[0m ├── access |672 |NA |nan |nan |NA |nan |NA | │\n", - "│ \u001b[36m \u001b[0m ├── opendir |9K |NA |nan |nan |NA |nan |NA | │\n", - "│ \u001b[36m \u001b[0m ├── __xstat |47K |NA |nan |nan |NA |nan |NA | │\n", - "│ \u001b[36m \u001b[0m ├── open |22K |NA |nan |nan |NA |nan |NA | │\n", - "│ \u001b[36m \u001b[0m ├── unlink |192 |NA |nan |nan |NA |nan |NA | │\n", - "│ \u001b[36m \u001b[0m ├── read |14K |1 |7 |137 |14 |310 |7KB | │\n", - "│ \u001b[36m \u001b[0m ├── close |22K |NA |nan |nan |NA |nan |NA | │\n", - "│ \u001b[36m \u001b[0m ├── mkdir |76 |NA |nan |nan |NA |nan |NA | │\n", - "│ \u001b[36m \u001b[0m ├── ftruncate |643 |NA |nan |nan |NA |nan |NA | │\n", - "│ \u001b[36m \u001b[0m ├── mmap |576 |NA |nan |nan |NA |nan |NA | │\n", - "│ \u001b[36m \u001b[0m ├── readlink |432 |NA |nan |nan |NA |nan |NA | │\n", - "│ \u001b[36m \u001b[0m ├── __fxstat |3K |NA |nan |nan |NA |nan |NA | │\n", - "│ \u001b[36m \u001b[0m ├── write |112K |1 |5KB |6KB |5KB |5KB |64KB | │\n", - "│ \u001b[36m \u001b[0m ├── lseek |224 |NA |nan |nan |NA |nan |NA | │\n", - "│ \u001b[36m \u001b[0m ├── fcntl |96 |NA |nan |nan |NA |nan |NA | │\n", - "│ \u001b[36m \u001b[0m └── rmdir |58 |NA |nan |nan |NA |nan |NA | │\n", + "│ \u001b[36m \u001b[0m ├── link |1 |NA |nan |nan |NA |nan |NA | │\n", + "│ \u001b[36m \u001b[0m ├── unlink |1 |NA |nan |nan |NA |nan |NA | │\n", + "│ \u001b[36m \u001b[0m ├── symlink |1 |NA |nan |nan |NA |nan |NA | │\n", + "│ \u001b[36m \u001b[0m ├── chmod |1 |NA |nan |nan |NA |nan |NA | │\n", + "│ \u001b[36m \u001b[0m ├── chown |1 |NA |nan |nan |NA |nan |NA | │\n", + "│ \u001b[36m \u001b[0m ├── lchown |1 |NA |nan |nan |NA |nan |NA | │\n", + "│ \u001b[36m \u001b[0m ├── utime |1 |NA |nan |nan |NA |nan |NA | │\n", + "│ \u001b[36m \u001b[0m ├── mkfifo |1 |NA |nan |nan |NA |nan |NA | │\n", + "│ \u001b[36m \u001b[0m ├── symlinkat |1 |NA |nan |nan |NA |nan |NA | │\n", + "│ \u001b[36m \u001b[0m ├── chdir |1 |NA |nan |nan |NA |nan |NA | │\n", + "│ \u001b[36m \u001b[0m ├── open |1 |NA |nan |nan |NA |nan |NA | │\n", + "│ \u001b[36m \u001b[0m └── remove |2 |NA |nan |nan |NA |nan |NA | │\n", "╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\n" ] }, diff --git a/include/dftracer/core/constants.h b/include/dftracer/core/constants.h index d888453..d8dc893 100644 --- a/include/dftracer/core/constants.h +++ b/include/dftracer/core/constants.h @@ -28,5 +28,6 @@ static const int EVENT_TYPE_SIZE = 128; static const unsigned int DFT_PATH_MAX = 1024 * 4; static const char SEPARATOR = ';'; +static const int HASH_OUTPUT = 16; #endif // DFTRACER_CONSTANTS_H diff --git a/src/dftracer/brahma/posix.h b/src/dftracer/brahma/posix.h index df6a869..802d03f 100644 --- a/src/dftracer/brahma/posix.h +++ b/src/dftracer/brahma/posix.h @@ -25,24 +25,24 @@ class POSIXDFTracer : public POSIX { static bool stop_trace; static std::shared_ptr instance; static const int MAX_FD = 1024; - uint16_t tracked_fd[MAX_FD]; + std::string tracked_fd[MAX_FD]; std::shared_ptr logger; bool trace_all_files; - inline uint16_t is_traced(int fd, const char *func) { - if (fd < 0) return 0; - uint16_t trace = tracked_fd[fd % MAX_FD]; - if (trace != 0) { + inline std::string is_traced(int fd, const char *func) { + if (fd < 0) return std::string(); + std::string trace = tracked_fd[fd % MAX_FD]; + if (trace.empty()) { DFTRACER_LOG_DEBUG( "Calling POSIXDFTracer.is_traced for %s and" " fd %d trace %d", - func, fd, trace != 0); + func, fd, !trace.empty()); } return trace; } - inline uint16_t is_traced(const char *filename, const char *func) { - if (stop_trace) return 0; + inline std::string is_traced(const char *filename, const char *func) { + if (stop_trace) return std::string(); if (trace_all_files) { return logger->hash_and_store(filename, METADATA_NAME_FILE_HASH); } else { @@ -56,7 +56,7 @@ class POSIXDFTracer : public POSIX { } } - inline void trace(int fd, uint16_t hash) { + inline void trace(int fd, std::string hash) { DFTRACER_LOG_DEBUG("Calling POSIXDFTracer.trace for %d and %d", fd, hash); if (fd == -1) return; tracked_fd[fd % MAX_FD] = hash; @@ -65,13 +65,13 @@ class POSIXDFTracer : public POSIX { inline void remove_trace(int fd) { DFTRACER_LOG_DEBUG("Calling POSIXDFTracer.remove_trace for %d", fd); if (fd == -1) return; - tracked_fd[fd % MAX_FD] = 0; + tracked_fd[fd % MAX_FD] = std::string(); } public: POSIXDFTracer(bool trace_all) : POSIX(), trace_all_files(trace_all) { DFTRACER_LOG_DEBUG("POSIX class intercepted", ""); - for (int i = 0; i < MAX_FD; ++i) tracked_fd[i] = 0; + for (int i = 0; i < MAX_FD; ++i) tracked_fd[i] = std::string(); logger = DFT_LOGGER_INIT(); } void finalize() { diff --git a/src/dftracer/brahma/stdio.h b/src/dftracer/brahma/stdio.h index db07bb4..dc1a43c 100644 --- a/src/dftracer/brahma/stdio.h +++ b/src/dftracer/brahma/stdio.h @@ -22,25 +22,25 @@ class STDIODFTracer : public STDIO { private: static bool stop_trace; static std::shared_ptr instance; - std::unordered_map tracked_fh; + std::unordered_map tracked_fh; std::shared_ptr logger; bool trace_all_files; - inline uint16_t is_traced(FILE *fh, const char *func) { + inline std::string is_traced(FILE *fh, const char *func) { DFTRACER_LOG_DEBUG("Calling STDIODFTracer.is_traced for %s", func); - if (stop_trace) return 0; - if (fh == NULL) return 0; + if (stop_trace) return std::string(); + if (fh == NULL) return std::string(); auto iter = tracked_fh.find(fh); if (iter != tracked_fh.end()) { return iter->second; } - return 0; + return std::string(); } - inline uint16_t is_traced(const char *filename, const char *func) { + inline std::string is_traced(const char *filename, const char *func) { DFTRACER_LOG_DEBUG("Calling STDIODFTracer.is_traced with filename for %s", func); - if (stop_trace) return 0; + if (stop_trace) return std::string(); if (trace_all_files) return logger->hash_and_store(filename, METADATA_NAME_FILE_HASH); else { @@ -49,7 +49,7 @@ class STDIODFTracer : public STDIO { } } - inline void trace(FILE *fh, uint16_t hash) { + inline void trace(FILE *fh, std::string hash) { DFTRACER_LOG_DEBUG("Calling STDIODFTracer.trace with hash %d", hash); tracked_fh.insert_or_assign(fh, hash); } diff --git a/src/dftracer/df_logger.h b/src/dftracer/df_logger.h index 556f072..09a666b 100644 --- a/src/dftracer/df_logger.h +++ b/src/dftracer/df_logger.h @@ -5,6 +5,7 @@ #ifndef DFTRACER_GENERIC_LOGGER_H #define DFTRACER_GENERIC_LOGGER_H +#include #include #include #include @@ -44,7 +45,7 @@ class DFTLogger { std::shared_ptr writer; uint32_t level; std::vector index_stack; - std::unordered_map computed_hash; + std::unordered_map computed_hash; std::atomic_int index; bool has_entry; #ifdef DFTRACER_MPI_ENABLE @@ -102,6 +103,19 @@ class DFTLogger { this->is_init = true; } ~DFTLogger() {} + + inline std::string get_hash(char *name) { + uint8_t result[HASH_OUTPUT]; + md5String(name, result); + std::string hash; + hash.reserve(HASH_OUTPUT + 1); + for (int i = 0; i < HASH_OUTPUT; ++i) { + sprintf(hash.data() + i, "%02x", result[i]); + } + hash.data()[HASH_OUTPUT] = '\0'; + return hash; + } + inline void update_log_file(std::string log_file, std::string exec_name, std::string cmd, ProcessID process_id = -1) { DFTRACER_LOG_DEBUG("DFTLogger.update_log_file %s", log_file.c_str()); @@ -111,15 +125,15 @@ class DFTLogger { tid = df_gettid(); } this->writer = dftracer::Singleton::get_instance(); - uint16_t hostname_hash; - uint16_t cmd_hash; - uint16_t exec_hash; + std::string hostname_hash; + std::string cmd_hash; + std::string exec_hash; if (this->writer != nullptr) { char hostname[256]; gethostname(hostname, 256); - md5String(hostname, &hostname_hash); + hostname_hash = get_hash(hostname); this->writer->initialize(log_file.data(), this->throw_error, - hostname_hash); + hostname_hash.c_str()); hostname_hash = hash_and_store(hostname, METADATA_NAME_HOSTNAME_HASH); char thread_name[128]; auto size = sprintf(thread_name, "%lu", this->process_id); @@ -218,16 +232,16 @@ class DFTLogger { return -1; } - inline uint16_t has_hash(ConstEventNameType key) { + inline std::string has_hash(ConstEventNameType key) { std::shared_lock lock(map_mtx); auto iter = computed_hash.find(key); - if (iter != computed_hash.end()) iter->second; - return 0; + if (iter != computed_hash.end()) return iter->second.c_str(); + return std::string(); } - inline void insert_hash(ConstEventNameType key, uint16_t hash) { + inline void insert_hash(ConstEventNameType key, std::string hash) { std::unique_lock lock(map_mtx); - computed_hash.insert_or_assign(key, hash); + computed_hash.insert_or_assign(key, hash.c_str()); } inline TimeResolution get_time() { @@ -321,38 +335,37 @@ class DFTLogger { } } - inline uint16_t hash_and_store(char *filename, ConstEventNameType name) { - if (filename == NULL) return 0; + inline std::string hash_and_store(char *filename, ConstEventNameType name) { + if (filename == NULL) return std::string(); char file[PATH_MAX]; strcpy(file, filename); file[PATH_MAX - 1] = '\0'; return hash_and_store_str(file, name); } - inline uint16_t hash_and_store_str(char file[PATH_MAX], - ConstEventNameType name) { - uint16_t hash = has_hash(file); - if (hash == 0) { - md5String(file, &hash); - insert_hash(file, hash); + inline std::string hash_and_store_str(char file[PATH_MAX], + ConstEventNameType name) { + std::string hash = has_hash(file); + if (hash.empty()) { + hash = get_hash(file); + insert_hash(file, hash.c_str()); if (this->writer != nullptr) { ThreadID tid = 0; if (dftracer_tid) { tid = df_gettid(); } int current_index = this->enter_event(); - this->writer->log_metadata(current_index, file, - std::to_string(hash).c_str(), name, - this->process_id, tid, false); + this->writer->log_metadata(current_index, file, hash.c_str(), name, + this->process_id, tid); this->exit_event(); } } return hash; } - inline uint16_t hash_and_store(const char *filename, - ConstEventNameType name) { - if (filename == NULL) return 0; + inline std::string hash_and_store(const char *filename, + ConstEventNameType name) { + if (filename == NULL) return std::string(); char file[PATH_MAX]; strcpy(file, filename); file[PATH_MAX - 1] = '\0'; @@ -386,15 +399,15 @@ class DFTLogger { #define DFT_LOGGER_UPDATE_HASH(value) \ if (trace && this->logger->include_metadata) { \ - uint16_t value##_hash = \ + std::string value##_hash = \ this->logger->hash_and_store(value, METADATA_NAME_FILE_HASH); \ DFT_LOGGER_UPDATE(value##_hash); \ } #define DFT_LOGGER_START(entity) \ DFTRACER_LOG_DEBUG("Calling function %s", __FUNCTION__); \ - uint16_t fhash = is_traced(entity, __FUNCTION__); \ - bool trace = fhash != 0; \ + std::string fhash = is_traced(entity, __FUNCTION__); \ + bool trace = !fhash.empty(); \ TimeResolution start_time = 0; \ std::unordered_map *metadata = nullptr; \ if (trace) { \ diff --git a/src/dftracer/utils/md5.cpp b/src/dftracer/utils/md5.cpp index ce00f86..7dab8a1 100644 --- a/src/dftracer/utils/md5.cpp +++ b/src/dftracer/utils/md5.cpp @@ -4,7 +4,7 @@ * structures. */ -#include +#include "md5.h" /* * Constants defined by the MD5 algorithm @@ -193,16 +193,17 @@ void md5Step(uint32_t *buffer, uint32_t *input) { * Functions that run the algorithm on the provided input and put the digest * into result. result should be able to store 16 bytes. */ -void md5String(char *input, uint16_t *result) { +void md5String(char *input, uint8_t *result) { MD5Context ctx; md5Init(&ctx); md5Update(&ctx, (uint8_t *)input, strlen(input)); md5Finalize(&ctx); + memcpy(result, ctx.digest, 16); } -void md5File(FILE *file, uint16_t *result) { - char *input_buffer = (char *)malloc(1024 * 1024); +void md5File(FILE *file, uint8_t *result) { + char *input_buffer = (char *)malloc(1024); size_t input_size = 0; MD5Context ctx; diff --git a/src/dftracer/utils/md5.h b/src/dftracer/utils/md5.h index d67766e..d61b95d 100644 --- a/src/dftracer/utils/md5.h +++ b/src/dftracer/utils/md5.h @@ -1,25 +1,24 @@ -#ifndef DFTRACER_MD5_H -#define DFTRACER_MD5_H +#ifndef MD5_H +#define MD5_H -#include -#include #include -#include +#include #include +#include -typedef struct { - uint64_t size; // Size of input in bytes - uint32_t buffer[4]; // Current accumulation of hash - uint8_t input[PATH_MAX]; // Input to be used in the next step - uint8_t digest[16]; // Result of algorithm -} MD5Context; +typedef struct{ + uint64_t size; // Size of input in bytes + uint32_t buffer[4]; // Current accumulation of hash + uint8_t input[64]; // Input to be used in the next step + uint8_t digest[16]; // Result of algorithm +}MD5Context; void md5Init(MD5Context *ctx); void md5Update(MD5Context *ctx, uint8_t *input, size_t input_len); void md5Finalize(MD5Context *ctx); void md5Step(uint32_t *buffer, uint32_t *input); -void md5String(char *input, uint16_t *result); -void md5File(FILE *file, uint16_t *result); +void md5String(char *input, uint8_t *result); +void md5File(FILE *file, uint8_t *result); -#endif // DFTRACER_MD5_H \ No newline at end of file +#endif \ No newline at end of file diff --git a/src/dftracer/writer/chrome_writer.cpp b/src/dftracer/writer/chrome_writer.cpp index 9be3a65..cee847a 100644 --- a/src/dftracer/writer/chrome_writer.cpp +++ b/src/dftracer/writer/chrome_writer.cpp @@ -22,7 +22,7 @@ template <> bool dftracer::Singleton::stop_creating_instances = false; void dftracer::ChromeWriter::initialize(char *filename, bool throw_error, - uint16_t hostname_hash) { + const char *hostname_hash) { this->hostname_hash = hostname_hash; this->throw_error = throw_error; this->filename = filename; @@ -214,9 +214,10 @@ void dftracer::ChromeWriter::convert_json( previous_index = current_index; auto written_size = sprintf( buffer.data() + current_index, - R"(%s{"id":%d,"name":"%s","cat":"%s","pid":%lu,"tid":%lu,"ts":%llu,"dur":%llu,"ph":"X","args":{"hhash":%d%s}})", + R"(%s{"id":%d,"name":"%s","cat":"%s","pid":%lu,"tid":%lu,"ts":%llu,"dur":%llu,"ph":"X","args":{"hhash":"%s"%s}})", is_first_char, index, event_name, category, process_id, thread_id, - start_time, duration, this->hostname_hash, all_stream.str().c_str()); + start_time, duration, this->hostname_hash.c_str(), + all_stream.str().c_str()); current_index += written_size; buffer[current_index] = '\n'; current_index++; @@ -255,13 +256,15 @@ void dftracer::ChromeWriter::convert_json_metadata( if (is_string) { written_size = sprintf( buffer.data() + current_index, - R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"hhash":%d,"name":"%s","value":"%s"}})", - is_first_char, index, ph, process_id, thread_id, name, value); + R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"hhash":"%s","name":"%s","value":"%s"}})", + is_first_char, index, ph, process_id, thread_id, + this->hostname_hash.c_str(), name, value); } else { written_size = sprintf( buffer.data() + current_index, - R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"hhash":%d,"name":"%s","value":%s}})", - is_first_char, index, ph, process_id, thread_id, name, value); + R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"hhash":"%s","name":"%s","value":%s}})", + is_first_char, index, ph, process_id, thread_id, + this->hostname_hash.c_str(), name, value); } current_index += written_size; buffer[current_index] = '\n'; diff --git a/src/dftracer/writer/chrome_writer.h b/src/dftracer/writer/chrome_writer.h index f3567e9..1c917ec 100644 --- a/src/dftracer/writer/chrome_writer.h +++ b/src/dftracer/writer/chrome_writer.h @@ -36,7 +36,7 @@ class ChromeWriter { bool enable_core_affinity; FILE *fh; - uint16_t hostname_hash; + std::string hostname_hash; static const int MAX_LINE_SIZE = 16 * 1024L; size_t write_buffer_size; @@ -67,7 +67,7 @@ class ChromeWriter { funlockfile(fh); if (written_elements != 1) { // GCOVR_EXCL_START DFTRACER_LOG_ERROR( - "unable to log write only %ld of %d trying to write %d with error " + "unable to log write only %ld of %d trying to write %ld with error " "code " "%d", written_elements, 1, current_index, errno); @@ -100,7 +100,7 @@ class ChromeWriter { } } ~ChromeWriter() { DFTRACER_LOG_DEBUG("Destructing ChromeWriter", ""); } - void initialize(char *filename, bool throw_error, uint16_t hostname_hash); + void initialize(char *filename, bool throw_error, const char *hostname_hash); void log(int index, ConstEventNameType event_name, ConstEventNameType category, TimeResolution start_time, diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ea72b7c..59634ac 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -32,7 +32,7 @@ function(set_common_properties test_name) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_LOG_FILE=${CMAKE_CURRENT_BINARY_DIR}/${test_name}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_ENABLE=1) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_INC_METADATA=1) - set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_BIND_SIGNALS=1) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_BIND_SIGNALS=0) endfunction() file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data) @@ -43,14 +43,14 @@ set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT LD_PRELOAD=${CMAKE_BI set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_INIT=PRELOAD) find_program(BASH_PROGRAM bash) -df_add_test(check_file_exists_${test_name} ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/check_file_at_least.sh ${CMAKE_CURRENT_BINARY_DIR}/${test_name}* 30) +df_add_test(check_file_exists_${test_name} ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/check_file_at_least.sh ${CMAKE_CURRENT_BINARY_DIR}/${test_name}* 29) set_tests_properties(check_file_exists_${test_name} PROPERTIES DEPENDS ${test_name}) set(test_name test_cpp_basic_app_only) df_add_test(${test_name} ${CMAKE_BINARY_DIR}/bin/test_cpp ${CMAKE_CURRENT_BINARY_DIR}/data 1) set_common_properties(${test_name}) -df_add_test(check_file_exists_${test_name} ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/check_file_at_least.sh ${CMAKE_CURRENT_BINARY_DIR}/${test_name}* 30) +df_add_test(check_file_exists_${test_name} ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/check_file_at_least.sh ${CMAKE_CURRENT_BINARY_DIR}/${test_name}* 29) set_tests_properties(check_file_exists_${test_name} PROPERTIES DEPENDS ${test_name}) set(test_name test_cpp_basic_app_only_yaml) @@ -104,7 +104,7 @@ set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT LD_PRELOAD=${CMAKE_BI set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_INIT=PRELOAD) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_INC_METADATA=1) -df_add_test(check_file_exists_${test_name} ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/check_file_at_least.sh ${CMAKE_CURRENT_BINARY_DIR}/${test_name}* 30) +df_add_test(check_file_exists_${test_name} ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/check_file_at_least.sh ${CMAKE_CURRENT_BINARY_DIR}/${test_name}* 29) set_tests_properties(check_file_exists_${test_name} PROPERTIES DEPENDS ${test_name}) set(test_name test_cpp_basic_affinity) @@ -115,7 +115,7 @@ set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_INIT=PRELOAD set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_INC_METADATA=1) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DFTRACER_SET_CORE_AFFINITY=1) -df_add_test(check_file_exists_${test_name} ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/check_file_at_least.sh ${CMAKE_CURRENT_BINARY_DIR}/${test_name}* 30) +df_add_test(check_file_exists_${test_name} ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/check_file_at_least.sh ${CMAKE_CURRENT_BINARY_DIR}/${test_name}* 29) set_tests_properties(check_file_exists_${test_name} PROPERTIES DEPENDS ${test_name}) set(test_name test_py_disable_only) From d0cff0bfbb526dc63dad1abe42b64dfccb12b28d Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Mon, 7 Oct 2024 21:08:20 -0700 Subject: [PATCH 08/15] Update requirements.txt (#217) * Update .readthedocs.yaml * Update requirements.txt --- .readthedocs.yaml | 6 +++--- docs/requirements.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f9dde6b..7b17d99 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -27,6 +27,6 @@ sphinx: # Optional but recommended, declare the Python requirements required # to build your documentation # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html -# python: -# install: -# - requirements: docs/requirements.txt \ No newline at end of file +python: + install: + - requirements: docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt index 70b23e3..27385d7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ +sphinx<7.0.0 sphinx-rtd-theme -Sphinx<7 \ No newline at end of file From 365c0e5f1cb3d1f39408fff12541dda740ee1b32 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Tue, 8 Oct 2024 00:58:12 -0700 Subject: [PATCH 09/15] Update dftracer_split --- script/dftracer_split | 45 ++++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/script/dftracer_split b/script/dftracer_split index 6756eb5..a204236 100755 --- a/script/dftracer_split +++ b/script/dftracer_split @@ -220,14 +220,15 @@ CHUNKS=() accumulated_size=0 temp_chunk="" scheduled_chunks=0 - +CHUNKS_LINES_TOTAL=${#CHUNKS_LINES[@]} chunk_index=1 chunk_index_file=".chunk-${chunk_index}.bak" line_number=0 +total_size=0 while IFS='|' read -r index file_name size start end; do line_number=$((line_number + 1)) - if [ $line_number -lt $file_processed ]; then + if [ $CHUNKS_LINES_TOTAL -gt 0 ] && [ $line_number -lt $file_processed ]; then if [[ $line_number -eq ${CHUNKS_LINES[$chunk_index]} ]]; then chunk_index=$((chunk_index + 1)) chunk_index_file=".chunk-${chunk_index}.bak" @@ -257,22 +258,16 @@ while IFS='|' read -r index file_name size start end; do progress_date_echo "Scheduling $index of $total to chunk $chunk_index with size $accumulated_size" i=$chunk_index scheduled_chunks=$chunk_index - { - if [ "$verbose" == "1" ]; then - date_echo "Processing chunk $i with size $size MB" - fi - chunk_file=$dest/${app_name}-$i.pfw - rm -f $chunk_file - touch $chunk_file - echo '[' > $chunk_file - while IFS='|' read -r file_name size start end; do - if [ "$verbose" == "1" ]; then - date_echo "[CHUNK $i] Extracting from $file_name from $start to $end with size $size" - fi - $zq_exec ${file_name}.pfw.gz --index-file ${file_name}.pfw.gz.zindex --raw "select a.line from LineOffsets a where a.line >= $start AND a.line <= $end AND a.length > 8;" | grep -v '^[[]\|^[]]' >> $chunk_file - total_size=$(bc -l <<< "scale=2; $total_size + $size") - done < ${chunk_index_file} - progress_date_echo "Chunk $i out of ${#CHUNKS_LINES[@]} done with size $total_size MB, path = $chunk_file" + { + chunk_file=$dest/${app_name}-$i.pfw + rm -f $chunk_file + touch $chunk_file + echo '[' > $chunk_file + while IFS='|' read -r file_name size start end; do + $zq_exec ${file_name}.pfw.gz --index-file ${file_name}.pfw.gz.zindex --raw "select a.line from LineOffsets a where a.line >= $start AND a.line <= $end AND a.length > 8;" >> $chunk_file + total_size=$(bc -l <<< "scale=2; $total_size + $size") + done < ${chunk_index_file} + progress_date_echo "Chunk $i out of ${#CHUNKS_LINES[@]} done with size $total_size MB, path = $chunk_file" } & chunk_index=$((chunk_index + 1)) accumulated_size=0 @@ -306,27 +301,21 @@ for i in $(seq $start $total_chunks); do done date_echo "Running $running_jobs jobs are now less than $JOBS_LIMIT" fi - #{ - if [ "$verbose" == "1" ]; then - date_echo "Processing chunk $i with size $size MB" - fi + { chunk_file=$dest/${app_name}-$i.pfw rm -f $chunk_file touch $chunk_file echo '[' > $chunk_file while IFS='|' read -r file_name size start end; do - if [ "$verbose" == "1" ]; then - date_echo "[CHUNK $i] Extracting from $file_name from $start to $end with size $size" - fi $zq_exec ${file_name}.pfw.gz --index-file ${file_name}.pfw.gz.zindex --raw "select a.line from LineOffsets a where a.line >= $start AND a.line <= $end AND a.length > 8;" | grep -v '^[[]\|^[]]' >> $chunk_file total_size=$(bc -l <<< "scale=2; $total_size + $size") done < ${chunk_index_file} echo ']' >> $chunk_file progress_date_echo "Chunk $i out of ${total_chunks} done with size $total_size MB, path = $chunk_file" - #} & + } & done wait - +echo "" date_echo "All chunks processed" date_echo Reindexing split files @@ -344,6 +333,7 @@ SPLIT_LINES_COUNT=$( $SCRIPT_DIR/dftracer_event_count -d $dest } &) wait +rm -f .chunk* counting.bak if [ $LINES_COUNT -ne $SPLIT_LINES_COUNT ]; then date_echo "Error: Original lines count $LINES_COUNT does not match split lines count $SPLIT_LINES_COUNT" @@ -352,5 +342,4 @@ else date_echo "Original lines count $LINES_COUNT matches split lines count $SPLIT_LINES_COUNT" fi popd > /dev/null -rm -f .chunk* counting.bak date_echo Done reindexing split files From 7d2f3a8532f84ce9322c23897ec983227344a688 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Tue, 8 Oct 2024 22:06:36 -0700 Subject: [PATCH 10/15] multithreaded exclusive lock (#218) * multithreaded exclusive lock * Added validate script --- CMakeLists.txt | 8 ++ docs/utilities.rst | 19 +++- script/dftracer_validate | 122 ++++++++++++++++++++++++++ setup.py | 3 +- src/dftracer/writer/chrome_writer.cpp | 6 +- src/dftracer/writer/chrome_writer.h | 6 +- 6 files changed, 155 insertions(+), 9 deletions(-) create mode 100755 script/dftracer_validate diff --git a/CMakeLists.txt b/CMakeLists.txt index 7487491..222eb53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -476,6 +476,14 @@ install( bin ) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/script/dftracer_validate ${EXECUTABLE_OUTPUT_PATH}/dftracer_validate COPYONLY) +install( + FILES + ${EXECUTABLE_OUTPUT_PATH}/dftracer_validate + DESTINATION + bin +) + #cmake_policy(SET CMP0079 NEW) # In case that we need more control over the target building order if(DFTRACER_ENABLE_TESTS) diff --git a/docs/utilities.rst b/docs/utilities.rst index 2cb8e37..c00c4d7 100644 --- a/docs/utilities.rst +++ b/docs/utilities.rst @@ -113,7 +113,7 @@ The script splits the traces into equal sized chunk optimized for analysis .. code-block:: bash - /bin/usage: dftracer_split [-fv] [-n app_name] [-d input_directory] [-o output_directory] [-s chunk_size] + /bin/dftracer_split [-fv] [-n app_name] [-d input_directory] [-o output_directory] [-s chunk_size] Arguments for this script are: @@ -133,10 +133,25 @@ The script will count number of valid events of traces .. code-block:: bash - /bin/usage: dftracer_event_count [-f] [-d input_directory] + /bin/dftracer_event_count [-f] [-d input_directory] Arguments for this script are: 1. **-d input_directory** specify input directories. should contain .pfw or .pfw.gz files. 2. **-f** force index creation 3. **-h** display help + +------------------ +Validating DFTracer traces +------------------ + +The script validates dftracer traces + +.. code-block:: bash + + /bin/dftracer_validate [-v] [-d input_directory] + +Arguments for this script are: + +1. **-d input_directory** specify input directories. should contain .pfw or .pfw.gz files. +3. **-h** display help diff --git a/script/dftracer_validate b/script/dftracer_validate new file mode 100755 index 0000000..36ac4bc --- /dev/null +++ b/script/dftracer_validate @@ -0,0 +1,122 @@ +#!/bin/bash + +# The script validates dftracer traces +# This has the following signature. +# +# usage: dftracer_validate [-v] [-d input_directory] +# -v enable verbose mode +# -h display help +# -d input_directory specify input directories. should contain .pfw or .pfw.gz files. + +date_echo() { + dt=$(date '+%d/%m/%Y %H:%M:%S'); + echo "$dt $@" +} + +progress_date_echo() { + dt=$(date '+%d/%m/%Y %H:%M:%S') + echo -ne "$dt $@"\\r +} + +LOG_DIR=$PWD + +PPWD=$PWD + +function usage { + echo "usage: $(basename $0) [-cv] [-d input_directory]" + echo " -v enable verbose mode" + echo " -h display help" + echo " -d input_directory specify input directories. should contain .pfw or .pfw.gz files." + exit 1 +} +while getopts ':cvfd:h' opt; do + case "$opt" in + d) + LOG_DIR="${OPTARG}" + ;; + v) + set -x + ;; + h) + usage + exit 0 + ;; + + :) + echo -e "option requires an argument.\n" + usage + exit 1 + ;; + + ?) + echo -e "Invalid command option.\n" + usage + exit 1 + ;; + esac +done +shift "$(($OPTIND -1))" +total=0 +for file in *.pfw*; do total=1; break; done + +# pfw_count=$(ls -1 $LOG_DIR/*.pfw 2> /dev/null | wc -l) +# gz_count=$(ls -1 $LOG_DIR/*.gz 2> /dev/null | wc -l) +# total=$((pfw_count + gz_count)) +if [ $total == 0 ]; then + date_echo "The folder does not contain any pfw or pfw.gz files." + exit 1 +fi + +pushd $LOG_DIR > /dev/null +JOBS_LIMIT=$(nproc --all) +files=("$LOG_DIR"/*.pfw*) +total=${#files[@]} +# loop over logs +for file_index in "${!files[@]}"; do + file=${files[$file_index]} + filename=$(basename -- "$file") + ext="${filename##*.}" + if [ "$ext" != "zindex" ]; then + running_jobs=$(jobs -rp | wc -l) + if [ $running_jobs -ge $JOBS_LIMIT ]; then + date_echo "waiting for Running $running_jobs jobs to be less than $JOBS_LIMIT" + while [ $running_jobs -ge $JOBS_LIMIT ] + do + sleep 1 + running_jobs=$(jobs -rp | wc -l) + done + date_echo "Running $running_jobs jobs are now less than $JOBS_LIMIT" + fi + # only look at files + if [ -f "$file" ]; then + # calculate basename and copy files + filename=$(basename -- "$file") + ext="${filename##*.}" + if [ "$ext" == "gz" ]; then + { + # if file is gz get the name + name=${filename%.pfw.gz} + if jq --slurp -e >/dev/null 2>&1 <<< $(gunzip -c $file | grep -v "\[" | grep -v "\]" | awk '{$file=$file;print}'); then + echo -ne ""; + else + echo "Failed to parse JSON for $file"; + fi + } & + else + { + # if file is pfw get the name + name=${filename%.pfw} + if jq --slurp -e >/dev/null 2>&1 <<< $(cat $file | grep -v "\[" | grep -v "\]" | awk '{$file=$file;print}'); then + echo -ne ""; + else + echo "Failed to parse JSON for $file"; + fi + } & + fi + fi + fi +done +popd > /dev/null +wait +echo "" +date_echo Validating traces finished diff --git a/setup.py b/setup.py index fcfec35..6274ab7 100644 --- a/setup.py +++ b/setup.py @@ -198,7 +198,8 @@ def build_extension(self, ext: CMakeExtension) -> None: 'script/dftracer_anonymize', 'script/dftracer_split', 'script/dftracer_create_index', - 'script/dftracer_event_count', ], + 'script/dftracer_event_count', + 'script/dftracer_validate', ], package_dir={"dftracer": "dftracer", "dftracer_dbg": "dftracer_dbg", "dfanalyzer": "dfanalyzer"}, diff --git a/src/dftracer/writer/chrome_writer.cpp b/src/dftracer/writer/chrome_writer.cpp index cee847a..9adb7b8 100644 --- a/src/dftracer/writer/chrome_writer.cpp +++ b/src/dftracer/writer/chrome_writer.cpp @@ -210,7 +210,7 @@ void dftracer::ChromeWriter::convert_json( all_stream << "," << meta_stream.str(); } { - std::unique_lock lock(mtx); + std::unique_lock lock(mtx); previous_index = current_index; auto written_size = sprintf( buffer.data() + current_index, @@ -224,7 +224,7 @@ void dftracer::ChromeWriter::convert_json( } } else { { - std::unique_lock lock(mtx); + std::unique_lock lock(mtx); previous_index = current_index; auto written_size = sprintf( buffer.data() + current_index, @@ -250,7 +250,7 @@ void dftracer::ChromeWriter::convert_json_metadata( char is_first_char[3] = " "; if (!is_first_write) is_first_char[0] = '\0'; { - std::unique_lock lock(mtx); + std::unique_lock lock(mtx); previous_index = current_index; auto written_size = 0; if (is_string) { diff --git a/src/dftracer/writer/chrome_writer.h b/src/dftracer/writer/chrome_writer.h index 1c917ec..1990ccb 100644 --- a/src/dftracer/writer/chrome_writer.h +++ b/src/dftracer/writer/chrome_writer.h @@ -24,7 +24,7 @@ namespace dftracer { class ChromeWriter { private: std::unordered_map metadata; - std::shared_mutex mtx; + std::mutex mtx; protected: bool throw_error; @@ -55,7 +55,7 @@ class ChromeWriter { bool is_first_write; inline size_t write_buffer_op(bool force = false) { - std::unique_lock lock(mtx); + std::unique_lock lock(mtx); if (current_index == 0 || (!force && current_index < write_buffer_size)) return 0; DFTRACER_LOG_DEBUG("ChromeWriter.write_buffer_op %s", @@ -94,7 +94,7 @@ class ChromeWriter { enable_compression = conf->compression; write_buffer_size = conf->write_buffer_size; { - std::unique_lock lock(mtx); + std::unique_lock lock(mtx); buffer = std::vector(write_buffer_size + MAX_LINE_SIZE); current_index = 0; } From 4b82a073187fbe0fd81288229f6b0d6eebc62244 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Wed, 9 Oct 2024 20:28:07 -0700 Subject: [PATCH 11/15] Bugfix: Escape special non-json-line compatible characters from hash strings. (#219) * remove quotes from strings * ignore multiple characters * added validation script --- CMakeLists.txt | 2 +- script/dftracer_create_index | 4 +-- script/dftracer_validate | 38 +++++++++++++++++------------ src/dftracer/brahma/posix.h | 3 ++- src/dftracer/core/dftracer_main.cpp | 6 +++-- src/dftracer/core/dftracer_main.h | 2 +- src/dftracer/df_logger.h | 27 ++++++++++++++++---- src/dftracer/dftracer.cpp | 6 ++--- 8 files changed, 58 insertions(+), 30 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 222eb53..3e232c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ project(dftracer LANGUAGES C CXX) # Convenience defines string(TOUPPER "${PROJECT_NAME}" UPPER_PROJECT_NAME) string(TOLOWER "${PROJECT_NAME}" LOWER_PROJECT_NAME) - +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-empty-body -Wno-format-extra-args") #------------------------------------------------------------------------------ # Internal Includes for header and libraries #------------------------------------------------------------------------------ diff --git a/script/dftracer_create_index b/script/dftracer_create_index index 9448ffd..402305f 100755 --- a/script/dftracer_create_index +++ b/script/dftracer_create_index @@ -135,7 +135,7 @@ for file_index in "${!files[@]}"; do if [ ! -f "$file.zindex" ]; then $zindex_exec $file --index-file file:$file.zindex --regex 'id:([0-9]+)' --numeric --unique fi - progress_date_echo "Created index for file $file $((file_index+1)) of $total" + progress_date_echo "Created index for file $name $((file_index+1)) of $total" } & else { @@ -148,7 +148,7 @@ for file_index in "${!files[@]}"; do $zindex_exec $file.gz --index-file file:$file.gz.zindex --regex 'id:([0-9]+)' --numeric --unique fi fi - progress_date_echo "Created index for file $file.gz $((file_index+1)) of $total" + progress_date_echo "Created index for file $name.gz $((file_index+1)) of $total" } & fi fi diff --git a/script/dftracer_validate b/script/dftracer_validate index 36ac4bc..f928826 100755 --- a/script/dftracer_validate +++ b/script/dftracer_validate @@ -1,4 +1,4 @@ -#!/bin/bash + #!/bin/bash # The script validates dftracer traces # This has the following signature. @@ -67,9 +67,13 @@ if [ $total == 0 ]; then exit 1 fi -pushd $LOG_DIR > /dev/null JOBS_LIMIT=$(nproc --all) -files=("$LOG_DIR"/*.pfw*) +if [ -d $LOG_DIR ]; then + files=("$LOG_DIR"/*.pfw*) + pushd $LOG_DIR > /dev/null +else + files=($LOG_DIR) +fi total=${#files[@]} # loop over logs for file_index in "${!files[@]}"; do @@ -79,13 +83,13 @@ for file_index in "${!files[@]}"; do if [ "$ext" != "zindex" ]; then running_jobs=$(jobs -rp | wc -l) if [ $running_jobs -ge $JOBS_LIMIT ]; then - date_echo "waiting for Running $running_jobs jobs to be less than $JOBS_LIMIT" + # date_echo "waiting for Running $running_jobs jobs to be less than $JOBS_LIMIT" while [ $running_jobs -ge $JOBS_LIMIT ] do sleep 1 running_jobs=$(jobs -rp | wc -l) done - date_echo "Running $running_jobs jobs are now less than $JOBS_LIMIT" + # date_echo "Running $running_jobs jobs are now less than $JOBS_LIMIT" fi # only look at files if [ -f "$file" ]; then @@ -94,29 +98,33 @@ for file_index in "${!files[@]}"; do ext="${filename##*.}" if [ "$ext" == "gz" ]; then { - # if file is gz get the name - name=${filename%.pfw.gz} - if jq --slurp -e >/dev/null 2>&1 <<< $(gunzip -c $file | grep -v "\[" | grep -v "\]" | awk '{$file=$file;print}'); then + # if file is gz get the name + name=${filename%.pfw.gz} + if jq -e -c > /dev/null <<< $(gunzip -c $file | grep -v "\[" | grep -v "\]"); then echo -ne ""; else - echo "Failed to parse JSON for $file"; - fi + echo "Failed to parse JSON for $name on line $line_number"; + fi + progress_date_echo "Completed $file_index of $total" } & else { - # if file is pfw get the name - name=${filename%.pfw} - if jq --slurp -e >/dev/null 2>&1 <<< $(cat $file | grep -v "\[" | grep -v "\]" | awk '{$file=$file;print}'); then + # if file is pfw get the name + name=${filename%.pfw} + if jq -e -c > /dev/null <<< $(cat $file | grep -v "\[" | grep -v "\]"); then echo -ne ""; else - echo "Failed to parse JSON for $file"; - fi + echo "Failed to parse JSON for $name on line $line_number"; + fi + progress_date_echo "Completed $file_index of $total" } & fi fi fi done +if [ -d $LOG_DIR ]; then popd > /dev/null +fi wait echo "" date_echo Validating traces finished diff --git a/src/dftracer/brahma/posix.h b/src/dftracer/brahma/posix.h index 802d03f..6130a3b 100644 --- a/src/dftracer/brahma/posix.h +++ b/src/dftracer/brahma/posix.h @@ -47,11 +47,12 @@ class POSIXDFTracer : public POSIX { return logger->hash_and_store(filename, METADATA_NAME_FILE_HASH); } else { const char *tracefile = is_traced_common(filename, func); - if (tracefile != nullptr) + if (tracefile != nullptr) { DFTRACER_LOG_DEBUG( "Calling POSIXDFTracer.is_traced with " "filename %s for %s trace %d", filename, func, tracefile != nullptr); + } return logger->hash_and_store(tracefile, METADATA_NAME_FILE_HASH); } } diff --git a/src/dftracer/core/dftracer_main.cpp b/src/dftracer/core/dftracer_main.cpp index 04f0ae5..72ce7de 100644 --- a/src/dftracer/core/dftracer_main.cpp +++ b/src/dftracer/core/dftracer_main.cpp @@ -89,7 +89,8 @@ void dftracer::DFTracerCore::log( } } -void dftracer::DFTracerCore::log_metadata(ConstEventNameType key, ConstEventNameType value) { +void dftracer::DFTracerCore::log_metadata(ConstEventNameType key, + ConstEventNameType value) { DFTRACER_LOG_DEBUG("DFTracerCore::log", ""); if (this->is_initialized && conf->enable) { if (logger != nullptr) { @@ -170,7 +171,8 @@ void dftracer::DFTracerCore::initialize(bool _bind, const char *_log_file, if (exec_cmd[index] == '\0') { if (!has_extracted) { strcpy(exec_name, basename(exec_cmd + last_index)); - if (exec_name[0] != '-' && strstr(exec_name, "python") == NULL && strstr(exec_name, "env") == NULL) { + if (exec_name[0] != '-' && strstr(exec_name, "python") == NULL && + strstr(exec_name, "env") == NULL) { has_extracted = true; DFTRACER_LOG_INFO("Extracted process_name %s", exec_name); } diff --git a/src/dftracer/core/dftracer_main.h b/src/dftracer/core/dftracer_main.h index 725e4cb..8fc7882 100644 --- a/src/dftracer/core/dftracer_main.h +++ b/src/dftracer/core/dftracer_main.h @@ -55,7 +55,7 @@ class DFTracerCore { void log(ConstEventNameType event_name, ConstEventNameType category, TimeResolution start_time, TimeResolution duration, std::unordered_map *metadata); - + void log_metadata(ConstEventNameType key, ConstEventNameType value); inline void enter_event() { logger->enter_event(); } diff --git a/src/dftracer/df_logger.h b/src/dftracer/df_logger.h index 09a666b..f851cd6 100644 --- a/src/dftracer/df_logger.h +++ b/src/dftracer/df_logger.h @@ -62,7 +62,7 @@ class DFTLogger { if (enable_core_affinity) { hwloc_cpuset_t set = hwloc_bitmap_alloc(); hwloc_get_cpubind(topology, set, HWLOC_CPUBIND_PROCESS); - for (unsigned id = hwloc_bitmap_first(set); id != -1; + for (int id = hwloc_bitmap_first(set); id != -1; id = hwloc_bitmap_next(set, id)) { cores.push_back(id); } @@ -321,10 +321,6 @@ class DFTLogger { if (dftracer_tid) { tid = df_gettid(); } - int local_index; - if (!include_metadata) { - local_index = index.load(); - } handle_mpi(tid); if (this->writer != nullptr) { this->writer->log_metadata(index_stack[level - 1], key, value, @@ -343,6 +339,26 @@ class DFTLogger { return hash_and_store_str(file, name); } + bool ignore_chars(char c) { + switch (c) { + case '(': + case ')': + case '\\': + case '"': + case '\'': + case '|': + return true; + default: + return false; + } + } + + void fix_str(char *str, size_t len) { + for (size_t i = 0; i < len && str[i] != '\0'; ++i) { + if (ignore_chars(str[i])) str[i] = ' '; + } + } + inline std::string hash_and_store_str(char file[PATH_MAX], ConstEventNameType name) { std::string hash = has_hash(file); @@ -354,6 +370,7 @@ class DFTLogger { if (dftracer_tid) { tid = df_gettid(); } + fix_str(file, PATH_MAX); int current_index = this->enter_event(); this->writer->log_metadata(current_index, file, hash.c_str(), name, this->process_id, tid); diff --git a/src/dftracer/dftracer.cpp b/src/dftracer/dftracer.cpp index 852e282..103327d 100644 --- a/src/dftracer/dftracer.cpp +++ b/src/dftracer/dftracer.cpp @@ -8,11 +8,11 @@ DFTracer::DFTracer(ConstEventNameType _name, ConstEventNameType _cat, int event_type) - : initialized(true), + : event_type(event_type), + initialized(true), name(_name), cat(_cat), - metadata(nullptr), - event_type(event_type) { + metadata(nullptr) { DFTRACER_LOG_DEBUG("DFTracer::DFTracer event %s cat %s ", _name, _cat); auto dftracer_core = DFTRACER_MAIN_SINGLETON(ProfilerStage::PROFILER_OTHER, ProfileType::PROFILER_CPP_APP); From 3a65307275b7f44b82b756775424cd7bf9dc9075 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Thu, 10 Oct 2024 21:04:27 -0700 Subject: [PATCH 12/15] release v1.0.6 (#220) --- CMakeLists.txt | 2 +- docs/conf.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e232c8..155b515 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ set(DFTRACER_PACKAGE_VERSION_MINOR "${DFTRACER_VERSION_PATCH}") set(DFTRACER_PACKAGE_STRING "${DFTRACER_PACKAGE_NAME} ${DFTRACER_PACKAGE_VERSION}") set(DFTRACER_PACKAGE_TARNAME "${DFTRACER_PACKAGE}") -set(DFTRACER_VERSION "(1, 0, 5)") +set(DFTRACER_VERSION "(1, 0, 6)") project(dftracer LANGUAGES C CXX) diff --git a/docs/conf.py b/docs/conf.py index 4824601..08aba8f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,7 +26,7 @@ # The short X.Y version version = u'0.0' # The full version, including alpha/beta/rc tags -release = u'1.0.5' +release = u'1.0.6' # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index 6274ab7..7fff7b0 100644 --- a/setup.py +++ b/setup.py @@ -158,7 +158,7 @@ def build_extension(self, ext: CMakeExtension) -> None: # logic and declaration, and simpler if you include description/version in a file. setup( name="pydftracer", - version="1.0.5", + version="1.0.6", description="I/O profiler for deep learning python apps. Specifically for dlio_benchmark.", long_description=long_description, long_description_content_type="text/markdown", From 478226949e03838f67c85166ac56cd792d9046fe Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Fri, 11 Oct 2024 02:36:06 -0700 Subject: [PATCH 13/15] Bugfix: fixed hash to be integers (#222) * fixed hash to be integers * optimize hash * missed type conversion for metadata. --- include/dftracer/core/constants.h | 1 + include/dftracer/core/typedef.h | 1 + src/dftracer/brahma/posix.h | 24 +++++----- src/dftracer/brahma/stdio.h | 18 ++++---- src/dftracer/df_logger.h | 63 ++++++++++++++------------- src/dftracer/writer/chrome_writer.cpp | 24 +++++----- src/dftracer/writer/chrome_writer.h | 4 +- 7 files changed, 74 insertions(+), 61 deletions(-) diff --git a/include/dftracer/core/constants.h b/include/dftracer/core/constants.h index d8dc893..b6b829e 100644 --- a/include/dftracer/core/constants.h +++ b/include/dftracer/core/constants.h @@ -29,5 +29,6 @@ static const int EVENT_TYPE_SIZE = 128; static const unsigned int DFT_PATH_MAX = 1024 * 4; static const char SEPARATOR = ';'; static const int HASH_OUTPUT = 16; +#define NO_HASH_DEFAULT 0 #endif // DFTRACER_CONSTANTS_H diff --git a/include/dftracer/core/typedef.h b/include/dftracer/core/typedef.h index 6a9f5e3..43b05c5 100644 --- a/include/dftracer/core/typedef.h +++ b/include/dftracer/core/typedef.h @@ -9,4 +9,5 @@ typedef unsigned long int ThreadID; typedef unsigned long int ProcessID; typedef char* EventNameType; typedef const char* ConstEventNameType; +typedef unsigned long long HashType; #endif // DFTRACER_TYPEDEF_H diff --git a/src/dftracer/brahma/posix.h b/src/dftracer/brahma/posix.h index 6130a3b..77e7fe1 100644 --- a/src/dftracer/brahma/posix.h +++ b/src/dftracer/brahma/posix.h @@ -6,7 +6,9 @@ #define DFTRACER_POSIX_H #include +#include #include +#include #include #include #include @@ -25,24 +27,24 @@ class POSIXDFTracer : public POSIX { static bool stop_trace; static std::shared_ptr instance; static const int MAX_FD = 1024; - std::string tracked_fd[MAX_FD]; + HashType tracked_fd[MAX_FD]; std::shared_ptr logger; bool trace_all_files; - inline std::string is_traced(int fd, const char *func) { - if (fd < 0) return std::string(); - std::string trace = tracked_fd[fd % MAX_FD]; - if (trace.empty()) { + inline HashType is_traced(int fd, const char *func) { + if (fd < 0) return NO_HASH_DEFAULT; + HashType trace = tracked_fd[fd % MAX_FD]; + if (trace == NO_HASH_DEFAULT) { DFTRACER_LOG_DEBUG( "Calling POSIXDFTracer.is_traced for %s and" " fd %d trace %d", - func, fd, !trace.empty()); + func, fd, trace != NO_HASH_DEFAULT); } return trace; } - inline std::string is_traced(const char *filename, const char *func) { - if (stop_trace) return std::string(); + inline HashType is_traced(const char *filename, const char *func) { + if (stop_trace) return NO_HASH_DEFAULT; if (trace_all_files) { return logger->hash_and_store(filename, METADATA_NAME_FILE_HASH); } else { @@ -57,7 +59,7 @@ class POSIXDFTracer : public POSIX { } } - inline void trace(int fd, std::string hash) { + inline void trace(int fd, HashType hash) { DFTRACER_LOG_DEBUG("Calling POSIXDFTracer.trace for %d and %d", fd, hash); if (fd == -1) return; tracked_fd[fd % MAX_FD] = hash; @@ -66,13 +68,13 @@ class POSIXDFTracer : public POSIX { inline void remove_trace(int fd) { DFTRACER_LOG_DEBUG("Calling POSIXDFTracer.remove_trace for %d", fd); if (fd == -1) return; - tracked_fd[fd % MAX_FD] = std::string(); + tracked_fd[fd % MAX_FD] = NO_HASH_DEFAULT; } public: POSIXDFTracer(bool trace_all) : POSIX(), trace_all_files(trace_all) { DFTRACER_LOG_DEBUG("POSIX class intercepted", ""); - for (int i = 0; i < MAX_FD; ++i) tracked_fd[i] = std::string(); + for (int i = 0; i < MAX_FD; ++i) tracked_fd[i] = NO_HASH_DEFAULT; logger = DFT_LOGGER_INIT(); } void finalize() { diff --git a/src/dftracer/brahma/stdio.h b/src/dftracer/brahma/stdio.h index dc1a43c..a6b7796 100644 --- a/src/dftracer/brahma/stdio.h +++ b/src/dftracer/brahma/stdio.h @@ -7,6 +7,8 @@ #include #include +#include +#include #include #include #include @@ -22,25 +24,25 @@ class STDIODFTracer : public STDIO { private: static bool stop_trace; static std::shared_ptr instance; - std::unordered_map tracked_fh; + std::unordered_map tracked_fh; std::shared_ptr logger; bool trace_all_files; - inline std::string is_traced(FILE *fh, const char *func) { + inline HashType is_traced(FILE *fh, const char *func) { DFTRACER_LOG_DEBUG("Calling STDIODFTracer.is_traced for %s", func); - if (stop_trace) return std::string(); - if (fh == NULL) return std::string(); + if (stop_trace) return NO_HASH_DEFAULT; + if (fh == NULL) return NO_HASH_DEFAULT; auto iter = tracked_fh.find(fh); if (iter != tracked_fh.end()) { return iter->second; } - return std::string(); + return NO_HASH_DEFAULT; } - inline std::string is_traced(const char *filename, const char *func) { + inline HashType is_traced(const char *filename, const char *func) { DFTRACER_LOG_DEBUG("Calling STDIODFTracer.is_traced with filename for %s", func); - if (stop_trace) return std::string(); + if (stop_trace) return NO_HASH_DEFAULT; if (trace_all_files) return logger->hash_and_store(filename, METADATA_NAME_FILE_HASH); else { @@ -49,7 +51,7 @@ class STDIODFTracer : public STDIO { } } - inline void trace(FILE *fh, std::string hash) { + inline void trace(FILE *fh, HashType hash) { DFTRACER_LOG_DEBUG("Calling STDIODFTracer.trace with hash %d", hash); tracked_fh.insert_or_assign(fh, hash); } diff --git a/src/dftracer/df_logger.h b/src/dftracer/df_logger.h index f851cd6..43556ae 100644 --- a/src/dftracer/df_logger.h +++ b/src/dftracer/df_logger.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include #include #include "core/enumeration.h" @@ -45,7 +47,7 @@ class DFTLogger { std::shared_ptr writer; uint32_t level; std::vector index_stack; - std::unordered_map computed_hash; + std::unordered_map computed_hash; std::atomic_int index; bool has_entry; #ifdef DFTRACER_MPI_ENABLE @@ -104,15 +106,15 @@ class DFTLogger { } ~DFTLogger() {} - inline std::string get_hash(char *name) { + inline HashType get_hash(char *name) { uint8_t result[HASH_OUTPUT]; md5String(name, result); - std::string hash; - hash.reserve(HASH_OUTPUT + 1); + char hash_str[HASH_OUTPUT + 1]; for (int i = 0; i < HASH_OUTPUT; ++i) { - sprintf(hash.data() + i, "%02x", result[i]); + sprintf(hash_str + i, "%02x", result[i]); } - hash.data()[HASH_OUTPUT] = '\0'; + hash_str[HASH_OUTPUT] = '\0'; + HashType hash = std::stoull(hash_str, nullptr, 16); return hash; } @@ -125,15 +127,15 @@ class DFTLogger { tid = df_gettid(); } this->writer = dftracer::Singleton::get_instance(); - std::string hostname_hash; - std::string cmd_hash; - std::string exec_hash; + HashType hostname_hash; + HashType cmd_hash; + HashType exec_hash; if (this->writer != nullptr) { char hostname[256]; gethostname(hostname, 256); hostname_hash = get_hash(hostname); this->writer->initialize(log_file.data(), this->throw_error, - hostname_hash.c_str()); + hostname_hash); hostname_hash = hash_and_store(hostname, METADATA_NAME_HOSTNAME_HASH); char thread_name[128]; auto size = sprintf(thread_name, "%lu", this->process_id); @@ -232,16 +234,16 @@ class DFTLogger { return -1; } - inline std::string has_hash(ConstEventNameType key) { + inline HashType has_hash(ConstEventNameType key) { std::shared_lock lock(map_mtx); auto iter = computed_hash.find(key); - if (iter != computed_hash.end()) return iter->second.c_str(); - return std::string(); + if (iter != computed_hash.end()) return iter->second; + return NO_HASH_DEFAULT; } - inline void insert_hash(ConstEventNameType key, std::string hash) { + inline void insert_hash(ConstEventNameType key, HashType hash) { std::unique_lock lock(map_mtx); - computed_hash.insert_or_assign(key, hash.c_str()); + computed_hash.insert_or_assign(key, hash); } inline TimeResolution get_time() { @@ -331,8 +333,8 @@ class DFTLogger { } } - inline std::string hash_and_store(char *filename, ConstEventNameType name) { - if (filename == NULL) return std::string(); + inline HashType hash_and_store(char *filename, ConstEventNameType name) { + if (filename == NULL) return NO_HASH_DEFAULT; char file[PATH_MAX]; strcpy(file, filename); file[PATH_MAX - 1] = '\0'; @@ -359,12 +361,12 @@ class DFTLogger { } } - inline std::string hash_and_store_str(char file[PATH_MAX], - ConstEventNameType name) { - std::string hash = has_hash(file); - if (hash.empty()) { + inline HashType hash_and_store_str(char file[PATH_MAX], + ConstEventNameType name) { + HashType hash = has_hash(file); + if (hash == NO_HASH_DEFAULT) { hash = get_hash(file); - insert_hash(file, hash.c_str()); + insert_hash(file, hash); if (this->writer != nullptr) { ThreadID tid = 0; if (dftracer_tid) { @@ -372,17 +374,18 @@ class DFTLogger { } fix_str(file, PATH_MAX); int current_index = this->enter_event(); - this->writer->log_metadata(current_index, file, hash.c_str(), name, - this->process_id, tid); + this->writer->log_metadata(current_index, file, + std::to_string(hash).c_str(), name, + this->process_id, tid, false); this->exit_event(); } } return hash; } - inline std::string hash_and_store(const char *filename, - ConstEventNameType name) { - if (filename == NULL) return std::string(); + inline HashType hash_and_store(const char *filename, + ConstEventNameType name) { + if (filename == NULL) return NO_HASH_DEFAULT; char file[PATH_MAX]; strcpy(file, filename); file[PATH_MAX - 1] = '\0'; @@ -416,15 +419,15 @@ class DFTLogger { #define DFT_LOGGER_UPDATE_HASH(value) \ if (trace && this->logger->include_metadata) { \ - std::string value##_hash = \ + HashType value##_hash = \ this->logger->hash_and_store(value, METADATA_NAME_FILE_HASH); \ DFT_LOGGER_UPDATE(value##_hash); \ } #define DFT_LOGGER_START(entity) \ DFTRACER_LOG_DEBUG("Calling function %s", __FUNCTION__); \ - std::string fhash = is_traced(entity, __FUNCTION__); \ - bool trace = !fhash.empty(); \ + HashType fhash = is_traced(entity, __FUNCTION__); \ + bool trace = fhash != NO_HASH_DEFAULT; \ TimeResolution start_time = 0; \ std::unordered_map *metadata = nullptr; \ if (trace) { \ diff --git a/src/dftracer/writer/chrome_writer.cpp b/src/dftracer/writer/chrome_writer.cpp index 9adb7b8..790a529 100644 --- a/src/dftracer/writer/chrome_writer.cpp +++ b/src/dftracer/writer/chrome_writer.cpp @@ -22,7 +22,7 @@ template <> bool dftracer::Singleton::stop_creating_instances = false; void dftracer::ChromeWriter::initialize(char *filename, bool throw_error, - const char *hostname_hash) { + HashType hostname_hash) { this->hostname_hash = hostname_hash; this->throw_error = throw_error; this->filename = filename; @@ -185,6 +185,11 @@ void dftracer::ChromeWriter::convert_json( << "\":" << std::any_cast(item.second) << ""; if (i < meta_size - 1) meta_stream << ","; + } else if (item.second.type() == typeid(HashType)) { + meta_stream << "\"" << item.first + << "\":" << std::any_cast(item.second) << ""; + if (i < meta_size - 1) meta_stream << ","; + } else if (item.second.type() == typeid(long)) { meta_stream << "\"" << item.first << "\":" << std::any_cast(item.second) << ""; @@ -214,10 +219,9 @@ void dftracer::ChromeWriter::convert_json( previous_index = current_index; auto written_size = sprintf( buffer.data() + current_index, - R"(%s{"id":%d,"name":"%s","cat":"%s","pid":%lu,"tid":%lu,"ts":%llu,"dur":%llu,"ph":"X","args":{"hhash":"%s"%s}})", + R"(%s{"id":%d,"name":"%s","cat":"%s","pid":%lu,"tid":%lu,"ts":%llu,"dur":%llu,"ph":"X","args":{"hhash":%llu%s}})", is_first_char, index, event_name, category, process_id, thread_id, - start_time, duration, this->hostname_hash.c_str(), - all_stream.str().c_str()); + start_time, duration, this->hostname_hash, all_stream.str().c_str()); current_index += written_size; buffer[current_index] = '\n'; current_index++; @@ -256,15 +260,15 @@ void dftracer::ChromeWriter::convert_json_metadata( if (is_string) { written_size = sprintf( buffer.data() + current_index, - R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"hhash":"%s","name":"%s","value":"%s"}})", - is_first_char, index, ph, process_id, thread_id, - this->hostname_hash.c_str(), name, value); + R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"hhash":%llu,"name":"%s","value":"%s"}})", + is_first_char, index, ph, process_id, thread_id, this->hostname_hash, + name, value); } else { written_size = sprintf( buffer.data() + current_index, - R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"hhash":"%s","name":"%s","value":%s}})", - is_first_char, index, ph, process_id, thread_id, - this->hostname_hash.c_str(), name, value); + R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"hhash":%llu,"name":"%s","value":%s}})", + is_first_char, index, ph, process_id, thread_id, this->hostname_hash, + name, value); } current_index += written_size; buffer[current_index] = '\n'; diff --git a/src/dftracer/writer/chrome_writer.h b/src/dftracer/writer/chrome_writer.h index 1990ccb..caabe8e 100644 --- a/src/dftracer/writer/chrome_writer.h +++ b/src/dftracer/writer/chrome_writer.h @@ -36,7 +36,7 @@ class ChromeWriter { bool enable_core_affinity; FILE *fh; - std::string hostname_hash; + HashType hostname_hash; static const int MAX_LINE_SIZE = 16 * 1024L; size_t write_buffer_size; @@ -100,7 +100,7 @@ class ChromeWriter { } } ~ChromeWriter() { DFTRACER_LOG_DEBUG("Destructing ChromeWriter", ""); } - void initialize(char *filename, bool throw_error, const char *hostname_hash); + void initialize(char *filename, bool throw_error, HashType hostname_hash); void log(int index, ConstEventNameType event_name, ConstEventNameType category, TimeResolution start_time, From 0773e2c8aee557b0ae4e53ad1b811a30370c8be3 Mon Sep 17 00:00:00 2001 From: Izzet Yildirim Date: Fri, 11 Oct 2024 15:38:47 -0500 Subject: [PATCH 14/15] Repository: Update README and CITATION (#223) * Enhance README.md * Remove coverage status from README.md for now * Delete unused .trace-0-of-2.pfw file * Add seaborn to DFAnalyzer requirements * Create CITATION.cff * Fix CITATION.cff * Update CITATION.cff * Update ci.yml --- .github/workflows/ci.yml | 4 +- .trace-0-of-2.pfw | 705 --------------------------------------- CITATION.cff | 89 +++++ README.md | 105 +++--- 4 files changed, 151 insertions(+), 752 deletions(-) delete mode 100644 .trace-0-of-2.pfw create mode 100644 CITATION.cff diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65436a6..d5264c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,8 @@ -name: DFTracer Build and Test +name: Build and Test on: pull_request: - branches: [ main, develop ] + branches: [main, develop] push: jobs: build-and-test: diff --git a/.trace-0-of-2.pfw b/.trace-0-of-2.pfw deleted file mode 100644 index ab9f75d..0000000 --- a/.trace-0-of-2.pfw +++ /dev/null @@ -1,705 +0,0 @@ -[ -{"name":"TFFramework.__init__","cat":"ai_framework","pid":33128,"tid":81567,"ts":108849,"dur":15,"ph":"X","args":{}} -{"name":"TFFramework.is_nativeio_available","cat":"ai_framework","pid":33128,"tid":81567,"ts":180755,"dur":9,"ph":"X","args":{}} -{"name":"FileStorage.__init__","cat":"storage","pid":33128,"tid":81567,"ts":294,"dur":180601,"ph":"X","args":{}} -{"name":"FileStorage.create_namespace","cat":"storage","pid":33128,"tid":81567,"ts":181075,"dur":65,"ph":"X","args":{}} -{"name":"TFFramework.is_nativeio_available","cat":"ai_framework","pid":33128,"tid":81567,"ts":307866,"dur":2,"ph":"X","args":{}} -{"name":"FileStorage.__init__","cat":"storage","pid":33128,"tid":81567,"ts":307842,"dur":95,"ph":"X","args":{}} -{"name":"DLIOBenchmark.__init__","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":170,"dur":307888,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":309597,"dur":10,"ph":"X","args":{}} -{"name":"FileStorage.create_node","cat":"storage","pid":33128,"tid":81567,"ts":309588,"dur":1547,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":311238,"dur":12,"ph":"X","args":{}} -{"name":"FileStorage.create_node","cat":"storage","pid":33128,"tid":81567,"ts":311227,"dur":908,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":312247,"dur":11,"ph":"X","args":{}} -{"name":"FileStorage.create_node","cat":"storage","pid":33128,"tid":81567,"ts":312222,"dur":1032,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":318717,"dur":12,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":322956,"dur":2532,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":318447,"dur":7,"ph":"X","args":{"step":1,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":318456,"dur":9026,"ph":"X","args":{"step":1,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":327906,"dur":12,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":328092,"dur":1894,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":327638,"dur":3,"ph":"X","args":{"step":2,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":327644,"dur":4467,"ph":"X","args":{"step":2,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":332570,"dur":13,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":332743,"dur":1957,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":332263,"dur":3,"ph":"X","args":{"step":3,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":332268,"dur":3948,"ph":"X","args":{"step":3,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":336667,"dur":13,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":336804,"dur":1884,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":336367,"dur":3,"ph":"X","args":{"step":4,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":336372,"dur":3930,"ph":"X","args":{"step":4,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":340789,"dur":13,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":340930,"dur":2336,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":340485,"dur":3,"ph":"X","args":{"step":5,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":340490,"dur":4431,"ph":"X","args":{"step":5,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":345383,"dur":13,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":345543,"dur":84544,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":345087,"dur":4,"ph":"X","args":{"step":6,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":345093,"dur":87084,"ph":"X","args":{"step":6,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":432684,"dur":13,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":432863,"dur":2600,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":432371,"dur":5,"ph":"X","args":{"step":7,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":432378,"dur":4739,"ph":"X","args":{"step":7,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":437608,"dur":14,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":437767,"dur":2431,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":437273,"dur":4,"ph":"X","args":{"step":8,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":437279,"dur":4716,"ph":"X","args":{"step":8,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":442475,"dur":14,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":442623,"dur":2477,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":442176,"dur":3,"ph":"X","args":{"step":9,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":442181,"dur":4639,"ph":"X","args":{"step":9,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":447276,"dur":13,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":447421,"dur":2582,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":446956,"dur":4,"ph":"X","args":{"step":10,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":446962,"dur":5019,"ph":"X","args":{"step":10,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":452412,"dur":13,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":452546,"dur":2510,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":452131,"dur":3,"ph":"X","args":{"step":11,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":452137,"dur":4737,"ph":"X","args":{"step":11,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":457315,"dur":30,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":457467,"dur":2435,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":457001,"dur":3,"ph":"X","args":{"step":12,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":457007,"dur":4654,"ph":"X","args":{"step":12,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":462075,"dur":13,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":462210,"dur":2004,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":461784,"dur":3,"ph":"X","args":{"step":13,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":461790,"dur":4201,"ph":"X","args":{"step":13,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":466420,"dur":14,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":466565,"dur":2635,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":466121,"dur":3,"ph":"X","args":{"step":14,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":466127,"dur":4821,"ph":"X","args":{"step":14,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":471399,"dur":14,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":471537,"dur":2365,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":471089,"dur":3,"ph":"X","args":{"step":15,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":471095,"dur":4386,"ph":"X","args":{"step":15,}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":475924,"dur":14,"ph":"X","args":{}} -{"name":"fopen","cat":"STDIO","pid":33128,"tid":81567,"ts":476076,"dur":2185,"ph":"X","args":{}} -{"name":"generate.iter","cat":"generator","pid":33128,"tid":81567,"ts":475609,"dur":3,"ph":"X","args":{"step":16,}} -{"name":"generate","cat":"generator","pid":33128,"tid":81567,"ts":475614,"dur":4334,"ph":"X","args":{"step":16,}} -{"name":"TFRecordGenerator.generate","cat":"generator","pid":33128,"tid":81567,"ts":309574,"dur":170535,"ph":"X","args":{"step":17,}} -{"name":"TFFramework.start_framework_profiler","cat":"ai_framework","pid":33128,"tid":81567,"ts":558232,"dur":15,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":559104,"dur":12,"ph":"X","args":{}} -{"name":"FileStorage.walk_node","cat":"storage","pid":33128,"tid":81567,"ts":559072,"dur":3140,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":562435,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_node","cat":"storage","pid":33128,"tid":81567,"ts":562415,"dur":854,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":563483,"dur":11,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":563623,"dur":9,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":563730,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":563874,"dur":9,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":563973,"dur":9,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":564086,"dur":9,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":564186,"dur":9,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":564277,"dur":9,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":564394,"dur":9,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":564478,"dur":26,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":564619,"dur":25,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":564723,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":564824,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":564915,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":565010,"dur":9,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":565146,"dur":9,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":565254,"dur":7,"ph":"X","args":{}} -{"name":"FileStorage.walk_node","cat":"storage","pid":33128,"tid":81567,"ts":565237,"dur":1587,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":566992,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_node","cat":"storage","pid":33128,"tid":81567,"ts":566982,"dur":196,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":567294,"dur":10,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":567435,"dur":9,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":567528,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":567616,"dur":9,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":567701,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":567801,"dur":24,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":567932,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":568011,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":568192,"dur":11,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":568295,"dur":9,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":568397,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":568474,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":568567,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":568650,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":568731,"dur":8,"ph":"X","args":{}} -{"name":"FileStorage.get_uri","cat":"storage","pid":33128,"tid":81567,"ts":568813,"dur":8,"ph":"X","args":{}} -{"name":"ConfigArguments.derive_configurations","cat":"config","pid":33128,"tid":81567,"ts":568893,"dur":3527,"ph":"X","args":{}} -{"name":"ConfigArguments.validate","cat":"config","pid":33128,"tid":81567,"ts":572499,"dur":36,"ph":"X","args":{}} -{"name":"DLIOBenchmark.initialize","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":308139,"dur":264468,"ph":"X","args":{}} -{"name":"TFDataLoader.__init__","cat":"data_loader","pid":33128,"tid":81567,"ts":649097,"dur":18,"ph":"X","args":{"epoch":1,}} -{"name":"TFDataLoader.__init__","cat":"data_loader","pid":33128,"tid":81567,"ts":649208,"dur":9,"ph":"X","args":{"epoch":1,}} -{"name":"TFFramework.is_nativeio_available","cat":"ai_framework","pid":33128,"tid":81567,"ts":649336,"dur":2,"ph":"X","args":{}} -{"name":"FileStorage.__init__","cat":"storage","pid":33128,"tid":81567,"ts":649290,"dur":102,"ph":"X","args":{}} -{"name":"TFFramework.init_loader","cat":"ai_framework","pid":33128,"tid":81567,"ts":573276,"dur":76183,"ph":"X","args":{}} -{"name":"ConfigArguments.build_sample_map","cat":"config","pid":33128,"tid":81567,"ts":649548,"dur":175,"ph":"X","args":{}} -{"name":"ConfigArguments.reconfigure","cat":"config","pid":33128,"tid":81567,"ts":649527,"dur":259,"ph":"X","args":{}} -{"name":"TFFramework.get_loader","cat":"ai_framework","pid":33128,"tid":81567,"ts":650147,"dur":10,"ph":"X","args":{}} -{"name":"fread","cat":"STDIO","pid":33128,"tid":81567,"ts":659302,"dur":17,"ph":"X","args":{}} -{"name":"fclose","cat":"STDIO","pid":33128,"tid":81567,"ts":659865,"dur":11,"ph":"X","args":{}} -{"name":"fclose","cat":"STDIO","pid":33128,"tid":81567,"ts":682671,"dur":16,"ph":"X","args":{}} -{"name":"fclose","cat":"STDIO","pid":33128,"tid":81567,"ts":767653,"dur":17,"ph":"X","args":{}} -{"name":"TensorflowDataset.__new__","cat":"data_loader","pid":33128,"tid":81567,"ts":1021989,"dur":42346,"ph":"X","args":{"epoch":1,}} -{"name":"TFDataLoader.read","cat":"data_loader","pid":33128,"tid":81567,"ts":650229,"dur":423897,"ph":"X","args":{"epoch":1,}} -{"name":"TFDataLoader.next","cat":"data_loader","pid":33128,"tid":81567,"ts":1074228,"dur":5,"ph":"X","args":{"epoch":1,}} -{"name":"TensorflowDataset._generator","cat":"data_loader","pid":33128,"tid":36424,"ts":1138555,"dur":21,"ph":"X","args":{"epoch":1,}} -{"name":"TensorflowDataset._generator","cat":"data_loader","pid":33128,"tid":91361,"ts":1141102,"dur":24,"ph":"X","args":{"epoch":1,}} -{"name":"TensorflowDataset._generator","cat":"data_loader","pid":33128,"tid":17643,"ts":1141944,"dur":17,"ph":"X","args":{"epoch":1,}} -{"name":"TensorflowDataset._generator","cat":"data_loader","pid":33128,"tid":6016,"ts":1142231,"dur":14,"ph":"X","args":{"epoch":1,}} -{"name":"TFReader.__init__","cat":"reader","pid":33128,"tid":36424,"ts":1317507,"dur":34,"ph":"X","args":{"epoch":1,}} -{"name":"TFReader.next","cat":"reader","pid":33128,"tid":36424,"ts":1317615,"dur":5,"ph":"X","args":{"step":1,"image_idx":0,"epoch":1,}} -{"name":"TFReader.__init__","cat":"reader","pid":33128,"tid":91361,"ts":1319901,"dur":33,"ph":"X","args":{"step":1,"image_idx":0,"epoch":1,}} -{"name":"TFReader.next","cat":"reader","pid":33128,"tid":91361,"ts":1320010,"dur":5,"ph":"X","args":{"step":1,"image_idx":0,"epoch":1,}} -{"name":"TFReader.__init__","cat":"reader","pid":33128,"tid":17643,"ts":1320773,"dur":31,"ph":"X","args":{"step":1,"image_idx":0,"epoch":1,}} -{"name":"TFReader.next","cat":"reader","pid":33128,"tid":17643,"ts":1320887,"dur":6,"ph":"X","args":{"step":1,"image_idx":0,"epoch":1,}} -{"name":"TFReader.__init__","cat":"reader","pid":33128,"tid":6016,"ts":1321536,"dur":31,"ph":"X","args":{"step":1,"image_idx":0,"epoch":1,}} -{"name":"TFReader.next","cat":"reader","pid":33128,"tid":6016,"ts":1321649,"dur":5,"ph":"X","args":{"step":1,"image_idx":0,"epoch":1,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":52001,"ts":1424934,"dur":733,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1425766,"dur":32,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1425853,"dur":18,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":50024,"ts":1426255,"dur":250,"ph":"X","args":{"ret":13,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":18359,"ts":1425187,"dur":1317,"ph":"X","args":{"ret":12,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":13092,"ts":1426374,"dur":133,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1426584,"dur":32,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1426651,"dur":28,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1426711,"dur":41,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1426760,"dur":18,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1426826,"dur":18,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1426891,"dur":19,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":56712,"ts":1430243,"dur":6531,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":9505,"ts":1428050,"dur":8941,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":50024,"ts":1437579,"dur":40,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":78042,"ts":1429152,"dur":8586,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":50024,"ts":1437838,"dur":703,"ph":"X","args":{"ret":13,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":13092,"ts":1438583,"dur":24,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1438611,"dur":33,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1438709,"dur":18,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":13092,"ts":1438824,"dur":17,"ph":"X","args":{"ret":15,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":18359,"ts":1438834,"dur":17,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":50024,"ts":1438849,"dur":20,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1438905,"dur":30,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1439048,"dur":20,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":18359,"ts":1439053,"dur":26,"ph":"X","args":{"ret":12,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":13092,"ts":1439157,"dur":13,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":80936,"ts":1426201,"dur":12968,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1439174,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1439320,"dur":17,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":50024,"ts":1439122,"dur":650,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1439842,"dur":33,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1439916,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":18359,"ts":1439399,"dur":587,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":52001,"ts":1439827,"dur":592,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":18359,"ts":1440221,"dur":237,"ph":"X","args":{"ret":10,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":13092,"ts":1439307,"dur":1151,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1440546,"dur":25,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1440607,"dur":29,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1440639,"dur":16,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1440682,"dur":17,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":52001,"ts":1440603,"dur":807,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1441479,"dur":26,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1441563,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":52001,"ts":1441636,"dur":640,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":52001,"ts":1442556,"dur":12,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1442611,"dur":25,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1442671,"dur":14,"ph":"X","args":{}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":1446357,"dur":27,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":20858,"ts":1440483,"dur":15496,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":50024,"ts":1456526,"dur":21,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":21581,"ts":1443315,"dur":13345,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":52001,"ts":1457161,"dur":17,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":50024,"ts":1456707,"dur":567,"ph":"X","args":{"ret":13,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":52001,"ts":1457320,"dur":13,"ph":"X","args":{"ret":12,}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":60202,"ts":1441837,"dur":15501,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1457340,"dur":31,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1457387,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1457496,"dur":17,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1457538,"dur":18,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":50024,"ts":1457615,"dur":16,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":18359,"ts":1457791,"dur":29,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":52001,"ts":1457632,"dur":434,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":50024,"ts":1457836,"dur":678,"ph":"X","args":{"ret":10,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":18359,"ts":1457975,"dur":546,"ph":"X","args":{"ret":12,}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":78042,"ts":1445230,"dur":13290,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1458580,"dur":24,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1458635,"dur":26,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1458724,"dur":16,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1458778,"dur":18,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":18359,"ts":1458866,"dur":335,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":52001,"ts":1458280,"dur":965,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1459363,"dur":23,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":13092,"ts":1459024,"dur":365,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":18359,"ts":1459393,"dur":15,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1459434,"dur":16,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1459548,"dur":24,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1459624,"dur":15,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":13092,"ts":1459677,"dur":543,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1460287,"dur":26,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1460354,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":13092,"ts":1460417,"dur":680,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":13092,"ts":1461266,"dur":14,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1461333,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1461415,"dur":15,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":21581,"ts":1462198,"dur":21413,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":52001,"ts":1484019,"dur":55,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":52001,"ts":1484229,"dur":14,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1484288,"dur":30,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1484394,"dur":33,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":52001,"ts":1484526,"dur":14,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":52001,"ts":1484712,"dur":844,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1485637,"dur":39,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1485717,"dur":23,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":60202,"ts":1460809,"dur":29904,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":18359,"ts":1491060,"dur":17,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":18359,"ts":1491225,"dur":13,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1491284,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1491349,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":18359,"ts":1491416,"dur":12,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":18359,"ts":1491633,"dur":24,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1491697,"dur":20,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1491751,"dur":13,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":21581,"ts":1489433,"dur":15368,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":52001,"ts":1505425,"dur":52,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":9505,"ts":1459415,"dur":46204,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":52001,"ts":1505665,"dur":14,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1505755,"dur":31,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1505825,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":52001,"ts":1505886,"dur":20,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":50024,"ts":1505907,"dur":17,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":50024,"ts":1506200,"dur":693,"ph":"X","args":{"ret":13,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":52001,"ts":1506097,"dur":800,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1506961,"dur":30,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1507020,"dur":38,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1507064,"dur":18,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1507104,"dur":16,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":50024,"ts":1507160,"dur":540,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":50024,"ts":1507865,"dur":14,"ph":"X","args":{"ret":16,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1507931,"dur":28,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1507999,"dur":16,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":78042,"ts":1466091,"dur":42870,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":13092,"ts":1509276,"dur":613,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":13092,"ts":1510173,"dur":634,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1510920,"dur":58,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1511019,"dur":24,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":13092,"ts":1511102,"dur":703,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":13092,"ts":1512065,"dur":15,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1512139,"dur":28,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1512215,"dur":15,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":9505,"ts":1509372,"dur":5578,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":50024,"ts":1515456,"dur":22,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":21581,"ts":1510613,"dur":4963,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":50024,"ts":1515627,"dur":549,"ph":"X","args":{"ret":16,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1516233,"dur":48,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1516323,"dur":15,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":56712,"ts":1496573,"dur":19774,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":52001,"ts":1516101,"dur":651,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":18359,"ts":1516852,"dur":29,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":50024,"ts":1516392,"dur":520,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":52001,"ts":1516955,"dur":15,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1517058,"dur":30,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1517129,"dur":16,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":52001,"ts":1517194,"dur":15,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":18359,"ts":1517050,"dur":676,"ph":"X","args":{"ret":12,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":50024,"ts":1517148,"dur":598,"ph":"X","args":{"ret":13,}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":9716,"ts":1513700,"dur":4091,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1517799,"dur":41,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1517869,"dur":37,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":52001,"ts":1517394,"dur":597,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1517984,"dur":17,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1518147,"dur":24,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1518024,"dur":22,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1518218,"dur":16,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":18359,"ts":1518193,"dur":619,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":13092,"ts":1518294,"dur":572,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":18359,"ts":1518975,"dur":14,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1519044,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1519111,"dur":14,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":13092,"ts":1519022,"dur":599,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1519683,"dur":25,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1519748,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":13092,"ts":1519811,"dur":494,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":13092,"ts":1520549,"dur":28,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1520619,"dur":21,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1520675,"dur":13,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":9505,"ts":1523471,"dur":5374,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":21581,"ts":1525568,"dur":4203,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":60202,"ts":1526880,"dur":3452,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":9716,"ts":1524522,"dur":6331,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":1446530,"dur":135628,"ph":"X","args":{}} -{"name":"_train.iter","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":1078299,"dur":367950,"ph":"X","args":{"step":1,}} -{"name":"_train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":1446257,"dur":136391,"ph":"X","args":{"step":1,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":13092,"ts":1583457,"dur":18,"ph":"X","args":{}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":1583448,"dur":13,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":13092,"ts":1583689,"dur":19,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1583764,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1583829,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":13092,"ts":1583891,"dur":13,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":13092,"ts":1584149,"dur":14,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1584207,"dur":35,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1584277,"dur":14,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":78042,"ts":1586707,"dur":1978,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":1583633,"dur":56629,"ph":"X","args":{}} -{"name":"_train.iter","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":1582754,"dur":387,"ph":"X","args":{"step":2,}} -{"name":"_train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":1583148,"dur":58211,"ph":"X","args":{"step":2,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":50024,"ts":1642174,"dur":16,"ph":"X","args":{}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":1642187,"dur":12,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":50024,"ts":1642395,"dur":13,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1642469,"dur":25,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1642546,"dur":14,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":50024,"ts":1642603,"dur":10,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":50024,"ts":1642833,"dur":11,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1642882,"dur":36,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1642952,"dur":13,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":9505,"ts":1645344,"dur":1599,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":1642328,"dur":56843,"ph":"X","args":{}} -{"name":"_train.iter","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":1641535,"dur":358,"ph":"X","args":{"step":3,}} -{"name":"_train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":1641899,"dur":57671,"ph":"X","args":{"step":3,}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":1700363,"dur":12,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":18359,"ts":1700366,"dur":707,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":18359,"ts":1701287,"dur":14,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1701376,"dur":25,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1701448,"dur":14,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":18359,"ts":1701504,"dur":740,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":18359,"ts":1702487,"dur":14,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1702552,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1702616,"dur":15,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":56712,"ts":1703148,"dur":2635,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":1700519,"dur":57348,"ph":"X","args":{}} -{"name":"_train.iter","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":1699674,"dur":390,"ph":"X","args":{"step":4,}} -{"name":"_train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":1700071,"dur":59574,"ph":"X","args":{"step":4,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":52001,"ts":1760526,"dur":22,"ph":"X","args":{}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":1760522,"dur":13,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":52001,"ts":1760801,"dur":17,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1760881,"dur":46,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1760968,"dur":17,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":52001,"ts":1761042,"dur":17,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":52001,"ts":1761196,"dur":814,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1762086,"dur":42,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":52001,"ts":1762201,"dur":16,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":21581,"ts":1763900,"dur":2068,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":1760707,"dur":57539,"ph":"X","args":{}} -{"name":"_train.iter","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":1759748,"dur":442,"ph":"X","args":{"step":5,}} -{"name":"_train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":1760197,"dur":60697,"ph":"X","args":{"step":5,}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":1821601,"dur":11,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":13092,"ts":1822581,"dur":16,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":13092,"ts":1822795,"dur":13,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1822867,"dur":25,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1822929,"dur":14,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":13092,"ts":1822994,"dur":10,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":13092,"ts":1823223,"dur":12,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1823275,"dur":22,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":13092,"ts":1823331,"dur":13,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":78042,"ts":1824079,"dur":2682,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":1821691,"dur":57118,"ph":"X","args":{}} -{"name":"_train.iter","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":1821114,"dur":379,"ph":"X","args":{"step":6,}} -{"name":"_train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":1821500,"dur":57712,"ph":"X","args":{"step":6,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":50024,"ts":1880063,"dur":16,"ph":"X","args":{}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":1880058,"dur":13,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":50024,"ts":1880287,"dur":13,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1880361,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1880427,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":50024,"ts":1880487,"dur":12,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":50024,"ts":1880725,"dur":28,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1880795,"dur":40,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":50024,"ts":1880872,"dur":14,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":9505,"ts":1883463,"dur":1768,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":1880217,"dur":57438,"ph":"X","args":{}} -{"name":"_train.iter","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":1879337,"dur":393,"ph":"X","args":{"step":7,}} -{"name":"_train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":1879736,"dur":59854,"ph":"X","args":{"step":7,}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":1940482,"dur":14,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":18359,"ts":1940487,"dur":764,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":18359,"ts":1941436,"dur":14,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1941521,"dur":28,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1941587,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":18359,"ts":1941659,"dur":14,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":18359,"ts":1941846,"dur":12,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1941904,"dur":24,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":18359,"ts":1941966,"dur":14,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":56712,"ts":1943538,"dur":2097,"ph":"X","args":{"epoch":1,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":1940580,"dur":56628,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":6016,"ts":2003274,"dur":30,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":6016,"ts":2004466,"dur":16,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":91361,"ts":2005602,"dur":19,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":6016,"ts":2006747,"dur":732,"ph":"X","args":{}} -{"name":"DLIOBenchmark._train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":649513,"dur":1361425,"ph":"X","args":{"step":8,}} -{"name":"TFFramework.get_loader","cat":"ai_framework","pid":33128,"tid":81567,"ts":2011899,"dur":8,"ph":"X","args":{}} -{"name":"TFDataLoader.finalize","cat":"data_loader","pid":33128,"tid":81567,"ts":2011994,"dur":3,"ph":"X","args":{"epoch":1,}} -{"name":"TFDataLoader.__init__","cat":"data_loader","pid":33128,"tid":81567,"ts":2012468,"dur":12,"ph":"X","args":{"epoch":2,}} -{"name":"TFDataLoader.__init__","cat":"data_loader","pid":33128,"tid":81567,"ts":2012560,"dur":9,"ph":"X","args":{"epoch":2,}} -{"name":"TFFramework.is_nativeio_available","cat":"ai_framework","pid":33128,"tid":81567,"ts":2012678,"dur":3,"ph":"X","args":{}} -{"name":"FileStorage.__init__","cat":"storage","pid":33128,"tid":81567,"ts":2012646,"dur":92,"ph":"X","args":{}} -{"name":"TFFramework.init_loader","cat":"ai_framework","pid":33128,"tid":81567,"ts":2012418,"dur":367,"ph":"X","args":{}} -{"name":"ConfigArguments.build_sample_map","cat":"config","pid":33128,"tid":81567,"ts":2012884,"dur":87,"ph":"X","args":{}} -{"name":"ConfigArguments.reconfigure","cat":"config","pid":33128,"tid":81567,"ts":2012864,"dur":232,"ph":"X","args":{}} -{"name":"TFFramework.get_loader","cat":"ai_framework","pid":33128,"tid":81567,"ts":2013545,"dur":7,"ph":"X","args":{}} -{"name":"TensorflowDataset.__new__","cat":"data_loader","pid":33128,"tid":81567,"ts":2018698,"dur":40121,"ph":"X","args":{"epoch":2,}} -{"name":"TFDataLoader.read","cat":"data_loader","pid":33128,"tid":81567,"ts":2013616,"dur":53016,"ph":"X","args":{"epoch":2,}} -{"name":"TFDataLoader.next","cat":"data_loader","pid":33128,"tid":81567,"ts":2066729,"dur":4,"ph":"X","args":{"epoch":2,}} -{"name":"TensorflowDataset._generator","cat":"data_loader","pid":33128,"tid":58970,"ts":2122497,"dur":15,"ph":"X","args":{"epoch":2,}} -{"name":"TFReader.__init__","cat":"reader","pid":33128,"tid":58970,"ts":2122686,"dur":45,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFReader.next","cat":"reader","pid":33128,"tid":58970,"ts":2122824,"dur":5,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TensorflowDataset._generator","cat":"data_loader","pid":33128,"tid":19440,"ts":2124294,"dur":30,"ph":"X","args":{"epoch":2,}} -{"name":"TFReader.__init__","cat":"reader","pid":33128,"tid":19440,"ts":2124480,"dur":23,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFReader.next","cat":"reader","pid":33128,"tid":19440,"ts":2124586,"dur":5,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TensorflowDataset._generator","cat":"data_loader","pid":33128,"tid":47953,"ts":2125795,"dur":23,"ph":"X","args":{"epoch":2,}} -{"name":"TFReader.__init__","cat":"reader","pid":33128,"tid":47953,"ts":2126012,"dur":42,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFReader.next","cat":"reader","pid":33128,"tid":47953,"ts":2126122,"dur":5,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TensorflowDataset._generator","cat":"data_loader","pid":33128,"tid":92626,"ts":2126760,"dur":15,"ph":"X","args":{"epoch":2,}} -{"name":"TFReader.__init__","cat":"reader","pid":33128,"tid":92626,"ts":2126954,"dur":22,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFReader.next","cat":"reader","pid":33128,"tid":92626,"ts":2127070,"dur":6,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":19599,"ts":2218091,"dur":636,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2218880,"dur":23,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2218959,"dur":16,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":11786,"ts":2218433,"dur":1294,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2219830,"dur":30,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2219909,"dur":17,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":80936,"ts":2219205,"dur":3055,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":19599,"ts":2222761,"dur":22,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":8029,"ts":2220290,"dur":2767,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":19599,"ts":2222931,"dur":655,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2223644,"dur":26,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2223709,"dur":15,"ph":"X","args":{}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":2223922,"dur":22,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":11786,"ts":2223511,"dur":606,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":19599,"ts":2223782,"dur":487,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":11786,"ts":2224274,"dur":556,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2224902,"dur":46,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2224995,"dur":16,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":19599,"ts":2224420,"dur":590,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2225123,"dur":28,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2225193,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":11786,"ts":2225087,"dur":524,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":11786,"ts":2225766,"dur":13,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2225825,"dur":25,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2225889,"dur":15,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":8029,"ts":2228127,"dur":2700,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":11786,"ts":2231359,"dur":20,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":24126,"ts":2230763,"dur":763,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2231625,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2231693,"dur":15,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":80936,"ts":2227078,"dur":4834,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":11786,"ts":2231539,"dur":709,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2232308,"dur":32,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2232380,"dur":16,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":93484,"ts":2232611,"dur":27,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2232703,"dur":34,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2232779,"dur":16,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":11786,"ts":2232446,"dur":614,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":19599,"ts":2232270,"dur":790,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":11786,"ts":2233258,"dur":890,"ph":"X","args":{"ret":12,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":19599,"ts":2233361,"dur":805,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2234222,"dur":28,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2234274,"dur":26,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2234313,"dur":29,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2234356,"dur":16,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":19599,"ts":2234455,"dur":419,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":19599,"ts":2235018,"dur":18,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2235081,"dur":23,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2235146,"dur":14,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":52001,"ts":2234442,"dur":4378,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":81901,"ts":2232452,"dur":7468,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":24126,"ts":2240482,"dur":16,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":93484,"ts":2240666,"dur":542,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":24126,"ts":2240689,"dur":553,"ph":"X","args":{"ret":10,}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":80936,"ts":2235828,"dur":5453,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2241324,"dur":27,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":93484,"ts":2241400,"dur":13,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2241416,"dur":16,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2241465,"dur":27,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":24126,"ts":2241521,"dur":12,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2241530,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":93484,"ts":2241648,"dur":459,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":19599,"ts":2242173,"dur":19,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":24126,"ts":2241720,"dur":587,"ph":"X","args":{"ret":10,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":93484,"ts":2242297,"dur":13,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2242377,"dur":26,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2242415,"dur":26,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":8029,"ts":2237218,"dur":5195,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2242444,"dur":15,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2242487,"dur":16,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":19599,"ts":2242332,"dur":652,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2243056,"dur":28,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2243124,"dur":16,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":11786,"ts":2242790,"dur":406,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":19599,"ts":2243188,"dur":517,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":11786,"ts":2243355,"dur":704,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2244124,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2244192,"dur":16,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":19599,"ts":2243869,"dur":557,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2244500,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2244568,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":11786,"ts":2244314,"dur":596,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":11786,"ts":2245114,"dur":14,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2245177,"dur":28,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2245245,"dur":16,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":38594,"ts":2245649,"dur":4835,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":11786,"ts":2251050,"dur":20,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":52001,"ts":2242944,"dur":8229,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":80936,"ts":2247411,"dur":3957,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":11786,"ts":2251229,"dur":625,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2251918,"dur":28,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2251986,"dur":15,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":81901,"ts":2243989,"dur":8257,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":19599,"ts":2251736,"dur":661,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":93484,"ts":2252560,"dur":17,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":11786,"ts":2252062,"dur":674,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":24126,"ts":2252589,"dur":510,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":93484,"ts":2252727,"dur":499,"ph":"X","args":{"ret":12,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":19599,"ts":2252607,"dur":681,"ph":"X","args":{"ret":10,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":24126,"ts":2253292,"dur":16,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2253407,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2253291,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2253362,"dur":28,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":11786,"ts":2252900,"dur":412,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2253486,"dur":15,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2253527,"dur":17,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2253574,"dur":18,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2253645,"dur":29,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":24126,"ts":2253743,"dur":15,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2253859,"dur":17,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":19599,"ts":2253824,"dur":489,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":93484,"ts":2253781,"dur":583,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":19599,"ts":2254500,"dur":15,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2254570,"dur":28,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2254640,"dur":16,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":93484,"ts":2254516,"dur":658,"ph":"X","args":{"ret":12,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":24126,"ts":2255196,"dur":15,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2255241,"dur":26,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2255345,"dur":39,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2255408,"dur":34,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2255467,"dur":31,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":80936,"ts":2255228,"dur":4865,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":81901,"ts":2256634,"dur":4427,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":38594,"ts":2254110,"dur":7714,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":19599,"ts":2262168,"dur":20,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":24126,"ts":2262289,"dur":17,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":52001,"ts":2258153,"dur":4758,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":19599,"ts":2262336,"dur":734,"ph":"X","args":{"ret":10,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":24126,"ts":2262470,"dur":657,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2263144,"dur":28,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2263211,"dur":29,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2263251,"dur":18,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2263299,"dur":18,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":93484,"ts":2263227,"dur":514,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":19599,"ts":2263354,"dur":431,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":24126,"ts":2263394,"dur":587,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":93484,"ts":2263952,"dur":589,"ph":"X","args":{"ret":10,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":19599,"ts":2263968,"dur":690,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2264643,"dur":25,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":24126,"ts":2264171,"dur":547,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2264728,"dur":28,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2264766,"dur":17,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2264825,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2264863,"dur":17,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2264945,"dur":18,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":93484,"ts":2264915,"dur":678,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":93484,"ts":2265780,"dur":13,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2265853,"dur":26,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2265929,"dur":14,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":81901,"ts":2266265,"dur":4165,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"close","cat":"POSIX","pid":33128,"tid":24126,"ts":2270768,"dur":18,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":80936,"ts":2265137,"dur":5992,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":52001,"ts":2267589,"dur":3754,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"open","cat":"POSIX","pid":33128,"tid":24126,"ts":2270927,"dur":583,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2271677,"dur":23,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2271744,"dur":16,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":24126,"ts":2271812,"dur":680,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":24126,"ts":2272778,"dur":14,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2272837,"dur":26,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2272908,"dur":33,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":81901,"ts":2273144,"dur":1806,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":2224039,"dur":58862,"ph":"X","args":{}} -{"name":"_train.iter","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2070739,"dur":153054,"ph":"X","args":{"step":1,}} -{"name":"_train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2223801,"dur":99606,"ph":"X","args":{"step":1,}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":2324231,"dur":11,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":11786,"ts":2324246,"dur":22,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":11786,"ts":2324523,"dur":13,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2324591,"dur":41,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2324669,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":11786,"ts":2324737,"dur":12,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":11786,"ts":2324880,"dur":11,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2324941,"dur":21,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2324998,"dur":14,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":38594,"ts":2326357,"dur":2699,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":2324349,"dur":56883,"ph":"X","args":{}} -{"name":"_train.iter","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2323529,"dur":378,"ph":"X","args":{"step":2,}} -{"name":"_train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2323914,"dur":57702,"ph":"X","args":{"step":2,}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":2382267,"dur":12,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":93484,"ts":2385196,"dur":745,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":93484,"ts":2386236,"dur":679,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2386973,"dur":41,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2387137,"dur":25,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":93484,"ts":2387209,"dur":649,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":93484,"ts":2388109,"dur":12,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2388164,"dur":24,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2388223,"dur":14,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":52001,"ts":2388414,"dur":1439,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":2382375,"dur":55415,"ph":"X","args":{}} -{"name":"_train.iter","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2381741,"dur":428,"ph":"X","args":{"step":3,}} -{"name":"_train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2382177,"dur":57239,"ph":"X","args":{"step":3,}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":2440005,"dur":11,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":24126,"ts":2441191,"dur":19,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":24126,"ts":2441420,"dur":717,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2442226,"dur":28,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2442319,"dur":16,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":24126,"ts":2442387,"dur":595,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":24126,"ts":2443247,"dur":15,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2443308,"dur":26,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2443374,"dur":39,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":30596,"ts":2444308,"dur":1774,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":2440185,"dur":56556,"ph":"X","args":{}} -{"name":"_train.iter","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2439522,"dur":392,"ph":"X","args":{"step":4,}} -{"name":"_train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2439921,"dur":58837,"ph":"X","args":{"step":4,}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":2499584,"dur":12,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":19599,"ts":2499593,"dur":21,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":19599,"ts":2499892,"dur":42,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2499982,"dur":30,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2500104,"dur":19,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":19599,"ts":2500183,"dur":15,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":19599,"ts":2500349,"dur":1404,"ph":"X","args":{"ret":12,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2501825,"dur":71,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":19599,"ts":2501972,"dur":14,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":80936,"ts":2503127,"dur":2055,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":2499684,"dur":57972,"ph":"X","args":{}} -{"name":"_train.iter","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2498884,"dur":391,"ph":"X","args":{"step":5,}} -{"name":"_train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2499282,"dur":59968,"ph":"X","args":{"step":5,}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":2559826,"dur":12,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":11786,"ts":2560833,"dur":19,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":11786,"ts":2561117,"dur":16,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2561182,"dur":29,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2561269,"dur":16,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":11786,"ts":2561337,"dur":14,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":11786,"ts":2561619,"dur":14,"ph":"X","args":{"ret":13,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2561679,"dur":24,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":11786,"ts":2561742,"dur":15,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":8029,"ts":2562609,"dur":2956,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":2559920,"dur":57857,"ph":"X","args":{}} -{"name":"_train.iter","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2559360,"dur":365,"ph":"X","args":{"step":6,}} -{"name":"_train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2559733,"dur":59027,"ph":"X","args":{"step":6,}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":2619732,"dur":13,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":93484,"ts":2619744,"dur":865,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":93484,"ts":2620893,"dur":815,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2621793,"dur":27,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2621861,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":93484,"ts":2621928,"dur":555,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":93484,"ts":2622767,"dur":14,"ph":"X","args":{"ret":10,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2622826,"dur":26,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":93484,"ts":2622889,"dur":14,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":18359,"ts":2623717,"dur":2191,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":2619880,"dur":58329,"ph":"X","args":{}} -{"name":"_train.iter","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2618949,"dur":415,"ph":"X","args":{"step":7,}} -{"name":"_train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2619371,"dur":60067,"ph":"X","args":{"step":7,}} -{"name":"TFFramework.trace_object","cat":"ai_framework","pid":33128,"tid":81567,"ts":2680305,"dur":12,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":24126,"ts":2680317,"dur":20,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":24126,"ts":2680615,"dur":15,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2680675,"dur":30,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2680746,"dur":15,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":24126,"ts":2680809,"dur":21,"ph":"X","args":{}} -{"name":"open","cat":"POSIX","pid":33128,"tid":24126,"ts":2681095,"dur":14,"ph":"X","args":{"ret":15,}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2681157,"dur":25,"ph":"X","args":{}} -{"name":"pread","cat":"POSIX","pid":33128,"tid":24126,"ts":2681223,"dur":16,"ph":"X","args":{}} -{"name":"TFReader.parse_image","cat":"reader","pid":33128,"tid":81901,"ts":2682634,"dur":3204,"ph":"X","args":{"epoch":2,"image_idx":0,"step":1,"image_size":484,}} -{"name":"TFFramework.compute","cat":"ai_framework","pid":33128,"tid":81567,"ts":2680402,"dur":57823,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":19440,"ts":2740319,"dur":25,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":58970,"ts":2741501,"dur":20,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":58970,"ts":2742687,"dur":595,"ph":"X","args":{}} -{"name":"close","cat":"POSIX","pid":33128,"tid":19440,"ts":2744510,"dur":595,"ph":"X","args":{}} -{"name":"DLIOBenchmark._train","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2012854,"dur":735163,"ph":"X","args":{"step":8,}} -{"name":"TFFramework.get_loader","cat":"ai_framework","pid":33128,"tid":81567,"ts":2749004,"dur":40,"ph":"X","args":{}} -{"name":"TFDataLoader.finalize","cat":"data_loader","pid":33128,"tid":81567,"ts":2749145,"dur":5,"ph":"X","args":{"epoch":2,}} -{"name":"DLIOBenchmark.run","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":572675,"dur":2176539,"ph":"X","args":{"step":8,}} -{"name":"TFFramework.stop_framework_profiler","cat":"ai_framework","pid":33128,"tid":81567,"ts":2750192,"dur":14,"ph":"X","args":{}} -{"name":"DLIOBenchmark.finalize","cat":"dlio_benchmark","pid":33128,"tid":81567,"ts":2749279,"dur":237094,"ph":"X","args":{"step":8,}} diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..1c8b0c6 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,89 @@ +cff-version: 1.2.0 +message: If you use this software, please cite the software and the paper. +title: "DFTracer: An Analysis-Friendly Data Flow Tracer for AI-Driven Workflows" +abstract: A multi-level dataflow tracer designed for capturing I/O calls from AI-driven workflows. +license: MIT +url: https://github.com/hariharan-devarajan/dftracer +repository-code: https://github.com/hariharan-devarajan/dftracer +contact: + - name: Hariharan Devarajan + email: mani.hariharan@gmail.com +authors: + - family-names: Devarajan + given-names: Hariharan + orcid: https://orcid.org/0000-0001-5625-3494 + - family-names: Pottier + given-names: Loic + orcid: https://orcid.org/0000-0002-7681-3521 + - family-names: Velusamy + given-names: Kaushik + orcid: https://orcid.org/0000-0003-4902-8760 + - family-names: Zheng + given-names: Huihuo + orcid: https://orcid.org/0000-0001-9008-9552 + - family-names: Yildirim + given-names: Izzet + orcid: https://orcid.org/0000-0003-3513-0764 + - family-names: Kogiou + given-names: Olga + orcid: https://orcid.org/0009-0009-6043-4437 + - family-names: Yu + given-names: Weikuan + orcid: https://orcid.org/0000-0002-8754-0311 + - family-names: Kougkas + given-names: Anthony + orcid: https://orcid.org/0000-0003-3943-663X + - family-names: Sun + given-names: Xian-He + orcid: https://orcid.org/0000-0002-1093-0792 + - family-names: Yeom + given-names: Jae Seung + orcid: https://orcid.org/0000-0001-5464-6040 + - family-names: Mohror + given-names: Kathryn + orcid: https://orcid.org/0000-0002-1366-1655 +preferred-citation: + type: conference-paper + title: "DFTracer: An Analysis-Friendly Data Flow Tracer for AI-Driven Workflows" + year: 2024 + authors: + - family-names: Devarajan + given-names: Hariharan + orcid: https://orcid.org/0000-0001-5625-3494 + - family-names: Pottier + given-names: Loic + orcid: https://orcid.org/0000-0002-7681-3521 + - family-names: Velusamy + given-names: Kaushik + orcid: https://orcid.org/0000-0003-4902-8760 + - family-names: Zheng + given-names: Huihuo + orcid: https://orcid.org/0000-0001-9008-9552 + - family-names: Yildirim + given-names: Izzet + orcid: https://orcid.org/0000-0003-3513-0764 + - family-names: Kogiou + given-names: Olga + orcid: https://orcid.org/0009-0009-6043-4437 + - family-names: Yu + given-names: Weikuan + orcid: https://orcid.org/0000-0002-8754-0311 + - family-names: Kougkas + given-names: Anthony + orcid: https://orcid.org/0000-0003-3943-663X + - family-names: Sun + given-names: Xian-He + orcid: https://orcid.org/0000-0002-1093-0792 + - family-names: Yeom + given-names: Jae Seung + orcid: https://orcid.org/0000-0001-5464-6040 + - family-names: Mohror + given-names: Kathryn + orcid: https://orcid.org/0000-0002-1366-1655 + conference: + name: "SC24: International Conference for High Performance Computing, Networking, Storage and Analysis" + city: Atlanta + region: GA + country: USA + date-start: 2024-11-17 + date-end: 2024-11-22 diff --git a/README.md b/README.md index a99661f..c1b15a8 100644 --- a/README.md +++ b/README.md @@ -1,46 +1,59 @@ -[![DFTracer Build and Test](https://github.com/hariharan-devarajan/dftracer/actions/workflows/ci.yml/badge.svg)](https://github.com/hariharan-devarajan/dftracer/actions/workflows/ci.yml) -[![Coverage Status](https://coveralls.io/repos/github/hariharan-devarajan/dftracer/badge.svg?branch=feature/apis)](https://coveralls.io/github/hariharan-devarajan/dftracer?branch=dev) +# DFTracer + +[![Build and Test](https://github.com/hariharan-devarajan/dftracer/actions/workflows/ci.yml/badge.svg)](https://github.com/hariharan-devarajan/dftracer/actions/workflows/ci.yml) [![Documentation Status](https://readthedocs.org/projects/dftracer/badge/?version=latest)](https://dftracer.readthedocs.io/en/latest/?badge=latest) +![PyPI - Version](https://img.shields.io/pypi/v/pydftracer?label=PyPI) +![PyPI - Wheel](https://img.shields.io/pypi/wheel/pydftracer?label=Wheel) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pydftracer?label=Python) +![PyPI - License](https://img.shields.io/pypi/l/pydftracer?label=License) + +## Overview + +DFTracer is a tracing tool designed to capture both application-code and I/O-call level events from workflows. It provides a unified tracing interface, optimized trace format, and compression mechanism to enable efficient distributed analysis for large-scale AI-driven workloads. + +## Prerequisites -# DFTracer v1.0.4 -A multi-level profiler for capturing application functions and low-level system I/O calls from deep learning workloads. +Requirements for DFTracer -Requirements for profiler -1. Python > 3.7 -2. pybind11 +1. Python>=3.7 +1. pybind11 + +Requirements for DFAnalyzer -Requirements for analyzer 1. bokeh>=2.4.2 -2. pybind11 -3. [zindex_py](https://github.com/hariharan-devarajan/zindex.git) -4. pandas>=2.0.3 -5. dask>=2023.5.0 -6. distributed -7. numpy>=1.24.3 -8. pyarrow>=12.0.1 -9. rich>=13.6.0 -10. python-intervals>=1.10.0.post1 -11. matplotlib>=3.7.3 +1. dask>=2023.5.0 +1. distributed +1. matplotlib>=3.7.3 +1. numpy>=1.24.3 +1. pandas>=2.0.3 +1. pyarrow>=12.0.1 +1. pybind11 +1. python-intervals>=1.10.0.post1 +1. rich>=13.6.0 +1. seaborn>=0.13.2 +1. [zindex_py](https://github.com/hariharan-devarajan/zindex.git) ## Installation -Users can easily install DFTracer using pip. This is the way most Python packages are installed. -This method would work for both native Python environments and Conda environments. +Users can easily install DFTracer using `pip`, the standard tool for installing Python packages. +This method works for both native Python and Conda environments. ### From PyPI ```bash pip install pydftracer +pip install pydftracer[dfanalyzer] ``` ### From Github ```bash -DFT_VERSION=develop -pip install git+https://github.com/hariharan-devarajan/dftracer.git@${DFT_VERSION} +DFTRACER_VERSION=develop +pip install git+https://github.com/hariharan-devarajan/dftracer.git@${DFTRACER_VERSION} +pip install git+https://github.com/hariharan-devarajan/dftracer.git@${DFTRACER_VERSION}#egg=pydftracer[dfanalyzer] ``` -### From source +### From Source ```bash git clone git@github.com:hariharan-devarajan/dftracer.git @@ -51,7 +64,7 @@ git checkout tags/ -b pip install . ``` -For more build instructions check [here](https://dftracer.readthedocs.io/en/latest/build.html). +For detailed build instructions, click [here](https://dftracer.readthedocs.io/en/latest/build.html). ## Usage @@ -80,7 +93,6 @@ def posix_calls(val): # NPZ calls internally calls POSIX calls. def npz_calls(index): - # print(f"{cwd}/data/demofile2.npz") path = f"{cwd}/data/demofile{index}.npz" if os.path.exists(path): os.remove(path) @@ -104,31 +116,39 @@ By default the DFTracer mode is set to `FUNCTION`. Example of running this configurations are: ```bash -# The process id, app_name and .pfw will be appended by the profiler for each app and process. -# The name of the final log file is ~/log_file--.pfw +# The process id, app_name and .pfw will be appended by DFTracer for each app and process. +# The name of the final log file will be ~/log_file--.pfw DFTRACER_LOG_FILE=~/log_file -# Colon separated paths for including for profiler +# Colon separated paths to include in the tracing DFTRACER_DATA_DIR=/dev/shm/:/p/gpfs1/$USER/dataset:$PWD/data -# Enable profiler +# Enable DFTracer DFTRACER_ENABLE=1 ``` -For more example check [Examples](https://dftracer.readthedocs.io/en/latest/examples.html). +For more examples, click [here](https://dftracer.readthedocs.io/en/latest/examples.html). + +## Documentation + +* Building DFTracer: [https://dftracer.readthedocs.io/en/latest/build.html](https://dftracer.readthedocs.io/en/latest/build.html) +* Integrating DFTracer: [https://dftracer.readthedocs.io/en/latest/examples.html](https://dftracer.readthedocs.io/en/latest/examples.html) +* Visualizing DFTracer Traces: [https://dftracer.readthedocs.io/en/latest/perfetto.html](https://dftracer.readthedocs.io/en/latest/perfetto.html) +* Building DFAnalyzer: [https://dftracer.readthedocs.io/en/latest/dfanalyzer_build.html](https://dftracer.readthedocs.io/en/latest/dfanalyzer_build.html) ## Citation and Reference -The original SC'24 paper describes the design and implementation of DFTracer code. Please cite this paper and the code if you use DFTracer for your research. + +The original SC'24 paper describes the design and implementation of the DFTracer code. Please cite this paper and the code if you use DFTracer in your research. ``` @inproceedings{devarajan_dftracer_2024, - address = {Atlanta, GA}, - title = {{DFTracer}: {An} {Analysis}-{Friendly} {Data} {Flow} {Tracer} for {AI}-{Driven} {Workflows}}, - shorttitle = {{DFTracer}}, - urldate = {2024-07-31}, - booktitle = {{SC24}: {International} {Conference} for {High} {Performance} {Computing}, {Networking}, {Storage} and {Analysis}}, - publisher = {IEEE}, - author = {Devarajan, Hariharan and Pottier, Loic and Velusamy, Kaushik and Zheng, Huihuo and Yildirim, Izzet and Kogiou, Olga and Yu, Weikuan and Kougkas, Anthony and Sun, Xian-He and Yeom, Jae Seung and Mohror, Kathryn}, - month = nov, - year = {2024}, + address = {Atlanta, GA}, + title = {{DFTracer}: {An} {Analysis}-{Friendly} {Data} {Flow} {Tracer} for {AI}-{Driven} {Workflows}}, + shorttitle = {{DFTracer}}, + urldate = {2024-07-31}, + booktitle = {{SC24}: {International} {Conference} for {High} {Performance} {Computing}, {Networking}, {Storage} and {Analysis}}, + publisher = {IEEE}, + author = {Devarajan, Hariharan and Pottier, Loic and Velusamy, Kaushik and Zheng, Huihuo and Yildirim, Izzet and Kogiou, Olga and Yu, Weikuan and Kougkas, Anthony and Sun, Xian-He and Yeom, Jae Seung and Mohror, Kathryn}, + month = nov, + year = {2024}, } @misc{devarajan_dftracer_code_2024, @@ -147,8 +167,3 @@ The original SC'24 paper describes the design and implementation of DFTracer cod ## Acknowledgments This work was performed under the auspices of the U.S. Department of Energy by Lawrence Livermore National Laboratory under Contract DE-AC52-07NA27344; and under the auspices of the National Cancer Institute (NCI) by Frederick National Laboratory for Cancer Research (FNLCR) under Contract 75N91019D00024. This research used resources of the Argonne Leadership Computing Facility, a U.S. Department of Energy (DOE) Office of Science user facility at Argonne National Laboratory and is based on research supported by the U.S. DOE Office of Science-Advanced Scientific Computing Research Program, under Contract No. DE-AC02-06CH11357. Office of Advanced Scientific Computing Research under the DOE Early Career Research Program. Also, This material is based upon work partially supported by LLNL LDRD 23-ERD-045 and 24-SI-005. LLNL-CONF-857447. - - -## License - -MIT License [LICENSE](./LICENSE) From fcca51bea8f8053bb9ef59b3184992a332c4ef3f Mon Sep 17 00:00:00 2001 From: Izzet Yildirim Date: Fri, 11 Oct 2024 15:59:28 -0500 Subject: [PATCH 15/15] Add DOI to CITATION.cff (#224) --- CITATION.cff | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CITATION.cff b/CITATION.cff index 1c8b0c6..1136436 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,6 +1,8 @@ cff-version: 1.2.0 message: If you use this software, please cite the software and the paper. title: "DFTracer: An Analysis-Friendly Data Flow Tracer for AI-Driven Workflows" +version: 1.0.6 +doi: 10.5281/zenodo.13917384 abstract: A multi-level dataflow tracer designed for capturing I/O calls from AI-driven workflows. license: MIT url: https://github.com/hariharan-devarajan/dftracer