Skip to content

Commit

Permalink
i#2006 generalize drcachesim: export drmemtrace analysis framework (#…
Browse files Browse the repository at this point in the history
…2780)

Exports a new drmemtrace_analyzer library along with all trace analysis
libraries as drmemtrace_basic_counts, drmemtrace_histogram,
drmemtrace_reuse_distance, drmemtrace_reuse_time, and drmemtrace_simulator.

Renames the drmemtrace_histogram executable to histogram_launcher to avoid
clashing with the new public library drmemtrace_ prefix.

Exports header files for building trace analysis tools and using our
provided tools in a new location tools/include/drmemtrace/.

Adds a new DynamoRIO CMake package command
use_DynamoRIO_drmemtrace() to facilitate building third-party trace
analysis tools.

Adds a new DynamoRIO CMake package command
configure_DynamoRIO_main_headers() to facilitate using just drfrontendlib
and finding dr_frontend.h, and fixes dr_frontend.h typedef issues when used
by itself.

Adds a build-and-test test of using the framework from a separate CMake
project.

Adds documentation of the new features.

Manually tested in a release package.  Tweaks the package.cmake for a
missing utils function.

Still missing: adding extensive doxygen comments in all of the headers, and
including the headers in genapi so they show up in the html docs.

Issue: #2006
  • Loading branch information
derekbruening authored Dec 20, 2017
1 parent 5ac6364 commit 84353f2
Show file tree
Hide file tree
Showing 11 changed files with 365 additions and 51 deletions.
9 changes: 7 additions & 2 deletions api/docs/release.dox
Original file line number Diff line number Diff line change
Expand Up @@ -210,8 +210,13 @@ Further non-compatibility-affecting changes include:
dr_register_kernel_xfer_event() with corresponding routines
drmgr_register_kernel_xfer_event() and drmgr_register_kernel_xfer_event_ex().
- Added a new option -ignore_all_libs to drcpusim.
- Added a new tool basic_counts to drcachesim for counting up the
different types of entries in a trace, broken down by thread.
- Added several new trace analysis tools to drcachesim: reuse distance,
reuse time, histogram, and trace basic counts.
- Added a trace analysis tool framework to facilitate creating custom
trace tools using the CMake function use_DynamoRIO_drmemtrace() and
exported drmemtrace_analyzer and analysis tool libraries.
- Added the CMake function configure_DynamoRIO_main_headers() to
facilitate using drfrontendlib by itself.
- Added instr_is_string_op() and instr_is_rep_string_op().
- Added dr_app_recurlock_lock().

Expand Down
20 changes: 15 additions & 5 deletions clients/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# **********************************************************
# Copyright (c) 2010-2016 Google, Inc. All rights reserved.
# Copyright (c) 2010-2017 Google, Inc. All rights reserved.
# Copyright (c) 2010 VMware, Inc. All rights reserved.
# **********************************************************

Expand Down Expand Up @@ -54,16 +54,26 @@ set(INSTALL_CLIENTS_BIN ${INSTALL_CLIENTS_BASE}/${INSTALL_BIN})
set(INSTALL_CLIENTS_LIB ${INSTALL_CLIENTS_BASE}/${INSTALL_LIB})
set(INSTALL_CLIENTS_CONFIG ${INSTALL_CLIENTS_BASE})
set(INSTALL_CLIENTS_CMAKE ${INSTALL_CLIENTS_BASE}/cmake)
# We install client header files to ext/include
set(INSTALL_CLIENTS_INCLUDE ${INSTALL_CLIENTS_BASE}/../ext/include)
# We install header files meant for use by DR clients to ext/include
set(INSTALL_CLIENTS_DR_INCLUDE ${INSTALL_CLIENTS_BASE}/../ext/include)
# For non-DR-client headers we use tools/include in the install package.
set(INSTALL_CLIENTS_NONDR_INCLUDE ${INSTALL_CLIENTS_BASE}/include)

macro(install_client_header header)
DR_install(FILES ${header} DESTINATION ${INSTALL_CLIENTS_INCLUDE})
macro(install_client_DR_header header)
DR_install(FILES ${header} DESTINATION ${INSTALL_CLIENTS_DR_INCLUDE})
# We also need a copy in the build dir for our --build-and-test (i#1586)
get_filename_component(file ${header} NAME)
configure_file(${header} ${PROJECT_BINARY_DIR}/ext/include/${file} COPYONLY)
endmacro()

macro(install_client_nonDR_header dest_dir src_header)
DR_install(FILES ${src_header} DESTINATION ${INSTALL_CLIENTS_NONDR_INCLUDE}/${dest_dir})
# We also need a copy in the build dir for our --build-and-test (i#1586)
get_filename_component(file ${src_header} NAME)
configure_file(${src_header} ${PROJECT_BINARY_DIR}/clients/include/${dest_dir}/${file}
COPYONLY)
endmacro()

disable_compiler_warnings()
# Clients don't include configure.h so they don't get DR defines
add_dr_defines()
Expand Down
121 changes: 82 additions & 39 deletions clients/drcachesim/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,21 @@ set(client_and_sim_srcs
common/trace_entry.cpp)

# i#2006: we split our tools into libraries for combining as desired in separate
# launchers.
add_library(reuse_distance STATIC tools/reuse_distance.cpp)
add_library(histogram STATIC tools/histogram.cpp)
add_library(reuse_time STATIC tools/reuse_time.cpp)
add_library(basic_counts STATIC tools/basic_counts.cpp)
# launchers. Since they are exported in the same dir as other tools like drcov,
# we use a drmemtrace_ prefix.
macro (add_exported_library name type)
add_library(${name} ${type} ${ARGN})
DR_export_target(${name})
install_exported_target(${name} ${INSTALL_CLIENTS_LIB})
endmacro ()

add_exported_library(drmemtrace_reuse_distance STATIC tools/reuse_distance.cpp)
add_exported_library(drmemtrace_histogram STATIC tools/histogram.cpp)
add_exported_library(drmemtrace_reuse_time STATIC tools/reuse_time.cpp)
add_exported_library(drmemtrace_basic_counts STATIC tools/basic_counts.cpp)

# We combine the cache and TLB simulators as they share code already.
add_library(simulator STATIC
add_exported_library(drmemtrace_simulator STATIC
simulator/simulator.cpp
simulator/cache.cpp
simulator/cache_lru.cpp
Expand All @@ -79,12 +87,12 @@ add_library(simulator STATIC
simulator/tlb_simulator.cpp
)

add_library(raw2trace STATIC
add_exported_library(drmemtrace_raw2trace STATIC
tracer/raw2trace.cpp
tracer/raw2trace_directory.cpp
)
configure_DynamoRIO_standalone(raw2trace)
target_link_libraries(raw2trace drfrontendlib)
configure_DynamoRIO_standalone(drmemtrace_raw2trace)
target_link_libraries(drmemtrace_raw2trace drfrontendlib)

set(drcachesim_srcs
launcher.cpp
Expand All @@ -107,8 +115,8 @@ add_executable(drcachesim ${drcachesim_srcs})
# In order to embed raw2trace we need to be standalone:
configure_DynamoRIO_standalone(drcachesim)
# Link in our tools:
target_link_libraries(drcachesim simulator reuse_distance histogram reuse_time
basic_counts raw2trace)
target_link_libraries(drcachesim drmemtrace_simulator drmemtrace_reuse_distance
drmemtrace_histogram drmemtrace_reuse_time drmemtrace_basic_counts drmemtrace_raw2trace)
# To avoid dup symbol errors between drinjectlib and the drdecode brought in
# by drfrontendlib we have to explicitly list drdecode up front:
target_link_libraries(drcachesim drdecode drinjectlib drconfiglib drfrontendlib)
Expand All @@ -118,35 +126,54 @@ use_DynamoRIO_extension(drcachesim drcovlib_static)
use_DynamoRIO_extension(drcachesim drutil_static)

# This is to avoid ../ and common/ in the #includes of headers that we
# may want to install into a single dir for 3rd-party tool integration.
# export in a single dir for 3rd-party tool integration.
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/common)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/reader)
include_directories(${CMAKE_CURRENT_SOURCE_DIR})

set(file_analyzer_tool_srcs
add_exported_library(drmemtrace_analyzer STATIC
analyzer.cpp
common/trace_entry.cpp
reader/reader.cpp
reader/file_reader.cpp
${zlib_reader}
)
# We get away w/ exporting the generically-named "utils.h" by putting into a
# drmemtrace/ subdir.
install_client_nonDR_header(drmemtrace common/utils.h)
install_client_nonDR_header(drmemtrace common/trace_entry.h)
install_client_nonDR_header(drmemtrace common/memref.h)
install_client_nonDR_header(drmemtrace reader/reader.h)
install_client_nonDR_header(drmemtrace analysis_tool.h)
install_client_nonDR_header(drmemtrace analyzer.h)
install_client_nonDR_header(drmemtrace tools/reuse_distance_create.h)
install_client_nonDR_header(drmemtrace tools/histogram_create.h)
install_client_nonDR_header(drmemtrace tools/reuse_time_create.h)
install_client_nonDR_header(drmemtrace tools/basic_counts_create.h)
install_client_nonDR_header(drmemtrace simulator/cache_simulator_create.h)
install_client_nonDR_header(drmemtrace simulator/tlb_simulator_create.h)

# We show one example of how to create a standalone analyzer of trace
# files that does not need to link with DR.
# We also use this to link in our trace_invariants sanity checker.
# i#2006: the final method for supporting 3rd-party tools is not yet decided.
add_executable(drmemtrace_histogram
${file_analyzer_tool_srcs}
add_executable(histogram_launcher
tools/histogram_launcher.cpp
tests/trace_invariants.cpp
)
target_link_libraries(drmemtrace_histogram histogram drfrontendlib)
use_DynamoRIO_extension(drmemtrace_histogram droption)
add_dependencies(drmemtrace_histogram api_headers)
target_link_libraries(histogram_launcher drmemtrace_analyzer drmemtrace_histogram
drfrontendlib)
use_DynamoRIO_extension(histogram_launcher droption)
add_dependencies(histogram_launcher api_headers)
# We have a companion test built using a separate --build-and-test CMake project in
# tests/analyzer_separate.cpp to better test 3rd-party usage.
set_property(GLOBAL PROPERTY DynamoRIO_drmemtrace_src_dir
"${CMAKE_CURRENT_SOURCE_DIR}/tests")
set_property(GLOBAL PROPERTY DynamoRIO_drmemtrace_build_dir
"${CMAKE_CURRENT_BINARY_DIR}/tests")

if (ZLIB_FOUND)
target_link_libraries(drcachesim ${ZLIB_LIBRARIES})
target_link_libraries(drmemtrace_histogram ${ZLIB_LIBRARIES})
target_link_libraries(histogram_launcher ${ZLIB_LIBRARIES})
endif ()

macro(add_drmemtrace name type)
Expand Down Expand Up @@ -176,14 +203,18 @@ endmacro()

add_drmemtrace(drmemtrace SHARED)
add_drmemtrace(drmemtrace_static STATIC)
install_client_header(tracer/drmemtrace.h)
# XXX i#2006: should we export drmemtrace.h into tools/include/drmemtace/tracer.h
# with the analysis framework headers, or keep it where it is in ext/include? For
# now I'm keeping it looking like an extension as it's for client use, not separate
# analysis tool use.
install_client_DR_header(tracer/drmemtrace.h)

add_executable(drraw2trace
tracer/raw2trace_launcher.cpp
tracer/instru.cpp
tracer/instru_online.cpp
)
target_link_libraries(drraw2trace raw2trace)
target_link_libraries(drraw2trace drmemtrace_raw2trace)
# To avoid dup symbol errors on some VS builds we list drdecode before DR:
target_link_libraries(drraw2trace drdecode)
configure_DynamoRIO_standalone(drraw2trace)
Expand Down Expand Up @@ -214,12 +245,13 @@ endmacro()

restore_nonclient_flags(drcachesim)
restore_nonclient_flags(drraw2trace)
restore_nonclient_flags(histogram_launcher)
restore_nonclient_flags(drmemtrace_simulator)
restore_nonclient_flags(drmemtrace_reuse_distance)
restore_nonclient_flags(drmemtrace_histogram)
restore_nonclient_flags(simulator)
restore_nonclient_flags(reuse_distance)
restore_nonclient_flags(histogram)
restore_nonclient_flags(reuse_time)
restore_nonclient_flags(basic_counts)
restore_nonclient_flags(drmemtrace_reuse_time)
restore_nonclient_flags(drmemtrace_basic_counts)
restore_nonclient_flags(drmemtrace_analyzer)

# We need to pass /EHsc and we pull in libcmtd into drcachesim from a dep lib.
# Thus we need to override the /MT with /MTd.
Expand All @@ -246,12 +278,13 @@ endmacro ()

add_win32_flags(drcachesim)
add_win32_flags(drraw2trace)
add_win32_flags(histogram_launcher)
add_win32_flags(drmemtrace_simulator)
add_win32_flags(drmemtrace_reuse_distance)
add_win32_flags(drmemtrace_histogram)
add_win32_flags(simulator)
add_win32_flags(reuse_distance)
add_win32_flags(histogram)
add_win32_flags(reuse_time)
add_win32_flags(basic_counts)
add_win32_flags(drmemtrace_reuse_time)
add_win32_flags(drmemtrace_basic_counts)
add_win32_flags(drmemtrace_analyzer)
if (WIN32 AND DEBUG)
get_target_property(sim_srcs drcachesim SOURCES)
get_target_property(raw2trace_srcs drraw2trace SOURCES)
Expand Down Expand Up @@ -337,9 +370,11 @@ endif ()
if (BUILD_TESTS)
add_executable(tool.drcachesim.unit_tests tests/drcachesim_unit_tests.cpp)
if (ZLIB_FOUND)
target_link_libraries(tool.drcachesim.unit_tests simulator drmemtrace_static ${ZLIB_LIBRARIES})
target_link_libraries(tool.drcachesim.unit_tests drmemtrace_simulator
drmemtrace_static ${ZLIB_LIBRARIES})
else ()
target_link_libraries(tool.drcachesim.unit_tests simulator drmemtrace_static)
target_link_libraries(tool.drcachesim.unit_tests drmemtrace_simulator
drmemtrace_static)
endif ()
add_win32_flags(tool.drcachesim.unit_tests)
add_test(NAME tool.drcachesim.unit_tests
Expand All @@ -355,14 +390,19 @@ if (BUILD_TESTS)
add_executable(tool.drcacheoff.burst_replace tests/burst_replace.cpp)
configure_DynamoRIO_static(tool.drcacheoff.burst_replace)
use_DynamoRIO_static_client(tool.drcacheoff.burst_replace drmemtrace_static)
target_link_libraries(tool.drcacheoff.burst_replace raw2trace)
target_link_libraries(tool.drcacheoff.burst_replace drmemtrace_raw2trace)
if (WIN32)
# burst_replace is unusual in cramming the tracer and post-processor into the
# same binary and we need some massaging to avoid duplicate symbol link errors.
target_link_libraries(tool.drcacheoff.burst_replace libcmtd)
if (DEBUG)
target_link_libraries(tool.drcacheoff.burst_replace libcmtd)
else ()
target_link_libraries(tool.drcacheoff.burst_replace libcmt)
endif ()
endif ()
add_win32_flags(tool.drcacheoff.burst_replace)
# Not really using an extension, just for including drmemtrace.h.
# Not really using an extension, just for including drmemtrace.h, which we
# don't want to include in use_DynamoRIO_drmemtrace().
use_DynamoRIO_extension(tool.drcacheoff.burst_replace drmemtrace_static)
use_DynamoRIO_extension(tool.drcacheoff.burst_replace drcovlib_static)

Expand All @@ -371,7 +411,7 @@ if (BUILD_TESTS)
use_DynamoRIO_static_client(tool.drcacheoff.burst_replaceall drmemtrace_static)
add_win32_flags(tool.drcacheoff.burst_replaceall)
use_DynamoRIO_extension(tool.drcacheoff.burst_replaceall drcontainers)
# Not really using an extension, just for including drmemtrace.h.
# See above.
use_DynamoRIO_extension(tool.drcacheoff.burst_replaceall drmemtrace_static)

add_executable(tool.drcacheoff.burst_threads tests/burst_threads.cpp)
Expand Down Expand Up @@ -401,7 +441,7 @@ if (BUILD_TESTS)
tracer/instru_online.cpp)
configure_DynamoRIO_standalone(tool.drcacheoff.raw2trace_io)
add_win32_flags(tool.drcacheoff.raw2trace_io)
target_link_libraries(tool.drcacheoff.raw2trace_io raw2trace)
target_link_libraries(tool.drcacheoff.raw2trace_io drmemtrace_raw2trace)
use_DynamoRIO_extension(tool.drcacheoff.raw2trace_io droption)
target_link_libraries(tool.drcacheoff.raw2trace_io drdecode)
use_DynamoRIO_extension(tool.drcacheoff.raw2trace_io drcovlib_static)
Expand Down Expand Up @@ -457,3 +497,6 @@ add_custom_command(
-D prog=${ops_path}
-P ${CMAKE_CURRENT_SOURCE_DIR}/../common/gendocs.cmake
VERBATIM)

# propagate to parent dir
set(exported_targets_append "${exported_targets_append}" PARENT_SCOPE)
19 changes: 18 additions & 1 deletion clients/drcachesim/drcachesim.dox.in
Original file line number Diff line number Diff line change
Expand Up @@ -522,7 +522,7 @@ and override the \p access(), \p child_access(), \p flush(), and/or
****************************************************************************
\section sec_drcachesim_newtool Creating New Analysis Tools

\p drcachesim provides an analysis tool framework to make it easy to create
\p drcachesim provides a \p drmemtrace analysis tool framework to make it easy to create
new trace analysis tools. A new tool should subclass <a
href="https://github.com/DynamoRIO/dynamorio/blob/master/clients/drcachesim/analysis_tool.h">analysis_tool_t</a>.
Two key functions should be overridden: \p process_memref, which operates
Expand All @@ -541,6 +541,23 @@ such as a thread exit or marker. There are built-in markers indicating
disruptions in user mode control flow such as signal handler entry and
exit.

CMake support is provided for including the headers and linking the
libraries of the \p drmemtrace framework. A new CMake function is defined
in the DynamoRIO package which sets the include directory for using the \p
drmemtrace/ headers:

\code
use_DynamoRIO_drmemtrace(mytool)
\endcode

The \p drmemtrace_analyzer library exported by the DynamoRIO package is the
main library to link when building a new tool. The tools described above
are also exported as the libraries \p drmemtrace_basic_counts, \p
drmemtrace_histogram, \p drmemtrace_reuse_distance, \p
drmemtrace_reuse_time, and \p drmemtrace_simulator and can be created using
the functions in the \p drmemtrace/toolname_create.h headers (e.g., \p
drmemtrace/reuse_distance_create.h).

****************************************************************************
\section sec_drcachesim_ops Simulator Parameters

Expand Down
64 changes: 64 additions & 0 deletions clients/drcachesim/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
# **********************************************************
# Copyright (c) 2015-2017 Google, Inc. All rights reserved.
# **********************************************************

# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# * Neither the name of Google, Inc. nor the names of its contributors may be
# used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
# DAMAGE.

# This is NOT a CMake file for building the various test suite executables in
# this tests/ directory: this is only for the --build-and-test test!

cmake_minimum_required(VERSION 2.6)

project(DynamoRIO_drmemtrace)

if (UNIX) # The shipped drmemtrace on Windows has no zlib support.
find_package(ZLIB)
if (NOT ZLIB_FOUND)
# Not a fatal error to succeed on AArch64 pre-commit and other places where
# zlib is not installed.
# XXX i#2006: Can we automate the zlib link somehow? Should we provide two versions
# of our libs, one with and one without? Can we include some version of zlib.a?
message(WARNING "zlib not found: linking will fail if drmemtrace has compressed "
"file support built-in")
endif()
endif ()

find_package(DynamoRIO)
if (NOT DynamoRIO_FOUND)
message(FATAL_ERROR "DynamoRIO package required to build")
endif()

configure_DynamoRIO_main_headers() # for dr_frontend.h

add_executable(analyzer_separate analyzer_separate.cpp)
use_DynamoRIO_extension(analyzer_separate droption) # for droption.h
use_DynamoRIO_drmemtrace(analyzer_separate) # for analysis framework headers
target_link_libraries(analyzer_separate drmemtrace_analyzer drmemtrace_histogram
drfrontendlib)
if (ZLIB_FOUND)
target_link_libraries(analyzer_separate ${ZLIB_LIBRARIES})
endif ()
Loading

0 comments on commit 84353f2

Please sign in to comment.