diff --git a/api/docs/API.doxy b/api/docs/API.doxy index cc85c688a40..b244cd87534 100644 --- a/api/docs/API.doxy +++ b/api/docs/API.doxy @@ -44,8 +44,11 @@ REPEAT_BRIEF = YES ABBREVIATE_BRIEF = ALWAYS_DETAILED_SEC = NO INLINE_INHERITED_MEMB = NO -FULL_PATH_NAMES = NO -STRIP_FROM_PATH = +# We want a hierarchy for the File List with the drcontainers headers +# grouped, and ext/ and drmemtrace/ separated out, so we do relative +# paths via full plus strip: +FULL_PATH_NAMES = YES +STRIP_FROM_PATH = ../../clients/include ../../../src STRIP_FROM_INC_PATH = SHORT_NAMES = NO JAVADOC_AUTOBRIEF = NO diff --git a/api/docs/CMakeLists.txt b/api/docs/CMakeLists.txt index 2e733f04703..b7108ce7be2 100644 --- a/api/docs/CMakeLists.txt +++ b/api/docs/CMakeLists.txt @@ -1,5 +1,5 @@ # ********************************************************** -# Copyright (c) 2010-2015 Google, Inc. All rights reserved. +# Copyright (c) 2010-2017 Google, Inc. All rights reserved. # Copyright (c) 2009-2010 VMware, Inc. All rights reserved. # ********************************************************** @@ -155,6 +155,7 @@ add_custom_command( # to work around i#84 be sure to put a space after -D for 1st arg at least ARGS -D srcdir=${CMAKE_CURRENT_SOURCE_DIR} -Doutfile=${doxyfile} -Dproj_srcdir=${PROJECT_SOURCE_DIR} -Dversion_number=${VERSION_NUMBER} + -Dproj_bindir=${PROJECT_BINARY_DIR} -Dheader_dir=${BUILD_INCLUDE} -Dgendox_dir=${CMAKE_CURRENT_BINARY_DIR} -DDOXYGEN_EXECUTABLE=${DOXYGEN_EXECUTABLE} -Ddoxygen_ver=${doxygen_ver} diff --git a/api/docs/CMake_doxyfile.cmake b/api/docs/CMake_doxyfile.cmake index 476f4ae6922..5b2068bf52a 100644 --- a/api/docs/CMake_doxyfile.cmake +++ b/api/docs/CMake_doxyfile.cmake @@ -1,5 +1,5 @@ # ********************************************************** -# Copyright (c) 2012-2015 Google, Inc. All rights reserved. +# Copyright (c) 2012-2017 Google, Inc. All rights reserved. # Copyright (c) 2009-2010 VMware, Inc. All rights reserved. # ********************************************************** @@ -33,6 +33,7 @@ # * srcdir # * outfile # * proj_srcdir +# * proj_bindir # * version_number # * header_dir # * gendox_dir @@ -51,6 +52,9 @@ foreach (dir ${dirs}) set(ext_dirs ${ext_dirs} ${dir}) endforeach (dir) +# Add drcachesim dirs (i#2006). +set(ext_dirs ${ext_dirs} "${proj_bindir}/clients/include/drmemtrace") + include("${srcdir}/CMake_doxyutils.cmake") set(input_paths srcdir proj_srcdir header_dir gendox_dir outdir) doxygen_path_xform(${DOXYGEN_EXECUTABLE} "${input_paths}") diff --git a/api/docs/release.dox b/api/docs/release.dox index b875875a2c0..e68f6fe02c3 100644 --- a/api/docs/release.dox +++ b/api/docs/release.dox @@ -183,7 +183,8 @@ Further non-compatibility-affecting changes include: drmemtrace_get_modlist_path(), and a separate rawtrace library for post-processing customization with raw2trace_t::handle_custom_data(), raw2trace_t::do_module_parsing(), raw2trace_t::do_conversion(), and - raw2trace_directory_t. + raw2trace_directory_t. A corresponding CMake function for finding the + tracer customization header is use_DynamoRIO_drmemtrace_tracer(). - Added a set_value() function to the \ref page_droption. - Added instrlist_get_auto_predicate() and instrlist_set_auto_predicate(). - Globally enabled auto predication in the drmgr instrumentation insertion event by diff --git a/clients/CMakeLists.txt b/clients/CMakeLists.txt index 8f5b29ab4ef..b776bac8b40 100644 --- a/clients/CMakeLists.txt +++ b/clients/CMakeLists.txt @@ -91,7 +91,11 @@ foreach (dir ${dirs}) endforeach (dir) # install subdirs -install_subdirs(${INSTALL_CLIENTS_LIB} ${INSTALL_CLIENTS_BIN}) +install_subdirs(${INSTALL_CLIENTS_LIB} ${INSTALL_CLIENTS_BIN} + PATTERN "histogram_launcher.debug" EXCLUDE + PATTERN "drcachesim_ops.debug" EXCLUDE + PATTERN "drcpusim_ops.debug" EXCLUDE + ) # propagate to parent dir set(exported_targets_append "${exported_targets_append}" PARENT_SCOPE) diff --git a/clients/drcachesim/CMakeLists.txt b/clients/drcachesim/CMakeLists.txt index f9a896c9da2..2e1d26eea38 100644 --- a/clients/drcachesim/CMakeLists.txt +++ b/clients/drcachesim/CMakeLists.txt @@ -152,6 +152,7 @@ install_client_nonDR_header(drmemtrace tools/reuse_time_create.h) install_client_nonDR_header(drmemtrace tools/basic_counts_create.h) install_client_nonDR_header(drmemtrace simulator/cache_simulator_create.h) install_client_nonDR_header(drmemtrace simulator/tlb_simulator_create.h) +install_client_nonDR_header(drmemtrace tracer/raw2trace.h) # We show one example of how to create a standalone analyzer of trace # files that does not need to link with DR. @@ -203,11 +204,9 @@ endmacro() add_drmemtrace(drmemtrace SHARED) add_drmemtrace(drmemtrace_static STATIC) -# XXX i#2006: should we export drmemtrace.h into tools/include/drmemtace/tracer.h -# with the analysis framework headers, or keep it where it is in ext/include? For -# now I'm keeping it looking like an extension as it's for client use, not separate -# analysis tool use. -install_client_DR_header(tracer/drmemtrace.h) +# We export drmemtrace.h to the same place as the analysis tool headers +# for simplicity, rather than sticking it into ext/include or sthg. +install_client_nonDR_header(drmemtrace tracer/drmemtrace.h) add_executable(drraw2trace tracer/raw2trace_launcher.cpp @@ -311,6 +310,7 @@ if (NOT DynamoRIO_INTERNAL OR NOT "${CMAKE_GENERATOR}" MATCHES "Ninja") endif () install_target(drcachesim ${INSTALL_CLIENTS_BIN}) +install_target(drraw2trace ${INSTALL_CLIENTS_BIN}) set(INSTALL_DRCACHESIM_CONFIG ${INSTALL_CLIENTS_BASE}) @@ -401,9 +401,7 @@ if (BUILD_TESTS) endif () endif () add_win32_flags(tool.drcacheoff.burst_replace) - # Not really using an extension, just for including drmemtrace.h, which we - # don't want to include in use_DynamoRIO_drmemtrace(). - use_DynamoRIO_extension(tool.drcacheoff.burst_replace drmemtrace_static) + use_DynamoRIO_drmemtrace_tracer(tool.drcacheoff.burst_replace) use_DynamoRIO_extension(tool.drcacheoff.burst_replace drcovlib_static) add_executable(tool.drcacheoff.burst_replaceall tests/burst_replaceall.cpp) @@ -411,8 +409,7 @@ if (BUILD_TESTS) use_DynamoRIO_static_client(tool.drcacheoff.burst_replaceall drmemtrace_static) add_win32_flags(tool.drcacheoff.burst_replaceall) use_DynamoRIO_extension(tool.drcacheoff.burst_replaceall drcontainers) - # See above. - use_DynamoRIO_extension(tool.drcacheoff.burst_replaceall drmemtrace_static) + use_DynamoRIO_drmemtrace_tracer(tool.drcacheoff.burst_replaceall) add_executable(tool.drcacheoff.burst_threads tests/burst_threads.cpp) configure_DynamoRIO_static(tool.drcacheoff.burst_threads) diff --git a/clients/drcachesim/analysis_tool.h b/clients/drcachesim/analysis_tool.h index b129d07af97..5425bf3649a 100644 --- a/clients/drcachesim/analysis_tool.h +++ b/clients/drcachesim/analysis_tool.h @@ -36,19 +36,42 @@ #ifndef _ANALYSIS_TOOL_H_ #define _ANALYSIS_TOOL_H_ 1 +/** + * @file drmemtrace/analysis_tool.h + * @brief DrMemtrace analysis tool base class. + */ + // To support installation of headers for analysis tools into a single // separate directory we omit common/ here and rely on -I. #include "memref.h" +/** + * The base class for a tool that analyzes a trace. A new tool should subclass this + * class. Two key functions should be overridden: process_memref(), which operates + * on a single trace entry, and print_results(), which is called just once after + * processing the entire trace and should present the results of analysis. In the + * default mode of operation, the #analyzer_t class iterates over the trace and calls + * the process_memref() function of each tool. An alternative mode is supported + * which exposes the iterator and allows a separate control infrastructure to be + * built. + */ class analysis_tool_t { public: - // Usage: errors encountered during the constructor will set a flag that should - // be queried via operator!. + /** + * Errors encountered during the constructor will set the success flag, which should + * be queried via operator!. + */ analysis_tool_t() : success(true) {}; - virtual ~analysis_tool_t() {}; + virtual ~analysis_tool_t() {}; /**< Destructor. */ + /** Returns whether the tool was created successfully. */ virtual bool operator!() { return !success; } + /** + * The heart of an analysis tool, this routine operates on a single trace entry and + * takes whatever actions the tool needs to perform its analysis. + */ virtual bool process_memref(const memref_t &memref) = 0; + /** This routine reports the results of the trace analysis. */ virtual bool print_results() = 0; protected: bool success; diff --git a/clients/drcachesim/analyzer.h b/clients/drcachesim/analyzer.h index 01ff60b68ec..5538052084a 100644 --- a/clients/drcachesim/analyzer.h +++ b/clients/drcachesim/analyzer.h @@ -38,44 +38,65 @@ #ifndef _ANALYZER_H_ #define _ANALYZER_H_ 1 +/** + * @file drmemtrace/analyzer.h + * @brief DrMemtrace top-level trace analysis driver. + */ + #include #include #include "analysis_tool.h" #include "reader.h" +/** + * An analyzer is the top-level driver of a set of trace analysis tools. + * It supports two different modes of operation: either it iterates over the + * trace and calls the process_memref() routine of each tool, or it exposes + * an iteration interface to external control code. + */ class analyzer_t { public: - // Usage: errors encountered during a constructor will set a flag that should - // be queried via operator!. If operator! returns true, get_error_string() - // can be used to try to obtain more information. + /** + * Usage: errors encountered during a constructor will set a flag that should + * be queried via operator!(). If operator!() returns true, get_error_string() + * can be used to try to obtain more information. + */ analyzer_t(); - virtual ~analyzer_t(); + virtual ~analyzer_t(); /**< Destructor. */ + /** Returns whether the analyzer was created successfully. */ virtual bool operator!(); + /** Returns further information on an error in initializing the analyzer. */ virtual std::string get_error_string(); - // We have two usage models: one where there are multiple tools and the - // trace iteration is performed by analyzer_t, and another where a single - // tool controls the iteration. - - // The default, simpler, multiple-tool-supporting model uses this constructor. - // The analyzer will reference the tools array passed in during its lifetime: - // it does not make a copy. - // The user must free them afterward. + /** + * We have two usage models: one where there are multiple tools and the + * trace iteration is performed by analyzer_t, and another where a single + * tool controls the iteration. + * + * The default, simpler, multiple-tool-supporting model uses this constructor. + * The analyzer will reference the tools array passed in during its lifetime: + * it does not make a copy. + * The user must free them afterward. + */ analyzer_t(const std::string &trace_file, analysis_tool_t **tools, int num_tools); + /** Launches the analysis process. */ virtual bool run(); + /** Presents the results of the analysis. */ virtual bool print_stats(); - // The alternate usage model exposes the iterator to a single tool. + /** The alternate usage model exposes the iterator to a single tool. */ analyzer_t(const std::string &trace_file); - // As the iterator is more heavyweight than regular container iterators - // we hold it internally and return a reference. Copying will fail to compile - // as reader_t is virtual, reminding the caller of begin() to use a reference. - // This usage model supports only a single user of the iterator: the - // multi-tool model above should be used if multiple tools are involved. + /** + * As the iterator is more heavyweight than regular container iterators + * we hold it internally and return a reference. Copying will fail to compile + * as reader_t is virtual, reminding the caller of begin() to use a reference. + * This usage model supports only a single user of the iterator: the + * multi-tool model above should be used if multiple tools are involved. + */ virtual reader_t & begin(); - virtual reader_t & end(); + virtual reader_t & end(); /** End iterator for the external-iterator usage model. */ protected: bool init_file_reader(const std::string &trace_file); diff --git a/clients/drcachesim/common/memref.h b/clients/drcachesim/common/memref.h index 7a9f56cce61..477e7d5f9fe 100644 --- a/clients/drcachesim/common/memref.h +++ b/clients/drcachesim/common/memref.h @@ -39,76 +39,88 @@ #include // for size_t #include "trace_entry.h" +/** + * @file drmemtrace/memref.h + * @brief DrMemtrace trace entry structures. + */ + // On some platforms, like MacOS, a thread id is 64 bits. // We just make both 64 bits to cover all our bases. -typedef int_least64_t memref_pid_t; -typedef int_least64_t memref_tid_t; - -// Each trace entry is one of the following. -// Although the pc of each data reference is provided, the trace also guarantees that -// an instruction entry immediately precedes the data references that it is -// responsible for, with no intervening trace entries. -// Offline traces further guarantee that an instruction entry for a branch -// instruction is always followed by an instruction entry for the branch's -// target (with any memory references for the branch in between of course) -// without a thread switch intervening, to make it simpler to identify branch -// targets. Online traces do not currently guarantee this. +typedef int_least64_t memref_pid_t; /**< Process id type. */ +typedef int_least64_t memref_tid_t; /**< Thread id type. */ +/** A trace entry representing a data load, store, or prefetch. */ struct _memref_data_t { - // TRACE_TYPE_READ, TRACE_TYPE_WRITE, and TRACE_TYPE_PREFETCH*: - // data references. - trace_type_t type; - memref_pid_t pid; - memref_tid_t tid; - addr_t addr; - size_t size; - addr_t pc; + trace_type_t type; /**< #TRACE_TYPE_READ, #TRACE_TYPE_WRITE, or type_is_prefetch(). */ + memref_pid_t pid; /**< Process id. */ + memref_tid_t tid; /**< Thread id. */ + addr_t addr; /**< Address of data being loaded or stored. */ + size_t size; /**< Size of data being loaded or stored. */ + addr_t pc; /**< Program counter of instruction performing load or store. */ }; +/** A trace entry representing an instruction fetch. */ struct _memref_instr_t { - // TRACE_TYPE_INSTR_* (minus BUNDLE): instruction fetch. - trace_type_t type; - memref_pid_t pid; - memref_tid_t tid; - addr_t addr; - size_t size; + trace_type_t type; /**< Matches type_is_instr() or #TRACE_TYPE_INSTR_NO_FETCH. */ + memref_pid_t pid; /**< Process id. */ + memref_tid_t tid; /**< Thread id. */ + addr_t addr; /**< The address of the instruction (i.e., program counter). */ + size_t size; /**< The length of the instruction. */ }; +/** A trace entry representing a software-requested explicit cache flush. */ struct _memref_flush_t { - // TRACE_TYPE_INSTR_FLUSH, TRACE_TYPE_DATA_FLUSH: explicit cache flush. - trace_type_t type; - memref_pid_t pid; - memref_tid_t tid; - addr_t addr; - size_t size; - addr_t pc; + trace_type_t type; /**< #TRACE_TYPE_INSTR_FLUSH or #TRACE_TYPE_DATA_FLUSH. */ + memref_pid_t pid; /**< Process id. */ + memref_tid_t tid; /**< Thread id. */ + addr_t addr; /**< The start address of the region being flushed. */ + size_t size; /**< The size of the region being flushed. */ + addr_t pc; /**< Program counter of the instruction requesting the flush. */ }; +/** A trace entry representing a thread exit. */ struct _memref_thread_exit_t { - // TRACE_TYPE_THREAD_EXIT. - trace_type_t type; - memref_pid_t pid; - memref_tid_t tid; + trace_type_t type; /**< #TRACE_TYPE_THREAD_EXIT. */ + memref_pid_t pid; /**< Process id. */ + memref_tid_t tid; /**< Thread id. */ }; +/** + * A trace entry containing metadata identifying some event that occurred at this + * point in the trace. The most common marker type represents a kernel-mediated + * control flow change such as a signal delivery, entry into an APC, callback, or + * exception dispatcher on Windows, or a system call that changes the context such as + * a signal return. + */ struct _memref_marker_t { - // TRACE_TYPE_MARKER. - trace_type_t type; - memref_pid_t pid; - memref_tid_t tid; - trace_marker_type_t marker_type; - uintptr_t marker_value; + trace_type_t type; /**< #TRACE_TYPE_MARKER. */ + memref_pid_t pid; /**< Process id. */ + memref_tid_t tid; /**< Thread id. */ + trace_marker_type_t marker_type; /**< Identifies the type of marker. */ + uintptr_t marker_value; /**< A value whose meaning depends on the marker type. */ }; +/** + * Each trace entry is one of the structures in this union. + * Each entry identifies the originating process and thread. + * Although the pc of each data reference is provided, the trace also guarantees that + * an instruction entry immediately precedes the data references that it is + * responsible for, with no intervening trace entries. + * Offline traces further guarantee that an instruction entry for a branch + * instruction is always followed by an instruction entry for the branch's + * target (with any memory references for the branch in between of course) + * without a thread switch intervening, to make it simpler to identify branch + * targets. Online traces do not currently guarantee this. + */ typedef union _memref_t { // The C standard allows us to reference the type field of any of these, and the // addr and size fields of data, instr, or flush generically if known to be one // of those types, due to the shared fields in our union of structs. - struct _memref_data_t data; - struct _memref_instr_t instr; - struct _memref_flush_t flush; - struct _memref_thread_exit_t exit; - struct _memref_marker_t marker; + struct _memref_data_t data; /**< A data load or store. */ + struct _memref_instr_t instr; /**< An insruction fetch. */ + struct _memref_flush_t flush; /**< A software-initiated cache flush. */ + struct _memref_thread_exit_t exit; /**< A thread exit. */ + struct _memref_marker_t marker; /**< A marker holding metadata. */ } memref_t; #endif /* _MEMREF_H_ */ diff --git a/clients/drcachesim/common/trace_entry.h b/clients/drcachesim/common/trace_entry.h index 61e103213db..b0537a232f7 100644 --- a/clients/drcachesim/common/trace_entry.h +++ b/clients/drcachesim/common/trace_entry.h @@ -46,12 +46,18 @@ #include #include "utils.h" -typedef uintptr_t addr_t; +/** + * @file drmemtrace/trace_entry.h + * @brief DrMemtrace trace entry enum types and definitions. + */ + +typedef uintptr_t addr_t; /**< The type of a memory address. */ -#define TRACE_ENTRY_VERSION 1 +#define TRACE_ENTRY_VERSION 1 /**< The version of the trace format. */ +/** The type of a trace entry in a #memref_t structure. */ // The type identifier for trace entries in the raw trace_entry_t passed to -// reader_t and the exposed memref_t passed to analysis tools. +// reader_t and the exposed #memref_t passed to analysis tools. // XXX: if we want to rely on a recent C++ standard we could try to get // this enum to be just 2 bytes instead of an int and give it qualified names. // N.B.: when adding new values, be sure to update trace_type_names[]. @@ -62,34 +68,33 @@ typedef enum { // assume we don't need the opcode. // These entries describe a memory reference as data: - TRACE_TYPE_READ, - TRACE_TYPE_WRITE, + TRACE_TYPE_READ, /**< A data load. */ + TRACE_TYPE_WRITE, /**< A data store. */ - // General prefetch to L1 data cache - TRACE_TYPE_PREFETCH, + TRACE_TYPE_PREFETCH, /**< A general prefetch to the level 1 data cache. */ // X86 specific prefetch - TRACE_TYPE_PREFETCHT0, - TRACE_TYPE_PREFETCHT1, - TRACE_TYPE_PREFETCHT2, - TRACE_TYPE_PREFETCHNTA, + TRACE_TYPE_PREFETCHT0, /**< An x86 prefetch to all levels of the cache. */ + TRACE_TYPE_PREFETCHT1, /**< An x86 prefetch to level 1 of the cache. */ + TRACE_TYPE_PREFETCHT2, /**< An x86 prefetch to level 2 of the cache. */ + TRACE_TYPE_PREFETCHNTA, /**< An x86 non-temporal prefetch. */ // ARM specific prefetch - TRACE_TYPE_PREFETCH_READ, - TRACE_TYPE_PREFETCH_WRITE, - TRACE_TYPE_PREFETCH_INSTR, + TRACE_TYPE_PREFETCH_READ, /**< An ARM load prefetch. */ + TRACE_TYPE_PREFETCH_WRITE, /**< An ARM store prefetch. */ + TRACE_TYPE_PREFETCH_INSTR, /**< An ARM insruction prefetch. */ // These entries describe an instruction fetch memory reference. // The trace_entry_t stream always has the instr fetch prior to data refs, // which the reader can use to obtain the PC for data references. // For memref_t, the instruction address is in the addr field. // An instruction *not* of the types below: - TRACE_TYPE_INSTR, + TRACE_TYPE_INSTR, /**< A non-branch instruction. */ // Particular categories of instructions: - TRACE_TYPE_INSTR_DIRECT_JUMP, - TRACE_TYPE_INSTR_INDIRECT_JUMP, - TRACE_TYPE_INSTR_CONDITIONAL_JUMP, - TRACE_TYPE_INSTR_DIRECT_CALL, - TRACE_TYPE_INSTR_INDIRECT_CALL, - TRACE_TYPE_INSTR_RETURN, + TRACE_TYPE_INSTR_DIRECT_JUMP, /**< A direct unconditional jump instruction. */ + TRACE_TYPE_INSTR_INDIRECT_JUMP, /**< An indirect jump instruction. */ + TRACE_TYPE_INSTR_CONDITIONAL_JUMP, /**< A conditional jump instruction. */ + TRACE_TYPE_INSTR_DIRECT_CALL, /**< A direct call instruction. */ + TRACE_TYPE_INSTR_INDIRECT_CALL, /**< An indirect call instruction. */ + TRACE_TYPE_INSTR_RETURN, /**< A return instruction. */ // These entries describe a bundle of consecutive instruction fetch // memory references. The trace stream always has a single instr fetch // prior to instr bundles which the reader can use to obtain the starting PC. @@ -106,9 +111,9 @@ typedef enum { // for the end address (exclusive) of flush. // The size field of both entries should be 0. // The _END entries are hidden by reader_t as memref_t has space for the size. - TRACE_TYPE_INSTR_FLUSH, + TRACE_TYPE_INSTR_FLUSH, /**< An instruction cache flush. */ TRACE_TYPE_INSTR_FLUSH_END, - TRACE_TYPE_DATA_FLUSH, + TRACE_TYPE_DATA_FLUSH, /**< A data cache flush. */ TRACE_TYPE_DATA_FLUSH_END, // These entries indicate that all subsequent memory references (until the @@ -118,7 +123,7 @@ typedef enum { TRACE_TYPE_THREAD, // This entry indicates that the thread whose id is in the addr field exited: - TRACE_TYPE_THREAD_EXIT, + TRACE_TYPE_THREAD_EXIT, /**< A thread exit. */ // These entries indicate which process the current thread belongs to. // The process id is in the addr field. @@ -132,36 +137,46 @@ typedef enum { // The final entry in an offline file or a pipe. TRACE_TYPE_FOOTER, - // Hardware-issued prefetch. + /** A hardware-issued prefetch (generated after tracing by a cache simulator). */ TRACE_TYPE_HARDWARE_PREFETCH, - // A marker containing metadata about this point in the trace. - // It includes a marker sub-type trace_marker_type_t and a value. + /** + * A marker containing metadata about this point in the trace. + * It includes a marker sub-type #trace_marker_type_t and a value. + */ TRACE_TYPE_MARKER, - // For core simulators, a trace includes instructions that do not incur - // instruction cache fetches, such as on each subsequent iteration of a - // rep string loop. + /** + * For core simulators, a trace includes instructions that do not incur + * instruction cache fetches, such as on each subsequent iteration of a + * rep string loop on x86. + */ TRACE_TYPE_INSTR_NO_FETCH, // An internal value used for online traces and turned by reader_t into // either TRACE_TYPE_INSTR or TRACE_TYPE_INSTR_NO_FETCH. TRACE_TYPE_INSTR_MAYBE_FETCH, - // We separate out the x86 sysenter instruction as it has a hardcoded - // return point that shows up as a discontinuity in the user mode program - // counter execution sequence. + /** + * We separate out the x86 sysenter instruction as it has a hardcoded + * return point that shows up as a discontinuity in the user mode program + * counter execution sequence. + */ TRACE_TYPE_INSTR_SYSENTER, } trace_type_t; -// The sub-type for TRACE_TYPE_MARKER. +/** The sub-type for TRACE_TYPE_MARKER. */ typedef enum { - // The subsequent instruction is the start of a handler for a kernel-initiated - // event: a signal handler on UNIX, or an APC, exception, or callback dispatcher - // on Windows. + /** + * The subsequent instruction is the start of a handler for a kernel-initiated + * event: a signal handler on UNIX, or an APC, exception, or callback dispatcher + * on Windows. + */ TRACE_MARKER_TYPE_KERNEL_EVENT, - // The subsequent instruction is the target of a system call that changes the - // context: a signal return on UNIX, or a callback return or NtContinue or - // NtSetContextThread on Windows. + /** + * The subsequent instruction is the target of a system call that changes the + * context: a signal return on UNIX, or a callback return or NtContinue or + * NtSetContextThread on Windows. + */ TRACE_MARKER_TYPE_KERNEL_XFER, // ... @@ -173,8 +188,10 @@ typedef enum { extern const char * const trace_type_names[]; -// Returns whether the type represents an instruction fetch. -// Deliberately excludes TRACE_TYPE_INSTR_NO_FETCH and TRACE_TYPE_INSTR_BUNDLE. +/** + * Returns whether the type represents an instruction fetch. + * Deliberately excludes TRACE_TYPE_INSTR_NO_FETCH and TRACE_TYPE_INSTR_BUNDLE. + */ static inline bool type_is_instr(const trace_type_t type) { @@ -182,12 +199,14 @@ type_is_instr(const trace_type_t type) type == TRACE_TYPE_INSTR_SYSENTER; } +/** Returns whether the type represents the fetch of a branch instruction. */ static inline bool type_is_instr_branch(const trace_type_t type) { return (type >= TRACE_TYPE_INSTR_DIRECT_JUMP && type <= TRACE_TYPE_INSTR_RETURN); } +/** Returns whether the type represents a prefetch request. */ static inline bool type_is_prefetch(const trace_type_t type) { diff --git a/clients/drcachesim/drcachesim.dox.in b/clients/drcachesim/drcachesim.dox.in index 87189e6c38b..a18ce3acea1 100644 --- a/clients/drcachesim/drcachesim.dox.in +++ b/clients/drcachesim/drcachesim.dox.in @@ -52,6 +52,7 @@ online and offline. - \ref sec_drcachesim_phys - \ref sec_drcachesim_core - \ref sec_drcachesim_extend + - \ref sec_drcachesim_tracer - \ref sec_drcachesim_newtool - \ref sec_drcachesim_ops - \ref sec_drcachesim_limit @@ -67,7 +68,8 @@ The analyzer consumes the traces (online or offline) and performs customized analysis. It is designed to be extensible, allowing users to easily implement a simulator for different devices, such as CPU caches, TLBs, page -caches, etc. (see \ref sec_drcachesim_extend). +caches, etc. (see \ref sec_drcachesim_extend), or to build arbitrary trace +analysis tools (see \ref sec_drcachesim_newtool). The default analyzer simulates the architectural behavior of caching devices for a target application (or multiple applications). @@ -519,23 +521,43 @@ class. To implement custom statistics, subclass \p caching_device_stats_t and override the \p access(), \p child_access(), \p flush(), and/or \p print_stats() methods. +**************************************************************************** +\section sec_drcachesim_tracer Customizing the Tracer + +The tracer supports customization for special-purpose i/o via +drmemtrace_replace_file_ops(), allowing traces to be written to locations +not supported by simple UNIX file operations. One option for using this +function is to create a new client which links with the provided +drmemtrace_static library, includes the \p drmemtrace/drmemtrace.h header via: + +\code +use_DynamoRIO_drmemtrace_tracer(mytool) +\endcode + +And includes its own dr_client_main() which calls +drmemtrace_client_main(). + +The tracer also supports storing custom data with each module (i.e., +library or executable) such as a build identifier via +drmemtrace_custom_module_data(). The custom data can be retrieved via +raw2trace_t::handle_custom_data() by creating a custom offline trace +post-processor using the #raw2trace_t class. + **************************************************************************** \section sec_drcachesim_newtool Creating New Analysis Tools -\p drcachesim provides a \p drmemtrace analysis tool framework to make it easy to create -new trace analysis tools. A new tool should subclass analysis_tool_t. -Two key functions should be overridden: \p process_memref, which operates -on a single trace entry, and \p print_results, which is called just once -after processing the entire trace and should present the results of -analysis. In the default mode of operation, the analyzer_t -class iterates over the trace and calls the \p process_memref function of -each tool. An alternative mode is supported which exposes the iterator and +\p drcachesim provides a \p drmemtrace analysis tool framework to make it +easy to create new trace analysis tools. A new tool should subclass +#analysis_tool_t. Two key functions should be overridden: +analysis_tool_t::process_memref(), which operates on a single trace entry, +and analysis_tool_t::print_results(), which is called just once after +processing the entire trace and should present the results of analysis. In +the default mode of operation, the #analyzer_t class iterates over the +trace and calls the analysis_tool_t::process_memref() function of each +tool. An alternative mode is supported which exposes the iterator and allows a separate control infrastructure to be built. -Each trace entry is of type memref_t +Each trace entry is of type #memref_t and represents one instruction or data reference or a metadata operation such as a thread exit or marker. There are built-in markers indicating disruptions in user mode control flow such as signal handler entry and @@ -555,8 +577,9 @@ main library to link when building a new tool. The tools described above are also exported as the libraries \p drmemtrace_basic_counts, \p drmemtrace_histogram, \p drmemtrace_reuse_distance, \p drmemtrace_reuse_time, and \p drmemtrace_simulator and can be created using -the functions in the \p drmemtrace/toolname_create.h headers (e.g., \p -drmemtrace/reuse_distance_create.h). +the basic_counts_tool_create(), histogram_tool_create(), +reuse_distance_tool_create(), reuse_time_tool_create(), +cache_simulator_create(), and tlb_simulator_create() functions. **************************************************************************** \section sec_drcachesim_ops Simulator Parameters diff --git a/clients/drcachesim/simulator/cache_simulator_create.h b/clients/drcachesim/simulator/cache_simulator_create.h index 95bae021d11..5bbbfe07f33 100644 --- a/clients/drcachesim/simulator/cache_simulator_create.h +++ b/clients/drcachesim/simulator/cache_simulator_create.h @@ -38,7 +38,16 @@ #include #include "analysis_tool.h" -// These options are currently documented in ../common/options.cpp. +/** + * @file drmemtrace/cache_simulator_create.h + * @brief DrMemtrace cache simulator creation. + */ + +/** + * Creates an instance of a cache simulator. + * The options are currently documented in \ref sec_drcachesim_ops. + */ +// The options are currently documented in ../common/options.cpp. analysis_tool_t * cache_simulator_create(unsigned int num_cores = 4, unsigned int line_size = 64, diff --git a/clients/drcachesim/simulator/tlb_simulator_create.h b/clients/drcachesim/simulator/tlb_simulator_create.h index b54f92c8c8e..1486db2acc0 100644 --- a/clients/drcachesim/simulator/tlb_simulator_create.h +++ b/clients/drcachesim/simulator/tlb_simulator_create.h @@ -38,7 +38,16 @@ #include #include "analysis_tool.h" -// These options are currently documented in ../common/options.cpp. +/** + * @file drmemtrace/tlb_simulator_create.h + * @brief DrMemtrace TLB simulator creation. + */ + +/** + * Creates an instance of a TLB simulator. + * The options are currently documented in \ref sec_drcachesim_ops. + */ +// The options are currently documented in ../common/options.cpp. analysis_tool_t * tlb_simulator_create(unsigned int num_cores = 4, uint64_t page_size = 4*1024, diff --git a/clients/drcachesim/tests/burst_replace.cpp b/clients/drcachesim/tests/burst_replace.cpp index 2e9aa02c696..15372000d35 100644 --- a/clients/drcachesim/tests/burst_replace.cpp +++ b/clients/drcachesim/tests/burst_replace.cpp @@ -39,7 +39,7 @@ * for us. */ #include "dr_api.h" -#include "drmemtrace.h" +#include "drmemtrace/drmemtrace.h" #include "drcovlib.h" #include "tracer/raw2trace.h" #include "tracer/raw2trace_directory.h" diff --git a/clients/drcachesim/tests/burst_replaceall.cpp b/clients/drcachesim/tests/burst_replaceall.cpp index 2d704772398..c9e6d975e79 100644 --- a/clients/drcachesim/tests/burst_replaceall.cpp +++ b/clients/drcachesim/tests/burst_replaceall.cpp @@ -39,7 +39,7 @@ * for us. */ #include "dr_api.h" -#include "drmemtrace.h" +#include "drmemtrace/drmemtrace.h" #include "drvector.h" #include "../../../suite/tests/client_tools.h" #include diff --git a/clients/drcachesim/tools/basic_counts_create.h b/clients/drcachesim/tools/basic_counts_create.h index 7c05d83228b..1af41004207 100644 --- a/clients/drcachesim/tools/basic_counts_create.h +++ b/clients/drcachesim/tools/basic_counts_create.h @@ -37,6 +37,15 @@ #include "analysis_tool.h" +/** + * @file drmemtrace/basic_counts_create.h + * @brief DrMemtrace basic counting trace analysis tool creation. + */ + +/** + * Creates an analysis tool which counts the number of instructions, loads, stores, + * prefetch, threads, and markers in the trace. + */ analysis_tool_t * basic_counts_tool_create(unsigned int verbose = 0); diff --git a/clients/drcachesim/tools/histogram_create.h b/clients/drcachesim/tools/histogram_create.h index 2c290169e8e..4f1b66c6efb 100644 --- a/clients/drcachesim/tools/histogram_create.h +++ b/clients/drcachesim/tools/histogram_create.h @@ -37,6 +37,15 @@ #include "analysis_tool.h" +/** + * @file drmemtrace/histogram_create.h + * @brief DrMemtrace tool that computes the most-referenced cache lines. + */ + +/** + * Creates an analysis tool which computes the most-referenced cache lines. + * The options are currently documented in \ref sec_drcachesim_ops. + */ // These options are currently documented in ../common/options.cpp. analysis_tool_t * histogram_tool_create(unsigned int line_size = 64, diff --git a/clients/drcachesim/tools/reuse_distance_create.h b/clients/drcachesim/tools/reuse_distance_create.h index e22c2eb54f9..6ce0ffb363d 100644 --- a/clients/drcachesim/tools/reuse_distance_create.h +++ b/clients/drcachesim/tools/reuse_distance_create.h @@ -37,6 +37,15 @@ #include "analysis_tool.h" +/** + * @file drmemtrace/reuse_distance_create.h + * @brief DrMemtrace reuse distance tool creation. + */ + +/** + * Creates an analysis tool which computes reuse distance. + * The options are currently documented in \ref sec_drcachesim_ops. + */ // These options are currently documented in ../common/options.cpp. analysis_tool_t * reuse_distance_tool_create(unsigned int line_size = 64, diff --git a/clients/drcachesim/tools/reuse_time_create.h b/clients/drcachesim/tools/reuse_time_create.h index 9a4a9a11882..dfd8463bab5 100644 --- a/clients/drcachesim/tools/reuse_time_create.h +++ b/clients/drcachesim/tools/reuse_time_create.h @@ -37,6 +37,17 @@ #include "analysis_tool.h" +/** + * @file drmemtrace/reuse_time_create.h + * @brief DrMemtrace reuse time (i.e., reuse distance without regard to + * uniqueness) tool creation. + */ + +/** + * Creates an analysis tool which computes reuse time (i.e., reuse + * distance without regard to uniqueness). The options are currently + * documented in \ref sec_drcachesim_ops. + */ // These options are currently documented in ../common/options.cpp. analysis_tool_t * reuse_time_tool_create(unsigned int line_size = 64, unsigned int verbose = 0); diff --git a/clients/drcachesim/tracer/drmemtrace.h b/clients/drcachesim/tracer/drmemtrace.h index df37a348124..18d524e7247 100644 --- a/clients/drcachesim/tracer/drmemtrace.h +++ b/clients/drcachesim/tracer/drmemtrace.h @@ -1,5 +1,5 @@ /* ********************************************************** - * (c) 2016 Google, Inc. All rights reserved. + * Copyright (c) 2016-2017 Google, Inc. All rights reserved. * **********************************************************/ /* @@ -41,13 +41,14 @@ /** * @file drmemtrace.h - * @brief Header for DynamoRIO Tracer Library + * @brief Header for customizing the DrMemtrace tracer. */ #ifdef __cplusplus extern "C" { #endif +/** Status return values from drmemtrace functions. */ typedef enum { DRMEMTRACE_SUCCESS, /**< Operation succeeded. */ DRMEMTRACE_ERROR, /**< Operation failed. */ diff --git a/clients/drcachesim/tracer/raw2trace.h b/clients/drcachesim/tracer/raw2trace.h index d3fad3bd3b1..cb2342e6c8a 100644 --- a/clients/drcachesim/tracer/raw2trace.h +++ b/clients/drcachesim/tracer/raw2trace.h @@ -36,6 +36,11 @@ #ifndef _RAW2TRACE_H_ #define _RAW2TRACE_H_ 1 +/** + * @file drmemtrace/raw2trace.h + * @brief DrMemtrace offline trace post-processing customization. + */ + #include "dr_api.h" #include "drmemtrace.h" #include "drcovlib.h" @@ -61,6 +66,11 @@ struct module_t { bool is_external; // If true, the data is embedded in drmodtrack custom fields. }; +/** + * The raw2trace class converts the raw offline trace format to the format + * expected by analysis tools. It requires access to the binary files for the + * libraries and executable that were present during tracing. + */ class raw2trace_t { public: // module_map, thread_files and out_file are all owned and opened/closed by the diff --git a/make/DynamoRIOConfig.cmake.in b/make/DynamoRIOConfig.cmake.in index 77de0996447..1c07fb230e8 100755 --- a/make/DynamoRIOConfig.cmake.in +++ b/make/DynamoRIOConfig.cmake.in @@ -113,6 +113,10 @@ # use_DynamoRIO_drmemtrace(myapp) # target_link_libraries(myapp drmemtrace_analyzer drmemtrace_reuse_distance) # +# To customize the tracer and add drmemtrace.h to the include path: +# +# use_DynamoRIO_drmemtrace_tracer(myapp) +# ########### # # Global changes: @@ -1230,6 +1234,16 @@ function (use_DynamoRIO_drmemtrace target) _DR_append_property_string(TARGET ${target} COMPILE_FLAGS "-std=c++11") endfunction () +function (use_DynamoRIO_drmemtrace_tracer target) + # The tracer header is in the same place as the analysis tool headers. + if (EXISTS ${DynamoRIO_cwd}/../tools/include) + include_directories(${DynamoRIO_cwd}/../tools/include) + else () + # It's called "clients/" in build dirs. + include_directories(${DynamoRIO_cwd}/../clients/include) + endif () +endfunction () + # Support co-located DRMF without having to separately specify it if (NOT DrMemoryFramework_DIR AND EXISTS "${DynamoRIO_DIR}/../drmemory/drmf") set(DrMemoryFramework_DIR "${DynamoRIO_DIR}/../drmemory/drmf") diff --git a/make/utils.cmake b/make/utils.cmake index 3e9e177bbb4..0105ea63ae3 100644 --- a/make/utils.cmake +++ b/make/utils.cmake @@ -1,5 +1,5 @@ # ********************************************************** -# Copyright (c) 2012-2016 Google, Inc. All rights reserved. +# Copyright (c) 2012-2017 Google, Inc. All rights reserved. # ********************************************************** # # Redistribution and use in source and binary forms, with or without @@ -120,7 +120,7 @@ macro (add_dr_defines) endforeach (config) endmacro (add_dr_defines) -function (install_subdirs tgt_lib tgt_bin) +macro (install_subdirs tgt_lib tgt_bin) # These cover all subdirs. # Subdirs just need to install their targets. DR_install(DIRECTORY ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/ @@ -131,21 +131,23 @@ function (install_subdirs tgt_lib tgt_bin) PATTERN "*.debug" PATTERN "*.pdb" REGEX ".*.dSYM/.*DWARF/.*" # too painful to get right # of backslash for literal . - ) - # We rely on our shared library targets being redirected to - # CMAKE_LIBRARY_OUTPUT_DIRECTORY in order to copy the shared lib pdbs - # and executable pdbs into the right places. Callers can use - # place_shared_lib_in_lib_dir() to accomplish this. - DR_install(DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ - DESTINATION ${tgt_bin} - FILE_PERMISSIONS ${owner_access} OWNER_EXECUTE GROUP_READ GROUP_EXECUTE - WORLD_READ WORLD_EXECUTE - FILES_MATCHING - PATTERN "*.debug" - PATTERN "*.pdb" - REGEX ".*.dSYM/.*DWARF/.*" # too painful to get right # of backslash for literal . - ) -endfunction (install_subdirs) + ${ARGN} + ) + # We rely on our shared library targets being redirected to + # CMAKE_LIBRARY_OUTPUT_DIRECTORY in order to copy the shared lib pdbs + # and executable pdbs into the right places. Callers can use + # place_shared_lib_in_lib_dir() to accomplish this. + DR_install(DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ + DESTINATION ${tgt_bin} + FILE_PERMISSIONS ${owner_access} OWNER_EXECUTE GROUP_READ GROUP_EXECUTE + WORLD_READ WORLD_EXECUTE + FILES_MATCHING + PATTERN "*.debug" + PATTERN "*.pdb" + REGEX ".*.dSYM/.*DWARF/.*" # too painful to get right # of backslash for literal . + ${ARGN} + ) +endmacro (install_subdirs) # Use this to put shared libraries in the lib dir to separate them from # executables in the output dir.