i#2006 drcachesim generalization: doxygenate drmemtrace framework

api/docs/API.doxy

@@ -44,8 +44,11 @@ REPEAT_BRIEF           = YES
 ABBREVIATE_BRIEF       =
 ALWAYS_DETAILED_SEC    = NO
 INLINE_INHERITED_MEMB  = NO
-FULL_PATH_NAMES        = NO
-STRIP_FROM_PATH        =
+# We want a hierarchy for the File List with the drcontainers headers
+# grouped, and ext/ and drmemtrace/ separated out, so we do relative
+# paths via full plus strip:
+FULL_PATH_NAMES        = YES
+STRIP_FROM_PATH        = ../../clients/include ../../../src
 STRIP_FROM_INC_PATH    =
 SHORT_NAMES            = NO
 JAVADOC_AUTOBRIEF      = NO

api/docs/CMakeLists.txt

@@ -1,5 +1,5 @@
 # **********************************************************
-# Copyright (c) 2010-2015 Google, Inc.    All rights reserved.
+# Copyright (c) 2010-2017 Google, Inc.    All rights reserved.
 # Copyright (c) 2009-2010 VMware, Inc.    All rights reserved.
 # **********************************************************
 
@@ -155,6 +155,7 @@ add_custom_command(
   # to work around i#84 be sure to put a space after -D for 1st arg at least
   ARGS -D srcdir=${CMAKE_CURRENT_SOURCE_DIR} -Doutfile=${doxyfile}
        -Dproj_srcdir=${PROJECT_SOURCE_DIR} -Dversion_number=${VERSION_NUMBER}
+       -Dproj_bindir=${PROJECT_BINARY_DIR}
        -Dheader_dir=${BUILD_INCLUDE} -Dgendox_dir=${CMAKE_CURRENT_BINARY_DIR}
        -DDOXYGEN_EXECUTABLE=${DOXYGEN_EXECUTABLE}
        -Ddoxygen_ver=${doxygen_ver}

api/docs/CMake_doxyfile.cmake

@@ -1,5 +1,5 @@
 # **********************************************************
-# Copyright (c) 2012-2015 Google, Inc.    All rights reserved.
+# Copyright (c) 2012-2017 Google, Inc.    All rights reserved.
 # Copyright (c) 2009-2010 VMware, Inc.    All rights reserved.
 # **********************************************************
 
@@ -33,6 +33,7 @@
 # * srcdir
 # * outfile
 # * proj_srcdir
+# * proj_bindir
 # * version_number
 # * header_dir
 # * gendox_dir
@@ -51,6 +52,9 @@ foreach (dir ${dirs})
   set(ext_dirs ${ext_dirs} ${dir})
 endforeach (dir)
 
+# Add drcachesim dirs (i#2006).
+set(ext_dirs ${ext_dirs} "${proj_bindir}/clients/include/drmemtrace")
+
 include("${srcdir}/CMake_doxyutils.cmake")
 set(input_paths srcdir proj_srcdir header_dir gendox_dir outdir)
 doxygen_path_xform(${DOXYGEN_EXECUTABLE} "${input_paths}")

api/docs/release.dox

@@ -183,7 +183,8 @@ Further non-compatibility-affecting changes include:
    drmemtrace_get_modlist_path(), and a separate rawtrace library for
    post-processing customization with raw2trace_t::handle_custom_data(),
    raw2trace_t::do_module_parsing(), raw2trace_t::do_conversion(), and
-   raw2trace_directory_t.
+   raw2trace_directory_t.  A corresponding CMake function for finding the
+   tracer customization header is use_DynamoRIO_drmemtrace_tracer().
  - Added a set_value() function to the \ref page_droption.
  - Added instrlist_get_auto_predicate() and instrlist_set_auto_predicate().
  - Globally enabled auto predication in the drmgr instrumentation insertion event by

clients/CMakeLists.txt

@@ -91,7 +91,11 @@ foreach (dir ${dirs})
 endforeach (dir)
 
 # install subdirs
-install_subdirs(${INSTALL_CLIENTS_LIB} ${INSTALL_CLIENTS_BIN})
+install_subdirs(${INSTALL_CLIENTS_LIB} ${INSTALL_CLIENTS_BIN}
+  PATTERN "histogram_launcher.debug" EXCLUDE
+  PATTERN "drcachesim_ops.debug" EXCLUDE
+  PATTERN "drcpusim_ops.debug" EXCLUDE
+  )
 
 # propagate to parent dir
 set(exported_targets_append "${exported_targets_append}" PARENT_SCOPE)

clients/drcachesim/CMakeLists.txt

@@ -152,6 +152,7 @@ install_client_nonDR_header(drmemtrace tools/reuse_time_create.h)
 install_client_nonDR_header(drmemtrace tools/basic_counts_create.h)
 install_client_nonDR_header(drmemtrace simulator/cache_simulator_create.h)
 install_client_nonDR_header(drmemtrace simulator/tlb_simulator_create.h)
+install_client_nonDR_header(drmemtrace tracer/raw2trace.h)
 
 # We show one example of how to create a standalone analyzer of trace
 # files that does not need to link with DR.
@@ -203,11 +204,9 @@ endmacro()
 
 add_drmemtrace(drmemtrace SHARED)
 add_drmemtrace(drmemtrace_static STATIC)
-# XXX i#2006: should we export drmemtrace.h into tools/include/drmemtace/tracer.h
-# with the analysis framework headers, or keep it where it is in ext/include?  For
-# now I'm keeping it looking like an extension as it's for client use, not separate
-# analysis tool use.
-install_client_DR_header(tracer/drmemtrace.h)
+# We export drmemtrace.h to the same place as the analysis tool headers
+# for simplicity, rather than sticking it into ext/include or sthg.
+install_client_nonDR_header(drmemtrace tracer/drmemtrace.h)
 
 add_executable(drraw2trace
   tracer/raw2trace_launcher.cpp
@@ -311,6 +310,7 @@ if (NOT DynamoRIO_INTERNAL OR NOT "${CMAKE_GENERATOR}" MATCHES "Ninja")
 endif ()
 
 install_target(drcachesim ${INSTALL_CLIENTS_BIN})
+install_target(drraw2trace ${INSTALL_CLIENTS_BIN})
 
 set(INSTALL_DRCACHESIM_CONFIG ${INSTALL_CLIENTS_BASE})
 
@@ -401,18 +401,15 @@ if (BUILD_TESTS)
       endif ()
     endif ()
     add_win32_flags(tool.drcacheoff.burst_replace)
-    # Not really using an extension, just for including drmemtrace.h, which we
-    # don't want to include in use_DynamoRIO_drmemtrace().
-    use_DynamoRIO_extension(tool.drcacheoff.burst_replace drmemtrace_static)
+    use_DynamoRIO_drmemtrace_tracer(tool.drcacheoff.burst_replace)
     use_DynamoRIO_extension(tool.drcacheoff.burst_replace drcovlib_static)
 
     add_executable(tool.drcacheoff.burst_replaceall tests/burst_replaceall.cpp)
     configure_DynamoRIO_static(tool.drcacheoff.burst_replaceall)
     use_DynamoRIO_static_client(tool.drcacheoff.burst_replaceall drmemtrace_static)
     add_win32_flags(tool.drcacheoff.burst_replaceall)
     use_DynamoRIO_extension(tool.drcacheoff.burst_replaceall drcontainers)
-    # See above.
-    use_DynamoRIO_extension(tool.drcacheoff.burst_replaceall drmemtrace_static)
+    use_DynamoRIO_drmemtrace_tracer(tool.drcacheoff.burst_replaceall)
 
     add_executable(tool.drcacheoff.burst_threads tests/burst_threads.cpp)
     configure_DynamoRIO_static(tool.drcacheoff.burst_threads)

clients/drcachesim/analysis_tool.h

@@ -36,19 +36,42 @@
 #ifndef _ANALYSIS_TOOL_H_
 #define _ANALYSIS_TOOL_H_ 1
 
+/**
+ * @file drmemtrace/analysis_tool.h
+ * @brief DrMemtrace analysis tool base class.
+ */
+
 // To support installation of headers for analysis tools into a single
 // separate directory we omit common/ here and rely on -I.
 #include "memref.h"
 
+/**
+ * The base class for a tool that analyzes a trace.  A new tool should subclass this
+ * class.  Two key functions should be overridden: process_memref(), which operates
+ * on a single trace entry, and print_results(), which is called just once after
+ * processing the entire trace and should present the results of analysis.  In the
+ * default mode of operation, the #analyzer_t class iterates over the trace and calls
+ * the process_memref() function of each tool.  An alternative mode is supported
+ * which exposes the iterator and allows a separate control infrastructure to be
+ * built.
+ */
 class analysis_tool_t
 {
  public:
-    // Usage: errors encountered during the constructor will set a flag that should
-    // be queried via operator!.
+    /**
+     * Errors encountered during the constructor will set the success flag, which should
+     * be queried via operator!.
+     */
     analysis_tool_t() : success(true) {};
-    virtual ~analysis_tool_t() {};
+    virtual ~analysis_tool_t() {}; /**< Destructor. */
+    /** Returns whether the tool was created successfully. */
     virtual bool operator!() { return !success; }
+    /**
+     * The heart of an analysis tool, this routine operates on a single trace entry and
+     * takes whatever actions the tool needs to perform its analysis.
+     */
     virtual bool process_memref(const memref_t &memref) = 0;
+    /** This routine reports the results of the trace analysis. */
     virtual bool print_results() = 0;
  protected:
     bool success;

clients/drcachesim/analyzer.h

@@ -38,44 +38,65 @@
 #ifndef _ANALYZER_H_
 #define _ANALYZER_H_ 1
 
+/**
+ * @file drmemtrace/analyzer.h
+ * @brief DrMemtrace top-level trace analysis driver.
+ */
+
 #include <iterator>
 #include <string>
 #include "analysis_tool.h"
 #include "reader.h"
 
+/**
+ * An analyzer is the top-level driver of a set of trace analysis tools.
+ * It supports two different modes of operation: either it iterates over the
+ * trace and calls the process_memref() routine of each tool, or it exposes
+ * an iteration interface to external control code.
+ */
 class analyzer_t
 {
  public:
-    // Usage: errors encountered during a constructor will set a flag that should
-    // be queried via operator!.  If operator! returns true, get_error_string()
-    // can be used to try to obtain more information.
+    /**
+     * Usage: errors encountered during a constructor will set a flag that should
+     * be queried via operator!().  If operator!() returns true, get_error_string()
+     * can be used to try to obtain more information.
+     */
     analyzer_t();
-    virtual ~analyzer_t();
+    virtual ~analyzer_t(); /**< Destructor. */
+    /** Returns whether the analyzer was created successfully. */
     virtual bool operator!();
+    /** Returns further information on an error in initializing the analyzer. */
     virtual std::string get_error_string();
 
-    // We have two usage models: one where there are multiple tools and the
-    // trace iteration is performed by analyzer_t, and another where a single
-    // tool controls the iteration.
-
-    // The default, simpler, multiple-tool-supporting model uses this constructor.
-    // The analyzer will reference the tools array passed in during its lifetime:
-    // it does not make a copy.
-    // The user must free them afterward.
+    /**
+     * We have two usage models: one where there are multiple tools and the
+     * trace iteration is performed by analyzer_t, and another where a single
+     * tool controls the iteration.
+     *
+     * The default, simpler, multiple-tool-supporting model uses this constructor.
+     * The analyzer will reference the tools array passed in during its lifetime:
+     * it does not make a copy.
+     * The user must free them afterward.
+     */
     analyzer_t(const std::string &trace_file, analysis_tool_t **tools,
                int num_tools);
+    /** Launches the analysis process. */
     virtual bool run();
+    /** Presents the results of the analysis. */
     virtual bool print_stats();
 
-    // The alternate usage model exposes the iterator to a single tool.
+    /** The alternate usage model exposes the iterator to a single tool. */
     analyzer_t(const std::string &trace_file);
-    // As the iterator is more heavyweight than regular container iterators
-    // we hold it internally and return a reference.  Copying will fail to compile
-    // as reader_t is virtual, reminding the caller of begin() to use a reference.
-    // This usage model supports only a single user of the iterator: the
-    // multi-tool model above should be used if multiple tools are involved.
+    /**
+     * As the iterator is more heavyweight than regular container iterators
+     * we hold it internally and return a reference.  Copying will fail to compile
+     * as reader_t is virtual, reminding the caller of begin() to use a reference.
+     * This usage model supports only a single user of the iterator: the
+     * multi-tool model above should be used if multiple tools are involved.
+     */
     virtual reader_t & begin();
-    virtual reader_t & end();
+    virtual reader_t & end(); /** End iterator for the external-iterator usage model. */
 
  protected:
     bool init_file_reader(const std::string &trace_file);

clients/drcachesim/common/memref.h

@@ -39,76 +39,88 @@
 #include <stddef.h> // for size_t
 #include "trace_entry.h"
 
+/**
+ * @file drmemtrace/memref.h
+ * @brief DrMemtrace trace entry structures.
+ */
+
 // On some platforms, like MacOS, a thread id is 64 bits.
 // We just make both 64 bits to cover all our bases.
-typedef int_least64_t memref_pid_t;
-typedef int_least64_t memref_tid_t;
-
-// Each trace entry is one of the following.
-// Although the pc of each data reference is provided, the trace also guarantees that
-// an instruction entry immediately precedes the data references that it is
-// responsible for, with no intervening trace entries.
-// Offline traces further guarantee that an instruction entry for a branch
-// instruction is always followed by an instruction entry for the branch's
-// target (with any memory references for the branch in between of course)
-// without a thread switch intervening, to make it simpler to identify branch
-// targets.  Online traces do not currently guarantee this.
+typedef int_least64_t memref_pid_t; /**< Process id type. */
+typedef int_least64_t memref_tid_t; /**< Thread id type. */
 
+/** A trace entry representing a data load, store, or prefetch. */
 struct _memref_data_t {
-    // TRACE_TYPE_READ, TRACE_TYPE_WRITE, and TRACE_TYPE_PREFETCH*:
-    // data references.
-    trace_type_t type;
-    memref_pid_t pid;
-    memref_tid_t tid;
-    addr_t addr;
-    size_t size;
-    addr_t pc;
+    trace_type_t type; /**< #TRACE_TYPE_READ, #TRACE_TYPE_WRITE, or type_is_prefetch(). */
+    memref_pid_t pid;  /**< Process id. */
+    memref_tid_t tid;  /**< Thread id. */
+    addr_t addr;       /**< Address of data being loaded or stored. */
+    size_t size;       /**< Size of data being loaded or stored. */
+    addr_t pc;         /**< Program counter of instruction performing load or store. */
 };
 
+/** A trace entry representing an instruction fetch. */
 struct _memref_instr_t {
-    // TRACE_TYPE_INSTR_* (minus BUNDLE): instruction fetch.
-    trace_type_t type;
-    memref_pid_t pid;
-    memref_tid_t tid;
-    addr_t addr;
-    size_t size;
+    trace_type_t type; /**< Matches type_is_instr() or #TRACE_TYPE_INSTR_NO_FETCH. */
+    memref_pid_t pid;  /**< Process id. */
+    memref_tid_t tid;  /**< Thread id. */
+    addr_t addr;       /**< The address of the instruction (i.e., program counter). */
+    size_t size;       /**< The length of the instruction. */
 };
 
+/** A trace entry representing a software-requested explicit cache flush. */
 struct _memref_flush_t {
-    // TRACE_TYPE_INSTR_FLUSH, TRACE_TYPE_DATA_FLUSH: explicit cache flush.
-    trace_type_t type;
-    memref_pid_t pid;
-    memref_tid_t tid;
-    addr_t addr;
-    size_t size;
-    addr_t pc;
+    trace_type_t type; /**< #TRACE_TYPE_INSTR_FLUSH or #TRACE_TYPE_DATA_FLUSH. */
+    memref_pid_t pid;  /**< Process id. */
+    memref_tid_t tid;  /**< Thread id. */
+    addr_t addr;       /**< The start address of the region being flushed. */
+    size_t size;       /**< The size of the region being flushed. */
+    addr_t pc;         /**< Program counter of the instruction requesting the flush. */
 };
 
+/** A trace entry representing a thread exit. */
 struct _memref_thread_exit_t {
-    // TRACE_TYPE_THREAD_EXIT.
-    trace_type_t type;
-    memref_pid_t pid;
-    memref_tid_t tid;
+    trace_type_t type; /**< #TRACE_TYPE_THREAD_EXIT. */
+    memref_pid_t pid;  /**< Process id. */
+    memref_tid_t tid;  /**< Thread id. */
 };
 
+/**
+ * A trace entry containing metadata identifying some event that occurred at this
+ * point in the trace.  The most common marker type represents a kernel-mediated
+ * control flow change such as a signal delivery, entry into an APC, callback, or
+ * exception dispatcher on Windows, or a system call that changes the context such as
+ * a signal return.
+ */
 struct _memref_marker_t {
-    // TRACE_TYPE_MARKER.
-    trace_type_t type;
-    memref_pid_t pid;
-    memref_tid_t tid;
-    trace_marker_type_t marker_type;
-    uintptr_t marker_value;
+    trace_type_t type; /**< #TRACE_TYPE_MARKER. */
+    memref_pid_t pid;  /**< Process id. */
+    memref_tid_t tid;  /**< Thread id. */
+    trace_marker_type_t marker_type; /**< Identifies the type of marker. */
+    uintptr_t marker_value; /**< A value whose meaning depends on the marker type. */
 };
 
+/**
+ * Each trace entry is one of the structures in this union.
+ * Each entry identifies the originating process and thread.
+ * Although the pc of each data reference is provided, the trace also guarantees that
+ * an instruction entry immediately precedes the data references that it is
+ * responsible for, with no intervening trace entries.
+ * Offline traces further guarantee that an instruction entry for a branch
+ * instruction is always followed by an instruction entry for the branch's
+ * target (with any memory references for the branch in between of course)
+ * without a thread switch intervening, to make it simpler to identify branch
+ * targets.  Online traces do not currently guarantee this.
+ */
 typedef union _memref_t {
     // The C standard allows us to reference the type field of any of these, and the
     // addr and size fields of data, instr, or flush generically if known to be one
     // of those types, due to the shared fields in our union of structs.
-    struct _memref_data_t data;
-    struct _memref_instr_t instr;
-    struct _memref_flush_t flush;
-    struct _memref_thread_exit_t exit;
-    struct _memref_marker_t marker;
+    struct _memref_data_t data;         /**< A data load or store. */
+    struct _memref_instr_t instr;       /**< An insruction fetch. */
+    struct _memref_flush_t flush;       /**< A software-initiated cache flush. */
+    struct _memref_thread_exit_t exit;  /**< A thread exit. */
+    struct _memref_marker_t marker;     /**< A marker holding metadata. */
 } memref_t;
 
 #endif /* _MEMREF_H_ */