i#5538 memtrace seek, part 7: Add instr count to tools

Adds a new memref_stream_t interface class which provides the record
and instruction count to drmemtrace analysis tools.  A pointer to this
interface is passed to new extended-argument versions of
analysis_tool_t's initialize() and parallel_shard_init() functions,
which are now what is called by the analyzer.  The base class
implementation of these new functions simply calls the old versions,
which are now deprecated but will continue to work.

This new interface is not just for convenience: the tool itself cannot
accurately count when the reader skips over records, as will happen
with seeking.  The counting must be done in the reader.  (If the tool
indeed wants to count only records/instrs that it actually sees, it
can continue using its own counters.)

Updates the view tool to use the new interface to obtain the record
ordinal, replacing its own counter.  The tool is expanded to print a
new column with the instruction ordinal.  The view tool test is
updated along with example output in the docs.

Issue: #5538

clients/drcachesim/CMakeLists.txt

@@ -250,6 +250,7 @@ link_with_pthread(drmemtrace_analyzer)
 install_client_nonDR_header(drmemtrace common/utils.h)
 install_client_nonDR_header(drmemtrace common/trace_entry.h)
 install_client_nonDR_header(drmemtrace common/memref.h)
+install_client_nonDR_header(drmemtrace common/memref_stream.h)
 install_client_nonDR_header(drmemtrace reader/reader.h)
 install_client_nonDR_header(drmemtrace analysis_tool.h)
 install_client_nonDR_header(drmemtrace analyzer.h)

clients/drcachesim/analysis_tool.h

@@ -44,6 +44,7 @@
 // To support installation of headers for analysis tools into a single
 // separate directory we omit common/ here and rely on -I.
 #include "memref.h"
+#include "memref_stream.h"
 #include <string>
 
 /**
@@ -93,17 +94,31 @@ class analysis_tool_t {
         : success_(true) {};
     virtual ~analysis_tool_t() {}; /**< Destructor. */
     /**
-     * Tools are encouraged to perform any initialization that might fail here rather
-     * than in the constructor.  On an error, this returns an error string.  On success,
-     * it returns "".
+     * \deprecated The version taking in a #memref_stream_t is the version
+     * called by the analyzer; this no-argument version is only called if the default
+     * implementation of the other version is left in place and it calls this version.  On
+     * an error, this returns an error string.  On success, it returns "".
      */
     virtual std::string
     initialize()
     {
         return "";
     }
+    /**
+     * Tools are encouraged to perform any initialization that might fail here rather
+     * than in the constructor.  The \p serial_query interface allows tools to query
+     * details of the underlying trace during serial operation; it is nullptr for
+     * parallel operation (a per-shard version is passed to parallel_shard_init()).  On
+     * an error, this returns an error string.  On success, it returns "".
+     */
+    virtual std::string
+    initialize(memref_stream_t *serial_query)
+    {
+        return initialize();
+    }
     /** Returns whether the tool was created successfully. */
-    virtual bool operator!()
+    virtual bool
+    operator!()
     {
         return !success_;
     }
@@ -136,7 +151,7 @@ class analysis_tool_t {
     /**
      * Returns whether this tool supports analyzing trace shards concurrently, or
      * whether it needs to see a single thread-interleaved stream of traced
-     * events.
+     * events.  This may be called prior to initialize().
      */
     virtual bool
     parallel_shard_supported()
@@ -168,20 +183,31 @@ class analysis_tool_t {
         return "";
     }
     /**
-     * Invoked once for each trace shard prior to calling parallel_shard_memref() for
-     * that shard, this allows a tool to create data local to a shard.  The \p
-     * shard_index is a unique identifier allowing shard data to be stored into a
-     * global table if desired (typically for aggregation use in print_results()).
-     * The \p worker_data is the return value of parallel_worker_init() for the
-     * worker thread who will exclusively operate on this shard.  The return value
-     * here will be passed to each invocation of parallel_shard_memref() for that
-     * same shard.
+     * \deprecated The version with a 3rd parameter in a #memref_stream_t is the version
+     * called by the analyzer; this 2-argument version is only called if the default
+     * implementation of the other version is left in place and it calls this version.
      */
     virtual void *
     parallel_shard_init(int shard_index, void *worker_data)
     {
         return nullptr;
     }
+    /**
+     * Invoked once for each trace shard prior to calling parallel_shard_memref() for
+     * that shard, this allows a tool to create data local to a shard.  The \p
+     * shard_index is a unique identifier allowing shard data to be stored into a global
+     * table if desired (typically for aggregation use in print_results()).  The \p
+     * worker_data is the return value of parallel_worker_init() for the worker thread
+     * who will exclusively operate on this shard.  The \p shard_query allows tools to
+     * query details of the underlying trace shard during parallel operation; it is
+     * valid only until parallel_shard_exit() is called.  The return value here will be
+     * passed to each invocation of parallel_shard_memref() for that same shard.
+     */
+    virtual void *
+    parallel_shard_init(int shard_index, void *worker_data, memref_stream_t *shard_query)
+    {
+        return parallel_shard_init(shard_index, worker_data);
+    }
     /**
      * Invoked once when all trace entries for a shard have been processed.  \p
      * shard_data is the value returned by parallel_shard_init() for this shard.

clients/drcachesim/analyzer.cpp

@@ -45,6 +45,7 @@
 #    include "reader/snappy_file_reader.h"
 #endif
 #include "common/utils.h"
+#include "memref_stream.h"
 
 #ifdef HAS_ZLIB
 // Even if the file is uncompressed, zlib's gzip interface is faster than
@@ -182,6 +183,11 @@ analyzer_t::analyzer_t(const std::string &trace_path, analysis_tool_t **tools,
     , parallel_(true)
     , worker_count_(worker_count)
 {
+    if (!init_file_reader(trace_path)) {
+        success_ = false;
+        error_string_ = "Failed to create reader";
+        return;
+    }
     for (int i = 0; i < num_tools; ++i) {
         if (tools_[i] == NULL || !*tools_[i]) {
             success_ = false;
@@ -190,15 +196,13 @@ analyzer_t::analyzer_t(const std::string &trace_path, analysis_tool_t **tools,
                 error_string_ += ": " + tools_[i]->get_error_string();
             return;
         }
-        const std::string error = tools_[i]->initialize();
+        const std::string error = tools_[i]->initialize(serial_trace_iter_.get());
         if (!error.empty()) {
             success_ = false;
             error_string_ = "Tool failed to initialize: " + error;
             return;
         }
     }
-    if (!init_file_reader(trace_path))
-        success_ = false;
 }
 
 analyzer_t::analyzer_t(const std::string &trace_path)
@@ -264,8 +268,10 @@ analyzer_t::process_tasks(std::vector<analyzer_shard_data_t *> *tasks)
             return;
         }
         std::vector<void *> shard_data(num_tools_);
-        for (int i = 0; i < num_tools_; ++i)
-            shard_data[i] = tools_[i]->parallel_shard_init(tdata->index, worker_data[i]);
+        for (int i = 0; i < num_tools_; ++i) {
+            shard_data[i] = tools_[i]->parallel_shard_init(tdata->index, worker_data[i],
+                                                           tdata->iter.get());
+        }
         VPRINT(this, 1, "shard_data[0] is %p\n", shard_data[0]);
         for (; *tdata->iter != *trace_end_; ++(*tdata->iter)) {
             for (int i = 0; i < num_tools_; ++i) {

clients/drcachesim/analyzer_multi.cpp

@@ -151,6 +151,10 @@ analyzer_multi_t::analyzer_multi_t()
         if (!init_file_reader(op_infile.get_value(), op_verbose.get_value()))
             success_ = false;
     }
+    if (!init_analysis_tools()) {
+        success_ = false;
+        return;
+    }
     // We can't call serial_trace_iter_->init() here as it blocks for ipc_reader_t.
 }
 
@@ -170,14 +174,10 @@ analyzer_multi_t::create_analysis_tools()
     tools_[0] = drmemtrace_analysis_tool_create();
     if (tools_[0] == NULL)
         return false;
-    std::string tool_error;
     if (!*tools_[0]) {
-        tool_error = tools_[0]->get_error_string();
+        std::string tool_error = tools_[0]->get_error_string();
         if (tool_error.empty())
             tool_error = "no error message provided.";
-    } else
-        tool_error = tools_[0]->initialize();
-    if (!tool_error.empty()) {
         error_string_ = "Tool failed to initialize: " + tool_error;
         delete tools_[0];
         tools_[0] = NULL;
@@ -229,8 +229,6 @@ analyzer_multi_t::create_analysis_tools()
             serial_schedule_file_.get(), cpu_schedule_file_.get());
         if (tools_[1] == NULL)
             return false;
-        if (!!*tools_[1])
-            tools_[1]->initialize();
         if (!*tools_[1]) {
             error_string_ = tools_[1]->get_error_string();
             delete tools_[1];
@@ -242,6 +240,28 @@ analyzer_multi_t::create_analysis_tools()
     return true;
 }
 
+bool
+analyzer_multi_t::init_analysis_tools()
+{
+    std::string tool_error = tools_[0]->initialize(serial_trace_iter_.get());
+    if (!tool_error.empty()) {
+        error_string_ = "Tool failed to initialize: " + tool_error;
+        delete tools_[0];
+        tools_[0] = NULL;
+        return false;
+    }
+    if (op_test_mode.get_value()) {
+        tools_[1]->initialize(serial_trace_iter_.get());
+        if (!*tools_[1]) {
+            error_string_ = tools_[1]->get_error_string();
+            delete tools_[1];
+            tools_[1] = NULL;
+            return false;
+        }
+    }
+    return true;
+}
+
 void
 analyzer_multi_t::destroy_analysis_tools()
 {

clients/drcachesim/analyzer_multi.h

@@ -49,6 +49,8 @@ class analyzer_multi_t : public analyzer_t {
 protected:
     bool
     create_analysis_tools();
+    bool
+    init_analysis_tools();
     void
     destroy_analysis_tools();
 

clients/drcachesim/common/memref_stream.h

@@ -0,0 +1,69 @@
+/* **********************************************************
+ * Copyright (c) 2022 Google, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Google, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* memref_stream: represent a memory trace analysis tool.
+ */
+
+#ifndef _MEMREF_STREAM_H_
+#define _MEMREF_STREAM_H_ 1
+
+/**
+ * @file drmemtrace/memref_stream.h
+ * @brief DrMemtrace interface for obtaining information from analysis
+ * tools on the full stream of memory reference records.
+ */
+
+/**
+ * This is an interface for obtaining information from analysis tools
+ * on the full stream of memory reference records.
+ */
+class memref_stream_t {
+public:
+    /** Destructor. */
+    virtual ~memref_stream_t()
+    {
+    }
+    /**
+     * Returns the count of #memref_t records from the start of the trace to this point.
+     * This includes records skipped over and not presented to any tool.
+     */
+    virtual uint64_t
+    get_record_ordinal() = 0;
+    /**
+     * Returns the count of instructions from the start of the trace to this point.
+     * This includes instructions skipped over and not presented to any tool.
+     */
+    virtual uint64_t
+    get_instruction_ordinal() = 0;
+};
+
+#endif /* _MEMREF_STREAM_H_ */