i#2062 memtrace nonmod part 2: Raw2trace parsing of encoding file

api/docs/release.dox

@@ -165,6 +165,7 @@ Further non-compatibility-affecting changes include:
    and available only on Intel processors that support the Intel@ Processor Trace
    feature.
  - Added drmemtrace_get_encoding_path().
+ - Added preliminary support for generated code to drmemtrace.
 
 The changes between version 9.0.1 and 9.0.0 include the following compatibility
 changes:

clients/drcachesim/analyzer_multi.cpp

@@ -106,8 +106,8 @@ analyzer_multi_t::analyzer_multi_t()
                 return;
             }
             raw2trace_t raw2trace(dir.modfile_bytes_, dir.in_files_, dir.out_files_,
-                                  nullptr, op_verbose.get_value(), op_jobs.get_value(),
-                                  op_alt_module_dir.get_value());
+                                  dir.encoding_file_, nullptr, op_verbose.get_value(),
+                                  op_jobs.get_value(), op_alt_module_dir.get_value());
             std::string error = raw2trace.do_conversion();
             if (!error.empty()) {
                 success_ = false;

clients/drcachesim/tests/burst_aarch64_sys.cpp

@@ -162,7 +162,7 @@ post_process()
         std::string dir_err = dir.initialize(raw_dir, outdir);
         assert(dir_err.empty());
         raw2trace_t raw2trace(dir.modfile_bytes_, dir.in_files_, dir.out_files_,
-                              dr_context);
+                              dir.encoding_file_, dr_context);
         std::string error = raw2trace.do_conversion();
         if (!error.empty()) {
             std::cerr << "raw2trace failed: " << error << "\n";

clients/drcachesim/tests/burst_gencode.cpp

@@ -39,16 +39,25 @@
 #include "configure.h"
 #include "dr_api.h"
 #include "drmemtrace/drmemtrace.h"
+#include "drmemtrace/raw2trace.h"
+#include "raw2trace_directory.h"
+#include "analyzer.h"
 #include <assert.h>
 #include <fstream>
 #include <iostream>
 #include <string>
-#include "tools.h" // Included after system headers to avoid printf warning.
+#undef ALIGN_FORWARD // Conflicts with drcachesim utils.h.
+#include "tools.h"   // Included after system headers to avoid printf warning.
+
+namespace {
 
 /***************************************************************************
  * Code generation.
  */
 
+// XXX i#2062: Remove this once we have encodings in the final trace.
+static app_pc gencode_start;
+
 class code_generator_t {
 public:
     explicit code_generator_t(bool verbose = false)
@@ -66,6 +75,15 @@ class code_generator_t {
         reinterpret_cast<void (*)()>(map_)();
     }
 
+    static constexpr int kGencodeMagic1 = 0x742;
+    static constexpr int kGencodeMagic2 = 0x427;
+
+    app_pc
+    get_gencode_start() const
+    {
+        return map_;
+    }
+
 private:
     byte *map_ = nullptr;
     size_t map_size_ = 0;
@@ -84,9 +102,15 @@ class code_generator_t {
 
         instrlist_t *ilist = instrlist_create(dc);
         reg_id_t base = IF_X86_ELSE(IF_X64_ELSE(DR_REG_RAX, DR_REG_EAX), DR_REG_R0);
+        reg_id_t base4imm = IF_X86_64_ELSE(reg_64_to_32(base), base);
         int ptrsz = static_cast<int>(sizeof(void *));
-        // TODO i#2062: Add more instructions and look for them in the final trace.
-        // For now we are testing that drmemtrace detects generated code.
+        // A two-immediate pattern we look for in the trace.
+        instrlist_append(ilist,
+                         XINST_CREATE_load_int(dc, opnd_create_reg(base4imm),
+                                               OPND_CREATE_INT32(kGencodeMagic1)));
+        instrlist_append(ilist,
+                         XINST_CREATE_load_int(dc, opnd_create_reg(base4imm),
+                                               OPND_CREATE_INT32(kGencodeMagic2)));
         instrlist_append(
             ilist,
             XINST_CREATE_move(dc, opnd_create_reg(base), opnd_create_reg(DR_REG_XSP)));
@@ -119,9 +143,6 @@ class code_generator_t {
  * Top-level tracing.
  */
 
-// TODO i#2062: Run raw2trace and examine the final trace looking for the gencode
-// instrs; use a unique start and end pattern to identify.
-
 static int
 do_some_work(const code_generator_t &gen)
 {
@@ -147,27 +168,107 @@ exit_cb(void *)
     assert(stream.good());
 }
 
-int
-main(int argc, const char *argv[])
+static std::string
+post_process()
+{
+    const char *raw_dir;
+    drmemtrace_status_t mem_res = drmemtrace_get_output_path(&raw_dir);
+    assert(mem_res == DRMEMTRACE_SUCCESS);
+    std::string outdir = std::string(raw_dir) + DIRSEP + "post_processed";
+    void *dr_context = dr_standalone_init();
+    // Use a new scope to free raw2trace_directory_t before dr_standalone_exit().
+    {
+        raw2trace_directory_t dir;
+        if (!dr_create_dir(outdir.c_str())) {
+            std::cerr << "Failed to create output dir";
+            assert(false);
+        }
+        std::string dir_err = dir.initialize(raw_dir, outdir);
+        assert(dir_err.empty());
+        raw2trace_t raw2trace(dir.modfile_bytes_, dir.in_files_, dir.out_files_,
+                              dir.encoding_file_, dr_context);
+        std::string error = raw2trace.do_conversion();
+        if (!error.empty()) {
+            std::cerr << "raw2trace failed: " << error << "\n";
+            assert(false);
+        }
+    }
+    dr_standalone_exit();
+    return outdir;
+}
+
+static std::string
+gather_trace()
 {
     if (!my_setenv("DYNAMORIO_OPTIONS", "-stderr_mask 0xc -client_lib ';;-offline"))
         std::cerr << "failed to set env var!\n";
-
     code_generator_t gen(false);
-    for (int i = 0; i < 3; i++) {
-        std::cerr << "pre-DR init\n";
-        dr_app_setup();
-        assert(!dr_app_running_under_dynamorio());
-        drmemtrace_status_t res = drmemtrace_buffer_handoff(nullptr, exit_cb, nullptr);
-        assert(res == DRMEMTRACE_SUCCESS);
-        std::cerr << "pre-DR start\n";
-        dr_app_start();
-        if (do_some_work(gen) < 0)
-            std::cerr << "error in computation\n";
-        std::cerr << "pre-DR detach\n";
-        dr_app_stop_and_cleanup();
-        std::cerr << "all done\n";
-    }
+    gencode_start = gen.get_gencode_start();
+    std::cerr << "pre-DR init\n";
+    dr_app_setup();
+    assert(!dr_app_running_under_dynamorio());
+    drmemtrace_status_t res = drmemtrace_buffer_handoff(nullptr, exit_cb, nullptr);
+    assert(res == DRMEMTRACE_SUCCESS);
+    std::cerr << "pre-DR start\n";
+    dr_app_start();
+    if (do_some_work(gen) < 0)
+        std::cerr << "error in computation\n";
+    std::cerr << "pre-DR detach\n";
+    dr_app_stop_and_cleanup();
+    std::cerr << "all done\n";
+    return post_process();
+}
 
+static int
+look_for_gencode(std::string trace_dir)
+{
+    void *dr_context = dr_standalone_init();
+    analyzer_t analyzer(trace_dir);
+    if (!analyzer) {
+        std::cerr << "Failed to initialize: " << analyzer.get_error_string() << "\n";
+    }
+    bool found_magic1 = false, found_magic2 = false;
+    bool have_instr_encodings = false; // XXX i#5520: See comment below.
+    for (reader_t &iter = analyzer.begin(); iter != analyzer.end(); ++iter) {
+        memref_t memref = *iter;
+        if (!type_is_instr(memref.instr.type)) {
+            found_magic1 = false;
+            continue;
+        }
+        app_pc pc = (app_pc)memref.instr.addr;
+        // TODO i#5520: Once we have instruction encodings in the final trace, we want
+        // to perform the decoding below and look for the immediates.
+        // Until then, we just look for the gencode PC.
+        if (pc == gencode_start) {
+            found_magic2 = true;
+        }
+        if (have_instr_encodings) {
+            instr_t instr;
+            instr_init(dr_context, &instr);
+            app_pc next_pc = decode(dr_context, pc, &instr);
+            assert(next_pc != nullptr && instr_valid(&instr));
+            ptr_int_t immed;
+            if (!found_magic1 && instr_is_mov_constant(&instr, &immed) &&
+                immed == code_generator_t::kGencodeMagic1)
+                found_magic1 = true;
+            else if (found_magic1 && instr_is_mov_constant(&instr, &immed) &&
+                     immed == code_generator_t::kGencodeMagic2)
+                found_magic2 = true;
+            else
+                found_magic1 = false;
+            instr_free(dr_context, &instr);
+        }
+    }
+    dr_standalone_exit();
+    assert(found_magic2);
     return 0;
 }
+
+} // namespace
+
+int
+main(int argc, const char *argv[])
+{
+    std::string trace_dir = gather_trace();
+    return look_for_gencode(trace_dir);
+}

clients/drcachesim/tests/burst_replace.cpp

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2016-2021 Google, Inc.  All rights reserved.
+ * Copyright (c) 2016-2022 Google, Inc.  All rights reserved.
  * **********************************************************/
 
 /*
@@ -196,7 +196,7 @@ post_process()
             dir.modfile_bytes_, parse_cb, process_cb, MAGIC_VALUE, free_cb);
         assert(module_mapper->get_last_error().empty());
         // Test back-compat of deprecated APIs.
-        raw2trace_t raw2trace(dir.modfile_bytes_, dir.in_files_, dir.out_files_, NULL);
+        raw2trace_t raw2trace(dir.modfile_bytes_, dir.in_files_, dir.out_files_);
         std::string error =
             raw2trace.handle_custom_data(parse_cb, process_cb, MAGIC_VALUE, free_cb);
         assert(error.empty());
@@ -218,8 +218,8 @@ post_process()
     raw2trace_directory_t dir;
     std::string dir_err = dir.initialize(raw_dir, "");
     assert(dir_err.empty());
-    raw2trace_t raw2trace(dir.modfile_bytes_, dir.in_files_, dir.out_files_, dr_context,
-                          0);
+    raw2trace_t raw2trace(dir.modfile_bytes_, dir.in_files_, dir.out_files_,
+                          dir.encoding_file_, dr_context, 0);
     std::string error =
         raw2trace.handle_custom_data(parse_cb, process_cb, MAGIC_VALUE, free_cb);
     assert(error.empty());

clients/drcachesim/tests/burst_traceopts.cpp

@@ -175,7 +175,7 @@ post_process(const std::string &out_subdir)
         std::string dir_err = dir.initialize(raw_dir, outdir);
         assert(dir_err.empty());
         raw2trace_t raw2trace(dir.modfile_bytes_, dir.in_files_, dir.out_files_,
-                              dr_context,
+                              dir.encoding_file_, dr_context,
                               0
 #    ifdef WINDOWS
                               /* FIXME i#3983: Creating threads in standalone mode

clients/drcachesim/tests/offline-gencode.templatex

@@ -2,15 +2,6 @@ pre-DR init
 pre-DR start
 pre-DR detach
 all done
-pre-DR init
-pre-DR start
-pre-DR detach
-all done
-pre-DR init
-pre-DR start
-pre-DR detach
-all done
-\[drmemtrace\]: WARNING: Skipping ifetch for instructions not in a module
 Basic counts tool results:
 Total counts:
 .*

clients/drcachesim/tests/raw2trace_io.cpp

@@ -148,7 +148,7 @@ test_raw2trace(raw2trace_directory_t *dir)
          * raw2trace::read_and_map_modules().
          */
         raw2trace_t raw2trace(dir->modfile_bytes_, dir->in_files_, dir->out_files_,
-                              GLOBAL_DCONTEXT, 1);
+                              dir->encoding_file_, GLOBAL_DCONTEXT, 1);
         std::string error = raw2trace.do_conversion();
         if (!error.empty()) {
             std::cerr << "raw2trace failed " << error << "\n";

clients/drcachesim/tests/raw2trace_unit_tests.cpp

@@ -65,36 +65,44 @@
 #    error Unsupported arch
 #endif
 
-// Subclasses raw2trace_t and replaces the module loading with a buffer
-// of encoded instr_t.
-class raw2trace_test_t : public raw2trace_t {
+// Subclasses raw2trace_t and module_mapper_t and replaces the module loading with
+// a buffer of encoded instr_t.
+class module_mapper_test_t : public module_mapper_t {
 public:
-    raw2trace_test_t(const std::vector<std::istream *> &input,
-                     const std::vector<std::ostream *> &output, instrlist_t &instrs,
-                     void *drcontext)
-        : raw2trace_t(nullptr, input, output, drcontext,
-                      // The sequences are small so we print everything for easier
-                      // debugging and viewing of what's going on.
-                      4)
+    module_mapper_test_t(instrlist_t &instrs, void *drcontext)
+        : module_mapper_t(nullptr)
     {
         byte *pc = instrlist_encode(drcontext, &instrs, decode_buf_, true);
         ASSERT(pc - decode_buf_ < MAX_DECODE_SIZE, "decode buffer overflow");
-        set_modvec_(&modules_);
     }
 
 protected:
-    std::string
+    void
     read_and_map_modules() override
     {
-        modules_.push_back(module_t("fake_exe", 0, decode_buf_, 0, MAX_DECODE_SIZE,
-                                    MAX_DECODE_SIZE, true));
-        return "";
+        modvec_.push_back(module_t("fake_exe", 0, decode_buf_, 0, MAX_DECODE_SIZE,
+                                   MAX_DECODE_SIZE, true));
     }
 
 private:
     static const int MAX_DECODE_SIZE = 1024;
     byte decode_buf_[MAX_DECODE_SIZE];
-    std::vector<module_t> modules_;
+};
+
+class raw2trace_test_t : public raw2trace_t {
+public:
+    raw2trace_test_t(const std::vector<std::istream *> &input,
+                     const std::vector<std::ostream *> &output, instrlist_t &instrs,
+                     void *drcontext)
+        : raw2trace_t(nullptr, input, output, INVALID_FILE, drcontext,
+                      // The sequences are small so we print everything for easier
+                      // debugging and viewing of what's going on.
+                      4)
+    {
+        module_mapper_ =
+            std::unique_ptr<module_mapper_t>(new module_mapper_test_t(instrs, drcontext));
+        set_modmap_(module_mapper_.get());
+    }
 };
 
 offline_entry_t

clients/drcachesim/tools/opcode_mix.cpp

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2017-2021 Google, Inc.  All rights reserved.
+ * Copyright (c) 2017-2022 Google, Inc.  All rights reserved.
  * **********************************************************/
 
 /*
@@ -71,6 +71,8 @@ opcode_mix_t::initialize()
     std::string error = directory_.initialize_module_file(module_file_path_);
     if (!error.empty())
         return "Failed to initialize directory: " + error;
+    // Non-module instruction entries will be in the final trace (i#2062,i#5520) so
+    // we do not need the encoding file and leave it as its default INVALID_FILE.
     module_mapper_ =
         module_mapper_t::create(directory_.modfile_bytes_, nullptr, nullptr, nullptr,
                                 nullptr, knob_verbose_, knob_alt_module_dir_);

clients/drcachesim/tools/view.cpp

@@ -93,6 +93,8 @@ view_t::initialize()
         // not available.
         return "";
     }
+    // Non-module instruction entries will be in the final trace (i#2062,i#5520) so
+    // we do not need the encoding file and leave it as its default INVALID_FILE.
     module_mapper_ =
         module_mapper_t::create(directory_.modfile_bytes_, nullptr, nullptr, nullptr,
                                 nullptr, knob_verbose_, knob_alt_module_dir_);