Merge branch 'master' into i5505-drir2trace

.github/workflows/ci-clang-format.yml

@@ -71,8 +71,11 @@ jobs:
 
     - name: Create Build Environment
       run: |
+        wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+        sudo add-apt-repository 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main'
+        sudo apt update
         sudo apt-get -y install doxygen vera++ zlib1g-dev libsnappy-dev \
-          liblz4-dev clang-format-12 libunwind-dev
+          liblz4-dev clang-format-14 libunwind-dev
 
     - name: Run Suite
       working-directory: ${{ github.workspace }}

.gitignore

@@ -1 +1,9 @@
+# macOS specific
 .DS_Store
+
+# IDE project files
+.cache
+.idea
+.vs
+.vscode
+build

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "third_party/libipt"]
     path = third_party/libipt
     url = https://github.com/intel/libipt.git
+[submodule "third_party/zlib"]
+    path = third_party/zlib
+    url = https://github.com/madler/zlib.git

CMakeLists.txt

@@ -1835,7 +1835,10 @@ endif ()
 
 if (BUILD_CLIENTS)
   # Must be prior to api/docs
-  add_subdirectory(clients)
+  if (NOT (MACOS AND AARCH64))
+    # TODO i#5383: Add M1 drsyms libraries, needed by drcov2lcov and func_trace.
+    add_subdirectory(clients)
+  endif ()
 endif (BUILD_CLIENTS)
 
 if (BUILD_DOCS)

api/docs/building.dox

@@ -46,7 +46,7 @@ To build DynamoRIO on Linux, use the following commands as a guide.  This builds
   # Install dependencies for Ubuntu 15+.  Adjust this command as appropriate for
   # other distributions (in particular, use "cmake3" for Ubuntu Trusty).
   $ sudo apt-get install cmake g++ g++-multilib doxygen git zlib1g-dev libunwind-dev libsnappy-dev liblz4-dev
-  # Get sources and initialize the submodule third_party/libipt.
+  # Get sources and initialize the submodules.
   $ git clone --recursive https://github.com/DynamoRIO/dynamorio.git
   # Make a separate build directory.  Building in the source directory is not
   # supported.

api/docs/code_style.dox

@@ -41,7 +41,7 @@ maintainable.
 
 # Automated Formatting
 
-We employ automated code formatting via [`clang-format` version 12.0](https://releases.llvm.org/12.0.0/tools/clang/docs/ClangFormat.html). The [`.clang-format` top-level file](https://github.com/DynamoRIO/dynamorio/blob/master/.clang-format) specifies the style rules for all `.h`, `.c`, and `.cpp` source code files.  Developers are expected to set up their editors to use `clang-format` when saving each file (see [the `clang-format` documentation](https://releases.llvm.org/6.0.0/tools/clang/docs/ClangFormat.html) for pointers to vim, emacs, and Visual Studio setup instructions).  Our test suite includes a format check and will fail any code whose formatting does not match the `clang-format` output.
+We employ automated code formatting via [`clang-format` version 14.0](https://releases.llvm.org/14.0.0/tools/clang/docs/ClangFormat.html). The [`.clang-format` top-level file](https://github.com/DynamoRIO/dynamorio/blob/master/.clang-format) specifies the style rules for all `.h`, `.c`, and `.cpp` source code files.  Developers are expected to set up their editors to use `clang-format` when saving each file (see [the `clang-format` documentation](https://releases.llvm.org/14.0.0/tools/clang/docs/ClangFormat.html) for pointers to vim, emacs, and Visual Studio setup instructions).  Our test suite includes a format check and will fail any code whose formatting does not match the `clang-format` output.
 
 # Legacy Code
 

api/docs/debug_memtrace.dox

@@ -108,9 +108,20 @@ Type cheat sheet (from trace_type_t enum):
 
 type_is_instr: 0xa-0x10 + 0x1e
 
+For .zip files, the data is split into the component files within the archive, in order.
+Each component repeats enough information (the final timestamp + cpu from the prior
+chunk, instruction encodings, etc.) to avoid having to examing prior chunks.
+Extract each component file in turn to view the data.  For example:
+```
+$ unzip -p drmemtrace.app.194439.1516.dir/trace/drmemtrace.app.194439.7393.trace.zip chunk.0000 | hexdump -v -e '"%010_ax | " 6/2 "%04x " "\n"' | awk '{printf "%s %s %s %s %s%s%s%s\n", $1, $2, $3, $4, $8, $7, $6, $5}' | head -3
+0000000000 | 0019 0000 0000000000000004
+000000000c | 001c 000c 0000000000000004
+0000000018 | 001c 0009 0000000000000040
+```
+
 ## Viewing instruction disassembly
 
-You can use the `view` analysis tool with `skip_refs` and `sim_refs` parameters to select a window, or for small traces you can re-post-process with `raw2trace` with high verbosity (`-verbose 4` is good).
+You can use the `view` analysis tool with `skip_refs` and `sim_refs` parameters to select a window, or for small traces you can re-post-process with `drraw2trace` with high verbosity (`-verbose 4` is good).
 
  ****************************************************************************
  */

api/docs/release.dox

@@ -166,6 +166,9 @@ Further non-compatibility-affecting changes include:
    feature.
  - Added drmemtrace_get_encoding_path().
  - Added preliminary support for generated code to drmemtrace.
+ - Changed the default drmemtrace offline file format from .gz to .zip and
+   added the option -chunk_instr_count to control the split of a file within
+   the .zip, which sets the granularity of a fast seek.
 
 The changes between version 9.0.1 and 9.0.0 include the following compatibility
 changes:

clients/drcachesim/CMakeLists.txt

@@ -54,15 +54,46 @@ if (ARM AND CMAKE_COMPILER_IS_GNUCC)
   endif (GCC_HAS_NO_PSABI)
 endif ()
 
-# i#2277: we use zlib if available to read compressed trace files.
-# XXX: we could ship with a zlib for Windows: today we simply don't support
-# compressed traces on Windows.
+# i#2277: we use zlib if available to write and read compressed trace files,
+# and now to write zipfiles for fast seeking.
 if (ZLIB_FOUND)
   add_definitions(-DHAS_ZLIB)
   include_directories(${ZLIB_INCLUDE_DIRS})
   set(zlib_reader reader/compressed_file_reader.cpp)
+
+  # We use minizip, supplied with zlib, to split offline traces into pieces
+  # inside zipfiles to support fast seeking.
+  set(minizip_dir "${PROJECT_SOURCE_DIR}/third_party/zlib/contrib/minizip")
+  if (NOT EXISTS "${minizip_dir}")
+    # XXX: Should we make this a fatal error, and build zlib on Windows,
+    # to eliminate having to support both .gz and .zip output?
+    # Maybe require zlib as well and stop supporting uncompressed output
+    # and remove HAS_ZLIB ifdefs from our code?
+    message(WARNING
+      "third_party/zlib submodule is not initialized: run 'git submodule init'; "
+      "until then, disabling zip output and fast seeking")
+    set(zip_reader "")
+    set(zlib_libs ${ZLIB_LIBRARIES})
+  else ()
+    file(GLOB minizip_srcs "${minizip_dir}/*.c")
+    if (NOT WIN32)
+      list(REMOVE_ITEM minizip_srcs "${minizip_dir}/iowin32.c")
+    endif ()
+    add_library(minizip STATIC ${minizip_srcs})
+    add_definitions(-DHAS_ZIP)
+    # We add "minizip/" to avoid collisions with system "zip.h" on Mac.
+    include_directories(${minizip_dir}/..)
+    DR_export_target(minizip)
+    install_exported_target(minizip ${INSTALL_CLIENTS_LIB})
+    set(zip_reader reader/zipfile_file_reader.cpp)
+    set(zlib_libs ${ZLIB_LIBRARIES} minizip)
+  endif ()
 else ()
+  # XXX: We could ship with a zlib for Windows, or build the third_party/zlib.
+  # Today we simply don't support compressed traces or fast seeking on Windows.
   set(zlib_reader "")
+  set(zip_reader "")
+  set(zlib_libs "")
 endif()
 
 if (libsnappy)
@@ -171,6 +202,7 @@ set(drcachesim_srcs
   reader/config_reader.cpp
   reader/file_reader.cpp
   ${zlib_reader}
+  ${zip_reader}
   ${snappy_reader}
   reader/ipc_reader.cpp
   simulator/analyzer_interface.cpp
@@ -211,6 +243,7 @@ add_exported_library(drmemtrace_analyzer STATIC
   reader/config_reader.cpp
   reader/file_reader.cpp
   ${zlib_reader}
+  ${zip_reader}
   ${snappy_reader}
   )
 target_link_libraries(drmemtrace_analyzer directory_iterator)
@@ -281,14 +314,13 @@ if (NOT APPLE)
   configure_DynamoRIO_static(opcode_mix_launcher)
 endif ()
 
-if (ZLIB_FOUND)
-  target_link_libraries(drcachesim ${ZLIB_LIBRARIES})
-  target_link_libraries(histogram_launcher ${ZLIB_LIBRARIES})
-  target_link_libraries(prefetch_analyzer_launcher ${ZLIB_LIBRARIES})
-  target_link_libraries(drmemtrace_raw2trace ${ZLIB_LIBRARIES})
-  if (NOT AARCH64 AND NOT APPLE)
-    target_link_libraries(opcode_mix_launcher ${ZLIB_LIBRARIES})
-  endif ()
+target_link_libraries(drcachesim ${zlib_libs})
+target_link_libraries(histogram_launcher ${zlib_libs})
+target_link_libraries(prefetch_analyzer_launcher ${zlib_libs})
+target_link_libraries(drmemtrace_raw2trace ${zlib_libs})
+target_link_libraries(drmemtrace_analyzer ${zlib_libs})
+if (NOT AARCH64 AND NOT APPLE)
+  target_link_libraries(opcode_mix_launcher ${zlib_libs})
 endif ()
 
 macro(add_drmemtrace name type)
@@ -340,9 +372,7 @@ macro(add_drmemtrace name type)
   if (libsnappy)
     target_link_libraries(${name} snappy)
   endif ()
-  if (ZLIB_FOUND)
-    target_link_libraries(${name} ${ZLIB_LIBRARIES})
-  endif ()
+  target_link_libraries(${name} ${zlib_libs})
   if (liblz4)
     target_link_libraries(${name} lz4)
   endif ()
@@ -562,13 +592,8 @@ endif ()
 if (BUILD_TESTS)
   add_executable(tool.drcachesim.unit_tests tests/drcachesim_unit_tests.cpp
     tests/cache_replacement_policy_unit_test.cpp)
-  if (ZLIB_FOUND)
-    target_link_libraries(tool.drcachesim.unit_tests drmemtrace_simulator
-      drmemtrace_static drmemtrace_analyzer ${ZLIB_LIBRARIES})
-  else ()
-    target_link_libraries(tool.drcachesim.unit_tests drmemtrace_simulator
-      drmemtrace_static drmemtrace_analyzer)
-  endif ()
+  target_link_libraries(tool.drcachesim.unit_tests drmemtrace_simulator
+    drmemtrace_static drmemtrace_analyzer ${zlib_libs})
   add_win32_flags(tool.drcachesim.unit_tests)
   add_test(NAME tool.drcachesim.unit_tests
            COMMAND tool.drcachesim.unit_tests)
@@ -597,13 +622,8 @@ if (BUILD_TESTS)
   if (NOT APPLE AND NOT DISABLE_FOR_BUG_2949)
     # Tests for the cache miss analyzer.
     add_executable(tool.drcachesim.miss_analyzer_unit_test tests/cache_miss_analyzer_test.cpp)
-    if (ZLIB_FOUND)
-      target_link_libraries(tool.drcachesim.miss_analyzer_unit_test drmemtrace_simulator
-        drmemtrace_static drmemtrace_analyzer ${ZLIB_LIBRARIES})
-    else ()
-      target_link_libraries(tool.drcachesim.miss_analyzer_unit_test drmemtrace_simulator
-        drmemtrace_static drmemtrace_analyzer)
-    endif ()
+    target_link_libraries(tool.drcachesim.miss_analyzer_unit_test drmemtrace_simulator
+      drmemtrace_static drmemtrace_analyzer ${zlib_libs})
     add_win32_flags(tool.drcachesim.miss_analyzer_unit_test)
     add_test(NAME tool.drcachesim.miss_analyzer_unit_test
              COMMAND tool.drcachesim.miss_analyzer_unit_test)

clients/drcachesim/analyzer.cpp

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2016-2020 Google, Inc.  All rights reserved.
+ * Copyright (c) 2016-2022 Google, Inc.  All rights reserved.
  * **********************************************************/
 
 /*
@@ -38,6 +38,9 @@
 #ifdef HAS_ZLIB
 #    include "reader/compressed_file_reader.h"
 #endif
+#ifdef HAS_ZIP
+#    include "reader/zipfile_file_reader.h"
+#endif
 #ifdef HAS_SNAPPY
 #    include "reader/snappy_file_reader.h"
 #endif
@@ -62,7 +65,7 @@ analyzer_t::analyzer_t()
     /* Nothing else: child class needs to initialize. */
 }
 
-#ifdef HAS_SNAPPY
+#if defined(HAS_SNAPPY) || defined(HAS_ZIP)
 static bool
 ends_with(const std::string &str, const std::string &with)
 {
@@ -76,9 +79,15 @@ ends_with(const std::string &str, const std::string &with)
 static std::unique_ptr<reader_t>
 get_reader(const std::string &path, int verbosity)
 {
-#ifdef HAS_SNAPPY
+#if defined(HAS_SNAPPY) || defined(HAS_ZIP)
+#    ifdef HAS_SNAPPY
     if (ends_with(path, ".sz"))
         return std::unique_ptr<reader_t>(new snappy_file_reader_t(path, verbosity));
+#    endif
+#    ifdef HAS_ZIP
+    if (ends_with(path, ".zip"))
+        return std::unique_ptr<reader_t>(new zipfile_file_reader_t(path, verbosity));
+#    endif
     // If path is a directory, and any file in it ends in .sz, return a snappy reader.
     if (directory_iterator_t::is_directory(path)) {
         directory_iterator_t end;
@@ -89,14 +98,22 @@ get_reader(const std::string &path, int verbosity)
             return nullptr;
         }
         for (; iter != end; ++iter) {
+#    ifdef HAS_SNAPPY
             if (ends_with(*iter, ".sz")) {
                 return std::unique_ptr<reader_t>(
                     new snappy_file_reader_t(path, verbosity));
             }
+#    endif
+#    ifdef HAS_ZIP
+            if (ends_with(*iter, ".zip")) {
+                return std::unique_ptr<reader_t>(
+                    new zipfile_file_reader_t(path, verbosity));
+            }
+#    endif
         }
     }
 #endif
-    // No snappy support, or didn't find a .sz file, try the default reader.
+    // No snappy/zlib support, or didn't find a .sz/.zip file.
     return std::unique_ptr<reader_t>(new default_file_reader_t(path, verbosity));
 }
 

clients/drcachesim/analyzer_multi.cpp

@@ -105,10 +105,11 @@ analyzer_multi_t::analyzer_multi_t()
                 error_string_ = "Directory setup failed: " + dir_err;
                 return;
             }
-            raw2trace_t raw2trace(dir.modfile_bytes_, dir.in_files_, dir.out_files_,
-                                  dir.encoding_file_, nullptr, op_verbose.get_value(),
-                                  op_jobs.get_value(), dir.get_syscall_pt_trace_dir(),
-                                  op_alt_module_dir.get_value());
+            raw2trace_t raw2trace(
+                dir.modfile_bytes_, dir.in_files_, dir.out_files_, dir.out_archives_,
+                dir.encoding_file_, nullptr, op_verbose.get_value(), op_jobs.get_value(),
+                dir.get_syscall_pt_trace_dir(), op_alt_module_dir.get_value(),
+                op_chunk_instr_count.get_value());
             std::string error = raw2trace.do_conversion();
             if (!error.empty()) {
                 success_ = false;

clients/drcachesim/common/archive_ostream.h

@@ -0,0 +1,56 @@
+/* **********************************************************
+ * Copyright (c) 2022 Google, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Google, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* archive_ostream_t: a wrapper around std::ostream which additionally
+ * provides archive functionality to combine multiple components inside one
+ * larger component (typically a file).
+ */
+
+#ifndef _ARCHIVE_OSTREAM_H_
+#define _ARCHIVE_OSTREAM_H_ 1
+
+#include <fstream>
+
+class archive_ostream_t : public std::ostream {
+public:
+    archive_ostream_t(std::basic_streambuf<char, std::char_traits<char>> *buf)
+        : std::ostream(buf)
+    {
+    }
+    // Closes any currently open component and opens a new one.  Future writes are
+    // appended to the new component.  Returns an empty string on success or a non-empty
+    // error description on failure.
+    virtual std::string
+    open_new_component(const std::string &name) = 0;
+};
+
+#endif /* _ARCHIVE_OSTREAM_H_ */

clients/drcachesim/common/options.cpp

@@ -100,6 +100,28 @@ droption_t<std::string> op_alt_module_dir(
     "analysis tools, or in the raw modules file for post-prcoessing of offline "
     "raw trace files.  This directory takes precedence over the recorded path.");
 
+droption_t<bytesize_t> op_chunk_instr_count(
+    DROPTION_SCOPE_FRONTEND, "chunk_instr_count", bytesize_t(10 * 1000 * 1000U),
+    // We do not support tiny chunks.  We do not support disabling chunks with a 0
+    // value, to simplify testing: although we're still having to support generating
+    // non-zip files for !HAS_ZLIB/!HAS_ZIP!
+    bytesize_t(1000),
+#ifdef X64
+    bytesize_t(1ULL << 63),
+#else
+    // We store this value in a marker which can only hold a pointer-sized value and
+    // thus is limited to 4G.
+    // XXX i#5634: This happens to timestamps too: what we should do is use multiple
+    // markers (need up to 3) to support 64-bit values in 32-bit builds.
+    bytesize_t(UINT_MAX),
+#endif
+    "Chunk instruction count",
+    "Specifies the size in instructions of the chunks into which a trace output file "
+    "is split inside a zipfile.  This is the granularity of a fast seek. "
+    "This only applies when generating .zip-format traces; when built without "
+    "support for writing .zip files, this option is ignored. "
+    "For 32-bit this cannot exceed 4G.");
+
 droption_t<std::string> op_funclist_file(
     DROPTION_SCOPE_ALL, "funclist_file", "",
     "Path to function map file for func_view tool",

clients/drcachesim/common/options.h

@@ -68,6 +68,7 @@ extern droption_t<std::string> op_infile;
 extern droption_t<std::string> op_indir;
 extern droption_t<std::string> op_module_file;
 extern droption_t<std::string> op_alt_module_dir;
+extern droption_t<bytesize_t> op_chunk_instr_count;
 extern droption_t<std::string> op_funclist_file;
 extern droption_t<unsigned int> op_num_cores;
 extern droption_t<unsigned int> op_line_size;