Support passing total code size through when doing SPMI diffs (#60124)

Currently the total code bytes shown for the base/diff is misleading
when SPMI is used because SPMI generates only .dasm files for method
contexts with diffs. This change:

* Adds -baseMetricsSummary and -diffMetricsSummary to SPMI, which
  specifies a file path to output metrics. Currently that's just the
  number of failing/succeeding compiles, and the total number of code
  bytes, but the hope is to use this for other metrics as well.
* Adds --override-total-base-metric and --override-total-diff-metric to
  jit-analyze, to support overriding the computed total base and total
  diff metric, since they are misleading when .dasm files are only
  created for differences
* Supports this from the superpmi.py wrapper script for asmdiffs when no
  explicit metric is provided: in this case, the script will get the
  metrics from SPMI and pass them to jit-analyze to override the
  computed totals.

The net result is that the displayed results from jit-analyze after SPMI
runs are less misleading and should be more comparable to similar PMI
runs.

src/coreclr/ToolBox/superpmi/superpmi-shared/standardpch.h

@@ -68,12 +68,14 @@
 #ifdef TARGET_UNIX
 #include "clr_std/string"
 #include "clr_std/algorithm"
+#include "clr_std/vector"
 #else // !TARGET_UNIX
 #ifndef USE_STL
 #define USE_STL
 #endif // USE_STL
 #include <string>
 #include <algorithm>
+#include <vector>
 #endif // !TARGET_UNIX
 
 #ifdef USE_MSVCDIS

src/coreclr/ToolBox/superpmi/superpmi/CMakeLists.txt

@@ -22,6 +22,7 @@ set(SUPERPMI_SOURCES
     jitdebugger.cpp
     jitinstance.cpp
     methodstatsemitter.cpp
+    metricssummary.cpp
     neardiffer.cpp
     parallelsuperpmi.cpp
     superpmi.cpp

src/coreclr/ToolBox/superpmi/superpmi/commandline.cpp

@@ -81,6 +81,16 @@ void CommandLine::DumpHelp(const char* program)
     printf("         t - method throughput time\n");
     printf("         * - all available method stats\n");
     printf("\n");
+    printf(" -metricsSummary <file name>, -baseMetricsSummary <file name.csv>\n");
+    printf("     Emit a summary of metrics to the specified file\n");
+    printf("     Currently includes:\n");
+    printf("       Total number of successful SPMI compiles\n");
+    printf("       Total number of failing SPMI compiles\n");
+    printf("       Total amount of ASM code in bytes\n");
+    printf("\n");
+    printf(" -diffMetricsSummary <file name>\n");
+    printf("     Same as above, but emit for the diff/second JIT");
+    printf("\n");
     printf(" -a[pplyDiff]\n");
     printf("     Compare the compile result generated from the provided JIT with the\n");
     printf("     compile result stored with the MC. If two JITs are provided, this\n");
@@ -374,6 +384,26 @@ bool CommandLine::Parse(int argc, char* argv[], /* OUT */ Options* o)
 
                 o->methodStatsTypes = argv[i];
             }
+            else if ((_strnicmp(&argv[i][1], "metricsSummary", argLen) == 0) || (_strnicmp(&argv[i][1], "baseMetricsSummary", argLen) == 0))
+            {
+                if (++i >= argc)
+                {
+                    DumpHelp(argv[0]);
+                    return false;
+                }
+
+                o->baseMetricsSummaryFile = argv[i];
+            }
+            else if ((_strnicmp(&argv[i][1], "diffMetricsSummary", argLen) == 0))
+            {
+                if (++i >= argc)
+                {
+                    DumpHelp(argv[0]);
+                    return false;
+                }
+
+                o->diffMetricsSummaryFile = argv[i];
+            }
             else if ((_strnicmp(&argv[i][1], "applyDiff", argLen) == 0))
             {
                 o->applyDiff = true;

src/coreclr/ToolBox/superpmi/superpmi/commandline.h

@@ -36,6 +36,8 @@ class CommandLine
             , indexes(nullptr)
             , hash(nullptr)
             , methodStatsTypes(nullptr)
+            , baseMetricsSummaryFile(nullptr)
+            , diffMetricsSummaryFile(nullptr)
             , mclFilename(nullptr)
             , diffMCLFilename(nullptr)
             , targetArchitecture(nullptr)
@@ -67,6 +69,8 @@ class CommandLine
         int*  indexes;
         char* hash;
         char* methodStatsTypes;
+        char* baseMetricsSummaryFile;
+        char* diffMetricsSummaryFile;
         char* mclFilename;
         char* diffMCLFilename;
         char* targetArchitecture;

src/coreclr/ToolBox/superpmi/superpmi/jitinstance.cpp

@@ -8,6 +8,7 @@
 #include "jithost.h"
 #include "errorhandling.h"
 #include "spmiutil.h"
+#include "metricssummary.h"
 
 JitInstance* JitInstance::InitJit(char*                         nameOfJit,
                                   bool                          breakOnAssert,
@@ -276,7 +277,7 @@ bool JitInstance::reLoad(MethodContext* firstContext)
     return true;
 }
 
-JitInstance::Result JitInstance::CompileMethod(MethodContext* MethodToCompile, int mcIndex, bool collectThroughput)
+JitInstance::Result JitInstance::CompileMethod(MethodContext* MethodToCompile, int mcIndex, bool collectThroughput, MetricsSummary* metrics)
 {
     struct Param : FilterSuperPMIExceptionsParam_CaptureException
     {
@@ -286,12 +287,14 @@ JitInstance::Result JitInstance::CompileMethod(MethodContext* MethodToCompile, i
         unsigned            flags;
         int                 mcIndex;
         bool                collectThroughput;
+        MetricsSummary*     metrics;
     } param;
     param.pThis             = this;
     param.result            = RESULT_SUCCESS; // assume success
     param.flags             = 0;
     param.mcIndex           = mcIndex;
     param.collectThroughput = collectThroughput;
+    param.metrics           = metrics;
 
     // store to instance field our raw values, so we can figure things out a bit later...
     mc = MethodToCompile;
@@ -365,11 +368,16 @@ JitInstance::Result JitInstance::CompileMethod(MethodContext* MethodToCompile, i
             pParam->pThis->mc->cr->recAllocGCInfoCapture();
 
             pParam->pThis->mc->cr->recMessageLog("Successful Compile");
+
+            pParam->metrics->SuccessfulCompiles++;
+            pParam->metrics->NumCodeBytes += NCodeSizeBlock;
         }
         else
         {
             LogDebug("compileMethod failed with result %d", jitResult);
             pParam->result = RESULT_ERROR;
+
+            pParam->metrics->FailingCompiles++;
         }
     }
     PAL_EXCEPT_FILTER(FilterSuperPMIExceptions_CaptureExceptionAndStop)

src/coreclr/ToolBox/superpmi/superpmi/jitinstance.h

@@ -60,7 +60,7 @@ class JitInstance
 
     bool resetConfig(MethodContext* firstContext);
 
-    Result CompileMethod(MethodContext* MethodToCompile, int mcIndex, bool collectThroughput);
+    Result CompileMethod(MethodContext* MethodToCompile, int mcIndex, bool collectThroughput, class MetricsSummary* summary);
 
     const WCHAR* getForceOption(const WCHAR* key);
     const WCHAR* getOption(const WCHAR* key);

src/coreclr/ToolBox/superpmi/superpmi/methodstatsemitter.h

@@ -24,4 +24,5 @@ class MethodStatsEmitter
     void Emit(int methodNumber, MethodContext* mc, ULONGLONG firstTime, ULONGLONG secondTime);
     void SetStatsTypes(char* types);
 };
+
 #endif

src/coreclr/ToolBox/superpmi/superpmi/metricssummary.cpp

@@ -0,0 +1,90 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#include "standardpch.h"
+#include "metricssummary.h"
+#include "logging.h"
+
+struct HandleCloser
+{
+    void operator()(HANDLE hFile)
+    {
+        CloseHandle(hFile);
+    }
+};
+
+struct FileHandleWrapper
+{
+    FileHandleWrapper(HANDLE hFile)
+        : hFile(hFile)
+    {
+    }
+
+    ~FileHandleWrapper()
+    {
+        CloseHandle(hFile);
+    }
+
+    HANDLE get() { return hFile; }
+
+private:
+    HANDLE hFile;
+};
+
+bool MetricsSummary::SaveToFile(const char* path)
+{
+    FileHandleWrapper file(CreateFile(path, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr));
+    if (file.get() == INVALID_HANDLE_VALUE)
+    {
+        return false;
+    }
+
+    char buffer[4096];
+    int len =
+        sprintf_s(buffer, sizeof(buffer), "Successful compiles,Failing compiles,Code bytes\n%d,%d,%lld\n",
+            SuccessfulCompiles, FailingCompiles, NumCodeBytes);
+    DWORD numWritten;
+    if (!WriteFile(file.get(), buffer, static_cast<DWORD>(len), &numWritten, nullptr) || numWritten != static_cast<DWORD>(len))
+    {
+        return false;
+    }
+
+    return true;
+}
+
+bool MetricsSummary::LoadFromFile(const char* path, MetricsSummary* metrics)
+{
+    FileHandleWrapper file(CreateFile(path, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr));
+    if (file.get() == INVALID_HANDLE_VALUE)
+    {
+        return false;
+    }
+
+    LARGE_INTEGER len;
+    if (!GetFileSizeEx(file.get(), &len))
+    {
+        return false;
+    }
+
+    std::vector<char> content(static_cast<size_t>(len.QuadPart));
+    DWORD numRead;
+    if (!ReadFile(file.get(), content.data(), static_cast<DWORD>(content.size()), &numRead, nullptr) || numRead != content.size())
+    {
+        return false;
+    }
+
+    if (sscanf_s(content.data(), "Successful compiles,Failing compiles,Code bytes\n%d,%d,%lld\n",
+        &metrics->SuccessfulCompiles, &metrics->FailingCompiles, &metrics->NumCodeBytes) != 3)
+    {
+        return false;
+    }
+
+    return true;
+}
+
+void MetricsSummary::AggregateFrom(const MetricsSummary& other)
+{
+    SuccessfulCompiles += other.SuccessfulCompiles;
+    FailingCompiles += other.FailingCompiles;
+    NumCodeBytes += other.NumCodeBytes;
+}

src/coreclr/ToolBox/superpmi/superpmi/metricssummary.h

@@ -0,0 +1,26 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#ifndef _MetricsSummary
+#define _MetricsSummary
+
+class MetricsSummary
+{
+public:
+    int SuccessfulCompiles;
+    int FailingCompiles;
+    long long NumCodeBytes;
+
+    MetricsSummary()
+        : SuccessfulCompiles(0)
+        , FailingCompiles(0)
+        , NumCodeBytes(0)
+    {
+    }
+
+    bool SaveToFile(const char* path);
+    static bool LoadFromFile(const char* path, MetricsSummary* metrics);
+    void AggregateFrom(const MetricsSummary& other);
+};
+
+#endif