Merge pull request #825 from Cysharp/feature/server_metrics

chore: benchmark server metrics

.github/scripts/run-benchmark-server.sh

@@ -9,12 +9,16 @@ set -euo pipefail
 
 function usage {
     echo "usage: $(basename $0) [options]"
+    echo "Required:"
+    echo "  --args          string      Arguments to pass when running the built binary (default: \"\")"
     echo "Options:"
     echo "  --help                      Show this help message"
 }
 
 while [ $# -gt 0 ]; do
   case $1 in
+    # required
+    --args) _ARGS=$2; shift 2; ;;
     # optional
     --help) usage; exit 1; ;;
     *) shift ;;
@@ -29,6 +33,7 @@ function print() {
 # parameter setup
 repo="MagicOnion"
 build_config="Release"
+args="${_ARGS:=""}"
 build_csproj="perf/BenchmarkApp/PerformanceTest.Server/PerformanceTest.Server.csproj"
 env_settings=""
 
@@ -105,10 +110,11 @@ pushd "$clone_path"
 popd
 
 # run dotnet app
-print "# Run $full_process_path"
+print "# Run $full_process_path $args"
 pushd "$output_dir"
-  # run background https://stackoverflow.com/questions/29142/getting-ssh-to-execute-a-command-in-the-background-on-target-machine
-  nohup "./$binary_name" > "${stdoutfile}" 2> "${stderrfile}" < /dev/null &
+  # use nohup to run background https://stackoverflow.com/questions/29142/getting-ssh-to-execute-a-command-in-the-background-on-target-machine
+  # shellcheck disable=SC2086
+  nohup "./$binary_name" $args > "${stdoutfile}" 2> "${stderrfile}" < /dev/null &
 
   # wait 10s will be enough to start the server or not
   sleep 10s

.github/workflows/benchmark.yml

@@ -28,29 +28,33 @@ jobs:
       matrix:
         include:
           # 1
-          - tags: "legend:h2c-linux,streams:1"
+          - tags: "legend:messagepack-h2c-linux,streams:1"
             channels: 28
             streams: 1
+            serialization: messagepack
           # 1x1
-          - tags: "legend:h2c-linux,streams:1x1"
+          - tags: "legend:messagepack-h2c-linux,streams:1x1"
             channels: 1
             streams: 1
+            serialization: messagepack
           # 70
-          - tags: "legend:h2c-linux,streams:70"
+          - tags: "legend:messagepack-h2c-linux,streams:70"
             channels: 28
             streams: 70
+            serialization: messagepack
           # 70x1
-          - tags: "legend:h2c-linux,streams:70x1"
+          - tags: "legend:messagepack-h2c-linux,streams:70x1"
             channels: 1
             streams: 70
+            serialization: messagepack
     uses: Cysharp/Actions/.github/workflows/benchmark.yaml@main
     with:
       dotnet-version: "8.0"
       environment: benchmark
       benchmark-name: "magiconion-${{ github.event.issue.number || (inputs.reuse && 'wf' || github.run_number) }}"
       benchmark-timeout: 20 # 10min (env prepare) + 5min (clone & benchmark) + 5min (spare)
       client-benchmark-script-path: ".github/scripts/run-benchmark-client.sh"
-      client-benchmark-script-args: "--args \"-u http://${BENCHMARK_SERVER_NAME}:5000 -s CI --channels ${{ matrix.channels }} --streams ${{ matrix.streams }} --validate true --tags '${{ matrix.tags }}'\""
+      client-benchmark-script-args: "--args \"-u http://${BENCHMARK_SERVER_NAME}:5000 -s CI --channels ${{ matrix.channels }} --streams ${{ matrix.streams }} --serialization ${{ matrix.serialization }} --validate true --tags ${{ matrix.tags }}\""
       server-benchmark-script-path: ".github/scripts/run-benchmark-server.sh"
-      server-benchmark-script-args: ""
+      server-benchmark-script-args: "--args \"--validate true --tags ${{ matrix.tags }}\""
     secrets: inherit

perf/BenchmarkApp/PerformanceTest.Client/Program.cs

@@ -278,7 +278,7 @@ static class DatadogMetricsRecorderExtensions
     public static async Task PutClientBenchmarkMetricsAsync(this DatadogMetricsRecorder recorder, ScenarioType scenario, ApplicationInformation applicationInfo, SerializationType serialization, PerformanceResult result)
     {
         var tags = MetricsTagCache.Get((recorder.TagBranch, recorder.TagLegend, recorder.TagStreams, scenario, applicationInfo, serialization), static x => [
-            $"legend:{x.scenario.ToString().ToLower()}-{x.serialization}-{x.TagLegend}{x.TagStreams}",
+            $"legend:{x.scenario.ToString().ToLower()}-{x.TagLegend}{x.TagStreams}",
             $"branch:{x.TagBranch}",
             $"streams:{x.TagStreams}",
             $"process_arch:{x.applicationInfo.ProcessArchitecture}",

perf/BenchmarkApp/PerformanceTest.Server/ProfileService.cs

@@ -0,0 +1,59 @@
+using PerformanceTest.Shared;
+using PerformanceTest.Shared.Reporting;
+
+namespace PerformanceTest.Server;
+
+class ProfileService : BackgroundService
+{
+    private DatadogMetricsRecorder datadog;
+    private readonly HardwarePerformanceReporter hardwarehardwarePerformanceReporter;
+    private readonly PeriodicTimer timer;
+
+    public ProfileService(TimeProvider timeProvider, IConfiguration configuration)
+    {
+        var tagString = configuration.GetValue<string>("Tags") ?? "";
+        var validate = configuration.GetValue<bool?>("Validate") ?? false;
+        datadog = DatadogMetricsRecorder.Create(tagString, validate);
+        hardwarehardwarePerformanceReporter = new HardwarePerformanceReporter();
+        timer = new PeriodicTimer(TimeSpan.FromSeconds(10), timeProvider);
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        hardwarehardwarePerformanceReporter.Start();
+        while (await timer.WaitForNextTickAsync(stoppingToken))
+        {
+            var result = hardwarehardwarePerformanceReporter.GetResult();
+            await datadog.PutServerBenchmarkMetricsAsync(ApplicationInformation.Current, result);
+        }
+    }
+}
+
+static class DatadogMetricsRecorderExtensions
+{
+    /// <summary>
+    /// Put Server Benchmark metrics to background. 
+    /// </summary>
+    /// <param name="recorder"></param>
+    /// <param name="applicationInfo"></param>
+    /// <param name="result"></param>
+    public static async Task PutServerBenchmarkMetricsAsync(this DatadogMetricsRecorder recorder, ApplicationInformation applicationInfo, HardwarePerformanceResult result)
+    {
+        var tags = MetricsTagCache.Get((recorder.TagBranch, recorder.TagLegend, recorder.TagStreams, applicationInfo), static x => [
+            $"legend:{x.TagLegend}{x.TagStreams}",
+            $"branch:{x.TagBranch}",
+            $"streams:{x.TagStreams}",
+            $"process_arch:{x.applicationInfo.ProcessArchitecture}",
+            $"process_count:{x.applicationInfo.ProcessorCount}",
+        ]);
+
+        // Don't want to await each put. Let's send it to queue and await when benchmark ends.
+        recorder.Record(recorder.SendAsync("benchmark.magiconion.server.cpu_usage_max", result.MaxCpuUsage, DatadogMetricsType.Gauge, tags, "percent"));
+        recorder.Record(recorder.SendAsync("benchmark.magiconion.server.cpu_usage_avg", result.MaxCpuUsage, DatadogMetricsType.Gauge, tags, "percent"));
+        recorder.Record(recorder.SendAsync("benchmark.magiconion.server.memory_usage_max", result.MaxMemoryUsageMB, DatadogMetricsType.Gauge, tags, "megabyte"));
+        recorder.Record(recorder.SendAsync("benchmark.magiconion.server.memory_usage_avg", result.AvgMemoryUsageMB, DatadogMetricsType.Gauge, tags, "megabyte"));
+
+        // wait until send complete
+        await recorder.WaitSaveAsync();
+    }
+}

perf/BenchmarkApp/PerformanceTest.Server/Program.cs

@@ -9,6 +9,11 @@
 }
 
 var builder = WebApplication.CreateBuilder(args);
+builder.Configuration.AddCommandLine(args, new Dictionary<string, string>()
+{
+    { "--tags", "Tags" },
+    { "--validate", "Validate" },
+});
 
 builder.Logging.ClearProviders();
 
@@ -18,7 +23,9 @@
 // Add services to the container.
 builder.Services.AddGrpc();
 builder.Services.AddMagicOnion();
+builder.Services.AddSingleton(TimeProvider.System);
 builder.Services.AddHostedService<StartupService>();
+builder.Services.AddHostedService<ProfileService>();
 
 var app = builder.Build();
 

perf/BenchmarkApp/PerformanceTest.Server/Properties/launchSettings.json

@@ -2,6 +2,7 @@
   "profiles": {
     "PerformanceTest.Server": {
       "commandName": "Project",
+      "commandLineArgs": "--tags legend:h2c-linux,streams:1",
       "dotnetRunMessages": true,
       "launchBrowser": false,
       "applicationUrl": "http://localhost:5000;https://localhost:5001",

perf/BenchmarkApp/PerformanceTest.Shared/Reporting/HardwarePerformanceReporter.cs

@@ -1,3 +1,4 @@
+using System.Collections.Concurrent;
 using System.Diagnostics;
 
 namespace PerformanceTest.Shared.Reporting;
@@ -7,8 +8,8 @@ public class HardwarePerformanceReporter
     private readonly TimeSpan samplingInterval;
     private readonly TimeProvider timeProvider;
     private readonly Process currentProcess;
-    private readonly List<double> cpuUsages;
-    private readonly List<double> memoryUsages;
+    private readonly ConcurrentBag<double> cpuUsages;
+    private readonly ConcurrentBag<double> memoryUsages;
     private CancellationTokenSource cancellationTokenSource;
     private bool running;
 
@@ -68,6 +69,7 @@ public HardwarePerformanceResult GetResult()
         var avgCpuUsage = cpuUsages.Count > 0 ? cpuUsages.Average() : 0d;
         var maxMemoryUsage = memoryUsages.Count > 0 ? memoryUsages.Max() / 1024 / 1024: 0d;
         var avgMemoryUsage = memoryUsages.Count > 0 ? memoryUsages.Average() / 1024 / 1024: 0d;
+
         cpuUsages.Clear();
         memoryUsages.Clear();