diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation.Tests/CppTest.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation.Tests/CppTest.cs new file mode 100644 index 00000000000..b56169afcbb --- /dev/null +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation.Tests/CppTest.cs @@ -0,0 +1,380 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using NUnit.Framework; + +namespace Azure.Sdk.Tools.PerfAutomation.Tests +{ + public class CppTest + { + [Test] + public async Task SetupDebugWindows() + { + var cpp = new Cpp(); + cpp.IsTest = true; + cpp.UtilMethodCall += Cpp_UtilMethodCallDebugWindows; + cpp.WorkingDirectory = "workingFolder"; + cpp.IsWindows = true; + cpp.ProcessorCount = 16; + string setupOutput = null; + string setupError = null; + object context = null; + + (setupOutput, setupError, context) = await cpp.SetupAsync("project", "languageVersion", "primaryPackage", null, true); + Assert.AreEqual(setupOutput, "output"); + Assert.AreEqual(setupError, "error"); + Assert.AreEqual(context, "exe"); + } + + private void Cpp_UtilMethodCallDebugWindows(object sender, Cpp.UtilEventArgs e) + { + if (e.MethodName == "DeleteIfExists") + { + Assert.AreEqual(e.Params[0], Path.Combine("workingFolder", "build")); + } + else if (e.MethodName == "CreateDirectory") + { + Assert.AreEqual(e.Params[0], Path.Combine("workingFolder", "build")); + } + else if (e.MethodName == "UpdatePackageVersions") + { + Assert.AreEqual(e.Params[0], "packageVersions"); + } + else if (e.MethodName == "RunAsync1") + { + Assert.AreEqual(e.Params[0], "cmake"); + Assert.AreEqual(e.Params[1], "-DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON .."); + Assert.AreEqual(e.Params[2], Path.Combine("workingFolder", "build")); + Assert.AreEqual(e.Params[3], "output"); + Assert.AreEqual(e.Params[4], "error"); + } + else if (e.MethodName == "RunAsync2") + { + Assert.AreEqual(e.Params[0], "cmake"); + Assert.AreEqual(e.Params[1], "--build . --parallel 16 --config Debug --target project"); + Assert.AreEqual(e.Params[2], Path.Combine("workingFolder", "build")); + Assert.AreEqual(e.Params[3], "output"); + Assert.AreEqual(e.Params[4], "error"); + } + else + { + Assert.Fail("Unknown method"); + } + } + + [Test] + public async Task SetupRetailWindows() + { + var cpp = new Cpp(); + cpp.IsTest = true; + cpp.UtilMethodCall += Cpp_UtilMethodCallRetailWindows; + cpp.WorkingDirectory = "workingFolder"; + cpp.IsWindows = true; + cpp.ProcessorCount = 16; + string setupOutput = null; + string setupError = null; + object context = null; + + (setupOutput, setupError, context) = await cpp.SetupAsync("project", "languageVersion", "primaryPackage", null, false); + Assert.AreEqual(setupOutput, "output"); + Assert.AreEqual(setupError, "error"); + Assert.AreEqual(context, "exe"); + } + + private void Cpp_UtilMethodCallRetailWindows(object sender, Cpp.UtilEventArgs e) + { + if (e.MethodName == "DeleteIfExists") + { + Assert.AreEqual(e.Params[0], Path.Combine("workingFolder", "build")); + } + else if (e.MethodName == "CreateDirectory") + { + Assert.AreEqual(e.Params[0], Path.Combine("workingFolder", "build")); + } + else if (e.MethodName == "UpdatePackageVersions") + { + Assert.AreEqual(e.Params[0], "packageVersions"); + } + else if (e.MethodName == "RunAsync1") + { + Assert.AreEqual(e.Params[0], "cmake"); + Assert.AreEqual(e.Params[1], "-DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON .."); + Assert.AreEqual(e.Params[2], Path.Combine("workingFolder", "build")); + Assert.AreEqual(e.Params[3], "output"); + Assert.AreEqual(e.Params[4], "error"); + } + else if (e.MethodName == "RunAsync2") + { + Assert.AreEqual(e.Params[0], "cmake"); + Assert.AreEqual(e.Params[1], "--build . --parallel 16 --config MinSizeRel --target project"); + Assert.AreEqual(e.Params[2], Path.Combine("workingFolder", "build")); + Assert.AreEqual(e.Params[3], "output"); + Assert.AreEqual(e.Params[4], "error"); + } + else + { + Assert.Fail("Unknown method"); + } + } + + [Test] + public async Task SetupDebugLinux() + { + var cpp = new Cpp(); + cpp.IsTest = true; + cpp.UtilMethodCall += Cpp_UtilMethodCallDebugLinux; + cpp.WorkingDirectory = "workingFolder"; + cpp.IsWindows = false; + cpp.ProcessorCount = 16; + string setupOutput = null; + string setupError = null; + object context = null; + + (setupOutput, setupError, context) = await cpp.SetupAsync("project", "languageVersion", "primaryPackage", null, true); + Assert.AreEqual(setupOutput, "output"); + Assert.AreEqual(setupError, "error"); + Assert.AreEqual(context, "exe"); + } + + private void Cpp_UtilMethodCallDebugLinux(object sender, Cpp.UtilEventArgs e) + { + if (e.MethodName == "DeleteIfExists") + { + Assert.AreEqual(e.Params[0], Path.Combine("workingFolder", "build")); + } + else if (e.MethodName == "CreateDirectory") + { + Assert.AreEqual(e.Params[0], Path.Combine("workingFolder", "build")); + } + else if (e.MethodName == "UpdatePackageVersions") + { + Assert.AreEqual(e.Params[0], "packageVersions"); + } + else if (e.MethodName == "RunAsync1") + { + Assert.AreEqual(e.Params[0], "cmake"); + Assert.AreEqual(e.Params[1], "-DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DCMAKE_BUILD_TYPE=Debug .."); + Assert.AreEqual(e.Params[2], Path.Combine("workingFolder", "build")); + Assert.AreEqual(e.Params[3], "output"); + Assert.AreEqual(e.Params[4], "error"); + } + else if (e.MethodName == "RunAsync2") + { + Assert.AreEqual(e.Params[0], "cmake"); + Assert.AreEqual(e.Params[1], "--build . --parallel 16 --target project"); + Assert.AreEqual(e.Params[2], Path.Combine("workingFolder", "build")); + Assert.AreEqual(e.Params[3], "output"); + Assert.AreEqual(e.Params[4], "error"); + } + else + { + Assert.Fail("Unknown method"); + } + } + + [Test] + public async Task SetupRetailLinux() + { + var cpp = new Cpp(); + cpp.IsTest = true; + cpp.UtilMethodCall += Cpp_UtilMethodCallRetailLinux; + cpp.WorkingDirectory = "workingFolder"; + cpp.IsWindows = false; + cpp.ProcessorCount = 16; + string setupOutput = null; + string setupError = null; + object context = null; + + (setupOutput, setupError, context) = await cpp.SetupAsync("project", "languageVersion", "primaryPackage", null, false); + Assert.AreEqual(setupOutput, "output"); + Assert.AreEqual(setupError, "error"); + Assert.AreEqual(context, "exe"); + } + + private void Cpp_UtilMethodCallRetailLinux(object sender, Cpp.UtilEventArgs e) + { + if (e.MethodName == "DeleteIfExists") + { + Assert.AreEqual(e.Params[0], Path.Combine("workingFolder", "build")); + } + else if (e.MethodName == "CreateDirectory") + { + Assert.AreEqual(e.Params[0], Path.Combine("workingFolder", "build")); + } + else if (e.MethodName == "UpdatePackageVersions") + { + Assert.AreEqual(e.Params[0], "packageVersions"); + } + else if (e.MethodName == "RunAsync1") + { + Assert.AreEqual(e.Params[0], "cmake"); + Assert.AreEqual(e.Params[1], "-DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DCMAKE_BUILD_TYPE=Release .."); + Assert.AreEqual(e.Params[2], Path.Combine("workingFolder", "build")); + Assert.AreEqual(e.Params[3], "output"); + Assert.AreEqual(e.Params[4], "error"); + } + else if (e.MethodName == "RunAsync2") + { + Assert.AreEqual(e.Params[0], "cmake"); + Assert.AreEqual(e.Params[1], "--build . --parallel 16 --target project"); + Assert.AreEqual(e.Params[2], Path.Combine("workingFolder", "build")); + Assert.AreEqual(e.Params[3], "output"); + Assert.AreEqual(e.Params[4], "error"); + } + else + { + Assert.Fail("Unknown method"); + } + } + + [Test] + public async Task RunProfileWindows() + { + var cpp = new Cpp(); + cpp.IsTest = true; + cpp.WorkingDirectory = "workingFolder"; + cpp.IsWindows = true; + cpp.ProcessorCount = 16; + + try + { + await cpp.RunAsync( + "project", + "languageVersion", + "primaryPackage", + new Dictionary(), + "testName", + "arguments", + true, + "profilerOptions", + "exe"); + Assert.Fail(); + } + catch (InvalidOperationException) + { + Assert.Pass(); + } + } + + [Test] + public async Task RunNoProfileWindows() + { + var cpp = new Cpp(); + cpp.IsTest = true; + cpp.UtilMethodCall += Cpp_RunUtilMethodCallNoProfileWindows; + cpp.WorkingDirectory = "workingFolder"; + cpp.IsWindows = true; + cpp.ProcessorCount = 16; + + var result = await cpp.RunAsync( + "project", + "languageVersion", + "primaryPackage", + new Dictionary(), + "testName", + "arguments", + false, + "profilerOptions", + "exe"); + + Assert.AreEqual(result.StandardOutput, "output (2.0 ops/s, 1.0 s/op)"); + Assert.AreEqual(result.StandardError, "error"); + Assert.AreEqual(result.OperationsPerSecond, 2); + } + private void Cpp_RunUtilMethodCallNoProfileWindows(object sender, Cpp.UtilEventArgs e) + { + if (e.MethodName == "RunAsync") + { + Assert.AreEqual(e.Params[0], "exe"); + Assert.AreEqual(e.Params[1], "testName arguments"); + Assert.AreEqual(e.Params[2], "workingFolder"); + } + else + { + Assert.Fail("Unknown method"); + } + } + + [Test] + public async Task RunProfileLinux() + { + var cpp = new Cpp(); + cpp.IsTest = true; + cpp.UtilMethodCall += Cpp_RunUtilMethodCallProfileLinux; + cpp.WorkingDirectory = "workingFolder"; + cpp.IsWindows = false; + cpp.ProcessorCount = 16; + + var result = await cpp.RunAsync( + "project", + "languageVersion", + "primaryPackage", + new Dictionary(), + "testName", + "arguments", + true, + "profilerOptions", + "exe"); + + Assert.AreEqual(result.StandardOutput, "output (2.0 ops/s, 1.0 s/op)"); + Assert.AreEqual(result.StandardError, "error"); + Assert.AreEqual(result.OperationsPerSecond, 2); + } + private void Cpp_RunUtilMethodCallProfileLinux(object sender, Cpp.UtilEventArgs e) + { + if (e.MethodName == "RunAsync") + { + Assert.AreEqual(e.Params[0], "valgrind"); + Assert.AreEqual(e.Params[1], "profilerOptions exe testName arguments"); + Assert.AreEqual(e.Params[2], "workingFolder"); + } + else + { + Assert.Fail("Unknown method"); + } + + } + + [Test] + public async Task RunNoProfileLinux() + { + var cpp = new Cpp(); + cpp.IsTest = true; + cpp.UtilMethodCall += Cpp_RunUtilMethodCallNoProfileLinux; + cpp.WorkingDirectory = "workingFolder"; + cpp.IsWindows = false; + cpp.ProcessorCount = 16; + + var result = await cpp.RunAsync( + "project", + "languageVersion", + "primaryPackage", + new Dictionary(), + "testName", + "arguments", + false, + "profilerOptions", + "exe"); + + Assert.AreEqual(result.StandardOutput, "output (2.0 ops/s, 1.0 s/op)"); + Assert.AreEqual(result.StandardError, "error"); + Assert.AreEqual(result.OperationsPerSecond, 2); + } + private void Cpp_RunUtilMethodCallNoProfileLinux(object sender, Cpp.UtilEventArgs e) + { + if (e.MethodName == "RunAsync") + { + Assert.AreEqual(e.Params[0], "exe"); + Assert.AreEqual(e.Params[1], "testName arguments"); + Assert.AreEqual(e.Params[2], "workingFolder"); + } + else + { + Assert.Fail("Unknown method"); + } + + } + } +} + diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Cpp.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Cpp.cs index df8db37c5e8..bca1f10054b 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Cpp.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Cpp.cs @@ -1,9 +1,11 @@ using Azure.Sdk.Tools.PerfAutomation.Models; +using Microsoft.Crank.Agent; using Newtonsoft.Json; using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -20,44 +22,108 @@ namespace Azure.Sdk.Tools.PerfAutomation // 5. ./vcpkg install curl LibXml2 openssl public class Cpp : LanguageBase { + public class UtilEventArgs : EventArgs + { + public UtilEventArgs(string methodName, string[] methodParams, bool isWindows) + { + this.MethodName = methodName; + this.Params = methodParams; + this.IsWindows = isWindows; + } + + public string MethodName { get; set; } + public string[] Params { get; set; } = null; + public bool IsWindows { get; set; } + } + private const string _buildDirectory = "build"; private const string _vcpkgFile = "vcpkg.json"; - + public bool IsTest { get; set; } = false; + public bool IsWindows { get; set; } = Util.IsWindows; + public int ProcessorCount { get; set; } = Environment.ProcessorCount; protected override Language Language => Language.Cpp; + public event EventHandler UtilMethodCall; public override async Task<(string output, string error, object context)> SetupAsync( string project, string languageVersion, string primaryPackage, - IDictionary packageVersions) + IDictionary packageVersions, + bool debug) { var buildDirectory = Path.Combine(WorkingDirectory, _buildDirectory); - Util.DeleteIfExists(buildDirectory); - Directory.CreateDirectory(buildDirectory); + if (IsTest) + { + UtilMethodCall(this, new UtilEventArgs("DeleteIfExists", new string[] { buildDirectory }, false)); + UtilMethodCall(this, new UtilEventArgs("CreateDirectory", new string[] { buildDirectory }, false)); + } + else + { + Util.DeleteIfExists(buildDirectory); + Directory.CreateDirectory(buildDirectory); + } var outputBuilder = new StringBuilder(); var errorBuilder = new StringBuilder(); - - await UpdatePackageVersions(packageVersions); + if (IsTest) + { + UtilMethodCall(this, new UtilEventArgs("UpdatePackageVersions", new string[] { "packageVersions" }, false)); + } + else + { + await UpdatePackageVersions(packageVersions); + } // Windows and Linux require different arguments to build Release config - var additionalGenerateArguments = Util.IsWindows ? "-DDISABLE_AZURE_CORE_OPENTELEMETRY=ON" : "-DCMAKE_BUILD_TYPE=Release"; - var additionalBuildArguments = Util.IsWindows ? "--config MinSizeRel" : String.Empty; + var additionalGenerateArguments = IsWindows ? "-DDISABLE_AZURE_CORE_OPENTELEMETRY=ON" : (debug ? "-DCMAKE_BUILD_TYPE=Debug" : "-DCMAKE_BUILD_TYPE=Release"); + var additionalBuildArguments = IsWindows ? (debug ? "--config Debug" : "--config MinSizeRel") : String.Empty; - await Util.RunAsync( - "cmake", $"-DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON {additionalGenerateArguments} ..", - buildDirectory, outputBuilder: outputBuilder, errorBuilder: errorBuilder); + if (IsTest) + { + outputBuilder.Append("output"); + errorBuilder.Append("error"); + UtilMethodCall(this, new UtilEventArgs( + "RunAsync1", + new string[] + { + "cmake", + $"-DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON {additionalGenerateArguments} ..", + buildDirectory, + outputBuilder.ToString(), + errorBuilder.ToString() + }, + IsWindows)); + + UtilMethodCall(this, new UtilEventArgs( + "RunAsync2", + new string[] + { + "cmake", + $"--build . --parallel {this.ProcessorCount} {additionalBuildArguments} --target {project}", + buildDirectory, + outputBuilder.ToString(), + errorBuilder.ToString() + }, + IsWindows)); + return (outputBuilder.ToString(), errorBuilder.ToString(), "exe"); + } + else + { + await Util.RunAsync( + "cmake", $"-DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON {additionalGenerateArguments} ..", + buildDirectory, outputBuilder: outputBuilder, errorBuilder: errorBuilder); - var result = await Util.RunAsync( - "cmake", $"--build . --parallel {Environment.ProcessorCount} {additionalBuildArguments} --target {project}", - buildDirectory, outputBuilder: outputBuilder, errorBuilder: errorBuilder); + var result = await Util.RunAsync( + "cmake", $"--build . --parallel {Environment.ProcessorCount} {additionalBuildArguments} --target {project}", + buildDirectory, outputBuilder: outputBuilder, errorBuilder: errorBuilder); - // Find path to perf test executable - var exeFileName = Util.IsWindows ? $"{project}.exe" : project; - var exe = Directory.GetFiles(buildDirectory, exeFileName, SearchOption.AllDirectories).Single(); + // Find path to perf test executable + var exeFileName = Util.IsWindows ? $"{project}.exe" : project; + var exe = Directory.GetFiles(buildDirectory, exeFileName, SearchOption.AllDirectories).Single(); - return (result.StandardOutput, result.StandardError, exe); + return (result.StandardOutput, result.StandardError, exe); + } } public override async Task RunAsync( @@ -68,12 +134,45 @@ public override async Task RunAsync( string testName, string arguments, bool profile, + string profilerOptions, object context) { - var perfExe = (string)context; + string perfExe = (string)context; + var profiledExe = perfExe; + + if (profile) + { + if (IsWindows) + { + throw new InvalidOperationException("Profiling available on linux alone at the moment"); + } + + profiledExe = perfExe; + perfExe = "valgrind"; - var result = await Util.RunAsync(perfExe, $"{testName} {arguments}", WorkingDirectory); + } + string finalParams = $"{testName} {arguments}"; + if (profile) + { + finalParams = $"{profilerOptions} {profiledExe} {finalParams}"; + } + ProcessResult result = new ProcessResult(0, String.Empty, String.Empty); + if (IsTest) + { + UtilMethodCall(this, new UtilEventArgs( + "RunAsync", + new string[] { + perfExe, + finalParams, + WorkingDirectory}, + IsWindows)); + result = new ProcessResult(0, "output (2.0 ops/s, 1.0 s/op)", "error"); + } + else + { + result = await Util.RunAsync(perfExe, finalParams, WorkingDirectory); + } IDictionary reportedVersions = new Dictionary(); // Completed 54 operations in a weighted-average of 1s (52.766473 ops/s, 0.0189514 s/op) @@ -113,7 +212,7 @@ public override async Task CleanupAsync(string project) var fullVcpkgPath = Path.Combine(WorkingDirectory, _vcpkgFile); var buildDirectory = Path.Combine(WorkingDirectory, _buildDirectory); Util.DeleteIfExists(buildDirectory); - //cleanup the vcpkg file + //cleanup the vcpkg file by restoring from git await Util.RunAsync("git", $"checkout -- {fullVcpkgPath}", WorkingDirectory); return; } diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/ILanguage.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/ILanguage.cs index b7d4ae900f2..bf5e97dc773 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/ILanguage.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/ILanguage.cs @@ -16,7 +16,8 @@ public interface ILanguage string project, string languageVersion, string primaryPackage, - IDictionary packageVersions); + IDictionary packageVersions, + bool debug); Task RunAsync( string project, @@ -26,6 +27,7 @@ Task RunAsync( string testName, string arguments, bool profile, + string profilerOptions, object context); } } diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Java.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Java.cs index 08d14beaea9..bd91f4fca8a 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Java.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Java.cs @@ -31,7 +31,8 @@ public class Java : LanguageBase string project, string languageVersion, string primaryPackage, - IDictionary packageVersions) + IDictionary packageVersions, + bool debug ) { var projectFile = Path.Combine(WorkingDirectory, project, "pom.xml"); @@ -113,6 +114,7 @@ public override async Task RunAsync( string testName, string arguments, bool profile, + string profilerOptions, object context) { var jarFile = (string)context; diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/JavaScript.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/JavaScript.cs index e1bc54601bd..c4a6a1cf31d 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/JavaScript.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/JavaScript.cs @@ -21,7 +21,8 @@ public class JavaScript : LanguageBase string project, string languageVersion, string primaryPackage, - IDictionary packageVersions) + IDictionary packageVersions, + bool debug) { var outputBuilder = new StringBuilder(); var errorBuilder = new StringBuilder(); @@ -132,6 +133,7 @@ public override async Task RunAsync( string testName, string arguments, bool profile, + string profilerOptions, object context) { var runtimePackageVersions = (Dictionary)context; diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/LanguageBase.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/LanguageBase.cs index 0918be74271..79893f1284d 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/LanguageBase.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/LanguageBase.cs @@ -24,12 +24,14 @@ public abstract Task RunAsync( string testName, string arguments, bool profile, + string profilerOptions, object context); public abstract Task<(string output, string error, object context)> SetupAsync( string project, string languageVersion, string primaryPackage, - IDictionary packageVersions); + IDictionary packageVersions, + bool debug); } } diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs index bf57bf9f566..d922857294a 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Net.cs @@ -19,7 +19,8 @@ public class Net : LanguageBase string project, string languageVersion, string primaryPackage, - IDictionary packageVersions) + IDictionary packageVersions, + bool debug) { var projectFile = Path.Combine(WorkingDirectory, project); @@ -94,6 +95,7 @@ public override async Task RunAsync( string testName, string arguments, bool profile, + string profilerOptions, object context) { var dllName = Path.GetFileNameWithoutExtension(project) + ".dll"; diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Program.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Program.cs index 99ef0c1e67e..90cb7f67b72 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Program.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Program.cs @@ -80,6 +80,9 @@ public class Options [Option("profile", HelpText = "Enables capture of profiling data")] public bool Profile { get; set; } + [Option("profilerOpt", HelpText = "Provides additional profiler parameters")] + public string ProfilerOptions { get; set; } + [Option("repo-root", Required = true, HelpText = "Path to root of repository in which to run tests")] public string RepoRoot { get; set; } @@ -131,7 +134,8 @@ static Task DisplayHelp(ParserResult result) private static async Task Run(Options options) { - if (options.Language == Language.JS) { + if (options.Language == Language.JS) + { // JS is async-only options.NoSync = true; } @@ -266,7 +270,7 @@ private static async Task RunPackageVersion( try { (setupOutput, setupError, context) = await _languages[language].SetupAsync( - project, languageVersion, primaryPackage, packageVersions); + project, languageVersion, primaryPackage, packageVersions, options.Debug); } catch (Exception e) { @@ -343,7 +347,7 @@ private static async Task RunPackageVersion( try { Console.WriteLine($"RunAsync({project}, {languageVersion}, " + - $"{test.Class}, {allArguments}, {context}, {options.Profile})"); + $"{test.Class}, {allArguments}, {context}, {options.Profile}, {options.ProfilerOptions})"); Console.WriteLine(); iterationResult = await _languages[language].RunAsync( @@ -354,6 +358,7 @@ private static async Task RunPackageVersion( test.Class, allArguments, options.Profile, + options.ProfilerOptions, context); } catch (Exception e) diff --git a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Python.cs b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Python.cs index 67a5db17094..9bf3268c6a0 100644 --- a/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Python.cs +++ b/tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Python.cs @@ -20,7 +20,8 @@ public class Python : LanguageBase string project, string languageVersion, string primaryPackage, - IDictionary packageVersions) + IDictionary packageVersions, + bool debug) { var projectDirectory = Path.Combine(WorkingDirectory, project); var env = Path.Combine(projectDirectory, _env); @@ -76,6 +77,7 @@ public override async Task RunAsync( string testName, string arguments, bool profile, + string profilerOptions, object context) { var projectDirectory = Path.Combine(WorkingDirectory, project); diff --git a/tools/perf-automation/tests.yml b/tools/perf-automation/tests.yml index 4dc6c31af1d..654814925c5 100644 --- a/tools/perf-automation/tests.yml +++ b/tools/perf-automation/tests.yml @@ -286,6 +286,12 @@ stages: - template: /eng/pipelines/templates/steps/vcpkg-clone.yml@azure-sdk-for-cpp parameters: RepoOwner: Microsoft + - ${{ if parameters.Profile }}: + - task: Bash@3 + inputs: + targetType: 'inline' + script: sudo apt install valgrind -y + displayName: Install valgrind ServiceDirectory: ${{ parameters.CppServiceDirectory }} PackageVersions: ${{ parameters.PackageVersions }} Tests: ${{ parameters.Tests }}