From 7e945473e67bea11e2b0bce1e17108bc264a3ac1 Mon Sep 17 00:00:00 2001 From: Jesse Squire Date: Mon, 7 Dec 2020 09:58:07 -0500 Subject: [PATCH] [Data Lake Performance] Test Scenarios for T2 (#17307) The focus of these changes is to create the initial set of performance test scenarios for the v12.x (track two) Data Lake client library. --- eng/.docsettings.yml | 1 + eng/Packages.Data.props | 2 +- .../Azure.Storage.Files.DataLake.Perf.csproj | 16 ++ .../Azure.Storage.Files.DataLake.Perf.sln | 43 +++++ .../Infrastructure/PerfTestEnvironment.cs | 69 ++++++++ .../Program.cs | 7 + .../Scenarios/Append.cs | 144 +++++++++++++++++ .../Scenarios/Read.cs | 117 ++++++++++++++ .../Scenarios/Upload.cs | 134 ++++++++++++++++ .../Scenarios/UploadFile.cs | 150 ++++++++++++++++++ .../perf/README.md | 15 ++ 11 files changed, 697 insertions(+), 1 deletion(-) create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Azure.Storage.Files.DataLake.Perf.csproj create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Azure.Storage.Files.DataLake.Perf.sln create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Infrastructure/PerfTestEnvironment.cs create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Program.cs create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/Append.cs create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/Read.cs create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/Upload.cs create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/UploadFile.cs create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/perf/README.md diff --git a/eng/.docsettings.yml b/eng/.docsettings.yml index 49ec02099f58d..536aabf4b12a1 100644 --- a/eng/.docsettings.yml +++ b/eng/.docsettings.yml @@ -158,6 +158,7 @@ known_content_issues: - ['sdk/storage/Azure.Storage.Blobs.Cryptography/README.md','#11492'] - ['sdk/storage/Azure.Storage.Common/README.md','#11492'] - ['sdk/storage/Azure.Storage.Files.DataLake/README.md','#11492'] + - ['sdk/storage/Azure.Storage.Files.DataLake/perf/README.md','#11492'] - ['sdk/storage/Azure.Storage.Files.Shares/README.md','#11492'] - ['sdk/storage/Azure.Storage.Queues/README.md','#11492'] - ['sdk/synapse/Microsoft.Azure.Synapse/README.md','#11492'] diff --git a/eng/Packages.Data.props b/eng/Packages.Data.props index 39002cf2b201c..64793b31d252c 100644 --- a/eng/Packages.Data.props +++ b/eng/Packages.Data.props @@ -28,7 +28,7 @@ - + diff --git a/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Azure.Storage.Files.DataLake.Perf.csproj b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Azure.Storage.Files.DataLake.Perf.csproj new file mode 100644 index 0000000000000..05b725f28c251 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Azure.Storage.Files.DataLake.Perf.csproj @@ -0,0 +1,16 @@ + + + 1.0.0 + Exe + latest + + + + + + + + + + + diff --git a/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Azure.Storage.Files.DataLake.Perf.sln b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Azure.Storage.Files.DataLake.Perf.sln new file mode 100644 index 0000000000000..6d66d3ee953cf --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Azure.Storage.Files.DataLake.Perf.sln @@ -0,0 +1,43 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30711.63 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Files.DataLake.Perf", "Azure.Storage.Files.DataLake.Perf.csproj", "{700A2B09-B9BE-4727-8CD8-9513EAA185C1}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "External", "External", "{D19AC4D9-9D47-4CBB-9D39-BBCCB73E0093}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Test.Perf", "..\..\..\..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj", "{19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Core.TestFramework", "..\..\..\..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj", "{59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {700A2B09-B9BE-4727-8CD8-9513EAA185C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {700A2B09-B9BE-4727-8CD8-9513EAA185C1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {700A2B09-B9BE-4727-8CD8-9513EAA185C1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {700A2B09-B9BE-4727-8CD8-9513EAA185C1}.Release|Any CPU.Build.0 = Release|Any CPU + {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}.Release|Any CPU.Build.0 = Release|Any CPU + {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D} = {D19AC4D9-9D47-4CBB-9D39-BBCCB73E0093} + {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6} = {D19AC4D9-9D47-4CBB-9D39-BBCCB73E0093} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {8E6A0374-AFF4-4DF0-8663-856605C48B62} + EndGlobalSection +EndGlobal diff --git a/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Infrastructure/PerfTestEnvironment.cs b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Infrastructure/PerfTestEnvironment.cs new file mode 100644 index 0000000000000..5c8493908ab1a --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Infrastructure/PerfTestEnvironment.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Azure.Core.TestFramework; + +namespace Azure.Storage.Files.DataLake.Perf +{ + /// + /// Represents the ambient environment in which the test suite is + /// being run, offering access to information such as environment + /// variables. + /// + /// + public sealed class PerfTestEnvironment : TestEnvironment + { + /// + /// The shared instance of the to be used during test runs. + /// + /// + public static PerfTestEnvironment Instance { get; } = new PerfTestEnvironment(); + + /// + /// The storage account endpoint suffix of the cloud to use for testing. + /// + /// + public new string StorageEndpointSuffix => base.StorageEndpointSuffix ?? "core.windows.net"; + + /// + /// The name of the Data Lake storage account to test against. + /// + /// + /// The Data Lake storage account name, read from the "DATALAKE_STORAGE_ACCOUNT_NAME" environment variable. + /// + public string DataLakeAccountName => GetVariable("DATALAKE_STORAGE_ACCOUNT_NAME"); + + /// + /// The shared access key of the Data Lake storage account to test against. + /// + /// + /// The Data Lake storage account key, read from the "DATALAKE_STORAGE_ACCOUNT_KEY" environment variable. + /// + public string DataLakeAccountKey => GetVariable("DATALAKE_STORAGE_ACCOUNT_KEY"); + + /// + /// The fully-qualified URI for the Data Lake storage account to test against. + /// + /// + public Uri DataLakeServiceUri { get; } + + /// + /// The credential for accessing the Data Lake storage account used for testing. + /// + /// + /// This credential is based on the configured Data Lake shared key. + /// + public StorageSharedKeyCredential DataLakeCredential { get; } + + /// + /// Initializes a new instance of the class. + /// + /// + public PerfTestEnvironment() : base("storage") + { + DataLakeServiceUri = new Uri($"{ Uri.UriSchemeHttps }{ Uri.SchemeDelimiter }{ DataLakeAccountName }.dfs.{ StorageEndpointSuffix }"); + DataLakeCredential = new StorageSharedKeyCredential(DataLakeAccountName, DataLakeAccountKey); + } + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Program.cs b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Program.cs new file mode 100644 index 0000000000000..1d1341bdc7bee --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Program.cs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Reflection; +using Azure.Test.Perf; + +await PerfProgram.Main(Assembly.GetEntryAssembly(), args); \ No newline at end of file diff --git a/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/Append.cs b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/Append.cs new file mode 100644 index 0000000000000..fd5f4b1b10483 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/Append.cs @@ -0,0 +1,144 @@ +//Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Test.Perf; + +namespace Azure.Storage.Files.DataLake.Perf.Scenarios +{ + /// + /// The performance test scenario focused on appending to an existing + /// file in Data Lake storage. + /// + /// + /// + /// + public sealed class Append : PerfTest + { + /// + /// The ambient test environment associated with the current execution. + /// + /// + private static PerfTestEnvironment TestEnvironment { get; } = PerfTestEnvironment.Instance; + + /// + /// The name of the file system to use across parallel executions of the scenario. + /// + /// + private static string FileSystemName { get; } = Guid.NewGuid().ToString(); + + /// + /// The client for interaction with the Data Lake file system. + /// + /// + private DataLakeFileSystemClient FileSystemClient { get; } + + /// + /// The client to use for interactions with the test file. + /// + /// + private DataLakeFileClient FileClient { get; } + + /// + /// The payload to use with a file being uploaded. + /// + /// + private Stream Payload { get; set; } + + /// + /// Initializes a new instance of the class. + /// + /// + /// The set of options to consider for configuring the scenario. + /// + public Append(SizeOptions options) : base(options) + { + var serviceClient = new DataLakeServiceClient(TestEnvironment.DataLakeServiceUri, TestEnvironment.DataLakeCredential); + + FileSystemClient = serviceClient.GetFileSystemClient(FileSystemName); + FileClient = FileSystemClient.GetFileClient(Path.GetRandomFileName()); + } + + /// + /// Performs the tasks needed to initialize and set up the environment for the test scenario. + /// When multiple instances are run in parallel, the setup will take place once, prior to the + /// execution of the first test instance. + /// + /// + public async override Task GlobalSetupAsync() + { + await base.GlobalSetupAsync(); + await FileSystemClient.CreateAsync(); + } + + /// + /// Performs the tasks needed to clean up the environment for the test scenario. + /// When multiple instances are run in parallel, the cleanup will take place once, + /// after the execution of all test instances. + /// + /// + public async override Task GlobalCleanupAsync() + { + await base.GlobalCleanupAsync(); + await FileSystemClient.DeleteAsync(); + } + + /// + /// Performs the tasks needed to initialize and set up the environment for an instance + /// of the test scenario. When multiple instances are run in parallel, setup will be + /// run once for each prior to its execution. + /// + /// + public async override Task SetupAsync() + { + await base.SetupAsync(); + Payload = RandomStream.Create(Options.Size); + + // Create the test file that will be used as the basis for uploading. + + using var randomStream = RandomStream.Create(1024); + + await FileClient.CreateAsync(); + await FileClient.UploadAsync(randomStream, true); + } + + /// + /// Performs the tasks needed to clean up the environment for an instance + /// of the test scenario. When multiple instances are run in parallel, cleanup + /// will be run once for each after execution has completed. + /// + /// + public async override Task CleanupAsync() + { + await base.CleanupAsync(); + Payload.Dispose(); + } + + /// + /// Executes the performance test scenario synchronously. + /// + /// + /// The token used to signal when cancellation is requested. + /// + public override void Run(CancellationToken cancellationToken) + { + Payload.Position = 0; + FileClient.Append(Payload, 0, cancellationToken:cancellationToken); + } + + /// + /// Executes the performance test scenario asynchronously. + /// + /// + /// The token used to signal when cancellation is requested. + /// + public async override Task RunAsync(CancellationToken cancellationToken) + { + Payload.Position = 0; + await FileClient.AppendAsync(Payload, 0, cancellationToken: cancellationToken); + } + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/Read.cs b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/Read.cs new file mode 100644 index 0000000000000..9facf6ed13037 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/Read.cs @@ -0,0 +1,117 @@ +//Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Test.Perf; + +namespace Azure.Storage.Files.DataLake.Perf.Scenarios +{ + /// + /// The performance test scenario focused on reading from Data Lake storage. + /// + /// + /// + /// + public sealed class Read : PerfTest + { + /// + /// The ambient test environment associated with the current execution. + /// + /// + private static PerfTestEnvironment TestEnvironment { get; } = PerfTestEnvironment.Instance; + + /// + /// The name of the file to use across parallel executions of the scenario. + /// + /// + private static string Filename { get; } = Path.GetRandomFileName(); + + /// + /// The name of the file system to use across parallel executions of the scenario. + /// + /// + private static string FileSystemName { get; } = Guid.NewGuid().ToString(); + + /// + /// The client for interaction with the Data Lake file system. + /// + /// + private DataLakeFileSystemClient FileSystemClient { get; } + + /// + /// The client to use for interactions with the test file. + /// + /// + private DataLakeFileClient FileClient { get; } + + /// + /// Initializes a new instance of the class. + /// + /// + /// The set of options to consider for configuring the scenario. + /// + public Read(SizeOptions options) : base(options) + { + var serviceClient = new DataLakeServiceClient(TestEnvironment.DataLakeServiceUri, TestEnvironment.DataLakeCredential); + + FileSystemClient = serviceClient.GetFileSystemClient(FileSystemName); + FileClient = FileSystemClient.GetFileClient(Filename); + } + + /// + /// Performs the tasks needed to initialize and set up the environment for the test scenario. + /// When multiple instances are run in parallel, the setup will take place once, prior to the + /// execution of the first test instance. + /// + /// + public async override Task GlobalSetupAsync() + { + await base.GlobalSetupAsync(); + await FileSystemClient.CreateAsync(); + + // Create the test file that will be used for reading. + + using var randomStream = RandomStream.Create(Options.Size); + + await FileClient.CreateAsync(); + await FileClient.UploadAsync(randomStream, true); + } + + /// + /// Performs the tasks needed to clean up the environment for the test scenario. + /// When multiple instances are run in parallel, the cleanup will take place once, + /// after the execution of all test instances. + /// + /// + public async override Task GlobalCleanupAsync() + { + await base.GlobalCleanupAsync(); + await FileSystemClient.DeleteAsync(); + } + + /// + /// Executes the performance test scenario synchronously. + /// + /// + /// The token used to signal when cancellation is requested. + /// + public override void Run(CancellationToken cancellationToken) + { + FileClient.Read(cancellationToken); + } + + /// + /// Executes the performance test scenario asynchronously. + /// + /// + /// The token used to signal when cancellation is requested. + /// + public async override Task RunAsync(CancellationToken cancellationToken) + { + await FileClient.ReadAsync(cancellationToken); + } + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/Upload.cs b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/Upload.cs new file mode 100644 index 0000000000000..5c6736d3b577d --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/Upload.cs @@ -0,0 +1,134 @@ +//Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Test.Perf; + +namespace Azure.Storage.Files.DataLake.Perf.Scenarios +{ + /// + /// The performance test scenario focused on uploading to Data Lake storage. + /// + /// + /// + /// + public sealed class Upload : PerfTest + { + /// + /// The ambient test environment associated with the current execution. + /// + /// + private static PerfTestEnvironment TestEnvironment { get; } = PerfTestEnvironment.Instance; + + /// + /// The name of the file system to use across parallel executions of the scenario. + /// + /// + private static string FileSystemName { get; } = Guid.NewGuid().ToString(); + + /// + /// The client for interaction with the Data Lake file system. + /// + /// + private DataLakeFileSystemClient FileSystemClient { get; } + + /// + /// The payload to use with a file being uploaded. + /// + /// + private Stream Payload { get; set; } + + /// + /// Initializes a new instance of the class. + /// + /// + /// The set of options to consider for configuring the scenario. + /// + public Upload(SizeOptions options) : base(options) + { + var serviceClient = new DataLakeServiceClient(TestEnvironment.DataLakeServiceUri, TestEnvironment.DataLakeCredential); + FileSystemClient = serviceClient.GetFileSystemClient(FileSystemName); + } + + /// + /// Performs the tasks needed to initialize and set up the environment for the test scenario. + /// When multiple instances are run in parallel, the setup will take place once, prior to the + /// execution of the first test instance. + /// + /// + public async override Task GlobalSetupAsync() + { + await base.GlobalSetupAsync(); + await FileSystemClient.CreateAsync(); + } + + /// + /// Performs the tasks needed to clean up the environment for the test scenario. + /// When multiple instances are run in parallel, the cleanup will take place once, + /// after the execution of all test instances. + /// + /// + public async override Task GlobalCleanupAsync() + { + await base.GlobalCleanupAsync(); + await FileSystemClient.DeleteAsync(); + } + + /// + /// Performs the tasks needed to initialize and set up the environment for an instance + /// of the test scenario. When multiple instances are run in parallel, setup will be + /// run once for each prior to its execution. + /// + /// + public async override Task SetupAsync() + { + await base.SetupAsync(); + Payload = RandomStream.Create(Options.Size); + } + + /// + /// Performs the tasks needed to clean up the environment for an instance + /// of the test scenario. When multiple instances are run in parallel, cleanup + /// will be run once for each after execution has completed. + /// + /// + public async override Task CleanupAsync() + { + await base.CleanupAsync(); + Payload.Dispose(); + } + + /// + /// Executes the performance test scenario synchronously. + /// + /// + /// The token used to signal when cancellation is requested. + /// + public override void Run(CancellationToken cancellationToken) + { + var fileClient = FileSystemClient.GetFileClient(Path.GetRandomFileName()); + Payload.Position = 0; + + fileClient.Create(cancellationToken: cancellationToken); + fileClient.Upload(Payload, true, cancellationToken); + } + + /// + /// Executes the performance test scenario asynchronously. + /// + /// + /// The token used to signal when cancellation is requested. + /// + public async override Task RunAsync(CancellationToken cancellationToken) + { + var fileClient = FileSystemClient.GetFileClient(Path.GetRandomFileName()); + Payload.Position = 0; + + await fileClient.CreateAsync(cancellationToken: cancellationToken); + await fileClient.UploadAsync(Payload, true, cancellationToken); + } + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/UploadFile.cs b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/UploadFile.cs new file mode 100644 index 0000000000000..b140ef64ac7f9 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/UploadFile.cs @@ -0,0 +1,150 @@ +//Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Test.Perf; + +namespace Azure.Storage.Files.DataLake.Perf.Scenarios +{ + /// + /// The performance test scenario focused on uploading to Data Lake storage. + /// + /// + /// + /// + public sealed class UploadFile : PerfTest + { + /// + /// The ambient test environment associated with the current execution. + /// + /// + private static PerfTestEnvironment TestEnvironment { get; } = PerfTestEnvironment.Instance; + + /// + /// The name of the file system to use across parallel executions of the scenario. + /// + /// + private static string FileSystemName { get; } = Guid.NewGuid().ToString(); + + /// + /// The client for interaction with the Data Lake file system. + /// + /// + private DataLakeFileSystemClient FileSystemClient { get; } + + /// + /// The path of the file to be uploaded. + /// + /// + private string UploadPath { get; set; } + + /// + /// Initializes a new instance of the class. + /// + /// + /// The set of options to consider for configuring the scenario. + /// + public UploadFile(SizeOptions options) : base(options) + { + var serviceClient = new DataLakeServiceClient(TestEnvironment.DataLakeServiceUri, TestEnvironment.DataLakeCredential); + FileSystemClient = serviceClient.GetFileSystemClient(FileSystemName); + } + + /// + /// Performs the tasks needed to initialize and set up the environment for the test scenario. + /// When multiple instances are run in parallel, the setup will take place once, prior to the + /// execution of the first test instance. + /// + /// + public async override Task GlobalSetupAsync() + { + await base.GlobalSetupAsync(); + await FileSystemClient.CreateAsync(); + } + + /// + /// Performs the tasks needed to clean up the environment for the test scenario. + /// When multiple instances are run in parallel, the cleanup will take place once, + /// after the execution of all test instances. + /// + /// + public async override Task GlobalCleanupAsync() + { + await base.GlobalCleanupAsync(); + await FileSystemClient.DeleteAsync(); + } + + /// + /// Performs the tasks needed to initialize and set up the environment for an instance + /// of the test scenario. When multiple instances are run in parallel, setup will be + /// run once for each prior to its execution. + /// + /// + public async override Task SetupAsync() + { + await base.SetupAsync(); + UploadPath = Path.GetTempFileName(); + + using var uploadFile = File.OpenWrite(UploadPath); + using var randomStream = RandomStream.Create(Options.Size); + + await randomStream.CopyToAsync(uploadFile); + uploadFile.Close(); + } + + /// + /// Performs the tasks needed to clean up the environment for an instance + /// of the test scenario. When multiple instances are run in parallel, cleanup + /// will be run once for each after execution has completed. + /// + /// + public async override Task CleanupAsync() + { + await base.CleanupAsync(); + + // Make a best effort to clean the upload file. If there is an issue, + // ignore it. The file exists in the system temporary area and should + // be cleaned up by the OS if necessary. + + try + { + File.Delete(UploadPath); + } + catch + { + // Ignore + } + } + + /// + /// Executes the performance test scenario synchronously. + /// + /// + /// The token used to signal when cancellation is requested. + /// + public override void Run(CancellationToken cancellationToken) + { + var fileClient = FileSystemClient.GetFileClient(Path.GetRandomFileName()); + + fileClient.Create(cancellationToken: cancellationToken); + fileClient.Upload(UploadPath, true, cancellationToken: cancellationToken); + } + + /// + /// Executes the performance test scenario asynchronously. + /// + /// + /// The token used to signal when cancellation is requested. + /// + public async override Task RunAsync(CancellationToken cancellationToken) + { + var fileClient = FileSystemClient.GetFileClient(Path.GetRandomFileName()); + + await fileClient.CreateAsync(cancellationToken: cancellationToken); + await fileClient.UploadAsync(UploadPath, true, cancellationToken); + } + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/perf/README.md b/sdk/storage/Azure.Storage.Files.DataLake/perf/README.md new file mode 100644 index 0000000000000..c9430493bc4a4 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/perf/README.md @@ -0,0 +1,15 @@ +# Azure Storage Files Data Lake performance tests + +The assets in this area comprise a set of performance tests for the [Azure Storage Files Data Lake client library for .NET](https://github.com/Azure/azure-sdk-for-net/tree/master/sdk/storage/Azure.Storage.Files.DataLake) and its associated ecosystem. The artifacts in this library are intended to be used primarily with the Azure SDK engineering system's testing infrastructure, by may also be run as stand-alone applications from the command line. + +## Contributing + +This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + +Please see our [contributing guide](https://github.com/Azure/azure-sdk-for-net/blob/master/sdk/eventhub/Azure.Messaging.EventHubs/CONTRIBUTING.md) for more information. + +![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-net%2Fsdk%2Feventhub%2FAzure.Storage.Files.DataLake.Perf%2FREADME.png)