-
Notifications
You must be signed in to change notification settings - Fork 4.8k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[Data Lake Performance] Test Scenarios for T2 #17307
Merged
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
16 changes: 16 additions & 0 deletions
16
....DataLake/perf/Azure.Storage.Files.DataLake.Perf/Azure.Storage.Files.DataLake.Perf.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
| <PropertyGroup> | ||
| <Version>1.0.0</Version> | ||
| <OutputType>Exe</OutputType> | ||
| <LangVersion>latest</LangVersion> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Azure.Storage.Files.DataLake" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj" /> | ||
| <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\..\..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj" /> | ||
| </ItemGroup> | ||
| </Project> |
43 changes: 43 additions & 0 deletions
43
...les.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Azure.Storage.Files.DataLake.Perf.sln
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| | ||
| Microsoft Visual Studio Solution File, Format Version 12.00 | ||
| # Visual Studio Version 16 | ||
| VisualStudioVersion = 16.0.30711.63 | ||
| MinimumVisualStudioVersion = 10.0.40219.1 | ||
| Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Files.DataLake.Perf", "Azure.Storage.Files.DataLake.Perf.csproj", "{700A2B09-B9BE-4727-8CD8-9513EAA185C1}" | ||
| EndProject | ||
| Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "External", "External", "{D19AC4D9-9D47-4CBB-9D39-BBCCB73E0093}" | ||
| EndProject | ||
| Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Test.Perf", "..\..\..\..\..\common\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj", "{19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}" | ||
| EndProject | ||
| Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Core.TestFramework", "..\..\..\..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj", "{59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}" | ||
| EndProject | ||
| Global | ||
| GlobalSection(SolutionConfigurationPlatforms) = preSolution | ||
| Debug|Any CPU = Debug|Any CPU | ||
| Release|Any CPU = Release|Any CPU | ||
| EndGlobalSection | ||
| GlobalSection(ProjectConfigurationPlatforms) = postSolution | ||
| {700A2B09-B9BE-4727-8CD8-9513EAA185C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU | ||
| {700A2B09-B9BE-4727-8CD8-9513EAA185C1}.Debug|Any CPU.Build.0 = Debug|Any CPU | ||
| {700A2B09-B9BE-4727-8CD8-9513EAA185C1}.Release|Any CPU.ActiveCfg = Release|Any CPU | ||
| {700A2B09-B9BE-4727-8CD8-9513EAA185C1}.Release|Any CPU.Build.0 = Release|Any CPU | ||
| {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU | ||
| {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}.Debug|Any CPU.Build.0 = Debug|Any CPU | ||
| {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}.Release|Any CPU.ActiveCfg = Release|Any CPU | ||
| {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D}.Release|Any CPU.Build.0 = Release|Any CPU | ||
| {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU | ||
| {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}.Debug|Any CPU.Build.0 = Debug|Any CPU | ||
| {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}.Release|Any CPU.ActiveCfg = Release|Any CPU | ||
| {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6}.Release|Any CPU.Build.0 = Release|Any CPU | ||
| EndGlobalSection | ||
| GlobalSection(SolutionProperties) = preSolution | ||
| HideSolutionNode = FALSE | ||
| EndGlobalSection | ||
| GlobalSection(NestedProjects) = preSolution | ||
| {19FA3DDB-41CB-4DA2-88A4-A12FEB07557D} = {D19AC4D9-9D47-4CBB-9D39-BBCCB73E0093} | ||
| {59CCD0D2-1F7B-4EB8-AAF5-A4CB01DD1BC6} = {D19AC4D9-9D47-4CBB-9D39-BBCCB73E0093} | ||
| EndGlobalSection | ||
| GlobalSection(ExtensibilityGlobals) = postSolution | ||
| SolutionGuid = {8E6A0374-AFF4-4DF0-8663-856605C48B62} | ||
| EndGlobalSection | ||
| EndGlobal |
69 changes: 69 additions & 0 deletions
69
...les.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Infrastructure/PerfTestEnvironment.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System; | ||
| using Azure.Core.TestFramework; | ||
|
|
||
| namespace Azure.Storage.Files.DataLake.Perf | ||
| { | ||
| /// <summary> | ||
| /// Represents the ambient environment in which the test suite is | ||
| /// being run, offering access to information such as environment | ||
| /// variables. | ||
| /// </summary> | ||
| /// | ||
| public sealed class PerfTestEnvironment : TestEnvironment | ||
| { | ||
| /// <summary> | ||
| /// The shared instance of the <see cref="PerfTestEnvironment"/> to be used during test runs. | ||
| /// </summary> | ||
| /// | ||
| public static PerfTestEnvironment Instance { get; } = new PerfTestEnvironment(); | ||
|
|
||
| /// <summary> | ||
| /// The storage account endpoint suffix of the cloud to use for testing. | ||
| /// </summary> | ||
| /// | ||
| public new string StorageEndpointSuffix => base.StorageEndpointSuffix ?? "core.windows.net"; | ||
|
|
||
| /// <summary> | ||
| /// The name of the Data Lake storage account to test against. | ||
| /// </summary> | ||
| /// | ||
| /// <value>The Data Lake storage account name, read from the "DATALAKE_STORAGE_ACCOUNT_NAME" environment variable.</value> | ||
| /// | ||
| public string DataLakeAccountName => GetVariable("DATALAKE_STORAGE_ACCOUNT_NAME"); | ||
|
|
||
| /// <summary> | ||
| /// The shared access key of the Data Lake storage account to test against. | ||
| /// </summary> | ||
| /// | ||
| /// <value>The Data Lake storage account key, read from the "DATALAKE_STORAGE_ACCOUNT_KEY" environment variable.</value> | ||
| /// | ||
| public string DataLakeAccountKey => GetVariable("DATALAKE_STORAGE_ACCOUNT_KEY"); | ||
|
|
||
| /// <summary> | ||
| /// The fully-qualified URI for the Data Lake storage account to test against. | ||
| /// </summary> | ||
| /// | ||
| public Uri DataLakeServiceUri { get; } | ||
|
|
||
| /// <summary> | ||
| /// The credential for accessing the Data Lake storage account used for testing. | ||
| /// </summary> | ||
| /// | ||
| /// <value>This credential is based on the configured Data Lake shared key.</value> | ||
| /// | ||
| public StorageSharedKeyCredential DataLakeCredential { get; } | ||
|
|
||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="PerfTestEnvironment"/> class. | ||
| /// </summary> | ||
| /// | ||
| public PerfTestEnvironment() : base("storage") | ||
| { | ||
| DataLakeServiceUri = new Uri($"{ Uri.UriSchemeHttps }{ Uri.SchemeDelimiter }{ DataLakeAccountName }.dfs.{ StorageEndpointSuffix }"); | ||
| DataLakeCredential = new StorageSharedKeyCredential(DataLakeAccountName, DataLakeAccountKey); | ||
| } | ||
| } | ||
| } |
7 changes: 7 additions & 0 deletions
7
sdk/storage/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Program.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System.Reflection; | ||
| using Azure.Test.Perf; | ||
|
|
||
| await PerfProgram.Main(Assembly.GetEntryAssembly(), args); |
144 changes: 144 additions & 0 deletions
144
...e/Azure.Storage.Files.DataLake/perf/Azure.Storage.Files.DataLake.Perf/Scenarios/Append.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| //Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System; | ||
| using System.IO; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Azure.Test.Perf; | ||
|
|
||
| namespace Azure.Storage.Files.DataLake.Perf.Scenarios | ||
| { | ||
| /// <summary> | ||
| /// The performance test scenario focused on appending to an existing | ||
| /// file in Data Lake storage. | ||
| /// </summary> | ||
| /// | ||
| /// <seealso cref="Azure.Test.Perf.PerfTest{SizeOptions}" /> | ||
| /// | ||
| public sealed class Append : PerfTest<SizeOptions> | ||
| { | ||
| /// <summary> | ||
| /// The ambient test environment associated with the current execution. | ||
| /// </summary> | ||
| /// | ||
| private static PerfTestEnvironment TestEnvironment { get; } = PerfTestEnvironment.Instance; | ||
|
|
||
| /// <summary> | ||
| /// The name of the file system to use across parallel executions of the scenario. | ||
| /// </summary> | ||
| /// | ||
| private static string FileSystemName { get; } = Guid.NewGuid().ToString(); | ||
|
|
||
| /// <summary> | ||
| /// The client for interaction with the Data Lake file system. | ||
| /// </summary> | ||
| /// | ||
| private DataLakeFileSystemClient FileSystemClient { get; } | ||
|
|
||
| /// <summary> | ||
| /// The client to use for interactions with the test file. | ||
| /// </summary> | ||
| /// | ||
| private DataLakeFileClient FileClient { get; } | ||
|
|
||
| /// <summary> | ||
| /// The payload to use with a file being uploaded. | ||
| /// </summary> | ||
| /// | ||
| private Stream Payload { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="Append"/> class. | ||
| /// </summary> | ||
| /// | ||
| /// <param name="options">The set of options to consider for configuring the scenario.</param> | ||
| /// | ||
| public Append(SizeOptions options) : base(options) | ||
| { | ||
| var serviceClient = new DataLakeServiceClient(TestEnvironment.DataLakeServiceUri, TestEnvironment.DataLakeCredential); | ||
|
|
||
| FileSystemClient = serviceClient.GetFileSystemClient(FileSystemName); | ||
| FileClient = FileSystemClient.GetFileClient(Path.GetRandomFileName()); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Performs the tasks needed to initialize and set up the environment for the test scenario. | ||
| /// When multiple instances are run in parallel, the setup will take place once, prior to the | ||
| /// execution of the first test instance. | ||
| /// </summary> | ||
| /// | ||
| public async override Task GlobalSetupAsync() | ||
| { | ||
| await base.GlobalSetupAsync(); | ||
| await FileSystemClient.CreateAsync(); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Performs the tasks needed to clean up the environment for the test scenario. | ||
| /// When multiple instances are run in parallel, the cleanup will take place once, | ||
| /// after the execution of all test instances. | ||
| /// </summary> | ||
| /// | ||
| public async override Task GlobalCleanupAsync() | ||
| { | ||
| await base.GlobalCleanupAsync(); | ||
| await FileSystemClient.DeleteAsync(); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Performs the tasks needed to initialize and set up the environment for an instance | ||
| /// of the test scenario. When multiple instances are run in parallel, setup will be | ||
| /// run once for each prior to its execution. | ||
| /// </summary> | ||
| /// | ||
| public async override Task SetupAsync() | ||
| { | ||
| await base.SetupAsync(); | ||
| Payload = RandomStream.Create(Options.Size); | ||
|
|
||
| // Create the test file that will be used as the basis for uploading. | ||
|
|
||
| using var randomStream = RandomStream.Create(1024); | ||
|
|
||
| await FileClient.CreateAsync(); | ||
| await FileClient.UploadAsync(randomStream, true); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Performs the tasks needed to clean up the environment for an instance | ||
| /// of the test scenario. When multiple instances are run in parallel, cleanup | ||
| /// will be run once for each after execution has completed. | ||
| /// </summary> | ||
| /// | ||
| public async override Task CleanupAsync() | ||
| { | ||
| await base.CleanupAsync(); | ||
| Payload.Dispose(); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Executes the performance test scenario synchronously. | ||
| /// </summary> | ||
| /// | ||
| /// <param name="cancellationToken">The token used to signal when cancellation is requested.</param> | ||
| /// | ||
| public override void Run(CancellationToken cancellationToken) | ||
| { | ||
| Payload.Position = 0; | ||
| FileClient.Append(Payload, 0, cancellationToken:cancellationToken); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Executes the performance test scenario asynchronously. | ||
| /// </summary> | ||
| /// | ||
| /// <param name="cancellationToken">The token used to signal when cancellation is requested.</param> | ||
| /// | ||
| public async override Task RunAsync(CancellationToken cancellationToken) | ||
| { | ||
| Payload.Position = 0; | ||
| await FileClient.AppendAsync(Payload, 0, cancellationToken: cancellationToken); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I did not find any other references to the package; this should not have a cascading impact.