Eventhub Stress Test Onboarding

eng/Packages.Data.props

@@ -187,6 +187,7 @@
     <PackageReference Update="CommandLineParser" Version="2.8.0" />
     <PackageReference Update="FluentAssertions" Version="5.10.3" />
     <PackageReference Update="FsCheck.Xunit" Version="2.14.0" />
+    <PackageReference Update="Microsoft.ApplicationInsights" Version="2.20.0" />
     <PackageReference Update="Microsoft.Azure.ApplicationInsights.Query" Version="1.0.0" />
     <PackageReference Update="Microsoft.AspNetCore" Version="2.2.0" />
     <PackageReference Update="Microsoft.AspNetCore.Mvc.Testing" Version="2.2.0" />

sdk/eventhub/Azure.Messaging.EventHubs.Shared/src/Azure.Messaging.EventHubs.Shared.Stress.projitems

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
+    <HasSharedItems>true</HasSharedItems>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <Import_RootNamespace>Azure.Messaging.EventHubs.Tests</Import_RootNamespace>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="$(MSBuildThisFileDirectory)Testing\EventGenerator.cs" Link="SharedSource\Stress\%(Filename)%(Extension)" />
+  </ItemGroup>
+</Project>

sdk/eventhub/Azure.Messaging.EventHubs.Shared/src/Testing/EventGenerator.cs

@@ -64,6 +64,47 @@ public static IEnumerable<EventData> CreateEvents(int numberOfEvents)
             }
         }
 
+        /// <summary>
+        ///   Creates a random set of events with random data and random body size within the specified constraints.
+        /// </summary>
+        ///
+        /// <param name="maximumBatchSize">The maximum size of the batch.</param>
+        /// <param name="numberOfEvents">The number of events to create.</param>
+        /// <param name="largeMessageRandomFactor">The percentage of events that should be large.</param>
+        /// <param name="minimumBodySize">The minimum size of the event body.</param>
+        /// <param name="maximumBodySize">The maximum size of the event body.</param>
+        ///
+        /// <returns>The requested set of events.</returns>
+        ///
+        public static IEnumerable<EventData> CreateEvents(long maximumBatchSize,
+                                                          int numberOfEvents,
+                                                          int largeMessageRandomFactor = 30,
+                                                          int minimumBodySize = 15,
+                                                          int maximumBodySize = 83886)
+        {
+            var activeMinimumBodySize = minimumBodySize;
+            var activeMaximumBodySize = maximumBodySize;
+            var totalBytesGenerated = 0;
+
+            if (RandomNumberGenerator.Value.Next(1, 100) < largeMessageRandomFactor)
+            {
+                activeMinimumBodySize = (int)Math.Ceiling(maximumBodySize * 0.65);
+            }
+            else
+            {
+                activeMaximumBodySize = (int)Math.Floor((maximumBatchSize * 1.0f) / numberOfEvents);
+            }
+
+            for (var index = 0; ((index < numberOfEvents) && (totalBytesGenerated <= maximumBatchSize)); ++index)
+            {
+                var buffer = new byte[RandomNumberGenerator.Value.Next(activeMinimumBodySize, activeMaximumBodySize)];
+                RandomNumberGenerator.Value.NextBytes(buffer);
+                totalBytesGenerated += buffer.Length;
+
+                yield return CreateEventFromBody(buffer);
+            }
+        }
+
         /// <summary>
         ///   Creates a set of events with random data and a small body size.
         /// </summary>

sdk/eventhub/Azure.Messaging.EventHubs/stress/.gitignore

@@ -0,0 +1,2 @@
+stress-test-resources.json
+charts

sdk/eventhub/Azure.Messaging.EventHubs/stress/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: stress-test-addons
+  repository: https://stresstestcharts.blob.core.windows.net/helm/
+  version: 0.1.13
+digest: sha256:007ec9983233e0f8c8ad8aa7f1df5f57b1b4c127d223d59d233b3c5366bd5173
+generated: "2022-04-12T11:20:37.5250017-07:00"

sdk/eventhub/Azure.Messaging.EventHubs/stress/Chart.yaml

@@ -0,0 +1,15 @@
+apiVersion: v2
+name: eventhub-net-stress-test-2
+version: 0.1.2
+description: Stress tests for Event Hubs for .NET
+
+dependencies:
+- name: stress-test-addons
+  version: 0.1.13
+  repository: https://stresstestcharts.blob.core.windows.net/helm/
+
+annotations:
+  stressTest: 'true'
+  namespace: 'net'
+  dockerbuilddir: '../../../..'
+  dockerfile: './Dockerfile'

sdk/eventhub/Azure.Messaging.EventHubs/stress/Dockerfile

@@ -0,0 +1,34 @@
+# right version of dotnet?
+FROM mcr.microsoft.com/dotnet/sdk:6.0-cbl-mariner1.0 AS build-env 
+
+# Copy in engineering system needed to build
+COPY ./eng /app/eng
+COPY ./tools /app/tools
+COPY ./*.proj /app
+COPY ./Directory.Build.props /app
+COPY ./Directory.Build.targets /app
+COPY ./NuGet.Config /app
+
+# Copy in Event Hubs Source Code
+COPY ./sdk/eventhub/Azure.Messaging.EventHubs/ /app/sdk/eventhub/Azure.Messaging.EventHubs/
+COPY ./sdk/eventhub/Azure.Messaging.EventHubs.Shared /app/sdk/eventhub/Azure.Messaging.EventHubs.Shared
+
+# Copy in Core Source Code
+COPY ./sdk/core /app/sdk/core
+
+# Run restore and build, copy the .dll files over so they can used in the running container
+WORKDIR /app
+RUN dotnet build './sdk/eventhub/Azure.Messaging.EventHubs/stress/src/' --configuration Release
+
+# Copy in the dll files to be ready to run
+FROM build-env as publish
+WORKDIR /app
+COPY --from=build-env /app/artifacts/bin/Azure.Messaging.EventHubs.Stress/Release/net6.0 /app/artifacts/bin/Azure.Messaging.EventHubs.Stress/Release/net6.0
+COPY --from=build-env /app/sdk/eventhub/Azure.Messaging.EventHubs.Shared /app/sdk/evethub/Azure.Messaging.EventHubs.Shared
+COPY --from=build-env /app/sdk/eventhub/Azure.Messaging.EventHubs/stress /app/sdk/eventhub/Azure.Messaging.EventHubs/stress
+
+WORKDIR /app/artifacts/bin/Azure.Messaging.EventHubs.Stress/Release/net6.0
+
+# The default is running just the "EventProducerTest"
+ENTRYPOINT ["dotnet Azure.Messaging.EventHubs.Stress.dll", "--tests"] 
+CMD ["EventProducerTest"]

sdk/eventhub/Azure.Messaging.EventHubs/stress/README.md

@@ -0,0 +1,85 @@
+# Event Hub Stress Tests
+The scenarios in this directory provide a suite of stress tests that test the Event Hubs producer and buffered producer client types for long-term durability and reliability. For more in-depth information about the Azure SDK stress test tools, see the [stress test README](https://github.com/Azure/azure-sdk-tools/blob/main/tools/stress-cluster/chaos/README.md).
+
+Test runs can call any of the following tests:
+- "EventProducerTest"
+- "EventBufferedProducerTest"
+- "BurstBufferedProducerTest"
+- "ConcurrentBufferedProducerTest"
+
+## Local Stress Test Runs
+### Setting up resources and packages
+```cmd
+(env) <git root>/sdk/eventhub/Azure.Messaging.EventHubs/stress/src> dotnet clean
+(env) <git root>/sdk/eventhub/Azure.Messaging.EventHubs/stress/src> dotnet publish
+```
+
+### Running Tests
+When tests are run locally, Azure resources need to be created prior to running the test. This can be done through the Azure CLI, an ARM pr bicep file, or the Azure Portal. The bicep file included in this directory can be used to [deploy all resources](https://docs.microsoft.com/en-us/azure/azure-resource-manager/bicep/deploy-to-resource-group?tabs=azure-cli) aside from the application insights portal.
+
+To run the compiled .dll file, navigate to the `<git root>/artifacts/bin/Azure.Messaging.EventHubs.Stress/Release/net6.0` directory.
+
+The user is required to input the connection strings upon request on the command line when the test is being run, or include them in a .env file. To use the CLI input, add the `-i` or `--interactive` flag to the call:
+
+```cmd
+(env) <git root>/artifacts/bin/Azure.Messaging.EventHubs.Stress/Release/net6.0> dotnet Azure.Messaging.EventHubs.Stress.dll --tests <test/s-to-run> --interactive
+```
+
+To see the variable names for the environment file, see `EnvironmentVariables.cs`, The path to the environment file needs to be stored in the environment variable `$ENV_FILE`. In this case, do not include the `--interactive flag`. For more information about what specific resources are needed for each test, see the "Scenario Information" section below. 
+
+The recommended approach is to run tests one at a time when running locally.
+To run any one test, run the following:
+```cmd
+(env) <git root>/sdk/eventhub/Azure.Messaging.EventHubs/stress/src> dotnet run -f netcoreapp3.1 BasicPublishReadTest local
+```
+
+## Deploy a stress test
+In order to deploy stress tests to be run in kubernetes clusters, run:
+```cmd 
+(env) <git root>/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 `
+>>     -Login `
+>>     -PushImages
+```
+This command requires Azure login credentials.
+
+## Scenario Information
+### Event Producer Test
+This test requires an event hub namespace, an event hub, and an application insights resource. Note that an event hub may experience throttling if too few partitions are used. This test creates 2 producers and has 5 concurrent processes per producer sending batches of events. This is a long-running, consistent volume test. To run this test in interactive mode, run the following:
+```cmd
+(env) <git root>/artifacts/bin/Azure.Messaging.EventHubs.Stress/Release/net6.0> dotnet Azure.Messaging.EventHubs.Stress.dll --tests EventProducerTest --interactive
+```
+
+### Event Buffered Producer Test
+This test requires an event hub namespace, an event hub, and an application insights resource. High CPU usage may be experienced if too few partitions are used. This test creates 2 producers and continuously sends to each producer separately. This is a long-running, consistent volume test. To run this test in interactive mode, run the following:
+```cmd
+(env) <git root>/artifacts/bin/Azure.Messaging.EventHubs.Stress/Release/net6.0> dotnet Azure.Messaging.EventHubs.Stress.dll --tests BufferedProducerTest --interactive
+```
+
+### Concurrent Buffered Producer Test
+This test requires an event hub namespace, an event hub, and an application insights resource. High CPU usage may be experienced if too few partitions are used. This test creates 2 producers and has 5 concurrent processes per producer continuously sending events. This is a long-running, consistent volume test. To run this test in interactive mode, run the following:
+```cmd
+(env) <git root>/artifacts/bin/Azure.Messaging.EventHubs.Stress/Release/net6.0> dotnet Azure.Messaging.EventHubs.Stress.dll --tests ConcurrentBufferedProducerTest --interactive
+```
+
+### Burst Buffered Producer Test
+This test requires an event hub namespace, an event hub, and an application insights resource. Note that this test may have a high CPU usage. This test creates 2 producers and sends sets of events to each producer separately every 15 minutes. This is a long-running, variable volume test. To run this test in interactive mode, run the following:
+```cmd
+(env) <git root>/artifacts/bin/Azure.Messaging.EventHubs.Stress/Release/net6.0> dotnet Azure.Messaging.EventHubs.Stress.dll --tests BurstBufferedProducerTest --interactive
+```
+
+## Seeing Metrics and Logging in App Insights
+All metrics and logging are sent to App Insights via the Instrumentation Key provided during the initialization of the test. A brief explanation of the metrics collection approach is described below.
+- Any exceptions that occur are tracked by the telemetry client as exception telemetry. They can be accessed through the logs by filtering for exceptions, or through the application insights portal in the "Exceptions" blade. If an exception occured during send, the exception telemetry will include a "process" property containing "send"
+- Successful enqueues and sends are tracked through Metrics. For the buffered producer, the total number of enqueues is tracked, and the actual sends include the ability to separate counts into partition Ids.
+
+## Contributing  
+
+This project welcomes contributions and suggestions.  Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.microsoft.com.
+
+When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [[email protected]](mailto:[email protected]) with any additional questions or comments.
+
+Please see our [contributing guide](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/eventhub/Azure.Messaging.EventHubs/CONTRIBUTING.md) for more information.
+
+![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-net%2Fsdk%2Feventhub%2FAzure.Messaging.EventHubs%2Fstress%2FREADME.png)

sdk/eventhub/Azure.Messaging.EventHubs/stress/src/Azure.Messaging.EventHubs.Stress.csproj

@@ -0,0 +1,29 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <Version>1.0.0</Version>
+    <OutputType>Exe</OutputType>
+    <LangVersion>latest</LangVersion>
+    <IsTestSupportProject>true</IsTestSupportProject>
+    <IsStressProject>false</IsStressProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="System.Memory.Data" />
+    <PackageReference Include="Azure.Storage.Blobs" />
+    <PackageReference Include="Microsoft.ApplicationInsights" />
+    <PackageReference Include="CommandLineParser" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Azure.Messaging.EventHubs.csproj" />
+  </ItemGroup>
+
+  <!-- Import Azure.Core shared source -->
+  <ItemGroup>
+    <Compile Include="$(AzureCoreSharedSources)Argument.cs" LinkBase="SharedSource\Azure.Core" />
+    <Compile Include="$(AzureCoreSharedSources)TaskExtensions.cs" LinkBase="SharedSource\Azure.Core" />
+  </ItemGroup>
+
+  <!-- Import Event Hubs shared source -->
+  <Import Project="$(MSBuildThisFileDirectory)..\..\..\Azure.Messaging.EventHubs.Shared\src\Azure.Messaging.EventHubs.Shared.Stress.projitems" Label="Stress" />
+</Project>

sdk/eventhub/Azure.Messaging.EventHubs/stress/src/Configurations/BufferedProducerTestConfig.cs

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+
+namespace Azure.Messaging.EventHubs.Stress
+{
+    internal class BufferedProducerTestConfig : TestConfig
+    {
+        // Test Configurations
+
+        // The number of events to generate and enqueue during each iteration of PerformEnqueue
+        public int EventEnqueueListSize = 50;
+        public long MaximumEventListSize = 1_000_000;
+
+        // Buffered Producer Configuration
+
+        public TimeSpan MaxWaitTime = TimeSpan.FromSeconds(5);
+    }
+}

sdk/eventhub/Azure.Messaging.EventHubs/stress/src/Configurations/EventProducerTestConfig.cs

@@ -0,0 +1,13 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+
+namespace Azure.Messaging.EventHubs.Stress
+{
+    internal class EventProducerTestConfig : TestConfig
+    {
+        // The number of events to generate and put into a batch during each iteration of PerformSend
+        public int PublishBatchSize = 50;
+    }
+}

sdk/eventhub/Azure.Messaging.EventHubs/stress/src/Configurations/TestConfig.cs

@@ -0,0 +1,32 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+
+namespace Azure.Messaging.EventHubs.Stress
+{
+    internal class TestConfig
+    {
+        // true if the given test should be run.
+        public bool Run;
+
+        // Resource Configurations
+
+        public string EventHubsConnectionString;
+        public string EventHub;
+        public string InstrumentationKey;
+
+        // Test Run Configurations
+
+        public int DurationInHours = 120;
+
+        // Publishing Configurations
+
+        public int ConcurrentSends = 5;
+        public int PublishingBodyMinBytes = 100;
+        public int PublishingBodyRegularMaxBytes = 262144;
+        public int LargeMessageRandomFactorPercent = 50;
+        public TimeSpan SendTimeout = TimeSpan.FromMinutes(3);
+        public TimeSpan? ProducerPublishingDelay = TimeSpan.FromMilliseconds(4000);
+    }
+}