From 6cd1f37168c4ff7fd65965e96ddc2e25d70836a1 Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Tue, 13 Apr 2021 17:03:49 -0700 Subject: [PATCH] add perf tests (#17519) * add perf tests * update * update * updates --- .../tests/perfstress_tests/README.md | 50 +++++++++++++++++++ .../tests/perfstress_tests/__init__.py | 0 .../tests/perfstress_tests/list_anomalies.py | 43 ++++++++++++++++ .../tests/perfstress_tests/list_incidents.py | 44 ++++++++++++++++ .../perfstress_tests/list_root_causes.py | 43 ++++++++++++++++ 5 files changed, 180 insertions(+) create mode 100644 sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md create mode 100644 sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/__init__.py create mode 100644 sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_anomalies.py create mode 100644 sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_incidents.py create mode 100644 sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_root_causes.py diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md new file mode 100644 index 000000000000..37e7f0d57a3b --- /dev/null +++ b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/README.md @@ -0,0 +1,50 @@ +# Metrics advisor Performance Tests + +In order to run the performance tests, the `azure-devtools` package must be installed. This is done as part of the `dev_requirements`. +Start be creating a new virtual environment for your perf tests. This will need to be a Python 3 environment, preferably >=3.7. + +### Setup for test resources + +These tests will run against a pre-configured metrics advisor service(see [Setting up Resource for Perf test Guide](https://microsoft.sharepoint.com/teams/AzureDeveloperExperience/_layouts/15/Doc.aspx?sourcedoc={b489fb65-ef3e-410c-a69e-c23fccf5fcca}&action=edit&wd=target%28Untitled%20Section.one%7C93e23c77-5539-4c7a-9059-efa745a43f58%2FSetting%20up%20Resource%20for%20Perf%20test%20Guide%7C395ad307-9fd1-4971-bcd5-6d336dde77cd%2F%29&wdorigin=703) about how to setup the instance and ingest data). The following environment variable will need to be set for the tests to access the live resources: +``` +AZURE_METRICS_ADVISOR_ENDPOINT= +AZURE_METRICS_ADVISOR_SUBSCRIPTION_KEY= +AZURE_METRICS_ADVISOR_API_KEY= +AZURE_METRICS_ADVISOR_ANOMALY_ALERT_CONFIGURATION_ID= +AZURE_METRICS_ADVISOR_ALERT_ID= +AZURE_METRICS_ADVISOR_ANOMALY_DETECTION_CONFIGURATION_ID= +AZURE_METRICS_ADVISOR_INCIDENT_ID= +``` + +### Setup for perf test runs + +```cmd +(env) ~/azure-ai-metricsadvisor> pip install -r dev_requirements.txt +(env) ~/azure-ai-metricsadvisor> pip install -e . +``` + +## Test commands + +When `azure-devtools` is installed, you will have access to the `perfstress` command line tool, which will scan the current module for runable perf tests. Only a specific test can be run at a time (i.e. there is no "run all" feature). + +```cmd +(env) ~/azure-ai-metricsadvisor> cd tests +(env) ~/azure-ai-metricsadvisor/tests> perfstress +``` +Using the `perfstress` command alone will list the available perf tests found. + +### Common perf command line options +These options are available for all perf tests: +- `--duration=10` Number of seconds to run as many operations (the "run" function) as possible. Default is 10. +- `--iterations=1` Number of test iterations to run. Default is 1. +- `--parallel=1` Number of tests to run in parallel. Default is 1. +- `--warmup=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5. +- `--sync` Whether to run the tests in sync or async. Default is False (async). This flag must be used for Storage legacy tests, which do not support async. +- `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted). + +## Example command +```cmd +(env) ~/azure-ai-metricsadvisor/tests> perfstress ListAnomaliesTest +(env) ~/azure-ai-metricsadvisor/tests> perfstress ListIncidentsTest +(env) ~/azure-ai-metricsadvisor/tests> perfstress ListRootCausesTest +``` diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/__init__.py b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_anomalies.py b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_anomalies.py new file mode 100644 index 000000000000..b74459b48d3d --- /dev/null +++ b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_anomalies.py @@ -0,0 +1,43 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os + +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.ai.metricsadvisor import MetricsAdvisorClient as SyncClient, MetricsAdvisorKeyCredential +from azure.ai.metricsadvisor.aio import MetricsAdvisorClient as AsyncClient + + +class ListAnomaliesTest(PerfStressTest): + def __init__(self, arguments): + super().__init__(arguments) + service_endpoint = os.getenv("AZURE_METRICS_ADVISOR_ENDPOINT") + subscription_key = os.getenv("AZURE_METRICS_ADVISOR_SUBSCRIPTION_KEY") + api_key = os.getenv("AZURE_METRICS_ADVISOR_API_KEY") + self.anomaly_alert_configuration_id = os.getenv("AZURE_METRICS_ADVISOR_ANOMALY_ALERT_CONFIGURATION_ID") + self.alert_id = os.getenv("AZURE_METRICS_ADVISOR_ALERT_ID") + self.service_client = SyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) + self.async_service_client = AsyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) + + async def close(self): + await self.async_service_client.close() + await super().close() + + def run_sync(self): + results = self.service_client.list_anomalies( + alert_configuration_id=self.anomaly_alert_configuration_id, + alert_id=self.alert_id, + ) + for _ in results: + pass + + async def run_async(self): + results = self.async_service_client.list_anomalies( + alert_configuration_id=self.anomaly_alert_configuration_id, + alert_id=self.alert_id, + ) + async for _ in results: + pass diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_incidents.py b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_incidents.py new file mode 100644 index 000000000000..8604236cbb53 --- /dev/null +++ b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_incidents.py @@ -0,0 +1,44 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import datetime +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.ai.metricsadvisor import MetricsAdvisorClient as SyncClient, MetricsAdvisorKeyCredential +from azure.ai.metricsadvisor.aio import MetricsAdvisorClient as AsyncClient + + +class ListIncidentsTest(PerfStressTest): + def __init__(self, arguments): + super().__init__(arguments) + service_endpoint = os.getenv("AZURE_METRICS_ADVISOR_ENDPOINT") + subscription_key = os.getenv("AZURE_METRICS_ADVISOR_SUBSCRIPTION_KEY") + api_key = os.getenv("AZURE_METRICS_ADVISOR_API_KEY") + self.anomaly_detection_configuration_id = os.getenv("AZURE_METRICS_ADVISOR_ANOMALY_DETECTION_CONFIGURATION_ID") + self.service_client = SyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) + self.async_service_client = AsyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) + + async def close(self): + await self.async_service_client.close() + await super().close() + + def run_sync(self): + results = self.service_client.list_incidents( + detection_configuration_id=self.anomaly_detection_configuration_id, + start_time=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc), + end_time=datetime.datetime(2020, 10, 21, tzinfo=datetime.timezone.utc), + ) + for _ in results: + pass + + async def run_async(self): + results = self.async_service_client.list_incidents( + detection_configuration_id=self.anomaly_detection_configuration_id, + start_time=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc), + end_time=datetime.datetime(2020, 10, 21, tzinfo=datetime.timezone.utc), + ) + async for _ in results: + pass diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_root_causes.py b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_root_causes.py new file mode 100644 index 000000000000..911a24979586 --- /dev/null +++ b/sdk/metricsadvisor/azure-ai-metricsadvisor/tests/perfstress_tests/list_root_causes.py @@ -0,0 +1,43 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os + +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.ai.metricsadvisor import MetricsAdvisorClient as SyncClient, MetricsAdvisorKeyCredential +from azure.ai.metricsadvisor.aio import MetricsAdvisorClient as AsyncClient + + +class ListRootCausesTest(PerfStressTest): + def __init__(self, arguments): + super().__init__(arguments) + service_endpoint = os.getenv("AZURE_METRICS_ADVISOR_ENDPOINT") + subscription_key = os.getenv("AZURE_METRICS_ADVISOR_SUBSCRIPTION_KEY") + api_key = os.getenv("AZURE_METRICS_ADVISOR_API_KEY") + self.anomaly_detection_configuration_id = os.getenv("AZURE_METRICS_ADVISOR_ANOMALY_DETECTION_CONFIGURATION_ID") + self.incident_id = os.getenv("AZURE_METRICS_ADVISOR_INCIDENT_ID") + self.service_client = SyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) + self.async_service_client = AsyncClient(service_endpoint, MetricsAdvisorKeyCredential(subscription_key, api_key)) + + async def close(self): + await self.async_service_client.close() + await super().close() + + def run_sync(self): + results = self.service_client.list_incident_root_causes( + detection_configuration_id=self.anomaly_detection_configuration_id, + incident_id=self.incident_id, + ) + for _ in results: + pass + + async def run_async(self): + results = self.async_service_client.list_incident_root_causes( + detection_configuration_id=self.anomaly_detection_configuration_id, + incident_id=self.incident_id, + ) + async for _ in results: + pass