From f61c5abff7d429011fda5ed7b353ce9d60a438b1 Mon Sep 17 00:00:00 2001 From: Warren Jones Date: Wed, 20 Oct 2021 20:38:25 -0700 Subject: [PATCH] Add Quantum CLI Extension Live E2E Client Validation Tests (#3943) * Add Quantum CLI extension tests.live files * Delete commented-out code * Add azdev setup * Fix azdev setup path params * Add Set-Location in Run.ps1 * Omit --cli param for azdev setup * Use environment variable for Subscription ID * Update environment variable names * Update 'Running locally' instructions in README.md * Skip test if provider not in AZURE_QUANTUM_CAPABILITIES * Convert capabilities string to lowercase * Use --xml-path param to save test output with the recordings * Handle multiple providers in capabilities logic * Test re-run * Test re-run, Take 2 * Remove test_results.xml --- .../tests/latest/test_quantum_workspace.py | 30 +++++++++++-------- .../azext_quantum/tests/latest/utils.py | 25 ++++++++++++---- src/quantum/tests.live/README.md | 18 +++++++++++ src/quantum/tests.live/Run.ps1 | 21 +++++++++++++ 4 files changed, 75 insertions(+), 19 deletions(-) create mode 100644 src/quantum/tests.live/README.md create mode 100644 src/quantum/tests.live/Run.ps1 diff --git a/src/quantum/azext_quantum/tests/latest/test_quantum_workspace.py b/src/quantum/azext_quantum/tests/latest/test_quantum_workspace.py index 5be3608e5e5..62a56f476a2 100644 --- a/src/quantum/azext_quantum/tests/latest/test_quantum_workspace.py +++ b/src/quantum/azext_quantum/tests/latest/test_quantum_workspace.py @@ -10,7 +10,7 @@ from azure.cli.testsdk import (ScenarioTest, ResourceGroupPreparer) from azure.cli.core.azclierror import RequiredArgumentMissingError, ResourceNotFoundError -from .utils import get_test_resource_group, get_test_workspace, get_test_workspace_location, get_test_workspace_storage, get_test_workspace_random_name +from .utils import get_test_resource_group, get_test_workspace, get_test_workspace_location, get_test_workspace_storage, get_test_workspace_random_name, get_test_capabilities, get_test_workspace_provider_sku_list, all_providers_are_in_capabilities from ..._version_check_helper import check_version from datetime import datetime from ...__init__ import CLI_REPORTED_VERSION @@ -56,18 +56,22 @@ def test_workspace_create_destroy(self): test_resource_group = get_test_resource_group() test_workspace_temp = get_test_workspace_random_name() test_storage_account = get_test_workspace_storage() - - # create - self.cmd(f'az quantum workspace create -g {test_resource_group} -w {test_workspace_temp} -l {test_location} -a {test_storage_account} -r "Microsoft/Basic" -o json --skip-role-assignment', checks=[ - self.check("name", test_workspace_temp), - self.check("provisioningState", "Accepted") # Status is accepted since we're not linking the storage account. - ]) - - # delete - self.cmd(f'az quantum workspace delete -g {test_resource_group} -w {test_workspace_temp} -o json', checks=[ - self.check("name", test_workspace_temp), - self.check("provisioningState", "Deleting") - ]) + test_provider_sku_list = get_test_workspace_provider_sku_list() + + if all_providers_are_in_capabilities(test_provider_sku_list, get_test_capabilities()): + # create + self.cmd(f'az quantum workspace create -g {test_resource_group} -w {test_workspace_temp} -l {test_location} -a {test_storage_account} -r {test_provider_sku_list} -o json --skip-role-assignment', checks=[ + self.check("name", test_workspace_temp), + self.check("provisioningState", "Accepted") # Status is accepted since we're not linking the storage account. + ]) + + # delete + self.cmd(f'az quantum workspace delete -g {test_resource_group} -w {test_workspace_temp} -o json', checks=[ + self.check("name", test_workspace_temp), + self.check("provisioningState", "Deleting") + ]) + else: + self.skipTest(f"Skipping test_workspace_create_destroy: One or more providers in '{test_provider_sku_list}' not found in AZURE_QUANTUM_CAPABILITIES") @live_only() def test_workspace_errors(self): diff --git a/src/quantum/azext_quantum/tests/latest/utils.py b/src/quantum/azext_quantum/tests/latest/utils.py index ca485311635..e9eb48e1f33 100644 --- a/src/quantum/azext_quantum/tests/latest/utils.py +++ b/src/quantum/azext_quantum/tests/latest/utils.py @@ -8,28 +8,41 @@ TEST_WORKSPACE_DEFAULT = "e2e-qsharp-tests" TEST_WORKSPACE_DEFAULT_LOCATION = "westus2" TEST_WORKSPACE_DEFAULT_STORAGE = "/subscriptions/916dfd6d-030c-4bd9-b579-7bb6d1926e97/resourceGroups/e2e-scenarios/providers/Microsoft.Storage/storageAccounts/e2etests" +TEST_WORKSPACE_DEFAULT_PROVIDER_SKU_LIST = "Microsoft/Basic" +TEST_CAPABILITIES_DEFAULT = "new.microsoft;submit.microsoft" def get_from_os_environment(env_name, default): import os return os.environ[env_name] if env_name in os.environ and os.environ[env_name] != "" else default def get_test_subscription_id(): - return get_from_os_environment("AZUREQUANTUM_SUBSCRIPTION_ID", TEST_SUBS_DEFAULT) + return get_from_os_environment("AZURE_QUANTUM_SUBSCRIPTION_ID", TEST_SUBS_DEFAULT) def get_test_resource_group(): - return get_from_os_environment("AZUREQUANTUM_WORKSPACE_RG", TEST_RG_DEFAULT) + return get_from_os_environment("AZURE_QUANTUM_WORKSPACE_RG", TEST_RG_DEFAULT) def get_test_workspace(): - return get_from_os_environment("AZUREQUANTUM_WORKSPACE_NAME", TEST_WORKSPACE_DEFAULT) + return get_from_os_environment("AZURE_QUANTUM_WORKSPACE_NAME", TEST_WORKSPACE_DEFAULT) def get_test_workspace_location(): - return get_from_os_environment("AZUREQUANTUM_WORKSPACE_LOCATION", TEST_WORKSPACE_DEFAULT_LOCATION) + return get_from_os_environment("AZURE_QUANTUM_WORKSPACE_LOCATION", TEST_WORKSPACE_DEFAULT_LOCATION) def get_test_workspace_storage(): - return get_from_os_environment("AZUREQUANTUM_WORKSPACE_STORAGE", TEST_WORKSPACE_DEFAULT_STORAGE) + return get_from_os_environment("AZURE_QUANTUM_WORKSPACE_STORAGE", TEST_WORKSPACE_DEFAULT_STORAGE) + +def get_test_workspace_provider_sku_list(): + return get_from_os_environment("AZURE_QUANTUM_WORKSPACE_PROVIDER_SKU_LIST", TEST_WORKSPACE_DEFAULT_PROVIDER_SKU_LIST) + +def get_test_capabilities(): + return get_from_os_environment("AZURE_QUANTUM_CAPABILITIES", TEST_CAPABILITIES_DEFAULT).lower() def get_test_workspace_random_name(): import random return "e2e-test-w" + str(random.randint(1000000, 9999999)) - +def all_providers_are_in_capabilities(provider_sku_string, capabilities_string): + for provide_sku_pair in provider_sku_string.split(';'): + provider = "new." + provide_sku_pair.split('/')[0].lower() + if provider not in capabilities_string: + return False + return True diff --git a/src/quantum/tests.live/README.md b/src/quantum/tests.live/README.md new file mode 100644 index 00000000000..f4027bfa371 --- /dev/null +++ b/src/quantum/tests.live/README.md @@ -0,0 +1,18 @@ +# Quantum CLI Extension Live tests + +This folder contains Run.ps1, a script the will run all of the live tests for the Quantum CLI Extension. Live tests are end-to-end tests that require an actual connection with Azure Quantum +to complete successfully. + + +## Running locally + +1. Set the following environment variable to the Subscription ID of the "QuArC Internal Consumption" subscription: + * `$Env:AZURE_QUANTUM_SUBSCRIPTION_ID = "[Subscription ID]"` + +2. Login to Azure using either: + * the [Azure Account extension in VS Code](https://marketplace.visualstudio.com/items?itemName=ms-vscode.azure-account) + * `az login` from the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/) + +3. Make this folder your current directory location: src\quantum\tests.live + +4. Use [`.\Run.ps1 -SkipInstall`](.\Run.ps1) to run all the tests. diff --git a/src/quantum/tests.live/Run.ps1 b/src/quantum/tests.live/Run.ps1 new file mode 100644 index 00000000000..7b5a97fd6ec --- /dev/null +++ b/src/quantum/tests.live/Run.ps1 @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +param ( + [Parameter()] + [switch] + $SkipInstall=$False +) + +# For debug, print all relevant environment variables: +Get-ChildItem env:AZURE*, env:*VERSION, env:*OUTDIR | Format-Table | Out-String | Write-Host + +# Run the Quantum CLI Extension tests in an azdev environment +Set-Location ../../.. +python -m venv env +env\Scripts\activate.ps1 +python -m pip install -U pip +pip install azdev +azdev setup --repo . +azdev extension add quantum +az account set -s $Env:AZURE_QUANTUM_SUBSCRIPTION_ID +azdev test quantum --live --verbose --xml-path src\quantum\azext_quantum\tests\latest\recordings