Smoke test isolation per package

common/smoketest/requirements-release.txt

@@ -6,3 +6,5 @@ azure-keyvault-certificates>=0.0.0b1
 azure-keyvault-keys>=0.0.0b1
 azure-keyvault-secrets>=0.0.0b1
 azure-storage-blob>=0.0.0b1
+# Required for template release to trigger smoke tests
+azure-template>=0.0.0b1

eng/pipelines/templates/jobs/smoke.tests.yml

@@ -1,9 +1,9 @@
 parameters:
   - name: Daily
     default: true
-  - name: Artifacts
+  - name: Artifact
     type: object
-    default: []
+    default: {}
   - name: ArtifactName
     type: string
     default: "not-specified"
@@ -16,16 +16,15 @@ jobs:
         name: "azsdk-pool-mms-ubuntu-2004-general"
         vmImage: "MMSUbuntu20.04"
       steps:
-        - ${{ each artifact in parameters.Artifacts }}:
-          - ${{ if and(ne(variables['Skip.Release'], 'true'), ne(artifact.skipPublishPackage, 'true')) }}:
-            - pwsh: |
-                $packages = Get-Content $(Build.SourcesDirectory)/common/smoketest/requirements-release.txt
-                if ($packages | Where-Object { $_ -match "${{ replace(artifact.name, '_', '-') }}" }) {
-                  Write-Host "Smoke tests will run for ${{ artifact.name }}"
-                  Write-Host "##vso[task.setvariable variable=RunSmokeTests;]true"
-                }
-              name: check_smoke_tests_${{ artifact.safeName }}
-              displayName: Check smoke test eligibility for ${{ artifact.name }}
+        - ${{ if and(ne(variables['Skip.Release'], 'true'), ne(parameters.Artifact.skipPublishPackage, 'true')) }}:
+          - pwsh: |
+              $packages = Get-Content $(Build.SourcesDirectory)/common/smoketest/requirements-release.txt
+              if ($packages | Where-Object { $_ -match "${{ replace(parameters.Artifact.name, '_', '-') }}" }) {
+                Write-Host "Smoke tests will run for ${{ parameters.Artifact.name }}"
+                Write-Host "##vso[task.setvariable variable=RunSmokeTests;]true"
+              }
+            name: check_smoke_tests_${{ parameters.Artifact.safeName }}
+            displayName: Check smoke test eligibility for ${{ parameters.Artifact.name }}
 
         - pwsh: |
             Write-Host "Setting RunSmokeTests to $($env:RunSmokeTests)"
@@ -34,89 +33,91 @@ jobs:
           env:
             RunSmokeTests: $(RunSmokeTests)
 
-  - job:
+  - job: run_smoke_test
+    displayName: Run Smoke Test
     ${{ if eq(parameters.Daily, false) }}:
       dependsOn: smoke_test_eligibility
       condition: and(succeeded(), eq(dependencies.smoke_test_eligibility.outputs['output_eligibility.RunSmokeTests'], true))
     strategy:
       matrix:
-        Python_27_Linux (AzureCloud):
-          PythonVersion: '2.7'
-          SkipAsyncInstall: true
-          Pool: "azsdk-pool-mms-ubuntu-2004-general"
-          OSVmImage: "MMSUbuntu20.04"
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_37_Linux (AzureCloud):
-          PythonVersion: '3.7'
-          Pool: "azsdk-pool-mms-ubuntu-2004-general"
-          OSVmImage: "MMSUbuntu20.04"
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
         Python_38_Linux (AzureCloud):
           PythonVersion: '3.8'
           Pool: "azsdk-pool-mms-ubuntu-2004-general"
           OSVmImage: "MMSUbuntu20.04"
           SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
           ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_38_Linux (AzureCloud Canary):
-          PythonVersion: '3.8'
-          Pool: "azsdk-pool-mms-ubuntu-2004-general"
-          OSVmImage: "MMSUbuntu20.04"
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources-preview)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-          Location: 'eastus2euap'
-        Python_37_Windows (AzureCloud):
-          PythonVersion: '3.7'
-          Pool: "azsdk-pool-mms-win-2019-general"
-          OSVmImage: "MMS2019"
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_38_Windows (AzureCloud):
-          PythonVersion: '3.8'
-          Pool: "azsdk-pool-mms-ubuntu-2004-general"
-          OSVmImage: "MMSUbuntu20.04"
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_37_Mac (AzureCloud):
-          PythonVersion: '3.7'
-          Pool: Azure Pipelines
-          OSVmImage: macOS-10.15
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_38_Mac (AzureCloud):
-          PythonVersion: '3.8'
-          Pool: Azure Pipelines
-          OSVmImage: macOS-10.15
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_38_Linux (AzureUSGovernment):
-          PythonVersion: '3.8'
-          Pool: "azsdk-pool-mms-ubuntu-2004-general"
-          OSVmImage: "MMSUbuntu120.04"
-          SubscriptionConfiguration: $(sub-config-gov-test-resources)
-          ArmTemplateParameters: $(azureUSGovernmentArmParameters)
-        Python_37_Windows (AzureUSGovernment):
-          PythonVersion: '3.7'
-          Pool: "azsdk-pool-mms-ubuntu-2004-general"
-          OSVmImage: "MMSUbuntu20.04"
-          SubscriptionConfiguration: $(sub-config-gov-test-resources)
-          ArmTemplateParameters: $(azureUSGovernmentArmParameters)
-        Python_38_Linux (AzureChinaCloud):
-          PythonVersion: '3.8'
-          Pool: "azsdk-pool-mms-ubuntu-2004-general"
-          OSVmImage: "MMSUbuntu20.04"
-          SubscriptionConfiguration: $(sub-config-cn-test-resources)
-          Location: 'chinanorth'
-          ArmTemplateParameters: $(azureChinaCloudArmParameters)
-        Python_37_Windows (AzureChinaCloud):
-          PythonVersion: '3.7'
-          Pool: "azsdk-pool-mms-ubuntu-2004-general"
-          OSVmImage: "MMSUbuntu20.04"
-          SubscriptionConfiguration: $(sub-config-cn-test-resources)
-          Location: 'chinanorth'
-          ArmTemplateParameters: $(azureChinaCloudArmParameters)
-
+        ${{ if eq(parameters.Daily, true) }}:
+          Python_27_Linux (AzureCloud):
+            PythonVersion: '2.7'
+            SkipAsyncInstall: true
+            Pool: "azsdk-pool-mms-ubuntu-2004-general"
+            OSVmImage: "MMSUbuntu20.04"
+            SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+            ArmTemplateParameters: $(azureCloudArmParameters)
+          Python_37_Linux (AzureCloud):
+            PythonVersion: '3.7'
+            Pool: "azsdk-pool-mms-ubuntu-2004-general"
+            OSVmImage: "MMSUbuntu20.04"
+            SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+            ArmTemplateParameters: $(azureCloudArmParameters)
+          Python_38_Linux (AzureCloud Canary):
+            PythonVersion: '3.8'
+            Pool: "azsdk-pool-mms-ubuntu-2004-general"
+            OSVmImage: "MMSUbuntu20.04"
+            SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources-preview)
+            ArmTemplateParameters: $(azureCloudArmParameters)
+            Location: 'eastus2euap'
+          Python_37_Windows (AzureCloud):
+            PythonVersion: '3.7'
+            Pool: "azsdk-pool-mms-win-2019-general"
+            OSVmImage: "MMS2019"
+            SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+            ArmTemplateParameters: $(azureCloudArmParameters)
+          Python_38_Windows (AzureCloud):
+            PythonVersion: '3.8'
+            Pool: "azsdk-pool-mms-ubuntu-2004-general"
+            OSVmImage: "MMSUbuntu20.04"
+            SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+            ArmTemplateParameters: $(azureCloudArmParameters)
+          Python_37_Mac (AzureCloud):
+            PythonVersion: '3.7'
+            Pool: Azure Pipelines
+            OSVmImage: macOS-10.15
+            SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+            ArmTemplateParameters: $(azureCloudArmParameters)
+          Python_38_Mac (AzureCloud):
+            PythonVersion: '3.8'
+            Pool: Azure Pipelines
+            OSVmImage: macOS-10.15
+            SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+            ArmTemplateParameters: $(azureCloudArmParameters)
+          Python_38_Linux (AzureUSGovernment):
+            PythonVersion: '3.8'
+            Pool: "azsdk-pool-mms-ubuntu-2004-general"
+            OSVmImage: "MMSUbuntu120.04"
+            SubscriptionConfiguration: $(sub-config-gov-test-resources)
+            ArmTemplateParameters: $(azureUSGovernmentArmParameters)
+          Python_37_Windows (AzureUSGovernment):
+            PythonVersion: '3.7'
+            Pool: "azsdk-pool-mms-ubuntu-2004-general"
+            OSVmImage: "MMSUbuntu20.04"
+            SubscriptionConfiguration: $(sub-config-gov-test-resources)
+            ArmTemplateParameters: $(azureUSGovernmentArmParameters)
+          Python_38_Linux (AzureChinaCloud):
+            PythonVersion: '3.8'
+            Pool: "azsdk-pool-mms-ubuntu-2004-general"
+            OSVmImage: "MMSUbuntu20.04"
+            SubscriptionConfiguration: $(sub-config-cn-test-resources)
+            Location: 'chinanorth'
+            ArmTemplateParameters: $(azureChinaCloudArmParameters)
+          Python_37_Windows (AzureChinaCloud):
+            PythonVersion: '3.7'
+            Pool: "azsdk-pool-mms-ubuntu-2004-general"
+            OSVmImage: "MMSUbuntu20.04"
+            SubscriptionConfiguration: $(sub-config-cn-test-resources)
+            Location: 'chinanorth'
+            ArmTemplateParameters: $(azureChinaCloudArmParameters)
+
     pool:
       name: $(Pool)
       vmImage: $(OSVmImage)

eng/pipelines/templates/stages/archetype-python-release.yml

@@ -222,16 +222,13 @@ stages:
                           PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases"
                           PRLabels: "auto-merge"
                           CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}'
+          - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}:
+            - template: /eng/pipelines/templates/jobs/smoke.tests.yml
+              parameters:
+                Daily: false
+                ArtifactName: ${{ parameters.ArtifactName }}
+                Artifact: ${{ artifact }}
 
-  - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}:
-    - stage: SmokeTest_Release_Packages
-      displayName: Smoke Test Release Packages
-      jobs:
-        - template: /eng/pipelines/templates/jobs/smoke.tests.yml
-          parameters:
-            Daily: false
-            ArtifactName: ${{ parameters.ArtifactName }}
-            Artifacts: ${{ parameters.Artifacts }}
 
   - stage: Integration
     dependsOn: ${{parameters.DependsOn}}