From ae43fcbcafc60f4f3752c0eaed147164c2b17591 Mon Sep 17 00:00:00 2001 From: Alan Zimmer <48699787+alzimmermsft@users.noreply.github.com> Date: Tue, 1 Feb 2022 09:58:30 -0800 Subject: [PATCH] Standardize Build and Test Steps for CI and Live Test Pipelines (#26769) Standardize Build and Test Steps for CI and Live Test Pipelines --- eng/pipelines/templates/jobs/ci.tests.yml | 147 ++++-------------- eng/pipelines/templates/jobs/ci.yml | 32 ++-- eng/pipelines/templates/jobs/live.tests.yml | 61 +++----- .../templates/stages/archetype-sdk-client.yml | 11 +- .../templates/stages/archetype-sdk-tests.yml | 24 +-- .../stages/cosmos-emulator-matrix.json | 2 +- .../templates/stages/cosmos-sdk-client.yml | 21 +-- .../templates/stages/platform-matrix.json | 10 +- .../templates/steps/build-and-test.yml | 141 +++++++++++++++++ .../steps/initialize-test-environment.yml | 4 +- .../steps/install-reporting-tools.yml | 3 +- eng/pipelines/templates/variables/globals.yml | 15 +- sdk/appconfiguration/tests.yml | 5 +- sdk/core/ci.yml | 2 +- sdk/core/tests.yml | 2 +- sdk/core/unsupported-version-matrix.json | 2 +- sdk/cosmos/tests.yml | 12 +- sdk/keyvault/ci.yml | 2 +- sdk/spring/spring-test-template.yml | 6 +- sdk/storage/platform-matrix.json | 12 +- sdk/storage/tests-template.yml | 3 - 21 files changed, 284 insertions(+), 233 deletions(-) create mode 100644 eng/pipelines/templates/steps/build-and-test.yml diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index bb4aed9ae4afe..d8cbd9cbb0f99 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -17,12 +17,6 @@ parameters: - name: PreTestSteps type: object default: [] - - name: BuildOptions - type: string - default: '$(DefaultOptions) -T 2C --quiet -DskipTests -Dgpg.skip -DtrimStackTrace=false -Dmaven.javadoc.skip=true -Dcodesnippet.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -Djacoco.skip=true' - - name: TestOptions - type: string - default: '$(DefaultTestOptions) -Dparallel-test-playback -DAZURE_TEST_DEBUG=$(IsDebug)' - name: Matrix type: object - name: CloudConfig @@ -40,6 +34,21 @@ parameters: - name: PreBuildSteps type: object default: [] + - name: TestGoals + type: string + default: $(TestGoals) + - name: TestOptions + type: string + default: $(TestOptions) + - name: AdditionalBuildOptions + type: string + default: $(AdditionalBuildOptions) + - name: TestParallelization + type: string + default: '1C' + - name: BuildParallelization + type: string + default: '2C' jobs: - job: 'Test' @@ -70,121 +79,21 @@ jobs: SDKType: ${{ parameters.SDKType }} - template: ../steps/install-reporting-tools.yml + parameters: + JdkVersion: $(JavaTestVersion) - ${{ parameters.PreBuildSteps }} - ${{ parameters.PreTestSteps }} - - task: PythonScript@0 - # The root pom.xml contains both track 1, data, and track 2, client libraries. - # This task calls a Python script that will go through all of the pom files - # in the repository looking, specifically, for ones whose parent is - # azure-client-sdk-parent and create a temporary pom file with just those libraries. - # Note: It excludes items within the /eng directory otherwise we'd be picking up - # jacoco and spotbugs which are unnecessary for the From Source run. - displayName: 'Generate Aggregate Pom for From Source Build' - inputs: - scriptPath: eng/scripts/generate_from_source_pom.py - arguments: '--project-list $(ProjectList)' - condition: and(succeeded(), eq(variables['ShouldRunSourceTests'],'true')) - - - task: Maven@3 - displayName: 'Build for non-From Source run' - inputs: - mavenPomFile: pom.xml - options: ${{ parameters.BuildOptions }} -pl $(ProjectList) -am - mavenOptions: '$(MemoryOptions) $(LoggingOptions)' - javaHomeOption: 'JDKVersion' - jdkVersionOption: $(JavaTestVersion) - jdkArchitectureOption: 'x64' - publishJUnitResults: false - goals: 'install' - # we want to run this when we aren't running from source and aren't generating Jacoco test coverage - condition: and(succeeded(), ne(variables['TestFromSource'], 'true'), not(and(eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['JavaTestVersion'], '1.11'), eq('${{ parameters.SDKType }}', 'client')))) - - - task: Maven@3 - displayName: 'Run tests' - inputs: - mavenPomFile: pom.xml - options: ${{ parameters.TestOptions }} $(TestOptions) $(AdditionalOptions) - mavenOptions: '$(MemoryOptions) $(LoggingOptions)' - javaHomeOption: 'JDKVersion' - jdkVersionOption: $(JavaTestVersion) - jdkArchitectureOption: 'x64' - publishJUnitResults: false - goals: $(TestGoals) $(AdditionalTestGoals) - # we want to run this when TestFromSource isn't true - condition: and(succeeded(), ne(variables['TestFromSource'],'true')) - - - task: Maven@3 - displayName: 'Generate aggregate code coverage report' - condition: and(eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['JavaTestVersion'], '1.11'), eq('${{ parameters.SDKType }}', 'client'), eq('${{ parameters.SkipAggregateReports }}', 'false')) - inputs: - mavenPomFile: sdk/${{ parameters.ServiceDirectory }}/pom.xml - options: '$(DefaultOptions) -Pcoverage' - mavenOptions: '$(MemoryOptions) $(LoggingOptions)' - javaHomeOption: 'JDKVersion' - jdkVersionOption: $(JavaTestVersion) - jdkArchitectureOption: 'x64' - goals: jacoco:report-aggregate - # we want to run this when TestFromSource isn't true - condition: and(succeeded(), ne(variables['TestFromSource'],'true')) - - - task: Maven@3 - displayName: 'Build for From Source run' - inputs: - mavenPomFile: ClientFromSourcePom.xml - goals: 'install' - # Build and install every track 2, client, library. The purpose is to have all of - # the libraries installed on the machine so we can simply run tests with the -amd - # switch and we can remove the SNAFU of dependencies that bloated the ci.yml files' - # AdditionalModules because we ended up having to list every transitive azure client - # dependency. Note: The -T 1C says spin up a build thread per core. Since we're building - # all of the track 2, client, libraries (which is now over 100) this effectively cuts the - # the build time in half. - options: ${{ parameters.BuildOptions }} - mavenOptions: '$(MemoryOptions) $(LoggingOptions)' - javaHomeOption: 'JDKVersion' - jdkVersionOption: $(JavaBuildVersion) - jdkArchitectureOption: 'x64' - publishJUnitResults: false - condition: and(succeeded(), and(eq(variables['TestFromSource'],'true'), eq(variables['ShouldRunSourceTests'],'true'))) - - - task: Maven@3 - displayName: 'Run tests for From Source run' - inputs: - mavenPomFile: ClientFromSourcePom.xml - # For the From Source runs we don't want the -am switch as we don't care about running tests for our dependencies - # but we do want the -amd switch because we want to run tests on things that depend on us. - options: ${{ parameters.TestOptions }} $(TestOptions) -amd -T 1C - mavenOptions: '$(MemoryOptions) $(LoggingOptions)' - javaHomeOption: 'JDKVersion' - jdkVersionOption: $(JavaTestVersion) - jdkArchitectureOption: 'x64' - publishJUnitResults: false - # The From Source runs only care about running tests for libraries within an SDK/area - # as well as libraries which depend on them and not the other stages in the Maven - # lifecycle that test or verify targets would execute. The libraries were already - # compiled and installed in the previous task, this tasks only runs tests. - goals: $(TestGoals) $(AdditionalTestGoals) - condition: and(succeeded(), and(eq(variables['TestFromSource'],'true'), eq(variables['ShouldRunSourceTests'],'true'))) - - - template: ../steps/upload-repository-on-failure.yml - - - template: ../steps/retain-heap-dump-hprofs.yml - - - task: PublishTestResults@2 - condition: and(always(), or(ne(variables['TestFromSource'],'true'), eq(variables['ShouldRunSourceTests'],'true'))) - inputs: - mergeTestResults: true - testRunTitle: $(Agent.JobName) - - # Azure DevOps only seems to respect the last code coverage result published, so only do this for Windows + Java LTS. - # Code coverage reporting is setup only for Track 2 modules. - - task: PublishCodeCoverageResults@1 - condition: and(eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['JavaTestVersion'], '1.11'), eq('${{ parameters.SDKType }}', 'client')) - inputs: - codeCoverageTool: JaCoCo - summaryFileLocation: sdk/${{ parameters.ServiceDirectory }}/target/site/test-coverage/jacoco.xml - reportDirectory: sdk/${{ parameters.ServiceDirectory }}/target/site/test-coverage/ - failIfCoverageEmpty: false + - template: /eng/pipelines/templates/steps/build-and-test.yml + parameters: + SDKType: ${{ parameters.SDKType }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + ParallelTestPlayback: 'true' + SkipAggregateReports: ${{ parameters.SkipAggregateReports }} + TestGoals: ${{ parameters.TestGoals }} + TestOptions: ${{ parameters.TestOptions }} + AdditionalBuildOptions: ${{ parameters.AdditionalBuildOptions }} + TestParallelization: ${{ parameters.TestParallelization }} + BuildParallelization: ${{ parameters.BuildParallelization }} diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index bddbcd47e91f0..7b13f6e74e2b1 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -34,18 +34,24 @@ parameters: - name: PreBuildSteps type: object default: [] - - name: MavenVerifyAdditionalOptions + - name: AdditionalLintingOptions type: string default: '' - - name: MavenBuildPackageParallelization - type: string - default: '1C' - - name: MavenBuildInstallParallelization + - name: BuildParallelization type: string default: '2C' - - name: SparkShading + - name: TestGoals type: string - default: '' + default: $(TestGoals) + - name: TestOptions + type: string + default: $(TestOptions) + - name: AdditionalBuildOptions + type: string + default: $(AdditionalBuildOptions) + - name: TestParallelization + type: string + default: '1C' jobs: - job: 'Build' @@ -160,7 +166,7 @@ jobs: inputs: mavenPomFile: pom.xml goals: 'deploy' - options: '$(DefaultOptions) -T ${{parameters.MavenBuildPackageParallelization}} ${{parameters.SparkShading}} --quiet -DskipTests -Dgenerate-overview -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -pl $(ProjectList) -am -DaltDeploymentRepository=id::default::file://$(System.DefaultWorkingDirectory)/build' # We include template-module so we ensure it always builds in CI + options: '$(DefaultOptions) -T ${{parameters.BuildParallelization}} ${{parameters.AdditionalBuildOptions}} --quiet -DskipTests -Dgenerate-overview -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -pl $(ProjectList) -am -DaltDeploymentRepository=id::default::file://$(System.DefaultWorkingDirectory)/build' # We include template-module so we ensure it always builds in CI mavenOptions: '$(MemoryOptions) $(LoggingOptions)' javaHomeOption: 'JDKVersion' jdkVersionOption: $(JavaBuildVersion) @@ -309,7 +315,7 @@ jobs: inputs: mavenPomFile: pom.xml goals: 'install' - options: '$(DefaultOptions) -T ${{parameters.MavenBuildInstallParallelization}} ${{parameters.SparkShading}} --quiet -DskipTests -Dgpg.skip -Dmaven.javadoc.skip=true -Dcodesnippet.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true -Drevapi.skip=true -pl $(ProjectList) -am' + options: '$(DefaultOptions) -T ${{parameters.BuildParallelization}} ${{parameters.AdditionalBuildOptions}} --quiet -DskipTests $(DefaultSkipOptions) -pl $(ProjectList) -am' mavenOptions: '$(MemoryOptions) $(LoggingOptions)' javaHomeOption: 'JDKVersion' jdkVersionOption: $(JavaBuildVersion) @@ -320,7 +326,7 @@ jobs: displayName: 'Run SpotBugs, Checkstyle, RevApi, and Javadoc' inputs: mavenPomFile: pom.xml - options: '$(DefaultOptions) --no-transfer-progress -DskipTests -Dgpg.skip -Dverify-readme -Dmaven.main.skip=true -Dmaven.test.skip=true ${{ parameters.MavenVerifyAdditionalOptions }} -pl $(ProjectList)' + options: '$(DefaultOptions) --no-transfer-progress -DskipTests -Dgpg.skip -Dverify-readme -Dmaven.main.skip=true -Dmaven.test.skip=true ${{ parameters.AdditionalLintingOptions }} -pl $(ProjectList)' mavenOptions: '$(MemoryOptions)' javaHomeOption: 'JDKVersion' jdkVersionOption: $(JavaBuildVersion) @@ -349,4 +355,8 @@ jobs: PreTestSteps: ${{ parameters.PreTestSteps }} SkipAggregateReports: ${{ parameters.SkipAggregateReports }} PreBuildSteps: ${{ parameters.PreBuildSteps }} - BuildOptions: '$(DefaultOptions) -T ${{parameters.MavenBuildInstallParallelization}} ${{parameters.SparkShading}} --quiet -DskipTests -Dgpg.skip -DtrimStackTrace=false -Dmaven.javadoc.skip=true -Dcodesnippet.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -Djacoco.skip=true' + TestGoals: ${{ parameters.TestGoals }} + TestOptions: ${{ parameters.TestOptions }} + AdditionalBuildOptions: ${{ parameters.AdditionalBuildOptions }} + TestParallelization: ${{ parameters.TestParallelization }} + BuildParallelization: ${{ parameters.BuildParallelization }} diff --git a/eng/pipelines/templates/jobs/live.tests.yml b/eng/pipelines/templates/jobs/live.tests.yml index d149a875484b5..87a1d779d7351 100644 --- a/eng/pipelines/templates/jobs/live.tests.yml +++ b/eng/pipelines/templates/jobs/live.tests.yml @@ -9,7 +9,6 @@ parameters: PostSteps: [] TimeoutInMinutes: 60 TestMode: 'LIVE' - TestStepMavenInputs: {} TestResultsFiles: '' CloudConfig: {} ArmTemplateParameters: '@{}' @@ -18,8 +17,11 @@ parameters: DependsOn: '' UsePlatformContainer: false DisableAzureResourceCreation: false - MavenBuildInstallParallelization: '1C' - SparkShading: '' + BuildParallelization: '2C' + TestGoals: $(TestGoals) + TestOptions: $(TestOptions) + AdditionalBuildOptions: $(AdditionalBuildOptions) + TestParallelization: '1' jobs: - job: @@ -52,6 +54,10 @@ jobs: ServiceDirectory: ${{ parameters.ServiceDirectory }} SDKType: ${{ parameters.SDKType }} + - template: ../steps/install-reporting-tools.yml + parameters: + JdkVersion: $(JavaTestVersion) + - ${{ if ne(parameters.DisableAzureResourceCreation, 'true') }}: - template: /eng/common/TestResources/build-test-resource-config.yml parameters: @@ -78,29 +84,22 @@ jobs: - ${{ parameters.PreSteps }} - - task: Maven@3 - displayName: 'Build and Install, JDK Version: $(JavaTestVersion)' - inputs: - mavenPomFile: pom.xml - goals: 'install' - options: '$(DefaultOptions) -T ${{parameters.MavenBuildInstallParallelization}} ${{parameters.SparkShading}} -DskipTests -Dgpg.skip -Dmaven.javadoc.skip=true -Dcodesnippet.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true -Drevapi.skip=true -pl $(ProjectList) -am' - mavenOptions: '$(MemoryOptions) $(LoggingOptions)' - javaHomeOption: 'JDKVersion' - jdkVersionOption: $(JavaTestVersion) - jdkArchitectureOption: 'x64' - publishJUnitResults: false - - - task: Maven@3 - displayName: 'Run tests: $(Agent.JobName)' - inputs: - mavenPomFile: pom.xml - ${{ insert }}: ${{ parameters.TestStepMavenInputs }} - env: - AZURE_TEST_MODE: ${{ parameters.TestMode }} - ${{ each var in parameters.EnvVars }}: - ${{ var.key }}: ${{ var.value }} - - - template: ../steps/retain-heap-dump-hprofs.yml + - template: /eng/pipelines/templates/steps/build-and-test.yml + parameters: + SDKType: ${{ parameters.SDKType }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + ParallelTestPlayback: 'false' + SkipAggregateReports: 'true' + BuildParallelization: ${{ parameters.BuildParallelization }} + IsLiveTest: true + TestGoals: ${{ parameters.TestGoals }} + TestOptions: ${{ parameters.TestOptions }} + AdditionalBuildOptions: ${{ parameters.AdditionalBuildOptions }} + TestParallelization: ${{ parameters.TestParallelization }} + TestEnvVars: + AZURE_TEST_MODE: ${{ parameters.TestMode }} + ${{ each var in parameters.EnvVars }}: + ${{ var.key }}: ${{ var.value }} - ${{ parameters.PostSteps }} @@ -116,13 +115,3 @@ jobs: parameters: ServiceDirectory: '${{ parameters.ServiceDirectory }}' SubscriptionConfiguration: $(SubscriptionConfiguration) - - - template: ../steps/upload-repository-on-failure.yml - - - task: PublishTestResults@2 - condition: always() - inputs: - mergeTestResults: true - testRunTitle: 'Live tests for ${{ parameters.ServiceDirectory }} ${{ parameters.CloudConfig.Cloud }} $(Agent.JobName)' - ${{ if ne(parameters.TestResultsFiles, '') }}: - testResultsFiles: ${{ parameters.TestResultsFiles }} diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 82fd0698d0113..6e9d16f544d74 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -47,7 +47,7 @@ parameters: - name: PreBuildSteps type: object default: [] -- name: MavenVerifyAdditionalOptions +- name: AdditionalLintingOptions type: string default: '' @@ -77,12 +77,13 @@ stages: MatrixReplace: - ${{ each replacement in parameters.MatrixReplace }}: - ${{ replacement }} - - ${{ if eq(parameters.SDKType, 'data') }}: - - TestGoals=.*/verify - - TestOptions=.*/-am + - AZURE_TEST.*=.*/ PreBuildSteps: ${{ parameters.PreBuildSteps }} - MavenVerifyAdditionalOptions: ${{ parameters.MavenVerifyAdditionalOptions }} + AdditionalLintingOptions: ${{ parameters.AdditionalLintingOptions }} + ${{ if eq(parameters.SDKType, 'data') }}: + TestGoals: 'verify' + TestOptions: '-am' # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch. diff --git a/eng/pipelines/templates/stages/archetype-sdk-tests.yml b/eng/pipelines/templates/stages/archetype-sdk-tests.yml index dbf9331c0d429..381859e298754 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-tests.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-tests.yml @@ -59,16 +59,6 @@ parameters: - name: TestMode type: string default: 'LIVE' -- name: TestStepMavenInputs - type: object - default: - options: '$(DefaultTestOptions)' - mavenOptions: '$(DefaultTestMavenOptions)' - javaHomeOption: 'JDKVersion' - jdkVersionOption: '$(JavaTestVersion)' - jdkArchitectureOption: 'x64' - publishJUnitResults: false - goals: 'verify' - name: CloudConfig type: object default: @@ -105,6 +95,15 @@ parameters: - name: PlatformPostSteps type: object default: [] +- name: TestGoals + type: string + default: $(TestGoals) +- name: TestOptions + type: string + default: $(TestOptions) +- name: AdditionalBuildOptions + type: string + default: $(AdditionalBuildOptions) stages: - ${{ each cloud in parameters.CloudConfig }}: @@ -132,11 +131,14 @@ stages: TestResourceDirectories: ${{ parameters.TestResourceDirectories }} TestSetupSteps: ${{ parameters.TestSetupSteps }} TestMode: ${{ parameters.TestMode }} - TestStepMavenInputs: ${{ parameters.TestStepMavenInputs }} Artifacts: ${{ parameters.Artifacts }} AdditionalModules: ${{ parameters.AdditionalModules }} TestResultsFiles: ${{ parameters.TestResultsFiles }} DisableAzureResourceCreation: ${{ parameters.DisableAzureResourceCreation }} + TestGoals: ${{ parameters.TestGoals }} + TestOptions: ${{ parameters.TestOptions }} + AdditionalBuildOptions: ${{ parameters.AdditionalBuildOptions }} + MatrixConfigs: # Enumerate platforms and additional platforms based on supported clouds (sparse platform<-->cloud matrix). - ${{ each config in parameters.MatrixConfigs }}: diff --git a/eng/pipelines/templates/stages/cosmos-emulator-matrix.json b/eng/pipelines/templates/stages/cosmos-emulator-matrix.json index 283dda23b5625..22e7b3503f476 100644 --- a/eng/pipelines/templates/stages/cosmos-emulator-matrix.json +++ b/eng/pipelines/templates/stages/cosmos-emulator-matrix.json @@ -18,7 +18,7 @@ "AdditionalArgs": "-DargLine=\"-DACCOUNT_HOST=https://localhost:8081/\"" }, "Spring Emulator Only Integration Tests": { - "ProfileFlag": "-P integration-test-emulator", + "ProfileFlag": "-Pintegration-test-emulator", "PROTOCOLS": "[\"Tcp\"]", "AdditionalArgs": "-Dspring-e2e -DargLine=\"-DACCOUNT_HOST=https://localhost:8081/ -DACCOUNT_KEY=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw== -DSECONDARY_ACCOUNT_KEY=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw== -DNEW_ACCOUNT_HOST=https://localhost:8081/ -DNEW_ACCOUNT_KEY=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw== -DNEW_SECONDARY_ACCOUNT_KEY=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==\"" }, diff --git a/eng/pipelines/templates/stages/cosmos-sdk-client.yml b/eng/pipelines/templates/stages/cosmos-sdk-client.yml index 6407eb47f7d61..0747d853cdeaa 100644 --- a/eng/pipelines/templates/stages/cosmos-sdk-client.yml +++ b/eng/pipelines/templates/stages/cosmos-sdk-client.yml @@ -38,13 +38,12 @@ stages: NonSparseParameters: Agent GenerateVMJobs: true MatrixFilters: - - TestFromSource=^$ + - TestFromSource=^$|false MatrixReplace: - - TestOptions=.*/-Punit - AZURE_TEST.*=.*/ - MavenBuildInstallParallelization: 1 - MavenBuildPackageParallelization: 1 - SparkShading: '-DsparkShading' + BuildParallelization: 1 + AdditionalBuildOptions: '-DsparkShading' + TestOptions: '-Punit' - stage: TestEmulator displayName: Test Emulator @@ -59,8 +58,8 @@ stages: Selection: all GenerateVMJobs: true AdditionalParameters: - MavenBuildInstallParallelization: 1 - SparkShading: '-DsparkShading' + BuildParallelization: 1 + AdditionalBuildOptions: '-DsparkShading' DisableAzureResourceCreation: true ServiceDirectory: cosmos Artifacts: ${{ parameters.Artifacts }} @@ -71,12 +70,8 @@ stages: # Increased timeout to 90 because of cosmos emulator taking 25-30 mins to download emulator # Issue filed to improve download speed: https://github.com/Azure/azure-sdk-for-java/issues/12970 TimeoutInMinutes: 90 - TestStepMavenInputs: - goals: clean verify - options: '$(DefaultTestOptions) $(ProfileFlag) $(AdditionalArgs)' - mavenOptions: '$(DefaultTestMavenOptions)' - mavenAuthenticateFeed: true - jdkVersionOption: $(JavaTestVersion) + TestGoals: 'clean verify' + TestOptions: '$(ProfileFlag) $(AdditionalArgs)' PreSteps: - template: /eng/common/pipelines/templates/steps/cosmos-emulator.yml diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json index b54ff7039fa18..216a12bcab053 100644 --- a/eng/pipelines/templates/stages/platform-matrix.json +++ b/eng/pipelines/templates/stages/platform-matrix.json @@ -10,8 +10,8 @@ }, "JavaTestVersion": [ "1.8", "1.11" ], "AZURE_TEST_HTTP_CLIENTS": [ "okhttp", "netty" ], + "TestFromSource": false, "TestGoals": "surefire:test", - "AdditionalTestGoals": "", "TestOptions": "" }, "exclude": [ @@ -28,8 +28,8 @@ "JavaTestVersion": "1.11", "AZURE_TEST_HTTP_CLIENTS": "netty", "TestFromSource": true, - "TestGoals": "surefire:test", - "TestOptions": "" + "TestGoals": "verify", + "TestOptions": "-Dmaven.main.skip=true -Dmaven.test.skip=true -DcreateSourcesJar=false" }, { "Agent": { @@ -37,9 +37,9 @@ }, "JavaTestVersion": "1.11", "AZURE_TEST_HTTP_CLIENTS": "netty", - "AdditionalOptions": "-am", + "TestFromSource": false, "TestGoals": "verify", - "TestOptions": "" + "TestOptions": "-Dmaven.main.skip=true -Dmaven.test.skip=true -DcreateSourcesJar=false" } ] } diff --git a/eng/pipelines/templates/steps/build-and-test.yml b/eng/pipelines/templates/steps/build-and-test.yml new file mode 100644 index 0000000000000..327b70105e794 --- /dev/null +++ b/eng/pipelines/templates/steps/build-and-test.yml @@ -0,0 +1,141 @@ +parameters: + - name: SDKType + type: string + default: client + - name: ServiceDirectory + type: string + default: 'not-specified' # Set a default that breaks in obvious ways. + - name: BuildParallelization + type: string + default: '2C' + - name: TestGoals + type: string + default: $(TestGoals) + - name: TestOptions + type: string + default: '' + - name: AdditionalBuildOptions + type: string + default: '' + - name: TestParallelization + type: string + default: '1C' + - name: ParallelTestPlayback + type: string + default: 'false' + - name: SkipAggregateReports + type: boolean + default: false + - name: TestResultsFiles + type: string + default: '' + - name: TestEnvVars + type: object + default: {} + - name: IsLiveTest + type: boolean + default: false + +steps: + - task: Maven@3 + displayName: 'Build for non-From Source run' + inputs: + mavenPomFile: pom.xml + options: $(DefaultOptions) --quiet -DskipTests $(DefaultSkipOptions) -Djacoco.skip=true -T ${{ parameters.BuildParallelization }} ${{ parameters.AdditionalBuildOptions }} -pl $(ProjectList) -am + mavenOptions: '$(MemoryOptions) $(LoggingOptions)' + javaHomeOption: 'JDKVersion' + jdkVersionOption: $(JavaTestVersion) + jdkArchitectureOption: 'x64' + publishJUnitResults: false + goals: 'install' + condition: and(succeeded(), ne(variables['TestFromSource'], 'true')) + + - task: Maven@3 + displayName: 'Run tests' + inputs: + mavenPomFile: pom.xml + options: '$(DefaultTestOptions) --quiet -DAZURE_TEST_DEBUG=$(IsDebug) -Dparallel-test-playback=${{ parameters.ParallelTestPlayback }} ${{ parameters.TestOptions }} -T ${{ parameters.TestParallelization }}' + mavenOptions: '$(MemoryOptions) $(LoggingOptions)' + javaHomeOption: 'JDKVersion' + jdkVersionOption: $(JavaTestVersion) + jdkArchitectureOption: 'x64' + publishJUnitResults: false + goals: ${{ parameters.TestGoals }} + env: ${{ parameters.TestEnvVars }} + # we want to run this when TestFromSource isn't true + condition: and(succeeded(), ne(variables['TestFromSource'], 'true')) + + - task: Maven@3 + displayName: 'Generate aggregate code coverage report' + condition: and(eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['JavaTestVersion'], '1.11'), eq('${{ parameters.SDKType }}', 'client'), eq('${{ parameters.SkipAggregateReports }}', 'false')) + inputs: + mavenPomFile: sdk/${{ parameters.ServiceDirectory }}/pom.xml + options: '$(DefaultOptions) -Pcoverage' + mavenOptions: '$(MemoryOptions) $(LoggingOptions)' + javaHomeOption: 'JDKVersion' + jdkVersionOption: $(JavaTestVersion) + jdkArchitectureOption: 'x64' + goals: jacoco:report-aggregate + + - task: Maven@3 + displayName: 'Build for From Source run' + inputs: + mavenPomFile: ClientFromSourcePom.xml + goals: 'install' + # Build and install every track 2, client, library. The purpose is to have all the libraries installed on the + # machine, so we can simply run tests with the -amd switch and we can remove the SNAFU of dependencies that + # bloated the ci.yml files' AdditionalModules because we ended up having to list every transitive azure client + # dependency. Note: The -T 1C says spin up a build thread per core. Since we're building + # all the track 2, client, libraries (which is now over 100) this effectively cuts the build time in half. + options: $(DefaultOptions) --quiet -DskipTests $(DefaultSkipOptions) -Djacoco.skip=true -T ${{ parameters.BuildParallelization }} ${{ parameters.AdditionalBuildOptions }} + mavenOptions: '$(MemoryOptions) $(LoggingOptions)' + javaHomeOption: 'JDKVersion' + jdkVersionOption: $(JavaBuildVersion) + jdkArchitectureOption: 'x64' + publishJUnitResults: false + condition: and(succeeded(), eq(variables['TestFromSource'], 'true')) + + - task: Maven@3 + displayName: 'Run tests for From Source run' + inputs: + mavenPomFile: ClientFromSourcePom.xml + # For the "From Source" runs we don't want the -am switch as we don't care about running tests for our + # dependencies, but we do want the -amd switch because we want to run tests on things that depend on us. + ${{ if eq(parameters.IsLiveTest, 'true') }}: + options: $(DefaultTestOptions) --quiet -DAZURE_TEST_DEBUG=$(IsDebug) -Dparallel-test-playback=${{ parameters.ParallelTestPlayback }} ${{ parameters.TestOptions }} -T ${{ parameters.TestParallelization }} + ${{ else }}: + options: $(DefaultTestOptions) --quiet -DAZURE_TEST_DEBUG=$(IsDebug) -Dparallel-test-playback=${{ parameters.ParallelTestPlayback }} ${{ parameters.TestOptions }} -amd -T ${{ parameters.TestParallelization }} + mavenOptions: '$(MemoryOptions) $(LoggingOptions)' + javaHomeOption: 'JDKVersion' + jdkVersionOption: $(JavaTestVersion) + jdkArchitectureOption: 'x64' + publishJUnitResults: false + # The "From Source" runs only care about running tests for libraries within an SDK/area + # as well as libraries which depend on them and not the other stages in the Maven + # lifecycle that test or verify targets would execute. The libraries were already + # compiled and installed in the previous task, this tasks only runs tests. + goals: ${{ parameters.TestGoals }} + env: ${{ parameters.TestEnvVars }} + condition: and(succeeded(), eq(variables['TestFromSource'], 'true')) + + - template: ../steps/upload-repository-on-failure.yml + + - template: ../steps/retain-heap-dump-hprofs.yml + + - task: PublishTestResults@2 + condition: always() + inputs: + mergeTestResults: true + testRunTitle: $(Agent.JobName) + ${{ if ne(parameters.TestResultsFiles, '') }}: + testResultsFiles: ${{ parameters.TestResultsFiles }} + + # Azure DevOps only seems to respect the last code coverage result published, so only do this for Windows + Java LTS. + # Code coverage reporting is set up only for Track 2 modules. + - task: PublishCodeCoverageResults@1 + condition: and(eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['JavaTestVersion'], '1.11'), eq('${{ parameters.SDKType }}', 'client')) + inputs: + codeCoverageTool: JaCoCo + summaryFileLocation: sdk/${{ parameters.ServiceDirectory }}/target/site/test-coverage/jacoco.xml + reportDirectory: sdk/${{ parameters.ServiceDirectory }}/target/site/test-coverage/ + failIfCoverageEmpty: false diff --git a/eng/pipelines/templates/steps/initialize-test-environment.yml b/eng/pipelines/templates/steps/initialize-test-environment.yml index 0f333ca3e633a..8967e6898577d 100644 --- a/eng/pipelines/templates/steps/initialize-test-environment.yml +++ b/eng/pipelines/templates/steps/initialize-test-environment.yml @@ -45,7 +45,7 @@ steps: inputs: scriptPath: 'eng/versioning/update_versions.py' arguments: '--update-type library --build-type client --sr' - condition: and(succeeded(), eq(variables['ShouldRunSourceTests'],'true')) + condition: and(succeeded(), eq(variables['TestFromSource'], 'true')) - template: generate-project-list.yml parameters: @@ -54,7 +54,7 @@ steps: JobType: 'test' SDKType: ${{ parameters.SDKType }} ServiceDirectory: ${{ parameters.ServiceDirectory }} - + - task: PythonScript@0 displayName: 'Generate directories variable for sparse checkout' inputs: diff --git a/eng/pipelines/templates/steps/install-reporting-tools.yml b/eng/pipelines/templates/steps/install-reporting-tools.yml index 124d9f7aa531a..0480abdd462f9 100644 --- a/eng/pipelines/templates/steps/install-reporting-tools.yml +++ b/eng/pipelines/templates/steps/install-reporting-tools.yml @@ -2,6 +2,7 @@ parameters: Options: '$(DefaultOptions)' MavenOptions: '$(MemoryOptions) $(LoggingOptions)' + JdkVersion: $(JavaBuildVersion) steps: - task: Maven@3 @@ -11,7 +12,7 @@ steps: options: ${{ parameters.Options }} mavenOptions: ${{ parameters.MavenOptions }} javaHomeOption: 'JDKVersion' - jdkVersionOption: '1.11' + jdkVersionOption: ${{ parameters.JdkVersion }} jdkArchitectureOption: 'x64' publishJUnitResults: false goals: 'install' diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml index e3ed79030f909..88dd23543952e 100644 --- a/eng/pipelines/templates/variables/globals.yml +++ b/eng/pipelines/templates/variables/globals.yml @@ -13,11 +13,20 @@ variables: DefaultOptions: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) --batch-mode --fail-at-end --settings eng/settings.xml $(WagonOptions)' LoggingOptions: '-Dorg.slf4j.simpleLogger.defaultLogLevel=info -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn' MemoryOptions: '-Xmx3072m' - DefaultTestOptions: '$(DefaultOptions) -Dgpg.skip -Dmaven.javadoc.skip=true -Dcodesnippet.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -DtrimStackTrace=false -pl $(ProjectList)' + DefaultSkipOptions: '-Dgpg.skip -Dmaven.javadoc.skip=true -Dcodesnippet.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -DtrimStackTrace=false' + DefaultTestOptions: '$(DefaultOptions) $(DefaultSkipOptions) -pl $(ProjectList)' DefaultTestMavenOptions: '$(MemoryOptions) $(LoggingOptions)' AdditionalOptions: '' - TestFromSource: false - AdditionalTestGoals: '' + + # Exists if needed in coalesce situations. + DefaultTestGoals: 'surefire:test' + # This will be overwritten by the test matrix, if configured. + TestGoals: $(DefaultTestGoals) + + # This will be overwritten by the test matrix, if configured. + TestOptions: '' + + AdditionalBuildOptions: '' skipComponentGovernanceDetection: true IsDebug: $[coalesce(variables['System.Debug'], 'false')] diff --git a/sdk/appconfiguration/tests.yml b/sdk/appconfiguration/tests.yml index 62bd91bcee258..085217d310b07 100644 --- a/sdk/appconfiguration/tests.yml +++ b/sdk/appconfiguration/tests.yml @@ -15,6 +15,5 @@ stages: AZURE_CLIENT_ID: $(aad-azure-sdk-test-client-id) AZURE_CLIENT_SECRET: $(aad-azure-sdk-test-client-secret) AZURE_TENANT_ID: $(aad-azure-sdk-test-tenant-id) - TestStepMavenInputs: - options: '-Dmaven.wagon.http.pool=false $(DefaultOptions) -Dmaven.javadoc.skip=true -Dcodesnippet.skip=true -Drevapi.skip=true -DskipSpringITs=false -pl $(ProjectList)' - goals: "verify" + TestGoals: 'verify' + TestOptions: '-DskipSpringITs=false' diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index f218d3d7ad10c..bba0b45e4f2f0 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -76,4 +76,4 @@ extends: - name: perf-test-core groupId: com.azure MatrixReplace: - - AdditionalTestGoals=.*/failsafe:integration-test failsafe:verify + - TestGoals=(surefire:test)/$1 failsafe:integration-test failsafe:verify diff --git a/sdk/core/tests.yml b/sdk/core/tests.yml index b01b4643766d1..ff7667b3ff34b 100644 --- a/sdk/core/tests.yml +++ b/sdk/core/tests.yml @@ -45,7 +45,7 @@ stages: groupId: com.azure safeName: azurecoretracingopentelemetry MatrixReplace: - - AdditionalTestGoals=.*/failsafe:integration-test failsafe:verify + - TestGoals=(surefire:test)/$1 failsafe:integration-test failsafe:verify - template: /eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: ServiceDirectory: core diff --git a/sdk/core/unsupported-version-matrix.json b/sdk/core/unsupported-version-matrix.json index 201de0d3fe235..d0878043fc118 100644 --- a/sdk/core/unsupported-version-matrix.json +++ b/sdk/core/unsupported-version-matrix.json @@ -1,6 +1,6 @@ { "displayNames": { - "2.9.10": "jackson2_9", + "2.9.10": "jackson2_9" }, "matrix": { "Agent": { diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index 427545455dc58..6f6f4e35518d7 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -31,10 +31,8 @@ stages: MaxParallel: 8 PreSteps: - template: /eng/pipelines/templates/steps/install-reporting-tools.yml - TestStepMavenInputs: - goals: verify - options: '$(DefaultTestOptions) $(ProfileFlag) $(AdditionalArgs)' - mavenOptions: '$(DefaultTestMavenOptions)' + TestGoals: 'verify' + TestOptions: '$(ProfileFlag) $(AdditionalArgs)' TestResultsFiles: '**/junitreports/TEST-*.xml' - template: /eng/pipelines/templates/stages/archetype-sdk-tests.yml @@ -58,7 +56,5 @@ stages: TimeoutInMinutes: 90 PreSteps: - template: /eng/pipelines/templates/steps/install-reporting-tools.yml - TestStepMavenInputs: - goals: clean verify - options: '$(DefaultTestOptions) $(ProfileFlag)' - mavenOptions: '$(DefaultTestMavenOptions)' + TestGoals: 'clean verify' + TestOptions: '$(ProfileFlag)' diff --git a/sdk/keyvault/ci.yml b/sdk/keyvault/ci.yml index 14b14448293d3..d203bb3697bcd 100644 --- a/sdk/keyvault/ci.yml +++ b/sdk/keyvault/ci.yml @@ -57,7 +57,7 @@ extends: template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: keyvault - MavenVerifyAdditionalOptions: '-DcreateSourcesJar=false' + AdditionalLintingOptions: '-DcreateSourcesJar=false' Artifacts: - name: azure-security-keyvault-administration groupId: com.azure diff --git a/sdk/spring/spring-test-template.yml b/sdk/spring/spring-test-template.yml index 01e96708a97ee..bb75f56e8e82b 100644 --- a/sdk/spring/spring-test-template.yml +++ b/sdk/spring/spring-test-template.yml @@ -100,10 +100,8 @@ stages: Clouds: ${{ parameters.Clouds }} TestResourceDirectories: ${{ parameters.TestResourceDirectories }} Artifacts: ${{ parameters.Artifacts }} - TestStepMavenInputs: - options: '$(DefaultTestOptions) -DskipSpringITs=false' - mavenOptions: '$(DefaultTestMavenOptions)' - goals: "verify" + TestGoals: 'verify' + TestOptions: '-DskipSpringITs=false' PreSteps: - script: | az cloud set --name AzureChinaCloud diff --git a/sdk/storage/platform-matrix.json b/sdk/storage/platform-matrix.json index ab26652d6f1d7..11da4004ac6eb 100644 --- a/sdk/storage/platform-matrix.json +++ b/sdk/storage/platform-matrix.json @@ -9,17 +9,20 @@ { "Agent": { "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" } }, "JavaTestVersion": "1.8", - "AZURE_TEST_HTTP_CLIENTS": "netty" + "AZURE_TEST_HTTP_CLIENTS": "netty", + "TestFromSource": false }, { "Agent": { "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" } }, "JavaTestVersion": "1.11", - "AZURE_TEST_HTTP_CLIENTS": "okhttp" + "AZURE_TEST_HTTP_CLIENTS": "okhttp", + "TestFromSource": false }, { "Agent": { "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" } }, "JavaTestVersion": "1.17", - "AZURE_TEST_HTTP_CLIENTS": "netty" + "AZURE_TEST_HTTP_CLIENTS": "netty", + "TestFromSource": false }, { "Agent": { @@ -27,8 +30,9 @@ }, "JavaTestVersion": ["1.8", "1.11"], "AZURE_TEST_HTTP_CLIENTS": "netty", + "TestFromSource": false, "StorageRunStressScenarios" : "true", - "AdditionalOptions": "-DskipTests" + "TestOptions": "-DskipTests" } ] } diff --git a/sdk/storage/tests-template.yml b/sdk/storage/tests-template.yml index 31dccc49a35e6..383c71e639d5f 100644 --- a/sdk/storage/tests-template.yml +++ b/sdk/storage/tests-template.yml @@ -53,9 +53,6 @@ stages: SubscriptionConfiguration: $(sub-config-storage-test-resources) Clouds: Preview SupportedClouds: Preview - TestStepMavenInputs: - options: '$(DefaultTestOptions) $(AdditionalOptions) -T 1C' - mavenOptions: '$(DefaultTestMavenOptions)' MatrixReplace: # Use dedicated storage pool in canadacentral with higher memory capacity - Pool=(.*)-general/$1-storage