From e837eedec51e32ef84c35be1d2e2854a94fe48ff Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Tue, 5 Jul 2022 14:57:13 +0800 Subject: [PATCH 1/3] Reformat --- azure-pipelines.yml | 1760 +++++++++++++++++++++---------------------- 1 file changed, 880 insertions(+), 880 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 89fcffba679..ee60076e104 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,11 +1,11 @@ resources: -- repo: self + - repo: self trigger: batch: true branches: include: - - '*' + - '*' pr: branches: @@ -13,882 +13,882 @@ pr: - '*' jobs: -- job: CheckPullRequest - displayName: "Check the Format of Pull Request Title and Content" - condition: and(succeeded(), in(variables['System.PullRequest.TargetBranch'], 'dev', 'release')) - - pool: - vmImage: 'ubuntu-20.04' - steps: - - bash: | - echo "Check Title of Pull Request: #$(System.PullRequest.PullRequestNumber)" - title=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.title') - echo $title - body=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.body') - echo $body - if [ "$(System.PullRequest.TargetBranch)" != "release" ] && echo $title | grep -iqF hotfix:; then - echo "Hotfix PR should target release branch." + - job: CheckPullRequest + displayName: "Check the Format of Pull Request Title and Content" + condition: and(succeeded(), in(variables['System.PullRequest.TargetBranch'], 'dev', 'release')) + + pool: + vmImage: 'ubuntu-20.04' + steps: + - bash: | + echo "Check Title of Pull Request: #$(System.PullRequest.PullRequestNumber)" + title=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.title') + echo $title + body=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.body') + echo $body + if [ "$(System.PullRequest.TargetBranch)" != "release" ] && echo $title | grep -iqF hotfix:; then + echo "Hotfix PR should target release branch." + exit 1 + fi + python scripts/ci/check_pull_request.py "$title" "$body" + + - job: RejectPullRequestToMasterBranch + displayName: "Reject Pull Request To Master Branch" + condition: and(succeeded(), eq(variables['System.PullRequest.TargetBranch'], 'master')) + + pool: + vmImage: 'ubuntu-20.04' + steps: + - bash: | + echo "Reject pull request directly to master branch" exit 1 - fi - python scripts/ci/check_pull_request.py "$title" "$body" - -- job: RejectPullRequestToMasterBranch - displayName: "Reject Pull Request To Master Branch" - condition: and(succeeded(), eq(variables['System.PullRequest.TargetBranch'], 'master')) - - pool: - vmImage: 'ubuntu-20.04' - steps: - - bash: | - echo "Reject pull request directly to master branch" - exit 1 - -- job: CredScan - displayName: "Credential Scan" - pool: - vmImage: "windows-2019" - steps: - - task: ms-codeanalysis.vss-microsoft-security-code-analysis-devops.build-task-credscan.CredScan@2 - displayName: 'Run Credential Scanner' - inputs: - toolMajorVersion: V2 - suppressionsFile: './scripts/ci/credscan/CredScanSuppressions.json' - toolVersionV2: '2.1.17' - - task: ms-codeanalysis.vss-microsoft-security-code-analysis-devops.build-task-postanalysis.PostAnalysis@1 - displayName: 'Post Analysis' - inputs: - AllTools: false - BinSkim: false - CredScan: true - RoslynAnalyzers: false - TSLint: false - ToolLogsNotFoundAction: 'Standard' - -- job: ExtractMetadata - displayName: Extract Metadata - - condition: succeeded() - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: Bash@3 - displayName: 'Extract Version' - inputs: - targetType: 'filePath' - filePath: scripts/release/get_version.sh - - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: metadata' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: metadata - -- job: VerifyLinuxRequirements - displayName: 'Verify src/azure-cli/requirements.*.Linux.txt' - condition: succeeded() - pool: - vmImage: 'ubuntu-20.04' - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3' - inputs: - versionSpec: 3.x - - - bash: ./scripts/ci/dependency_check.sh - displayName: 'Verify src/azure-cli/requirements.py3.Linux.txt' - -- job: VerifyDarwinRequirements - displayName: 'Verify src/azure-cli/requirements.*.Darwin.txt' - condition: succeeded() - pool: - vmImage: 'macOS-10.15' - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3' - inputs: - versionSpec: 3.x - - - bash: ./scripts/ci/dependency_check.sh - displayName: 'Verify src/azure-cli/requirements.py3.Darwin.txt' - -- job: VerifyWindowsRequirements - displayName: 'Verify src/azure-cli/requirements.*.Windows.txt' - condition: succeeded() - pool: - vmImage: 'windows-2019' - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: 3.10 - - - task: BatchScript@1 - inputs: - filename: ./scripts/ci/dependency_check.bat - displayName: 'Verify src/azure-cli/requirements.py3.Windows.txt' - -- job: VerifyVersions - displayName: Verify Command Module Versions - condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/release'), eq(variables['System.PullRequest.TargetBranch'], 'release'))) - - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: 3.10 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev verify history - - displayName: 'Verify History' - -- job: BuildWindowsMSI - displayName: Build Windows MSI - - dependsOn: ExtractMetadata - condition: succeeded() - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - script: | - set /p CLI_VERSION=<$(System.ArtifactsDirectory)/metadata/version - set - - build_scripts/windows/scripts/build.cmd - displayName: 'Build Windows MSI' - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: 'build_scripts/windows/out/' - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: MSI' - inputs: - TargetPath: 'build_scripts/windows/out/' - ArtifactName: msi - -- job: TestWindowsMSI - displayName: Test Windows MSI - - dependsOn: BuildWindowsMSI - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/msi' - artifactName: msi - - - task: PowerShell@2 - displayName: Install and Load CLI - inputs: - targetType: inline - script: | - if (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")) { - # Start another Powershell process as Admin and execute this script again - $arguments = "& '" +$myinvocation.mycommand.definition + "'" - Start-Process powershell -Verb runAs -ArgumentList $arguments - # Stop if the PowerShell is not run as Admin - Break - } - # The following are executed by elevated PowerShell - az --version - - $InstallArgs = @( - "/i" - "`"$env:SYSTEM_ARTIFACTSDIRECTORY\msi\Microsoft Azure CLI.msi`"" - "/q" - "/norestart" - "/l*v" - ".\install_logs.txt" - ) - $pre_installed_version=az version --query '\"azure-cli\"' -o tsv - $to_be_installed_version=Get-Content $(System.ArtifactsDirectory)/metadata/version - if ($pre_installed_version -eq $to_be_installed_version){ - # See https://docs.microsoft.com/windows/win32/msi/reinstallmode about options of REINSTALLMODE - $reinstall_option="REINSTALL=ALL REINSTALLMODE=emus" - $InstallArgs += $reinstall_option - } - Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow - $install_time=Measure-Command {Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow} | select -expand TotalSeconds - $installed_version=az version --query '\"azure-cli\"' -o tsv - if ($installed_version -ne $to_be_installed_version){ - echo "The MSI failed to install." - Exit 1 - } - echo 'Install time(seconds):' $install_time - az --version - # Test bundled pip with extension installation - az extension add -n rdbms-connect - az self-test - - Get-Content .\install_logs.txt - -- job: BuildDockerImage - displayName: Build Docker Image - - dependsOn: ExtractMetadata - condition: succeeded() - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: Bash@3 - displayName: 'Bash Script' - inputs: - targetType: 'filePath' - filePath: scripts/release/docker/pipeline.sh - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - DockerImagesToScan: 'clibuild$BUILD_BUILDNUMBER:latest' - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: docker image' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: docker - -- job: TestDockerImage - displayName: Test Docker Image - - dependsOn: BuildDockerImage - condition: succeeded() - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Docker Image' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: docker - - - - bash: | - set -exv - - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - IMAGE_NAME=clibuild$BUILD_BUILDNUMBER:latest - TAR_FILE=$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$CLI_VERSION.tar - - echo "== Test docker image ==" - - docker load < $TAR_FILE - docker run $IMAGE_NAME /bin/bash -c "time az self-test && time az --version && sleep 5" - displayName: 'Bash Script' - -- job: BuildPythonWheel - displayName: Build Python Wheels - - dependsOn: ExtractMetadata - condition: succeeded() - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: 3.10 - - - - script: | - if [[ "$(Build.Reason)" == "PullRequest" ]]; then - branch=$(System.PullRequest.TargetBranch) - else - branch=$(Build.SourceBranchName) - fi - scripts/release/pypi/build.sh $branch - displayName: 'Run Wheel Build Script' - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: pypi' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: pypi - -- job: TestPythonWheel - displayName: Test Python Wheels - - dependsOn: BuildPythonWheel - condition: succeeded() - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - task: DownloadPipelineArtifact@1 - displayName: 'Download PyPI Packages' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/pypi' - artifactName: pypi - - - - bash: | - #!/usr/bin/env bash - - # Verify the pip wheels - - set -ex - - CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` - PYPI_FILES=$(cd $BUILD_ARTIFACTSTAGINGDIRECTORY/pypi; pwd) - - echo "== Testing pip install on Python 3.7 ==" - docker run \ - --rm -v $PYPI_FILES:/mnt/pypi python:3.7 \ - /bin/bash -c "ls /mnt/pypi && pip install -f /mnt/pypi -q azure-cli==$CLI_VERSION.* && az self-test && az --version && sleep 5" - - echo "== Testing pip install on Python 3.9 ==" - docker run \ - --rm -v $PYPI_FILES:/mnt/pypi python:3.9 \ - /bin/bash -c "ls /mnt/pypi && pip install -f /mnt/pypi -q azure-cli==$CLI_VERSION.* && az self-test && az --version && sleep 5" - - echo "== Testing pip install on Python 3.10 ==" - docker run \ - --rm -v $PYPI_FILES:/mnt/pypi python:3.10 \ - /bin/bash -c "ls /mnt/pypi && pip install -f /mnt/pypi -q azure-cli==$CLI_VERSION.* && az self-test && az --version && sleep 5" - - displayName: 'Test pip Install' - -- job: TestCore - displayName: Unit Test for Core - timeoutInMinutes: 10 - pool: - vmImage: 'ubuntu-20.04' - strategy: - matrix: - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - steps: - - template: .azure-pipelines/templates/automation_test.yml - parameters: - pythonVersion: '$(python.version)' - module: 'azure-cli-core' - -- job: TestTelemetry - displayName: Unit Test for Telemetry - timeoutInMinutes: 10 - pool: - vmImage: 'ubuntu-20.04' - strategy: - matrix: - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - steps: - - template: .azure-pipelines/templates/automation_test.yml - parameters: - pythonVersion: '$(python.version)' - module: 'azure-cli-telemetry' - -- job: IntegrationTestAgainstProfiles - displayName: Integration Test against Profiles - dependsOn: BuildPythonWheel - condition: succeeded() - timeoutInMinutes: 20 - - pool: - vmImage: 'ubuntu-20.04' - strategy: - matrix: - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - bash: pip install --upgrade pip wheel - displayName: 'Install pip and wheel' - - bash: ./scripts/ci/test_profile_integration.sh - displayName: 'Run Integration Test against Profiles' - -- job: TestExtensionsLoading - displayName: Test Extensions Loading - condition: succeeded() - timeoutInMinutes: 40 - - pool: - vmImage: 'ubuntu-20.04' - strategy: - matrix: - Python310: - python.version: '3.10' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - bash: pip install --upgrade pip wheel - displayName: 'Install pip and wheel' - - bash: ./scripts/ci/test_extensions.sh - displayName: 'Load extensions' - -- job: BuildHomebrewFormula - displayName: Build Homebrew Formula - - dependsOn: BuildPythonWheel - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - bash: | - #!/bin/bash - - root=$(cd $(dirname $0); pwd) - - set -evx - - CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` - HOMEBREW_UPSTREAM_URL=`curl -Ls -o /dev/null -w %{url_effective} https://api.github.com/repos/Azure/azure-cli/tarball/$BUILD_SOURCEVERSION` - - docker_files=$(cd $BUILD_SOURCESDIRECTORY/scripts/release/homebrew/docker; pwd) - src_files=$(cd $BUILD_SOURCESDIRECTORY/src; pwd) - - echo "Generating formula in docker container ... " - docker run -v $docker_files:/mnt/scripts \ - -v $src_files:/mnt/src \ - -e CLI_VERSION=$CLI_VERSION \ - -e HOMEBREW_UPSTREAM_URL=$HOMEBREW_UPSTREAM_URL \ - --name azurecli \ - python:3.10 \ - /mnt/scripts/run.sh - - # clean up - rm -rf $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata - - docker cp azurecli:azure-cli.rb $BUILD_ARTIFACTSTAGINGDIRECTORY/azure-cli.rb - docker rm --force azurecli - displayName: 'Build homebrew formula' - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: homebrew' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: homebrew - -- job: TestHomebrewFormula - displayName: Test Homebrew Formula - - dependsOn: BuildHomebrewFormula - condition: succeeded() - pool: - vmImage: 'macOS-10.15' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Homebrew' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' - artifactName: homebrew - - - - bash: | - set -ev - - echo == Remove pre-installed azure-cli == - brew uninstall azure-cli - - echo == Install azure-cli.rb formula == - brew install --build-from-source $SYSTEM_ARTIFACTSDIRECTORY/homebrew/azure-cli.rb - - echo == Az Version == - az --version - - echo == Run Self-Test == - az self-test - - displayName: 'Bash Script' - -- job: TestHomebrewPackage - displayName: Test Homebrew Package - timeoutInMinutes: 180 - dependsOn: BuildHomebrewFormula - # condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - condition: false - pool: - vmImage: 'macOS-10.15' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' - artifactName: homebrew - - - - bash: ./scripts/release/homebrew/test_homebrew_package.sh - - - displayName: 'Test Homebrew Package' - - -- job: BuildRpmPackageMariner - displayName: Build Rpm Package Mariner - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - vmImage: 'ubuntu-20.04' - strategy: - matrix: - 1.0: - image: cblmariner.azurecr.io/base/core - tag: 1.0 - 2.0: - image: mcr.microsoft.com/cbl-mariner/base/core - tag: 2.0 - steps: - - task: Bash@3 - displayName: 'Build Rpm Package: Mariner' - inputs: - targetType: 'filePath' - filePath: scripts/release/rpm/pipeline_mariner.sh - env: - IMAGE: $(image) - TAG: $(tag) - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm-mariner' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: rpm-mariner$(tag) - - -# TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. -- job: BuildRpmPackageUbi8 - displayName: Build Rpm Package Red Hat Universal Base Image 8 - # Do not run this job for Pull Requests due to the slowness - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: Bash@3 - displayName: 'Build Rpm Package' - inputs: - targetType: 'filePath' - filePath: scripts/release/rpm/pipeline.sh - env: - IMAGE: ubi8 - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: rpm-ubi8 - - -- job: TestRpmPackage - displayName: Test Rpm Package - timeoutInMinutes: 120 - dependsOn: BuildRpmPackageUbi8 - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' - artifactName: rpm-ubi8 - - - - bash: | - set -ex - - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=azure-cli-$CLI_VERSION-1.el8.x86_64.rpm - RPM_FILE=$SYSTEM_ARTIFACTSDIRECTORY/rpm/$RPM_NAME - - echo "== Test rpm package on Red Hat UBI 8 ==" - - IMAGE=registry.access.redhat.com/ubi8/ubi:8.4 - docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" - - displayName: 'Test Rpm Package' - -- job: BuildDebPackages - displayName: Build Deb Packages - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - vmImage: 'ubuntu-20.04' - strategy: - matrix: - # https://wiki.ubuntu.com/Releases - Bionic: - # 18.04 - deb_system: ubuntu - distro: bionic - Focal: - # 20.04 - deb_system: ubuntu - distro: focal - Impish: - # 21.10 - deb_system: ubuntu - distro: impish - Jammy: - # 22.04 - deb_system: ubuntu - distro: jammy - - # https://wiki.debian.org/DebianReleases - Buster: - # 10 - deb_system: debian - distro: buster - Bullseye: - # 11 - deb_system: debian - distro: bullseye - steps: - - task: Bash@3 - displayName: 'Build $(deb_system) $(distro) Package' - inputs: - targetType: 'filePath' - filePath: scripts/release/debian/pipeline.sh - env: - DISTRO: $(distro) - DISTRO_BASE_IMAGE: $(deb_system):$(distro) - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: debian' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(deb_system)-$(distro) - -- job: TestDebPackages - displayName: Test Deb Packages - dependsOn: - - BuildDebPackages - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - strategy: - matrix: - Focal: - deb_system: ubuntu - distro: focal - Jammy: - deb_system: ubuntu - distro: jammy - Buster: - deb_system: debian - distro: buster - Bullseye: - deb_system: debian - distro: bullseye - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download $(deb_system):$(distro) Build' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: $(deb_system)-$(distro) - - - task: Bash@3 - displayName: 'Test $(deb_system) $(distro) Package' - env: - DISTRO: $(distro) - DISTRO_BASE_IMAGE: $(deb_system):$(distro) - inputs: - targetType: 'inline' - script: | - set -exv - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - echo "== Test debian package on ${DISTRO} ==" - docker pull ${DISTRO_BASE_IMAGE} - docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" - -- job: CheckStyle - displayName: "Check CLI Style" - - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: 3.10 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev style - -- job: CheckHeaders - displayName: "Check License, History, and DocMap" - - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: 3.10 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev verify license - azdev verify history - azdev verify document-map - -- job: PerformanceCheck - displayName: "PerformanceCheck" - strategy: - matrix: - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev perf load-times - displayName: "Load Performance" - # - bash: | - # set -ev - # . env/bin/activate - - # azdev perf benchmark "version" "network vnet -h" "rest -h" "storage account" - # displayName: "Execution Performance" - -- job: CheckLinter - displayName: "Check CLI Linter" - - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: 3.10 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - python scripts/ci/service_name.py - if [[ "$(System.PullRequest.TargetBranch)" != "" ]]; then - # If CI is set to shallow fetch, target branch should be expilictly fetched. - git fetch origin --depth=1 $(System.PullRequest.TargetBranch) - azdev linter --ci-exclusions --min-severity medium --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) - else - azdev linter --ci-exclusions --min-severity medium - fi - -- job: VerifySphinxDocumentGenerator - displayName: "Verify Sphinx Document Generator" - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: 3.10 - - bash: pip install --upgrade pip wheel - displayName: "Install pip and wheel" - - bash: ./scripts/ci/test_ref_doc.sh - displayName: "Verify Sphinx Document Generator" + + - job: CredScan + displayName: "Credential Scan" + pool: + vmImage: "windows-2019" + steps: + - task: ms-codeanalysis.vss-microsoft-security-code-analysis-devops.build-task-credscan.CredScan@2 + displayName: 'Run Credential Scanner' + inputs: + toolMajorVersion: V2 + suppressionsFile: './scripts/ci/credscan/CredScanSuppressions.json' + toolVersionV2: '2.1.17' + - task: ms-codeanalysis.vss-microsoft-security-code-analysis-devops.build-task-postanalysis.PostAnalysis@1 + displayName: 'Post Analysis' + inputs: + AllTools: false + BinSkim: false + CredScan: true + RoslynAnalyzers: false + TSLint: false + ToolLogsNotFoundAction: 'Standard' + + - job: ExtractMetadata + displayName: Extract Metadata + + condition: succeeded() + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: Bash@3 + displayName: 'Extract Version' + inputs: + targetType: 'filePath' + filePath: scripts/release/get_version.sh + + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: metadata' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: metadata + + - job: VerifyLinuxRequirements + displayName: 'Verify src/azure-cli/requirements.*.Linux.txt' + condition: succeeded() + pool: + vmImage: 'ubuntu-20.04' + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3' + inputs: + versionSpec: 3.x + + - bash: ./scripts/ci/dependency_check.sh + displayName: 'Verify src/azure-cli/requirements.py3.Linux.txt' + + - job: VerifyDarwinRequirements + displayName: 'Verify src/azure-cli/requirements.*.Darwin.txt' + condition: succeeded() + pool: + vmImage: 'macOS-10.15' + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3' + inputs: + versionSpec: 3.x + + - bash: ./scripts/ci/dependency_check.sh + displayName: 'Verify src/azure-cli/requirements.py3.Darwin.txt' + + - job: VerifyWindowsRequirements + displayName: 'Verify src/azure-cli/requirements.*.Windows.txt' + condition: succeeded() + pool: + vmImage: 'windows-2019' + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + + - task: BatchScript@1 + inputs: + filename: ./scripts/ci/dependency_check.bat + displayName: 'Verify src/azure-cli/requirements.py3.Windows.txt' + + - job: VerifyVersions + displayName: Verify Command Module Versions + condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/release'), eq(variables['System.PullRequest.TargetBranch'], 'release'))) + + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev verify history + + displayName: 'Verify History' + + - job: BuildWindowsMSI + displayName: Build Windows MSI + + dependsOn: ExtractMetadata + condition: succeeded() + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - script: | + set /p CLI_VERSION=<$(System.ArtifactsDirectory)/metadata/version + set + + build_scripts/windows/scripts/build.cmd + displayName: 'Build Windows MSI' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: 'build_scripts/windows/out/' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: MSI' + inputs: + TargetPath: 'build_scripts/windows/out/' + ArtifactName: msi + + - job: TestWindowsMSI + displayName: Test Windows MSI + + dependsOn: BuildWindowsMSI + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/msi' + artifactName: msi + + - task: PowerShell@2 + displayName: Install and Load CLI + inputs: + targetType: inline + script: | + if (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")) { + # Start another Powershell process as Admin and execute this script again + $arguments = "& '" +$myinvocation.mycommand.definition + "'" + Start-Process powershell -Verb runAs -ArgumentList $arguments + # Stop if the PowerShell is not run as Admin + Break + } + # The following are executed by elevated PowerShell + az --version + + $InstallArgs = @( + "/i" + "`"$env:SYSTEM_ARTIFACTSDIRECTORY\msi\Microsoft Azure CLI.msi`"" + "/q" + "/norestart" + "/l*v" + ".\install_logs.txt" + ) + $pre_installed_version=az version --query '\"azure-cli\"' -o tsv + $to_be_installed_version=Get-Content $(System.ArtifactsDirectory)/metadata/version + if ($pre_installed_version -eq $to_be_installed_version){ + # See https://docs.microsoft.com/windows/win32/msi/reinstallmode about options of REINSTALLMODE + $reinstall_option="REINSTALL=ALL REINSTALLMODE=emus" + $InstallArgs += $reinstall_option + } + Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow + $install_time=Measure-Command {Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow} | select -expand TotalSeconds + $installed_version=az version --query '\"azure-cli\"' -o tsv + if ($installed_version -ne $to_be_installed_version){ + echo "The MSI failed to install." + Exit 1 + } + echo 'Install time(seconds):' $install_time + az --version + # Test bundled pip with extension installation + az extension add -n rdbms-connect + az self-test + + Get-Content .\install_logs.txt + + - job: BuildDockerImage + displayName: Build Docker Image + + dependsOn: ExtractMetadata + condition: succeeded() + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: Bash@3 + displayName: 'Bash Script' + inputs: + targetType: 'filePath' + filePath: scripts/release/docker/pipeline.sh + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + DockerImagesToScan: 'clibuild$BUILD_BUILDNUMBER:latest' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: docker image' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: docker + + - job: TestDockerImage + displayName: Test Docker Image + + dependsOn: BuildDockerImage + condition: succeeded() + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Docker Image' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: docker + + + - bash: | + set -exv + + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + IMAGE_NAME=clibuild$BUILD_BUILDNUMBER:latest + TAR_FILE=$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$CLI_VERSION.tar + + echo "== Test docker image ==" + + docker load < $TAR_FILE + docker run $IMAGE_NAME /bin/bash -c "time az self-test && time az --version && sleep 5" + displayName: 'Bash Script' + + - job: BuildPythonWheel + displayName: Build Python Wheels + + dependsOn: ExtractMetadata + condition: succeeded() + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + + + - script: | + if [[ "$(Build.Reason)" == "PullRequest" ]]; then + branch=$(System.PullRequest.TargetBranch) + else + branch=$(Build.SourceBranchName) + fi + scripts/release/pypi/build.sh $branch + displayName: 'Run Wheel Build Script' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: pypi' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: pypi + + - job: TestPythonWheel + displayName: Test Python Wheels + + dependsOn: BuildPythonWheel + condition: succeeded() + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download PyPI Packages' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/pypi' + artifactName: pypi + + + - bash: | + #!/usr/bin/env bash + + # Verify the pip wheels + + set -ex + + CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` + PYPI_FILES=$(cd $BUILD_ARTIFACTSTAGINGDIRECTORY/pypi; pwd) + + echo "== Testing pip install on Python 3.7 ==" + docker run \ + --rm -v $PYPI_FILES:/mnt/pypi python:3.7 \ + /bin/bash -c "ls /mnt/pypi && pip install -f /mnt/pypi -q azure-cli==$CLI_VERSION.* && az self-test && az --version && sleep 5" + + echo "== Testing pip install on Python 3.9 ==" + docker run \ + --rm -v $PYPI_FILES:/mnt/pypi python:3.9 \ + /bin/bash -c "ls /mnt/pypi && pip install -f /mnt/pypi -q azure-cli==$CLI_VERSION.* && az self-test && az --version && sleep 5" + + echo "== Testing pip install on Python 3.10 ==" + docker run \ + --rm -v $PYPI_FILES:/mnt/pypi python:3.10 \ + /bin/bash -c "ls /mnt/pypi && pip install -f /mnt/pypi -q azure-cli==$CLI_VERSION.* && az self-test && az --version && sleep 5" + + displayName: 'Test pip Install' + + - job: TestCore + displayName: Unit Test for Core + timeoutInMinutes: 10 + pool: + vmImage: 'ubuntu-20.04' + strategy: + matrix: + Python39: + python.version: '3.9' + Python310: + python.version: '3.10' + steps: + - template: .azure-pipelines/templates/automation_test.yml + parameters: + pythonVersion: '$(python.version)' + module: 'azure-cli-core' + + - job: TestTelemetry + displayName: Unit Test for Telemetry + timeoutInMinutes: 10 + pool: + vmImage: 'ubuntu-20.04' + strategy: + matrix: + Python39: + python.version: '3.9' + Python310: + python.version: '3.10' + steps: + - template: .azure-pipelines/templates/automation_test.yml + parameters: + pythonVersion: '$(python.version)' + module: 'azure-cli-telemetry' + + - job: IntegrationTestAgainstProfiles + displayName: Integration Test against Profiles + dependsOn: BuildPythonWheel + condition: succeeded() + timeoutInMinutes: 20 + + pool: + vmImage: 'ubuntu-20.04' + strategy: + matrix: + Python39: + python.version: '3.9' + Python310: + python.version: '3.10' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - bash: pip install --upgrade pip wheel + displayName: 'Install pip and wheel' + - bash: ./scripts/ci/test_profile_integration.sh + displayName: 'Run Integration Test against Profiles' + + - job: TestExtensionsLoading + displayName: Test Extensions Loading + condition: succeeded() + timeoutInMinutes: 40 + + pool: + vmImage: 'ubuntu-20.04' + strategy: + matrix: + Python310: + python.version: '3.10' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - bash: pip install --upgrade pip wheel + displayName: 'Install pip and wheel' + - bash: ./scripts/ci/test_extensions.sh + displayName: 'Load extensions' + + - job: BuildHomebrewFormula + displayName: Build Homebrew Formula + + dependsOn: BuildPythonWheel + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - bash: | + #!/bin/bash + + root=$(cd $(dirname $0); pwd) + + set -evx + + CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` + HOMEBREW_UPSTREAM_URL=`curl -Ls -o /dev/null -w %{url_effective} https://api.github.com/repos/Azure/azure-cli/tarball/$BUILD_SOURCEVERSION` + + docker_files=$(cd $BUILD_SOURCESDIRECTORY/scripts/release/homebrew/docker; pwd) + src_files=$(cd $BUILD_SOURCESDIRECTORY/src; pwd) + + echo "Generating formula in docker container ... " + docker run -v $docker_files:/mnt/scripts \ + -v $src_files:/mnt/src \ + -e CLI_VERSION=$CLI_VERSION \ + -e HOMEBREW_UPSTREAM_URL=$HOMEBREW_UPSTREAM_URL \ + --name azurecli \ + python:3.10 \ + /mnt/scripts/run.sh + + # clean up + rm -rf $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata + + docker cp azurecli:azure-cli.rb $BUILD_ARTIFACTSTAGINGDIRECTORY/azure-cli.rb + docker rm --force azurecli + displayName: 'Build homebrew formula' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: homebrew' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: homebrew + + - job: TestHomebrewFormula + displayName: Test Homebrew Formula + + dependsOn: BuildHomebrewFormula + condition: succeeded() + pool: + vmImage: 'macOS-10.15' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Homebrew' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' + artifactName: homebrew + + + - bash: | + set -ev + + echo == Remove pre-installed azure-cli == + brew uninstall azure-cli + + echo == Install azure-cli.rb formula == + brew install --build-from-source $SYSTEM_ARTIFACTSDIRECTORY/homebrew/azure-cli.rb + + echo == Az Version == + az --version + + echo == Run Self-Test == + az self-test + + displayName: 'Bash Script' + + - job: TestHomebrewPackage + displayName: Test Homebrew Package + timeoutInMinutes: 180 + dependsOn: BuildHomebrewFormula + # condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + condition: false + pool: + vmImage: 'macOS-10.15' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' + artifactName: homebrew + + + - bash: ./scripts/release/homebrew/test_homebrew_package.sh + + + displayName: 'Test Homebrew Package' + + + - job: BuildRpmPackageMariner + displayName: Build Rpm Package Mariner + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + vmImage: 'ubuntu-20.04' + strategy: + matrix: + 1.0: + image: cblmariner.azurecr.io/base/core + tag: 1.0 + 2.0: + image: mcr.microsoft.com/cbl-mariner/base/core + tag: 2.0 + steps: + - task: Bash@3 + displayName: 'Build Rpm Package: Mariner' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline_mariner.sh + env: + IMAGE: $(image) + TAG: $(tag) + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: rpm-mariner' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: rpm-mariner$(tag) + + + # TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. + - job: BuildRpmPackageUbi8 + displayName: Build Rpm Package Red Hat Universal Base Image 8 + # Do not run this job for Pull Requests due to the slowness + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: Bash@3 + displayName: 'Build Rpm Package' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline.sh + env: + IMAGE: ubi8 + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: rpm' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: rpm-ubi8 + + + - job: TestRpmPackage + displayName: Test Rpm Package + timeoutInMinutes: 120 + dependsOn: BuildRpmPackageUbi8 + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' + artifactName: rpm-ubi8 + + + - bash: | + set -ex + + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + RPM_NAME=azure-cli-$CLI_VERSION-1.el8.x86_64.rpm + RPM_FILE=$SYSTEM_ARTIFACTSDIRECTORY/rpm/$RPM_NAME + + echo "== Test rpm package on Red Hat UBI 8 ==" + + IMAGE=registry.access.redhat.com/ubi8/ubi:8.4 + docker pull $IMAGE + docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" + + displayName: 'Test Rpm Package' + + - job: BuildDebPackages + displayName: Build Deb Packages + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + vmImage: 'ubuntu-20.04' + strategy: + matrix: + # https://wiki.ubuntu.com/Releases + Bionic: + # 18.04 + deb_system: ubuntu + distro: bionic + Focal: + # 20.04 + deb_system: ubuntu + distro: focal + Impish: + # 21.10 + deb_system: ubuntu + distro: impish + Jammy: + # 22.04 + deb_system: ubuntu + distro: jammy + + # https://wiki.debian.org/DebianReleases + Buster: + # 10 + deb_system: debian + distro: buster + Bullseye: + # 11 + deb_system: debian + distro: bullseye + steps: + - task: Bash@3 + displayName: 'Build $(deb_system) $(distro) Package' + inputs: + targetType: 'filePath' + filePath: scripts/release/debian/pipeline.sh + env: + DISTRO: $(distro) + DISTRO_BASE_IMAGE: $(deb_system):$(distro) + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: debian' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(deb_system)-$(distro) + + - job: TestDebPackages + displayName: Test Deb Packages + dependsOn: + - BuildDebPackages + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + strategy: + matrix: + Focal: + deb_system: ubuntu + distro: focal + Jammy: + deb_system: ubuntu + distro: jammy + Buster: + deb_system: debian + distro: buster + Bullseye: + deb_system: debian + distro: bullseye + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download $(deb_system):$(distro) Build' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/debian' + artifactName: $(deb_system)-$(distro) + + - task: Bash@3 + displayName: 'Test $(deb_system) $(distro) Package' + env: + DISTRO: $(distro) + DISTRO_BASE_IMAGE: $(deb_system):$(distro) + inputs: + targetType: 'inline' + script: | + set -exv + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + echo "== Test debian package on ${DISTRO} ==" + docker pull ${DISTRO_BASE_IMAGE} + docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" + + - job: CheckStyle + displayName: "Check CLI Style" + + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev style + + - job: CheckHeaders + displayName: "Check License, History, and DocMap" + + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev verify license + azdev verify history + azdev verify document-map + + - job: PerformanceCheck + displayName: "PerformanceCheck" + strategy: + matrix: + Python39: + python.version: '3.9' + Python310: + python.version: '3.10' + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev perf load-times + displayName: "Load Performance" + # - bash: | + # set -ev + # . env/bin/activate + + # azdev perf benchmark "version" "network vnet -h" "rest -h" "storage account" + # displayName: "Execution Performance" + + - job: CheckLinter + displayName: "Check CLI Linter" + + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + python scripts/ci/service_name.py + if [[ "$(System.PullRequest.TargetBranch)" != "" ]]; then + # If CI is set to shallow fetch, target branch should be expilictly fetched. + git fetch origin --depth=1 $(System.PullRequest.TargetBranch) + azdev linter --ci-exclusions --min-severity medium --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) + else + azdev linter --ci-exclusions --min-severity medium + fi + + - job: VerifySphinxDocumentGenerator + displayName: "Verify Sphinx Document Generator" + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + - bash: pip install --upgrade pip wheel + displayName: "Install pip and wheel" + - bash: ./scripts/ci/test_ref_doc.sh + displayName: "Verify Sphinx Document Generator" From a43a67d72112e99225a536ba97a21720e623b4f5 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Tue, 5 Jul 2022 16:27:42 +0800 Subject: [PATCH 2/3] Remove sequence value indent --- azure-pipelines.yml | 1762 +++++++++++++++++++++---------------------- 1 file changed, 881 insertions(+), 881 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ee60076e104..9bef01b1f73 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,894 +1,894 @@ resources: - - repo: self +- repo: self trigger: batch: true branches: include: - - '*' + - '*' pr: branches: include: - - '*' + - '*' jobs: - - job: CheckPullRequest - displayName: "Check the Format of Pull Request Title and Content" - condition: and(succeeded(), in(variables['System.PullRequest.TargetBranch'], 'dev', 'release')) - - pool: - vmImage: 'ubuntu-20.04' - steps: - - bash: | - echo "Check Title of Pull Request: #$(System.PullRequest.PullRequestNumber)" - title=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.title') - echo $title - body=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.body') - echo $body - if [ "$(System.PullRequest.TargetBranch)" != "release" ] && echo $title | grep -iqF hotfix:; then - echo "Hotfix PR should target release branch." - exit 1 - fi - python scripts/ci/check_pull_request.py "$title" "$body" - - - job: RejectPullRequestToMasterBranch - displayName: "Reject Pull Request To Master Branch" - condition: and(succeeded(), eq(variables['System.PullRequest.TargetBranch'], 'master')) - - pool: - vmImage: 'ubuntu-20.04' - steps: - - bash: | - echo "Reject pull request directly to master branch" - exit 1 - - - job: CredScan - displayName: "Credential Scan" - pool: - vmImage: "windows-2019" - steps: - - task: ms-codeanalysis.vss-microsoft-security-code-analysis-devops.build-task-credscan.CredScan@2 - displayName: 'Run Credential Scanner' - inputs: - toolMajorVersion: V2 - suppressionsFile: './scripts/ci/credscan/CredScanSuppressions.json' - toolVersionV2: '2.1.17' - - task: ms-codeanalysis.vss-microsoft-security-code-analysis-devops.build-task-postanalysis.PostAnalysis@1 - displayName: 'Post Analysis' - inputs: - AllTools: false - BinSkim: false - CredScan: true - RoslynAnalyzers: false - TSLint: false - ToolLogsNotFoundAction: 'Standard' - - - job: ExtractMetadata - displayName: Extract Metadata - - condition: succeeded() - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: Bash@3 - displayName: 'Extract Version' - inputs: - targetType: 'filePath' - filePath: scripts/release/get_version.sh - - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: metadata' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: metadata - - - job: VerifyLinuxRequirements - displayName: 'Verify src/azure-cli/requirements.*.Linux.txt' - condition: succeeded() - pool: - vmImage: 'ubuntu-20.04' - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3' - inputs: - versionSpec: 3.x - - - bash: ./scripts/ci/dependency_check.sh - displayName: 'Verify src/azure-cli/requirements.py3.Linux.txt' - - - job: VerifyDarwinRequirements - displayName: 'Verify src/azure-cli/requirements.*.Darwin.txt' - condition: succeeded() - pool: - vmImage: 'macOS-10.15' - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3' - inputs: - versionSpec: 3.x - - - bash: ./scripts/ci/dependency_check.sh - displayName: 'Verify src/azure-cli/requirements.py3.Darwin.txt' - - - job: VerifyWindowsRequirements - displayName: 'Verify src/azure-cli/requirements.*.Windows.txt' - condition: succeeded() - pool: - vmImage: 'windows-2019' - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: 3.10 - - - task: BatchScript@1 - inputs: - filename: ./scripts/ci/dependency_check.bat - displayName: 'Verify src/azure-cli/requirements.py3.Windows.txt' - - - job: VerifyVersions - displayName: Verify Command Module Versions - condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/release'), eq(variables['System.PullRequest.TargetBranch'], 'release'))) - - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: 3.10 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev verify history - - displayName: 'Verify History' - - - job: BuildWindowsMSI - displayName: Build Windows MSI - - dependsOn: ExtractMetadata - condition: succeeded() - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - script: | - set /p CLI_VERSION=<$(System.ArtifactsDirectory)/metadata/version - set - - build_scripts/windows/scripts/build.cmd - displayName: 'Build Windows MSI' - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: 'build_scripts/windows/out/' - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: MSI' - inputs: - TargetPath: 'build_scripts/windows/out/' - ArtifactName: msi - - - job: TestWindowsMSI - displayName: Test Windows MSI - - dependsOn: BuildWindowsMSI - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/msi' - artifactName: msi - - - task: PowerShell@2 - displayName: Install and Load CLI - inputs: - targetType: inline - script: | - if (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")) { - # Start another Powershell process as Admin and execute this script again - $arguments = "& '" +$myinvocation.mycommand.definition + "'" - Start-Process powershell -Verb runAs -ArgumentList $arguments - # Stop if the PowerShell is not run as Admin - Break - } - # The following are executed by elevated PowerShell - az --version - - $InstallArgs = @( - "/i" - "`"$env:SYSTEM_ARTIFACTSDIRECTORY\msi\Microsoft Azure CLI.msi`"" - "/q" - "/norestart" - "/l*v" - ".\install_logs.txt" - ) - $pre_installed_version=az version --query '\"azure-cli\"' -o tsv - $to_be_installed_version=Get-Content $(System.ArtifactsDirectory)/metadata/version - if ($pre_installed_version -eq $to_be_installed_version){ - # See https://docs.microsoft.com/windows/win32/msi/reinstallmode about options of REINSTALLMODE - $reinstall_option="REINSTALL=ALL REINSTALLMODE=emus" - $InstallArgs += $reinstall_option - } - Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow - $install_time=Measure-Command {Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow} | select -expand TotalSeconds - $installed_version=az version --query '\"azure-cli\"' -o tsv - if ($installed_version -ne $to_be_installed_version){ - echo "The MSI failed to install." - Exit 1 - } - echo 'Install time(seconds):' $install_time - az --version - # Test bundled pip with extension installation - az extension add -n rdbms-connect - az self-test - - Get-Content .\install_logs.txt - - - job: BuildDockerImage - displayName: Build Docker Image - - dependsOn: ExtractMetadata - condition: succeeded() - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: Bash@3 - displayName: 'Bash Script' - inputs: - targetType: 'filePath' - filePath: scripts/release/docker/pipeline.sh - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - DockerImagesToScan: 'clibuild$BUILD_BUILDNUMBER:latest' - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: docker image' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: docker - - - job: TestDockerImage - displayName: Test Docker Image - - dependsOn: BuildDockerImage - condition: succeeded() - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Docker Image' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: docker - - - - bash: | - set -exv - - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - IMAGE_NAME=clibuild$BUILD_BUILDNUMBER:latest - TAR_FILE=$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$CLI_VERSION.tar - - echo "== Test docker image ==" - - docker load < $TAR_FILE - docker run $IMAGE_NAME /bin/bash -c "time az self-test && time az --version && sleep 5" - displayName: 'Bash Script' - - - job: BuildPythonWheel - displayName: Build Python Wheels - - dependsOn: ExtractMetadata - condition: succeeded() - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: 3.10 - - - - script: | - if [[ "$(Build.Reason)" == "PullRequest" ]]; then - branch=$(System.PullRequest.TargetBranch) - else - branch=$(Build.SourceBranchName) - fi - scripts/release/pypi/build.sh $branch - displayName: 'Run Wheel Build Script' - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: pypi' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: pypi - - - job: TestPythonWheel - displayName: Test Python Wheels - - dependsOn: BuildPythonWheel - condition: succeeded() - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - task: DownloadPipelineArtifact@1 - displayName: 'Download PyPI Packages' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/pypi' - artifactName: pypi - - - - bash: | - #!/usr/bin/env bash - - # Verify the pip wheels - - set -ex - - CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` - PYPI_FILES=$(cd $BUILD_ARTIFACTSTAGINGDIRECTORY/pypi; pwd) - - echo "== Testing pip install on Python 3.7 ==" - docker run \ - --rm -v $PYPI_FILES:/mnt/pypi python:3.7 \ - /bin/bash -c "ls /mnt/pypi && pip install -f /mnt/pypi -q azure-cli==$CLI_VERSION.* && az self-test && az --version && sleep 5" - - echo "== Testing pip install on Python 3.9 ==" - docker run \ - --rm -v $PYPI_FILES:/mnt/pypi python:3.9 \ - /bin/bash -c "ls /mnt/pypi && pip install -f /mnt/pypi -q azure-cli==$CLI_VERSION.* && az self-test && az --version && sleep 5" - - echo "== Testing pip install on Python 3.10 ==" - docker run \ - --rm -v $PYPI_FILES:/mnt/pypi python:3.10 \ - /bin/bash -c "ls /mnt/pypi && pip install -f /mnt/pypi -q azure-cli==$CLI_VERSION.* && az self-test && az --version && sleep 5" - - displayName: 'Test pip Install' - - - job: TestCore - displayName: Unit Test for Core - timeoutInMinutes: 10 - pool: - vmImage: 'ubuntu-20.04' - strategy: - matrix: - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - steps: - - template: .azure-pipelines/templates/automation_test.yml - parameters: - pythonVersion: '$(python.version)' - module: 'azure-cli-core' - - - job: TestTelemetry - displayName: Unit Test for Telemetry - timeoutInMinutes: 10 - pool: - vmImage: 'ubuntu-20.04' - strategy: - matrix: - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - steps: - - template: .azure-pipelines/templates/automation_test.yml - parameters: - pythonVersion: '$(python.version)' - module: 'azure-cli-telemetry' - - - job: IntegrationTestAgainstProfiles - displayName: Integration Test against Profiles - dependsOn: BuildPythonWheel - condition: succeeded() - timeoutInMinutes: 20 - - pool: - vmImage: 'ubuntu-20.04' - strategy: - matrix: - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - bash: pip install --upgrade pip wheel - displayName: 'Install pip and wheel' - - bash: ./scripts/ci/test_profile_integration.sh - displayName: 'Run Integration Test against Profiles' - - - job: TestExtensionsLoading - displayName: Test Extensions Loading - condition: succeeded() - timeoutInMinutes: 40 - - pool: - vmImage: 'ubuntu-20.04' - strategy: - matrix: - Python310: - python.version: '3.10' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - bash: pip install --upgrade pip wheel - displayName: 'Install pip and wheel' - - bash: ./scripts/ci/test_extensions.sh - displayName: 'Load extensions' - - - job: BuildHomebrewFormula - displayName: Build Homebrew Formula - - dependsOn: BuildPythonWheel - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - bash: | - #!/bin/bash - - root=$(cd $(dirname $0); pwd) - - set -evx - - CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` - HOMEBREW_UPSTREAM_URL=`curl -Ls -o /dev/null -w %{url_effective} https://api.github.com/repos/Azure/azure-cli/tarball/$BUILD_SOURCEVERSION` - - docker_files=$(cd $BUILD_SOURCESDIRECTORY/scripts/release/homebrew/docker; pwd) - src_files=$(cd $BUILD_SOURCESDIRECTORY/src; pwd) - - echo "Generating formula in docker container ... " - docker run -v $docker_files:/mnt/scripts \ - -v $src_files:/mnt/src \ - -e CLI_VERSION=$CLI_VERSION \ - -e HOMEBREW_UPSTREAM_URL=$HOMEBREW_UPSTREAM_URL \ - --name azurecli \ - python:3.10 \ - /mnt/scripts/run.sh - - # clean up - rm -rf $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata - - docker cp azurecli:azure-cli.rb $BUILD_ARTIFACTSTAGINGDIRECTORY/azure-cli.rb - docker rm --force azurecli - displayName: 'Build homebrew formula' - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: homebrew' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: homebrew - - - job: TestHomebrewFormula - displayName: Test Homebrew Formula - - dependsOn: BuildHomebrewFormula - condition: succeeded() - pool: - vmImage: 'macOS-10.15' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Homebrew' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' - artifactName: homebrew - - - - bash: | - set -ev - - echo == Remove pre-installed azure-cli == - brew uninstall azure-cli - - echo == Install azure-cli.rb formula == - brew install --build-from-source $SYSTEM_ARTIFACTSDIRECTORY/homebrew/azure-cli.rb - - echo == Az Version == - az --version - - echo == Run Self-Test == - az self-test - - displayName: 'Bash Script' - - - job: TestHomebrewPackage - displayName: Test Homebrew Package - timeoutInMinutes: 180 - dependsOn: BuildHomebrewFormula - # condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - condition: false - pool: - vmImage: 'macOS-10.15' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' - artifactName: homebrew - - - - bash: ./scripts/release/homebrew/test_homebrew_package.sh - - - displayName: 'Test Homebrew Package' - - - - job: BuildRpmPackageMariner - displayName: Build Rpm Package Mariner - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - vmImage: 'ubuntu-20.04' - strategy: - matrix: - 1.0: - image: cblmariner.azurecr.io/base/core - tag: 1.0 - 2.0: - image: mcr.microsoft.com/cbl-mariner/base/core - tag: 2.0 - steps: - - task: Bash@3 - displayName: 'Build Rpm Package: Mariner' - inputs: - targetType: 'filePath' - filePath: scripts/release/rpm/pipeline_mariner.sh - env: - IMAGE: $(image) - TAG: $(tag) - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm-mariner' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: rpm-mariner$(tag) - - - # TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. - - job: BuildRpmPackageUbi8 - displayName: Build Rpm Package Red Hat Universal Base Image 8 - # Do not run this job for Pull Requests due to the slowness - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: Bash@3 - displayName: 'Build Rpm Package' - inputs: - targetType: 'filePath' - filePath: scripts/release/rpm/pipeline.sh - env: - IMAGE: ubi8 - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: rpm-ubi8 - - - - job: TestRpmPackage - displayName: Test Rpm Package - timeoutInMinutes: 120 - dependsOn: BuildRpmPackageUbi8 - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' - artifactName: rpm-ubi8 - - - - bash: | - set -ex - - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=azure-cli-$CLI_VERSION-1.el8.x86_64.rpm - RPM_FILE=$SYSTEM_ARTIFACTSDIRECTORY/rpm/$RPM_NAME - - echo "== Test rpm package on Red Hat UBI 8 ==" - - IMAGE=registry.access.redhat.com/ubi8/ubi:8.4 - docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" - - displayName: 'Test Rpm Package' - - - job: BuildDebPackages - displayName: Build Deb Packages - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - vmImage: 'ubuntu-20.04' - strategy: - matrix: - # https://wiki.ubuntu.com/Releases - Bionic: - # 18.04 - deb_system: ubuntu - distro: bionic - Focal: - # 20.04 - deb_system: ubuntu - distro: focal - Impish: - # 21.10 - deb_system: ubuntu - distro: impish - Jammy: - # 22.04 - deb_system: ubuntu - distro: jammy - - # https://wiki.debian.org/DebianReleases - Buster: - # 10 - deb_system: debian - distro: buster - Bullseye: - # 11 - deb_system: debian - distro: bullseye - steps: - - task: Bash@3 - displayName: 'Build $(deb_system) $(distro) Package' - inputs: - targetType: 'filePath' - filePath: scripts/release/debian/pipeline.sh - env: - DISTRO: $(distro) - DISTRO_BASE_IMAGE: $(deb_system):$(distro) - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: debian' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(deb_system)-$(distro) - - - job: TestDebPackages - displayName: Test Deb Packages - dependsOn: - - BuildDebPackages - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - strategy: - matrix: - Focal: - deb_system: ubuntu - distro: focal - Jammy: - deb_system: ubuntu - distro: jammy - Buster: - deb_system: debian - distro: buster - Bullseye: - deb_system: debian - distro: bullseye - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download $(deb_system):$(distro) Build' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: $(deb_system)-$(distro) - - - task: Bash@3 - displayName: 'Test $(deb_system) $(distro) Package' - env: - DISTRO: $(distro) - DISTRO_BASE_IMAGE: $(deb_system):$(distro) - inputs: - targetType: 'inline' - script: | - set -exv - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - echo "== Test debian package on ${DISTRO} ==" - docker pull ${DISTRO_BASE_IMAGE} - docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" - - - job: CheckStyle - displayName: "Check CLI Style" - - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: 3.10 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev style - - - job: CheckHeaders - displayName: "Check License, History, and DocMap" - - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: 3.10 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev verify license - azdev verify history - azdev verify document-map - - - job: PerformanceCheck - displayName: "PerformanceCheck" - strategy: - matrix: - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev perf load-times - displayName: "Load Performance" - # - bash: | - # set -ev - # . env/bin/activate - - # azdev perf benchmark "version" "network vnet -h" "rest -h" "storage account" - # displayName: "Execution Performance" - - - job: CheckLinter - displayName: "Check CLI Linter" - - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: 3.10 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - python scripts/ci/service_name.py - if [[ "$(System.PullRequest.TargetBranch)" != "" ]]; then - # If CI is set to shallow fetch, target branch should be expilictly fetched. - git fetch origin --depth=1 $(System.PullRequest.TargetBranch) - azdev linter --ci-exclusions --min-severity medium --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) - else - azdev linter --ci-exclusions --min-severity medium - fi - - - job: VerifySphinxDocumentGenerator +- job: CheckPullRequest + displayName: "Check the Format of Pull Request Title and Content" + condition: and(succeeded(), in(variables['System.PullRequest.TargetBranch'], 'dev', 'release')) + + pool: + vmImage: 'ubuntu-20.04' + steps: + - bash: | + echo "Check Title of Pull Request: #$(System.PullRequest.PullRequestNumber)" + title=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.title') + echo $title + body=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.body') + echo $body + if [ "$(System.PullRequest.TargetBranch)" != "release" ] && echo $title | grep -iqF hotfix:; then + echo "Hotfix PR should target release branch." + exit 1 + fi + python scripts/ci/check_pull_request.py "$title" "$body" + +- job: RejectPullRequestToMasterBranch + displayName: "Reject Pull Request To Master Branch" + condition: and(succeeded(), eq(variables['System.PullRequest.TargetBranch'], 'master')) + + pool: + vmImage: 'ubuntu-20.04' + steps: + - bash: | + echo "Reject pull request directly to master branch" + exit 1 + +- job: CredScan + displayName: "Credential Scan" + pool: + vmImage: "windows-2019" + steps: + - task: ms-codeanalysis.vss-microsoft-security-code-analysis-devops.build-task-credscan.CredScan@2 + displayName: 'Run Credential Scanner' + inputs: + toolMajorVersion: V2 + suppressionsFile: './scripts/ci/credscan/CredScanSuppressions.json' + toolVersionV2: '2.1.17' + - task: ms-codeanalysis.vss-microsoft-security-code-analysis-devops.build-task-postanalysis.PostAnalysis@1 + displayName: 'Post Analysis' + inputs: + AllTools: false + BinSkim: false + CredScan: true + RoslynAnalyzers: false + TSLint: false + ToolLogsNotFoundAction: 'Standard' + +- job: ExtractMetadata + displayName: Extract Metadata + + condition: succeeded() + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: Bash@3 + displayName: 'Extract Version' + inputs: + targetType: 'filePath' + filePath: scripts/release/get_version.sh + + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: metadata' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: metadata + +- job: VerifyLinuxRequirements + displayName: 'Verify src/azure-cli/requirements.*.Linux.txt' + condition: succeeded() + pool: + vmImage: 'ubuntu-20.04' + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3' + inputs: + versionSpec: 3.x + + - bash: ./scripts/ci/dependency_check.sh + displayName: 'Verify src/azure-cli/requirements.py3.Linux.txt' + +- job: VerifyDarwinRequirements + displayName: 'Verify src/azure-cli/requirements.*.Darwin.txt' + condition: succeeded() + pool: + vmImage: 'macOS-10.15' + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3' + inputs: + versionSpec: 3.x + + - bash: ./scripts/ci/dependency_check.sh + displayName: 'Verify src/azure-cli/requirements.py3.Darwin.txt' + +- job: VerifyWindowsRequirements + displayName: 'Verify src/azure-cli/requirements.*.Windows.txt' + condition: succeeded() + pool: + vmImage: 'windows-2019' + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + + - task: BatchScript@1 + inputs: + filename: ./scripts/ci/dependency_check.bat + displayName: 'Verify src/azure-cli/requirements.py3.Windows.txt' + +- job: VerifyVersions + displayName: Verify Command Module Versions + condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/release'), eq(variables['System.PullRequest.TargetBranch'], 'release'))) + + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev verify history + + displayName: 'Verify History' + +- job: BuildWindowsMSI + displayName: Build Windows MSI + + dependsOn: ExtractMetadata + condition: succeeded() + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - script: | + set /p CLI_VERSION=<$(System.ArtifactsDirectory)/metadata/version + set + + build_scripts/windows/scripts/build.cmd + displayName: 'Build Windows MSI' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: 'build_scripts/windows/out/' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: MSI' + inputs: + TargetPath: 'build_scripts/windows/out/' + ArtifactName: msi + +- job: TestWindowsMSI + displayName: Test Windows MSI + + dependsOn: BuildWindowsMSI + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/msi' + artifactName: msi + + - task: PowerShell@2 + displayName: Install and Load CLI + inputs: + targetType: inline + script: | + if (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")) { + # Start another Powershell process as Admin and execute this script again + $arguments = "& '" +$myinvocation.mycommand.definition + "'" + Start-Process powershell -Verb runAs -ArgumentList $arguments + # Stop if the PowerShell is not run as Admin + Break + } + # The following are executed by elevated PowerShell + az --version + + $InstallArgs = @( + "/i" + "`"$env:SYSTEM_ARTIFACTSDIRECTORY\msi\Microsoft Azure CLI.msi`"" + "/q" + "/norestart" + "/l*v" + ".\install_logs.txt" + ) + $pre_installed_version=az version --query '\"azure-cli\"' -o tsv + $to_be_installed_version=Get-Content $(System.ArtifactsDirectory)/metadata/version + if ($pre_installed_version -eq $to_be_installed_version){ + # See https://docs.microsoft.com/windows/win32/msi/reinstallmode about options of REINSTALLMODE + $reinstall_option="REINSTALL=ALL REINSTALLMODE=emus" + $InstallArgs += $reinstall_option + } + Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow + $install_time=Measure-Command {Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow} | select -expand TotalSeconds + $installed_version=az version --query '\"azure-cli\"' -o tsv + if ($installed_version -ne $to_be_installed_version){ + echo "The MSI failed to install." + Exit 1 + } + echo 'Install time(seconds):' $install_time + az --version + # Test bundled pip with extension installation + az extension add -n rdbms-connect + az self-test + + Get-Content .\install_logs.txt + +- job: BuildDockerImage + displayName: Build Docker Image + + dependsOn: ExtractMetadata + condition: succeeded() + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: Bash@3 + displayName: 'Bash Script' + inputs: + targetType: 'filePath' + filePath: scripts/release/docker/pipeline.sh + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + DockerImagesToScan: 'clibuild$BUILD_BUILDNUMBER:latest' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: docker image' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: docker + +- job: TestDockerImage + displayName: Test Docker Image + + dependsOn: BuildDockerImage + condition: succeeded() + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Docker Image' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: docker + + + - bash: | + set -exv + + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + IMAGE_NAME=clibuild$BUILD_BUILDNUMBER:latest + TAR_FILE=$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$CLI_VERSION.tar + + echo "== Test docker image ==" + + docker load < $TAR_FILE + docker run $IMAGE_NAME /bin/bash -c "time az self-test && time az --version && sleep 5" + displayName: 'Bash Script' + +- job: BuildPythonWheel + displayName: Build Python Wheels + + dependsOn: ExtractMetadata + condition: succeeded() + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + + + - script: | + if [[ "$(Build.Reason)" == "PullRequest" ]]; then + branch=$(System.PullRequest.TargetBranch) + else + branch=$(Build.SourceBranchName) + fi + scripts/release/pypi/build.sh $branch + displayName: 'Run Wheel Build Script' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: pypi' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: pypi + +- job: TestPythonWheel + displayName: Test Python Wheels + + dependsOn: BuildPythonWheel + condition: succeeded() + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download PyPI Packages' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/pypi' + artifactName: pypi + + + - bash: | + #!/usr/bin/env bash + + # Verify the pip wheels + + set -ex + + CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` + PYPI_FILES=$(cd $BUILD_ARTIFACTSTAGINGDIRECTORY/pypi; pwd) + + echo "== Testing pip install on Python 3.7 ==" + docker run \ + --rm -v $PYPI_FILES:/mnt/pypi python:3.7 \ + /bin/bash -c "ls /mnt/pypi && pip install -f /mnt/pypi -q azure-cli==$CLI_VERSION.* && az self-test && az --version && sleep 5" + + echo "== Testing pip install on Python 3.9 ==" + docker run \ + --rm -v $PYPI_FILES:/mnt/pypi python:3.9 \ + /bin/bash -c "ls /mnt/pypi && pip install -f /mnt/pypi -q azure-cli==$CLI_VERSION.* && az self-test && az --version && sleep 5" + + echo "== Testing pip install on Python 3.10 ==" + docker run \ + --rm -v $PYPI_FILES:/mnt/pypi python:3.10 \ + /bin/bash -c "ls /mnt/pypi && pip install -f /mnt/pypi -q azure-cli==$CLI_VERSION.* && az self-test && az --version && sleep 5" + + displayName: 'Test pip Install' + +- job: TestCore + displayName: Unit Test for Core + timeoutInMinutes: 10 + pool: + vmImage: 'ubuntu-20.04' + strategy: + matrix: + Python39: + python.version: '3.9' + Python310: + python.version: '3.10' + steps: + - template: .azure-pipelines/templates/automation_test.yml + parameters: + pythonVersion: '$(python.version)' + module: 'azure-cli-core' + +- job: TestTelemetry + displayName: Unit Test for Telemetry + timeoutInMinutes: 10 + pool: + vmImage: 'ubuntu-20.04' + strategy: + matrix: + Python39: + python.version: '3.9' + Python310: + python.version: '3.10' + steps: + - template: .azure-pipelines/templates/automation_test.yml + parameters: + pythonVersion: '$(python.version)' + module: 'azure-cli-telemetry' + +- job: IntegrationTestAgainstProfiles + displayName: Integration Test against Profiles + dependsOn: BuildPythonWheel + condition: succeeded() + timeoutInMinutes: 20 + + pool: + vmImage: 'ubuntu-20.04' + strategy: + matrix: + Python39: + python.version: '3.9' + Python310: + python.version: '3.10' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - bash: pip install --upgrade pip wheel + displayName: 'Install pip and wheel' + - bash: ./scripts/ci/test_profile_integration.sh + displayName: 'Run Integration Test against Profiles' + +- job: TestExtensionsLoading + displayName: Test Extensions Loading + condition: succeeded() + timeoutInMinutes: 40 + + pool: + vmImage: 'ubuntu-20.04' + strategy: + matrix: + Python310: + python.version: '3.10' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - bash: pip install --upgrade pip wheel + displayName: 'Install pip and wheel' + - bash: ./scripts/ci/test_extensions.sh + displayName: 'Load extensions' + +- job: BuildHomebrewFormula + displayName: Build Homebrew Formula + + dependsOn: BuildPythonWheel + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - bash: | + #!/bin/bash + + root=$(cd $(dirname $0); pwd) + + set -evx + + CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` + HOMEBREW_UPSTREAM_URL=`curl -Ls -o /dev/null -w %{url_effective} https://api.github.com/repos/Azure/azure-cli/tarball/$BUILD_SOURCEVERSION` + + docker_files=$(cd $BUILD_SOURCESDIRECTORY/scripts/release/homebrew/docker; pwd) + src_files=$(cd $BUILD_SOURCESDIRECTORY/src; pwd) + + echo "Generating formula in docker container ... " + docker run -v $docker_files:/mnt/scripts \ + -v $src_files:/mnt/src \ + -e CLI_VERSION=$CLI_VERSION \ + -e HOMEBREW_UPSTREAM_URL=$HOMEBREW_UPSTREAM_URL \ + --name azurecli \ + python:3.10 \ + /mnt/scripts/run.sh + + # clean up + rm -rf $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata + + docker cp azurecli:azure-cli.rb $BUILD_ARTIFACTSTAGINGDIRECTORY/azure-cli.rb + docker rm --force azurecli + displayName: 'Build homebrew formula' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: homebrew' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: homebrew + +- job: TestHomebrewFormula + displayName: Test Homebrew Formula + + dependsOn: BuildHomebrewFormula + condition: succeeded() + pool: + vmImage: 'macOS-10.15' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Homebrew' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' + artifactName: homebrew + + + - bash: | + set -ev + + echo == Remove pre-installed azure-cli == + brew uninstall azure-cli + + echo == Install azure-cli.rb formula == + brew install --build-from-source $SYSTEM_ARTIFACTSDIRECTORY/homebrew/azure-cli.rb + + echo == Az Version == + az --version + + echo == Run Self-Test == + az self-test + + displayName: 'Bash Script' + +- job: TestHomebrewPackage + displayName: Test Homebrew Package + timeoutInMinutes: 180 + dependsOn: BuildHomebrewFormula + # condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + condition: false + pool: + vmImage: 'macOS-10.15' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' + artifactName: homebrew + + + - bash: ./scripts/release/homebrew/test_homebrew_package.sh + + + displayName: 'Test Homebrew Package' + + +- job: BuildRpmPackageMariner + displayName: Build Rpm Package Mariner + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + vmImage: 'ubuntu-20.04' + strategy: + matrix: + 1.0: + image: cblmariner.azurecr.io/base/core + tag: 1.0 + 2.0: + image: mcr.microsoft.com/cbl-mariner/base/core + tag: 2.0 + steps: + - task: Bash@3 + displayName: 'Build Rpm Package: Mariner' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline_mariner.sh + env: + IMAGE: $(image) + TAG: $(tag) + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: rpm-mariner' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: rpm-mariner$(tag) + + +# TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. +- job: BuildRpmPackageUbi8 + displayName: Build Rpm Package Red Hat Universal Base Image 8 + # Do not run this job for Pull Requests due to the slowness + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: Bash@3 + displayName: 'Build Rpm Package' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline.sh + env: + IMAGE: ubi8 + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: rpm' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: rpm-ubi8 + + +- job: TestRpmPackage + displayName: Test Rpm Package + timeoutInMinutes: 120 + dependsOn: BuildRpmPackageUbi8 + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' + artifactName: rpm-ubi8 + + + - bash: | + set -ex + + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + RPM_NAME=azure-cli-$CLI_VERSION-1.el8.x86_64.rpm + RPM_FILE=$SYSTEM_ARTIFACTSDIRECTORY/rpm/$RPM_NAME + + echo "== Test rpm package on Red Hat UBI 8 ==" + + IMAGE=registry.access.redhat.com/ubi8/ubi:8.4 + docker pull $IMAGE + docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" + + displayName: 'Test Rpm Package' + +- job: BuildDebPackages + displayName: Build Deb Packages + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + vmImage: 'ubuntu-20.04' + strategy: + matrix: + # https://wiki.ubuntu.com/Releases + Bionic: + # 18.04 + deb_system: ubuntu + distro: bionic + Focal: + # 20.04 + deb_system: ubuntu + distro: focal + Impish: + # 21.10 + deb_system: ubuntu + distro: impish + Jammy: + # 22.04 + deb_system: ubuntu + distro: jammy + + # https://wiki.debian.org/DebianReleases + Buster: + # 10 + deb_system: debian + distro: buster + Bullseye: + # 11 + deb_system: debian + distro: bullseye + steps: + - task: Bash@3 + displayName: 'Build $(deb_system) $(distro) Package' + inputs: + targetType: 'filePath' + filePath: scripts/release/debian/pipeline.sh + env: + DISTRO: $(distro) + DISTRO_BASE_IMAGE: $(deb_system):$(distro) + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: debian' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(deb_system)-$(distro) + +- job: TestDebPackages + displayName: Test Deb Packages + dependsOn: + - BuildDebPackages + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + strategy: + matrix: + Focal: + deb_system: ubuntu + distro: focal + Jammy: + deb_system: ubuntu + distro: jammy + Buster: + deb_system: debian + distro: buster + Bullseye: + deb_system: debian + distro: bullseye + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download $(deb_system):$(distro) Build' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/debian' + artifactName: $(deb_system)-$(distro) + + - task: Bash@3 + displayName: 'Test $(deb_system) $(distro) Package' + env: + DISTRO: $(distro) + DISTRO_BASE_IMAGE: $(deb_system):$(distro) + inputs: + targetType: 'inline' + script: | + set -exv + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + echo "== Test debian package on ${DISTRO} ==" + docker pull ${DISTRO_BASE_IMAGE} + docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" + +- job: CheckStyle + displayName: "Check CLI Style" + + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev style + +- job: CheckHeaders + displayName: "Check License, History, and DocMap" + + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev verify license + azdev verify history + azdev verify document-map + +- job: PerformanceCheck + displayName: "PerformanceCheck" + strategy: + matrix: + Python39: + python.version: '3.9' + Python310: + python.version: '3.10' + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev perf load-times + displayName: "Load Performance" + # - bash: | + # set -ev + # . env/bin/activate + + # azdev perf benchmark "version" "network vnet -h" "rest -h" "storage account" + # displayName: "Execution Performance" + +- job: CheckLinter + displayName: "Check CLI Linter" + + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + python scripts/ci/service_name.py + if [[ "$(System.PullRequest.TargetBranch)" != "" ]]; then + # If CI is set to shallow fetch, target branch should be expilictly fetched. + git fetch origin --depth=1 $(System.PullRequest.TargetBranch) + azdev linter --ci-exclusions --min-severity medium --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) + else + azdev linter --ci-exclusions --min-severity medium + fi + +- job: VerifySphinxDocumentGenerator + displayName: "Verify Sphinx Document Generator" + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.10' + inputs: + versionSpec: 3.10 + - bash: pip install --upgrade pip wheel + displayName: "Install pip and wheel" + - bash: ./scripts/ci/test_ref_doc.sh displayName: "Verify Sphinx Document Generator" - pool: - vmImage: 'ubuntu-20.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.10' - inputs: - versionSpec: 3.10 - - bash: pip install --upgrade pip wheel - displayName: "Install pip and wheel" - - bash: ./scripts/ci/test_ref_doc.sh - displayName: "Verify Sphinx Document Generator" From c822129e5be2ca9b6b70516621975ea925f4a739 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Tue, 5 Jul 2022 17:29:29 +0800 Subject: [PATCH 3/3] Fix azure-pipelines-full-tests.yml --- azure-pipelines-full-tests.yml | 60 +++++++++++++++++----------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/azure-pipelines-full-tests.yml b/azure-pipelines-full-tests.yml index 17bf0e7d6e1..3e7dd55b214 100644 --- a/azure-pipelines-full-tests.yml +++ b/azure-pipelines-full-tests.yml @@ -10,7 +10,7 @@ trigger: pr: branches: include: - - '*' + - '*' jobs: - job: AutomationTest20200901 @@ -25,11 +25,11 @@ jobs: Python310: python.version: '3.10' steps: - - template: .azure-pipelines/templates/automation_test.yml - parameters: - pythonVersion: '$(python.version)' - profile: '2020-09-01-hybrid' - fullTest: true + - template: .azure-pipelines/templates/automation_test.yml + parameters: + pythonVersion: '$(python.version)' + profile: '2020-09-01-hybrid' + fullTest: true - job: AutomationTest20190301 displayName: Automation Test (Profile 2019-03-01) @@ -43,11 +43,11 @@ jobs: Python310: python.version: '3.10' steps: - - template: .azure-pipelines/templates/automation_test.yml - parameters: - pythonVersion: '$(python.version)' - profile: '2019-03-01-hybrid' - fullTest: true + - template: .azure-pipelines/templates/automation_test.yml + parameters: + pythonVersion: '$(python.version)' + profile: '2019-03-01-hybrid' + fullTest: true - job: AutomationTest20180301 displayName: Automation Test (Profile 2018-03-01) @@ -61,11 +61,11 @@ jobs: Python310: python.version: '3.10' steps: - - template: .azure-pipelines/templates/automation_test.yml - parameters: - pythonVersion: '$(python.version)' - profile: '2018-03-01-hybrid' - fullTest: true + - template: .azure-pipelines/templates/automation_test.yml + parameters: + pythonVersion: '$(python.version)' + profile: '2018-03-01-hybrid' + fullTest: true - job: AutomationFullTestPython39ProfileLatest displayName: Automation Full Test Python39 Profile Latest @@ -92,13 +92,13 @@ jobs: pool: vmImage: 'ubuntu-20.04' steps: - - template: .azure-pipelines/templates/automation_test.yml - parameters: - pythonVersion: '3.9' - profile: 'latest' - instance_cnt: '8' - instance_idx: '$(Instance_idx)' - fullTest: true + - template: .azure-pipelines/templates/automation_test.yml + parameters: + pythonVersion: '3.9' + profile: 'latest' + instance_cnt: '8' + instance_idx: '$(Instance_idx)' + fullTest: true - job: AutomationFullTestPython310ProfileLatest displayName: Automation Full Test Python310 Profile Latest @@ -125,10 +125,10 @@ jobs: pool: vmImage: 'ubuntu-20.04' steps: - - template: .azure-pipelines/templates/automation_test.yml - parameters: - pythonVersion: '3.10' - profile: 'latest' - instance_cnt: '8' - instance_idx: '$(Instance_idx)' - fullTest: true + - template: .azure-pipelines/templates/automation_test.yml + parameters: + pythonVersion: '3.10' + profile: 'latest' + instance_cnt: '8' + instance_idx: '$(Instance_idx)' + fullTest: true