From d2ebd135085b6e6b51952372390f567a71fc0340 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Thu, 19 May 2022 18:06:25 +0800 Subject: [PATCH] [ci] Paralize azure pipeline (#1043) cherry-pick from master branch: https://github.com/Azure/sonic-sairedis/pull/1040 --- .../build-docker-sonic-vs-template.yml | 25 ++++-- .azure-pipelines/build-swss-template.yml | 80 ++++++++++++------- .azure-pipelines/build-template.yml | 21 ++--- .../test-docker-sonic-vs-template.yml | 44 ++++++---- azure-pipelines.yml | 45 +++++++++-- 5 files changed, 143 insertions(+), 72 deletions(-) diff --git a/.azure-pipelines/build-docker-sonic-vs-template.yml b/.azure-pipelines/build-docker-sonic-vs-template.yml index 347c61c2f..163230a66 100644 --- a/.azure-pipelines/build-docker-sonic-vs-template.yml +++ b/.azure-pipelines/build-docker-sonic-vs-template.yml @@ -32,40 +32,48 @@ jobs: vmImage: 'ubuntu-20.04' steps: + - checkout: self + clean: true - task: DownloadPipelineArtifact@2 inputs: source: specific project: build - pipeline: 9 + pipeline: Azure.sonic-swss-common artifact: ${{ parameters.swss_common_artifact_name }} + path: $(Build.ArtifactStagingDirectory)/download runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' + runBranch: 'refs/heads/$(BUILD_BRANCH)' displayName: "Download sonic swss common deb packages" - task: DownloadPipelineArtifact@2 inputs: artifact: ${{ parameters.sairedis_artifact_name }} + path: $(Build.ArtifactStagingDirectory)/download displayName: "Download sonic sairedis deb packages" - task: DownloadPipelineArtifact@2 inputs: artifact: ${{ parameters.swss_artifact_name }} + path: $(Build.ArtifactStagingDirectory)/download displayName: "Download sonic swss artifact" - task: DownloadPipelineArtifact@2 inputs: source: specific project: build - pipeline: 142 + pipeline: Azure.sonic-buildimage.official.vs artifact: sonic-buildimage.vs runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' - displayName: "Download sonic buildimage" + runBranch: 'refs/heads/$(BUILD_BRANCH)' + path: $(Build.ArtifactStagingDirectory)/download + patterns: '**/target/docker-sonic-vs.gz' + displayName: "Download sonic-buildimage docker-sonic-vs" - script: | + set -ex echo $(Build.DefinitionName).$(Build.BuildNumber) - docker load < ../target/docker-sonic-vs.gz + docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz mkdir -p .azure-pipelines/docker-sonic-vs/debs - cp -v ../*.deb .azure-pipelines/docker-sonic-vs/debs + cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs pushd .azure-pipelines @@ -74,7 +82,8 @@ jobs: popd docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz - displayName: "Build sonic-docker-vs" + rm -rf $(Build.ArtifactStagingDirectory)/download + displayName: "Build docker-sonic-vs" - publish: $(Build.ArtifactStagingDirectory)/ artifact: ${{ parameters.artifact_name }} diff --git a/.azure-pipelines/build-swss-template.yml b/.azure-pipelines/build-swss-template.yml index 6c1c3173d..96c4c9fe3 100644 --- a/.azure-pipelines/build-swss-template.yml +++ b/.azure-pipelines/build-swss-template.yml @@ -21,6 +21,9 @@ parameters: - name: sonic_slave type: string +- name: debian_version + type: string + - name: sairedis_artifact_name type: string @@ -38,13 +41,22 @@ jobs: pool: ${{ if ne(parameters.pool, 'default') }}: name: ${{ parameters.pool }} - ${{ if eq(parameters.pool, 'default') }}: + ${{ else }}: vmImage: 'ubuntu-20.04' container: image: sonicdev-microsoft.azurecr.io:443/${{ parameters.sonic_slave }}:latest steps: + - checkout: sonic-swss + submodules: true + clean: true + - script: | + set -ex + git checkout $(BUILD_BRANCH) + git submodule update + git status + displayName: Set up sonic-swss branch - script: | sudo apt-get install -y libhiredis0.14 libhiredis-dev sudo apt-get install -y libzmq5 libzmq3-dev @@ -60,51 +72,57 @@ jobs: inputs: source: specific project: build - pipeline: 9 + pipeline: Azure.sonic-swss-common artifact: ${{ parameters.swss_common_artifact_name }} runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' + runBranch: 'refs/heads/$(BUILD_BRANCH)' + path: $(Build.ArtifactStagingDirectory)/download + patterns: | + libswsscommon_1.0.0_*.deb + libswsscommon-dev_1.0.0*.deb displayName: "Download sonic swss common deb packages" - task: DownloadPipelineArtifact@2 inputs: artifact: ${{ parameters.sairedis_artifact_name }} - displayName: "Download sonic sairedis deb packages" + path: $(Build.ArtifactStagingDirectory)/download + patterns: | + libsaivs_*.deb + libsaivs-dev_*.deb + libsairedis_*.deb + libsairedis-dev_*.deb + libsaimetadata_*.deb + libsaimetadata-dev_*.deb + syncd-vs_*.deb + displayName: "Download pre-stage built ${{ parameters.sairedis_artifact_name }}" - task: DownloadPipelineArtifact@2 inputs: source: specific project: build - pipeline: 1 - artifact: sonic-buildimage.vs + pipeline: Azure.sonic-buildimage.common_libs runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' - displayName: "Download sonic buildimage" + runBranch: 'refs/heads/$(BUILD_BRANCH)' + path: $(Build.ArtifactStagingDirectory)/download + artifact: common-lib + patterns: | + target/debs/${{ parameters.debian_version }}/libnl-3*.deb + target/debs/${{ parameters.debian_version }}/libnl-genl*.deb + target/debs/${{ parameters.debian_version }}/libnl-route*.deb + target/debs/${{ parameters.debian_version }}/libnl-nf*.deb + displayName: "Download common libs" + - script: | - sudo dpkg -i target/debs/buster/libnl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb - sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb - sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb - sudo dpkg -i libsaivs_*.deb - sudo dpkg -i libsaivs-dev_*.deb - sudo dpkg -i libsairedis_*.deb - sudo dpkg -i libsairedis-dev_*.deb - sudo dpkg -i libsaimetadata_*.deb - sudo dpkg -i libsaimetadata-dev_*.deb - sudo dpkg -i syncd-vs_*.deb - workingDirectory: $(Pipeline.Workspace) + set -ex + sudo dpkg -i $(find ./download -name *.deb) + rm -rf download || true + workingDirectory: $(Build.ArtifactStagingDirectory) displayName: "Install libnl3, sonic swss common, and sairedis" - - checkout: sonic-swss - path: s - submodules: true - script: | + set -ex + rm ../*.deb || true ./autogen.sh - dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb . + dpkg-buildpackage -us -uc -b -j$(nproc) + mv ../*.deb $(Build.ArtifactStagingDirectory) displayName: "Compile sonic swss" - - publish: $(System.DefaultWorkingDirectory)/ + - publish: $(Build.ArtifactStagingDirectory) artifact: ${{ parameters.artifact_name }} displayName: "Archive swss debian packages" diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 355897ea9..c6b6fbaa9 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -51,7 +51,7 @@ jobs: pool: ${{ if ne(parameters.pool, 'default') }}: name: ${{ parameters.pool }} - ${{ if eq(parameters.pool, 'default') }}: + ${{ else }}: vmImage: 'ubuntu-20.04' container: @@ -97,22 +97,25 @@ jobs: inputs: source: specific project: build - pipeline: 9 + pipeline: Azure.sonic-swss-common artifact: ${{ parameters.swss_common_artifact_name }} runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' - path: '$(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }}' + runBranch: 'refs/heads/$(BUILD_BRANCH)' + path: $(Build.ArtifactStagingDirectory)/download displayName: "Download sonic swss common deb packages" - script: | - cd $(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }} - sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb - sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb - workingDirectory: $(Pipeline.Workspace) + set -ex + sudo dpkg -i download/libswsscommon_1.0.0_${{ parameters.arch }}.deb + sudo dpkg -i download/libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb + rm -rf download + workingDirectory: $(Build.ArtifactStagingDirectory) displayName: "Install sonic swss Common" - script: | set -ex + rm ../*.deb || true ./autogen.sh - fakeroot debian/rules DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' CFLAGS="" CXXFLAGS="" binary-syncd-vs && cp ../*.deb . + fakeroot debian/rules DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' CFLAGS="" CXXFLAGS="" binary-syncd-vs + mv ../*.deb . displayName: "Compile sonic sairedis with coverage enabled" - script: | sudo cp azsyslog.conf /etc/rsyslog.conf diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml index 3ba78bcc4..1447c34e5 100644 --- a/.azure-pipelines/test-docker-sonic-vs-template.yml +++ b/.azure-pipelines/test-docker-sonic-vs-template.yml @@ -1,7 +1,7 @@ parameters: - name: timeout type: number - default: 240 + default: 360 - name: log_artifact_name type: string @@ -11,37 +11,46 @@ jobs: displayName: vstest timeoutInMinutes: ${{ parameters.timeout }} - pool: - vmImage: 'ubuntu-20.04' + pool: sonic-common steps: + - script: | + ls -A1 | xargs -I{} sudo rm -rf {} + displayName: "Clean workspace" + - checkout: self + clean: true + displayName: "Checkout sonic-sairedis" + - checkout: sonic-swss + clean: true + displayName: "Checkout sonic-swss" + - script: | + set -ex + cd sonic-swss + git checkout $(BUILD_BRANCH) + displayName: Set up sonic-swss branch - task: DownloadPipelineArtifact@2 inputs: artifact: docker-sonic-vs - displayName: "Download docker sonic vs image" - + path: $(Build.ArtifactStagingDirectory)/download + displayName: "Download pre-stage built docker-sonic-vs" - task: DownloadPipelineArtifact@2 inputs: source: specific project: build - pipeline: 9 + pipeline: Azure.sonic-swss-common artifact: sonic-swss-common.amd64.ubuntu20_04 + path: $(Build.ArtifactStagingDirectory)/download runVersion: 'latestFromBranch' - runBranch: 'refs/heads/master' + runBranch: 'refs/heads/$(BUILD_BRANCH)' displayName: "Download sonic swss common deb packages" - - checkout: self - displayName: "Checkout sonic-sairedis" - - checkout: sonic-swss - displayName: "Checkout sonic-swss" - - script: | - set -x + set -ex sudo sonic-sairedis/.azure-pipelines/build_and_install_module.sh sudo apt-get install -y libhiredis0.14 - sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb || apt-get install -f - sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb + sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f + sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb # install packages for vs test sudo apt-get install -y net-tools bridge-utils vlan @@ -50,12 +59,13 @@ jobs: displayName: "Install dependencies" - script: | - set -x - sudo docker load -i ../docker-sonic-vs.gz + set -ex + sudo docker load -i $(Build.ArtifactStagingDirectory)/download/docker-sonic-vs.gz docker ps ip netns list pushd sonic-swss/tests sudo py.test -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) + rm -rf $(Build.ArtifactStagingDirectory)/download displayName: "Run vs tests" - task: PublishTestResults@2 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 804b5810a..02dfbed8f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,10 +3,29 @@ # Add steps that build, run tests, deploy, and more: # https://aka.ms/yaml +pr: +- master +- 202??? +- 201??? + trigger: + batch: true branches: include: - - "*" + - master + - 202??? + - 201??? + +# this part need to be set in UI +schedules: +- cron: "0 0 * * 6" + displayName: Weekly build + branches: + include: + - master + - 202??? + - 201??? + always: true resources: repositories: @@ -15,6 +34,17 @@ resources: name: Azure/sonic-swss endpoint: build +parameters: + - name: debian_version + type: string + default: buster +variables: + - name: BUILD_BRANCH + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + value: $(System.PullRequest.TargetBranch) + ${{ else }}: + value: $(Build.SourceBranchName) + stages: - stage: Build @@ -22,7 +52,7 @@ stages: - template: .azure-pipelines/build-template.yml parameters: arch: amd64 - sonic_slave: sonic-slave-buster + sonic_slave: sonic-slave-${{ parameters.debian_version }} swss_common_artifact_name: sonic-swss-common artifact_name: sonic-sairedis syslog_artifact_name: sonic-sairedis.syslog @@ -36,9 +66,9 @@ stages: - template: .azure-pipelines/build-template.yml parameters: arch: armhf - timeout: 240 + timeout: 180 pool: sonicbld-armhf - sonic_slave: sonic-slave-buster-armhf + sonic_slave: sonic-slave-${{ parameters.debian_version }}-armhf swss_common_artifact_name: sonic-swss-common.armhf artifact_name: sonic-sairedis.armhf syslog_artifact_name: sonic-sairedis.syslog.armhf @@ -46,9 +76,9 @@ stages: - template: .azure-pipelines/build-template.yml parameters: arch: arm64 - timeout: 240 + timeout: 180 pool: sonicbld-arm64 - sonic_slave: sonic-slave-buster-arm64 + sonic_slave: sonic-slave-${{ parameters.debian_version }}-arm64 swss_common_artifact_name: sonic-swss-common.arm64 artifact_name: sonic-sairedis.arm64 syslog_artifact_name: sonic-sairedis.syslog.arm64 @@ -60,10 +90,11 @@ stages: - template: .azure-pipelines/build-swss-template.yml parameters: arch: amd64 - sonic_slave: sonic-slave-buster + sonic_slave: sonic-slave-${{ parameters.debian_version }} swss_common_artifact_name: sonic-swss-common sairedis_artifact_name: sonic-sairedis artifact_name: sonic-swss + debian_version: ${{ parameters.debian_version }} - stage: BuildDocker dependsOn: BuildSwss