[ci] Paralize azure pipeline (sonic-net#1043)

cherry-pick from master branch: sonic-net#1040
liushilongbuaa · May 19, 2022 · d2ebd13 · d2ebd13
1 parent 74e0056
commit d2ebd13
Show file tree

Hide file tree

Showing 5 changed files with 143 additions and 72 deletions.
diff --git a/.azure-pipelines/build-docker-sonic-vs-template.yml b/.azure-pipelines/build-docker-sonic-vs-template.yml
@@ -32,40 +32,48 @@ jobs:
     vmImage: 'ubuntu-20.04'
 
   steps:
+  - checkout: self
+    clean: true
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
       project: build
-      pipeline: 9
+      pipeline: Azure.sonic-swss-common
       artifact: ${{ parameters.swss_common_artifact_name }}
+      path: $(Build.ArtifactStagingDirectory)/download
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
     displayName: "Download sonic swss common deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       artifact: ${{ parameters.sairedis_artifact_name }}
+      path: $(Build.ArtifactStagingDirectory)/download
     displayName: "Download sonic sairedis deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       artifact: ${{ parameters.swss_artifact_name }}
+      path: $(Build.ArtifactStagingDirectory)/download
     displayName: "Download sonic swss artifact"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
       project: build
-      pipeline: 142
+      pipeline: Azure.sonic-buildimage.official.vs
       artifact: sonic-buildimage.vs
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
-    displayName: "Download sonic buildimage"
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      patterns: '**/target/docker-sonic-vs.gz'
+    displayName: "Download sonic-buildimage docker-sonic-vs"
   - script: |
+      set -ex
       echo $(Build.DefinitionName).$(Build.BuildNumber)
 
-      docker load < ../target/docker-sonic-vs.gz
+      docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz
 
       mkdir -p .azure-pipelines/docker-sonic-vs/debs
 
-      cp -v ../*.deb .azure-pipelines/docker-sonic-vs/debs
+      cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs
 
       pushd .azure-pipelines
 
@@ -74,7 +82,8 @@ jobs:
       popd
 
       docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
-    displayName: "Build sonic-docker-vs"
+      rm -rf $(Build.ArtifactStagingDirectory)/download
+    displayName: "Build docker-sonic-vs"
 
   - publish: $(Build.ArtifactStagingDirectory)/
     artifact: ${{ parameters.artifact_name }}

diff --git a/.azure-pipelines/build-swss-template.yml b/.azure-pipelines/build-swss-template.yml
@@ -21,6 +21,9 @@ parameters:
 - name: sonic_slave
   type: string
 
+- name: debian_version
+  type: string
+
 - name: sairedis_artifact_name
   type: string
 
@@ -38,13 +41,22 @@ jobs:
   pool:
     ${{ if ne(parameters.pool, 'default') }}:
       name: ${{ parameters.pool }}
-    ${{ if eq(parameters.pool, 'default') }}:
+    ${{ else }}:
       vmImage: 'ubuntu-20.04'
 
   container:
     image: sonicdev-microsoft.azurecr.io:443/${{ parameters.sonic_slave }}:latest
 
   steps:
+  - checkout: sonic-swss
+    submodules: true
+    clean: true
+  - script: |
+      set -ex
+      git checkout $(BUILD_BRANCH)
+      git submodule update
+      git status
+    displayName: Set up sonic-swss branch
   - script: |
       sudo apt-get install -y libhiredis0.14 libhiredis-dev
       sudo apt-get install -y libzmq5 libzmq3-dev
@@ -60,51 +72,57 @@ jobs:
     inputs:
       source: specific
       project: build
-      pipeline: 9
+      pipeline: Azure.sonic-swss-common
       artifact: ${{ parameters.swss_common_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      patterns: |
+        libswsscommon_1.0.0_*.deb
+        libswsscommon-dev_1.0.0*.deb
     displayName: "Download sonic swss common deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       artifact: ${{ parameters.sairedis_artifact_name }}
-    displayName: "Download sonic sairedis deb packages"
+      path: $(Build.ArtifactStagingDirectory)/download
+      patterns: |
+        libsaivs_*.deb
+        libsaivs-dev_*.deb
+        libsairedis_*.deb
+        libsairedis-dev_*.deb
+        libsaimetadata_*.deb
+        libsaimetadata-dev_*.deb
+        syncd-vs_*.deb
+    displayName: "Download pre-stage built ${{ parameters.sairedis_artifact_name }}"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
       project: build
-      pipeline: 1
-      artifact: sonic-buildimage.vs
+      pipeline: Azure.sonic-buildimage.common_libs
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
-    displayName: "Download sonic buildimage"
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      artifact: common-lib
+      patterns: |
+        target/debs/${{ parameters.debian_version }}/libnl-3*.deb
+        target/debs/${{ parameters.debian_version }}/libnl-genl*.deb
+        target/debs/${{ parameters.debian_version }}/libnl-route*.deb
+        target/debs/${{ parameters.debian_version }}/libnl-nf*.deb
+    displayName: "Download common libs"
+
   - script: |
-      sudo dpkg -i target/debs/buster/libnl-3-200_*.deb
-      sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb
-      sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb
-      sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb
-      sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb
-      sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb
-      sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb
-      sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb
-      sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb
-      sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
-      sudo dpkg -i libsaivs_*.deb
-      sudo dpkg -i libsaivs-dev_*.deb
-      sudo dpkg -i libsairedis_*.deb
-      sudo dpkg -i libsairedis-dev_*.deb
-      sudo dpkg -i libsaimetadata_*.deb
-      sudo dpkg -i libsaimetadata-dev_*.deb
-      sudo dpkg -i syncd-vs_*.deb
-    workingDirectory: $(Pipeline.Workspace)
+      set -ex
+      sudo dpkg -i $(find ./download -name *.deb)
+      rm -rf download || true
+    workingDirectory: $(Build.ArtifactStagingDirectory)
     displayName: "Install libnl3, sonic swss common, and sairedis"
-  - checkout: sonic-swss
-    path: s
-    submodules: true
   - script: |
+      set -ex
+      rm ../*.deb || true
       ./autogen.sh
-      dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb .
+      dpkg-buildpackage -us -uc -b -j$(nproc)
+      mv ../*.deb $(Build.ArtifactStagingDirectory)
     displayName: "Compile sonic swss"
-  - publish: $(System.DefaultWorkingDirectory)/
+  - publish: $(Build.ArtifactStagingDirectory)
     artifact: ${{ parameters.artifact_name }}
     displayName: "Archive swss debian packages"
diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml
@@ -51,7 +51,7 @@ jobs:
   pool:
     ${{ if ne(parameters.pool, 'default') }}:
       name: ${{ parameters.pool }}
-    ${{ if eq(parameters.pool, 'default') }}:
+    ${{ else }}:
       vmImage: 'ubuntu-20.04'
 
   container:
@@ -97,22 +97,25 @@ jobs:
     inputs:
       source: specific
       project: build
-      pipeline: 9
+      pipeline: Azure.sonic-swss-common
       artifact: ${{ parameters.swss_common_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
-      path: '$(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }}'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
     displayName: "Download sonic swss common deb packages"
   - script: |
-      cd $(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }}
-      sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb
-      sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
-    workingDirectory: $(Pipeline.Workspace)
+      set -ex
+      sudo dpkg -i download/libswsscommon_1.0.0_${{ parameters.arch }}.deb
+      sudo dpkg -i download/libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
+      rm -rf download
+    workingDirectory: $(Build.ArtifactStagingDirectory)
     displayName: "Install sonic swss Common"
   - script: |
       set -ex
+      rm ../*.deb || true
       ./autogen.sh
-      fakeroot debian/rules DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' CFLAGS="" CXXFLAGS="" binary-syncd-vs && cp ../*.deb .
+      fakeroot debian/rules DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' CFLAGS="" CXXFLAGS="" binary-syncd-vs
+      mv ../*.deb .
     displayName: "Compile sonic sairedis with coverage enabled"
   - script: |
       sudo cp azsyslog.conf /etc/rsyslog.conf

diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml
@@ -1,7 +1,7 @@
 parameters:
 - name: timeout
   type: number
-  default: 240
+  default: 360
 
 - name: log_artifact_name
   type: string
@@ -11,37 +11,46 @@ jobs:
   displayName: vstest
   timeoutInMinutes: ${{ parameters.timeout }}
 
-  pool:
-    vmImage: 'ubuntu-20.04'
+  pool: sonic-common
 
   steps:
+  - script: |
+      ls -A1 | xargs -I{} sudo rm -rf {}
+    displayName: "Clean workspace"
+  - checkout: self
+    clean: true
+    displayName: "Checkout sonic-sairedis"
+  - checkout: sonic-swss
+    clean: true
+    displayName: "Checkout sonic-swss"
+  - script: |
+      set -ex
+      cd sonic-swss
+      git checkout $(BUILD_BRANCH)
+    displayName: Set up sonic-swss branch
   - task: DownloadPipelineArtifact@2
     inputs:
       artifact: docker-sonic-vs
-    displayName: "Download docker sonic vs image"
-
+      path: $(Build.ArtifactStagingDirectory)/download
+    displayName: "Download pre-stage built docker-sonic-vs"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
       project: build
-      pipeline: 9
+      pipeline: Azure.sonic-swss-common
       artifact: sonic-swss-common.amd64.ubuntu20_04
+      path: $(Build.ArtifactStagingDirectory)/download
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
     displayName: "Download sonic swss common deb packages"
 
-  - checkout: self
-    displayName: "Checkout sonic-sairedis"
-  - checkout: sonic-swss
-    displayName: "Checkout sonic-swss"
-
   - script: |
-      set -x
+      set -ex
       sudo sonic-sairedis/.azure-pipelines/build_and_install_module.sh
 
       sudo apt-get install -y libhiredis0.14
-      sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb || apt-get install -f
-      sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb
+      sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f
+      sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb
 
       # install packages for vs test
       sudo apt-get install -y net-tools bridge-utils vlan
@@ -50,12 +59,13 @@ jobs:
     displayName: "Install dependencies"
 
   - script: |
-      set -x
-      sudo docker load -i ../docker-sonic-vs.gz
+      set -ex
+      sudo docker load -i $(Build.ArtifactStagingDirectory)/download/docker-sonic-vs.gz
       docker ps
       ip netns list
       pushd sonic-swss/tests
       sudo py.test -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
+      rm -rf $(Build.ArtifactStagingDirectory)/download
     displayName: "Run vs tests"
 
   - task: PublishTestResults@2