Backport all CI matrix changes from 7.0 to reelase/6.0

eng/pipelines/common/platform-matrix.yml

@@ -91,7 +91,7 @@ jobs:
       targetRid: linux-musl-x64
       platform: Linux_musl_x64
       container:
-        image: alpine-3.9-WithNode-20210714125437-9b5bbc2
+        image: alpine-3.13-WithNode-20210910135845-c401c85
         registry: mcr
       jobParameters:
         runtimeFlavor: ${{ parameters.runtimeFlavor }}
@@ -205,6 +205,32 @@ jobs:
         ${{ insert }}: ${{ parameters.jobParameters }}
         buildingOnSourceBuildImage: true
 
+# Linux s390x
+
+- ${{ if or(containsValue(parameters.platforms, 'Linux_s390x'), in(parameters.platformGroup, 'all', 'gcstress')) }}:
+  - template: xplat-setup.yml
+    parameters:
+      jobTemplate: ${{ parameters.jobTemplate }}
+      helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
+      variables: ${{ parameters.variables }}
+      osGroup: Linux
+      archType: s390x
+      targetRid: linux-s390x
+      platform: Linux_s390x
+      container:
+        image: ubuntu-18.04-cross-s390x-20201102145728-d6e0352
+        registry: mcr
+      jobParameters:
+        runtimeFlavor: ${{ parameters.runtimeFlavor }}
+        stagedBuild: ${{ parameters.stagedBuild }}
+        buildConfig: ${{ parameters.buildConfig }}
+        ${{ if eq(parameters.passPlatforms, true) }}:
+          platforms: ${{ parameters.platforms }}
+        helixQueueGroup: ${{ parameters.helixQueueGroup }}
+        crossBuild: true
+        crossrootfsDir: '/crossrootfs/s390x'
+        ${{ insert }}: ${{ parameters.jobParameters }}
+
 # WebAssembly
 
 - ${{ if containsValue(parameters.platforms, 'Browser_wasm') }}:

eng/pipelines/coreclr/templates/helix-queues-setup.yml

@@ -47,53 +47,56 @@ jobs:
       - ${{ if eq(variables['System.TeamProject'], 'public') }}:
         - (Ubuntu.1804.Arm32.Open)[email protected]/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440
       - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        - (Debian.9.Arm32)[email protected]/dotnet-buildtools/prereqs:debian-9-helix-arm32v7-bfcd90a-20200121150037
+        - (Debian.10.Arm32)[email protected]/dotnet-buildtools/prereqs:debian-10-helix-arm32v7-20210304164340-6616c63
+        - (Debian.11.Arm32)[email protected]/dotnet-buildtools/prereqs:debian-11-helix-arm32v7-20210304164347-5a7c380
         - (Ubuntu.1804.Arm32)[email protected]/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440
 
     # Linux arm64
     - ${{ if eq(parameters.platform, 'Linux_arm64') }}:
       - ${{ if eq(variables['System.TeamProject'], 'public') }}:
         - (Ubuntu.1804.Arm64.Open)[email protected]/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8-20210531091519-97d8652
       - ${{ if and(eq(variables['System.TeamProject'], 'public'), notIn(parameters.jobParameters.helixQueueGroup, 'pr', 'ci', 'libraries')) }}:
-        - (Debian.9.Arm64.Open)[email protected]/dotnet-buildtools/prereqs:debian-9-helix-arm64v8-bfcd90a-20200121150055
+        - (Debian.10.Arm64.Open)[email protected]/dotnet-buildtools/prereqs:debian-10-helix-arm64v8-20210304164340-56c6673
+        - (Debian.11.Arm64.Open)[email protected]/dotnet-buildtools/prereqs:debian-11-helix-arm64v8-20210304164340-5a7c380
       - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        - (Debian.9.Arm64)[email protected]/dotnet-buildtools/prereqs:debian-9-helix-arm64v8-bfcd90a-20200121150055
+        - (Debian.10.Arm64)[email protected]/dotnet-buildtools/prereqs:debian-10-helix-arm64v8-20210304164340-56c6673
+        - (Debian.11.Arm64)[email protected]/dotnet-buildtools/prereqs:debian-11-helix-arm64v8-20210304164340-5a7c380
         - (Ubuntu.1804.Arm64)[email protected]/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8-20210531091519-97d8652
 
     # Linux musl x64
     - ${{ if eq(parameters.platform, 'Linux_musl_x64') }}:
       - ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        - (Alpine.312.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.12-helix-20200602002622-e06dc59
+        - (Alpine.314.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64-20210910135833-1848e19
       - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        - (Alpine.312.Amd64)[email protected]/dotnet-buildtools/prereqs:alpine-3.12-helix-20200602002622-e06dc59
+        - (Alpine.314.Amd64)[email protected]/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64-20210910135833-1848e19
 
     # Linux musl arm32
     - ${{ if eq(parameters.platform, 'Linux_musl_arm') }}:
       - ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        - (Alpine.313.Arm32.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.13-helix-arm32v7-20210414141857-1ea6b0a
+        - (Alpine.314.Arm32.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.14-helix-arm32v7-20210910135806-8a6f4f3
       - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        - (Alpine.313.Arm32)[email protected]/dotnet-buildtools/prereqs:alpine-3.13-helix-arm32v7-20210414141857-1ea6b0a
+        - (Alpine.314.Arm32)[email protected]/dotnet-buildtools/prereqs:alpine-3.14-helix-arm32v7-20210910135806-8a6f4f3
 
     # Linux musl arm64
     - ${{ if eq(parameters.platform, 'Linux_musl_arm64') }}:
       - ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        - (Alpine.312.Arm64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.12-helix-arm64v8-20200602002604-25f8a3e
+        - (Alpine.314.Arm64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.14-helix-arm64v8-20210910135810-8a6f4f3
       - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        - (Alpine.312.Arm64)[email protected]/dotnet-buildtools/prereqs:alpine-3.12-helix-arm64v8-20200602002604-25f8a3e
+        - (Alpine.314.Arm64)[email protected]/dotnet-buildtools/prereqs:alpine-3.14-helix-arm64v8-20210910135810-8a6f4f3
 
     # Linux x64
     - ${{ if eq(parameters.platform, 'Linux_x64') }}:
       - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(parameters.jobParameters.helixQueueGroup, 'pr', 'ci', 'libraries')) }}:
         - Ubuntu.1804.Amd64.Open
       - ${{ if and(eq(variables['System.TeamProject'], 'public'), notIn(parameters.jobParameters.helixQueueGroup, 'pr', 'ci', 'libraries')) }}:
-        - Debian.9.Amd64.Open
-        - Ubuntu.1604.Amd64.Open
+        - (Debian.10.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:debian-10-helix-amd64-20210304164434-56c6673
+        - (Debian.11.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64-20210304164428-5a7c380
         - Ubuntu.1804.Amd64.Open
         - (Centos.8.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:centos-8-helix-20201229003624-c1bf759
         - RedHat.7.Amd64.Open
       - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        - Debian.9.Amd64
-        - Ubuntu.1604.Amd64
+        - (Debian.10.Amd64)[email protected]/dotnet-buildtools/prereqs:debian-10-helix-amd64-20210304164434-56c6673
+        - (Debian.11.Amd64)[email protected]/dotnet-buildtools/prereqs:debian-11-helix-amd64-20210304164428-5a7c380
         - Ubuntu.1804.Amd64
         - (Centos.8.Amd64)[email protected]/dotnet-buildtools/prereqs:centos-8-helix-20201229003624-c1bf759
         - (Fedora.34.Amd64)[email protected]/dotnet-buildtools/prereqs:fedora-34-helix-20210728124700-4f64125

eng/pipelines/coreclr/templates/run-performance-job.yml

@@ -23,7 +23,7 @@ parameters:
   runKind: ''                     # required -- test category
   logicalMachine: ''              # required -- Used to specify a which pool of machines the test should run against
   javascriptEngine: 'NoJS'
-  
+
 jobs:
 - template: xplat-pipeline-job.yml
   parameters:
@@ -36,7 +36,7 @@ jobs:
     enableTelemetry: ${{ parameters.enableTelemetry }}
     enablePublishBuildArtifacts: true
     continueOnError: ${{ parameters.continueOnError }}
-    
+
     ${{ if ne(parameters.displayName, '') }}:
       displayName: '${{ parameters.displayName }}'
     ${{ if eq(parameters.displayName, '') }}:
@@ -57,7 +57,7 @@ jobs:
     - HelixApiAccessToken: ''
     - HelixPreCommandStemWindows: 'set ORIGPYPATH=%PYTHONPATH%;py -m pip install -U pip;py -3 -m venv %HELIX_WORKITEM_PAYLOAD%\.venv;call %HELIX_WORKITEM_PAYLOAD%\.venv\Scripts\activate.bat;set PYTHONPATH=;py -3 -m pip install -U pip;py -3 -m pip install azure.storage.blob==12.0.0;py -3 -m pip install azure.storage.queue==12.0.0;set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"'
     - HelixPreCommandStemLinux: 'export ORIGPYPATH=$PYTHONPATH;python3 -m pip install -U pip;sudo apt-get -y install python3-venv;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install azure.storage.blob==12.0.0;pip3 install azure.storage.queue==12.0.0;sudo apt-get update;sudo apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates;curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash -;sudo apt-get -y install nodejs;sudo apt-get -y install npm;npm install --prefix $HELIX_WORKITEM_PAYLOAD jsvu -g;$HELIX_WORKITEM_PAYLOAD/bin/jsvu --os=linux64 --engines=v8,javascriptcore;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"'
-    - HelixPreCommandStemMsul: 'export ORIGPYPATH=$PYTHONPATH;sudo apk add icu-libs krb5-libs libgcc libintl libssl1.1 libstdc++ zlib;sudo apk add cargo --repository http://dl-cdn.alpinelinux.org/alpine/v3.12/community ;sudo apk add libgdiplus --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing; python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install azure.storage.blob==12.7.1;pip3 install azure.storage.queue==12.1.5;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"'
+    - HelixPreCommandStemMusl: 'export ORIGPYPATH=$PYTHONPATH;sudo apk add icu-libs krb5-libs libgcc libintl libssl1.1 libstdc++ zlib cargo;sudo apk add libgdiplus --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing; python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install azure.storage.blob==12.7.1;pip3 install azure.storage.queue==12.1.5;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"'
     - ExtraMSBuildLogsWindows: 'set MSBUILDDEBUGCOMM=1;set "MSBUILDDEBUGPATH=%HELIX_WORKITEM_UPLOAD_ROOT%"'
     - ExtraMSBuildLogsLinux: 'export MSBUILDDEBUGCOMM=1;export "MSBUILDDEBUGPATH=$HELIX_WORKITEM_UPLOAD_ROOT"'
     - HelixPreCommand: ''
@@ -70,7 +70,7 @@ jobs:
         - IsInternal: -Internal
       - ${{ if ne(parameters.osGroup, 'windows') }}:
         - ${{ if eq(parameters.osSubGroup, '_musl') }}:
-          - HelixPreCommand: $(HelixPreCommandStemMsul);$(ExtraMSBuildLogsLinux)
+          - HelixPreCommand: $(HelixPreCommandStemMusl);$(ExtraMSBuildLogsLinux)
           - HelixPostCommand: 'export PYTHONPATH=$ORIGPYPATH'
           - IsInternal: --internal
         - ${{ if ne(parameters.osSubGroup, '_musl') }}:
@@ -85,7 +85,7 @@ jobs:
         - HelixPreCommand: $(ExtraMSBuildLogsWindows)
       - ${{ if ne(parameters.osGroup, 'windows') }}:
           - HelixPreCommand: $(ExtraMSBuildLogsLinux);npm install --prefix $HELIX_WORKITEM_PAYLOAD jsvu -g;$HELIX_WORKITEM_PAYLOAD/bin/jsvu --os=linux64 --engines=v8,javascriptcore
-    
+
 
     - ${{ if and(eq(parameters.codeGenType, 'Interpreter'), eq(parameters.runtimeType, 'mono')) }}:
       - ${{ if eq( parameters.osGroup, 'windows') }}:

eng/pipelines/libraries/helix-queues-setup.yml

@@ -29,7 +29,8 @@ jobs:
     # Linux arm
     - ${{ if eq(parameters.platform, 'Linux_arm') }}:
       - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}:
-        - (Debian.9.Arm32.Open)[email protected]/dotnet-buildtools/prereqs:debian-9-helix-arm32v7-bfcd90a-20200121150037
+        - (Debian.10.Arm32.Open)[email protected]/dotnet-buildtools/prereqs:debian-10-helix-arm32v7-20210304164340-6616c63
+        - (Debian.11.Arm32.Open)[email protected]/dotnet-buildtools/prereqs:debian-11-helix-arm32v7-20210304164347-5a7c380
 
     # Linux arm64
     - ${{ if eq(parameters.platform, 'Linux_arm64') }}:
@@ -40,15 +41,14 @@ jobs:
 
     # Linux musl x64
     - ${{ if eq(parameters.platform, 'Linux_musl_x64') }}:
-      - ${{ if eq(parameters.jobParameters.isFullMatrix, false) }}:
-        - (Alpine.312.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.12-helix-20200602002622-e06dc59
+      - (Alpine.314.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64-20210910135833-1848e19
       - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}:
-        - (Alpine.312.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.12-helix-20200601195603-e06dc59
+        - (Alpine.313.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.13-helix-amd64-20210910135845-8a6f4f3
 
     # Linux musl arm64
-    - ${{ if eq(parameters.platform, 'Linux_musl_arm64') }}:
-      - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}:
-        - (Alpine.312.Arm64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.12-helix-arm64v8-20200602002604-25f8a3e
+    - ${{ if and(eq(parameters.platform, 'Linux_musl_arm64'), eq(parameters.jobParameters.isFullMatrix, true)) }}:
+      - (Alpine.313.Arm64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.13-helix-arm64v8-20210910135808-8a6f4f3
+      - (Alpine.314.Arm64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.14-helix-arm64v8-20210910135810-8a6f4f3
 
     # Linux x64
     - ${{ if eq(parameters.platform, 'Linux_x64') }}:
@@ -57,33 +57,34 @@ jobs:
           - (Centos.8.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:centos-8-helix-20201229003624-c1bf759
           - RedHat.7.Amd64.Open
           - SLES.15.Amd64.Open
-          - (Fedora.34.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:fedora-34-helix-20210728124700-4f64125
+          - (Fedora.34.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:fedora-34-helix-20210913123654-4f64125
           - (Ubuntu.1910.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:ubuntu-19.10-helix-amd64-cfcfd50-20191030180623
           - (Debian.10.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:debian-10-helix-amd64-bfcd90a-20200121150006
         - ${{ if or(ne(parameters.jobParameters.testScope, 'outerloop'), ne(parameters.jobParameters.runtimeFlavor, 'mono')) }}:
           - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}:
+            - (Centos.7.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:centos-7-mlnet-helix-20210714125435-dde38af
             - (Centos.8.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:centos-8-helix-20201229003624-c1bf759
             - RedHat.7.Amd64.Open
-            - Debian.9.Amd64.Open
-            - Ubuntu.1604.Amd64.Open
             - Ubuntu.1804.Amd64.Open
             - SLES.12.Amd64.Open
             - SLES.15.Amd64.Open
-            - (Fedora.34.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:fedora-34-helix-20210728124700-4f64125
+            - (Fedora.34.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:fedora-34-helix-20210913123654-4f64125
             - (Ubuntu.1910.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:ubuntu-19.10-helix-amd64-cfcfd50-20191030180623
-            - (Debian.10.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:debian-10-helix-amd64-bfcd90a-20200121150006
+            - (Debian.10.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:debian-10-helix-amd64-20210304164434-56c6673
+            - (Debian.11.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:debian-11-helix-amd64-20210304164428-5a7c380
             - (Mariner.1.0.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:cbl-mariner-1.0-helix-20210528192219-92bf620
           - ${{ if eq(parameters.jobParameters.isFullMatrix, false) }}:
-            - (Centos.8.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:centos-8-helix-20201229003624-c1bf759
+            - (Centos.7.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:centos-7-mlnet-helix-20210714125435-dde38af
             - RedHat.7.Amd64.Open
-            - (Debian.10.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:debian-10-helix-amd64-bfcd90a-20200121150006
-            - Ubuntu.1604.Amd64.Open
+            - (Debian.10.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:debian-10-helix-amd64-20210304164434-56c6673
             - Ubuntu.1804.Amd64.Open
-            - SLES.15.Amd64.Open
-            - (Fedora.34.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:fedora-34-helix-20210728124700-4f64125
       - ${{ if or(eq(parameters.jobParameters.interpreter, 'true'), eq(parameters.jobParameters.isSingleFile, true)) }}:
         # Limiting interp runs as we don't need as much coverage.
-        - Debian.9.Amd64.Open
+        - (Debian.10.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:debian-10-helix-amd64-20210304164434-56c6673
+
+    # Linux s390x
+    - ${{ if eq(parameters.platform, 'Linux_s390x') }}:
+        - Ubuntu.2004.S390X.Experimental.Open
 
     # OSX arm64
     - ${{ if eq(parameters.platform, 'OSX_arm64') }}:
@@ -92,11 +93,9 @@ jobs:
     # OSX x64
     - ${{ if eq(parameters.platform, 'OSX_x64') }}:
       - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}:
-        - OSX.1013.Amd64.Open
         - OSX.1014.Amd64.Open
         - OSX.1015.Amd64.Open
       - ${{ if eq(parameters.jobParameters.isFullMatrix, false) }}:
-        - OSX.1014.Amd64.Open
         - OSX.1015.Amd64.Open
 
     # Android
@@ -139,6 +138,7 @@ jobs:
           - ${{ if ne(parameters.jobParameters.isFullMatrix, true) }}:
             - Windows.81.Amd64.Open
             - Windows.10.Amd64.Server19H1.ES.Open
+            - Windows.11.Amd64.ClientPre.Open
             - ${{ if eq(parameters.jobParameters.testScope, 'outerloop') }}:
               - (Windows.Server.Core.1909.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:windowsservercore-2004-helix-amd64-20200904200251-272704c
             - ${{ if ne(parameters.jobParameters.runtimeFlavor, 'mono') }}: