Add GCStress testing for libraries tests on checked CoreCLR

eng/pipelines/coreclr/libraries-gcstress-extra.yml

@@ -0,0 +1,60 @@
+trigger: none
+
+pr: none
+
+# This pipeline currently has too many failures to be enabled by schedule.
+# schedules:
+# - cron: "0 10 * * 0"
+#   displayName: Sun at 2:00 AM (UTC-8:00)
+#   branches:
+#     include:
+#     - master
+#   always: true
+
+jobs:
+#
+# Checkout repository
+#
+- template: /eng/pipelines/common/checkout-job.yml
+
+#
+# Build CoreCLR checked and libraries Release
+#
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
+    buildConfig: checked
+    platformGroup: gcstress
+
+#
+# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms.
+#
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/libraries/build-test-job.yml
+    buildConfig: Release
+    platforms:
+    - Linux_x64
+    - Windows_NT_x64
+    jobParameters:
+      liveRuntimeBuildConfig: checked
+      testScope: innerloop
+
+#
+# Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes
+#
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/libraries/run-test-job.yml
+    buildConfig: Release
+    platformGroup: gcstress
+    helixQueueGroup: libraries
+    helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+    jobParameters:
+      # Default timeout is 150 minutes (2.5 hours), which is not enough for stress.
+      timeoutInMinutes: 600
+      testScope: innerloop
+      liveRuntimeBuildConfig: checked
+      dependsOnTestBuildConfiguration: Release
+      dependsOnTestArchitecture: x64
+      coreclrTestGroup: gcstress-extra

eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml

@@ -0,0 +1,60 @@
+trigger: none
+
+pr: none
+
+# This pipeline currently has too many failures to be enabled by schedule.
+# schedules:
+# - cron: "0 10 * * 6"
+#   displayName: Sat at 2:00 AM (UTC-8:00)
+#   branches:
+#     include:
+#     - master
+#   always: true
+
+jobs:
+#
+# Checkout repository
+#
+- template: /eng/pipelines/common/checkout-job.yml
+
+#
+# Build CoreCLR checked and libraries Release
+#
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
+    buildConfig: checked
+    platformGroup: gcstress
+
+#
+# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms.
+#
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/libraries/build-test-job.yml
+    buildConfig: Release
+    platforms:
+    - Linux_x64
+    - Windows_NT_x64
+    jobParameters:
+      liveRuntimeBuildConfig: checked
+      testScope: innerloop
+
+#
+# Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes
+#
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/libraries/run-test-job.yml
+    buildConfig: Release
+    platformGroup: gcstress
+    helixQueueGroup: libraries
+    helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+    jobParameters:
+      # Default timeout is 150 minutes (2.5 hours), which is not enough for stress.
+      timeoutInMinutes: 600
+      testScope: innerloop
+      liveRuntimeBuildConfig: checked
+      dependsOnTestBuildConfiguration: Release
+      dependsOnTestArchitecture: x64
+      coreclrTestGroup: gcstress0x3-gcstress0xc

eng/pipelines/libraries/run-test-job.yml

@@ -77,9 +77,24 @@ jobs:
           parameters:
             displayName: Test Assets
             cleanUnpackFolder: false
-            unpackFolder: $(Build.SourcesDirectory)/artifacts
             artifactName: $(librariesTestsArtifactName)
             artifactFileName: $(librariesTestsArtifactName)$(archiveExtension)
+            unpackFolder: $(Build.SourcesDirectory)/artifacts
+
+        - ${{ if in(parameters.coreclrTestGroup, 'gcstress0x3-gcstress0xc', 'gcstress-extra') }}:
+          # We need to find and download the GC stress dependencies (namely, coredistools). Put them
+          # in the 'sharedFramework' directory where we unpacked the CoreCLR build artifacts. The 'sharedFramework'
+          # directory is what is copied into the testhost.
+          - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
+            - script: $(Build.SourcesDirectory)\src\coreclr\tests\setup-stress-dependencies.cmd
+                      /arch ${{ parameters.archType }}
+                      /outputdir $(_runtimeDownloadPath)/sharedFramework
+              displayName: Download GC stress dependencies
+          - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
+            - script: $(Build.SourcesDirectory)/src/coreclr/tests/setup-stress-dependencies.sh
+                      --arch=${{ parameters.archType }}
+                      --outputDir=$(_runtimeDownloadPath)/sharedFramework
+              displayName: Download GC stress dependencies
 
         - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
           - script: $(_buildScript)
@@ -106,7 +121,7 @@ jobs:
             # eng/pipelines/common/templates/runtimes/run-test-job.yml (with 'testGroup' replaced by 'coreclrTestGroup'
             # for clarity), and should remain in sync. This is only a subset; only the testGroups that are
             # used to test the libraries have been added here. More could be added if we decided to test the
-            # libraries with more stress modes (e.g., GCStress). The scenario tags are interpreted by
+            # libraries with more stress modes. The scenario tags are interpreted by
             # src\coreclr\tests\testenvironment.proj.
             #
             # The one difference here compared to eng/pipelines/common/templates/runtimes/run-test-job.yml is
@@ -144,3 +159,18 @@ jobs:
               - jitstress2_jitstressregs0x10
               - jitstress2_jitstressregs0x80
               - jitstress2_jitstressregs0x1000
+            ${{ if in(parameters.coreclrTestGroup, 'gcstress0x3-gcstress0xc') }}:
+              scenarios:
+              # Disable gcstress0x3 for now; it causes lots of test timeouts. Investigate this after
+              # gcstress0xc runs are clean. Tracking issue: https://github.com/dotnet/runtime/issues/38903.
+              # - gcstress0x3
+              - gcstress0xc
+            ${{ if in(parameters.coreclrTestGroup, 'gcstress-extra') }}:
+              scenarios:
+              - heapverify1
+              - gcstress0xc_zapdisable
+              - gcstress0xc_zapdisable_jitstress2
+              - gcstress0xc_zapdisable_heapverify1
+              - gcstress0xc_jitstress1
+              - gcstress0xc_jitstress2
+              - gcstress0xc_jitminopts_heapverify1

src/coreclr/tests/setup-stress-dependencies.sh

@@ -16,7 +16,7 @@ function print_usage {
     echo ''
     echo 'Command line:'
     echo ''
-    echo './setup-gcstress.sh --arch=<TargetArch> --outputDir=<coredistools_lib_install_path>'
+    echo './setup-stress-dependencies.sh --arch=<TargetArch> --outputDir=<coredistools_lib_install_path>'
     echo ''
     echo 'Required arguments:'
     echo '  --arch=<TargetArch>        : Target arch for the build'

src/libraries/sendtohelixhelp.proj

@@ -13,12 +13,23 @@
     <HelixBuild Condition="'$(HelixBuild)' == ''">default</HelixBuild>
 
     <!-- Normal jobs have a 30 minute (1800 sec.) timeout for arm/arm64, and 15 minute (900 sec.) timeout otherwise.
-         Stress modes can take considerably longer, so double those numbers.
+         Stress modes can take considerably longer, so double those numbers. And GCStress is even slower.
     -->
-    <TimeoutInSeconds Condition="'$(Scenario)' == '' and ('$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'arm')">1800</TimeoutInSeconds>
-    <TimeoutInSeconds Condition="'$(Scenario)' != '' and ('$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'arm')">3600</TimeoutInSeconds>
+    <TimeoutInSeconds Condition="
+        '$(Scenario)' == 'gcstress0x3' or
+        '$(Scenario)' == 'gcstress0xc' or
+        '$(Scenario)' == 'heapverify1' or
+        '$(Scenario)' == 'gcstress0xc_zapdisable' or
+        '$(Scenario)' == 'gcstress0xc_zapdisable_jitstress2' or
+        '$(Scenario)' == 'gcstress0xc_zapdisable_heapverify1' or
+        '$(Scenario)' == 'gcstress0xc_jitstress1' or
+        '$(Scenario)' == 'gcstress0xc_jitstress2' or
+        '$(Scenario)' == 'gcstress0xc_jitminopts_heapverify1'">5400</TimeoutInSeconds>
+    <TimeoutInSeconds Condition="'$(Scenario)' == '' and '$(TimeoutInSeconds)' == '' and ('$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'arm')">1800</TimeoutInSeconds>
+    <TimeoutInSeconds Condition="'$(Scenario)' != '' and '$(TimeoutInSeconds)' == '' and ('$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'arm')">3600</TimeoutInSeconds>
     <TimeoutInSeconds Condition="'$(Scenario)' == '' and '$(TimeoutInSeconds)' == ''">900</TimeoutInSeconds>
     <TimeoutInSeconds Condition="'$(Scenario)' != '' and '$(TimeoutInSeconds)' == ''">1800</TimeoutInSeconds>
+
     <_timeoutSpan>$([System.TimeSpan]::FromSeconds($(TimeoutInSeconds)))</_timeoutSpan>
 
     <!-- We need to enable xunit reporter so that it parses test results