[Enhancement] Parallel test jobs for CI (opensearch-project#2861)

* Split multiple tests into separate gradle tasks.
  * Tasks are configured in "splitTestConfig" map in build.gradle file.
    Map allows to use all patterns from TestFilter like:
    includeTestsMatching, excludeTestsMatching, includeTest etc.
  * Tasks are automatically generated from "splitTestConfig" map.
  * Two new Gradle tasks: listTasksAsJSON and listTasksAsParam to
    output task names to console. First one outputs them as a JSON
    and second - in gradlew "-x <TASK>" format to use in CLI.
  * Patterns included in tasks are automatically excluded from main
    "test" task but at the same time generated tasks are dependencies
    for "test". Running "gradlew test" will run whole suite at once.
* CI pipeline has been configured to accomodate all changes.
  * New 'master' task to generate list of jobs to run in parallel.
  * Updated matrix strategy to include task name to start.

Signed-off-by: Pawel Gudel <[email protected]>

.github/workflows/ci.yml

@@ -6,13 +6,34 @@ env:
   GRADLE_OPTS: -Dhttp.keepAlive=false
 
 jobs:
-  build:
-    name: build
+  generate-test-list:
+    runs-on: ubuntu-latest
+    outputs:
+      separateTestsNames: ${{ steps.set-matrix.outputs.separateTestsNames }}
+    steps:
+    - name: Set up JDK for build and test
+      uses: actions/setup-java@v2
+      with:
+        distribution: temurin # Temurin is a distribution of adoptium
+        java-version: 17
+
+    - name: Checkout security
+      uses: actions/checkout@v2
+
+    - name: Generate list of tasks
+      id: set-matrix
+      run: |
+        echo "separateTestsNames=$(./gradlew listTasksAsJSON -q --console=plain | tail -n 1)" >> $GITHUB_OUTPUT
+
+  test:
+    name: test
+    needs: generate-test-list
     strategy:
       fail-fast: false
       matrix:
+        gradle_task: ${{ fromJson(needs.generate-test-list.outputs.separateTestsNames) }}
+        platform: [windows-latest, ubuntu-latest]
         jdk: [11, 17]
-        platform: ["ubuntu-latest", "windows-latest"]
     runs-on: ${{ matrix.platform }}
 
     steps:
@@ -29,12 +50,8 @@ jobs:
       uses: gradle/gradle-build-action@v2
       with:
         arguments: |
-          build test -Dbuild.snapshot=false
-          -x integrationTest
-          -x spotlessCheck
-          -x checkstyleMain
-          -x checkstyleTest
-          -x spotbugsMain
+          ${{ matrix.gradle_task }} -Dbuild.snapshot=false
+          -x test
 
     - name: Coverage
       uses: codecov/codecov-action@v1
@@ -59,7 +76,7 @@ jobs:
       fail-fast: false
       matrix:
         jdk: [17]
-        platform: ["ubuntu-latest", "windows-latest"]
+        platform: [ubuntu-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
 
     steps:
@@ -78,18 +95,13 @@ jobs:
       with:
         arguments: |
           integrationTest -Dbuild.snapshot=false
-          -x spotlessCheck
-          -x checkstyleMain
-          -x checkstyleTest
-          -x spotbugsMain
 
   backward-compatibility:
-
     strategy:
       fail-fast: false
       matrix:
         jdk: [11, 17]
-        platform: ["ubuntu-latest", "windows-latest"]
+        platform: [ubuntu-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
 
     steps:

build.gradle

@@ -12,6 +12,7 @@
 
 import com.diffplug.gradle.spotless.JavaExtension
 import org.opensearch.gradle.test.RestIntegTestTask
+import groovy.json.JsonBuilder
 
 buildscript {
     ext {
@@ -105,12 +106,85 @@ tasks.whenTaskAdded {task ->
     }
 }
 
+def splitTestConfig = [
+    ciSecurityIntegrationTest: [
+        description: "Runs integration tests from all classes.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.*Integ*"
+            ],
+            excludeTestsMatching: [
+                "org.opensearch.security.sanity.tests.*"
+            ]
+        ]
+    ],
+    crossClusterTest: [
+        description: "Runs cross-cluster tests.",
+        filters: [
+            includeTestsMatching: [
+                    "org.opensearch.security.ccstest.*"
+            ]
+        ]
+    ],
+    dlicDlsflsTest: [
+        description: "Runs Document- and Field-Level Security tests.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.dlic.dlsfls.*"
+            ]
+        ]
+    ],
+    dlicRestApiTest: [
+        description: "Runs REST Management API tests.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.dlic.rest.*"
+            ]
+        ]
+    ],
+    indicesTest: [
+        description: "Runs indices tests from all classes.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.*indices*"
+            ],
+            excludeTestsMatching: [
+                "org.opensearch.security.sanity.tests.*"
+            ]
+        ]
+    ],
+    opensslCITest: [
+        description: "Runs portion of SSL tests related to OpenSSL. Explained in https://github.com/opensearch-project/security/pull/2301",
+        include: '**/OpenSSL*.class'
+    ],
+    sslTest: [
+        description: "Runs most of the SSL tests.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.ssl.*"
+            ],
+            excludeTestsMatching: [
+                "org.opensearch.security.ssl.OpenSSL*"
+            ]
+        ]
+    ]
+] as ConfigObject
+
+List<String> taskNames = splitTestConfig.keySet() as List
+
+task listTasksAsJSON {
+    // We are using `doLast` to explicitly specify when we
+    // want this action to be started. Without it the output
+    // is not shown at all or can be mixed with other outputs.
+    doLast {
+        System.out.println(new JsonBuilder(["citest"] + taskNames))
+    }
+}
 
 test {
     include '**/*.class'
     filter {
         excludeTestsMatching "org.opensearch.security.sanity.tests.*"
-        excludeTestsMatching "org.opensearch.security.ssl.OpenSSL*"
     }
     maxParallelForks = 8
     jvmArgs += "-Xmx3072m"
@@ -138,14 +212,37 @@ test {
     }
 }
 
-//add new task that runs OpenSSL tests
-task opensslTest(type: Test) {
-    include '**/OpenSSL*.class'
-    retry {
+task copyExtraTestResources(dependsOn: testClasses) {
+
+    copy {
+        from 'src/test/resources'
+        into 'build/testrun/test/src/test/resources'
+    }
+
+    taskNames.each { testName ->
+        copy {
+            from 'src/test/resources'
+            into "build/testrun/${testName}/src/test/resources"
+        }
+    }
+
+    copy {
+        from 'src/test/resources'
+        into 'build/testrun/citest/src/test/resources'
+    }
+}
+
+def setCommonTestConfig(Test task) {
+    task.maxParallelForks = 8
+    task.jvmArgs += "-Xmx3072m"
+    if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+        task.jvmArgs += "--add-opens=java.base/java.io=ALL-UNNAMED"
+    }
+    task.retry {
         failOnPassedAfterRetry = false
         maxRetries = 5
     }
-    jacoco {
+    task.jacoco {
         excludes = [
                 "com.sun.jndi.dns.*",
                 "com.sun.security.sasl.gsskerb.*",
@@ -160,21 +257,58 @@ task opensslTest(type: Test) {
                 "sun.util.resources.provider.*"
         ]
     }
+    task.dependsOn copyExtraTestResources
+    task.finalizedBy jacocoTestReport
 }
 
-task copyExtraTestResources(dependsOn: testClasses) {
-    copy {
-        from 'src/test/resources'
-        into 'build/testrun/test/src/test/resources'
+task citest(type: Test) {
+    group = "Github Actions tests"
+    description = "Runs the test suite on classes not covered by rest of the task in this group."
+    include '**/*.class'
+    filter {
+        excludeTestsMatching "org.opensearch.security.sanity.tests.*"
+        excludeTestsMatching "org.opensearch.security.ssl.OpenSSL*"
+        splitTestConfig.each { entry ->
+            entry.value.filters.each{ test ->
+                if (test.key == "includeTestsMatching") {
+                    test.value.each{
+                        excludeTestsMatching "${it}"
+                    }
+                } else if (test.key == "includeTest") {
+                    test.value.each{
+                        excludeTest "${it}"
+                    }
+                }
+            }
+        }
+    }
+    setCommonTestConfig(it)
+}
+
+splitTestConfig.each{ testName, testCfg ->
+    task "${testName}"(type: Test) {
+        group = testCfg.group ?: "Github Actions tests"
+        description = testCfg.description
+        include testCfg.include ?: '**/*.class'
+        filter {
+            testCfg.filters.each{ filter, values ->
+                values.each{ value ->
+                    "${filter}" "${value}"
+                }
+            }
+        }
+        setCommonTestConfig(it)
     }
 }
-tasks.test.dependsOn(copyExtraTestResources, opensslTest)
+
+tasks.test.dependsOn(copyExtraTestResources)
 
 jacoco {
     reportsDirectory = file("$buildDir/reports/jacoco")
 }
 
 jacocoTestReport {
+    getExecutionData().setFrom(fileTree(buildDir).include("/jacoco/*.exec"))
     reports {
         xml.required = true
     }