[Enhancement] Parallel test jobs for CI

.github/workflows/ci.yml

@@ -6,13 +6,86 @@ env:
   GRADLE_OPTS: -Dhttp.keepAlive=false
 
 jobs:
+  generate-test-list:
+    runs-on: ubuntu-latest
+    outputs:
+      separateTestsNames: ${{ steps.set-matrix.outputs.separateTestsNames }}
+      testsNamesExclude: ${{ steps.set-matrix.outputs.testsNamesExclude }}
+    steps:
+    - name: Set up JDK for build and test
+      uses: actions/setup-java@v2
+      with:
+        distribution: temurin # Temurin is a distribution of adoptium
+        java-version: 17
+
+    - name: Checkout security
+      uses: actions/checkout@v2
+
+    - name: Generate list of tasks
+      id: set-matrix
+      run: |
+        echo "separateTestsNames=$(./gradlew listTasksAsJSON -q --console=plain | tail -n 1)" >> $GITHUB_OUTPUT
+
   build:
     name: build
+    needs: generate-test-list
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [11, 17]
+        platform: ["ubuntu-latest", "windows-latest"]
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+    - name: Set up JDK for build and test
+      uses: actions/setup-java@v2
+      with:
+        distribution: temurin # Temurin is a distribution of adoptium
+        java-version: ${{ matrix.jdk }}
+
+    - name: Checkout security
+      uses: actions/checkout@v2
+
+    - name: Build and Test
+      uses: gradle/gradle-build-action@v2
+      with:
+        arguments: |
+          build citest -Dbuild.snapshot=false
+          -x integrationTest
+          -x spotlessCheck
+          -x checkstyleMain
+          -x checkstyleTest
+          -x spotbugsMain
+          -x spotbugsIntegrationTest
+          -x checkstyleIntegrationTest
+          -x test
+
+    - name: Coverage
+      uses: codecov/codecov-action@v1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ./build/reports/jacoco/test/jacocoTestReport.xml
+
+    - uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: ${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports
+        path: |
+          ./build/reports/
+
+    - name: check archive for debugging
+      if: always()
+      run: echo "Check the artifact ${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports for detailed test results"
+
+  build-test:
+    name: build-test
+    needs: generate-test-list
     strategy:
       fail-fast: false
       matrix:
         jdk: [11, 17]
         platform: ["ubuntu-latest", "windows-latest"]
+        gradle_task: ${{ fromJson(needs.generate-test-list.outputs.separateTestsNames) }}
     runs-on: ${{ matrix.platform }}
 
     steps:
@@ -29,12 +102,15 @@ jobs:
       uses: gradle/gradle-build-action@v2
       with:
         arguments: |
-          build test -Dbuild.snapshot=false
+          build ${{ matrix.gradle_task }} -Dbuild.snapshot=false
           -x integrationTest
           -x spotlessCheck
           -x checkstyleMain
           -x checkstyleTest
           -x spotbugsMain
+          -x spotbugsIntegrationTest
+          -x checkstyleIntegrationTest
+          -x test
 
     - name: Coverage
       uses: codecov/codecov-action@v1
@@ -84,7 +160,6 @@ jobs:
           -x spotbugsMain
 
   backward-compatibility:
-
     strategy:
       fail-fast: false
       matrix:

build.gradle

@@ -12,6 +12,7 @@
 
 import com.diffplug.gradle.spotless.JavaExtension
 import org.opensearch.gradle.test.RestIntegTestTask
+import groovy.json.JsonBuilder
 
 buildscript {
     ext {
@@ -105,6 +106,83 @@ tasks.whenTaskAdded {task ->
     }
 }
 
+def splitTestConfig = [
+    dlicDlsflsTest: [
+        description: "Runs Document- and Field-Level Security tests.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.dlic.dlsfls.*"
+            ]
+        ]
+    ],
+    dlicRestApiTest: [
+        description: "Runs REST Management API tests.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.dlic.rest.*"
+            ]
+        ]
+    ],
+    crossClusterTest: [
+        description: "Runs cross-cluster tests.",
+        filters: [
+            includeTestsMatching: [
+                    "org.opensearch.security.ccstest.*"
+            ]
+        ]
+    ],
+    sslTest: [
+        description: "Runs most of the SSL tests.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.ssl.*"
+            ],
+            excludeTestsMatching: [
+                "org.opensearch.security.ssl.OpenSSL*"
+            ]
+        ]
+    ],
+    opensslCiTest: [
+        description: "Runs portion of SSL tests related to OpenSSL. Explained in https://github.com/opensearch-project/security/pull/2301",
+        include: '**/OpenSSL*.class'
+    ],
+    securityIntegrationTest: [
+        description: "Runs integration tests from all classes.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.*Integ*"
+            ],
+            excludeTestsMatching: [
+                "org.opensearch.security.sanity.tests.*"
+            ]
+        ]
+    ],
+    indicesTest: [
+        description: "Runs indices tests from all classes.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.*indices*"
+            ],
+            excludeTestsMatching: [
+                "org.opensearch.security.sanity.tests.*"
+            ]
+        ]
+    ]
+] as ConfigObject
+
+List<String> taskNames = splitTestConfig.keySet() as List
+
+task listTasksAsJSON {
+    doLast {
+        System.out.println(new JsonBuilder(taskNames))
+    }
+}
+
+task listTasksAsParam {
+    doLast {
+        System.out.println('-x ' + (taskNames).join(' -x '))
+    }
+}
 
 test {
     include '**/*.class'
@@ -163,18 +241,123 @@ task opensslTest(type: Test) {
 }
 
 task copyExtraTestResources(dependsOn: testClasses) {
+
     copy {
         from 'src/test/resources'
         into 'build/testrun/test/src/test/resources'
     }
+
+    taskNames.each { testName ->
+        copy {
+            from 'src/test/resources'
+            into "build/testrun/${testName}/src/test/resources"
+        }
+    }
+
+    copy {
+        from 'src/test/resources'
+        into 'build/testrun/citest/src/test/resources'
+    }
+}
+
+task citest(type: Test) {
+    group = "Github Actions tests"
+    description = "Runs the test suite on classes not covered by rest of the task in this group."
+    include '**/*.class'
+    filter {
+        excludeTestsMatching "org.opensearch.security.sanity.tests.*"
+        excludeTestsMatching "org.opensearch.security.ssl.OpenSSL*"
+        splitTestConfig.each { entry ->
+            entry.value.each{ test ->
+                if (test.key == "includeTestsMatching") {
+                    test.value.each{
+                        excludeTestsMatching "${it}"
+                    }
+                } else if (test.key == "includeTest") {
+                    test.value.each{
+                        excludeTest "${it}"
+                    }
+                }
+            }
+        }
+    }
+    maxParallelForks = 8
+    jvmArgs += "-Xmx3072m"
+    if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+        jvmArgs += "--add-opens=java.base/java.io=ALL-UNNAMED"
+    }
+    retry {
+        failOnPassedAfterRetry = false
+        maxRetries = 5
+    }
+    jacoco {
+        excludes = [
+                "com.sun.jndi.dns.*",
+                "com.sun.security.sasl.gsskerb.*",
+                "java.sql.*",
+                "javax.script.*",
+                "org.jcp.xml.dsig.internal.dom.*",
+                "sun.nio.cs.ext.*",
+                "sun.security.ec.*",
+                "sun.security.jgss.*",
+                "sun.security.pkcs11.*",
+                "sun.security.smartcardio.*",
+                "sun.util.resources.provider.*"
+        ]
+    }
+    dependsOn copyExtraTestResources
+    finalizedBy jacocoTestReport
+}
+
+splitTestConfig.each{ testName, testCfg ->
+    task "${testName}"(type: Test) {
+        group = testCfg.group ?: "Github Actions tests"
+        description = testCfg.description
+        include testCfg.include ?: '**/*.class'
+        filter {
+            testCfg.filters.each{ filter, values ->
+                values.each{ value ->
+                    "${filter}" "${value}"
+                }
+            }
+        }
+        maxParallelForks = 8
+        jvmArgs += "-Xmx3072m"
+        if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+            jvmArgs += "--add-opens=java.base/java.io=ALL-UNNAMED"
+        }
+        retry {
+            failOnPassedAfterRetry = false
+            maxRetries = 5
+        }
+        jacoco {
+            excludes = [
+                "com.sun.jndi.dns.*",
+                "com.sun.security.sasl.gsskerb.*",
+                "java.sql.*",
+                "javax.script.*",
+                "org.jcp.xml.dsig.internal.dom.*",
+                "sun.nio.cs.ext.*",
+                "sun.security.ec.*",
+                "sun.security.jgss.*",
+                "sun.security.pkcs11.*",
+                "sun.security.smartcardio.*",
+                "sun.util.resources.provider.*"
+            ]
+        }
+        dependsOn copyExtraTestResources
+        finalizedBy jacocoTestReport
+    }
 }
+
 tasks.test.dependsOn(copyExtraTestResources, opensslTest)
 
 jacoco {
     reportsDirectory = file("$buildDir/reports/jacoco")
 }
 
 jacocoTestReport {
+    getExecutionData().setFrom(fileTree(buildDir).include("/jacoco/*.exec"))
     reports {
         xml.required = true
     }