[Backport] Parallel test jobs for CI (#2861)

.github/workflows/ci.yml

@@ -6,13 +6,34 @@ env:
   GRADLE_OPTS: -Dhttp.keepAlive=false
 
 jobs:
-  build:
-    name: build
+  generate-test-list:
+    runs-on: ubuntu-latest
+    outputs:
+      separateTestsNames: ${{ steps.set-matrix.outputs.separateTestsNames }}
+    steps:
+    - name: Set up JDK for build and test
+      uses: actions/setup-java@v2
+      with:
+        distribution: temurin # Temurin is a distribution of adoptium
+        java-version: 17
+
+    - name: Checkout security
+      uses: actions/checkout@v2
+
+    - name: Generate list of tasks
+      id: set-matrix
+      run: |
+        echo "separateTestsNames=$(./gradlew listTasksAsJSON -q --console=plain | tail -n 1)" >> $GITHUB_OUTPUT
+
+  test:
+    name: test
+    needs: generate-test-list
     strategy:
       fail-fast: false
       matrix:
+        gradle_task: ${{ fromJson(needs.generate-test-list.outputs.separateTestsNames) }}
+        platform: [windows-latest, ubuntu-latest]
         jdk: [11, 17]
-        platform: ["ubuntu-latest", "windows-latest", "macos-latest"]
     runs-on: ${{ matrix.platform }}
 
     steps:
@@ -29,11 +50,8 @@ jobs:
       uses: gradle/gradle-build-action@v2
       with:
         arguments: |
-          build test -Dbuild.snapshot=false
-          -x spotlessCheck
-          -x checkstyleMain
-          -x checkstyleTest
-          -x spotbugsMain
+          ${{ matrix.gradle_task }} -Dbuild.snapshot=false
+          -x test
 
     - name: Coverage
       uses: codecov/codecov-action@v1
@@ -52,8 +70,40 @@ jobs:
       if: always()
       run: echo "Check the artifact ${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports for detailed test results"
 
+  integration-tests:
+    name: integration-tests
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [17]
+        platform: [ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+    - name: Set up JDK for build and test
+      uses: actions/setup-java@v2
+      with:
+        distribution: temurin # Temurin is a distribution of adoptium
+        java-version: ${{ matrix.jdk }}
+
+    - name: Checkout security
+      uses: actions/checkout@v2
+
+    - name: Build and Test
+      uses: gradle/gradle-build-action@v2
+      continue-on-error: true # Until retries are enable do not fail the workflow https://github.com/opensearch-project/security/issues/2184
+      with:
+        arguments: |
+          integrationTest -Dbuild.snapshot=false
+
   backward-compatibility:
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [11, 17]
+        platform: [ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.platform }}
+
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-java@v1

build.gradle

@@ -10,6 +10,7 @@
  */
 
 import org.opensearch.gradle.test.RestIntegTestTask
+import groovy.json.JsonBuilder
 
 buildscript {
     ext {
@@ -96,12 +97,91 @@ forbiddenPatterns.enabled = false
 testingConventions.enabled = false
 // Conflicts between runtime kafka-clients:x.y.z & testRuntime kafka-clients:x.y.z:test
 jarHell.enabled = false
+tasks.whenTaskAdded {task ->
+    if(task.name.contains("forbiddenApisIntegrationTest")) {
+        task.enabled = false
+    }
+}
+
+def splitTestConfig = [
+    ciSecurityIntegrationTest: [
+        description: "Runs integration tests from all classes.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.*Integ*"
+            ],
+            excludeTestsMatching: [
+                "org.opensearch.security.sanity.tests.*"
+            ]
+        ]
+    ],
+    crossClusterTest: [
+        description: "Runs cross-cluster tests.",
+        filters: [
+            includeTestsMatching: [
+                    "org.opensearch.security.ccstest.*"
+            ]
+        ]
+    ],
+    dlicDlsflsTest: [
+        description: "Runs Document- and Field-Level Security tests.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.dlic.dlsfls.*"
+            ]
+        ]
+    ],
+    dlicRestApiTest: [
+        description: "Runs REST Management API tests.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.dlic.rest.*"
+            ]
+        ]
+    ],
+    indicesTest: [
+        description: "Runs indices tests from all classes.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.*indices*"
+            ],
+            excludeTestsMatching: [
+                "org.opensearch.security.sanity.tests.*"
+            ]
+        ]
+    ],
+    opensslCITest: [
+        description: "Runs portion of SSL tests related to OpenSSL. Explained in https://github.com/opensearch-project/security/pull/2301",
+        include: '**/OpenSSL*.class'
+    ],
+    sslTest: [
+        description: "Runs most of the SSL tests.",
+        filters: [
+            includeTestsMatching: [
+                "org.opensearch.security.ssl.*"
+            ],
+            excludeTestsMatching: [
+                "org.opensearch.security.ssl.OpenSSL*"
+            ]
+        ]
+    ]
+] as ConfigObject
+
+List<String> taskNames = splitTestConfig.keySet() as List
+
+task listTasksAsJSON {
+    // We are using `doLast` to explicitly specify when we
+    // want this action to be started. Without it the output
+    // is not shown at all or can be mixed with other outputs.
+    doLast {
+        System.out.println(new JsonBuilder(["citest"] + taskNames))
+    }
+}
 
 test {
     include '**/*.class'
     filter {
         excludeTestsMatching "org.opensearch.security.sanity.tests.*"
-        excludeTestsMatching "org.opensearch.security.ssl.OpenSSL*"
     }
     maxParallelForks = 8
     jvmArgs += "-Xmx3072m"
@@ -129,14 +209,37 @@ test {
     }
 }
 
-//add new task that runs OpenSSL tests
-task opensslTest(type: Test) {
-    include '**/OpenSSL*.class'
-    retry {
+task copyExtraTestResources(dependsOn: testClasses) {
+
+    copy {
+        from 'src/test/resources'
+        into 'build/testrun/test/src/test/resources'
+    }
+
+    taskNames.each { testName ->
+        copy {
+            from 'src/test/resources'
+            into "build/testrun/${testName}/src/test/resources"
+        }
+    }
+
+    copy {
+        from 'src/test/resources'
+        into 'build/testrun/citest/src/test/resources'
+    }
+}
+
+def setCommonTestConfig(Test task) {
+    task.maxParallelForks = 8
+    task.jvmArgs += "-Xmx3072m"
+    if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+        task.jvmArgs += "--add-opens=java.base/java.io=ALL-UNNAMED"
+    }
+    task.retry {
         failOnPassedAfterRetry = false
         maxRetries = 5
     }
-    jacoco {
+    task.jacoco {
         excludes = [
             "com.sun.jndi.dns.*",
             "com.sun.security.sasl.gsskerb.*",
@@ -151,21 +254,58 @@ task opensslTest(type: Test) {
             "sun.util.resources.provider.*"
         ]
     }
+    task.dependsOn copyExtraTestResources
+    task.finalizedBy jacocoTestReport
 }
 
-task copyExtraTestResources(dependsOn: testClasses) {
-    copy {
-        from 'src/test/resources'
-        into 'build/testrun/test/src/test/resources'
+task citest(type: Test) {
+    group = "Github Actions tests"
+    description = "Runs the test suite on classes not covered by rest of the task in this group."
+    include '**/*.class'
+    filter {
+        excludeTestsMatching "org.opensearch.security.sanity.tests.*"
+        excludeTestsMatching "org.opensearch.security.ssl.OpenSSL*"
+        splitTestConfig.each { entry ->
+            entry.value.filters.each{ test ->
+                if (test.key == "includeTestsMatching") {
+                    test.value.each{
+                        excludeTestsMatching "${it}"
+                    }
+                } else if (test.key == "includeTest") {
+                    test.value.each{
+                        excludeTest "${it}"
+                    }
+                }
+            }
+        }
+    }
+    setCommonTestConfig(it)
+}
+
+splitTestConfig.each{ testName, testCfg ->
+    task "${testName}"(type: Test) {
+        group = testCfg.group ?: "Github Actions tests"
+        description = testCfg.description
+        include testCfg.include ?: '**/*.class'
+        filter {
+            testCfg.filters.each{ filter, values ->
+                values.each{ value ->
+                    "${filter}" "${value}"
+                }
+            }
+        }
+        setCommonTestConfig(it)
     }
 }
-tasks.test.dependsOn(copyExtraTestResources, opensslTest)
+
+tasks.test.dependsOn(copyExtraTestResources)
 
 jacoco {
     reportsDirectory = file("$buildDir/reports/jacoco")
 }
 
 jacocoTestReport {
+    getExecutionData().setFrom(fileTree(buildDir).include("/jacoco/*.exec"))
     reports {
         xml.required = true
     }