Split tests into multiple tasks

* Split multiple tests into separate gradle tasks. * Tasks are configured in "splitTestConfig" map in build.gradle file. Map allows to use all patterns from TestFilter like: includeTestsMatching, excludeTestsMatching, includeTest etc. * Tasks are automatically generated from "splitTestConfig" map. * Two new Gradle tasks: listTasksAsJSON and listTasksAsParam to output task names to console. First one outputs them as a JSON and second - in gradlew "-x <TASK>" format to use in CLI. * Patterns included in tasks are automatically excluded from main "test" task but at the same time generated tasks are dependencies for "test". Running "gradlew test" will run whole suite at once. * CI pipeline has been configured to accomodate all changes. * New 'master' task to generate list of jobs to run in parallel. * Updated matrix strategy to include task name to start. Signed-off-by: Pawel Gudel <[email protected]>
opensearch-project · Jun 14, 2023 · 59e1cbd · 59e1cbd
1 parent 0bc7158
commit 59e1cbd
Show file tree

Hide file tree

Showing 2 changed files with 177 additions and 3 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -6,8 +6,30 @@ env:
   GRADLE_OPTS: -Dhttp.keepAlive=false
 
 jobs:
+  generate-test-list:
+    runs-on: ubuntu-latest
+    outputs:
+      separateTestsNames: ${{ steps.set-matrix.outputs.separateTestsNames }}
+      testsNamesExclude: ${{ steps.set-matrix.outputs.testsNamesExclude }}
+    steps:
+    - name: Set up JDK for build and test
+      uses: actions/setup-java@v2
+      with:
+        distribution: temurin # Temurin is a distribution of adoptium
+        java-version: 17
+
+    - name: Checkout security
+      uses: actions/checkout@v2
+
+    - name: Generate list of tasks
+      id: set-matrix
+      run: |
+        echo "separateTestsNames=$(./gradlew listTasksAsJSON -q --console=plain | tail -n 1)" >> $GITHUB_OUTPUT
+        echo "testsNamesExclude=$(./gradlew listTasksAsParam -q --console=plain | tail -n 1)" >> $GITHUB_OUTPUT
+
   build:
     name: build
+    needs: generate-test-list
     strategy:
       fail-fast: false
       matrix:
@@ -29,12 +51,14 @@ jobs:
       uses: gradle/gradle-build-action@v2
       with:
         arguments: |
-          build test -Dbuild.snapshot=false
+          build test ${{ needs.generate-test-list.outputs.testsNamesExclude }} -Dbuild.snapshot=false
           -x integrationTest
           -x spotlessCheck
           -x checkstyleMain
           -x checkstyleTest
           -x spotbugsMain
+          -x spotbugsIntegrationTest
+          -x checkstyleIntegrationTest
 
     - name: Coverage
       uses: codecov/codecov-action@v1
@@ -53,6 +77,52 @@ jobs:
       if: always()
       run: echo "Check the artifact ${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports for detailed test results"
 
+  build-test:
+    name: build-test
+    needs: generate-test-list
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [11, 17]
+        platform: ["ubuntu-latest", "windows-latest"]
+        gradle_task: ${{ fromJson(needs.generate-test-list.outputs.separateTestsNames) }}
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+    - name: Set up JDK for build and test
+      uses: actions/setup-java@v2
+      with:
+        distribution: temurin # Temurin is a distribution of adoptium
+        java-version: ${{ matrix.jdk }}
+
+    - name: Checkout security
+      uses: actions/checkout@v2
+
+    - name: Build and Test
+      uses: gradle/gradle-build-action@v2
+      with:
+        arguments: |
+          build ${{ matrix.gradle_task }} -Dbuild.snapshot=false
+          -x integrationTest
+          -x spotlessCheck
+          -x checkstyleMain
+          -x checkstyleTest
+          -x spotbugsMain
+          -x spotbugsIntegrationTest
+          -x checkstyleIntegrationTest
+          -x test
+
+    - uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: ${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports
+        path: |
+          ./build/reports/
+
+    - name: check archive for debugging
+      if: always()
+      run: echo "Check the artifact ${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports for detailed test results"
+
   integration-tests:
     name: integration-tests
     strategy:
@@ -84,7 +154,6 @@ jobs:
           -x spotbugsMain
 
   backward-compatibility:
-
     strategy:
       fail-fast: false
       matrix:

diff --git a/build.gradle b/build.gradle
@@ -12,6 +12,7 @@
 
 import com.diffplug.gradle.spotless.JavaExtension
 import org.opensearch.gradle.test.RestIntegTestTask
+import groovy.json.JsonBuilder
 
 buildscript {
     ext {
@@ -152,12 +153,79 @@ tasks.whenTaskAdded {task ->
     }
 }
 
+def splitTestConfig = [
+    dlicDlsflsTest: [
+        includeTestsMatching: [
+            "org.opensearch.security.dlic.dlsfls.*"
+        ]
+    ],
+    dlicRestApiTest: [
+        includeTestsMatching: [
+            "org.opensearch.security.dlic.rest.*"
+        ]
+    ],
+    crossClusterTest: [
+        includeTestsMatching: [
+            "org.opensearch.security.ccstest.*"
+        ]
+    ],
+    sslTest: [
+        includeTestsMatching: [
+            "org.opensearch.security.ssl.*"
+        ]
+    ],
+    securityIntegrationTest: [
+        includeTestsMatching: [
+            "org.opensearch.security.*Integ*"
+        ],
+        excludeTestsMatching: [
+            "org.opensearch.security.sanity.tests.*",
+            "org.opensearch.security.ssl.OpenSSL*",
+        ]
+    ],
+    indicesTest: [
+        includeTestsMatching: [
+            "org.opensearch.security.*indices*"
+        ],
+        excludeTestsMatching: [
+            "org.opensearch.security.sanity.tests.*",
+            "org.opensearch.security.ssl.OpenSSL*",
+        ]
+    ]
+] as ConfigObject
+
+List<String> taskNames = splitTestConfig.keySet() as List
+
+task listTasksAsJSON {
+    doLast {
+        System.out.println(new JsonBuilder(taskNames))
+    }
+}
+
+task listTasksAsParam {
+    doLast {
+        System.out.println('-x ' + (taskNames).join(' -x '))
+    }
+}
 
 test {
     include '**/*.class'
     filter {
         excludeTestsMatching "org.opensearch.security.sanity.tests.*"
         excludeTestsMatching "org.opensearch.security.ssl.OpenSSL*"
+        splitTestConfig.each { entry ->
+            entry.value.each{ test ->
+                if (test.key == "includeTestsMatching") {
+                    test.value.each{ 
+                        excludeTestsMatching "${it}"
+                    }
+                } else if (test.key == "includeTest") {
+                    test.value.each{
+                        excludeTest "${it}"
+                    }
+                }
+            }
+        }
     }
     maxParallelForks = 8
     jvmArgs += "-Xmx3072m"
@@ -209,13 +277,50 @@ task opensslTest(type: Test) {
     }
 }
 
+splitTestConfig.each{ testName, filterCfg ->
+    task "${testName}"(type: Test) {
+        include '**/*.class'
+        filter {
+            filterCfg.each{ filter, values ->
+                values.each{ value ->
+                    "${filter}" "${value}"
+                }
+            }
+        }
+        maxParallelForks = 8
+        jvmArgs += "-Xmx3072m"
+        if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+            jvmArgs += "--add-opens=java.base/java.io=ALL-UNNAMED"
+        }
+        retry {
+            failOnPassedAfterRetry = false
+            maxRetries = 5
+        }
+        jacoco {
+            excludes = [
+                "com.sun.jndi.dns.*",
+                "com.sun.security.sasl.gsskerb.*",
+                "java.sql.*",
+                "javax.script.*",
+                "org.jcp.xml.dsig.internal.dom.*",
+                "sun.nio.cs.ext.*",
+                "sun.security.ec.*",
+                "sun.security.jgss.*",
+                "sun.security.pkcs11.*",
+                "sun.security.smartcardio.*",
+                "sun.util.resources.provider.*"
+            ]
+        }
+    }
+}
+
 task copyExtraTestResources(dependsOn: testClasses) {
     copy {
         from 'src/test/resources'
         into 'build/testrun/test/src/test/resources'
     }
 }
-tasks.test.dependsOn(copyExtraTestResources, opensslTest)
+tasks.test.dependsOn(copyExtraTestResources, opensslTest, taskNames)
 
 jacoco {
     reportsDirectory = file("$buildDir/reports/jacoco")