From e4f4817e0d2d9caf80da0d0cb1da73e4913f62e3 Mon Sep 17 00:00:00 2001 From: pawel-gudel-eliatra <136344230+pawel-gudel-eliatra@users.noreply.github.com> Date: Wed, 28 Jun 2023 22:41:46 +0200 Subject: [PATCH] [Enhancement] Parallel test jobs for CI (#2861) * Split multiple tests into separate gradle tasks. * Tasks are configured in "splitTestConfig" map in build.gradle file. Map allows to use all patterns from TestFilter like: includeTestsMatching, excludeTestsMatching, includeTest etc. * Tasks are automatically generated from "splitTestConfig" map. * Two new Gradle tasks: listTasksAsJSON and listTasksAsParam to output task names to console. First one outputs them as a JSON and second - in gradlew "-x " format to use in CLI. * Patterns included in tasks are automatically excluded from main "test" task but at the same time generated tasks are dependencies for "test". Running "gradlew test" will run whole suite at once. * CI pipeline has been configured to accomodate all changes. * New 'master' task to generate list of jobs to run in parallel. * Updated matrix strategy to include task name to start. Signed-off-by: Pawel Gudel --- .github/workflows/ci.yml | 44 +++++++---- build.gradle | 156 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 173 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6465c5d4e..0a6cd5b141 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,13 +6,34 @@ env: GRADLE_OPTS: -Dhttp.keepAlive=false jobs: - build: - name: build + generate-test-list: + runs-on: ubuntu-latest + outputs: + separateTestsNames: ${{ steps.set-matrix.outputs.separateTestsNames }} + steps: + - name: Set up JDK for build and test + uses: actions/setup-java@v2 + with: + distribution: temurin # Temurin is a distribution of adoptium + java-version: 17 + + - name: Checkout security + uses: actions/checkout@v2 + + - name: Generate list of tasks + id: set-matrix + run: | + echo "separateTestsNames=$(./gradlew listTasksAsJSON -q --console=plain | tail -n 1)" >> $GITHUB_OUTPUT + + test: + name: test + needs: generate-test-list strategy: fail-fast: false matrix: + gradle_task: ${{ fromJson(needs.generate-test-list.outputs.separateTestsNames) }} + platform: [windows-latest, ubuntu-latest] jdk: [11, 17] - platform: ["ubuntu-latest", "windows-latest"] runs-on: ${{ matrix.platform }} steps: @@ -29,12 +50,8 @@ jobs: uses: gradle/gradle-build-action@v2 with: arguments: | - build test -Dbuild.snapshot=false - -x integrationTest - -x spotlessCheck - -x checkstyleMain - -x checkstyleTest - -x spotbugsMain + ${{ matrix.gradle_task }} -Dbuild.snapshot=false + -x test - name: Coverage uses: codecov/codecov-action@v1 @@ -59,7 +76,7 @@ jobs: fail-fast: false matrix: jdk: [17] - platform: ["ubuntu-latest", "windows-latest"] + platform: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.platform }} steps: @@ -78,18 +95,13 @@ jobs: with: arguments: | integrationTest -Dbuild.snapshot=false - -x spotlessCheck - -x checkstyleMain - -x checkstyleTest - -x spotbugsMain backward-compatibility: - strategy: fail-fast: false matrix: jdk: [11, 17] - platform: ["ubuntu-latest", "windows-latest"] + platform: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.platform }} steps: diff --git a/build.gradle b/build.gradle index 9f2971db49..708d84f6d8 100644 --- a/build.gradle +++ b/build.gradle @@ -12,6 +12,7 @@ import com.diffplug.gradle.spotless.JavaExtension import org.opensearch.gradle.test.RestIntegTestTask +import groovy.json.JsonBuilder buildscript { ext { @@ -105,12 +106,85 @@ tasks.whenTaskAdded {task -> } } +def splitTestConfig = [ + ciSecurityIntegrationTest: [ + description: "Runs integration tests from all classes.", + filters: [ + includeTestsMatching: [ + "org.opensearch.security.*Integ*" + ], + excludeTestsMatching: [ + "org.opensearch.security.sanity.tests.*" + ] + ] + ], + crossClusterTest: [ + description: "Runs cross-cluster tests.", + filters: [ + includeTestsMatching: [ + "org.opensearch.security.ccstest.*" + ] + ] + ], + dlicDlsflsTest: [ + description: "Runs Document- and Field-Level Security tests.", + filters: [ + includeTestsMatching: [ + "org.opensearch.security.dlic.dlsfls.*" + ] + ] + ], + dlicRestApiTest: [ + description: "Runs REST Management API tests.", + filters: [ + includeTestsMatching: [ + "org.opensearch.security.dlic.rest.*" + ] + ] + ], + indicesTest: [ + description: "Runs indices tests from all classes.", + filters: [ + includeTestsMatching: [ + "org.opensearch.security.*indices*" + ], + excludeTestsMatching: [ + "org.opensearch.security.sanity.tests.*" + ] + ] + ], + opensslCITest: [ + description: "Runs portion of SSL tests related to OpenSSL. Explained in https://github.com/opensearch-project/security/pull/2301", + include: '**/OpenSSL*.class' + ], + sslTest: [ + description: "Runs most of the SSL tests.", + filters: [ + includeTestsMatching: [ + "org.opensearch.security.ssl.*" + ], + excludeTestsMatching: [ + "org.opensearch.security.ssl.OpenSSL*" + ] + ] + ] +] as ConfigObject + +List taskNames = splitTestConfig.keySet() as List + +task listTasksAsJSON { + // We are using `doLast` to explicitly specify when we + // want this action to be started. Without it the output + // is not shown at all or can be mixed with other outputs. + doLast { + System.out.println(new JsonBuilder(["citest"] + taskNames)) + } +} test { include '**/*.class' filter { excludeTestsMatching "org.opensearch.security.sanity.tests.*" - excludeTestsMatching "org.opensearch.security.ssl.OpenSSL*" } maxParallelForks = 8 jvmArgs += "-Xmx3072m" @@ -138,14 +212,37 @@ test { } } -//add new task that runs OpenSSL tests -task opensslTest(type: Test) { - include '**/OpenSSL*.class' - retry { +task copyExtraTestResources(dependsOn: testClasses) { + + copy { + from 'src/test/resources' + into 'build/testrun/test/src/test/resources' + } + + taskNames.each { testName -> + copy { + from 'src/test/resources' + into "build/testrun/${testName}/src/test/resources" + } + } + + copy { + from 'src/test/resources' + into 'build/testrun/citest/src/test/resources' + } +} + +def setCommonTestConfig(Test task) { + task.maxParallelForks = 8 + task.jvmArgs += "-Xmx3072m" + if (JavaVersion.current() > JavaVersion.VERSION_1_8) { + task.jvmArgs += "--add-opens=java.base/java.io=ALL-UNNAMED" + } + task.retry { failOnPassedAfterRetry = false maxRetries = 5 } - jacoco { + task.jacoco { excludes = [ "com.sun.jndi.dns.*", "com.sun.security.sasl.gsskerb.*", @@ -160,21 +257,58 @@ task opensslTest(type: Test) { "sun.util.resources.provider.*" ] } + task.dependsOn copyExtraTestResources + task.finalizedBy jacocoTestReport } -task copyExtraTestResources(dependsOn: testClasses) { - copy { - from 'src/test/resources' - into 'build/testrun/test/src/test/resources' +task citest(type: Test) { + group = "Github Actions tests" + description = "Runs the test suite on classes not covered by rest of the task in this group." + include '**/*.class' + filter { + excludeTestsMatching "org.opensearch.security.sanity.tests.*" + excludeTestsMatching "org.opensearch.security.ssl.OpenSSL*" + splitTestConfig.each { entry -> + entry.value.filters.each{ test -> + if (test.key == "includeTestsMatching") { + test.value.each{ + excludeTestsMatching "${it}" + } + } else if (test.key == "includeTest") { + test.value.each{ + excludeTest "${it}" + } + } + } + } + } + setCommonTestConfig(it) +} + +splitTestConfig.each{ testName, testCfg -> + task "${testName}"(type: Test) { + group = testCfg.group ?: "Github Actions tests" + description = testCfg.description + include testCfg.include ?: '**/*.class' + filter { + testCfg.filters.each{ filter, values -> + values.each{ value -> + "${filter}" "${value}" + } + } + } + setCommonTestConfig(it) } } -tasks.test.dependsOn(copyExtraTestResources, opensslTest) + +tasks.test.dependsOn(copyExtraTestResources) jacoco { reportsDirectory = file("$buildDir/reports/jacoco") } jacocoTestReport { + getExecutionData().setFrom(fileTree(buildDir).include("/jacoco/*.exec")) reports { xml.required = true }