[Backport] Parallel test jobs for CI (#2861) (#2936)

* [Enhancement] Parallel test jobs for CI (#2861)

* Split multiple tests into separate gradle tasks.
  * Tasks are configured in "splitTestConfig" map in build.gradle file.
    Map allows to use all patterns from TestFilter like:
    includeTestsMatching, excludeTestsMatching, includeTest etc.
  * Tasks are automatically generated from "splitTestConfig" map.
  * Two new Gradle tasks: listTasksAsJSON and listTasksAsParam to
    output task names to console. First one outputs them as a JSON
    and second - in gradlew "-x <TASK>" format to use in CLI.
  * Patterns included in tasks are automatically excluded from main
    "test" task but at the same time generated tasks are dependencies
    for "test". Running "gradlew test" will run whole suite at once.
* CI pipeline has been configured to accomodate all changes.
  * New 'master' task to generate list of jobs to run in parallel.
  * Updated matrix strategy to include task name to start.

Signed-off-by: Pawel Gudel <[email protected]>
(cherry picked from commit e4f4817)
Signed-off-by: Pawel Gudel <[email protected]>

* Generalize Backwards Compatibility tests so we can test from any version to any version

With an issue reported indicating that there are serialization issue between 1.3 and 2.X, making sure that we can reproduce the errors. This new workflow(s) will make sure that we aren't breaking BWC with changes we are adding to 2.X and will give us the flexibility to test certain migration workflows.

Fixing an issue where the BWC tests were not actually building or executing causing the clusters to spin up and then immediately spin down. We will need to invest more energy into running multiple kinds of security plugin specific scenarios through the test system.

Signed-off-by: Peter Nied <[email protected]>

* Remove seperate integration test workflow

Signed-off-by: Peter Nied <[email protected]>

* Adjust BWC setting to point to correct BWC versions

Signed-off-by: Peter Nied <[email protected]>

* Fix spotless issues

Signed-off-by: Peter Nied <[email protected]>

---------

Signed-off-by: Pawel Gudel <[email protected]>
Signed-off-by: Peter Nied <[email protected]>
Co-authored-by: Peter Nied <[email protected]>

.github/actions/create-bwc-build/action.yaml

@@ -0,0 +1,43 @@
+name: 'Create a backwards compatible ready build'
+description: 'Checkouts the official version of a the Security plugin and builds it so it can be used for BWC tests'
+
+inputs:
+  plugin-branch:
+    description: 'The branch of the plugin that should be built, e.g "2.2", "1.x"'
+    required: true
+
+outputs:
+  built-version:
+    description: 'The version of OpenSearch that was associated with this branch'
+    value: ${{ steps.get-opensearch-version.outputs.version }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Enable Longpaths if on Windows
+      if: ${{ runner.os == 'Windows' }}
+      run: git config --system core.longpaths true
+      shell: pwsh
+
+    - uses: actions/checkout@v3
+      with:
+        repository: opensearch-project/security
+        ref: ${{ inputs.plugin-branch }}
+        path: ${{ inputs.plugin-branch }}
+
+    - name: Build
+      uses: gradle/gradle-build-action@v2
+      with:
+        arguments: assemble -Dbuild.snapshot=false
+        build-root-directory: ${{ inputs.plugin-branch }}
+
+    - id: get-opensearch-version
+      uses: peternied/get-opensearch-version@v1
+      with:
+        working-directory: ${{ inputs.plugin-branch }}
+
+    - name: Copy current distro into the expected folder
+      run: |
+        mkdir -p ./bwc-test/src/test/resources/${{ steps.get-opensearch-version.outputs.version }}
+        cp ${{ inputs.plugin-branch }}/build/distributions/opensearch-security-${{ steps.get-opensearch-version.outputs.version }}.zip ./bwc-test/src/test/resources/${{ steps.get-opensearch-version.outputs.version }}
+      shell: bash

.github/actions/run-bwc-suite/action.yaml

@@ -0,0 +1,46 @@
+name: 'Runs the backward bompatiblity test suite'
+description: 'Tests backwards compability between a previous and next version of this plugin'
+
+inputs:
+  plugin-previous-branch:
+    description: 'The branch of the plugin that should be built for the previous version, e.g "2.2", "1.x"'
+    required: true
+
+  plugin-next-branch:
+    description: 'The branch of the plugin that should be built for the next version, e.g "2.3", "main"'
+    required: true
+
+  report-artifact-name:
+    description: 'The name of the artifacts for this run, e.g. "BWC-2.1-to-2.4-results"'
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+
+    - id: build-previous
+      uses: ./.github/actions/create-bwc-build
+      with:
+        plugin-branch: ${{ inputs.plugin-previous-branch }}
+
+    - id: build-next
+      uses: ./.github/actions/create-bwc-build
+      with:
+        plugin-branch: ${{ inputs.plugin-next-branch }}
+
+    - name: Run BWC tests
+      uses: gradle/gradle-build-action@v2
+      with:
+        arguments: |
+          bwcTestSuite
+          -Dtests.security.manager=false
+          -Dbwc.version.previous=${{ steps.build-previous.outputs.built-version }}
+          -Dbwc.version.next=${{ steps.build-next.outputs.built-version }} -i
+        build-root-directory: bwc-test
+
+    - uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: ${{ inputs.report-artifact-name }}
+        path: |
+          ./bwc-test/build/reports/

.github/workflows/bwc-tests.yml

@@ -0,0 +1,42 @@
+name: Backwards Compability Suite
+
+on: [workflow_dispatch]
+
+jobs:
+  last-supported-major-to-current:
+    name: Make sure that the last supported major version can move to the most current version
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/setup-java@v1
+      with:
+        java-version: 11
+
+    - name: Checkout Security Repo
+      uses: actions/checkout@v2
+
+    - id: build-previous
+      uses: ./.github/actions/run-bwc-suite
+      with:
+        plugin-previous-branch: "1.3"
+        plugin-next-branch: "2.x"
+        report-artifact-name: BWC-Last-Supported-Major
+
+  current-to-next-unreleased-major:
+    name: Make sure that the current version is compatible with the next major version
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/setup-java@v1
+      with:
+        java-version: 11
+
+    - name: Checkout Security Repo
+      uses: actions/checkout@v2
+
+    - id: build-previous
+      uses: ./.github/actions/run-bwc-suite
+      with:
+        plugin-previous-branch: "2.x"
+        plugin-next-branch: "main"
+        report-artifact-name: BWC-Next-Major

.github/workflows/ci.yml

@@ -6,13 +6,34 @@ env:
   GRADLE_OPTS: -Dhttp.keepAlive=false
 
 jobs:
-  build:
-    name: build
+  generate-test-list:
+    runs-on: ubuntu-latest
+    outputs:
+      separateTestsNames: ${{ steps.set-matrix.outputs.separateTestsNames }}
+    steps:
+    - name: Set up JDK for build and test
+      uses: actions/setup-java@v2
+      with:
+        distribution: temurin # Temurin is a distribution of adoptium
+        java-version: 17
+
+    - name: Checkout security
+      uses: actions/checkout@v2
+
+    - name: Generate list of tasks
+      id: set-matrix
+      run: |
+        echo "separateTestsNames=$(./gradlew listTasksAsJSON -q --console=plain | tail -n 1)" >> $GITHUB_OUTPUT
+
+  test:
+    name: test
+    needs: generate-test-list
     strategy:
       fail-fast: false
       matrix:
+        gradle_task: ${{ fromJson(needs.generate-test-list.outputs.separateTestsNames) }}
+        platform: [windows-latest, ubuntu-latest]
         jdk: [11, 17]
-        platform: ["ubuntu-latest", "windows-latest", "macos-latest"]
     runs-on: ${{ matrix.platform }}
 
     steps:
@@ -29,11 +50,8 @@ jobs:
       uses: gradle/gradle-build-action@v2
       with:
         arguments: |
-          build test -Dbuild.snapshot=false
-          -x spotlessCheck
-          -x checkstyleMain
-          -x checkstyleTest
-          -x spotbugsMain
+          ${{ matrix.gradle_task }} -Dbuild.snapshot=false
+          -x test
 
     - name: Coverage
       uses: codecov/codecov-action@v1
@@ -53,25 +71,27 @@ jobs:
       run: echo "Check the artifact ${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports for detailed test results"
 
   backward-compatibility:
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [11, 17]
+        platform: [ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.platform }}
+
     steps:
-    - uses: actions/checkout@v2
     - uses: actions/setup-java@v1
       with:
-        java-version: 11
-    - run: ./gradlew clean build -Dbuild.snapshot=false -x test
-    - run: |
-        echo "Running backwards compatibility tests ..."
-        security_plugin_version_no_snapshot=$(./gradlew properties -q | grep -E '^version:' | awk '{print $2}' | sed 's/-SNAPSHOT//g')
-        bwc_version=2.8.0
-        cp -r build/ ./bwc-test/
-        mkdir ./bwc-test/src/test/resources/security_plugin_version_no_snapshot
-        cp build/distributions/opensearch-security-${security_plugin_version_no_snapshot}.zip ./bwc-test/src/test/resources/${security_plugin_version_no_snapshot}
-        mkdir bwc-test/src/test/resources/${bwc_version}.0
-        wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${bwc_version}/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-security-${bwc_version}.0.zip
-        mv opensearch-security-${bwc_version}.0.zip bwc-test/src/test/resources/${bwc_version}.0/
-        cd bwc-test/
-        ./gradlew bwcTestSuite -Dtests.security.manager=false
+        java-version: ${{ matrix.jdk }}
+
+    - name: Checkout Security Repo
+      uses: actions/checkout@v2
+
+    - id: build-previous
+      uses: ./.github/actions/run-bwc-suite
+      with:
+        plugin-previous-branch: "2.8"
+        plugin-next-branch: "2.x"
+        report-artifact-name: bwc-${{ matrix.platform }}-jdk${{ matrix.jdk }}
 
   code-ql:
     runs-on: ubuntu-latest