Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Generalize Backwards Compatibility tests so we can test from any version to any version #2253

Merged
merged 4 commits into from
Nov 18, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions .github/actions/create-bwc-build/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
name: 'Create a backwards compatible ready build'
description: 'Checkouts the official version of a the Security plugin and builds it so it can be used for BWC tests'

inputs:
plugin-branch:
description: 'The branch of the plugin that should be built, e.g "2.2", "1.x"'
required: true
peternied marked this conversation as resolved.
Show resolved Hide resolved

outputs:
built-version:
description: 'The version of OpenSearch that was associated with this branch'
value: ${{ steps.get-opensearch-version.outputs.version }}

runs:
using: "composite"
steps:
- name: Enable Longpaths if on Windows
if: ${{ runner.os == 'Windows' }}
run: git config --system core.longpaths true
shell: pwsh

- uses: actions/checkout@v3
with:
repository: opensearch-project/security
ref: ${{ inputs.plugin-branch }}
peternied marked this conversation as resolved.
Show resolved Hide resolved
path: ${{ inputs.plugin-branch }}

- name: Build
uses: gradle/gradle-build-action@v2
with:
arguments: assemble -Dbuild.snapshot=false
build-root-directory: ${{ inputs.plugin-branch }}

- id: get-opensearch-version
uses: peternied/get-opensearch-version@v1
with:
working-directory: ${{ inputs.plugin-branch }}

- name: Copy current distro into the expected folder
run: |
mkdir -p ./bwc-test/src/test/resources/${{ steps.get-opensearch-version.outputs.version }}
cp ${{ inputs.plugin-branch }}/build/distributions/opensearch-security-${{ steps.get-opensearch-version.outputs.version }}.zip ./bwc-test/src/test/resources/${{ steps.get-opensearch-version.outputs.version }}
shell: bash
46 changes: 46 additions & 0 deletions .github/actions/run-bwc-suite/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: 'Runs the backward bompatiblity test suite'
description: 'Tests backwards compability between a previous and next version of this plugin'

inputs:
plugin-previous-branch:
description: 'The branch of the plugin that should be built for the previous version, e.g "2.2", "1.x"'
required: true

plugin-next-branch:
description: 'The branch of the plugin that should be built for the next version, e.g "2.3", "main"'
peternied marked this conversation as resolved.
Show resolved Hide resolved
required: true

report-artifact-name:
description: 'The name of the artifacts for this run, e.g. "BWC-2.1-to-2.4-results"'
required: true

runs:
using: "composite"
steps:

- id: build-previous
uses: ./.github/actions/create-bwc-build
with:
plugin-branch: ${{ inputs.plugin-previous-branch }}

- id: build-next
uses: ./.github/actions/create-bwc-build
with:
plugin-branch: ${{ inputs.plugin-next-branch }}

- name: Run BWC tests
uses: gradle/gradle-build-action@v2
with:
arguments: |
bwcTestSuite
-Dtests.security.manager=false
-Dbwc.version.previous=${{ steps.build-previous.outputs.built-version }}
-Dbwc.version.next=${{ steps.build-next.outputs.built-version }} -i
build-root-directory: bwc-test

- uses: actions/upload-artifact@v3
if: always()
peternied marked this conversation as resolved.
Show resolved Hide resolved
with:
name: ${{ inputs.report-artifact-name }}
path: |
./bwc-test/build/reports/
42 changes: 42 additions & 0 deletions .github/workflows/bwc-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
name: Backwards Compability Suite

on: [workflow_dispatch]

jobs:
last-supported-major-to-current:
name: Make sure that the last supported major version can move to the most current version
runs-on: ubuntu-latest

steps:
- uses: actions/setup-java@v1
with:
java-version: 11
peternied marked this conversation as resolved.
Show resolved Hide resolved

- name: Checkout Security Repo
uses: actions/checkout@v2

- id: build-previous
uses: ./.github/actions/run-bwc-suite
with:
plugin-previous-branch: "1.3"
plugin-next-branch: "2.x"
report-artifact-name: BWC-Last-Supported-Major

current-to-next-unreleased-major:
name: Make sure that the current version is compatible with the next major version
runs-on: ubuntu-latest

steps:
- uses: actions/setup-java@v1
with:
java-version: 11

- name: Checkout Security Repo
uses: actions/checkout@v2

- id: build-previous
uses: ./.github/actions/run-bwc-suite
with:
plugin-previous-branch: "2.x"
plugin-next-branch: "main"
report-artifact-name: BWC-Next-Major
34 changes: 19 additions & 15 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,24 +84,28 @@ jobs:
-x spotbugsMain

backward-compatibility:
runs-on: ubuntu-latest

strategy:
fail-fast: false
matrix:
jdk: [11, 17]
platform: ["ubuntu-latest", "windows-latest"]
runs-on: ${{ matrix.platform }}

steps:
- uses: actions/checkout@v2
- uses: actions/setup-java@v1
with:
java-version: 11
- run: ./gradlew clean build -Dbuild.snapshot=false -x test -x integrationTest
- run: |
echo "Running backwards compatibility tests ..."
security_plugin_version_no_snapshot=$(./gradlew properties -q | grep -E '^version:' | awk '{print $2}' | sed 's/-SNAPSHOT//g')
cp -r build/ ./bwc-test/
mkdir ./bwc-test/src/test/resources/security_plugin_version_no_snapshot
cp build/distributions/opensearch-security-${security_plugin_version_no_snapshot}.zip ./bwc-test/src/test/resources/${security_plugin_version_no_snapshot}
mkdir bwc-test/src/test/resources/2.4.0.0
wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.4.0/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-security-2.4.0.0.zip
mv opensearch-security-2.4.0.0.zip bwc-test/src/test/resources/2.4.0.0/
cd bwc-test/
./gradlew bwcTestSuite -Dtests.security.manager=false
java-version: ${{ matrix.jdk }}

- name: Checkout Security Repo
uses: actions/checkout@v2

- id: build-previous
uses: ./.github/actions/run-bwc-suite
with:
plugin-previous-branch: "2.x"
plugin-next-branch: "main"
report-artifact-name: bwc-${{ matrix.platform }}-jdk${{ matrix.jdk }}

code-ql:
runs-on: ubuntu-latest
Expand Down
85 changes: 27 additions & 58 deletions bwc-test/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,7 @@ apply plugin: 'java'

apply plugin: 'opensearch.testclusters'

compileTestJava.enabled = false

ext {
projectSubstitutions = [:]
licenseFile = rootProject.file('LICENSE.TXT')
noticeFile = rootProject.file('NOTICE')
}
Expand Down Expand Up @@ -70,19 +67,41 @@ repositories {
}

dependencies {
testImplementation 'com.google.guava:guava:30.0-jre'
testImplementation "org.opensearch.test:framework:${opensearch_version}"
testImplementation 'org.apache.logging.log4j:log4j-core:2.17.1'
}

String bwcVersion = "2.4.0.0";
loggerUsageCheck.enabled = false
testingConventions.enabled = false
validateNebulaPom.enabled = false

String previousVersion = System.getProperty("bwc.version.previous", "2.4.0.0")
String nextVersion = System.getProperty("bwc.version.next", "3.0.0.0")
peternied marked this conversation as resolved.
Show resolved Hide resolved

String bwcVersion = previousVersion
String baseName = "securityBwcCluster"
String bwcFilePath = "src/test/resources/"
String projectVersion = "3.0.0.0"
String projectVersion = nextVersion

String previousOpenSearch = extractVersion(previousVersion);
String nextOpenSearch = extractVersion(nextVersion);

println previousOpenSearch + nextOpenSearch;


// Extracts the OpenSearch version from a plugin version string, 2.4.0.0 -> 2.4.0.
def String extractVersion(versionStr) {
def versionMatcher = versionStr =~ /(.+?)(\.\d+)$/
versionMatcher.find()
return versionMatcher.group(1)
}

2.times {i ->
testClusters {
"${baseName}$i" {
testDistribution = "ARCHIVE"
versions = ["2.4.0","3.0.0"]
versions = [previousOpenSearch, nextOpenSearch]
numberOfNodes = 3
plugin(provider(new Callable<RegularFile>() {
@Override
Expand Down Expand Up @@ -148,24 +167,7 @@ List<Provider<RegularFile>> plugins = [
// Creates a test cluster with 3 nodes of the old version.
2.times {i ->
task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
exclude '**/*Test*'
exclude '**/*Sanity*'
useCluster testClusters."${baseName}$i"
if (System.getProperty("mixedCluster") != null) {
peternied marked this conversation as resolved.
Show resolved Hide resolved
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAMixedCluster")
}
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
if (System.getProperty("fullRestartCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAnUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'old_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'old'
systemProperty 'tests.plugin_bwc_version', bwcVersion
Expand All @@ -178,23 +180,11 @@ List<Provider<RegularFile>> plugins = [
// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
// This is also used as a one third upgraded cluster for a rolling upgrade.
task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
exclude '**/*Test*'
exclude '**/*Sanity*'
dependsOn "${baseName}#oldVersionClusterTask0"
useCluster testClusters."${baseName}0"
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
if (System.getProperty("mixedCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAMixedCluster")
}
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'first'
systemProperty 'tests.plugin_bwc_version', bwcVersion
Expand All @@ -206,18 +196,11 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
// This is used for rolling upgrade.
task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
exclude '**/*Test*'
exclude '**/*Sanity*'
dependsOn "${baseName}#mixedClusterTask"
useCluster testClusters."${baseName}0"
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'second'
systemProperty 'tests.plugin_bwc_version', bwcVersion
Expand All @@ -229,18 +212,11 @@ task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTas
// This results in a fully upgraded cluster.
// This is used for rolling upgrade.
task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
exclude '**/*Test*'
exclude '**/*Sanity*'
dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
useCluster testClusters."${baseName}0"
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'third'
systemProperty 'tests.plugin_bwc_version', bwcVersion
Expand All @@ -251,28 +227,21 @@ task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask)
// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
// at the same time resulting in a fully upgraded cluster.
tasks.register("${baseName}#fullRestartClusterTask", StandaloneRestIntegTestTask) {
exclude '**/*Test*'
exclude '**/*Sanity*'
dependsOn "${baseName}#oldVersionClusterTask1"
useCluster testClusters."${baseName}1"
doFirst {
testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
}
if (System.getProperty("fullRestartCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAnUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
systemProperty 'tests.plugin_bwc_version', bwcVersion
systemProperty 'tests.rest.bwcsuite_round', 'first'
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
}

// A bwc test suite which runs all the bwc tasks combined.
task bwcTestSuite(type: StandaloneRestIntegTestTask) {
exclude '**/*Test*'
exclude '**/*Sanity*'
exclude '**/**' // Do not run any tests as part of this aggregate task
dependsOn tasks.named("${baseName}#mixedClusterTask")
dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
dependsOn tasks.named("${baseName}#fullRestartClusterTask")
Expand Down
Loading