[Backport 2.x] Integration tests framework

.github/workflows/ci.yml

@@ -1,6 +1,12 @@
 name: CI
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - main
+      - 1.*
+      - 2.*
+  pull_request:
 
 env:
   GRADLE_OPTS: -Dhttp.keepAlive=false
@@ -18,7 +24,7 @@ jobs:
         java-version: 17
 
     - name: Checkout security
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Generate list of tasks
       id: set-matrix
@@ -44,15 +50,14 @@ jobs:
         java-version: ${{ matrix.jdk }}
 
     - name: Checkout security
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Build and Test
       uses: gradle/gradle-build-action@v2
       with:
         cache-disabled: true
         arguments: |
           ${{ matrix.gradle_task }} -Dbuild.snapshot=false
-          -x test
 
     - name: Coverage
       uses: Wandalen/[email protected]
@@ -76,6 +81,32 @@ jobs:
       if: always()
       run: echo "Check the artifact ${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports for detailed test results"
 
+  integration-tests:
+    name: integration-tests
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [11, 17]
+        platform: [ubuntu-latest] # Removing windows temporarily
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+    - name: Set up JDK for build and test
+      uses: actions/setup-java@v3
+      with:
+        distribution: temurin # Temurin is a distribution of adoptium
+        java-version: ${{ matrix.jdk }}
+
+    - name: Checkout security
+      uses: actions/checkout@v4
+
+    - name: Build and Test
+      uses: gradle/gradle-build-action@v2
+      with:
+        cache-disabled: true
+        arguments: |
+          integrationTest -Dbuild.snapshot=false
+
   backward-compatibility:
     strategy:
       fail-fast: false
@@ -91,7 +122,7 @@ jobs:
         java-version: ${{ matrix.jdk }}
 
     - name: Checkout Security Repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - id: build-previous
       uses: ./.github/actions/run-bwc-suite
@@ -105,21 +136,21 @@ jobs:
   code-ql:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-java@v3
       with:
         distribution: temurin # Temurin is a distribution of adoptium
         java-version: 11
-    - uses: github/codeql-action/init@v1
+    - uses: github/codeql-action/init@v2
       with:
         languages: java
-    - run: ./gradlew clean build -Dbuild.snapshot=false -x test
-    - uses: github/codeql-action/analyze@v1
+    - run: ./gradlew clean assemble -Dbuild.snapshot=false
+    - uses: github/codeql-action/analyze@v2
 
   build-artifact-names:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-java@v3
       with:

.github/workflows/code-hygiene.yml

@@ -43,7 +43,7 @@ jobs:
       - uses: gradle/gradle-build-action@v2
         with:
           cache-disabled: true
-          arguments: checkstyleMain checkstyleTest
+          arguments: checkstyleMain checkstyleTest checkstyleIntegrationTest
 
   spotbugs:
     runs-on: ubuntu-latest

build.gradle

@@ -434,6 +434,14 @@ configurations {
             force "com.github.luben:zstd-jni:${versions.zstd}"
             force "org.xerial.snappy:snappy-java:1.1.10.3"
             force "com.google.guava:guava:${guava_version}"
+
+            // TODO: Seems like this should be removable
+            force "org.apache.httpcomponents:httpclient-cache:4.5.13"
+            force "org.apache.httpcomponents:httpclient:4.5.13"
+            force "org.apache.httpcomponents:fluent-hc:4.5.13"
+            force "org.apache.httpcomponents:httpcore:4.4.16"
+            force "org.apache.httpcomponents:httpcore-nio:4.4.16"
+            force "org.apache.httpcomponents:httpasyncclient:4.1.5"
         }
     }
 
@@ -449,6 +457,7 @@ sourceSets {
             srcDir file ('src/integrationTest/java')
             compileClasspath += sourceSets.main.output
             runtimeClasspath += sourceSets.main.output
+
         }
         resources {
             srcDir file('src/integrationTest/resources')
@@ -466,6 +475,7 @@ task integrationTest(type: Test) {
     systemProperty "java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"
     testClassesDirs = sourceSets.integrationTest.output.classesDirs
     classpath = sourceSets.integrationTest.runtimeClasspath
+    jvmArgs += "-Djdk.internal.httpclient.disableHostnameVerification"
     retry {
         failOnPassedAfterRetry = false
         maxRetries = 2
@@ -639,6 +649,11 @@ dependencies {
         exclude(group: 'org.hamcrest', module: 'hamcrest')
     }
     integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14'
+    integrationTestImplementation "org.apache.httpcomponents:httpclient-cache:4.5.13"
+    integrationTestImplementation "org.apache.httpcomponents:httpclient:4.5.13"
+    integrationTestImplementation "org.apache.httpcomponents:fluent-hc:4.5.13"
+    integrationTestImplementation "org.apache.httpcomponents:httpcore:4.4.13"
+    integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5"
 
     //Checkstyle
     checkstyle 'com.puppycrawl.tools:checkstyle:10.12.1'

src/integrationTest/java/org/opensearch/common/logging/NodeAndClusterIdConverter.java

@@ -0,0 +1,29 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*
+*/
+package org.opensearch.common.logging;
+
+/**
+* Class uses to override OpenSearch NodeAndClusterIdConverter Log4j2 plugin in order to disable plugin and limit number of
+* warn messages like "...ApplierService#updateTask][T#1] WARN ClusterApplierService:628 - failed to notify ClusterStateListener..."
+* during tests execution.
+*
+* The class is rather a temporary solution and the real one should be developed in scope of:
+* https://github.com/opensearch-project/OpenSearch/pull/4322
+*/
+import org.apache.logging.log4j.core.LogEvent;
+
+class NodeAndClusterIdConverter {
+
+    public NodeAndClusterIdConverter() {}
+
+    public static void setNodeIdAndClusterId(String nodeId, String clusterUUID) {}
+
+    public void format(LogEvent event, StringBuilder toAppendTo) {}
+}

src/integrationTest/java/org/opensearch/node/PluginAwareNode.java

@@ -0,0 +1,52 @@
+/*
+* Copyright 2015-2018 _floragunn_ GmbH
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/*
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*
+* Modifications Copyright OpenSearch Contributors. See
+* GitHub history for details.
+*/
+
+package org.opensearch.node;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import org.opensearch.common.settings.Settings;
+import org.opensearch.plugins.Plugin;
+
+public class PluginAwareNode extends Node {
+
+    private final boolean clusterManagerEligible;
+
+    @SafeVarargs
+    public PluginAwareNode(boolean clusterManagerEligible, final Settings preparedSettings, final Class<? extends Plugin>... plugins) {
+        super(
+            InternalSettingsPreparer.prepareEnvironment(preparedSettings, Collections.emptyMap(), null, () -> System.getenv("HOSTNAME")),
+            Arrays.asList(plugins),
+            true
+        );
+        this.clusterManagerEligible = clusterManagerEligible;
+    }
+
+    public boolean isClusterManagerEligible() {
+        return clusterManagerEligible;
+    }
+}

src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java

@@ -0,0 +1,60 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*
+*/
+package org.opensearch.security;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Objects;
+
+class ConfigurationFiles {
+
+    public static void createRoleMappingFile(File destination) {
+        String resource = "roles_mapping.yml";
+        copyResourceToFile(resource, destination);
+    }
+
+    public static Path createConfigurationDirectory() {
+        try {
+            Path tempDirectory = Files.createTempDirectory("test-security-config");
+            String[] configurationFiles = {
+                "config.yml",
+                "action_groups.yml",
+                "config.yml",
+                "internal_users.yml",
+                "roles.yml",
+                "roles_mapping.yml",
+                "security_tenants.yml",
+                "tenants.yml" };
+            for (String fileName : configurationFiles) {
+                Path configFileDestination = tempDirectory.resolve(fileName);
+                copyResourceToFile(fileName, configFileDestination.toFile());
+            }
+            return tempDirectory.toAbsolutePath();
+        } catch (IOException ex) {
+            throw new RuntimeException("Cannot create directory with security plugin configuration.", ex);
+        }
+    }
+
+    private static void copyResourceToFile(String resource, File destination) {
+        try (InputStream input = ConfigurationFiles.class.getClassLoader().getResourceAsStream(resource)) {
+            Objects.requireNonNull(input, "Cannot find source resource " + resource);
+            try (OutputStream output = new FileOutputStream(destination)) {
+                input.transferTo(output);
+            }
+        } catch (IOException e) {
+            throw new RuntimeException("Cannot create file with security plugin configuration", e);
+        }
+    }
+}