New test framework

Signed-Off-By: Nils Bandener <[email protected]>

build.gradle

@@ -224,10 +224,51 @@ configurations.all {
         force "io.netty:netty-common:${versions.netty}"
         force "io.netty:netty-handler:${versions.netty}"
         force "io.netty:netty-transport:${versions.netty}"
-        force "io.netty:netty-transport-native-unix-common:${versions.netty}"
+        }
+}
+
+//create source set 'newTest'
+//add classes from the main source set to the compilation and runtime classpaths of the newTest
+sourceSets {
+    newTest {
+        compileClasspath += sourceSets.main.output
+        runtimeClasspath += sourceSets.main.output
     }
 }
 
+//add new task that runs new tests
+task newTest(type: Test) {
+    description = 'Run new tests.'
+    group = 'verification'
+    testClassesDirs = sourceSets.newTest.output.classesDirs
+    classpath = sourceSets.newTest.runtimeClasspath
+
+    //run the newTest task after the test task
+    shouldRunAfter test
+}
+
+//run the newTest task before the check task
+check.dependsOn newTest
+
+configurations {
+    all {
+        resolutionStrategy {
+            force 'commons-codec:commons-codec:1.14'
+            force 'org.apache.santuario:xmlsec:2.2.3'
+            force 'org.cryptacular:cryptacular:1.2.4'
+            force 'net.minidev:json-smart:2.4.7'
+            force 'commons-cli:commons-cli:1.3.1'
+            force 'org.apache.httpcomponents:httpcore:4.4.12'
+            force "org.apache.commons:commons-lang3:3.4"
+            force "org.springframework:spring-core:5.3.20"
+            force "com.google.guava:guava:30.0-jre"
+        }
+    }
+
+    //use testImplementation dependencies in the newTestImplementation configuration
+    newTestImplementation.extendsFrom testImplementation
+}
+
 dependencies {
     implementation 'jakarta.annotation:jakarta.annotation-api:1.3.5'
     implementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}"

src/main/java/org/opensearch/security/support/PemKeyReader.java

@@ -274,15 +274,19 @@ public static X509Certificate[] loadCertificatesFromFile(String file) throws Exc
             return null;
         }
 
-        CertificateFactory fact = CertificateFactory.getInstance("X.509");
         try(FileInputStream is = new FileInputStream(file)) {
-            Collection<? extends Certificate> certs = fact.generateCertificates(is);
-            X509Certificate[] x509Certs = new X509Certificate[certs.size()];
-            int i=0;
-            for(Certificate cert: certs) {
-                x509Certs[i++] = (X509Certificate) cert;
-            }
-            return x509Certs;
+        	return loadCertificatesFromStream(is);
+        }
+
+    }
+
+    public static X509Certificate[] loadCertificatesFromFile(File file) throws Exception {
+        if(file == null) {
+            return null;
+        }
+
+        try(FileInputStream is = new FileInputStream(file)) {
+        	return loadCertificatesFromStream(is);
         }
 
     }

src/newTest/java/org/opensearch/node/PluginAwareNode.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2015-2018 _floragunn_ GmbH
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.node;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import org.opensearch.common.settings.Settings;
+import org.opensearch.plugins.Plugin;
+
+public class PluginAwareNode extends Node {
+
+    private final boolean clusterManagerEligible;
+
+    @SafeVarargs
+    public PluginAwareNode(boolean clusterManagerEligible, final Settings preparedSettings, final Class<? extends Plugin>... plugins) {
+    	super(InternalSettingsPreparer.prepareEnvironment(preparedSettings, Collections.emptyMap(), null, () -> System.getenv("HOSTNAME")), Arrays.asList(plugins), true);
+        this.clusterManagerEligible = clusterManagerEligible;
+    }
+
+
+    public boolean isClusterManagerEligible() {
+        return clusterManagerEligible;
+    }
+}

src/newTest/java/org/opensearch/test/AbstractIntegrationTest.java

@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.test;
+
+import org.junit.runner.RunWith;
+import org.opensearch.test.framework.TestSecurityConfig;
+import org.opensearch.test.framework.TestSecurityConfig.Role;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+
+@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+public class AbstractIntegrationTest {
+
+	/**
+	 * Auth domain with HTTPS Basic and the internal user backend
+	 */
+	protected final static TestSecurityConfig.AuthcDomain AUTHC_HTTPBASIC_INTERNAL = new TestSecurityConfig.AuthcDomain("basic", 0)
+			.httpAuthenticator("basic").backend("internal");
+
+	/**
+	 * Admin user with full access to all indices
+	 */
+	protected final static TestSecurityConfig.User USER_ADMIN = new TestSecurityConfig.User("admin")
+			.roles(new Role("allaccess").indexPermissions("*").on("*").clusterPermissions("*"));
+
+}

src/newTest/java/org/opensearch/test/GenericIntegrationTest.java

@@ -0,0 +1,79 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.test;
+
+import org.apache.http.HttpStatus;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.opensearch.test.framework.TestIndex;
+import org.opensearch.test.framework.TestSecurityConfig;
+import org.opensearch.test.framework.TestSecurityConfig.Role;
+import org.opensearch.test.framework.cluster.ClusterConfiguration;
+import org.opensearch.test.framework.cluster.LocalCluster;
+import org.opensearch.test.framework.cluster.TestRestClient;
+import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse;
+
+import com.fasterxml.jackson.core.JsonPointer;
+
+/**
+ * WIP
+ * Generic test class that demonstrates how to use the test framework to 
+ * set up a test cluster with users, roles, indices and data, and how to
+ * implement tests. One main goal here is to make tests self-contained.
+ */
+public class GenericIntegrationTest extends AbstractIntegrationTest {
+
+    // define indices used in this test
+    private final static TestIndex INDEX_A = TestIndex.name("index-a").build();
+    private final static TestIndex INDEX_B = TestIndex.name("index-b").build();
+
+    private final static TestSecurityConfig.User INDEX_A_USER = new TestSecurityConfig.User("index_a_user")
+            .roles(new Role("index_a_role").indexPermissions("*").on(INDEX_A).clusterPermissions("*"));
+
+
+    // build our test cluster as a ClassRule
+    @ClassRule
+    public static LocalCluster cluster = new LocalCluster.Builder().clusterConfiguration(ClusterConfiguration.THREE_MASTERS)
+    	.authc(AUTHC_HTTPBASIC_INTERNAL)
+    	.users(USER_ADMIN, INDEX_A_USER)
+    	.indices(INDEX_A, INDEX_B).build();
+
+    @Test
+    public void testAdminUserHasAccessToAllIndices() throws Exception {
+        try (TestRestClient client = cluster.getRestClient(USER_ADMIN)) {
+            HttpResponse response = client.get("*/_search?pretty");
+            Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK);
+        }        
+    }
+
+    @Test
+    public void testIndexAUserHasOnlyAccessToIndexA() throws Exception {
+        try (TestRestClient client = cluster.getRestClient(INDEX_A_USER)) {
+            HttpResponse response = client.get("index-a/_search?pretty");
+            Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK);
+
+            // demo: work with JSON response body and check values
+            JsonPointer jsonPointer = JsonPointer.compile("/_source/hits/value");
+            int hits = response.toJsonNode().at(jsonPointer).asInt();
+
+            response = client.get("index-b/_search?pretty");
+            Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_FORBIDDEN);                       
+        }
+    }    
+
+    @AfterClass
+    public static void close() {
+    	cluster.close();
+    }
+}

src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java

@@ -0,0 +1,71 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.test;
+
+import org.apache.http.HttpStatus;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.opensearch.test.framework.TestSecurityConfig;
+import org.opensearch.test.framework.TestSecurityConfig.Role;
+import org.opensearch.test.framework.cluster.ClusterConfiguration;
+import org.opensearch.test.framework.cluster.LocalCluster;
+import org.opensearch.test.framework.cluster.TestRestClient;
+import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse;
+
+/**
+ * This is a port for the test
+ * org.opensearch.security.privileges.PrivilegesEvaluatorTest to the new test
+ * framework for direct comparison
+ *
+ */
+public class PrivilegesEvaluatorTest extends AbstractIntegrationTest {
+
+	protected final static TestSecurityConfig.User NEGATIVE_LOOKAHEAD = new TestSecurityConfig.User(
+			"negative_lookahead_user")
+			.roles(new Role("negative_lookahead_role").indexPermissions("read").on("/^(?!t.*).*/")
+					.clusterPermissions("cluster_composite_ops"));
+
+	protected final static TestSecurityConfig.User NEGATED_REGEX = new TestSecurityConfig.User("negated_regex_user")
+			.roles(new Role("negated_regex_role").indexPermissions("read").on("/^[a-z].*/")
+					.clusterPermissions("cluster_composite_ops"));
+
+	@ClassRule
+	public static LocalCluster cluster = new LocalCluster.Builder()
+			.clusterConfiguration(ClusterConfiguration.THREE_MASTERS).authc(AUTHC_HTTPBASIC_INTERNAL)
+			.users(NEGATIVE_LOOKAHEAD, NEGATED_REGEX).build();
+
+	@Test
+	public void testNegativeLookaheadPattern() throws Exception {
+
+		try (TestRestClient client = cluster.getRestClient(NEGATIVE_LOOKAHEAD)) {
+			HttpResponse response = client.get("*/_search");
+			Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_FORBIDDEN);
+
+			response = client.get("r*/_search");
+			Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK);
+		}
+	}
+
+	@Test
+	public void testRegexPattern() throws Exception {
+
+		try (TestRestClient client = cluster.getRestClient(NEGATED_REGEX)) {
+			HttpResponse response = client.get("*/_search");
+			Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_FORBIDDEN);
+
+			response = client.get("r*/_search");
+			Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK);
+		}
+
+	}
+}

src/newTest/java/org/opensearch/test/SecurityRolesTests.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2015-2018 _floragunn_ GmbH
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.test;
+
+import org.apache.http.HttpStatus;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.opensearch.test.framework.TestSecurityConfig;
+import org.opensearch.test.framework.TestSecurityConfig.Role;
+import org.opensearch.test.framework.cluster.ClusterConfiguration;
+import org.opensearch.test.framework.cluster.LocalCluster;
+import org.opensearch.test.framework.cluster.TestRestClient;
+import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse;
+
+import com.fasterxml.jackson.core.JsonPointer;
+
+public class SecurityRolesTests extends AbstractIntegrationTest {
+
+	protected final static TestSecurityConfig.User USER_SR = new TestSecurityConfig.User("sr_user").roles(
+			new Role("abc_ber").indexPermissions("*").on("*").clusterPermissions("*"),
+			new Role("def_efg").indexPermissions("*").on("*").clusterPermissions("*"));
+
+	@ClassRule
+	public static LocalCluster cluster = new LocalCluster.Builder()
+			.clusterConfiguration(ClusterConfiguration.THREE_MASTERS).anonymousAuth(true)
+			.authc(AUTHC_HTTPBASIC_INTERNAL).users(USER_SR).build();
+
+	@Test
+	public void testSecurityRolesAnon() throws Exception {
+
+		try (TestRestClient client = cluster.getRestClient(USER_SR)) {
+			HttpResponse response = client.getAuthInfo();
+			Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK);
+
+			// Check username
+			JsonPointer jsonPointer = JsonPointer.compile("/user_name");
+			String username = response.toJsonNode().at(jsonPointer).asText();
+			Assert.assertEquals("sr_user", username);
+
+			// Check security roles
+			jsonPointer = JsonPointer.compile("/roles/0");
+			String securityRole = response.toJsonNode().at(jsonPointer).asText();
+			Assert.assertEquals("user_sr_user__abc_ber", securityRole);
+
+			jsonPointer = JsonPointer.compile("/roles/1");
+			securityRole = response.toJsonNode().at(jsonPointer).asText();
+			Assert.assertEquals("user_sr_user__def_efg", securityRole);
+
+		}
+	}
+
+}