From 1efe73481e61f584024a5b0ee48c23362b9ce012 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Tue, 26 Jul 2022 16:17:53 +0200 Subject: [PATCH 01/47] New test framework Signed-Off-By: Nils Bandener --- build.gradle | 43 +- .../security/support/PemKeyReader.java | 20 +- .../org/opensearch/node/PluginAwareNode.java | 49 ++ .../test/AbstractIntegrationTest.java | 36 ++ .../test/GenericIntegrationTest.java | 79 +++ .../test/PrivilegesEvaluatorTest.java | 71 ++ .../opensearch/test/SecurityRolesTests.java | 77 +++ .../opensearch/test/framework/TestIndex.java | 87 +++ .../test/framework/TestSecurityConfig.java | 609 +++++++++++++++++ .../framework/certificate/Certificates.java | 165 +++++ .../certificate/TestCertificates.java | 63 ++ .../cluster/ClusterConfiguration.java | 131 ++++ .../test/framework/cluster/ClusterInfo.java | 45 ++ .../cluster/ContextHeaderDecoratorClient.java | 60 ++ .../test/framework/cluster/LocalCluster.java | 373 +++++++++++ .../cluster/LocalOpenSearchCluster.java | 610 ++++++++++++++++++ ...inimumSecuritySettingsSupplierFactory.java | 80 +++ .../framework/cluster/NestedValueMap.java | 518 +++++++++++++++ .../cluster/NodeSettingsSupplier.java | 34 + .../cluster/OpenSearchClientProvider.java | 133 ++++ .../test/framework/cluster/PortAllocator.java | 157 +++++ .../test/framework/cluster/SocketUtils.java | 319 +++++++++ .../framework/cluster/TestRestClient.java | 366 +++++++++++ 23 files changed, 4116 insertions(+), 9 deletions(-) create mode 100644 src/newTest/java/org/opensearch/node/PluginAwareNode.java create mode 100644 src/newTest/java/org/opensearch/test/AbstractIntegrationTest.java create mode 100644 src/newTest/java/org/opensearch/test/GenericIntegrationTest.java create mode 100644 src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java create mode 100644 src/newTest/java/org/opensearch/test/SecurityRolesTests.java create mode 100644 src/newTest/java/org/opensearch/test/framework/TestIndex.java create mode 100644 src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java create mode 100644 src/newTest/java/org/opensearch/test/framework/certificate/Certificates.java create mode 100644 src/newTest/java/org/opensearch/test/framework/certificate/TestCertificates.java create mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/ClusterConfiguration.java create mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/ClusterInfo.java create mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java create mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java create mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java create mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java create mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java create mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java create mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java create mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/PortAllocator.java create mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/SocketUtils.java create mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java diff --git a/build.gradle b/build.gradle index 02e32182ef..0ec6023315 100644 --- a/build.gradle +++ b/build.gradle @@ -235,10 +235,51 @@ configurations.all { force "io.netty:netty-common:${versions.netty}" force "io.netty:netty-handler:${versions.netty}" force "io.netty:netty-transport:${versions.netty}" - force "io.netty:netty-transport-native-unix-common:${versions.netty}" + } +} + +//create source set 'newTest' +//add classes from the main source set to the compilation and runtime classpaths of the newTest +sourceSets { + newTest { + compileClasspath += sourceSets.main.output + runtimeClasspath += sourceSets.main.output } } +//add new task that runs new tests +task newTest(type: Test) { + description = 'Run new tests.' + group = 'verification' + testClassesDirs = sourceSets.newTest.output.classesDirs + classpath = sourceSets.newTest.runtimeClasspath + + //run the newTest task after the test task + shouldRunAfter test +} + +//run the newTest task before the check task +check.dependsOn newTest + +configurations { + all { + resolutionStrategy { + force 'commons-codec:commons-codec:1.14' + force 'org.apache.santuario:xmlsec:2.2.3' + force 'org.cryptacular:cryptacular:1.2.4' + force 'net.minidev:json-smart:2.4.7' + force 'commons-cli:commons-cli:1.3.1' + force 'org.apache.httpcomponents:httpcore:4.4.12' + force "org.apache.commons:commons-lang3:3.4" + force "org.springframework:spring-core:5.3.20" + force "com.google.guava:guava:30.0-jre" + } + } + + //use testImplementation dependencies in the newTestImplementation configuration + newTestImplementation.extendsFrom testImplementation +} + dependencies { implementation 'jakarta.annotation:jakarta.annotation-api:1.3.5' implementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}" diff --git a/src/main/java/org/opensearch/security/support/PemKeyReader.java b/src/main/java/org/opensearch/security/support/PemKeyReader.java index 53eeb21736..66d1af8799 100644 --- a/src/main/java/org/opensearch/security/support/PemKeyReader.java +++ b/src/main/java/org/opensearch/security/support/PemKeyReader.java @@ -274,15 +274,19 @@ public static X509Certificate[] loadCertificatesFromFile(String file) throws Exc return null; } - CertificateFactory fact = CertificateFactory.getInstance("X.509"); try(FileInputStream is = new FileInputStream(file)) { - Collection certs = fact.generateCertificates(is); - X509Certificate[] x509Certs = new X509Certificate[certs.size()]; - int i=0; - for(Certificate cert: certs) { - x509Certs[i++] = (X509Certificate) cert; - } - return x509Certs; + return loadCertificatesFromStream(is); + } + + } + + public static X509Certificate[] loadCertificatesFromFile(File file) throws Exception { + if(file == null) { + return null; + } + + try(FileInputStream is = new FileInputStream(file)) { + return loadCertificatesFromStream(is); } } diff --git a/src/newTest/java/org/opensearch/node/PluginAwareNode.java b/src/newTest/java/org/opensearch/node/PluginAwareNode.java new file mode 100644 index 0000000000..39c93e6b8e --- /dev/null +++ b/src/newTest/java/org/opensearch/node/PluginAwareNode.java @@ -0,0 +1,49 @@ +/* + * Copyright 2015-2018 _floragunn_ GmbH + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.node; + +import java.util.Arrays; +import java.util.Collections; + +import org.opensearch.common.settings.Settings; +import org.opensearch.plugins.Plugin; + +public class PluginAwareNode extends Node { + + private final boolean clusterManagerEligible; + + @SafeVarargs + public PluginAwareNode(boolean clusterManagerEligible, final Settings preparedSettings, final Class... plugins) { + super(InternalSettingsPreparer.prepareEnvironment(preparedSettings, Collections.emptyMap(), null, () -> System.getenv("HOSTNAME")), Arrays.asList(plugins), true); + this.clusterManagerEligible = clusterManagerEligible; + } + + + public boolean isClusterManagerEligible() { + return clusterManagerEligible; + } +} diff --git a/src/newTest/java/org/opensearch/test/AbstractIntegrationTest.java b/src/newTest/java/org/opensearch/test/AbstractIntegrationTest.java new file mode 100644 index 0000000000..3f13fe261a --- /dev/null +++ b/src/newTest/java/org/opensearch/test/AbstractIntegrationTest.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test; + +import org.junit.runner.RunWith; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.Role; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class AbstractIntegrationTest { + + /** + * Auth domain with HTTPS Basic and the internal user backend + */ + protected final static TestSecurityConfig.AuthcDomain AUTHC_HTTPBASIC_INTERNAL = new TestSecurityConfig.AuthcDomain("basic", 0) + .httpAuthenticator("basic").backend("internal"); + + /** + * Admin user with full access to all indices + */ + protected final static TestSecurityConfig.User USER_ADMIN = new TestSecurityConfig.User("admin") + .roles(new Role("allaccess").indexPermissions("*").on("*").clusterPermissions("*")); + +} diff --git a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java new file mode 100644 index 0000000000..cfddf84ad5 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test; + +import org.apache.http.HttpStatus; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Test; +import org.opensearch.test.framework.TestIndex; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.Role; +import org.opensearch.test.framework.cluster.ClusterConfiguration; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; + +import com.fasterxml.jackson.core.JsonPointer; + +/** + * WIP + * Generic test class that demonstrates how to use the test framework to + * set up a test cluster with users, roles, indices and data, and how to + * implement tests. One main goal here is to make tests self-contained. + */ +public class GenericIntegrationTest extends AbstractIntegrationTest { + + // define indices used in this test + private final static TestIndex INDEX_A = TestIndex.name("index-a").build(); + private final static TestIndex INDEX_B = TestIndex.name("index-b").build(); + + private final static TestSecurityConfig.User INDEX_A_USER = new TestSecurityConfig.User("index_a_user") + .roles(new Role("index_a_role").indexPermissions("*").on(INDEX_A).clusterPermissions("*")); + + + // build our test cluster as a ClassRule + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterConfiguration(ClusterConfiguration.THREE_MASTERS) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USER_ADMIN, INDEX_A_USER) + .indices(INDEX_A, INDEX_B).build(); + + @Test + public void testAdminUserHasAccessToAllIndices() throws Exception { + try (TestRestClient client = cluster.getRestClient(USER_ADMIN)) { + HttpResponse response = client.get("*/_search?pretty"); + Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); + } + } + + @Test + public void testIndexAUserHasOnlyAccessToIndexA() throws Exception { + try (TestRestClient client = cluster.getRestClient(INDEX_A_USER)) { + HttpResponse response = client.get("index-a/_search?pretty"); + Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); + + // demo: work with JSON response body and check values + JsonPointer jsonPointer = JsonPointer.compile("/_source/hits/value"); + int hits = response.toJsonNode().at(jsonPointer).asInt(); + + response = client.get("index-b/_search?pretty"); + Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_FORBIDDEN); + } + } + + @AfterClass + public static void close() { + cluster.close(); + } +} diff --git a/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java b/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java new file mode 100644 index 0000000000..44307c6a02 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test; + +import org.apache.http.HttpStatus; +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Test; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.Role; +import org.opensearch.test.framework.cluster.ClusterConfiguration; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; + +/** + * This is a port for the test + * org.opensearch.security.privileges.PrivilegesEvaluatorTest to the new test + * framework for direct comparison + * + */ +public class PrivilegesEvaluatorTest extends AbstractIntegrationTest { + + protected final static TestSecurityConfig.User NEGATIVE_LOOKAHEAD = new TestSecurityConfig.User( + "negative_lookahead_user") + .roles(new Role("negative_lookahead_role").indexPermissions("read").on("/^(?!t.*).*/") + .clusterPermissions("cluster_composite_ops")); + + protected final static TestSecurityConfig.User NEGATED_REGEX = new TestSecurityConfig.User("negated_regex_user") + .roles(new Role("negated_regex_role").indexPermissions("read").on("/^[a-z].*/") + .clusterPermissions("cluster_composite_ops")); + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder() + .clusterConfiguration(ClusterConfiguration.THREE_MASTERS).authc(AUTHC_HTTPBASIC_INTERNAL) + .users(NEGATIVE_LOOKAHEAD, NEGATED_REGEX).build(); + + @Test + public void testNegativeLookaheadPattern() throws Exception { + + try (TestRestClient client = cluster.getRestClient(NEGATIVE_LOOKAHEAD)) { + HttpResponse response = client.get("*/_search"); + Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_FORBIDDEN); + + response = client.get("r*/_search"); + Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); + } + } + + @Test + public void testRegexPattern() throws Exception { + + try (TestRestClient client = cluster.getRestClient(NEGATED_REGEX)) { + HttpResponse response = client.get("*/_search"); + Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_FORBIDDEN); + + response = client.get("r*/_search"); + Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); + } + + } +} diff --git a/src/newTest/java/org/opensearch/test/SecurityRolesTests.java b/src/newTest/java/org/opensearch/test/SecurityRolesTests.java new file mode 100644 index 0000000000..b5de319912 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/SecurityRolesTests.java @@ -0,0 +1,77 @@ +/* + * Copyright 2015-2018 _floragunn_ GmbH + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test; + +import org.apache.http.HttpStatus; +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Test; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.Role; +import org.opensearch.test.framework.cluster.ClusterConfiguration; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; + +import com.fasterxml.jackson.core.JsonPointer; + +public class SecurityRolesTests extends AbstractIntegrationTest { + + protected final static TestSecurityConfig.User USER_SR = new TestSecurityConfig.User("sr_user").roles( + new Role("abc_ber").indexPermissions("*").on("*").clusterPermissions("*"), + new Role("def_efg").indexPermissions("*").on("*").clusterPermissions("*")); + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder() + .clusterConfiguration(ClusterConfiguration.THREE_MASTERS).anonymousAuth(true) + .authc(AUTHC_HTTPBASIC_INTERNAL).users(USER_SR).build(); + + @Test + public void testSecurityRolesAnon() throws Exception { + + try (TestRestClient client = cluster.getRestClient(USER_SR)) { + HttpResponse response = client.getAuthInfo(); + Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); + + // Check username + JsonPointer jsonPointer = JsonPointer.compile("/user_name"); + String username = response.toJsonNode().at(jsonPointer).asText(); + Assert.assertEquals("sr_user", username); + + // Check security roles + jsonPointer = JsonPointer.compile("/roles/0"); + String securityRole = response.toJsonNode().at(jsonPointer).asText(); + Assert.assertEquals("user_sr_user__abc_ber", securityRole); + + jsonPointer = JsonPointer.compile("/roles/1"); + securityRole = response.toJsonNode().at(jsonPointer).asText(); + Assert.assertEquals("user_sr_user__def_efg", securityRole); + + } + } + +} diff --git a/src/newTest/java/org/opensearch/test/framework/TestIndex.java b/src/newTest/java/org/opensearch/test/framework/TestIndex.java new file mode 100644 index 0000000000..035418c2b4 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/TestIndex.java @@ -0,0 +1,87 @@ +/* + * Copyright 2021-2022 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework; + +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; + +public class TestIndex { + + private final String name; + private final Settings settings; + + public TestIndex(String name, Settings settings) { + this.name = name; + this.settings = settings; + + } + + public void create(Client client) { + if (!client.admin().indices().exists(new IndicesExistsRequest(name)).actionGet().isExists()) { + client.admin().indices().create(new CreateIndexRequest(name).settings(settings)).actionGet(); + } + } + + public String getName() { + return name; + } + + + public static Builder name(String name) { + return new Builder().name(name); + } + + public static class Builder { + private String name; + private Settings.Builder settings = Settings.builder(); + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder setting(String name, int value) { + settings.put(name, value); + return this; + } + + public Builder shards(int value) { + settings.put("index.number_of_shards", 5); + return this; + } + + public TestIndex build() { + return new TestIndex(name, settings.build()); + } + + } + +} diff --git a/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java new file mode 100644 index 0000000000..88e86b51b3 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -0,0 +1,609 @@ +/* + * Copyright 2021 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework; + +import java.nio.ByteBuffer; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bouncycastle.crypto.generators.OpenBSDBCrypt; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.support.WriteRequest.RefreshPolicy; +import org.opensearch.client.Client; +import org.opensearch.common.Strings; +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.security.action.configupdate.ConfigUpdateAction; +import org.opensearch.security.action.configupdate.ConfigUpdateRequest; +import org.opensearch.security.action.configupdate.ConfigUpdateResponse; +import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.test.framework.cluster.NestedValueMap; +import org.opensearch.test.framework.cluster.NestedValueMap.Path; +import org.opensearch.test.framework.cluster.OpenSearchClientProvider.UserCredentialsHolder; + +public class TestSecurityConfig { + + private static final Logger log = LogManager.getLogger(TestSecurityConfig.class); + + private NestedValueMap overrideSecurityConfigSettings; + private NestedValueMap overrideUserSettings; + private NestedValueMap overrideRoleSettings; + private NestedValueMap overrideRoleMappingSettings; + private String indexName = ".opendistro_security"; + private Map> variableSuppliers = new HashMap<>(); + + public TestSecurityConfig() { + + } + + public TestSecurityConfig configIndexName(String configIndexName) { + this.indexName = configIndexName; + return this; + } + + public TestSecurityConfig var(String name, Supplier variableSupplier) { + this.variableSuppliers.put(name, variableSupplier); + return this; + } + + public TestSecurityConfig securityConfigSettings(String keyPath, Object value, Object... more) { + if (overrideSecurityConfigSettings == null) { + overrideSecurityConfigSettings = new NestedValueMap(); + } + + overrideSecurityConfigSettings.put(NestedValueMap.Path.parse(keyPath), value); + + for (int i = 0; i < more.length - 1; i += 2) { + overrideSecurityConfigSettings.put(NestedValueMap.Path.parse(String.valueOf(more[i])), more[i + 1]); + } + + return this; + } + + public TestSecurityConfig xff(String proxies) { + if (overrideSecurityConfigSettings == null) { + overrideSecurityConfigSettings = new NestedValueMap(); + } + + overrideSecurityConfigSettings.put(new NestedValueMap.Path("config", "dynamic", "http", "xff"), + NestedValueMap.of("enabled", true, "internalProxies", proxies)); + + return this; + } + + public TestSecurityConfig anonymousAuth(boolean anonymousAuthEnabled) { + if (overrideSecurityConfigSettings == null) { + overrideSecurityConfigSettings = new NestedValueMap(); + } + + overrideSecurityConfigSettings.put(new NestedValueMap.Path("config", "dynamic", "http"), + NestedValueMap.of("anonymous_auth_enabled", anonymousAuthEnabled)); + + return this; + } + + public TestSecurityConfig authc(AuthcDomain authcDomain) { + if (overrideSecurityConfigSettings == null) { + overrideSecurityConfigSettings = new NestedValueMap(); + } + + overrideSecurityConfigSettings.put(new NestedValueMap.Path("config", "dynamic", "authc"), authcDomain.toMap()); + + return this; + } + + public TestSecurityConfig user(User user) { + if (user.roleNames != null) { + return this.user(user.name, user.password, user.attributes, user.roleNames); + } else { + return this.user(user.name, user.password, user.attributes, user.roles); + } + } + + public TestSecurityConfig user(String name, String password, String... sgRoles) { + return user(name, password, null, sgRoles); + } + + public TestSecurityConfig user(String name, String password, Map attributes, String... securityRoles) { + if (overrideUserSettings == null) { + overrideUserSettings = new NestedValueMap(); + } + + overrideUserSettings.put(new NestedValueMap.Path(name, "hash"), hash(password.toCharArray())); + + if (securityRoles != null && securityRoles.length > 0) { + overrideUserSettings.put(new NestedValueMap.Path(name, "opensearch_security_roles"), securityRoles); + } + + if (attributes != null && attributes.size() != 0) { + for (Map.Entry attr : attributes.entrySet()) { + overrideUserSettings.put(new NestedValueMap.Path(name, "attributes", attr.getKey()), attr.getValue()); + } + } + + return this; + } + + public TestSecurityConfig user(String name, String password, Role... sgRoles) { + return user(name, password, null, sgRoles); + } + + public TestSecurityConfig user(String name, String password, Map attributes, Role... sgRoles) { + if (overrideUserSettings == null) { + overrideUserSettings = new NestedValueMap(); + } + + overrideUserSettings.put(new NestedValueMap.Path(name, "hash"), hash(password.toCharArray())); + + if (sgRoles != null && sgRoles.length > 0) { + String roleNamePrefix = "user_" + name + "__"; + + overrideUserSettings.put(new NestedValueMap.Path(name, "opendistro_security_roles"), + Arrays.asList(sgRoles).stream().map((r) -> roleNamePrefix + r.name).collect(Collectors.toList())); + roles(roleNamePrefix, sgRoles); + } + + if (attributes != null && attributes.size() != 0) { + for (Map.Entry attr : attributes.entrySet()) { + overrideUserSettings.put(new NestedValueMap.Path(name, "attributes", attr.getKey()), attr.getValue()); + } + } + + return this; + } + + public TestSecurityConfig roles(Role... roles) { + return roles("", roles); + } + + public TestSecurityConfig roles(String roleNamePrefix, Role... roles) { + if (overrideRoleSettings == null) { + overrideRoleSettings = new NestedValueMap(); + } + + for (Role role : roles) { + + String name = roleNamePrefix + role.name; + + if (role.clusterPermissions.size() > 0) { + overrideRoleSettings.put(new NestedValueMap.Path(name, "cluster_permissions"), role.clusterPermissions); + } + + if (role.indexPermissions.size() > 0) { + overrideRoleSettings.put(new NestedValueMap.Path(name, "index_permissions"), + role.indexPermissions.stream().map((p) -> p.toJsonMap()).collect(Collectors.toList())); + } + } + + return this; + } + + public TestSecurityConfig roleMapping(RoleMapping... roleMappings) { + if (overrideRoleMappingSettings == null) { + overrideRoleMappingSettings = new NestedValueMap(); + } + + for (RoleMapping roleMapping : roleMappings) { + + String name = roleMapping.name; + + if (roleMapping.backendRoles.size() > 0) { + overrideRoleMappingSettings.put(new NestedValueMap.Path(name, "backend_roles"), + roleMapping.backendRoles); + } + + if (roleMapping.users.size() > 0) { + overrideRoleMappingSettings.put(new NestedValueMap.Path(name, "users"), roleMapping.users); + } + } + + return this; + } + + public TestSecurityConfig roleToRoleMapping(Role role, String... backendRoles) { + return this.roleMapping(new RoleMapping(role.name).backendRoles(backendRoles)); + } + + public static class User implements UserCredentialsHolder { + private String name; + private String password; + private Role[] roles; + private String[] roleNames; + private Map attributes = new HashMap<>(); + + public User(String name) { + this.name = name; + this.password = "secret"; + } + + public User password(String password) { + this.password = password; + return this; + } + + public User roles(Role... roles) { + this.roles = roles; + return this; + } + + public User roles(String... roles) { + this.roleNames = roles; + return this; + } + + public User attr(String key, Object value) { + this.attributes.put(key, value); + return this; + } + + public String getName() { + return name; + } + + public String getPassword() { + return password; + } + + public Set getRoleNames() { + Set result = new HashSet(); + + if (roleNames != null) { + result.addAll(Arrays.asList(roleNames)); + } + + if (roles != null) { + result.addAll(Arrays.asList(roles).stream().map(Role::getName).collect(Collectors.toSet())); + } + + return result; + } + + } + + public static class Role { + public static Role ALL_ACCESS = new Role("all_access").clusterPermissions("*").indexPermissions("*").on("*"); + + private String name; + private List clusterPermissions = new ArrayList<>(); + + private List indexPermissions = new ArrayList<>(); + + public Role(String name) { + this.name = name; + } + + public Role clusterPermissions(String... clusterPermissions) { + this.clusterPermissions.addAll(Arrays.asList(clusterPermissions)); + return this; + } + + public IndexPermission indexPermissions(String... indexPermissions) { + return new IndexPermission(this, indexPermissions); + } + + public String getName() { + return name; + } + } + + public static class RoleMapping { + private String name; + private List backendRoles = new ArrayList<>(); + private List users = new ArrayList<>(); + + public RoleMapping(String name) { + this.name = name; + } + + public RoleMapping backendRoles(String... backendRoles) { + this.backendRoles.addAll(Arrays.asList(backendRoles)); + return this; + } + + public RoleMapping users(String... users) { + this.users.addAll(Arrays.asList(users)); + return this; + } + + } + + public static class IndexPermission { + private List allowedActions; + private List indexPatterns; + private Role role; + private String dlsQuery; + private List fls; + private List maskedFields; + + IndexPermission(Role role, String... allowedActions) { + this.allowedActions = Arrays.asList(allowedActions); + this.role = role; + } + + public IndexPermission dls(String dlsQuery) { + this.dlsQuery = dlsQuery; + return this; + } + + public IndexPermission fls(String... fls) { + this.fls = Arrays.asList(fls); + return this; + } + + public IndexPermission maskedFields(String... maskedFields) { + this.maskedFields = Arrays.asList(maskedFields); + return this; + } + + public Role on(String... indexPatterns) { + this.indexPatterns = Arrays.asList(indexPatterns); + this.role.indexPermissions.add(this); + return this.role; + } + + public Role on(TestIndex... testindices) { + this.indexPatterns = Arrays.asList(testindices).stream().map(TestIndex::getName).collect(Collectors.toList()); + this.role.indexPermissions.add(this); + return this.role; + } + + public NestedValueMap toJsonMap() { + NestedValueMap result = new NestedValueMap(); + + result.put("index_patterns", indexPatterns); + result.put("allowed_actions", allowedActions); + + if (dlsQuery != null) { + result.put("dls", dlsQuery); + } + + if (fls != null) { + result.put("fls", fls); + } + + if (maskedFields != null) { + result.put("masked_fields", maskedFields); + } + + return result; + } + + } + + public static class AuthcDomain { + + private final String id; + private boolean enabled = true; + private boolean transportEnabled = true; + private int order; + private List skipUsers = new ArrayList<>(); + private HttpAuthenticator httpAuthenticator; + private AuthenticationBackend authenticationBackend; + + public AuthcDomain(String id, int order) { + this.id = id; + this.order = order; + } + + public AuthcDomain httpAuthenticator(String type) { + this.httpAuthenticator = new HttpAuthenticator(type); + return this; + } + + public AuthcDomain challengingAuthenticator(String type) { + this.httpAuthenticator = new HttpAuthenticator(type).challenge(true); + return this; + } + + public AuthcDomain httpAuthenticator(HttpAuthenticator httpAuthenticator) { + this.httpAuthenticator = httpAuthenticator; + return this; + } + + public AuthcDomain backend(String type) { + this.authenticationBackend = new AuthenticationBackend(type); + return this; + } + + public AuthcDomain backend(AuthenticationBackend authenticationBackend) { + this.authenticationBackend = authenticationBackend; + return this; + } + + public AuthcDomain skipUsers(String... users) { + this.skipUsers.addAll(Arrays.asList(users)); + return this; + } + + NestedValueMap toMap() { + NestedValueMap result = new NestedValueMap(); + result.put(new NestedValueMap.Path(id, "http_enabled"), enabled); + result.put(new NestedValueMap.Path(id, "transport_enabled"), transportEnabled); + result.put(new NestedValueMap.Path(id, "order"), order); + + if (httpAuthenticator != null) { + result.put(new NestedValueMap.Path(id, "http_authenticator"), httpAuthenticator.toMap()); + } + + if (authenticationBackend != null) { + result.put(new NestedValueMap.Path(id, "authentication_backend"), authenticationBackend.toMap()); + } + + + if (skipUsers != null && skipUsers.size() > 0) { + result.put(new NestedValueMap.Path(id, "skip_users"), skipUsers); + } + + return result; + } + + public static class HttpAuthenticator { + private final String type; + private boolean challenge; + private NestedValueMap config = new NestedValueMap(); + + public HttpAuthenticator(String type) { + this.type = type; + } + + public HttpAuthenticator challenge(boolean challenge) { + this.challenge = challenge; + return this; + } + + public HttpAuthenticator config(Map config) { + this.config.putAllFromAnyMap(config); + return this; + } + + public HttpAuthenticator config(String key, Object value) { + this.config.put(Path.parse(key), value); + return this; + } + + NestedValueMap toMap() { + NestedValueMap result = new NestedValueMap(); + result.put("type", type); + result.put("challenge", challenge); + result.put("config", config); + return result; + } + } + + public static class AuthenticationBackend { + private final String type; + private NestedValueMap config = new NestedValueMap(); + + public AuthenticationBackend(String type) { + this.type = type; + } + + public AuthenticationBackend config(Map config) { + this.config.putAllFromAnyMap(config); + return this; + } + + public AuthenticationBackend config(String key, Object value) { + this.config.put(Path.parse(key), value); + return this; + } + + NestedValueMap toMap() { + NestedValueMap result = new NestedValueMap(); + result.put("type", type); + result.put("config", config); + return result; + } + } + } + + public TestSecurityConfig clone() { + TestSecurityConfig result = new TestSecurityConfig(); + result.indexName = indexName; + result.overrideRoleSettings = overrideRoleSettings != null ? overrideRoleSettings.clone() : null; + result.overrideSecurityConfigSettings = overrideSecurityConfigSettings != null ? overrideSecurityConfigSettings.clone() : null; + result.overrideUserSettings = overrideUserSettings != null ? overrideUserSettings.clone() : null; + + return result; + } + + public void initIndex(Client client) { + Map settings = new HashMap<>(); + if (indexName.startsWith(".")) { + settings.put("index.hidden", true); + } + client.admin().indices().create(new CreateIndexRequest(indexName).settings(settings)).actionGet(); + + writeConfigToIndex(client, CType.CONFIG, overrideSecurityConfigSettings); + writeConfigToIndex(client, CType.ROLES, overrideRoleSettings); + writeConfigToIndex(client, CType.INTERNALUSERS, overrideUserSettings); + writeConfigToIndex(client, CType.ROLESMAPPING, overrideRoleMappingSettings); + writeConfigToIndex(client, CType.ACTIONGROUPS); + writeConfigToIndex(client, CType.TENANTS); + + ConfigUpdateResponse configUpdateResponse = client.execute(ConfigUpdateAction.INSTANCE, + new ConfigUpdateRequest(CType.lcStringValues().toArray(new String[0]))).actionGet(); + + if (configUpdateResponse.hasFailures()) { + throw new RuntimeException("ConfigUpdateResponse produced failures: " + configUpdateResponse.failures()); + } + } + + + private static String hash(final char[] clearTextPassword) { + final byte[] salt = new byte[16]; + new SecureRandom().nextBytes(salt); + final String hash = OpenBSDBCrypt.generate((Objects.requireNonNull(clearTextPassword)), salt, 12); + Arrays.fill(salt, (byte) 0); + Arrays.fill(clearTextPassword, '\0'); + return hash; + } + + + private void writeConfigToIndex(Client client, CType configType) { + writeConfigToIndex(client, configType, NestedValueMap.createNonCloningMap()); + } + + private void writeConfigToIndex(Client client, CType configType, NestedValueMap overrides) { + try { + + NestedValueMap config = NestedValueMap.of(new NestedValueMap.Path("_meta", "type"), configType.toLCString(), + new NestedValueMap.Path("_meta", "config_version"), 2); + + if (overrides != null) { + config.overrideLeafs(overrides); + } + + XContentBuilder builder = XContentFactory.jsonBuilder().map(config); + String json = Strings.toString(builder); + + log.info("Writing " + configType + ":\n" + json); + + client.index(new IndexRequest(indexName).id(configType.toLCString()) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(configType.toLCString(), + BytesReference.fromByteBuffer(ByteBuffer.wrap(json.getBytes("utf-8"))))) + .actionGet(); + } catch (Exception e) { + throw new RuntimeException("Error while initializing config for " + indexName, e); + } + } +} diff --git a/src/newTest/java/org/opensearch/test/framework/certificate/Certificates.java b/src/newTest/java/org/opensearch/test/framework/certificate/Certificates.java new file mode 100644 index 0000000000..c24eebd1b7 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/certificate/Certificates.java @@ -0,0 +1,165 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.certificate; + +/** + * Contains static certificates for the test cluster. + * Note: This is WIP and will be replaced by classes + * that can generate certificates on the fly. This + * class will be removed after that. + */ +public class Certificates { + + final static String ROOT_CA_CERTIFICATE = "-----BEGIN CERTIFICATE-----\n" + + "MIID/jCCAuagAwIBAgIBATANBgkqhkiG9w0BAQsFADCBjzETMBEGCgmSJomT8ixk\n" + + "ARkWA2NvbTEXMBUGCgmSJomT8ixkARkWB2V4YW1wbGUxGTAXBgNVBAoMEEV4YW1w\n" + + "bGUgQ29tIEluYy4xITAfBgNVBAsMGEV4YW1wbGUgQ29tIEluYy4gUm9vdCBDQTEh\n" + + "MB8GA1UEAwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENBMB4XDTE4MDQyMjAzNDM0\n" + + "NloXDTI4MDQxOTAzNDM0NlowgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJ\n" + + "kiaJk/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEw\n" + + "HwYDVQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1w\n" + + "bGUgQ29tIEluYy4gUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC\n" + + "ggEBAK/u+GARP5innhpXK0c0q7s1Su1VTEaIgmZr8VWI6S8amf5cU3ktV7WT9SuV\n" + + "TsAm2i2A5P+Ctw7iZkfnHWlsC3HhPUcd6mvzGZ4moxnamM7r+a9otRp3owYoGStX\n" + + "ylVTQusAjbq9do8CMV4hcBTepCd+0w0v4h6UlXU8xjhj1xeUIz4DKbRgf36q0rv4\n" + + "VIX46X72rMJSETKOSxuwLkov1ZOVbfSlPaygXIxqsHVlj1iMkYRbQmaTib6XWHKf\n" + + "MibDaqDejOhukkCjzpptGZOPFQ8002UtTTNv1TiaKxkjMQJNwz6jfZ53ws3fh1I0\n" + + "RWT6WfM4oeFRFnyFRmc4uYTUgAkCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAf\n" + + "BgNVHSMEGDAWgBSSNQzgDx4rRfZNOfN7X6LmEpdAczAdBgNVHQ4EFgQUkjUM4A8e\n" + + "K0X2TTnze1+i5hKXQHMwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4IB\n" + + "AQBoQHvwsR34hGO2m8qVR9nQ5Klo5HYPyd6ySKNcT36OZ4AQfaCGsk+SecTi35QF\n" + + "RHL3g2qffED4tKR0RBNGQSgiLavmHGCh3YpDupKq2xhhEeS9oBmQzxanFwWFod4T\n" + + "nnsG2cCejyR9WXoRzHisw0KJWeuNlwjUdJY0xnn16srm1zL/M/f0PvCyh9HU1mF1\n" + + "ivnOSqbDD2Z7JSGyckgKad1Omsg/rr5XYtCeyJeXUPcmpeX6erWJJNTUh6yWC/hY\n" + + "G/dFC4xrJhfXwz6Z0ytUygJO32bJG4Np2iGAwvvgI9EfxzEv/KP+FGrJOvQJAq4/\n" + + "BU36ZAa80W/8TBnqZTkNnqZV\n" + + "-----END CERTIFICATE-----\n" + + ""; + + final static String NODE_CERTIFICATE = "-----BEGIN CERTIFICATE-----\n" + + "MIIEyTCCA7GgAwIBAgIGAWLrc1O2MA0GCSqGSIb3DQEBCwUAMIGPMRMwEQYKCZIm\n" + + "iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ\n" + + "RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290\n" + + "IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwHhcNMTgwNDIy\n" + + "MDM0MzQ3WhcNMjgwNDE5MDM0MzQ3WjBeMRIwEAYKCZImiZPyLGQBGRYCZGUxDTAL\n" + + "BgNVBAcMBHRlc3QxDTALBgNVBAoMBG5vZGUxDTALBgNVBAsMBG5vZGUxGzAZBgNV\n" + + "BAMMEm5vZGUtMC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC\n" + + "AQoCggEBAJa+f476vLB+AwK53biYByUwN+40D8jMIovGXm6wgT8+9Sbs899dDXgt\n" + + "9CE1Beo65oP1+JUz4c7UHMrCY3ePiDt4cidHVzEQ2g0YoVrQWv0RedS/yx/DKhs8\n" + + "Pw1O715oftP53p/2ijD5DifFv1eKfkhFH+lwny/vMSNxellpl6NxJTiJVnQ9HYOL\n" + + "gf2t971ITJHnAuuxUF48HcuNovW4rhtkXef8kaAN7cE3LU+A9T474ULNCKkEFPIl\n" + + "ZAKN3iJNFdVsxrTU+CUBHzk73Do1cCkEvJZ0ZFjp0Z3y8wLY/gqWGfGVyA9l2CUq\n" + + "eIZNf55PNPtGzOrvvONiui48vBKH1LsCAwEAAaOCAVkwggFVMIG8BgNVHSMEgbQw\n" + + "gbGAFJI1DOAPHitF9k0583tfouYSl0BzoYGVpIGSMIGPMRMwEQYKCZImiZPyLGQB\n" + + "GRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhhbXBs\n" + + "ZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENBMSEw\n" + + "HwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCAQEwHQYDVR0OBBYEFKyv\n" + + "78ZmFjVKM9g7pMConYH7FVBHMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgXg\n" + + "MCAGA1UdJQEB/wQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA1BgNVHREELjAsiAUq\n" + + "AwQFBYISbm9kZS0wLmV4YW1wbGUuY29tgglsb2NhbGhvc3SHBH8AAAEwDQYJKoZI\n" + + "hvcNAQELBQADggEBAIOKuyXsFfGv1hI/Lkpd/73QNqjqJdxQclX57GOMWNbOM5H0\n" + + "5/9AOIZ5JQsWULNKN77aHjLRr4owq2jGbpc/Z6kAd+eiatkcpnbtbGrhKpOtoEZy\n" + + "8KuslwkeixpzLDNISSbkeLpXz4xJI1ETMN/VG8ZZP1bjzlHziHHDu0JNZ6TnNzKr\n" + + "XzCGMCohFfem8vnKNnKUneMQMvXd3rzUaAgvtf7Hc2LTBlf4fZzZF1EkwdSXhaMA\n" + + "1lkfHiqOBxtgeDLxCHESZ2fqgVqsWX+t3qHQfivcPW6txtDyrFPRdJOGhiMGzT/t\n" + + "e/9kkAtQRgpTb3skYdIOOUOV0WGQ60kJlFhAzIs=\n" + + "-----END CERTIFICATE-----\n" + + ""; + + final static String NODE_KEY = "-----BEGIN PRIVATE KEY-----\n" + + "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCWvn+O+rywfgMC\n" + + "ud24mAclMDfuNA/IzCKLxl5usIE/PvUm7PPfXQ14LfQhNQXqOuaD9fiVM+HO1BzK\n" + + "wmN3j4g7eHInR1cxENoNGKFa0Fr9EXnUv8sfwyobPD8NTu9eaH7T+d6f9oow+Q4n\n" + + "xb9Xin5IRR/pcJ8v7zEjcXpZaZejcSU4iVZ0PR2Di4H9rfe9SEyR5wLrsVBePB3L\n" + + "jaL1uK4bZF3n/JGgDe3BNy1PgPU+O+FCzQipBBTyJWQCjd4iTRXVbMa01PglAR85\n" + + "O9w6NXApBLyWdGRY6dGd8vMC2P4KlhnxlcgPZdglKniGTX+eTzT7Rszq77zjYrou\n" + + "PLwSh9S7AgMBAAECggEABwiohxFoEIwws8XcdKqTWsbfNTw0qFfuHLuK2Htf7IWR\n" + + "htlzn66F3F+4jnwc5IsPCoVFriCXnsEC/usHHSMTZkL+gJqxlNaGdin6DXS/aiOQ\n" + + "nb69SaQfqNmsz4ApZyxVDqsQGkK0vAhDAtQVU45gyhp/nLLmmqP8lPzMirOEodmp\n" + + "U9bA8t/ttrzng7SVAER42f6IVpW0iTKTLyFii0WZbq+ObViyqib9hVFrI6NJuQS+\n" + + "IelcZB0KsSi6rqIjXg1XXyMiIUcSlhq+GfEa18AYgmsbPwMbExate7/8Ci7ZtCbh\n" + + "lx9bves2+eeqq5EMm3sMHyhdcg61yzd5UYXeZhwJkQKBgQDS9YqrAtztvLY2gMgv\n" + + "d+wOjb9awWxYbQTBjx33kf66W+pJ+2j8bI/XX2CpZ98w/oq8VhMqbr9j5b8MfsrF\n" + + "EoQvedA4joUo8sXd4j1mR2qKF4/KLmkgy6YYusNP2UrVSw7sh77bzce+YaVVoO/e\n" + + "0wIVTHuD/QZ6fG6MasOqcbl6hwKBgQC27cQruaHFEXR/16LrMVAX+HyEEv44KOCZ\n" + + "ij5OE4P7F0twb+okngG26+OJV3BtqXf0ULlXJ+YGwXCRf6zUZkld3NMy3bbKPgH6\n" + + "H/nf3BxqS2tudj7+DV52jKtisBghdvtlKs56oc9AAuwOs37DvhptBKUPdzDDqfys\n" + + "Qchv5JQdLQKBgERev+pcqy2Bk6xmYHrB6wdseS/4sByYeIoi0BuEfYH4eB4yFPx6\n" + + "UsQCbVl6CKPgWyZe3ydJbU37D8gE78KfFagtWoZ56j4zMF2RDUUwsB7BNCDamce/\n" + + "OL2bCeG/Erm98cBG3lxufOX+z47I8fTNfkdY2k8UmhzoZwurLm73HJ3RAoGBAKsp\n" + + "6yamuXF2FbYRhUXgjHsBbTD/vJO72/yO2CGiLRpi/5mjfkjo99269trp0C8sJSub\n" + + "5PBiSuADXFsoRgUv+HI1UAEGaCTwxFTQWrRWdtgW3d0sE2EQDVWL5kmfT9TwSeat\n" + + "mSoyAYR5t3tCBNkPJhbgA7pm4mASzHQ50VyxWs25AoGBAKPFx9X2oKhYQa+mW541\n" + + "bbqRuGFMoXIIcr/aeM3LayfLETi48o5NDr2NDP11j4yYuz26YLH0Dj8aKpWuehuH\n" + + "uB27n6j6qu0SVhQi6mMJBe1JrKbzhqMKQjYOoy8VsC2gdj5pCUP/kLQPW7zm9diX\n" + + "CiKTtKgPIeYdigor7V3AHcVT\n" + + "-----END PRIVATE KEY-----\n" + + ""; + + final static String ADMIN_CERTIFICATE = "-----BEGIN CERTIFICATE-----\n" + + "MIIEdzCCA1+gAwIBAgIGAWLrc1O4MA0GCSqGSIb3DQEBCwUAMIGPMRMwEQYKCZIm\n" + + "iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ\n" + + "RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290\n" + + "IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwHhcNMTgwNDIy\n" + + "MDM0MzQ3WhcNMjgwNDE5MDM0MzQ3WjBNMQswCQYDVQQGEwJkZTENMAsGA1UEBwwE\n" + + "dGVzdDEPMA0GA1UECgwGY2xpZW50MQ8wDQYDVQQLDAZjbGllbnQxDTALBgNVBAMM\n" + + "BGtpcmswggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDCwgBOoO88uMM8\n" + + "dREJsk58Yt4Jn0zwQ2wUThbvy3ICDiEWhiAhUbg6dTggpS5vWWJto9bvaaqgMVoh\n" + + "ElfYHdTDncX3UQNBEP8tqzHON6BFEFSGgJRGLd6f5dri6rK32nCotYS61CFXBFxf\n" + + "WumXjSukjyrcTsdkR3C5QDo2oN7F883MOQqRENPzAtZi9s3jNX48u+/e3yvJzXsB\n" + + "GS9Qmsye6C71enbIujM4CVwDT/7a5jHuaUp6OuNCFbdRPnu/wLYwOS2/yOtzAqk7\n" + + "/PFnPCe7YOa10ShnV/jx2sAHhp7ZQBJgFkkgnIERz9Ws74Au+EbptWnsWuB+LqRL\n" + + "x5G02IzpAgMBAAGjggEYMIIBFDCBvAYDVR0jBIG0MIGxgBSSNQzgDx4rRfZNOfN7\n" + + "X6LmEpdAc6GBlaSBkjCBjzETMBEGCgmSJomT8ixkARkWA2NvbTEXMBUGCgmSJomT\n" + + "8ixkARkWB2V4YW1wbGUxGTAXBgNVBAoMEEV4YW1wbGUgQ29tIEluYy4xITAfBgNV\n" + + "BAsMGEV4YW1wbGUgQ29tIEluYy4gUm9vdCBDQTEhMB8GA1UEAwwYRXhhbXBsZSBD\n" + + "b20gSW5jLiBSb290IENBggEBMB0GA1UdDgQWBBRsdhuHn3MGDvZxOe22+1wliCJB\n" + + "mDAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIF4DAWBgNVHSUBAf8EDDAKBggr\n" + + "BgEFBQcDAjANBgkqhkiG9w0BAQsFAAOCAQEAkPrUTKKn+/6g0CjhTPBFeX8mKXhG\n" + + "zw5z9Oq+xnwefZwxV82E/tgFsPcwXcJIBg0f43BaVSygPiV7bXqWhxASwn73i24z\n" + + "lveIR4+z56bKIhP6c3twb8WWR9yDcLu2Iroin7dYEm3dfVUrhz/A90WHr6ddwmLL\n" + + "3gcFF2kBu3S3xqM5OmN/tqRXFmo+EvwrdJRiTh4Fsf0tX1ZT07rrGvBFYktK7Kma\n" + + "lqDl4UDCF1UWkiiFubc0Xw+DR6vNAa99E0oaphzvCmITU1wITNnYZTKzVzQ7vUCq\n" + + "kLmXOFLTcxTQpptxSo5xDD3aTpzWGCvjExCKpXQtsITUOYtZc02AGjjPOQ==\n" + + "-----END CERTIFICATE-----\n" + + ""; + + final static String ADMIN_KEY = "-----BEGIN PRIVATE KEY-----\n" + + "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDCwgBOoO88uMM8\n" + + "dREJsk58Yt4Jn0zwQ2wUThbvy3ICDiEWhiAhUbg6dTggpS5vWWJto9bvaaqgMVoh\n" + + "ElfYHdTDncX3UQNBEP8tqzHON6BFEFSGgJRGLd6f5dri6rK32nCotYS61CFXBFxf\n" + + "WumXjSukjyrcTsdkR3C5QDo2oN7F883MOQqRENPzAtZi9s3jNX48u+/e3yvJzXsB\n" + + "GS9Qmsye6C71enbIujM4CVwDT/7a5jHuaUp6OuNCFbdRPnu/wLYwOS2/yOtzAqk7\n" + + "/PFnPCe7YOa10ShnV/jx2sAHhp7ZQBJgFkkgnIERz9Ws74Au+EbptWnsWuB+LqRL\n" + + "x5G02IzpAgMBAAECggEAEzwnMkeBbqqDgyRqFbO/PgMNvD7i0b/28V0dCtCPEVY6\n" + + "klzrg3RCERP5V9AN8VVkppYjPkCzZ2A4b0JpMUu7ncOmr7HCnoSCj2IfEyePSVg+\n" + + "4OHbbcBOAoDTHiI2myM/M9++8izNS34qGV4t6pfjaDyeQQ/5cBVWNBWnKjS34S5H\n" + + "rJWpAcDgxYk5/ah2Xs2aULZlXDMxbSikjrv+n4JIYTKFQo8ydzL8HQDBRmXAFLjC\n" + + "gNOSHf+5u1JdpY3uPIxK1ugVf8zPZ4/OEB23j56uu7c8+sZ+kZwfRWAQmMhFVG/y\n" + + "OXxoT5mOruBsAw29m2Ijtxg252/YzSTxiDqFziB/eQKBgQDjeVAdi55GW/bvhuqn\n" + + "xME/An8E3hI/FyaaITrMQJUBjiCUaStTEqUgQ6A7ZfY/VX6qafOX7sli1svihrXC\n" + + "uelmKrdve/CFEEqzX9JWWRiPiQ0VZD+EQRsJvX85Tw2UGvVUh6dO3UGPS0BhplMD\n" + + "jeVpyXgZ7Gy5we+DWjfwhYrCmwKBgQDbLmQhRy+IdVljObZmv3QtJ0cyxxZETWzU\n" + + "MKmgBFvcRw+KvNwO+Iy0CHEbDu06Uj63kzI2bK3QdINaSrjgr8iftXIQpBmcgMF+\n" + + "a1l5HtHlCp6RWd55nWQOEvn36IGN3cAaQkXuh4UYM7QfEJaAbzJhyJ+wXA3jWqUd\n" + + "8bDTIAZ0ywKBgFuZ44gyTAc7S2JDa0Up90O/ZpT4NFLRqMrSbNIJg7d/m2EIRNkM\n" + + "HhCzCthAg/wXGo3XYq+hCdnSc4ICCzmiEfoBY6LyPvXmjJ5VDOeWs0xBvVIK74T7\n" + + "jr7KX2wdiHNGs9pZUidw89CXVhK8nptEzcheyA1wZowbK68yamph7HHXAoGBAK3x\n" + + "7D9Iyl1mnDEWPT7f1Gh9UpDm1TIRrDvd/tBihTCVKK13YsFy2d+LD5Bk0TpGyUVR\n" + + "STlOGMdloFUJFh4jA3pUOpkgUr8Uo/sbYN+x6Ov3+I3sH5aupRhSURVA7YhUIz/z\n" + + "tqIt5R+m8Nzygi6dkQNvf+Qruk3jw0S3ahizwsvvAoGAL7do6dTLp832wFVxkEf4\n" + + "gg1M6DswfkgML5V/7GQ3MkIX/Hrmiu+qSuHhDGrp9inZdCDDYg5+uy1+2+RBMRZ3\n" + + "vDUUacvc4Fep05zp7NcjgU5y+/HWpuKVvLIlZAO1MBY4Xinqqii6RdxukIhxw7eT\n" + + "C6TPL5KAcV1R/XAihDhI18Y=\n" + + "-----END PRIVATE KEY-----\n" + + ""; +} diff --git a/src/newTest/java/org/opensearch/test/framework/certificate/TestCertificates.java b/src/newTest/java/org/opensearch/test/framework/certificate/TestCertificates.java new file mode 100644 index 0000000000..9ca0aa54e8 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/certificate/TestCertificates.java @@ -0,0 +1,63 @@ +/* + * Copyright 2021 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.opensearch.test.framework.certificate; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Provides TLS certificates required in test cases. + * WIP At the moment the certificates are hard coded. + * This will be replaced by classes + * that can generate certificates on the fly. + */ +public class TestCertificates { + + public File getRootCertificate() throws IOException { + return createTempFile("root", ".cert", Certificates.ROOT_CA_CERTIFICATE); + } + + public File getNodeCertificate(int node) throws IOException { + return createTempFile("node-" + node, ".cert", Certificates.NODE_CERTIFICATE); + } + + public File getNodeKey(int node) throws IOException { + return createTempFile("node-" + node, ".key", Certificates.NODE_KEY); + } + + public File getAdminCertificate() throws IOException { + return createTempFile("admin", ".cert", Certificates.ADMIN_CERTIFICATE); + } + + public File getAdminKey() throws IOException { + return createTempFile("admin", ".key", Certificates.ADMIN_KEY); + } + + public String[] getAdminDNs() throws IOException { + return new String[] {"CN=kirk,OU=client,O=client,L=test,C=de"}; + } + + private File createTempFile(String name, String suffix, String contents) throws IOException { + Path path = Files.createTempFile(name, suffix); + Files.writeString(path, contents); + return path.toFile(); + + } +} diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/ClusterConfiguration.java b/src/newTest/java/org/opensearch/test/framework/cluster/ClusterConfiguration.java new file mode 100644 index 0000000000..08e8eb31c1 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/cluster/ClusterConfiguration.java @@ -0,0 +1,131 @@ +/* + * Copyright 2015-2017 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.cluster; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; + +import org.opensearch.index.reindex.ReindexPlugin; +import org.opensearch.join.ParentJoinPlugin; +import org.opensearch.percolator.PercolatorPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.search.aggregations.matrix.MatrixAggregationPlugin; +import org.opensearch.security.OpenSearchSecurityPlugin; +import org.opensearch.transport.Netty4Plugin; + +import com.google.common.collect.Lists; + +public enum ClusterConfiguration { + //first one needs to be a master + //HUGE(new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(false, true,false), new NodeSettings(false, true, false)), + + //3 nodes (1m, 2d) + DEFAULT(new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true)), + + //1 node (1md) + SINGLENODE(new NodeSettings(true, true)), + + //4 node (1m, 2d, 1c) + CLIENTNODE(new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true), new NodeSettings(false, false)), + + THREE_MASTERS(new NodeSettings(true, false), new NodeSettings(true, false), new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true)); + + private List nodeSettings = new LinkedList<>(); + + private ClusterConfiguration(NodeSettings... settings) { + nodeSettings.addAll(Arrays.asList(settings)); + } + + public List getNodeSettings() { + return Collections.unmodifiableList(nodeSettings); + } + + public List getMasterNodeSettings() { + return Collections.unmodifiableList(nodeSettings.stream().filter(a -> a.masterNode).collect(Collectors.toList())); + } + + public List getNonMasterNodeSettings() { + return Collections.unmodifiableList(nodeSettings.stream().filter(a -> !a.masterNode).collect(Collectors.toList())); + } + + public int getNodes() { + return nodeSettings.size(); + } + + public int getMasterNodes() { + return (int) nodeSettings.stream().filter(a -> a.masterNode).count(); + } + + public int getDataNodes() { + return (int) nodeSettings.stream().filter(a -> a.dataNode).count(); + } + + public int getClientNodes() { + return (int) nodeSettings.stream().filter(a -> !a.masterNode && !a.dataNode).count(); + } + + public static class NodeSettings { + public boolean masterNode; + public boolean dataNode; + public List> plugins = Lists.newArrayList(Netty4Plugin.class, OpenSearchSecurityPlugin.class, MatrixAggregationPlugin.class, + ParentJoinPlugin.class, PercolatorPlugin.class, ReindexPlugin.class); + + public NodeSettings(boolean masterNode, boolean dataNode) { + super(); + this.masterNode = masterNode; + this.dataNode = dataNode; + } + + public NodeSettings(boolean masterNode, boolean dataNode, List> additionalPlugins) { + this(masterNode, dataNode); + + this.plugins.addAll(additionalPlugins); + } + + @SuppressWarnings("unchecked") + public Class[] getPlugins() { + return plugins.toArray(new Class[0]); + } + + @SuppressWarnings("unchecked") + public Class[] getPlugins(List> additionalPlugins) { + List> plugins = new ArrayList<>(this.plugins); + + if (additionalPlugins != null) { + plugins.addAll(additionalPlugins); + } + + return plugins.toArray(new Class[0]); + } + } +} diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/ClusterInfo.java b/src/newTest/java/org/opensearch/test/framework/cluster/ClusterInfo.java new file mode 100644 index 0000000000..e82ee82391 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/cluster/ClusterInfo.java @@ -0,0 +1,45 @@ +/* + * Copyright 2015-2018 _floragunn_ GmbH + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.cluster; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.opensearch.common.transport.TransportAddress; + +// TODO: change to getter/setter +public class ClusterInfo { + public int numNodes; + public String httpHost = null; + public int httpPort = -1; + public Set httpAdresses = new HashSet(); + public String nodeHost; + public int nodePort; + public String clustername; + public List tcpMasterPortsOnly; +} diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java b/src/newTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java new file mode 100644 index 0000000000..76805d1ea6 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java @@ -0,0 +1,60 @@ +package org.opensearch.test.framework.cluster; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.opensearch.action.ActionListener; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionResponse; +import org.opensearch.action.ActionType; +import org.opensearch.action.support.ContextPreservingActionListener; +import org.opensearch.client.Client; +import org.opensearch.client.FilterClient; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.common.util.concurrent.ThreadContext.StoredContext; + +public class ContextHeaderDecoratorClient extends FilterClient { + + private Map headers; + + public ContextHeaderDecoratorClient(Client in, Map headers) { + super(in); + this.headers = headers != null ? headers : Collections.emptyMap(); + } + + public ContextHeaderDecoratorClient(Client in, String... headers) { + this(in, arrayToMap(headers)); + } + + @Override + protected void doExecute(ActionType action, Request request, + ActionListener listener) { + + ThreadContext threadContext = threadPool().getThreadContext(); + ContextPreservingActionListener wrappedListener = new ContextPreservingActionListener<>(threadContext.newRestorableContext(true), listener); + + try (StoredContext ctx = threadContext.stashContext()) { + threadContext.putHeader(this.headers); + super.doExecute(action, request, wrappedListener); + } + } + + private static Map arrayToMap(String[] headers) { + if (headers == null) { + return null; + } + + if (headers.length % 2 != 0) { + throw new IllegalArgumentException("The headers array must consist of key-value pairs"); + } + + Map result = new HashMap<>(headers.length / 2); + + for (int i = 0; i < headers.length; i += 2) { + result.put(headers[i], headers[i + 1]); + } + + return result; + } +} diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java new file mode 100644 index 0000000000..3fddec2f15 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -0,0 +1,373 @@ +/* + * Copyright 2015-2021 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.cluster; + +import java.io.File; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.rules.ExternalResource; +import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; +import org.opensearch.node.PluginAwareNode; +import org.opensearch.plugins.Plugin; +import org.opensearch.security.support.ConfigConstants; +import org.opensearch.test.framework.TestIndex; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.Role; +import org.opensearch.test.framework.TestSecurityConfig.RoleMapping; +import org.opensearch.test.framework.certificate.TestCertificates; + + +public class LocalCluster extends ExternalResource implements AutoCloseable, OpenSearchClientProvider { + + private static final Logger log = LogManager.getLogger(LocalCluster.class); + + static { + System.setProperty("security.default_init.dir", new File("./securityconfig").getAbsolutePath()); + } + + protected static final AtomicLong num = new AtomicLong(); + + private final List> plugins; + private final ClusterConfiguration clusterConfiguration; + private final TestSecurityConfig testSecurityConfig; + private Settings nodeOverride; + private final String clusterName; + private final MinimumSecuritySettingsSupplierFactory minimumOpenSearchSettingsSupplierFactory; + private final TestCertificates testCertificates; + private final List clusterDependencies; + private final Map remotes; + private volatile LocalOpenSearchCluster localOpenSearchCluster; + private final List testIndices; + + private LocalCluster(String clusterName, TestSecurityConfig testSgConfig, Settings nodeOverride, + ClusterConfiguration clusterConfiguration, List> plugins, TestCertificates testCertificates, + List clusterDependencies, Map remotes, List testIndices) { + this.plugins = plugins; + this.testCertificates = testCertificates; + this.clusterConfiguration = clusterConfiguration; + this.testSecurityConfig = testSgConfig; + this.nodeOverride = nodeOverride; + this.clusterName = clusterName; + this.minimumOpenSearchSettingsSupplierFactory = new MinimumSecuritySettingsSupplierFactory(testCertificates); + this.remotes = remotes; + this.clusterDependencies = clusterDependencies; + this.testIndices = testIndices; + } + + @Override + public void before() throws Throwable { + if (localOpenSearchCluster == null) { + for (LocalCluster dependency : clusterDependencies) { + if (!dependency.isStarted()) { + dependency.before(); + } + } + + for (Map.Entry entry : remotes.entrySet()) { + @SuppressWarnings("resource") + InetSocketAddress transportAddress = entry.getValue().localOpenSearchCluster.masterNode().getTransportAddress(); + nodeOverride = Settings.builder().put(nodeOverride) + .putList("cluster.remote." + entry.getKey() + ".seeds", transportAddress.getHostString() + ":" + transportAddress.getPort()) + .build(); + } + + start(); + } + } + + @Override + protected void after() { + if (localOpenSearchCluster != null && localOpenSearchCluster.isStarted()) { + try { + Thread.sleep(1234); + localOpenSearchCluster.destroy(); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + localOpenSearchCluster = null; + } + } + } + + @Override + public void close() { + if (localOpenSearchCluster != null && localOpenSearchCluster.isStarted()) { + try { + Thread.sleep(100); + localOpenSearchCluster.destroy(); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + localOpenSearchCluster = null; + } + } + } + + @Override + public String getClusterName() { + return clusterName; + } + + @Override + public InetSocketAddress getHttpAddress() { + return localOpenSearchCluster.clientNode().getHttpAddress(); + } + + @Override + public InetSocketAddress getTransportAddress() { + return localOpenSearchCluster.clientNode().getTransportAddress(); + } + + public Client getInternalNodeClient() { + return localOpenSearchCluster.clientNode().getInternalNodeClient(); + } + + public PluginAwareNode node() { + return this.localOpenSearchCluster.masterNode().esNode(); + } + + public List nodes() { + return this.localOpenSearchCluster.getAllNodes(); + } + + public LocalOpenSearchCluster.Node getNodeByName(String name) { + return this.localOpenSearchCluster.getNodeByName(name); + } + + public LocalOpenSearchCluster.Node getRandomClientNode() { + return this.localOpenSearchCluster.randomClientNode(); + } + + public boolean isStarted() { + return localOpenSearchCluster != null; + } + + public Random getRandom() { + return localOpenSearchCluster.getRandom(); + } + + private void start() { + try { + localOpenSearchCluster = new LocalOpenSearchCluster(clusterName, clusterConfiguration, + minimumOpenSearchSettingsSupplierFactory.minimumOpenSearchSettings(nodeOverride), plugins, testCertificates); + + localOpenSearchCluster.start(); + + + if (testSecurityConfig != null) { + initSecurityIndex(testSecurityConfig); + } + + try (Client client = getInternalNodeClient()) { + for (TestIndex index : this.testIndices) { + index.create(client); + } + } + + } catch (Exception e) { + log.error("Local ES cluster start failed", e); + throw new RuntimeException(e); + } + } + + private void initSecurityIndex(TestSecurityConfig testSecurityConfig) { + log.info("Initializing OpenSearch Security index"); + Client client = new ContextHeaderDecoratorClient(this.getInternalNodeClient(), Map.of(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER , "true")); + testSecurityConfig.initIndex(client); + } + + public static class Builder { + + private final Settings.Builder nodeOverrideSettingsBuilder = Settings.builder(); + private final List> plugins = new ArrayList<>(); + private Map remoteClusters = new HashMap<>(); + private List clusterDependencies = new ArrayList<>(); + private List testIndices = new ArrayList<>(); + private ClusterConfiguration clusterConfiguration = ClusterConfiguration.DEFAULT; + private TestSecurityConfig testSecurityConfig = new TestSecurityConfig(); + private String clusterName = "local_cluster"; + private TestCertificates testCertificates; + + public Builder() { + this.testCertificates = new TestCertificates(); + } + + public Builder dependsOn(Object object) { + // We just want to make sure that the object is already done + if (object == null) { + throw new IllegalStateException("Dependency not fulfilled"); + } + return this; + } + + public Builder clusterConfiguration(ClusterConfiguration clusterConfiguration) { + this.clusterConfiguration = clusterConfiguration; + return this; + } + + public Builder singleNode() { + this.clusterConfiguration = ClusterConfiguration.SINGLENODE; + return this; + } + + public Builder sgConfig(TestSecurityConfig testSgConfig) { + this.testSecurityConfig = testSgConfig; + return this; + } + + public Builder nodeSettings(Object... settings) { + for (int i = 0; i < settings.length - 1; i += 2) { + String key = String.valueOf(settings[i]); + Object value = settings[i + 1]; + + if (value instanceof List) { + List values = ((List) value).stream().map(String::valueOf).collect(Collectors.toList()); + nodeOverrideSettingsBuilder.putList(key, values); + } else { + nodeOverrideSettingsBuilder.put(key, String.valueOf(value)); + } + } + + return this; + } + + public Builder plugin(Class plugin) { + this.plugins.add(plugin); + + return this; + } + + public Builder remote(String name, LocalCluster anotherCluster) { + remoteClusters.put(name, anotherCluster); + + clusterDependencies.add(anotherCluster); + + return this; + } + + public Builder indices(TestIndex... indices) { + this.testIndices.addAll(Arrays.asList(indices)); + return this; + } + + public Builder users(TestSecurityConfig.User... users) { + for (TestSecurityConfig.User user : users) { + testSecurityConfig.user(user); + } + return this; + } + + public Builder user(TestSecurityConfig.User user) { + testSecurityConfig.user(user); + return this; + } + + public Builder user(String name, String password, String... sgRoles) { + testSecurityConfig.user(name, password, sgRoles); + return this; + } + + public Builder user(String name, String password, Role... sgRoles) { + testSecurityConfig.user(name, password, sgRoles); + return this; + } + + public Builder roles(Role... roles) { + testSecurityConfig.roles(roles); + return this; + } + + public Builder roleMapping(RoleMapping... mappings) { + testSecurityConfig.roleMapping(mappings); + return this; + } + + public Builder roleToRoleMapping(Role role, String... backendRoles) { + testSecurityConfig.roleToRoleMapping(role, backendRoles); + return this; + } + + public Builder authc(TestSecurityConfig.AuthcDomain authc) { + testSecurityConfig.authc(authc); + return this; + } + + public Builder var(String name, Supplier value) { + testSecurityConfig.var(name, value); + return this; + } + + public Builder clusterName(String clusterName) { + this.clusterName = clusterName; + return this; + } + + public Builder configIndexName(String configIndexName) { + testSecurityConfig.configIndexName(configIndexName); + return this; + } + + public Builder anonymousAuth(boolean anonAuthEnabled) { + testSecurityConfig.anonymousAuth(anonAuthEnabled); + return this; + } + + public LocalCluster build() { + try { + + clusterName += "_" + num.incrementAndGet(); + + return new LocalCluster(clusterName, testSecurityConfig, nodeOverrideSettingsBuilder.build(), clusterConfiguration, plugins, + testCertificates, clusterDependencies, remoteClusters, testIndices); + } catch (Exception e) { + log.error("Failed to build LocalCluster", e); + throw new RuntimeException(e); + } + } + + } + + @Override + public TestCertificates getTestCertificates() { + return testCertificates; + } + +} diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java new file mode 100644 index 0000000000..4ae2c8b956 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -0,0 +1,610 @@ +/* + * Copyright 2015-2021 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.cluster; + +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Random; +import java.util.SortedSet; +import java.util.concurrent.CompletableFuture; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchTimeoutException; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.action.admin.cluster.node.info.NodeInfo; +import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.client.AdminClient; +import org.opensearch.client.Client; +import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.common.Strings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.transport.TransportAddress; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.http.BindHttpException; +import org.opensearch.http.HttpInfo; +import org.opensearch.node.PluginAwareNode; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.framework.certificate.TestCertificates; +import org.opensearch.test.framework.cluster.ClusterConfiguration.NodeSettings; +import org.opensearch.transport.BindTransportException; +import org.opensearch.transport.TransportInfo; + +import com.google.common.net.InetAddresses; + + +public class LocalOpenSearchCluster { + + static { + System.setProperty("opensearch.enforce.bootstrap.checks", "true"); + } + + private static final Logger log = LogManager.getLogger(LocalOpenSearchCluster.class); + + private final String clusterName; + private final ClusterConfiguration clusterConfiguration; + private final NodeSettingsSupplier nodeSettingsSupplier; + private final List> additionalPlugins; + private final List allNodes = new ArrayList<>(); + private final List masterNodes = new ArrayList<>(); + private final List dataNodes = new ArrayList<>(); + private final List clientNodes = new ArrayList<>(); + private final TestCertificates testCertificates; + + private File clusterHomeDir; + private List seedHosts; + private List initialMasterHosts; + private int retry = 0; + private boolean started; + private Random random = new Random(); + + public LocalOpenSearchCluster(String clusterName, ClusterConfiguration clusterConfiguration, NodeSettingsSupplier nodeSettingsSupplier, + List> additionalPlugins, TestCertificates testCertificates) { + this.clusterName = clusterName; + this.clusterConfiguration = clusterConfiguration; + this.nodeSettingsSupplier = nodeSettingsSupplier; + this.additionalPlugins = additionalPlugins; + this.testCertificates = testCertificates; + try { + this.clusterHomeDir = Files.createTempDirectory("local_cluster_" + clusterName).toFile(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + public void start() throws Exception { + log.info("Starting {}", clusterName); + + int forkNumber = getUnitTestForkNumber(); + int masterNodeCount = clusterConfiguration.getMasterNodes(); + int nonMasterNodeCount = clusterConfiguration.getDataNodes() + clusterConfiguration.getClientNodes(); + + SortedSet masterNodeTransportPorts = PortAllocator.TCP.allocate(clusterName, Math.max(masterNodeCount, 4), 5000 + forkNumber * 1000 + 300); + SortedSet masterNodeHttpPorts = PortAllocator.TCP.allocate(clusterName, masterNodeCount, 5000 + forkNumber * 1000 + 200); + + this.seedHosts = toHostList(masterNodeTransportPorts); + this.initialMasterHosts = toHostList(masterNodeTransportPorts.stream().limit(masterNodeCount).collect(Collectors.toSet())); + + started = true; + + CompletableFuture masterNodeFuture = startNodes(clusterConfiguration.getMasterNodeSettings(), masterNodeTransportPorts, + masterNodeHttpPorts); + + SortedSet nonMasterNodeTransportPorts = PortAllocator.TCP.allocate(clusterName, nonMasterNodeCount, 5000 + forkNumber * 1000 + 310); + SortedSet nonMasterNodeHttpPorts = PortAllocator.TCP.allocate(clusterName, nonMasterNodeCount, 5000 + forkNumber * 1000 + 210); + + CompletableFuture nonMasterNodeFuture = startNodes(clusterConfiguration.getNonMasterNodeSettings(), nonMasterNodeTransportPorts, + nonMasterNodeHttpPorts); + + CompletableFuture.allOf(masterNodeFuture, nonMasterNodeFuture).join(); + + if (isNodeFailedWithPortCollision()) { + log.info("Detected port collision for master node. Retrying."); + + retry(); + return; + } + + log.info("Startup finished. Waiting for GREEN"); + + waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(10), allNodes.size()); + putDefaultTemplate(); + + log.info("Started: {}", this); + + } + + public String getClusterName() { + return clusterName; + } + + public boolean isStarted() { + return started; + } + + public void stop() { + + for (Node node : clientNodes) { + node.stop(); + } + + for (Node node : dataNodes) { + node.stop(); + } + + for (Node node : masterNodes) { + node.stop(); + } + } + + public void destroy() { + stop(); + clientNodes.clear(); + dataNodes.clear(); + masterNodes.clear(); + + try { + FileUtils.deleteDirectory(clusterHomeDir); + } catch (IOException e) { + log.warn("Error while deleting " + clusterHomeDir, e); + } + } + + public Node clientNode() { + return findRunningNode(clientNodes, dataNodes, masterNodes); + } + + public Node randomClientNode() { + return randomRunningNode(clientNodes, dataNodes, masterNodes); + } + + public Node masterNode() { + return findRunningNode(masterNodes); + } + + public List getAllNodes() { + return Collections.unmodifiableList(allNodes); + } + + public Node getNodeByName(String name) { + return allNodes.stream().filter(node -> node.getNodeName().equals(name)).findAny().orElseThrow(() -> new RuntimeException( + "No such node with name: " + name + "; available: " + allNodes.stream().map(Node::getNodeName).collect(Collectors.toList()))); + } + + private boolean isNodeFailedWithPortCollision() { + return allNodes.stream().anyMatch(Node::isPortCollision); + } + + private void retry() throws Exception { + retry++; + + if (retry > 10) { + throw new RuntimeException("Detected port collisions for master node. Giving up."); + } + + stop(); + + this.allNodes.clear(); + this.masterNodes.clear(); + this.dataNodes.clear(); + this.clientNodes.clear(); + this.seedHosts = null; + this.initialMasterHosts = null; + this.clusterHomeDir = Files.createTempDirectory("local_cluster_" + clusterName + "_retry_" + retry).toFile(); + + start(); + } + + @SafeVarargs + private final Node findRunningNode(List nodes, List... moreNodes) { + for (Node node : nodes) { + if (node.isRunning()) { + return node; + } + } + + if (moreNodes != null && moreNodes.length > 0) { + for (List nodesList : moreNodes) { + for (Node node : nodesList) { + if (node.isRunning()) { + return node; + } + } + } + } + + return null; + } + + @SafeVarargs + private final Node randomRunningNode(List nodes, List... moreNodes) { + ArrayList runningNodes = new ArrayList<>(); + + for (Node node : nodes) { + if (node.isRunning()) { + runningNodes.add(node); + } + } + + if (moreNodes != null && moreNodes.length > 0) { + for (List nodesList : moreNodes) { + for (Node node : nodesList) { + if (node.isRunning()) { + runningNodes.add(node); + } + } + } + } + + if (runningNodes.size() == 0) { + return null; + } + + int index = this.random.nextInt(runningNodes.size()); + + return runningNodes.get(index); + } + + private void putDefaultTemplate() { + String defaultTemplate = "{\n" + " \"index_patterns\": [\"*\"],\n" + " \"order\": -1,\n" + " \"settings\": {\n" + + " \"number_of_shards\": \"5\",\n" + " \"number_of_replicas\": \"1\"\n" + " }\n" // + + " }"; + + AcknowledgedResponse templateAck = clientNode().getInternalNodeClient().admin().indices() + .putTemplate(new PutIndexTemplateRequest("default").source(defaultTemplate, XContentType.JSON)).actionGet(); + + if (!templateAck.isAcknowledged()) { + throw new RuntimeException("Default template could not be created"); + } + } + + private CompletableFuture startNodes(List nodeSettingList, SortedSet transportPorts, SortedSet httpPorts) { + Iterator transportPortIterator = transportPorts.iterator(); + Iterator httpPortIterator = httpPorts.iterator(); + List> futures = new ArrayList<>(); + + for (NodeSettings nodeSettings : nodeSettingList) { + Node node = new Node(nodeSettings, transportPortIterator.next(), httpPortIterator.next()); + futures.add(node.start()); + } + + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + } + + public ClusterInfo waitForCluster(ClusterHealthStatus status, TimeValue timeout, int expectedNodeCount) throws IOException { + ClusterInfo clusterInfo = new ClusterInfo(); + Client client = clientNode().getInternalNodeClient(); + + try { + log.debug("waiting for cluster state {} and {} nodes", status.name(), expectedNodeCount); + AdminClient adminClient = client.admin(); + + final ClusterHealthResponse healthResponse = adminClient.cluster().prepareHealth().setWaitForStatus(status).setTimeout(timeout) + .setMasterNodeTimeout(timeout).setWaitForNodes("" + expectedNodeCount).execute().actionGet(); + + if (log.isDebugEnabled()) { + log.debug("Current ClusterState:\n{}", Strings.toString(healthResponse)); + } + + if (healthResponse.isTimedOut()) { + throw new IOException( + "cluster state is " + healthResponse.getStatus().name() + " with " + healthResponse.getNumberOfNodes() + " nodes"); + } else { + log.debug("... cluster state ok {} with {} nodes", healthResponse.getStatus().name(), healthResponse.getNumberOfNodes()); + } + + assertEquals(expectedNodeCount, healthResponse.getNumberOfNodes()); + + final NodesInfoResponse nodesInfoResponse = adminClient.cluster().nodesInfo(new NodesInfoRequest()).actionGet(); + + final List nodes = nodesInfoResponse.getNodes(); + + final List masterNodes = nodes.stream() + .filter(nodeInfo -> nodeInfo.getNode().getRoles().contains(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE)).collect(Collectors.toList()); + + final List dataNodes = nodes.stream().filter(nodeInfo -> nodeInfo.getNode().getRoles().contains(DiscoveryNodeRole.DATA_ROLE) + && !nodeInfo.getNode().getRoles().contains(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE)).collect(Collectors.toList()); + + final List clientNodes = nodes.stream() + .filter(nodeInfo -> !nodeInfo.getNode().getRoles().contains(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE) + && !nodeInfo.getNode().getRoles().contains(DiscoveryNodeRole.DATA_ROLE)) + .collect(Collectors.toList()); + + for (NodeInfo nodeInfo : masterNodes) { + final TransportAddress is = nodeInfo.getInfo(TransportInfo.class).getAddress().publishAddress(); + clusterInfo.nodePort = is.getPort(); + clusterInfo.nodeHost = is.getAddress(); + } + + Predicate hasHttpAddressPredicate = nodeInfo -> nodeInfo.getInfo(HttpInfo.class) != null + && nodeInfo.getInfo(HttpInfo.class).address() != null; + + if (!clientNodes.isEmpty()) { + NodeInfo nodeInfo = clientNodes.get(0); + if (hasHttpAddressPredicate.test(nodeInfo)) { + final TransportAddress his = nodeInfo.getInfo(HttpInfo.class).address().publishAddress(); + clusterInfo.httpPort = his.getPort(); + clusterInfo.httpHost = his.getAddress(); + } else { + throw new RuntimeException("no http host/port for client node"); + } + } else if (!dataNodes.isEmpty()) { + for (NodeInfo nodeInfo : dataNodes) { + if (hasHttpAddressPredicate.test(nodeInfo)) { + final TransportAddress his = nodeInfo.getInfo(HttpInfo.class).address().publishAddress(); + clusterInfo.httpPort = his.getPort(); + clusterInfo.httpHost = his.getAddress(); + break; + } + } + } else { + for (NodeInfo nodeInfo : nodes) { + if (hasHttpAddressPredicate.test(nodeInfo)) { + final TransportAddress his = nodeInfo.getInfo(HttpInfo.class).address().publishAddress(); + clusterInfo.httpPort = his.getPort(); + clusterInfo.httpHost = his.getAddress(); + break; + } + } + } + + for (NodeInfo nodeInfo : nodes) { + clusterInfo.httpAdresses.add(nodeInfo.getInfo(HttpInfo.class).address().publishAddress()); + } + } catch (final OpenSearchTimeoutException e) { + throw new IOException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations"); + } + return clusterInfo; + } + + @Override + public String toString() { + return "\nES Cluster " + clusterName + "\nmaster nodes: " + masterNodes + "\n data nodes: " + dataNodes + "\nclient nodes: " + clientNodes + + "\n"; + } + + private static List toHostList(Collection ports) { + return ports.stream().map(port -> "127.0.0.1:" + port).collect(Collectors.toList()); + } + + private String createNextNodeName(NodeSettings nodeSettings) { + List nodes; + String nodeType; + + if (nodeSettings.masterNode) { + nodes = this.masterNodes; + nodeType = "master"; + } else if (nodeSettings.dataNode) { + nodes = this.dataNodes; + nodeType = "data"; + } else { + nodes = this.clientNodes; + nodeType = "client"; + } + + return nodeType + "_" + nodes.size(); + } + + public class Node implements OpenSearchClientProvider { + private final String nodeName; + private final NodeSettings nodeSettings; + private final File nodeHomeDir; + private final File dataDir; + private final File logsDir; + private final int transportPort; + private final int httpPort; + private final InetSocketAddress httpAddress; + private final InetSocketAddress transportAddress; + private PluginAwareNode node; + private boolean running = false; + private boolean portCollision = false; + + Node(NodeSettings nodeSettings, int transportPort, int httpPort) { + this.nodeName = createNextNodeName(nodeSettings); + this.nodeSettings = nodeSettings; + this.nodeHomeDir = new File(clusterHomeDir, nodeName); + this.dataDir = new File(this.nodeHomeDir, "data"); + this.logsDir = new File(this.nodeHomeDir, "logs"); + this.transportPort = transportPort; + this.httpPort = httpPort; + InetAddress hostAddress = InetAddresses.forString("127.0.0.1"); + this.httpAddress = new InetSocketAddress(hostAddress, httpPort); + this.transportAddress = new InetSocketAddress(hostAddress, transportPort); + + if (nodeSettings.masterNode) { + masterNodes.add(this); + } else if (nodeSettings.dataNode) { + dataNodes.add(this); + } else { + clientNodes.add(this); + } + allNodes.add(this); + } + + CompletableFuture start() { + CompletableFuture completableFuture = new CompletableFuture<>(); + + this.node = new PluginAwareNode(nodeSettings.masterNode, getEsSettings(), nodeSettings.getPlugins(additionalPlugins)); + + new Thread(new Runnable() { + + @Override + public void run() { + try { + node.start(); + running = true; + completableFuture.complete("initialized"); + } catch (BindTransportException | BindHttpException e) { + log.warn("Port collision detected for {}", this, e); + portCollision = true; + try { + node.close(); + } catch (IOException e1) { + log.error(e1); + } + + node = null; + PortAllocator.TCP.reserve(transportPort, httpPort); + + completableFuture.complete("retry"); + + } catch (Throwable e) { + log.error("Unable to start {}", this, e); + node = null; + completableFuture.completeExceptionally(e); + } + } + }).start(); + + return completableFuture; + } + + public Client getInternalNodeClient() { + return node.client(); + } + + public PluginAwareNode esNode() { + return node; + } + + public boolean isRunning() { + return running; + } + + public X getInjectable(Class clazz) { + return node.injector().getInstance(clazz); + } + + public void stop() { + try { + log.info("Stopping {}", this); + + running = false; + + if (node != null) { + node.close(); + node = null; + Thread.sleep(10); + } + + } catch (Throwable e) { + log.warn("Error while stopping " + this, e); + } + } + + @Override + public String toString() { + String state = running ? "RUNNING" : node != null ? "INITIALIZING" : "STOPPED"; + + return nodeName + " " + state + " [" + transportPort + ", " + httpPort + "]"; + } + + public boolean isPortCollision() { + return portCollision; + } + + public String getNodeName() { + return nodeName; + } + + @Override + public InetSocketAddress getHttpAddress() { + return httpAddress; + } + + @Override + public InetSocketAddress getTransportAddress() { + return transportAddress; + } + + private Settings getEsSettings() { + Settings settings = getMinimalEsSettings(); + + if (nodeSettingsSupplier != null) { + // TODO node number + return Settings.builder().put(settings).put(nodeSettingsSupplier.get(0)).build(); + } + + return settings; + } + + private Settings getMinimalEsSettings() { + return Settings.builder().put("node.name", nodeName).put("node.data", nodeSettings.dataNode).put("node.master", nodeSettings.masterNode) + .put("cluster.name", clusterName).put("path.home", nodeHomeDir.toPath()).put("path.data", dataDir.toPath()) + .put("path.logs", logsDir.toPath()).putList("cluster.initial_master_nodes", initialMasterHosts) + .put("discovery.initial_state_timeout", "8s").putList("discovery.seed_hosts", seedHosts).put("transport.tcp.port", transportPort) + .put("http.port", httpPort).put("cluster.routing.allocation.disk.threshold_enabled", false) + .put("discovery.probe.connect_timeout", "10s").put("discovery.probe.handshake_timeout", "10s").put("http.cors.enabled", true) + .build(); + } + + @Override + public String getClusterName() { + return clusterName; + } + + @Override + public TestCertificates getTestCertificates() { + return testCertificates; + } + } + + private static int getUnitTestForkNumber() { + String forkno = System.getProperty("forkno"); + + if (forkno != null && forkno.length() > 0) { + return Integer.parseInt(forkno.split("_")[1]); + } else { + return 42; + } + } + + public Random getRandom() { + return random; + } + +} diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java b/src/newTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java new file mode 100644 index 0000000000..37728c14cc --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java @@ -0,0 +1,80 @@ +/* + * Copyright 2021 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.cluster; + +import java.io.IOException; + +import org.opensearch.common.settings.Settings; +import org.opensearch.test.framework.certificate.TestCertificates; + +public class MinimumSecuritySettingsSupplierFactory { + + private TestCertificates testCertificates; + + public MinimumSecuritySettingsSupplierFactory(TestCertificates testCertificates) { + if (testCertificates == null) { + throw new IllegalArgumentException("certificates must not be null"); + } + this.testCertificates = testCertificates; + + } + + public NodeSettingsSupplier minimumOpenSearchSettings(Settings other) { + return i -> minimumOpenSearchSettingsBuilder(i, false).put(other).build(); + } + + public NodeSettingsSupplier minimumOpenSearchSettingsSslOnly(Settings other) { + return i -> minimumOpenSearchSettingsBuilder(i, true).put(other).build(); + } + + private Settings.Builder minimumOpenSearchSettingsBuilder(int node, boolean sslOnly) { + + Settings.Builder builder = Settings.builder(); + + // TODO: At the moment the test node certificates have an OID set, so we do not need to + // specify any node_dns here. Once we make generating and specifying + try { + builder.put("plugins.security.ssl.transport.pemtrustedcas_filepath", testCertificates.getRootCertificate().getAbsolutePath()); + builder.put("plugins.security.ssl.transport.pemcert_filepath", testCertificates.getNodeCertificate(node).getAbsolutePath()); + builder.put("plugins.security.ssl.transport.pemkey_filepath", testCertificates.getNodeKey(node).getAbsolutePath()); + + builder.put("plugins.security.ssl.http.enabled", true); + builder.put("plugins.security.ssl.http.pemtrustedcas_filepath", testCertificates.getRootCertificate().getAbsolutePath()); + builder.put("plugins.security.ssl.http.pemcert_filepath", testCertificates.getNodeCertificate(node).getAbsolutePath()); + builder.put("plugins.security.ssl.http.pemkey_filepath", testCertificates.getNodeKey(node).getAbsolutePath()); + + builder.putList("plugins.security.authcz.admin_dn", testCertificates.getAdminDNs()); + } catch (IOException e) { + throw new IllegalArgumentException("Invalid test certificates provided on local cluster start"); + } + + return builder; + + } +} diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java b/src/newTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java new file mode 100644 index 0000000000..0ed52e203f --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java @@ -0,0 +1,518 @@ +/* + * Copyright 2021-2022 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.cluster; + +import java.io.File; +import java.lang.reflect.Array; +import java.net.URI; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import com.google.common.collect.MapMaker; + +public class NestedValueMap extends HashMap { + + private static final long serialVersionUID = 2953312818482932741L; + + private Map originalToCloneMap; + private final boolean cloneWhilePut; + private boolean writable = true; + + public NestedValueMap() { + originalToCloneMap = new MapMaker().weakKeys().makeMap(); + cloneWhilePut = true; + } + + public NestedValueMap(int initialCapacity) { + super(initialCapacity); + originalToCloneMap = new MapMaker().weakKeys().makeMap(); + cloneWhilePut = true; + } + + NestedValueMap(Map originalToCloneMap, boolean cloneWhilePut) { + this.originalToCloneMap = originalToCloneMap; + this.cloneWhilePut = cloneWhilePut; + } + + NestedValueMap(int initialCapacity, Map originalToCloneMap, boolean cloneWhilePut) { + super(initialCapacity); + this.originalToCloneMap = originalToCloneMap; + this.cloneWhilePut = cloneWhilePut; + } + + @Override + public NestedValueMap clone() { + NestedValueMap result = new NestedValueMap(Math.max(this.size(), 10), + this.originalToCloneMap != null ? new MapMaker().weakKeys().makeMap() : null, this.cloneWhilePut); + + result.putAll(this); + + return result; + } + + public NestedValueMap without(String... keys) { + NestedValueMap result = new NestedValueMap(Math.max(this.size(), 10), + this.originalToCloneMap != null ? new MapMaker().weakKeys().makeMap() : null, this.cloneWhilePut); + + Set withoutKeySet = new HashSet<>(Arrays.asList(keys)); + + for (Map.Entry entry : this.entrySet()) { + if (!withoutKeySet.contains(entry.getKey())) { + result.put(entry.getKey(), entry.getValue()); + } + } + + return result; + } + + public static NestedValueMap copy(Map data) { + NestedValueMap result = new NestedValueMap(data.size()); + + result.putAllFromAnyMap(data); + + return result; + } + + public static NestedValueMap copy(Object data) { + if (data instanceof Map) { + return copy((Map) data); + } else { + NestedValueMap result = new NestedValueMap(); + result.put("_value", data); + return result; + } + } + + public static NestedValueMap createNonCloningMap() { + return new NestedValueMap(null, false); + } + + public static NestedValueMap createUnmodifieableMap(Map data) { + NestedValueMap result = new NestedValueMap(data.size()); + + result.putAllFromAnyMap(data); + result.seal(); + + return result; + } + +// public static NestedValueMap fromJsonString(String jsonString) throws IOException, DocumentParseException, UnexpectedDocumentStructureException { +// return NestedValueMap.copy(DocReader.json().readObject(jsonString)); +// } +// +// public static NestedValueMap fromYaml(String yamlString) throws IOException, DocumentParseException { +// return NestedValueMap.copy(DocReader.yaml().read(yamlString)); +// } +// +// public static NestedValueMap fromYaml(InputStream inputSteam) throws DocumentParseException, IOException { +// return NestedValueMap.copy(DocReader.yaml().read(inputSteam)); +// } + + public static NestedValueMap of(String key1, Object value1) { + NestedValueMap result = new NestedValueMap(1); + result.put(key1, value1); + return result; + } + + public static NestedValueMap of(String key1, Object value1, String key2, Object value2) { + NestedValueMap result = new NestedValueMap(2); + result.put(key1, value1); + result.put(key2, value2); + return result; + } + + public static NestedValueMap of(String key1, Object value1, String key2, Object value2, String key3, Object value3) { + NestedValueMap result = new NestedValueMap(3); + result.put(key1, value1); + result.put(key2, value2); + result.put(key3, value3); + + return result; + } + + public static NestedValueMap of(String key1, Object value1, String key2, Object value2, String key3, Object value3, Object... furtherEntries) { + NestedValueMap result = new NestedValueMap(3 + furtherEntries.length); + result.put(key1, value1); + result.put(key2, value2); + result.put(key3, value3); + + for (int i = 0; i < furtherEntries.length - 1; i += 2) { + result.put(String.valueOf(furtherEntries[i]), furtherEntries[i + 1]); + } + + return result; + } + + public static NestedValueMap of(Path key1, Object value1) { + NestedValueMap result = new NestedValueMap(1); + result.put(key1, value1); + return result; + } + + public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2) { + NestedValueMap result = new NestedValueMap(2); + result.put(key1, value1); + result.put(key2, value2); + return result; + } + + public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2, Path key3, Object value3) { + NestedValueMap result = new NestedValueMap(3); + result.put(key1, value1); + result.put(key2, value2); + result.put(key3, value3); + + return result; + } + + public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2, Path key3, Object value3, Object... furtherEntries) { + NestedValueMap result = new NestedValueMap(3 + furtherEntries.length); + result.put(key1, value1); + result.put(key2, value2); + result.put(key3, value3); + + for (int i = 0; i < furtherEntries.length - 1; i += 2) { + result.put(Path.parse(String.valueOf(furtherEntries[i])), furtherEntries[i + 1]); + } + + return result; + } + + public Object put(String key, Map data) { + checkWritable(); + + Object result = this.get(key); + NestedValueMap subMap = this.getOrCreateSubMapAt(key, data.size()); + + subMap.putAllFromAnyMap(data); + return result; + } + + public void putAll(Map map) { + checkWritable(); + + for (Map.Entry entry : map.entrySet()) { + String key = String.valueOf(entry.getKey()); + put(key, entry.getValue()); + } + } + + public void putAllFromAnyMap(Map map) { + checkWritable(); + + for (Map.Entry entry : map.entrySet()) { + String key = String.valueOf(entry.getKey()); + put(key, entry.getValue()); + } + } + + public void overrideLeafs(NestedValueMap map) { + checkWritable(); + + for (Map.Entry entry : map.entrySet()) { + String key = String.valueOf(entry.getKey()); + + if (entry.getValue() instanceof NestedValueMap) { + NestedValueMap subMap = (NestedValueMap) entry.getValue(); + + getOrCreateSubMapAt(key, subMap.size()).overrideLeafs(subMap); + } else { + put(key, entry.getValue()); + } + } + } + + public Object put(String key, Object object) { + checkWritable(); + + if (object instanceof Map) { + return put(key, (Map) object); + } + + return super.put(key, deepCloneObject(object)); + } + + public void put(Path path, Object object) { + checkWritable(); + + if (path.isEmpty()) { + if (object instanceof Map) { + putAllFromAnyMap((Map) object); + } else { + throw new IllegalArgumentException("put([], " + object + "): If an empty path is given, the object must be of type map"); + } + + } else { + NestedValueMap subMap = getOrCreateSubMapAtPath(path.withoutLast()); + subMap.put(path.getLast(), object); + } + } + + public Object get(Path path) { + if (path.isEmpty()) { + return this; + } else if (path.length() == 1) { + return this.get(path.getFirst()); + } else { + Object subObject = this.get(path.getFirst()); + + if (subObject instanceof NestedValueMap) { + return ((NestedValueMap) subObject).get(path.withoutFirst()); + } else { + return null; + } + } + } + + public void seal() { + if (!this.writable) { + return; + } + + this.writable = false; + this.originalToCloneMap = null; + + for (Object value : this.values()) { + if (value instanceof NestedValueMap) { + NestedValueMap subMap = (NestedValueMap) value; + subMap.seal(); + } else if (value instanceof Iterable) { + for (Object subValue : ((Iterable) value)) { + if (subValue instanceof NestedValueMap) { + NestedValueMap subMap = (NestedValueMap) subValue; + subMap.seal(); + } + } + } + } + } + +// public String toJsonString() { +// return DocWriter.json().writeAsString(this); +// } +// +// public String toYamlString() { +// return DocWriter.yaml().writeAsString(this); +// } + + private Object deepCloneObject(Object object) { + if (!cloneWhilePut || object == null || isImmutable(object)) { + return object; + } + + Object clone = this.originalToCloneMap.get(object); + + if (clone != null) { + return clone; + } + + if (object instanceof Set) { + Set set = (Set) object; + Set copy = new HashSet<>(set.size()); + this.originalToCloneMap.put(object, copy); + + for (Object element : set) { + copy.add(deepCloneObject(element)); + } + + return copy; + } else if (object instanceof Map) { + Map map = (Map) object; + NestedValueMap copy = new NestedValueMap(map.size(), this.originalToCloneMap, this.cloneWhilePut); + this.originalToCloneMap.put(object, copy); + + for (Map.Entry entry : map.entrySet()) { + copy.put((String) deepCloneObject(String.valueOf(entry.getKey())), deepCloneObject(entry.getValue())); + } + + return copy; + } else if (object instanceof Collection) { + Collection collection = (Collection) object; + ArrayList copy = new ArrayList<>(collection.size()); + this.originalToCloneMap.put(object, copy); + + for (Object element : collection) { + copy.add(deepCloneObject(element)); + } + + return copy; + } else if (object.getClass().isArray()) { + int length = Array.getLength(object); + Object copy = Array.newInstance(object.getClass().getComponentType(), length); + this.originalToCloneMap.put(object, copy); + + for (int i = 0; i < length; i++) { + Array.set(copy, i, deepCloneObject(Array.get(object, i))); + } + + return copy; + } else { + // Hope the best + + return object; + } + } + + private boolean isImmutable(Object object) { + return object instanceof String || object instanceof Number || object instanceof Boolean || object instanceof Void || object instanceof Class + || object instanceof Character || object instanceof Enum || object instanceof File || object instanceof UUID || object instanceof URL + || object instanceof URI; + } + + private NestedValueMap getOrCreateSubMapAt(String key, int capacity) { + Object value = this.get(key); + + if (value instanceof NestedValueMap) { + return (NestedValueMap) value; + } else { + if (value instanceof Map) { + capacity = Math.max(capacity, ((Map) value).size()); + } + + NestedValueMap mapValue = new NestedValueMap(capacity, this.originalToCloneMap, this.cloneWhilePut); + + if (value instanceof Map) { + mapValue.putAllFromAnyMap((Map) value); + } + + super.put(key, mapValue); + return mapValue; + } + + } + + private NestedValueMap getOrCreateSubMapAtPath(Path path) { + if (path.isEmpty()) { + return this; + } + + String pathElement = path.getFirst(); + Path remainingPath = path.withoutFirst(); + + Object value = this.get(pathElement); + + if (value instanceof NestedValueMap) { + NestedValueMap mapValue = (NestedValueMap) value; + if (remainingPath.isEmpty()) { + return mapValue; + } else { + return mapValue.getOrCreateSubMapAtPath(remainingPath); + } + } else { + NestedValueMap mapValue = new NestedValueMap(this.originalToCloneMap, this.cloneWhilePut); + super.put(pathElement, mapValue); + + if (remainingPath.isEmpty()) { + return mapValue; + } else { + return mapValue.getOrCreateSubMapAtPath(remainingPath); + } + } + } + + private void checkWritable() { + if (!writable) { + throw new UnsupportedOperationException("Map is not writable"); + } + } + + public static class Path { + private String[] elements; + private int start; + private int end; + + public Path(String... elements) { + this.elements = elements; + this.start = 0; + this.end = elements.length; + } + + private Path(String[] elements, int start, int end) { + this.elements = elements; + this.start = start; + this.end = end; + } + + public String getFirst() { + if (this.start >= this.end) { + return null; + } + + return this.elements[start]; + } + + public String getLast() { + if (this.start >= this.end) { + return null; + } + + return this.elements[end - 1]; + } + + public Path withoutFirst() { + if (this.start >= this.end - 1) { + return new Path(null, 0, 0); + } + + return new Path(elements, start + 1, end); + } + + public Path withoutLast() { + if (this.start >= this.end - 1) { + return new Path(null, 0, 0); + } + + return new Path(elements, start, end - 1); + } + + public int length() { + return this.end - this.start; + } + + public boolean isEmpty() { + return this.start == this.end; + } + + public static Path parse(String path) { + if (path.length() == 0) { + return new Path(new String [0]); + } else { + return new Path(path.split("\\.")); + } + } + } + +} diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java b/src/newTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java new file mode 100644 index 0000000000..828eb0f05a --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java @@ -0,0 +1,34 @@ +/* + * Copyright 2015-2018 _floragunn_ GmbH + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.cluster; + +import org.opensearch.common.settings.Settings; + +@FunctionalInterface +public interface NodeSettingsSupplier { + Settings get(int i); +} diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java b/src/newTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java new file mode 100644 index 0000000000..c98d052c57 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java @@ -0,0 +1,133 @@ +/* + * Copyright 2020 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.cluster; + +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.security.KeyStore; +import java.security.cert.X509Certificate; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; + +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; +import org.opensearch.security.support.PemKeyReader; +import org.opensearch.test.framework.certificate.TestCertificates; + + +public interface OpenSearchClientProvider { + + String getClusterName(); + + TestCertificates getTestCertificates(); + + InetSocketAddress getHttpAddress(); + + InetSocketAddress getTransportAddress(); + + default URI getHttpAddressAsURI() { + InetSocketAddress address = getHttpAddress(); + return URI.create("https://" + address.getHostString() + ":" + address.getPort()); + } + + default TestRestClient getRestClient(UserCredentialsHolder user, Header... headers) { + return getRestClient(user.getName(), user.getPassword(), headers); + } + + default TestRestClient getRestClient(String user, String password, String tenant) { + return getRestClient(user, password, new BasicHeader("sgtenant", tenant)); + } + + default TestRestClient getRestClient(String user, String password, Header... headers) { + BasicHeader basicAuthHeader = getBasicAuthHeader(user, password); + if (headers != null && headers.length > 0) { + List
concatenatedHeaders = Stream.concat(Stream.of(basicAuthHeader), Stream.of(headers)).collect(Collectors.toList()); + return getRestClient(concatenatedHeaders); + } + return getRestClient(basicAuthHeader); + } + + default TestRestClient getRestClient(Header... headers) { + return getRestClient(Arrays.asList(headers)); + } + + default TestRestClient getRestClient(List
headers) { + return createGenericClientRestClient(headers); + } + + default TestRestClient createGenericClientRestClient(List
headers) { + return new TestRestClient(getHttpAddress(), headers, getSSLContext()); + } + + default BasicHeader getBasicAuthHeader(String user, String password) { + return new BasicHeader("Authorization", + "Basic " + Base64.getEncoder().encodeToString((user + ":" + Objects.requireNonNull(password)).getBytes(StandardCharsets.UTF_8))); + } + + private SSLContext getSSLContext() { + X509Certificate[] trustCertificates; + + try { + trustCertificates = PemKeyReader.loadCertificatesFromFile(getTestCertificates().getRootCertificate() ); + + TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); + + ks.load(null); + + for (int i = 0; i < trustCertificates.length; i++) { + ks.setCertificateEntry("caCert-" + i, trustCertificates[i]); + } + + tmf.init(ks); + + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, tmf.getTrustManagers(), null); + return sslContext; + + } catch (Exception e) { + throw new RuntimeException("Error loading root CA ", e); + } + } + + public interface UserCredentialsHolder { + String getName(); + String getPassword(); + } + +} diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/PortAllocator.java b/src/newTest/java/org/opensearch/test/framework/cluster/PortAllocator.java new file mode 100644 index 0000000000..04aba5211d --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/cluster/PortAllocator.java @@ -0,0 +1,157 @@ +/* + * Copyright 2021 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.cluster; + +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.opensearch.test.framework.cluster.SocketUtils.SocketType; + + +public class PortAllocator { + + public static final PortAllocator TCP = new PortAllocator(SocketType.TCP, Duration.ofSeconds(100)); + public static final PortAllocator UDP = new PortAllocator(SocketType.UDP, Duration.ofSeconds(100)); + + private final SocketType socketType; + private final Duration timeoutDuration; + private final Map allocatedPorts = new HashMap<>(); + + PortAllocator(SocketType socketType, Duration timeoutDuration) { + this.socketType = socketType; + this.timeoutDuration = timeoutDuration; + } + + public SortedSet allocate(String clientName, int numRequested, int minPort) { + + int startPort = minPort; + + while (!isAvailable(startPort)) { + startPort += 10; + } + + SortedSet foundPorts = new TreeSet<>(); + + for (int currentPort = startPort; foundPorts.size() < numRequested && currentPort < SocketUtils.PORT_RANGE_MAX + && (currentPort - startPort) < 10000; currentPort++) { + if (allocate(clientName, currentPort)) { + foundPorts.add(currentPort); + } + } + + if (foundPorts.size() < numRequested) { + throw new IllegalStateException("Could not find " + numRequested + " free ports starting at " + minPort + " for " + clientName); + } + + return foundPorts; + } + + public int allocateSingle(String clientName, int minPort) { + + int startPort = minPort; + + for (int currentPort = startPort; currentPort < SocketUtils.PORT_RANGE_MAX && (currentPort - startPort) < 10000; currentPort++) { + if (allocate(clientName, currentPort)) { + return currentPort; + } + } + + throw new IllegalStateException("Could not find free port starting at " + minPort + " for " + clientName); + + } + + public void reserve(int... ports) { + + for (int port : ports) { + allocate("reserved", port); + } + } + + private boolean isInUse(int port) { + boolean result = !this.socketType.isPortAvailable(port); + + if (result) { + synchronized (this) { + allocatedPorts.put(port, new AllocatedPort("external")); + } + } + + return result; + } + + private boolean isAvailable(int port) { + return !isAllocated(port) && !isInUse(port); + } + + private synchronized boolean isAllocated(int port) { + AllocatedPort allocatedPort = this.allocatedPorts.get(port); + + return allocatedPort != null && !allocatedPort.isTimedOut(); + } + + private synchronized boolean allocate(String clientName, int port) { + + AllocatedPort allocatedPort = allocatedPorts.get(port); + + if (allocatedPort != null && allocatedPort.isTimedOut()) { + allocatedPort = null; + allocatedPorts.remove(port); + } + + if (allocatedPort == null && !isInUse(port)) { + allocatedPorts.put(port, new AllocatedPort(clientName)); + return true; + } else { + return false; + } + } + + private class AllocatedPort { + final String client; + final Instant allocatedAt; + + AllocatedPort(String client) { + this.client = client; + this.allocatedAt = Instant.now(); + } + + boolean isTimedOut() { + return allocatedAt.plus(timeoutDuration).isBefore(Instant.now()); + } + + @Override + public String toString() { + return "AllocatedPort [client=" + client + ", allocatedAt=" + allocatedAt + "]"; + } + } +} diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/SocketUtils.java b/src/newTest/java/org/opensearch/test/framework/cluster/SocketUtils.java new file mode 100644 index 0000000000..36718b6a43 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/cluster/SocketUtils.java @@ -0,0 +1,319 @@ +/* + * Copyright 2002-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.cluster; + +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.util.Random; +import java.util.SortedSet; +import java.util.TreeSet; + +import javax.net.ServerSocketFactory; + +/** + * Simple utility methods for working with network sockets — for example, + * for finding available ports on {@code localhost}. + * + *

Within this class, a TCP port refers to a port for a {@link ServerSocket}; + * whereas, a UDP port refers to a port for a {@link DatagramSocket}. + * + * @author Sam Brannen + * @author Ben Hale + * @author Arjen Poutsma + * @author Gunnar Hillert + * @author Gary Russell + * @since 4.0 + */ +public class SocketUtils { + + /** + * The default minimum value for port ranges used when finding an available + * socket port. + */ + public static final int PORT_RANGE_MIN = 1024; + + /** + * The default maximum value for port ranges used when finding an available + * socket port. + */ + public static final int PORT_RANGE_MAX = 65535; + + + private static final Random random = new Random(System.currentTimeMillis()); + + + /** + * Although {@code SocketUtils} consists solely of static utility methods, + * this constructor is intentionally {@code public}. + *

Rationale

+ *

Static methods from this class may be invoked from within XML + * configuration files using the Spring Expression Language (SpEL) and the + * following syntax. + * 

<bean id="bean1" ... p:port="#{T(org.springframework.util.SocketUtils).findAvailableTcpPort(12000)}" />
+ * If this constructor were {@code private}, you would be required to supply + * the fully qualified class name to SpEL's {@code T()} function for each usage. + * Thus, the fact that this constructor is {@code public} allows you to reduce + * boilerplate configuration with SpEL as can be seen in the following example. + * 
<bean id="socketUtils" class="org.springframework.util.SocketUtils" />
+     * <bean id="bean1" ... p:port="#{socketUtils.findAvailableTcpPort(12000)}" />
+     * <bean id="bean2" ... p:port="#{socketUtils.findAvailableTcpPort(30000)}" />
+ */ + public SocketUtils() { + /* no-op */ + } + + + /** + * Find an available TCP port randomly selected from the range + * [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. + * @return an available TCP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableTcpPort() { + return findAvailableTcpPort(PORT_RANGE_MIN); + } + + /** + * Find an available TCP port randomly selected from the range + * [{@code minPort}, {@value #PORT_RANGE_MAX}]. + * @param minPort the minimum port number + * @return an available TCP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableTcpPort(int minPort) { + return findAvailableTcpPort(minPort, PORT_RANGE_MAX); + } + + /** + * Find an available TCP port randomly selected from the range + * [{@code minPort}, {@code maxPort}]. + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return an available TCP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableTcpPort(int minPort, int maxPort) { + return SocketType.TCP.findAvailablePort(minPort, maxPort); + } + + /** + * Find the requested number of available TCP ports, each randomly selected + * from the range [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. + * @param numRequested the number of available ports to find + * @return a sorted set of available TCP port numbers + * @throws IllegalStateException if the requested number of available ports could not be found + */ + public static SortedSet findAvailableTcpPorts(int numRequested) { + return findAvailableTcpPorts(numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + /** + * Find the requested number of available TCP ports, each randomly selected + * from the range [{@code minPort}, {@code maxPort}]. + * @param numRequested the number of available ports to find + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return a sorted set of available TCP port numbers + * @throws IllegalStateException if the requested number of available ports could not be found + */ + public static SortedSet findAvailableTcpPorts(int numRequested, int minPort, int maxPort) { + return SocketType.TCP.findAvailablePorts(numRequested, minPort, maxPort); + } + + /** + * Find an available UDP port randomly selected from the range + * [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. + * @return an available UDP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableUdpPort() { + return findAvailableUdpPort(PORT_RANGE_MIN); + } + + /** + * Find an available UDP port randomly selected from the range + * [{@code minPort}, {@value #PORT_RANGE_MAX}]. + * @param minPort the minimum port number + * @return an available UDP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableUdpPort(int minPort) { + return findAvailableUdpPort(minPort, PORT_RANGE_MAX); + } + + /** + * Find an available UDP port randomly selected from the range + * [{@code minPort}, {@code maxPort}]. + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return an available UDP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableUdpPort(int minPort, int maxPort) { + return SocketType.UDP.findAvailablePort(minPort, maxPort); + } + + /** + * Find the requested number of available UDP ports, each randomly selected + * from the range [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. + * @param numRequested the number of available ports to find + * @return a sorted set of available UDP port numbers + * @throws IllegalStateException if the requested number of available ports could not be found + */ + public static SortedSet findAvailableUdpPorts(int numRequested) { + return findAvailableUdpPorts(numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + /** + * Find the requested number of available UDP ports, each randomly selected + * from the range [{@code minPort}, {@code maxPort}]. + * @param numRequested the number of available ports to find + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return a sorted set of available UDP port numbers + * @throws IllegalStateException if the requested number of available ports could not be found + */ + public static SortedSet findAvailableUdpPorts(int numRequested, int minPort, int maxPort) { + return SocketType.UDP.findAvailablePorts(numRequested, minPort, maxPort); + } + + + public enum SocketType { + + TCP { + @Override + protected boolean isPortAvailable(int port) { + try { + ServerSocket serverSocket = ServerSocketFactory.getDefault().createServerSocket( + port, 1, InetAddress.getByName("localhost")); + serverSocket.close(); + return true; + } + catch (Exception ex) { + return false; + } + } + }, + + UDP { + @Override + protected boolean isPortAvailable(int port) { + try { + DatagramSocket socket = new DatagramSocket(port, InetAddress.getByName("localhost")); + socket.close(); + return true; + } + catch (Exception ex) { + return false; + } + } + }; + + /** + * Determine if the specified port for this {@code SocketType} is + * currently available on {@code localhost}. + */ + protected abstract boolean isPortAvailable(int port); + + /** + * Find a pseudo-random port number within the range + * [{@code minPort}, {@code maxPort}]. + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return a random port number within the specified range + */ + private int findRandomPort(int minPort, int maxPort) { + int portRange = maxPort - minPort; + return minPort + random.nextInt(portRange + 1); + } + + /** + * Find an available port for this {@code SocketType}, randomly selected + * from the range [{@code minPort}, {@code maxPort}]. + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return an available port number for this socket type + * @throws IllegalStateException if no available port could be found + */ + int findAvailablePort(int minPort, int maxPort) { + //Assert.assertTrue(minPort > 0, "'minPort' must be greater than 0"); + //Assert.isTrue(maxPort >= minPort, "'maxPort' must be greater than or equal to 'minPort'"); + //Assert.isTrue(maxPort <= PORT_RANGE_MAX, "'maxPort' must be less than or equal to " + PORT_RANGE_MAX); + + int portRange = maxPort - minPort; + int candidatePort; + int searchCounter = 0; + do { + if (searchCounter > portRange) { + throw new IllegalStateException(String.format( + "Could not find an available %s port in the range [%d, %d] after %d attempts", + name(), minPort, maxPort, searchCounter)); + } + candidatePort = findRandomPort(minPort, maxPort); + searchCounter++; + } + while (!isPortAvailable(candidatePort)); + + return candidatePort; + } + + /** + * Find the requested number of available ports for this {@code SocketType}, + * each randomly selected from the range [{@code minPort}, {@code maxPort}]. + * @param numRequested the number of available ports to find + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return a sorted set of available port numbers for this socket type + * @throws IllegalStateException if the requested number of available ports could not be found + */ + SortedSet findAvailablePorts(int numRequested, int minPort, int maxPort) { + //Assert.isTrue(minPort > 0, "'minPort' must be greater than 0"); + //Assert.isTrue(maxPort > minPort, "'maxPort' must be greater than 'minPort'"); + //Assert.isTrue(maxPort <= PORT_RANGE_MAX, "'maxPort' must be less than or equal to " + PORT_RANGE_MAX); + //Assert.isTrue(numRequested > 0, "'numRequested' must be greater than 0"); + //Assert.isTrue((maxPort - minPort) >= numRequested, + // "'numRequested' must not be greater than 'maxPort' - 'minPort'"); + + SortedSet availablePorts = new TreeSet<>(); + int attemptCount = 0; + while ((++attemptCount <= numRequested + 100) && availablePorts.size() < numRequested) { + availablePorts.add(findAvailablePort(minPort, maxPort)); + } + + if (availablePorts.size() != numRequested) { + throw new IllegalStateException(String.format( + "Could not find %d available %s ports in the range [%d, %d]", + numRequested, name(), minPort, maxPort)); + } + + return availablePorts; + } + } + +} \ No newline at end of file diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java new file mode 100644 index 0000000000..1c207baaa5 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -0,0 +1,366 @@ +/* + * Copyright 2021 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.cluster; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import javax.net.ssl.SSLContext; + +import org.apache.commons.io.IOUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpOptions; +import org.apache.http.client.methods.HttpPatch; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.config.SocketConfig; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.Strings; +import org.opensearch.common.xcontent.ToXContentObject; +import org.opensearch.security.DefaultObjectMapper; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; + +public class TestRestClient implements AutoCloseable { + + private static final Logger log = LogManager.getLogger(TestRestClient.class); + + private boolean enableHTTPClientSSL = true; + private boolean enableHTTPClientSSLv3Only = false; + private boolean sendHTTPClientCertificate = false; + private InetSocketAddress nodeHttpAddress; + private RequestConfig requestConfig; + private List
headers = new ArrayList<>(); + private Header CONTENT_TYPE_JSON = new BasicHeader("Content-Type", "application/json"); + private boolean trackResources = false; + private SSLContext sslContext; + private Set puttedResourcesSet = new HashSet<>(); + private List puttedResourcesList = new ArrayList<>(); + + public TestRestClient(InetSocketAddress nodeHttpAddress, List
headers, SSLContext sslContext) { + this.nodeHttpAddress = nodeHttpAddress; + this.headers.addAll(headers); + this.sslContext = sslContext; + } + + public HttpResponse get(String path, Header... headers) throws Exception { + return executeRequest(new HttpGet(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse getAuthInfo( Header... headers) throws Exception { + return executeRequest(new HttpGet(getHttpServerUri() + "/_opendistro/_security/authinfo?pretty"), headers); + } + + public HttpResponse head(String path, Header... headers) throws Exception { + return executeRequest(new HttpHead(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse options(String path, Header... headers) throws Exception { + return executeRequest(new HttpOptions(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse putJson(String path, String body, Header... headers) throws Exception { + HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); + uriRequest.setEntity(new StringEntity(body)); + + HttpResponse response = executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); + + if (response.getStatusCode() < 400 && trackResources && !puttedResourcesSet.contains(path)) { + puttedResourcesSet.add(path); + puttedResourcesList.add(path); + } + + return response; + } + + public HttpResponse putJson(String path, ToXContentObject body) throws Exception { + return putJson(path, Strings.toString(body)); + } + + public HttpResponse put(String path) throws Exception { + HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); + HttpResponse response = executeRequest(uriRequest); + + if (response.getStatusCode() < 400 && trackResources && !puttedResourcesSet.contains(path)) { + puttedResourcesSet.add(path); + puttedResourcesList.add(path); + } + + return response; + } + + public HttpResponse delete(String path, Header... headers) throws Exception { + return executeRequest(new HttpDelete(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse postJson(String path, String body, Header... headers) throws Exception { + HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); + uriRequest.setEntity(new StringEntity(body)); + return executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); + } + + public HttpResponse postJson(String path, ToXContentObject body) throws Exception { + return postJson(path, Strings.toString(body)); + } + + public HttpResponse post(String path) throws Exception { + HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); + return executeRequest(uriRequest); + } + + public HttpResponse patch(String path, String body) throws Exception { + HttpPatch uriRequest = new HttpPatch(getHttpServerUri() + "/" + path); + uriRequest.setEntity(new StringEntity(body)); + return executeRequest(uriRequest, CONTENT_TYPE_JSON); + } + + public HttpResponse executeRequest(HttpUriRequest uriRequest, Header... requestSpecificHeaders) throws Exception { + + CloseableHttpClient httpClient = null; + try { + + httpClient = getHTTPClient(); + + if (requestSpecificHeaders != null && requestSpecificHeaders.length > 0) { + for (int i = 0; i < requestSpecificHeaders.length; i++) { + Header h = requestSpecificHeaders[i]; + uriRequest.addHeader(h); + } + } + + for (Header header : headers) { + uriRequest.addHeader(header); + } + + HttpResponse res = new HttpResponse(httpClient.execute(uriRequest)); + log.debug(res.getBody()); + return res; + } finally { + + if (httpClient != null) { + httpClient.close(); + } + } + } + + public TestRestClient trackResources() { + trackResources = true; + return this; + } + + private void cleanupResources() { + if (puttedResourcesList.size() > 0) { + log.info("Cleaning up " + puttedResourcesList); + + for (String resource : Lists.reverse(puttedResourcesList)) { + try { + delete(resource); + } catch (Exception e) { + log.error("Error cleaning up created resources " + resource, e); + } + } + } + } + + protected final String getHttpServerUri() { + return "http" + (enableHTTPClientSSL ? "s" : "") + "://" + nodeHttpAddress.getHostString() + ":" + nodeHttpAddress.getPort(); + } + + protected final CloseableHttpClient getHTTPClient() throws Exception { + + final HttpClientBuilder hcb = HttpClients.custom(); + + String[] protocols = null; + + if (enableHTTPClientSSLv3Only) { + protocols = new String[] { "SSLv3" }; + } else { + protocols = new String[] { "TLSv1", "TLSv1.1", "TLSv1.2" }; + } + + final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(this.sslContext, protocols, null, + NoopHostnameVerifier.INSTANCE); + + hcb.setSSLSocketFactory(sslsf); + + hcb.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(60 * 1000).build()); + + if (requestConfig != null) { + hcb.setDefaultRequestConfig(requestConfig); + } + + return hcb.build(); + } + + private Header[] mergeHeaders(Header header, Header... headers) { + + if (headers == null || headers.length == 0) { + return new Header[] { header }; + } else { + Header[] result = new Header[headers.length + 1]; + result[0] = header; + System.arraycopy(headers, 0, result, 1, headers.length); + return result; + } + } + + public static class HttpResponse { + private final CloseableHttpResponse inner; + private final String body; + private final Header[] header; + private final int statusCode; + private final String statusReason; + + public HttpResponse(CloseableHttpResponse inner) throws IllegalStateException, IOException { + super(); + this.inner = inner; + final HttpEntity entity = inner.getEntity(); + if (entity == null) { //head request does not have a entity + this.body = ""; + } else { + this.body = IOUtils.toString(entity.getContent(), StandardCharsets.UTF_8); + } + this.header = inner.getAllHeaders(); + this.statusCode = inner.getStatusLine().getStatusCode(); + this.statusReason = inner.getStatusLine().getReasonPhrase(); + inner.close(); + } + + public String getContentType() { + Header h = getInner().getFirstHeader("content-type"); + if (h != null) { + return h.getValue(); + } + return null; + } + + public boolean isJsonContentType() { + String ct = getContentType(); + if (ct == null) { + return false; + } + return ct.contains("application/json"); + } + + public CloseableHttpResponse getInner() { + return inner; + } + + public String getBody() { + return body; + } + + public Header[] getHeader() { + return header; + } + + public int getStatusCode() { + return statusCode; + } + + public String getStatusReason() { + return statusReason; + } + + public List
getHeaders() { + return header == null ? Collections.emptyList() : Arrays.asList(header); + } + + public JsonNode toJsonNode() throws JsonProcessingException, IOException { + return DefaultObjectMapper.objectMapper.readTree(getBody()); + } + + @Override + public String toString() { + return "HttpResponse [inner=" + inner + ", body=" + body + ", header=" + Arrays.toString(header) + ", statusCode=" + statusCode + + ", statusReason=" + statusReason + "]"; + } + + } + + @Override + public String toString() { + return "TestRestClient [server=" + getHttpServerUri() + ", node=" + nodeHttpAddress + "]"; + } + + public RequestConfig getRequestConfig() { + return requestConfig; + } + + public void setRequestConfig(RequestConfig requestConfig) { + this.requestConfig = requestConfig; + } + + public void setLocalAddress(InetAddress inetAddress) { + if (requestConfig == null) { + requestConfig = RequestConfig.custom().setLocalAddress(inetAddress).build(); + } else { + requestConfig = RequestConfig.copy(requestConfig).setLocalAddress(inetAddress).build(); + } + } + + @Override + public void close() throws IOException { + cleanupResources(); + } + + public boolean isSendHTTPClientCertificate() { + return sendHTTPClientCertificate; + } + + public void setSendHTTPClientCertificate(boolean sendHTTPClientCertificate) { + this.sendHTTPClientCertificate = sendHTTPClientCertificate; + } + +} From c3ae9a627102d765091c5d287106ba27adf173ba Mon Sep 17 00:00:00 2001 From: Jochen Kressin Date: Mon, 1 Aug 2022 18:43:46 +0200 Subject: [PATCH 02/47] removed abstract base class, prefer composition over inheritance Signed-off-by: Jochen Kressin --- .../test/AbstractIntegrationTest.java | 36 ------------------- .../test/GenericIntegrationTest.java | 11 ++++-- .../test/PrivilegesEvaluatorTest.java | 10 ++++-- .../opensearch/test/SecurityRolesTests.java | 8 ++++- .../test/framework/TestSecurityConfig.java | 10 +++++- 5 files changed, 33 insertions(+), 42 deletions(-) delete mode 100644 src/newTest/java/org/opensearch/test/AbstractIntegrationTest.java diff --git a/src/newTest/java/org/opensearch/test/AbstractIntegrationTest.java b/src/newTest/java/org/opensearch/test/AbstractIntegrationTest.java deleted file mode 100644 index 3f13fe261a..0000000000 --- a/src/newTest/java/org/opensearch/test/AbstractIntegrationTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.test; - -import org.junit.runner.RunWith; -import org.opensearch.test.framework.TestSecurityConfig; -import org.opensearch.test.framework.TestSecurityConfig.Role; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; - -@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) -@ThreadLeakScope(ThreadLeakScope.Scope.NONE) -public class AbstractIntegrationTest { - - /** - * Auth domain with HTTPS Basic and the internal user backend - */ - protected final static TestSecurityConfig.AuthcDomain AUTHC_HTTPBASIC_INTERNAL = new TestSecurityConfig.AuthcDomain("basic", 0) - .httpAuthenticator("basic").backend("internal"); - - /** - * Admin user with full access to all indices - */ - protected final static TestSecurityConfig.User USER_ADMIN = new TestSecurityConfig.User("admin") - .roles(new Role("allaccess").indexPermissions("*").on("*").clusterPermissions("*")); - -} diff --git a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java index cfddf84ad5..56b28ba20c 100644 --- a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java +++ b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java @@ -16,6 +16,7 @@ import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; +import org.junit.runner.RunWith; import org.opensearch.test.framework.TestIndex; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; @@ -24,15 +25,21 @@ import org.opensearch.test.framework.cluster.TestRestClient; import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import com.fasterxml.jackson.core.JsonPointer; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.*; +import static org.opensearch.test.framework.TestSecurityConfig.User.*; + /** * WIP * Generic test class that demonstrates how to use the test framework to * set up a test cluster with users, roles, indices and data, and how to * implement tests. One main goal here is to make tests self-contained. */ -public class GenericIntegrationTest extends AbstractIntegrationTest { +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class GenericIntegrationTest { // define indices used in this test private final static TestIndex INDEX_A = TestIndex.name("index-a").build(); @@ -46,7 +53,7 @@ public class GenericIntegrationTest extends AbstractIntegrationTest { @ClassRule public static LocalCluster cluster = new LocalCluster.Builder().clusterConfiguration(ClusterConfiguration.THREE_MASTERS) .authc(AUTHC_HTTPBASIC_INTERNAL) - .users(USER_ADMIN, INDEX_A_USER) + .users(INDEX_A_USER) .indices(INDEX_A, INDEX_B).build(); @Test diff --git a/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java b/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java index 44307c6a02..34bc546fba 100644 --- a/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java +++ b/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java @@ -11,10 +11,13 @@ package org.opensearch.test; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; + import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; +import org.junit.runner.RunWith; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.cluster.ClusterConfiguration; @@ -22,13 +25,16 @@ import org.opensearch.test.framework.cluster.TestRestClient; import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + /** * This is a port for the test * org.opensearch.security.privileges.PrivilegesEvaluatorTest to the new test * framework for direct comparison - * */ -public class PrivilegesEvaluatorTest extends AbstractIntegrationTest { +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class PrivilegesEvaluatorTest { protected final static TestSecurityConfig.User NEGATIVE_LOOKAHEAD = new TestSecurityConfig.User( "negative_lookahead_user") diff --git a/src/newTest/java/org/opensearch/test/SecurityRolesTests.java b/src/newTest/java/org/opensearch/test/SecurityRolesTests.java index b5de319912..7b82fb41c7 100644 --- a/src/newTest/java/org/opensearch/test/SecurityRolesTests.java +++ b/src/newTest/java/org/opensearch/test/SecurityRolesTests.java @@ -26,10 +26,13 @@ package org.opensearch.test; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; + import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; +import org.junit.runner.RunWith; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.cluster.ClusterConfiguration; @@ -37,9 +40,12 @@ import org.opensearch.test.framework.cluster.TestRestClient; import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import com.fasterxml.jackson.core.JsonPointer; -public class SecurityRolesTests extends AbstractIntegrationTest { +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class SecurityRolesTests { protected final static TestSecurityConfig.User USER_SR = new TestSecurityConfig.User("sr_user").roles( new Role("abc_ber").indexPermissions("*").on("*").clusterPermissions("*"), diff --git a/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java index 88e86b51b3..b64fea1090 100644 --- a/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -56,6 +56,7 @@ import org.opensearch.security.action.configupdate.ConfigUpdateRequest; import org.opensearch.security.action.configupdate.ConfigUpdateResponse; import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.cluster.NestedValueMap; import org.opensearch.test.framework.cluster.NestedValueMap.Path; import org.opensearch.test.framework.cluster.OpenSearchClientProvider.UserCredentialsHolder; @@ -244,6 +245,10 @@ public TestSecurityConfig roleToRoleMapping(Role role, String... backendRoles) { } public static class User implements UserCredentialsHolder { + + public final static TestSecurityConfig.User USER_ADMIN = new TestSecurityConfig.User("admin") + .roles(new Role("allaccess").indexPermissions("*").on("*").clusterPermissions("*")); + private String name; private String password; private Role[] roles; @@ -411,6 +416,9 @@ public NestedValueMap toJsonMap() { public static class AuthcDomain { + public final static AuthcDomain AUTHC_HTTPBASIC_INTERNAL = new TestSecurityConfig.AuthcDomain("basic", 0) + .httpAuthenticator("basic").backend("internal"); + private final String id; private boolean enabled = true; private boolean transportEnabled = true; @@ -418,7 +426,7 @@ public static class AuthcDomain { private List skipUsers = new ArrayList<>(); private HttpAuthenticator httpAuthenticator; private AuthenticationBackend authenticationBackend; - + public AuthcDomain(String id, int order) { this.id = id; this.order = order; From 6db0e64615f847d5dedfce3d9abca2d39c84a8bb Mon Sep 17 00:00:00 2001 From: Jochen Kressin Date: Mon, 1 Aug 2022 20:03:31 +0200 Subject: [PATCH 03/47] moved Jackson dependencies from test classes to TestRestClient Signed-off-by: Jochen Kressin --- .../test/GenericIntegrationTest.java | 11 +++---- .../opensearch/test/SecurityRolesTests.java | 12 +++---- .../framework/cluster/TestRestClient.java | 32 ++++++++++++++++++- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java index 56b28ba20c..6be35ff423 100644 --- a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java +++ b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java @@ -11,6 +11,9 @@ package org.opensearch.test; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.TestSecurityConfig.User.USER_ADMIN; + import org.apache.http.HttpStatus; import org.junit.AfterClass; import org.junit.Assert; @@ -26,10 +29,6 @@ import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import com.fasterxml.jackson.core.JsonPointer; - -import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.*; -import static org.opensearch.test.framework.TestSecurityConfig.User.*; /** * WIP @@ -71,8 +70,8 @@ public void testIndexAUserHasOnlyAccessToIndexA() throws Exception { Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); // demo: work with JSON response body and check values - JsonPointer jsonPointer = JsonPointer.compile("/_source/hits/value"); - int hits = response.toJsonNode().at(jsonPointer).asInt(); + int hits = response.getIntFromJsonBody("/_source/hits/value"); + Assert.assertEquals(0, hits); response = client.get("index-b/_search?pretty"); Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_FORBIDDEN); diff --git a/src/newTest/java/org/opensearch/test/SecurityRolesTests.java b/src/newTest/java/org/opensearch/test/SecurityRolesTests.java index 7b82fb41c7..37b4459e5c 100644 --- a/src/newTest/java/org/opensearch/test/SecurityRolesTests.java +++ b/src/newTest/java/org/opensearch/test/SecurityRolesTests.java @@ -41,7 +41,6 @@ import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import com.fasterxml.jackson.core.JsonPointer; @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) @@ -63,18 +62,15 @@ public void testSecurityRolesAnon() throws Exception { HttpResponse response = client.getAuthInfo(); Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); - // Check username - JsonPointer jsonPointer = JsonPointer.compile("/user_name"); - String username = response.toJsonNode().at(jsonPointer).asText(); + // Check username + String username = response.getTextFromJsonBody("/user_name"); Assert.assertEquals("sr_user", username); // Check security roles - jsonPointer = JsonPointer.compile("/roles/0"); - String securityRole = response.toJsonNode().at(jsonPointer).asText(); + String securityRole = response.getTextFromJsonBody("/roles/0"); Assert.assertEquals("user_sr_user__abc_ber", securityRole); - jsonPointer = JsonPointer.compile("/roles/1"); - securityRole = response.toJsonNode().at(jsonPointer).asText(); + securityRole = response.getTextFromJsonBody("/roles/0"); Assert.assertEquals("user_sr_user__def_efg", securityRole); } diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index 1c207baaa5..207b257f42 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -317,10 +317,40 @@ public List
getHeaders() { return header == null ? Collections.emptyList() : Arrays.asList(header); } - public JsonNode toJsonNode() throws JsonProcessingException, IOException { + public String getTextFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asText(); + } + + public int getIntFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asInt(); + } + + public Boolean getBooleanFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asBoolean(); + } + + public Double getDoubleFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asDouble(); + } + + public Long getLongFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asLong(); + } + + private JsonNode getJsonNodeAt(String jsonPointer) { + try { + return toJsonNode().at(jsonPointer); + } catch (IOException e) { + throw new IllegalArgumentException("Cound not convert response body to JSON node ",e); + } + } + + private JsonNode toJsonNode() throws JsonProcessingException, IOException { return DefaultObjectMapper.objectMapper.readTree(getBody()); } + + @Override public String toString() { return "HttpResponse [inner=" + inner + ", body=" + body + ", header=" + Arrays.toString(header) + ", statusCode=" + statusCode From 1fa6c0abc248d8ee3601b0823c68eead87b45fee Mon Sep 17 00:00:00 2001 From: Jochen Kressin Date: Mon, 1 Aug 2022 20:15:32 +0200 Subject: [PATCH 04/47] removed unused code, replaced some old ES references Signed-off-by: Jochen Kressin --- .../test/framework/TestSecurityConfig.java | 8 +- .../test/framework/cluster/ClusterInfo.java | 45 ---------- .../test/framework/cluster/LocalCluster.java | 5 -- .../cluster/LocalOpenSearchCluster.java | 89 ++----------------- .../framework/cluster/TestRestClient.java | 7 -- 5 files changed, 8 insertions(+), 146 deletions(-) delete mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/ClusterInfo.java diff --git a/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java index b64fea1090..38523bb13f 100644 --- a/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -70,8 +70,7 @@ public class TestSecurityConfig { private NestedValueMap overrideRoleSettings; private NestedValueMap overrideRoleMappingSettings; private String indexName = ".opendistro_security"; - private Map> variableSuppliers = new HashMap<>(); - + public TestSecurityConfig() { } @@ -81,11 +80,6 @@ public TestSecurityConfig configIndexName(String configIndexName) { return this; } - public TestSecurityConfig var(String name, Supplier variableSupplier) { - this.variableSuppliers.put(name, variableSupplier); - return this; - } - public TestSecurityConfig securityConfigSettings(String keyPath, Object value, Object... more) { if (overrideSecurityConfigSettings == null) { overrideSecurityConfigSettings = new NestedValueMap(); diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/ClusterInfo.java b/src/newTest/java/org/opensearch/test/framework/cluster/ClusterInfo.java deleted file mode 100644 index e82ee82391..0000000000 --- a/src/newTest/java/org/opensearch/test/framework/cluster/ClusterInfo.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2015-2018 _floragunn_ GmbH - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.test.framework.cluster; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.opensearch.common.transport.TransportAddress; - -// TODO: change to getter/setter -public class ClusterInfo { - public int numNodes; - public String httpHost = null; - public int httpPort = -1; - public Set httpAdresses = new HashSet(); - public String nodeHost; - public int nodePort; - public String clustername; - public List tcpMasterPortsOnly; -} diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 3fddec2f15..2ed877c021 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -330,11 +330,6 @@ public Builder authc(TestSecurityConfig.AuthcDomain authc) { return this; } - public Builder var(String name, Supplier value) { - testSecurityConfig.var(name, value); - return this; - } - public Builder clusterName(String clusterName) { this.clusterName = clusterName; return this; diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index 4ae2c8b956..6c92b12789 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -152,7 +152,6 @@ public void start() throws Exception { log.info("Startup finished. Waiting for GREEN"); waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(10), allNodes.size()); - putDefaultTemplate(); log.info("Started: {}", this); @@ -289,19 +288,6 @@ private final Node randomRunningNode(List nodes, List... moreNodes) return runningNodes.get(index); } - private void putDefaultTemplate() { - String defaultTemplate = "{\n" + " \"index_patterns\": [\"*\"],\n" + " \"order\": -1,\n" + " \"settings\": {\n" - + " \"number_of_shards\": \"5\",\n" + " \"number_of_replicas\": \"1\"\n" + " }\n" // - + " }"; - - AcknowledgedResponse templateAck = clientNode().getInternalNodeClient().admin().indices() - .putTemplate(new PutIndexTemplateRequest("default").source(defaultTemplate, XContentType.JSON)).actionGet(); - - if (!templateAck.isAcknowledged()) { - throw new RuntimeException("Default template could not be created"); - } - } - private CompletableFuture startNodes(List nodeSettingList, SortedSet transportPorts, SortedSet httpPorts) { Iterator transportPortIterator = transportPorts.iterator(); Iterator httpPortIterator = httpPorts.iterator(); @@ -315,11 +301,10 @@ private CompletableFuture startNodes(List nodeSettingList, S return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); } - public ClusterInfo waitForCluster(ClusterHealthStatus status, TimeValue timeout, int expectedNodeCount) throws IOException { - ClusterInfo clusterInfo = new ClusterInfo(); + public void waitForCluster(ClusterHealthStatus status, TimeValue timeout, int expectedNodeCount) throws IOException { Client client = clientNode().getInternalNodeClient(); - try { + log.debug("waiting for cluster state {} and {} nodes", status.name(), expectedNodeCount); AdminClient adminClient = client.admin(); @@ -338,67 +323,7 @@ public ClusterInfo waitForCluster(ClusterHealthStatus status, TimeValue timeout, } assertEquals(expectedNodeCount, healthResponse.getNumberOfNodes()); - - final NodesInfoResponse nodesInfoResponse = adminClient.cluster().nodesInfo(new NodesInfoRequest()).actionGet(); - - final List nodes = nodesInfoResponse.getNodes(); - - final List masterNodes = nodes.stream() - .filter(nodeInfo -> nodeInfo.getNode().getRoles().contains(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE)).collect(Collectors.toList()); - - final List dataNodes = nodes.stream().filter(nodeInfo -> nodeInfo.getNode().getRoles().contains(DiscoveryNodeRole.DATA_ROLE) - && !nodeInfo.getNode().getRoles().contains(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE)).collect(Collectors.toList()); - - final List clientNodes = nodes.stream() - .filter(nodeInfo -> !nodeInfo.getNode().getRoles().contains(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE) - && !nodeInfo.getNode().getRoles().contains(DiscoveryNodeRole.DATA_ROLE)) - .collect(Collectors.toList()); - - for (NodeInfo nodeInfo : masterNodes) { - final TransportAddress is = nodeInfo.getInfo(TransportInfo.class).getAddress().publishAddress(); - clusterInfo.nodePort = is.getPort(); - clusterInfo.nodeHost = is.getAddress(); - } - - Predicate hasHttpAddressPredicate = nodeInfo -> nodeInfo.getInfo(HttpInfo.class) != null - && nodeInfo.getInfo(HttpInfo.class).address() != null; - - if (!clientNodes.isEmpty()) { - NodeInfo nodeInfo = clientNodes.get(0); - if (hasHttpAddressPredicate.test(nodeInfo)) { - final TransportAddress his = nodeInfo.getInfo(HttpInfo.class).address().publishAddress(); - clusterInfo.httpPort = his.getPort(); - clusterInfo.httpHost = his.getAddress(); - } else { - throw new RuntimeException("no http host/port for client node"); - } - } else if (!dataNodes.isEmpty()) { - for (NodeInfo nodeInfo : dataNodes) { - if (hasHttpAddressPredicate.test(nodeInfo)) { - final TransportAddress his = nodeInfo.getInfo(HttpInfo.class).address().publishAddress(); - clusterInfo.httpPort = his.getPort(); - clusterInfo.httpHost = his.getAddress(); - break; - } - } - } else { - for (NodeInfo nodeInfo : nodes) { - if (hasHttpAddressPredicate.test(nodeInfo)) { - final TransportAddress his = nodeInfo.getInfo(HttpInfo.class).address().publishAddress(); - clusterInfo.httpPort = his.getPort(); - clusterInfo.httpHost = his.getAddress(); - break; - } - } - } - - for (NodeInfo nodeInfo : nodes) { - clusterInfo.httpAdresses.add(nodeInfo.getInfo(HttpInfo.class).address().publishAddress()); - } - } catch (final OpenSearchTimeoutException e) { - throw new IOException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations"); - } - return clusterInfo; + } @Override @@ -468,7 +393,7 @@ public class Node implements OpenSearchClientProvider { CompletableFuture start() { CompletableFuture completableFuture = new CompletableFuture<>(); - this.node = new PluginAwareNode(nodeSettings.masterNode, getEsSettings(), nodeSettings.getPlugins(additionalPlugins)); + this.node = new PluginAwareNode(nodeSettings.masterNode, getOpenSearchSettings(), nodeSettings.getPlugins(additionalPlugins)); new Thread(new Runnable() { @@ -561,8 +486,8 @@ public InetSocketAddress getTransportAddress() { return transportAddress; } - private Settings getEsSettings() { - Settings settings = getMinimalEsSettings(); + private Settings getOpenSearchSettings() { + Settings settings = getMinimalOpenSearchSettings(); if (nodeSettingsSupplier != null) { // TODO node number @@ -572,7 +497,7 @@ private Settings getEsSettings() { return settings; } - private Settings getMinimalEsSettings() { + private Settings getMinimalOpenSearchSettings() { return Settings.builder().put("node.name", nodeName).put("node.data", nodeSettings.dataNode).put("node.master", nodeSettings.masterNode) .put("cluster.name", clusterName).put("path.home", nodeHomeDir.toPath()).put("path.data", dataDir.toPath()) .put("path.logs", logsDir.toPath()).putList("cluster.initial_master_nodes", initialMasterHosts) diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index 207b257f42..8f486916e2 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -77,7 +77,6 @@ public class TestRestClient implements AutoCloseable { private static final Logger log = LogManager.getLogger(TestRestClient.class); private boolean enableHTTPClientSSL = true; - private boolean enableHTTPClientSSLv3Only = false; private boolean sendHTTPClientCertificate = false; private InetSocketAddress nodeHttpAddress; private RequestConfig requestConfig; @@ -223,12 +222,6 @@ protected final CloseableHttpClient getHTTPClient() throws Exception { String[] protocols = null; - if (enableHTTPClientSSLv3Only) { - protocols = new String[] { "SSLv3" }; - } else { - protocols = new String[] { "TLSv1", "TLSv1.1", "TLSv1.2" }; - } - final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(this.sslContext, protocols, null, NoopHostnameVerifier.INSTANCE); From 895e3960151b712f3885ec91e5873b9ef083497d Mon Sep 17 00:00:00 2001 From: Jochen Kressin Date: Mon, 1 Aug 2022 21:55:52 +0200 Subject: [PATCH 05/47] replaced JUnit asserts with Hamcrest asserts Signed-off-by: Jochen Kressin --- .../test/GenericIntegrationTest.java | 25 ++++++------------- .../test/PrivilegesEvaluatorTest.java | 18 +++++-------- .../opensearch/test/SecurityRolesTests.java | 19 +++++++------- 3 files changed, 22 insertions(+), 40 deletions(-) diff --git a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java index 6be35ff423..77bfc04516 100644 --- a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java +++ b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java @@ -11,12 +11,12 @@ package org.opensearch.test; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; import static org.opensearch.test.framework.TestSecurityConfig.User.USER_ADMIN; import org.apache.http.HttpStatus; -import org.junit.AfterClass; -import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; @@ -26,7 +26,6 @@ import org.opensearch.test.framework.cluster.ClusterConfiguration; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; -import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; @@ -58,28 +57,18 @@ public class GenericIntegrationTest { @Test public void testAdminUserHasAccessToAllIndices() throws Exception { try (TestRestClient client = cluster.getRestClient(USER_ADMIN)) { - HttpResponse response = client.get("*/_search?pretty"); - Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); + assertThat(client.get("*/_search?pretty").getStatusCode(), equalTo(HttpStatus.SC_OK)); } } @Test public void testIndexAUserHasOnlyAccessToIndexA() throws Exception { - try (TestRestClient client = cluster.getRestClient(INDEX_A_USER)) { - HttpResponse response = client.get("index-a/_search?pretty"); - Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); - + try (TestRestClient client = cluster.getRestClient(INDEX_A_USER)) { + assertThat(client.get("index-a/_search?pretty").getStatusCode(), equalTo(HttpStatus.SC_OK)); // demo: work with JSON response body and check values - int hits = response.getIntFromJsonBody("/_source/hits/value"); - Assert.assertEquals(0, hits); - - response = client.get("index-b/_search?pretty"); - Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_FORBIDDEN); + assertThat(client.get("index-a/_search?pretty").getIntFromJsonBody("/_source/hits/value"), equalTo(0)); + assertThat(client.get("index-b/_search?pretty"), equalTo(HttpStatus.SC_FORBIDDEN)); } } - @AfterClass - public static void close() { - cluster.close(); - } } diff --git a/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java b/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java index 34bc546fba..d21be816d8 100644 --- a/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java +++ b/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java @@ -11,10 +11,11 @@ package org.opensearch.test; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; import org.apache.http.HttpStatus; -import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; @@ -23,7 +24,6 @@ import org.opensearch.test.framework.cluster.ClusterConfiguration; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; -import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; @@ -54,11 +54,8 @@ public class PrivilegesEvaluatorTest { public void testNegativeLookaheadPattern() throws Exception { try (TestRestClient client = cluster.getRestClient(NEGATIVE_LOOKAHEAD)) { - HttpResponse response = client.get("*/_search"); - Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_FORBIDDEN); - - response = client.get("r*/_search"); - Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); + assertThat(client.get("*/_search").getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); + assertThat(client.get("r*/_search").getStatusCode(), equalTo(HttpStatus.SC_OK)); } } @@ -66,11 +63,8 @@ public void testNegativeLookaheadPattern() throws Exception { public void testRegexPattern() throws Exception { try (TestRestClient client = cluster.getRestClient(NEGATED_REGEX)) { - HttpResponse response = client.get("*/_search"); - Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_FORBIDDEN); - - response = client.get("r*/_search"); - Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); + assertThat(client.get("*/_search").getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); + assertThat(client.get("r*/_search").getStatusCode(), equalTo(HttpStatus.SC_OK)); } } diff --git a/src/newTest/java/org/opensearch/test/SecurityRolesTests.java b/src/newTest/java/org/opensearch/test/SecurityRolesTests.java index 37b4459e5c..e29c281655 100644 --- a/src/newTest/java/org/opensearch/test/SecurityRolesTests.java +++ b/src/newTest/java/org/opensearch/test/SecurityRolesTests.java @@ -28,6 +28,9 @@ import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.ClassRule; @@ -56,22 +59,18 @@ public class SecurityRolesTests { .authc(AUTHC_HTTPBASIC_INTERNAL).users(USER_SR).build(); @Test - public void testSecurityRolesAnon() throws Exception { + public void testSecurityRoles() throws Exception { try (TestRestClient client = cluster.getRestClient(USER_SR)) { HttpResponse response = client.getAuthInfo(); - Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); + assertThat(response.getStatusCode(), equalTo(HttpStatus.SC_OK)); // Check username - String username = response.getTextFromJsonBody("/user_name"); - Assert.assertEquals("sr_user", username); - - // Check security roles - String securityRole = response.getTextFromJsonBody("/roles/0"); - Assert.assertEquals("user_sr_user__abc_ber", securityRole); + assertThat(response.getTextFromJsonBody("/user_name"), equalTo("sr_user")); - securityRole = response.getTextFromJsonBody("/roles/0"); - Assert.assertEquals("user_sr_user__def_efg", securityRole); + // Check security roles + assertThat(response.getTextFromJsonBody("/roles/0"), equalTo("user_sr_user__abc_ber")); + assertThat(response.getTextFromJsonBody("/roles/1"), equalTo("user_sr_user__def_efg")); } } From b4cf9e97fda8885e70e8cbee7df95b8014ff87c4 Mon Sep 17 00:00:00 2001 From: Jochen Kressin Date: Mon, 1 Aug 2022 22:08:40 +0200 Subject: [PATCH 06/47] renamed ClusterConfiguration to ClusterManager Signed-off-by: Jochen Kressin --- .../org/opensearch/test/GenericIntegrationTest.java | 4 ++-- .../org/opensearch/test/PrivilegesEvaluatorTest.java | 4 ++-- .../java/org/opensearch/test/SecurityRolesTests.java | 4 ++-- .../{ClusterConfiguration.java => ClusterManager.java} | 4 ++-- .../test/framework/cluster/LocalCluster.java | 10 +++++----- .../test/framework/cluster/LocalOpenSearchCluster.java | 6 +++--- 6 files changed, 16 insertions(+), 16 deletions(-) rename src/newTest/java/org/opensearch/test/framework/cluster/{ClusterConfiguration.java => ClusterManager.java} (98%) diff --git a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java index 77bfc04516..c01ec32fa8 100644 --- a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java +++ b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java @@ -23,7 +23,7 @@ import org.opensearch.test.framework.TestIndex; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; -import org.opensearch.test.framework.cluster.ClusterConfiguration; +import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; @@ -49,7 +49,7 @@ public class GenericIntegrationTest { // build our test cluster as a ClassRule @ClassRule - public static LocalCluster cluster = new LocalCluster.Builder().clusterConfiguration(ClusterConfiguration.THREE_MASTERS) + public static LocalCluster cluster = new LocalCluster.Builder().clusterConfiguration(ClusterManager.THREE_MASTERS) .authc(AUTHC_HTTPBASIC_INTERNAL) .users(INDEX_A_USER) .indices(INDEX_A, INDEX_B).build(); diff --git a/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java b/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java index d21be816d8..cbea84ff2f 100644 --- a/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java +++ b/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java @@ -21,7 +21,7 @@ import org.junit.runner.RunWith; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; -import org.opensearch.test.framework.cluster.ClusterConfiguration; +import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; @@ -47,7 +47,7 @@ public class PrivilegesEvaluatorTest { @ClassRule public static LocalCluster cluster = new LocalCluster.Builder() - .clusterConfiguration(ClusterConfiguration.THREE_MASTERS).authc(AUTHC_HTTPBASIC_INTERNAL) + .clusterConfiguration(ClusterManager.THREE_MASTERS).authc(AUTHC_HTTPBASIC_INTERNAL) .users(NEGATIVE_LOOKAHEAD, NEGATED_REGEX).build(); @Test diff --git a/src/newTest/java/org/opensearch/test/SecurityRolesTests.java b/src/newTest/java/org/opensearch/test/SecurityRolesTests.java index e29c281655..956b3eb17f 100644 --- a/src/newTest/java/org/opensearch/test/SecurityRolesTests.java +++ b/src/newTest/java/org/opensearch/test/SecurityRolesTests.java @@ -38,7 +38,7 @@ import org.junit.runner.RunWith; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; -import org.opensearch.test.framework.cluster.ClusterConfiguration; +import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; @@ -55,7 +55,7 @@ public class SecurityRolesTests { @ClassRule public static LocalCluster cluster = new LocalCluster.Builder() - .clusterConfiguration(ClusterConfiguration.THREE_MASTERS).anonymousAuth(true) + .clusterConfiguration(ClusterManager.THREE_MASTERS).anonymousAuth(true) .authc(AUTHC_HTTPBASIC_INTERNAL).users(USER_SR).build(); @Test diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/ClusterConfiguration.java b/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java similarity index 98% rename from src/newTest/java/org/opensearch/test/framework/cluster/ClusterConfiguration.java rename to src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java index 08e8eb31c1..5d6deb7f3f 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/ClusterConfiguration.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java @@ -45,7 +45,7 @@ import com.google.common.collect.Lists; -public enum ClusterConfiguration { +public enum ClusterManager { //first one needs to be a master //HUGE(new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(false, true,false), new NodeSettings(false, true, false)), @@ -62,7 +62,7 @@ public enum ClusterConfiguration { private List nodeSettings = new LinkedList<>(); - private ClusterConfiguration(NodeSettings... settings) { + private ClusterManager(NodeSettings... settings) { nodeSettings.addAll(Arrays.asList(settings)); } diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 2ed877c021..ee01344bfb 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -66,7 +66,7 @@ public class LocalCluster extends ExternalResource implements AutoCloseable, Ope protected static final AtomicLong num = new AtomicLong(); private final List> plugins; - private final ClusterConfiguration clusterConfiguration; + private final ClusterManager clusterConfiguration; private final TestSecurityConfig testSecurityConfig; private Settings nodeOverride; private final String clusterName; @@ -78,7 +78,7 @@ public class LocalCluster extends ExternalResource implements AutoCloseable, Ope private final List testIndices; private LocalCluster(String clusterName, TestSecurityConfig testSgConfig, Settings nodeOverride, - ClusterConfiguration clusterConfiguration, List> plugins, TestCertificates testCertificates, + ClusterManager clusterConfiguration, List> plugins, TestCertificates testCertificates, List clusterDependencies, Map remotes, List testIndices) { this.plugins = plugins; this.testCertificates = testCertificates; @@ -221,7 +221,7 @@ public static class Builder { private Map remoteClusters = new HashMap<>(); private List clusterDependencies = new ArrayList<>(); private List testIndices = new ArrayList<>(); - private ClusterConfiguration clusterConfiguration = ClusterConfiguration.DEFAULT; + private ClusterManager clusterConfiguration = ClusterManager.DEFAULT; private TestSecurityConfig testSecurityConfig = new TestSecurityConfig(); private String clusterName = "local_cluster"; private TestCertificates testCertificates; @@ -238,13 +238,13 @@ public Builder dependsOn(Object object) { return this; } - public Builder clusterConfiguration(ClusterConfiguration clusterConfiguration) { + public Builder clusterConfiguration(ClusterManager clusterConfiguration) { this.clusterConfiguration = clusterConfiguration; return this; } public Builder singleNode() { - this.clusterConfiguration = ClusterConfiguration.SINGLENODE; + this.clusterConfiguration = ClusterManager.SINGLENODE; return this; } diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index 6c92b12789..147837a2cf 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -70,7 +70,7 @@ import org.opensearch.node.PluginAwareNode; import org.opensearch.plugins.Plugin; import org.opensearch.test.framework.certificate.TestCertificates; -import org.opensearch.test.framework.cluster.ClusterConfiguration.NodeSettings; +import org.opensearch.test.framework.cluster.ClusterManager.NodeSettings; import org.opensearch.transport.BindTransportException; import org.opensearch.transport.TransportInfo; @@ -86,7 +86,7 @@ public class LocalOpenSearchCluster { private static final Logger log = LogManager.getLogger(LocalOpenSearchCluster.class); private final String clusterName; - private final ClusterConfiguration clusterConfiguration; + private final ClusterManager clusterConfiguration; private final NodeSettingsSupplier nodeSettingsSupplier; private final List> additionalPlugins; private final List allNodes = new ArrayList<>(); @@ -102,7 +102,7 @@ public class LocalOpenSearchCluster { private boolean started; private Random random = new Random(); - public LocalOpenSearchCluster(String clusterName, ClusterConfiguration clusterConfiguration, NodeSettingsSupplier nodeSettingsSupplier, + public LocalOpenSearchCluster(String clusterName, ClusterManager clusterConfiguration, NodeSettingsSupplier nodeSettingsSupplier, List> additionalPlugins, TestCertificates testCertificates) { this.clusterName = clusterName; this.clusterConfiguration = clusterConfiguration; From c79c68ead4d04de498f0eca376d9f30b309d893d Mon Sep 17 00:00:00 2001 From: Jochen Kressin Date: Tue, 2 Aug 2022 15:54:39 +0200 Subject: [PATCH 07/47] throw exception if index already exists Signed-off-by: Jochen Kressin --- .../java/org/opensearch/test/GenericIntegrationTest.java | 2 +- .../java/org/opensearch/test/framework/TestIndex.java | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java index c01ec32fa8..17c78239ea 100644 --- a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java +++ b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java @@ -51,7 +51,7 @@ public class GenericIntegrationTest { @ClassRule public static LocalCluster cluster = new LocalCluster.Builder().clusterConfiguration(ClusterManager.THREE_MASTERS) .authc(AUTHC_HTTPBASIC_INTERNAL) - .users(INDEX_A_USER) + .users(USER_ADMIN, INDEX_A_USER) .indices(INDEX_A, INDEX_B).build(); @Test diff --git a/src/newTest/java/org/opensearch/test/framework/TestIndex.java b/src/newTest/java/org/opensearch/test/framework/TestIndex.java index 035418c2b4..1dbf3f3ca9 100644 --- a/src/newTest/java/org/opensearch/test/framework/TestIndex.java +++ b/src/newTest/java/org/opensearch/test/framework/TestIndex.java @@ -44,11 +44,9 @@ public TestIndex(String name, Settings settings) { } - public void create(Client client) { - if (!client.admin().indices().exists(new IndicesExistsRequest(name)).actionGet().isExists()) { - client.admin().indices().create(new CreateIndexRequest(name).settings(settings)).actionGet(); - } - } + public void create(Client client) { + client.admin().indices().create(new CreateIndexRequest(name).settings(settings)).actionGet(); + } public String getName() { return name; From 8f94acfb9a745507b3287db4d7252a09cdce98d7 Mon Sep 17 00:00:00 2001 From: Jochen Kressin Date: Tue, 2 Aug 2022 16:16:25 +0200 Subject: [PATCH 08/47] fixed incorrect test Signed-off-by: Jochen Kressin --- .../java/org/opensearch/test/GenericIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java index 17c78239ea..98fd63b084 100644 --- a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java +++ b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java @@ -67,7 +67,7 @@ public void testIndexAUserHasOnlyAccessToIndexA() throws Exception { assertThat(client.get("index-a/_search?pretty").getStatusCode(), equalTo(HttpStatus.SC_OK)); // demo: work with JSON response body and check values assertThat(client.get("index-a/_search?pretty").getIntFromJsonBody("/_source/hits/value"), equalTo(0)); - assertThat(client.get("index-b/_search?pretty"), equalTo(HttpStatus.SC_FORBIDDEN)); + assertThat(client.get("index-b/_search?pretty").getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); } } From f1cef31e82402a2d60574de5402b812961c97a24 Mon Sep 17 00:00:00 2001 From: Jochen Kressin Date: Tue, 2 Aug 2022 16:28:47 +0200 Subject: [PATCH 09/47] fixed code hygiene issues Signed-off-by: Jochen Kressin --- .../test/GenericIntegrationTest.java | 12 +++---- .../test/PrivilegesEvaluatorTest.java | 10 +++--- .../opensearch/test/SecurityRolesTests.java | 12 +++---- .../opensearch/test/framework/TestIndex.java | 1 - .../test/framework/TestSecurityConfig.java | 3 +- .../framework/cluster/ClusterManager.java | 5 ++- .../test/framework/cluster/LocalCluster.java | 2 +- .../cluster/LocalOpenSearchCluster.java | 18 ++--------- .../cluster/OpenSearchClientProvider.java | 2 +- .../test/framework/cluster/SocketUtils.java | 2 +- .../framework/cluster/TestRestClient.java | 31 +++++-------------- 11 files changed, 33 insertions(+), 65 deletions(-) diff --git a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java index 98fd63b084..b993a7a317 100644 --- a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java +++ b/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java @@ -11,15 +11,12 @@ package org.opensearch.test; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; -import static org.opensearch.test.framework.TestSecurityConfig.User.USER_ADMIN; - +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.apache.http.HttpStatus; import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; + import org.opensearch.test.framework.TestIndex; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; @@ -27,7 +24,10 @@ import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.TestSecurityConfig.User.USER_ADMIN; /** * WIP diff --git a/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java b/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java index cbea84ff2f..b3521af808 100644 --- a/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java +++ b/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java @@ -11,21 +11,21 @@ package org.opensearch.test; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; - +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.apache.http.HttpStatus; import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; + import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; /** * This is a port for the test diff --git a/src/newTest/java/org/opensearch/test/SecurityRolesTests.java b/src/newTest/java/org/opensearch/test/SecurityRolesTests.java index 956b3eb17f..416b6580bd 100644 --- a/src/newTest/java/org/opensearch/test/SecurityRolesTests.java +++ b/src/newTest/java/org/opensearch/test/SecurityRolesTests.java @@ -26,16 +26,12 @@ package org.opensearch.test; -import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; - +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.apache.http.HttpStatus; -import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; + import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.cluster.ClusterManager; @@ -43,7 +39,9 @@ import org.opensearch.test.framework.cluster.TestRestClient; import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) diff --git a/src/newTest/java/org/opensearch/test/framework/TestIndex.java b/src/newTest/java/org/opensearch/test/framework/TestIndex.java index 1dbf3f3ca9..97ace5e4b8 100644 --- a/src/newTest/java/org/opensearch/test/framework/TestIndex.java +++ b/src/newTest/java/org/opensearch/test/framework/TestIndex.java @@ -29,7 +29,6 @@ package org.opensearch.test.framework; import org.opensearch.action.admin.indices.create.CreateIndexRequest; -import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.opensearch.client.Client; import org.opensearch.common.settings.Settings; diff --git a/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java index 38523bb13f..ef8afc33c3 100644 --- a/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -38,12 +38,12 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bouncycastle.crypto.generators.OpenBSDBCrypt; + import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.WriteRequest.RefreshPolicy; @@ -56,7 +56,6 @@ import org.opensearch.security.action.configupdate.ConfigUpdateRequest; import org.opensearch.security.action.configupdate.ConfigUpdateResponse; import org.opensearch.security.securityconf.impl.CType; -import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.cluster.NestedValueMap; import org.opensearch.test.framework.cluster.NestedValueMap.Path; import org.opensearch.test.framework.cluster.OpenSearchClientProvider.UserCredentialsHolder; diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java b/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java index 5d6deb7f3f..dfb4e95937 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java @@ -35,6 +35,8 @@ import java.util.List; import java.util.stream.Collectors; +import com.google.common.collect.Lists; + import org.opensearch.index.reindex.ReindexPlugin; import org.opensearch.join.ParentJoinPlugin; import org.opensearch.percolator.PercolatorPlugin; @@ -43,8 +45,6 @@ import org.opensearch.security.OpenSearchSecurityPlugin; import org.opensearch.transport.Netty4Plugin; -import com.google.common.collect.Lists; - public enum ClusterManager { //first one needs to be a master //HUGE(new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(false, true,false), new NodeSettings(false, true, false)), @@ -108,7 +108,6 @@ public NodeSettings(boolean masterNode, boolean dataNode) { public NodeSettings(boolean masterNode, boolean dataNode, List> additionalPlugins) { this(masterNode, dataNode); - this.plugins.addAll(additionalPlugins); } diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index ee01344bfb..590f0ae280 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -37,12 +37,12 @@ import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.junit.rules.ExternalResource; + import org.opensearch.client.Client; import org.opensearch.common.settings.Settings; import org.opensearch.node.PluginAwareNode; diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index 147837a2cf..3c42a85835 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -28,8 +28,6 @@ package org.opensearch.test.framework.cluster; -import static org.junit.Assert.assertEquals; - import java.io.File; import java.io.IOException; import java.net.InetAddress; @@ -43,38 +41,28 @@ import java.util.Random; import java.util.SortedSet; import java.util.concurrent.CompletableFuture; -import java.util.function.Predicate; import java.util.stream.Collectors; +import com.google.common.net.InetAddresses; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.OpenSearchTimeoutException; + import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; -import org.opensearch.action.admin.cluster.node.info.NodeInfo; -import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest; -import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.AdminClient; import org.opensearch.client.Client; import org.opensearch.cluster.health.ClusterHealthStatus; -import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.http.BindHttpException; -import org.opensearch.http.HttpInfo; import org.opensearch.node.PluginAwareNode; import org.opensearch.plugins.Plugin; import org.opensearch.test.framework.certificate.TestCertificates; import org.opensearch.test.framework.cluster.ClusterManager.NodeSettings; import org.opensearch.transport.BindTransportException; -import org.opensearch.transport.TransportInfo; -import com.google.common.net.InetAddresses; +import static org.junit.Assert.assertEquals; public class LocalOpenSearchCluster { diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java b/src/newTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java index c98d052c57..02817a1dfa 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java @@ -35,7 +35,6 @@ import java.security.cert.X509Certificate; import java.util.Arrays; import java.util.Base64; -import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -46,6 +45,7 @@ import org.apache.http.Header; import org.apache.http.message.BasicHeader; + import org.opensearch.security.support.PemKeyReader; import org.opensearch.test.framework.certificate.TestCertificates; diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/SocketUtils.java b/src/newTest/java/org/opensearch/test/framework/cluster/SocketUtils.java index 36718b6a43..35ef48f022 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/SocketUtils.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/SocketUtils.java @@ -316,4 +316,4 @@ SortedSet findAvailablePorts(int numRequested, int minPort, int maxPort } } -} \ No newline at end of file +} diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index 8f486916e2..dbe13d5cdf 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -41,6 +41,8 @@ import javax.net.ssl.SSLContext; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import org.apache.commons.io.IOUtils; import org.apache.http.Header; import org.apache.http.HttpEntity; @@ -64,14 +66,11 @@ import org.apache.http.message.BasicHeader; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; + import org.opensearch.common.Strings; import org.opensearch.common.xcontent.ToXContentObject; import org.opensearch.security.DefaultObjectMapper; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Lists; - public class TestRestClient implements AutoCloseable { private static final Logger log = LogManager.getLogger(TestRestClient.class); @@ -198,20 +197,6 @@ public TestRestClient trackResources() { return this; } - private void cleanupResources() { - if (puttedResourcesList.size() > 0) { - log.info("Cleaning up " + puttedResourcesList); - - for (String resource : Lists.reverse(puttedResourcesList)) { - try { - delete(resource); - } catch (Exception e) { - log.error("Error cleaning up created resources " + resource, e); - } - } - } - } - protected final String getHttpServerUri() { return "http" + (enableHTTPClientSSL ? "s" : "") + "://" + nodeHttpAddress.getHostString() + ":" + nodeHttpAddress.getPort(); } @@ -373,11 +358,6 @@ public void setLocalAddress(InetAddress inetAddress) { } } - @Override - public void close() throws IOException { - cleanupResources(); - } - public boolean isSendHTTPClientCertificate() { return sendHTTPClientCertificate; } @@ -386,4 +366,9 @@ public void setSendHTTPClientCertificate(boolean sendHTTPClientCertificate) { this.sendHTTPClientCertificate = sendHTTPClientCertificate; } + @Override + public void close() throws Exception { + // TODO: Is there anything to clean up here? + } + } From 6f4dae9967b60a47a152c98f822bd20ed281d4f1 Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Mon, 8 Aug 2022 16:13:28 +0200 Subject: [PATCH 10/47] TestRestClient uses unchecked exceptions Signed-off-by: Lukasz Soszynski --- .../cluster/RestClientException.java | 7 +++ .../framework/cluster/TestRestClient.java | 57 ++++++++++--------- 2 files changed, 38 insertions(+), 26 deletions(-) create mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/RestClientException.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/RestClientException.java b/src/newTest/java/org/opensearch/test/framework/cluster/RestClientException.java new file mode 100644 index 0000000000..78cc158863 --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/cluster/RestClientException.java @@ -0,0 +1,7 @@ +package org.opensearch.test.framework.cluster; + +class RestClientException extends RuntimeException { + public RestClientException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index dbe13d5cdf..69e5746a5c 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -29,6 +29,7 @@ package org.opensearch.test.framework.cluster; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; @@ -46,6 +47,7 @@ import org.apache.commons.io.IOUtils; import org.apache.http.Header; import org.apache.http.HttpEntity; +import org.apache.http.client.ClientProtocolException; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; @@ -92,25 +94,25 @@ public TestRestClient(InetSocketAddress nodeHttpAddress, List
headers, S this.sslContext = sslContext; } - public HttpResponse get(String path, Header... headers) throws Exception { + public HttpResponse get(String path, Header... headers) { return executeRequest(new HttpGet(getHttpServerUri() + "/" + path), headers); } - public HttpResponse getAuthInfo( Header... headers) throws Exception { + public HttpResponse getAuthInfo( Header... headers) { return executeRequest(new HttpGet(getHttpServerUri() + "/_opendistro/_security/authinfo?pretty"), headers); } - public HttpResponse head(String path, Header... headers) throws Exception { + public HttpResponse head(String path, Header... headers) { return executeRequest(new HttpHead(getHttpServerUri() + "/" + path), headers); } - public HttpResponse options(String path, Header... headers) throws Exception { + public HttpResponse options(String path, Header... headers) { return executeRequest(new HttpOptions(getHttpServerUri() + "/" + path), headers); } - public HttpResponse putJson(String path, String body, Header... headers) throws Exception { + public HttpResponse putJson(String path, String body, Header... headers) { HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); - uriRequest.setEntity(new StringEntity(body)); + uriRequest.setEntity(toStringEntity(body)); HttpResponse response = executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); @@ -122,11 +124,19 @@ public HttpResponse putJson(String path, String body, Header... headers) throws return response; } - public HttpResponse putJson(String path, ToXContentObject body) throws Exception { + private StringEntity toStringEntity(String body) { + try { + return new StringEntity(body); + } catch (UnsupportedEncodingException e) { + throw new RestClientException("Cannot create string entity", e); + } + } + + public HttpResponse putJson(String path, ToXContentObject body) { return putJson(path, Strings.toString(body)); } - public HttpResponse put(String path) throws Exception { + public HttpResponse put(String path) { HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); HttpResponse response = executeRequest(uriRequest); @@ -138,37 +148,35 @@ public HttpResponse put(String path) throws Exception { return response; } - public HttpResponse delete(String path, Header... headers) throws Exception { + public HttpResponse delete(String path, Header... headers) { return executeRequest(new HttpDelete(getHttpServerUri() + "/" + path), headers); } - public HttpResponse postJson(String path, String body, Header... headers) throws Exception { + public HttpResponse postJson(String path, String body, Header... headers) { HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); - uriRequest.setEntity(new StringEntity(body)); + uriRequest.setEntity(toStringEntity(body)); return executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); } - public HttpResponse postJson(String path, ToXContentObject body) throws Exception { + public HttpResponse postJson(String path, ToXContentObject body) { return postJson(path, Strings.toString(body)); } - public HttpResponse post(String path) throws Exception { + public HttpResponse post(String path) { HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); return executeRequest(uriRequest); } - public HttpResponse patch(String path, String body) throws Exception { + public HttpResponse patch(String path, String body) { HttpPatch uriRequest = new HttpPatch(getHttpServerUri() + "/" + path); - uriRequest.setEntity(new StringEntity(body)); + uriRequest.setEntity(toStringEntity(body)); return executeRequest(uriRequest, CONTENT_TYPE_JSON); } - public HttpResponse executeRequest(HttpUriRequest uriRequest, Header... requestSpecificHeaders) throws Exception { + public HttpResponse executeRequest(HttpUriRequest uriRequest, Header... requestSpecificHeaders) { - CloseableHttpClient httpClient = null; - try { + try(CloseableHttpClient httpClient = getHTTPClient()) { - httpClient = getHTTPClient(); if (requestSpecificHeaders != null && requestSpecificHeaders.length > 0) { for (int i = 0; i < requestSpecificHeaders.length; i++) { @@ -184,11 +192,8 @@ public HttpResponse executeRequest(HttpUriRequest uriRequest, Header... requestS HttpResponse res = new HttpResponse(httpClient.execute(uriRequest)); log.debug(res.getBody()); return res; - } finally { - - if (httpClient != null) { - httpClient.close(); - } + } catch (IOException e) { + throw new RestClientException("Error occured during HTTP request execution", e); } } @@ -201,7 +206,7 @@ protected final String getHttpServerUri() { return "http" + (enableHTTPClientSSL ? "s" : "") + "://" + nodeHttpAddress.getHostString() + ":" + nodeHttpAddress.getPort(); } - protected final CloseableHttpClient getHTTPClient() throws Exception { + protected final CloseableHttpClient getHTTPClient() { final HttpClientBuilder hcb = HttpClients.custom(); @@ -367,7 +372,7 @@ public void setSendHTTPClientCertificate(boolean sendHTTPClientCertificate) { } @Override - public void close() throws Exception { + public void close() { // TODO: Is there anything to clean up here? } From bbf5dc97eee90b4c31c1db0352d285c0f74c4026 Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Mon, 8 Aug 2022 16:15:56 +0200 Subject: [PATCH 11/47] List of plugins in ClusterManager is immutable. Signed-off-by: Lukasz Soszynski --- .../framework/cluster/ClusterManager.java | 50 ++++++++++--------- .../cluster/LocalOpenSearchCluster.java | 4 +- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java b/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java index dfb4e95937..ce9903b59b 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java @@ -28,15 +28,14 @@ package org.opensearch.test.framework.cluster; -import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; -import com.google.common.collect.Lists; - import org.opensearch.index.reindex.ReindexPlugin; import org.opensearch.join.ParentJoinPlugin; import org.opensearch.percolator.PercolatorPlugin; @@ -45,6 +44,8 @@ import org.opensearch.security.OpenSearchSecurityPlugin; import org.opensearch.transport.Netty4Plugin; +import static java.util.Collections.unmodifiableList; + public enum ClusterManager { //first one needs to be a master //HUGE(new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(false, true,false), new NodeSettings(false, true, false)), @@ -67,15 +68,15 @@ private ClusterManager(NodeSettings... settings) { } public List getNodeSettings() { - return Collections.unmodifiableList(nodeSettings); + return unmodifiableList(nodeSettings); } public List getMasterNodeSettings() { - return Collections.unmodifiableList(nodeSettings.stream().filter(a -> a.masterNode).collect(Collectors.toList())); + return unmodifiableList(nodeSettings.stream().filter(a -> a.masterNode).collect(Collectors.toList())); } public List getNonMasterNodeSettings() { - return Collections.unmodifiableList(nodeSettings.stream().filter(a -> !a.masterNode).collect(Collectors.toList())); + return unmodifiableList(nodeSettings.stream().filter(a -> !a.masterNode).collect(Collectors.toList())); } public int getNodes() { @@ -95,36 +96,39 @@ public int getClientNodes() { } public static class NodeSettings { - public boolean masterNode; - public boolean dataNode; - public List> plugins = Lists.newArrayList(Netty4Plugin.class, OpenSearchSecurityPlugin.class, MatrixAggregationPlugin.class, - ParentJoinPlugin.class, PercolatorPlugin.class, ReindexPlugin.class); + + private final static List> DEFAULT_PLUGINS = List.of(Netty4Plugin.class, OpenSearchSecurityPlugin.class, + MatrixAggregationPlugin.class, ParentJoinPlugin.class, PercolatorPlugin.class, ReindexPlugin.class); + public final boolean masterNode; + public final boolean dataNode; + public final List> plugins; public NodeSettings(boolean masterNode, boolean dataNode) { + this(masterNode, dataNode, Collections.emptyList()); + } + + public NodeSettings(boolean masterNode, boolean dataNode, List> additionalPlugins) { super(); this.masterNode = masterNode; this.dataNode = dataNode; + this.plugins = mergePlugins(additionalPlugins, DEFAULT_PLUGINS); } - public NodeSettings(boolean masterNode, boolean dataNode, List> additionalPlugins) { - this(masterNode, dataNode); - this.plugins.addAll(additionalPlugins); + private List> mergePlugins(Collection>...plugins) { + List> mergedPlugins = Arrays.stream(plugins) + .filter(Objects::nonNull) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + return unmodifiableList(mergedPlugins); } @SuppressWarnings("unchecked") public Class[] getPlugins() { return plugins.toArray(new Class[0]); } - - @SuppressWarnings("unchecked") - public Class[] getPlugins(List> additionalPlugins) { - List> plugins = new ArrayList<>(this.plugins); - - if (additionalPlugins != null) { - plugins.addAll(additionalPlugins); - } - - return plugins.toArray(new Class[0]); + + public Class[] pluginsWithAddition(List> additionalPlugins) { + return mergePlugins(plugins, additionalPlugins).toArray(Class[]::new); } } } diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index 3c42a85835..9ab3a0533f 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -380,8 +380,8 @@ public class Node implements OpenSearchClientProvider { CompletableFuture start() { CompletableFuture completableFuture = new CompletableFuture<>(); - - this.node = new PluginAwareNode(nodeSettings.masterNode, getOpenSearchSettings(), nodeSettings.getPlugins(additionalPlugins)); + Class[] mergedPlugins = nodeSettings.pluginsWithAddition(additionalPlugins); + this.node = new PluginAwareNode(nodeSettings.masterNode, getOpenSearchSettings(), mergedPlugins); new Thread(new Runnable() { From 02897495b66a8aef62dc3fd38e0dce7601755a35 Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Tue, 9 Aug 2022 15:44:19 +0200 Subject: [PATCH 12/47] Unused and overloaded version of user method removed from LocalCluste cluss. Signed-off-by: Lukasz Soszynski --- .../test/framework/cluster/LocalCluster.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 590f0ae280..c72a6985ee 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -295,21 +295,6 @@ public Builder users(TestSecurityConfig.User... users) { return this; } - public Builder user(TestSecurityConfig.User user) { - testSecurityConfig.user(user); - return this; - } - - public Builder user(String name, String password, String... sgRoles) { - testSecurityConfig.user(name, password, sgRoles); - return this; - } - - public Builder user(String name, String password, Role... sgRoles) { - testSecurityConfig.user(name, password, sgRoles); - return this; - } - public Builder roles(Role... roles) { testSecurityConfig.roles(roles); return this; From f81ccc4aeab1a6431cfef111226431f50830a303 Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Tue, 9 Aug 2022 16:42:35 +0200 Subject: [PATCH 13/47] Unused constructor removed from class ContextHeaderDecoratorClient Signed-off-by: Lukasz Soszynski --- .../cluster/ContextHeaderDecoratorClient.java | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java b/src/newTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java index 76805d1ea6..2c788598d9 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java @@ -1,7 +1,6 @@ package org.opensearch.test.framework.cluster; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import org.opensearch.action.ActionListener; @@ -23,10 +22,6 @@ public ContextHeaderDecoratorClient(Client in, Map headers) { this.headers = headers != null ? headers : Collections.emptyMap(); } - public ContextHeaderDecoratorClient(Client in, String... headers) { - this(in, arrayToMap(headers)); - } - @Override protected void doExecute(ActionType action, Request request, ActionListener listener) { @@ -39,22 +34,4 @@ protected void super.doExecute(action, request, wrappedListener); } } - - private static Map arrayToMap(String[] headers) { - if (headers == null) { - return null; - } - - if (headers.length % 2 != 0) { - throw new IllegalArgumentException("The headers array must consist of key-value pairs"); - } - - Map result = new HashMap<>(headers.length / 2); - - for (int i = 0; i < headers.length; i += 2) { - result.put(headers[i], headers[i + 1]); - } - - return result; - } } From 3e6179fadc063c4aef06a25de4122c4a0461f8eb Mon Sep 17 00:00:00 2001 From: Kacper Trochimiak Date: Mon, 8 Aug 2022 09:35:09 +0200 Subject: [PATCH 14/47] fix build.gradle - new test configuration, dependencies, desable forbiddenApisNewTest Signed-off-by: Kacper Trochimiak --- build.gradle | 79 +++++++++++-------- .../framework/cluster/TestRestClient.java | 1 - 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/build.gradle b/build.gradle index 0ec6023315..757edcb753 100644 --- a/build.gradle +++ b/build.gradle @@ -92,6 +92,12 @@ forbiddenPatterns.enabled = false testingConventions.enabled = false // Conflicts between runtime kafka-clients:3.0.1 & testRuntime kafka-clients:3.0.1:test jarHell.enabled = false +tasks.whenTaskAdded {task -> + if(task.name.contains("forbiddenApisNewTest")) { + task.enabled = false + } +} + test { include '**/*.class' @@ -221,29 +227,40 @@ bundlePlugin { } } -configurations.all { - resolutionStrategy { - force 'commons-codec:commons-codec:1.14' - force 'org.slf4j:slf4j-api:1.7.30' - force 'org.scala-lang:scala-library:2.13.8' - force 'commons-io:commons-io:2.11.0' - force "com.fasterxml.jackson:jackson-bom:${versions.jackson}" - force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" - force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}" - force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" - force "io.netty:netty-buffer:${versions.netty}" - force "io.netty:netty-common:${versions.netty}" - force "io.netty:netty-handler:${versions.netty}" - force "io.netty:netty-transport:${versions.netty}" +configurations { + all { + resolutionStrategy { + force 'commons-codec:commons-codec:1.14' + force 'org.slf4j:slf4j-api:1.7.30' + force 'org.scala-lang:scala-library:2.13.8' + force 'commons-io:commons-io:2.11.0' + force "com.fasterxml.jackson:jackson-bom:${versions.jackson}" + force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}" + force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + force "io.netty:netty-buffer:${versions.netty}" + force "io.netty:netty-common:${versions.netty}" + force "io.netty:netty-handler:${versions.netty}" + force "io.netty:netty-transport:${versions.netty}" } + } + + newTestImplementation.extendsFrom implementation + newTestRuntimeOnly.extendsFrom runtimeOnly } //create source set 'newTest' //add classes from the main source set to the compilation and runtime classpaths of the newTest sourceSets { newTest { - compileClasspath += sourceSets.main.output - runtimeClasspath += sourceSets.main.output + java { + compileClasspath += sourceSets.main.output + runtimeClasspath += sourceSets.main.output + srcDir file ('src/newTest/java') + } + resources { + srcDir file('src/newTest/resources') + } } } @@ -261,25 +278,6 @@ task newTest(type: Test) { //run the newTest task before the check task check.dependsOn newTest -configurations { - all { - resolutionStrategy { - force 'commons-codec:commons-codec:1.14' - force 'org.apache.santuario:xmlsec:2.2.3' - force 'org.cryptacular:cryptacular:1.2.4' - force 'net.minidev:json-smart:2.4.7' - force 'commons-cli:commons-cli:1.3.1' - force 'org.apache.httpcomponents:httpcore:4.4.12' - force "org.apache.commons:commons-lang3:3.4" - force "org.springframework:spring-core:5.3.20" - force "com.google.guava:guava:30.0-jre" - } - } - - //use testImplementation dependencies in the newTestImplementation configuration - newTestImplementation.extendsFrom testImplementation -} - dependencies { implementation 'jakarta.annotation:jakarta.annotation-api:1.3.5' implementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}" @@ -403,6 +401,17 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" compileOnly "org.opensearch:opensearch:${opensearch_version}" + + //new test framework: + newTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.7.1') { + exclude(group: 'junit', module: 'junit') + } + newTestImplementation 'junit:junit:4.13.2' + newTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}" + newTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}" + newTestImplementation 'commons-io:commons-io:2.7' + newTestImplementation 'org.apache.logging.log4j:log4j-core:2.17.1' + newTestImplementation 'org.hamcrest:hamcrest:2.1' } jar { diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index 69e5746a5c..7871747a08 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -47,7 +47,6 @@ import org.apache.commons.io.IOUtils; import org.apache.http.Header; import org.apache.http.HttpEntity; -import org.apache.http.client.ClientProtocolException; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; From f373131b9cb872af0959b17fa55c46731f1bbd59 Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Wed, 10 Aug 2022 17:22:21 +0200 Subject: [PATCH 15/47] Try with resources was used to close OpenSearch clinet in the LocalCluster class. Signed-off-by: Lukasz Soszynski --- .../org/opensearch/test/framework/cluster/LocalCluster.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index c72a6985ee..6810a45bc6 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -210,8 +210,9 @@ private void start() { private void initSecurityIndex(TestSecurityConfig testSecurityConfig) { log.info("Initializing OpenSearch Security index"); - Client client = new ContextHeaderDecoratorClient(this.getInternalNodeClient(), Map.of(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER , "true")); - testSecurityConfig.initIndex(client); + try(Client client = new ContextHeaderDecoratorClient(this.getInternalNodeClient(), Map.of(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER , "true"))) { + testSecurityConfig.initIndex(client); + } } public static class Builder { From d135827357d7819dd301e85c5d4215209d55b5e2 Mon Sep 17 00:00:00 2001 From: Kacper Trochimiak Date: Thu, 11 Aug 2022 10:51:01 +0200 Subject: [PATCH 16/47] TestCertificates - throw RuntimeException instead of IOException Signed-off-by: Kacper Trochimiak --- .../certificate/TestCertificates.java | 25 +++++++++++-------- ...inimumSecuritySettingsSupplierFactory.java | 24 ++++++++---------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/newTest/java/org/opensearch/test/framework/certificate/TestCertificates.java b/src/newTest/java/org/opensearch/test/framework/certificate/TestCertificates.java index 9ca0aa54e8..d5fe9a527c 100644 --- a/src/newTest/java/org/opensearch/test/framework/certificate/TestCertificates.java +++ b/src/newTest/java/org/opensearch/test/framework/certificate/TestCertificates.java @@ -30,34 +30,37 @@ */ public class TestCertificates { - public File getRootCertificate() throws IOException { + public File getRootCertificate() { return createTempFile("root", ".cert", Certificates.ROOT_CA_CERTIFICATE); } - public File getNodeCertificate(int node) throws IOException { + public File getNodeCertificate(int node) { return createTempFile("node-" + node, ".cert", Certificates.NODE_CERTIFICATE); } - public File getNodeKey(int node) throws IOException { + public File getNodeKey(int node) { return createTempFile("node-" + node, ".key", Certificates.NODE_KEY); } - public File getAdminCertificate() throws IOException { + public File getAdminCertificate() { return createTempFile("admin", ".cert", Certificates.ADMIN_CERTIFICATE); } - public File getAdminKey() throws IOException { + public File getAdminKey() { return createTempFile("admin", ".key", Certificates.ADMIN_KEY); } - public String[] getAdminDNs() throws IOException { + public String[] getAdminDNs() { return new String[] {"CN=kirk,OU=client,O=client,L=test,C=de"}; } - private File createTempFile(String name, String suffix, String contents) throws IOException { - Path path = Files.createTempFile(name, suffix); - Files.writeString(path, contents); - return path.toFile(); - + private File createTempFile(String name, String suffix, String contents) { + try { + Path path = Files.createTempFile(name, suffix); + Files.writeString(path, contents); + return path.toFile(); + } catch (IOException e) { + throw new RuntimeException(String.format("Error while creating a temp file: %s%s", name, suffix), e); + } } } diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java b/src/newTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java index 37728c14cc..f31eb2b440 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java @@ -59,20 +59,16 @@ private Settings.Builder minimumOpenSearchSettingsBuilder(int node, boolean sslO // TODO: At the moment the test node certificates have an OID set, so we do not need to // specify any node_dns here. Once we make generating and specifying - try { - builder.put("plugins.security.ssl.transport.pemtrustedcas_filepath", testCertificates.getRootCertificate().getAbsolutePath()); - builder.put("plugins.security.ssl.transport.pemcert_filepath", testCertificates.getNodeCertificate(node).getAbsolutePath()); - builder.put("plugins.security.ssl.transport.pemkey_filepath", testCertificates.getNodeKey(node).getAbsolutePath()); - - builder.put("plugins.security.ssl.http.enabled", true); - builder.put("plugins.security.ssl.http.pemtrustedcas_filepath", testCertificates.getRootCertificate().getAbsolutePath()); - builder.put("plugins.security.ssl.http.pemcert_filepath", testCertificates.getNodeCertificate(node).getAbsolutePath()); - builder.put("plugins.security.ssl.http.pemkey_filepath", testCertificates.getNodeKey(node).getAbsolutePath()); - - builder.putList("plugins.security.authcz.admin_dn", testCertificates.getAdminDNs()); - } catch (IOException e) { - throw new IllegalArgumentException("Invalid test certificates provided on local cluster start"); - } + builder.put("plugins.security.ssl.transport.pemtrustedcas_filepath", testCertificates.getRootCertificate().getAbsolutePath()); + builder.put("plugins.security.ssl.transport.pemcert_filepath", testCertificates.getNodeCertificate(node).getAbsolutePath()); + builder.put("plugins.security.ssl.transport.pemkey_filepath", testCertificates.getNodeKey(node).getAbsolutePath()); + + builder.put("plugins.security.ssl.http.enabled", true); + builder.put("plugins.security.ssl.http.pemtrustedcas_filepath", testCertificates.getRootCertificate().getAbsolutePath()); + builder.put("plugins.security.ssl.http.pemcert_filepath", testCertificates.getNodeCertificate(node).getAbsolutePath()); + builder.put("plugins.security.ssl.http.pemkey_filepath", testCertificates.getNodeKey(node).getAbsolutePath()); + + builder.putList("plugins.security.authcz.admin_dn", testCertificates.getAdminDNs()); return builder; From 69dfa1880b9ed90ba6a596d97da6e3e43bbcab68 Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Thu, 11 Aug 2022 12:21:03 +0200 Subject: [PATCH 17/47] Nodes objects are stored in one list (instead of four) in class LocalOpenSearchCluster. Methods which return random node removed. Signed-off-by: Lukasz Soszynski --- .../framework/cluster/ClusterManager.java | 12 ++ .../test/framework/cluster/LocalCluster.java | 6 +- .../cluster/LocalOpenSearchCluster.java | 130 +++++++----------- .../test/framework/cluster/NodeType.java | 5 + 4 files changed, 67 insertions(+), 86 deletions(-) create mode 100644 src/newTest/java/org/opensearch/test/framework/cluster/NodeType.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java b/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java index ce9903b59b..a9d4153b32 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java @@ -45,6 +45,9 @@ import org.opensearch.transport.Netty4Plugin; import static java.util.Collections.unmodifiableList; +import static org.opensearch.test.framework.cluster.NodeType.CLIENT; +import static org.opensearch.test.framework.cluster.NodeType.DATA; +import static org.opensearch.test.framework.cluster.NodeType.MASTER; public enum ClusterManager { //first one needs to be a master @@ -113,6 +116,15 @@ public NodeSettings(boolean masterNode, boolean dataNode, List> mergePlugins(Collection>...plugins) { List> mergedPlugins = Arrays.stream(plugins) diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 6810a45bc6..967c234c6e 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -165,17 +165,13 @@ public PluginAwareNode node() { } public List nodes() { - return this.localOpenSearchCluster.getAllNodes(); + return this.localOpenSearchCluster.getNodes(); } public LocalOpenSearchCluster.Node getNodeByName(String name) { return this.localOpenSearchCluster.getNodeByName(name); } - public LocalOpenSearchCluster.Node getRandomClientNode() { - return this.localOpenSearchCluster.randomClientNode(); - } - public boolean isStarted() { return localOpenSearchCluster != null; } diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index 9ab3a0533f..b920a766c2 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -38,6 +38,8 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Locale; +import java.util.Objects; import java.util.Random; import java.util.SortedSet; import java.util.concurrent.CompletableFuture; @@ -62,8 +64,11 @@ import org.opensearch.test.framework.cluster.ClusterManager.NodeSettings; import org.opensearch.transport.BindTransportException; +import static java.util.Objects.requireNonNull; import static org.junit.Assert.assertEquals; - +import static org.opensearch.test.framework.cluster.NodeType.CLIENT; +import static org.opensearch.test.framework.cluster.NodeType.DATA; +import static org.opensearch.test.framework.cluster.NodeType.MASTER; public class LocalOpenSearchCluster { @@ -77,10 +82,7 @@ public class LocalOpenSearchCluster { private final ClusterManager clusterConfiguration; private final NodeSettingsSupplier nodeSettingsSupplier; private final List> additionalPlugins; - private final List allNodes = new ArrayList<>(); - private final List masterNodes = new ArrayList<>(); - private final List dataNodes = new ArrayList<>(); - private final List clientNodes = new ArrayList<>(); + private final List nodes = new ArrayList<>(); private final TestCertificates testCertificates; private File clusterHomeDir; @@ -104,6 +106,16 @@ public LocalOpenSearchCluster(String clusterName, ClusterManager clusterConfigur } } + private List getNodesByType(NodeType nodeType) { + return nodes.stream() + .filter(currentNode -> currentNode.hasAssignedType(nodeType)) + .collect(Collectors.toList()); + } + + private long countNodesByType(NodeType nodeType) { + return getNodesByType(nodeType).stream().count(); + } + public void start() throws Exception { log.info("Starting {}", clusterName); @@ -139,7 +151,7 @@ public void start() throws Exception { log.info("Startup finished. Waiting for GREEN"); - waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(10), allNodes.size()); + waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(10), nodes.size()); log.info("Started: {}", this); @@ -154,25 +166,22 @@ public boolean isStarted() { } public void stop() { - - for (Node node : clientNodes) { + for (Node node : getNodesByType(CLIENT)) { node.stop(); } - for (Node node : dataNodes) { + for (Node node : getNodesByType(DATA)) { node.stop(); } - for (Node node : masterNodes) { + for (Node node : getNodesByType(MASTER)) { node.stop(); } } public void destroy() { stop(); - clientNodes.clear(); - dataNodes.clear(); - masterNodes.clear(); + nodes.clear(); try { FileUtils.deleteDirectory(clusterHomeDir); @@ -182,28 +191,24 @@ public void destroy() { } public Node clientNode() { - return findRunningNode(clientNodes, dataNodes, masterNodes); - } - - public Node randomClientNode() { - return randomRunningNode(clientNodes, dataNodes, masterNodes); + return findRunningNode(getNodesByType(CLIENT), getNodesByType(DATA), getNodesByType(MASTER)); } public Node masterNode() { - return findRunningNode(masterNodes); + return findRunningNode(getNodesByType(MASTER)); } - public List getAllNodes() { - return Collections.unmodifiableList(allNodes); + public List getNodes() { + return Collections.unmodifiableList(nodes); } public Node getNodeByName(String name) { - return allNodes.stream().filter(node -> node.getNodeName().equals(name)).findAny().orElseThrow(() -> new RuntimeException( - "No such node with name: " + name + "; available: " + allNodes.stream().map(Node::getNodeName).collect(Collectors.toList()))); + return nodes.stream().filter(node -> node.getNodeName().equals(name)).findAny().orElseThrow(() -> new RuntimeException( + "No such node with name: " + name + "; available: " + nodes.stream().map(Node::getNodeName).collect(Collectors.toList()))); } private boolean isNodeFailedWithPortCollision() { - return allNodes.stream().anyMatch(Node::isPortCollision); + return nodes.stream().anyMatch(Node::isPortCollision); } private void retry() throws Exception { @@ -215,10 +220,7 @@ private void retry() throws Exception { stop(); - this.allNodes.clear(); - this.masterNodes.clear(); - this.dataNodes.clear(); - this.clientNodes.clear(); + this.nodes.clear(); this.seedHosts = null; this.initialMasterHosts = null; this.clusterHomeDir = Files.createTempDirectory("local_cluster_" + clusterName + "_retry_" + retry).toFile(); @@ -246,35 +248,6 @@ private final Node findRunningNode(List nodes, List... moreNodes) { return null; } - - @SafeVarargs - private final Node randomRunningNode(List nodes, List... moreNodes) { - ArrayList runningNodes = new ArrayList<>(); - - for (Node node : nodes) { - if (node.isRunning()) { - runningNodes.add(node); - } - } - - if (moreNodes != null && moreNodes.length > 0) { - for (List nodesList : moreNodes) { - for (Node node : nodesList) { - if (node.isRunning()) { - runningNodes.add(node); - } - } - } - } - - if (runningNodes.size() == 0) { - return null; - } - - int index = this.random.nextInt(runningNodes.size()); - - return runningNodes.get(index); - } private CompletableFuture startNodes(List nodeSettingList, SortedSet transportPorts, SortedSet httpPorts) { Iterator transportPortIterator = transportPorts.iterator(); @@ -316,33 +289,30 @@ public void waitForCluster(ClusterHealthStatus status, TimeValue timeout, int ex @Override public String toString() { + String masterNodes = nodeByTypeToString(MASTER); + String dataNodes = nodeByTypeToString(DATA); + String clientNodes = nodeByTypeToString(CLIENT); return "\nES Cluster " + clusterName + "\nmaster nodes: " + masterNodes + "\n data nodes: " + dataNodes + "\nclient nodes: " + clientNodes + "\n"; } + private String nodeByTypeToString(NodeType type) { + return getNodesByType(type).stream().map(Objects::toString).collect(Collectors.joining(", ")); + } + private static List toHostList(Collection ports) { return ports.stream().map(port -> "127.0.0.1:" + port).collect(Collectors.toList()); } private String createNextNodeName(NodeSettings nodeSettings) { - List nodes; - String nodeType; - - if (nodeSettings.masterNode) { - nodes = this.masterNodes; - nodeType = "master"; - } else if (nodeSettings.dataNode) { - nodes = this.dataNodes; - nodeType = "data"; - } else { - nodes = this.clientNodes; - nodeType = "client"; - } - - return nodeType + "_" + nodes.size(); + NodeType type = nodeSettings.recognizeNodeType(); + long nodeTypeCount = countNodesByType(type); + String nodeType = type.name().toLowerCase(Locale.ROOT); + return nodeType + "_" + nodeTypeCount; } public class Node implements OpenSearchClientProvider { + private final NodeType nodeType; private final String nodeName; private final NodeSettings nodeSettings; private final File nodeHomeDir; @@ -357,7 +327,7 @@ public class Node implements OpenSearchClientProvider { private boolean portCollision = false; Node(NodeSettings nodeSettings, int transportPort, int httpPort) { - this.nodeName = createNextNodeName(nodeSettings); + this.nodeName = createNextNodeName(requireNonNull(nodeSettings, "Node settings are required.")); this.nodeSettings = nodeSettings; this.nodeHomeDir = new File(clusterHomeDir, nodeName); this.dataDir = new File(this.nodeHomeDir, "data"); @@ -368,14 +338,12 @@ public class Node implements OpenSearchClientProvider { this.httpAddress = new InetSocketAddress(hostAddress, httpPort); this.transportAddress = new InetSocketAddress(hostAddress, transportPort); - if (nodeSettings.masterNode) { - masterNodes.add(this); - } else if (nodeSettings.dataNode) { - dataNodes.add(this); - } else { - clientNodes.add(this); - } - allNodes.add(this); + this.nodeType = nodeSettings.recognizeNodeType(); + nodes.add(this); + } + + boolean hasAssignedType(NodeType type) { + return requireNonNull(type, "Node type is required.").equals(this.nodeType); } CompletableFuture start() { diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/NodeType.java b/src/newTest/java/org/opensearch/test/framework/cluster/NodeType.java new file mode 100644 index 0000000000..0d7e39c40c --- /dev/null +++ b/src/newTest/java/org/opensearch/test/framework/cluster/NodeType.java @@ -0,0 +1,5 @@ +package org.opensearch.test.framework.cluster; + +enum NodeType { + CLIENT, DATA, MASTER +} From 9c1a35d10a9eb0ac253043cbcf752a565f73bfbe Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Thu, 11 Aug 2022 12:55:04 +0200 Subject: [PATCH 18/47] OpenSearch claster is stopped immediatly, without invocations of thread sleep. Signed-off-by: Lukasz Soszynski --- .../test/framework/cluster/LocalCluster.java | 2 - .../cluster/LocalOpenSearchCluster.java | 39 ++++++++++++------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 967c234c6e..85bf2ebc0d 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -117,7 +117,6 @@ public void before() throws Throwable { protected void after() { if (localOpenSearchCluster != null && localOpenSearchCluster.isStarted()) { try { - Thread.sleep(1234); localOpenSearchCluster.destroy(); } catch (Exception e) { throw new RuntimeException(e); @@ -131,7 +130,6 @@ protected void after() { public void close() { if (localOpenSearchCluster != null && localOpenSearchCluster.isStarted()) { try { - Thread.sleep(100); localOpenSearchCluster.destroy(); } catch (Exception e) { throw new RuntimeException(e); diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index b920a766c2..e4891b92a2 100644 --- a/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -43,6 +43,7 @@ import java.util.Random; import java.util.SortedSet; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import com.google.common.net.InetAddresses; @@ -166,17 +167,19 @@ public boolean isStarted() { } public void stop() { + List> stopFutures = new ArrayList<>(); for (Node node : getNodesByType(CLIENT)) { - node.stop(); + stopFutures.add(node.stop(2, TimeUnit.SECONDS)); } for (Node node : getNodesByType(DATA)) { - node.stop(); + stopFutures.add(node.stop(2, TimeUnit.SECONDS)); } for (Node node : getNodesByType(MASTER)) { - node.stop(); + stopFutures.add(node.stop(2, TimeUnit.SECONDS)); } + CompletableFuture.allOf(stopFutures.toArray(size -> new CompletableFuture[size])).join(); } public void destroy() { @@ -400,21 +403,27 @@ public X getInjectable(Class clazz) { return node.injector().getInstance(clazz); } - public void stop() { - try { - log.info("Stopping {}", this); + public CompletableFuture stop(long timeout, TimeUnit timeUnit) { + return CompletableFuture.supplyAsync(() -> { + try { + log.info("Stopping {}", this); - running = false; + running = false; - if (node != null) { - node.close(); - node = null; - Thread.sleep(10); + if (node != null) { + node.close(); + boolean stopped = node.awaitClose(timeout, timeUnit); + node = null; + return stopped; + } else { + return false; + } + } catch (Throwable e) { + String message = "Error while stopping " + this; + log.warn(message, e); + throw new RuntimeException(message, e); } - - } catch (Throwable e) { - log.warn("Error while stopping " + this, e); - } + }); } @Override From 26f098b08022b7102ecdbd2092966a745fb9e667 Mon Sep 17 00:00:00 2001 From: Kacper Trochimiak Date: Thu, 11 Aug 2022 15:16:13 +0200 Subject: [PATCH 19/47] rename newTest to integrationTest Signed-off-by: Kacper Trochimiak --- build.gradle | 48 +++++++++---------- .../org/opensearch/node/PluginAwareNode.java | 0 .../test/GenericIntegrationTest.java | 0 .../test/PrivilegesEvaluatorTest.java | 0 .../opensearch/test/SecurityRolesTests.java | 0 .../opensearch/test/framework/TestIndex.java | 0 .../test/framework/TestSecurityConfig.java | 0 .../framework/certificate/Certificates.java | 0 .../certificate/TestCertificates.java | 0 .../framework/cluster/ClusterManager.java | 0 .../cluster/ContextHeaderDecoratorClient.java | 0 .../test/framework/cluster/LocalCluster.java | 0 .../cluster/LocalOpenSearchCluster.java | 0 ...inimumSecuritySettingsSupplierFactory.java | 0 .../framework/cluster/NestedValueMap.java | 0 .../cluster/NodeSettingsSupplier.java | 0 .../test/framework/cluster/NodeType.java | 0 .../cluster/OpenSearchClientProvider.java | 0 .../test/framework/cluster/PortAllocator.java | 0 .../cluster/RestClientException.java | 0 .../test/framework/cluster/SocketUtils.java | 0 .../framework/cluster/TestRestClient.java | 0 22 files changed, 24 insertions(+), 24 deletions(-) rename src/{newTest => integrationTest}/java/org/opensearch/node/PluginAwareNode.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/GenericIntegrationTest.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/PrivilegesEvaluatorTest.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/SecurityRolesTests.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/TestIndex.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/TestSecurityConfig.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/certificate/Certificates.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/certificate/TestCertificates.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/cluster/ClusterManager.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/cluster/LocalCluster.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/cluster/NestedValueMap.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/cluster/NodeType.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/cluster/PortAllocator.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/cluster/RestClientException.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/cluster/SocketUtils.java (100%) rename src/{newTest => integrationTest}/java/org/opensearch/test/framework/cluster/TestRestClient.java (100%) diff --git a/build.gradle b/build.gradle index 757edcb753..87a22d109c 100644 --- a/build.gradle +++ b/build.gradle @@ -93,7 +93,7 @@ testingConventions.enabled = false // Conflicts between runtime kafka-clients:3.0.1 & testRuntime kafka-clients:3.0.1:test jarHell.enabled = false tasks.whenTaskAdded {task -> - if(task.name.contains("forbiddenApisNewTest")) { + if(task.name.contains("forbiddenApisIntegrationTest")) { task.enabled = false } } @@ -245,38 +245,38 @@ configurations { } } - newTestImplementation.extendsFrom implementation - newTestRuntimeOnly.extendsFrom runtimeOnly + integrationTestImplementation.extendsFrom implementation + integrationTestRuntimeOnly.extendsFrom runtimeOnly } -//create source set 'newTest' -//add classes from the main source set to the compilation and runtime classpaths of the newTest +//create source set 'integrationTest' +//add classes from the main source set to the compilation and runtime classpaths of the integrationTest sourceSets { - newTest { + integrationTest { java { compileClasspath += sourceSets.main.output runtimeClasspath += sourceSets.main.output - srcDir file ('src/newTest/java') + srcDir file ('src/integrationTest/java') } resources { - srcDir file('src/newTest/resources') + srcDir file('src/integrationTest/resources') } } } -//add new task that runs new tests -task newTest(type: Test) { - description = 'Run new tests.' +//add new task that runs integration tests +task integrationTest(type: Test) { + description = 'Run integration tests.' group = 'verification' - testClassesDirs = sourceSets.newTest.output.classesDirs - classpath = sourceSets.newTest.runtimeClasspath + testClassesDirs = sourceSets.integrationTest.output.classesDirs + classpath = sourceSets.integrationTest.runtimeClasspath - //run the newTest task after the test task + //run the integrationTest task after the test task shouldRunAfter test } -//run the newTest task before the check task -check.dependsOn newTest +//run the integrationTest task before the check task +check.dependsOn integrationTest dependencies { implementation 'jakarta.annotation:jakarta.annotation-api:1.3.5' @@ -402,16 +402,16 @@ dependencies { compileOnly "org.opensearch:opensearch:${opensearch_version}" - //new test framework: - newTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.7.1') { + //integration test framework: + integrationTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.7.1') { exclude(group: 'junit', module: 'junit') } - newTestImplementation 'junit:junit:4.13.2' - newTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}" - newTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}" - newTestImplementation 'commons-io:commons-io:2.7' - newTestImplementation 'org.apache.logging.log4j:log4j-core:2.17.1' - newTestImplementation 'org.hamcrest:hamcrest:2.1' + integrationTestImplementation 'junit:junit:4.13.2' + integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}" + integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}" + integrationTestImplementation 'commons-io:commons-io:2.7' + integrationTestImplementation 'org.apache.logging.log4j:log4j-core:2.17.1' + integrationTestImplementation 'org.hamcrest:hamcrest:2.1' } jar { diff --git a/src/newTest/java/org/opensearch/node/PluginAwareNode.java b/src/integrationTest/java/org/opensearch/node/PluginAwareNode.java similarity index 100% rename from src/newTest/java/org/opensearch/node/PluginAwareNode.java rename to src/integrationTest/java/org/opensearch/node/PluginAwareNode.java diff --git a/src/newTest/java/org/opensearch/test/GenericIntegrationTest.java b/src/integrationTest/java/org/opensearch/test/GenericIntegrationTest.java similarity index 100% rename from src/newTest/java/org/opensearch/test/GenericIntegrationTest.java rename to src/integrationTest/java/org/opensearch/test/GenericIntegrationTest.java diff --git a/src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java b/src/integrationTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java similarity index 100% rename from src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java rename to src/integrationTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java diff --git a/src/newTest/java/org/opensearch/test/SecurityRolesTests.java b/src/integrationTest/java/org/opensearch/test/SecurityRolesTests.java similarity index 100% rename from src/newTest/java/org/opensearch/test/SecurityRolesTests.java rename to src/integrationTest/java/org/opensearch/test/SecurityRolesTests.java diff --git a/src/newTest/java/org/opensearch/test/framework/TestIndex.java b/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/TestIndex.java rename to src/integrationTest/java/org/opensearch/test/framework/TestIndex.java diff --git a/src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/TestSecurityConfig.java rename to src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java diff --git a/src/newTest/java/org/opensearch/test/framework/certificate/Certificates.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/Certificates.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/certificate/Certificates.java rename to src/integrationTest/java/org/opensearch/test/framework/certificate/Certificates.java diff --git a/src/newTest/java/org/opensearch/test/framework/certificate/TestCertificates.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/certificate/TestCertificates.java rename to src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/cluster/ClusterManager.java rename to src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java rename to src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/cluster/LocalCluster.java rename to src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java rename to src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java rename to src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java rename to src/integrationTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java rename to src/integrationTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/NodeType.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/cluster/NodeType.java rename to src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java rename to src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/PortAllocator.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/cluster/PortAllocator.java rename to src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/RestClientException.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/RestClientException.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/cluster/RestClientException.java rename to src/integrationTest/java/org/opensearch/test/framework/cluster/RestClientException.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/SocketUtils.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtils.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/cluster/SocketUtils.java rename to src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtils.java diff --git a/src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java similarity index 100% rename from src/newTest/java/org/opensearch/test/framework/cluster/TestRestClient.java rename to src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java From 69e3d3175ea98614c3c873af31ded775fab1d8a7 Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Thu, 11 Aug 2022 14:10:48 +0200 Subject: [PATCH 20/47] Method org.opensearch.test.framework.cluster.LocalOpenSearchCluster.Node#start returns StartStage enum instead of string. Signed-off-by: Lukasz Soszynski --- .../test/framework/cluster/LocalOpenSearchCluster.java | 10 +++++----- .../opensearch/test/framework/cluster/StartStage.java | 6 ++++++ 2 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index e4891b92a2..1ed88d00a0 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -255,7 +255,7 @@ private final Node findRunningNode(List nodes, List... moreNodes) { private CompletableFuture startNodes(List nodeSettingList, SortedSet transportPorts, SortedSet httpPorts) { Iterator transportPortIterator = transportPorts.iterator(); Iterator httpPortIterator = httpPorts.iterator(); - List> futures = new ArrayList<>(); + List> futures = new ArrayList<>(); for (NodeSettings nodeSettings : nodeSettingList) { Node node = new Node(nodeSettings, transportPortIterator.next(), httpPortIterator.next()); @@ -349,8 +349,8 @@ boolean hasAssignedType(NodeType type) { return requireNonNull(type, "Node type is required.").equals(this.nodeType); } - CompletableFuture start() { - CompletableFuture completableFuture = new CompletableFuture<>(); + CompletableFuture start() { + CompletableFuture completableFuture = new CompletableFuture<>(); Class[] mergedPlugins = nodeSettings.pluginsWithAddition(additionalPlugins); this.node = new PluginAwareNode(nodeSettings.masterNode, getOpenSearchSettings(), mergedPlugins); @@ -361,7 +361,7 @@ public void run() { try { node.start(); running = true; - completableFuture.complete("initialized"); + completableFuture.complete(StartStage.INITIALIZED); } catch (BindTransportException | BindHttpException e) { log.warn("Port collision detected for {}", this, e); portCollision = true; @@ -374,7 +374,7 @@ public void run() { node = null; PortAllocator.TCP.reserve(transportPort, httpPort); - completableFuture.complete("retry"); + completableFuture.complete(StartStage.RETRY); } catch (Throwable e) { log.error("Unable to start {}", this, e); diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java new file mode 100644 index 0000000000..2a0ee0020f --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java @@ -0,0 +1,6 @@ +package org.opensearch.test.framework.cluster; + +enum StartStage { + INITIALIZED, + RETRY +} From 216153e23088fbdd2a6dfb9d9ceea7af046a5302 Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Thu, 11 Aug 2022 17:30:46 +0200 Subject: [PATCH 21/47] Method org.opensearch.test.framework.cluster.LocalCluster.Builder#nodeSettings accepts map instead of varargs. Signed-off-by: Lukasz Soszynski --- .../opensearch/test/framework/cluster/LocalCluster.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 85bf2ebc0d..4381156c71 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -248,18 +248,15 @@ public Builder sgConfig(TestSecurityConfig testSgConfig) { return this; } - public Builder nodeSettings(Object... settings) { - for (int i = 0; i < settings.length - 1; i += 2) { - String key = String.valueOf(settings[i]); - Object value = settings[i + 1]; - + public Builder nodeSettings(Map settings) { + settings.forEach((key, value) -> { if (value instanceof List) { List values = ((List) value).stream().map(String::valueOf).collect(Collectors.toList()); nodeOverrideSettingsBuilder.putList(key, values); } else { nodeOverrideSettingsBuilder.put(key, String.valueOf(value)); } - } + }); return this; } From 85f599f1623488a86b90022fee86de83d30601fe Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Thu, 11 Aug 2022 17:49:03 +0200 Subject: [PATCH 22/47] Method getUnitTestForkNumber removed, system property forkno is no longer used. Signed-off-by: Lukasz Soszynski --- .../cluster/LocalOpenSearchCluster.java | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index 1ed88d00a0..40e04c9f4b 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -120,12 +120,11 @@ private long countNodesByType(NodeType nodeType) { public void start() throws Exception { log.info("Starting {}", clusterName); - int forkNumber = getUnitTestForkNumber(); int masterNodeCount = clusterConfiguration.getMasterNodes(); int nonMasterNodeCount = clusterConfiguration.getDataNodes() + clusterConfiguration.getClientNodes(); - SortedSet masterNodeTransportPorts = PortAllocator.TCP.allocate(clusterName, Math.max(masterNodeCount, 4), 5000 + forkNumber * 1000 + 300); - SortedSet masterNodeHttpPorts = PortAllocator.TCP.allocate(clusterName, masterNodeCount, 5000 + forkNumber * 1000 + 200); + SortedSet masterNodeTransportPorts = PortAllocator.TCP.allocate(clusterName, Math.max(masterNodeCount, 4), 5000 + 42 * 1000 + 300); + SortedSet masterNodeHttpPorts = PortAllocator.TCP.allocate(clusterName, masterNodeCount, 5000 + 42 * 1000 + 200); this.seedHosts = toHostList(masterNodeTransportPorts); this.initialMasterHosts = toHostList(masterNodeTransportPorts.stream().limit(masterNodeCount).collect(Collectors.toSet())); @@ -135,8 +134,8 @@ public void start() throws Exception { CompletableFuture masterNodeFuture = startNodes(clusterConfiguration.getMasterNodeSettings(), masterNodeTransportPorts, masterNodeHttpPorts); - SortedSet nonMasterNodeTransportPorts = PortAllocator.TCP.allocate(clusterName, nonMasterNodeCount, 5000 + forkNumber * 1000 + 310); - SortedSet nonMasterNodeHttpPorts = PortAllocator.TCP.allocate(clusterName, nonMasterNodeCount, 5000 + forkNumber * 1000 + 210); + SortedSet nonMasterNodeTransportPorts = PortAllocator.TCP.allocate(clusterName, nonMasterNodeCount, 5000 + 42 * 1000 + 310); + SortedSet nonMasterNodeHttpPorts = PortAllocator.TCP.allocate(clusterName, nonMasterNodeCount, 5000 + 42 * 1000 + 210); CompletableFuture nonMasterNodeFuture = startNodes(clusterConfiguration.getNonMasterNodeSettings(), nonMasterNodeTransportPorts, nonMasterNodeHttpPorts); @@ -482,16 +481,6 @@ public TestCertificates getTestCertificates() { return testCertificates; } } - - private static int getUnitTestForkNumber() { - String forkno = System.getProperty("forkno"); - - if (forkno != null && forkno.length() > 0) { - return Integer.parseInt(forkno.split("_")[1]); - } else { - return 42; - } - } public Random getRandom() { return random; From c32984e77e7c1902055fed95421224feb3af88ae Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Tue, 16 Aug 2022 14:43:45 +0200 Subject: [PATCH 23/47] ClusterBlockException does not appear in the course of integration tests. Log4j2 reads configuration from src/integrationTest/resources/log4j2-test.properties Signed-off-by: Lukasz Soszynski --- build.gradle | 3 +++ .../test/framework/cluster/LocalCluster.java | 8 +++++--- src/integrationTest/resources/log4j2-test.properties | 10 ++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 src/integrationTest/resources/log4j2-test.properties diff --git a/build.gradle b/build.gradle index 87a22d109c..27346ab8c9 100644 --- a/build.gradle +++ b/build.gradle @@ -261,6 +261,9 @@ sourceSets { resources { srcDir file('src/integrationTest/resources') } + processIntegrationTestResources { + duplicatesStrategy(DuplicatesStrategy.INCLUDE) + } } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 4381156c71..15e8905050 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -195,7 +195,7 @@ private void start() { index.create(client); } } - + } catch (Exception e) { log.error("Local ES cluster start failed", e); throw new RuntimeException(e); @@ -326,8 +326,10 @@ public LocalCluster build() { try { clusterName += "_" + num.incrementAndGet(); - - return new LocalCluster(clusterName, testSecurityConfig, nodeOverrideSettingsBuilder.build(), clusterConfiguration, plugins, + Settings settings = nodeOverrideSettingsBuilder + .put(ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, false) + .build(); + return new LocalCluster(clusterName, testSecurityConfig, settings, clusterConfiguration, plugins, testCertificates, clusterDependencies, remoteClusters, testIndices); } catch (Exception e) { log.error("Failed to build LocalCluster", e); diff --git a/src/integrationTest/resources/log4j2-test.properties b/src/integrationTest/resources/log4j2-test.properties new file mode 100644 index 0000000000..aba0156605 --- /dev/null +++ b/src/integrationTest/resources/log4j2-test.properties @@ -0,0 +1,10 @@ +status = info +name = Integration test logging configuration + +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %threadName %-5p %c{1}:%L - %m%n + +rootLogger.level = warn +rootLogger.appenderRef.stdout.ref = consoleLogger From 5b22c519e0d309c332463283b3d976350eeb02d0 Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Tue, 16 Aug 2022 16:42:34 +0200 Subject: [PATCH 24/47] Unused code related to flag trackResources removed from class TestRestClient. Signed-off-by: Lukasz Soszynski --- .../framework/cluster/TestRestClient.java | 29 ++----------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index 7871747a08..7704794059 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -36,9 +36,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Set; import javax.net.ssl.SSLContext; @@ -82,10 +80,7 @@ public class TestRestClient implements AutoCloseable { private RequestConfig requestConfig; private List
headers = new ArrayList<>(); private Header CONTENT_TYPE_JSON = new BasicHeader("Content-Type", "application/json"); - private boolean trackResources = false; private SSLContext sslContext; - private Set puttedResourcesSet = new HashSet<>(); - private List puttedResourcesList = new ArrayList<>(); public TestRestClient(InetSocketAddress nodeHttpAddress, List
headers, SSLContext sslContext) { this.nodeHttpAddress = nodeHttpAddress; @@ -112,15 +107,7 @@ public HttpResponse options(String path, Header... headers) { public HttpResponse putJson(String path, String body, Header... headers) { HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); uriRequest.setEntity(toStringEntity(body)); - - HttpResponse response = executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); - - if (response.getStatusCode() < 400 && trackResources && !puttedResourcesSet.contains(path)) { - puttedResourcesSet.add(path); - puttedResourcesList.add(path); - } - - return response; + return executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); } private StringEntity toStringEntity(String body) { @@ -137,14 +124,7 @@ public HttpResponse putJson(String path, ToXContentObject body) { public HttpResponse put(String path) { HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); - HttpResponse response = executeRequest(uriRequest); - - if (response.getStatusCode() < 400 && trackResources && !puttedResourcesSet.contains(path)) { - puttedResourcesSet.add(path); - puttedResourcesList.add(path); - } - - return response; + return executeRequest(uriRequest); } public HttpResponse delete(String path, Header... headers) { @@ -196,11 +176,6 @@ public HttpResponse executeRequest(HttpUriRequest uriRequest, Header... requestS } } - public TestRestClient trackResources() { - trackResources = true; - return this; - } - protected final String getHttpServerUri() { return "http" + (enableHTTPClientSSL ? "s" : "") + "://" + nodeHttpAddress.getHostString() + ":" + nodeHttpAddress.getPort(); } From 075adfa0b8b08fb2dc9330aec1b235f38b151257 Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Tue, 16 Aug 2022 17:27:32 +0200 Subject: [PATCH 25/47] Redundant code in method org.opensearch.test.framework.cluster.LocalCluster#after replaced by close() invocation Signed-off-by: Lukasz Soszynski --- .../test/framework/cluster/LocalCluster.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 15e8905050..057863087c 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -115,15 +115,7 @@ public void before() throws Throwable { @Override protected void after() { - if (localOpenSearchCluster != null && localOpenSearchCluster.isStarted()) { - try { - localOpenSearchCluster.destroy(); - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - localOpenSearchCluster = null; - } - } + close(); } @Override From f217c354abd307e42f846395576231f4814ae8b8 Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Tue, 16 Aug 2022 17:38:59 +0200 Subject: [PATCH 26/47] Rename from ClusterConfiguration to ClusterManager finished. Signed-off-by: Lukasz Soszynski --- .../test/GenericIntegrationTest.java | 2 +- .../test/PrivilegesEvaluatorTest.java | 2 +- .../opensearch/test/SecurityRolesTests.java | 2 +- .../test/framework/cluster/LocalCluster.java | 18 +++++++++--------- .../cluster/LocalOpenSearchCluster.java | 16 +++++++++------- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/test/GenericIntegrationTest.java b/src/integrationTest/java/org/opensearch/test/GenericIntegrationTest.java index b993a7a317..05346623e9 100644 --- a/src/integrationTest/java/org/opensearch/test/GenericIntegrationTest.java +++ b/src/integrationTest/java/org/opensearch/test/GenericIntegrationTest.java @@ -49,7 +49,7 @@ public class GenericIntegrationTest { // build our test cluster as a ClassRule @ClassRule - public static LocalCluster cluster = new LocalCluster.Builder().clusterConfiguration(ClusterManager.THREE_MASTERS) + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_MASTERS) .authc(AUTHC_HTTPBASIC_INTERNAL) .users(USER_ADMIN, INDEX_A_USER) .indices(INDEX_A, INDEX_B).build(); diff --git a/src/integrationTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java b/src/integrationTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java index b3521af808..9bd469cabb 100644 --- a/src/integrationTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java +++ b/src/integrationTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java @@ -47,7 +47,7 @@ public class PrivilegesEvaluatorTest { @ClassRule public static LocalCluster cluster = new LocalCluster.Builder() - .clusterConfiguration(ClusterManager.THREE_MASTERS).authc(AUTHC_HTTPBASIC_INTERNAL) + .clusterManager(ClusterManager.THREE_MASTERS).authc(AUTHC_HTTPBASIC_INTERNAL) .users(NEGATIVE_LOOKAHEAD, NEGATED_REGEX).build(); @Test diff --git a/src/integrationTest/java/org/opensearch/test/SecurityRolesTests.java b/src/integrationTest/java/org/opensearch/test/SecurityRolesTests.java index 416b6580bd..22742aea0c 100644 --- a/src/integrationTest/java/org/opensearch/test/SecurityRolesTests.java +++ b/src/integrationTest/java/org/opensearch/test/SecurityRolesTests.java @@ -53,7 +53,7 @@ public class SecurityRolesTests { @ClassRule public static LocalCluster cluster = new LocalCluster.Builder() - .clusterConfiguration(ClusterManager.THREE_MASTERS).anonymousAuth(true) + .clusterManager(ClusterManager.THREE_MASTERS).anonymousAuth(true) .authc(AUTHC_HTTPBASIC_INTERNAL).users(USER_SR).build(); @Test diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 057863087c..0d003accc0 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -66,7 +66,7 @@ public class LocalCluster extends ExternalResource implements AutoCloseable, Ope protected static final AtomicLong num = new AtomicLong(); private final List> plugins; - private final ClusterManager clusterConfiguration; + private final ClusterManager clusterManager; private final TestSecurityConfig testSecurityConfig; private Settings nodeOverride; private final String clusterName; @@ -78,11 +78,11 @@ public class LocalCluster extends ExternalResource implements AutoCloseable, Ope private final List testIndices; private LocalCluster(String clusterName, TestSecurityConfig testSgConfig, Settings nodeOverride, - ClusterManager clusterConfiguration, List> plugins, TestCertificates testCertificates, + ClusterManager clusterManager, List> plugins, TestCertificates testCertificates, List clusterDependencies, Map remotes, List testIndices) { this.plugins = plugins; this.testCertificates = testCertificates; - this.clusterConfiguration = clusterConfiguration; + this.clusterManager = clusterManager; this.testSecurityConfig = testSgConfig; this.nodeOverride = nodeOverride; this.clusterName = clusterName; @@ -172,7 +172,7 @@ public Random getRandom() { private void start() { try { - localOpenSearchCluster = new LocalOpenSearchCluster(clusterName, clusterConfiguration, + localOpenSearchCluster = new LocalOpenSearchCluster(clusterName, clusterManager, minimumOpenSearchSettingsSupplierFactory.minimumOpenSearchSettings(nodeOverride), plugins, testCertificates); localOpenSearchCluster.start(); @@ -208,7 +208,7 @@ public static class Builder { private Map remoteClusters = new HashMap<>(); private List clusterDependencies = new ArrayList<>(); private List testIndices = new ArrayList<>(); - private ClusterManager clusterConfiguration = ClusterManager.DEFAULT; + private ClusterManager clusterManager = ClusterManager.DEFAULT; private TestSecurityConfig testSecurityConfig = new TestSecurityConfig(); private String clusterName = "local_cluster"; private TestCertificates testCertificates; @@ -225,13 +225,13 @@ public Builder dependsOn(Object object) { return this; } - public Builder clusterConfiguration(ClusterManager clusterConfiguration) { - this.clusterConfiguration = clusterConfiguration; + public Builder clusterManager(ClusterManager clusterManager) { + this.clusterManager = clusterManager; return this; } public Builder singleNode() { - this.clusterConfiguration = ClusterManager.SINGLENODE; + this.clusterManager = ClusterManager.SINGLENODE; return this; } @@ -321,7 +321,7 @@ public LocalCluster build() { Settings settings = nodeOverrideSettingsBuilder .put(ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, false) .build(); - return new LocalCluster(clusterName, testSecurityConfig, settings, clusterConfiguration, plugins, + return new LocalCluster(clusterName, testSecurityConfig, settings, clusterManager, plugins, testCertificates, clusterDependencies, remoteClusters, testIndices); } catch (Exception e) { log.error("Failed to build LocalCluster", e); diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index 40e04c9f4b..fd16698d42 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -80,7 +80,7 @@ public class LocalOpenSearchCluster { private static final Logger log = LogManager.getLogger(LocalOpenSearchCluster.class); private final String clusterName; - private final ClusterManager clusterConfiguration; + private final ClusterManager clusterManager; private final NodeSettingsSupplier nodeSettingsSupplier; private final List> additionalPlugins; private final List nodes = new ArrayList<>(); @@ -93,10 +93,10 @@ public class LocalOpenSearchCluster { private boolean started; private Random random = new Random(); - public LocalOpenSearchCluster(String clusterName, ClusterManager clusterConfiguration, NodeSettingsSupplier nodeSettingsSupplier, + public LocalOpenSearchCluster(String clusterName, ClusterManager clusterManager, NodeSettingsSupplier nodeSettingsSupplier, List> additionalPlugins, TestCertificates testCertificates) { this.clusterName = clusterName; - this.clusterConfiguration = clusterConfiguration; + this.clusterManager = clusterManager; this.nodeSettingsSupplier = nodeSettingsSupplier; this.additionalPlugins = additionalPlugins; this.testCertificates = testCertificates; @@ -120,8 +120,8 @@ private long countNodesByType(NodeType nodeType) { public void start() throws Exception { log.info("Starting {}", clusterName); - int masterNodeCount = clusterConfiguration.getMasterNodes(); - int nonMasterNodeCount = clusterConfiguration.getDataNodes() + clusterConfiguration.getClientNodes(); + int masterNodeCount = clusterManager.getMasterNodes(); + int nonMasterNodeCount = clusterManager.getDataNodes() + clusterManager.getClientNodes(); SortedSet masterNodeTransportPorts = PortAllocator.TCP.allocate(clusterName, Math.max(masterNodeCount, 4), 5000 + 42 * 1000 + 300); SortedSet masterNodeHttpPorts = PortAllocator.TCP.allocate(clusterName, masterNodeCount, 5000 + 42 * 1000 + 200); @@ -131,13 +131,15 @@ public void start() throws Exception { started = true; - CompletableFuture masterNodeFuture = startNodes(clusterConfiguration.getMasterNodeSettings(), masterNodeTransportPorts, + CompletableFuture masterNodeFuture = startNodes( + clusterManager.getMasterNodeSettings(), masterNodeTransportPorts, masterNodeHttpPorts); SortedSet nonMasterNodeTransportPorts = PortAllocator.TCP.allocate(clusterName, nonMasterNodeCount, 5000 + 42 * 1000 + 310); SortedSet nonMasterNodeHttpPorts = PortAllocator.TCP.allocate(clusterName, nonMasterNodeCount, 5000 + 42 * 1000 + 210); - CompletableFuture nonMasterNodeFuture = startNodes(clusterConfiguration.getNonMasterNodeSettings(), nonMasterNodeTransportPorts, + CompletableFuture nonMasterNodeFuture = startNodes( + clusterManager.getNonMasterNodeSettings(), nonMasterNodeTransportPorts, nonMasterNodeHttpPorts); CompletableFuture.allOf(masterNodeFuture, nonMasterNodeFuture).join(); From 65df948bbcfad94e4cb0b0b691a38e4b9e5eaefb Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Wed, 17 Aug 2022 10:35:40 +0200 Subject: [PATCH 27/47] Correction in class MinimumSecuritySettingsSupplierFactory to pass checkstyle tests, licence header added where missing, test classes moved to proper packages. Signed-off-by: Lukasz Soszynski --- .../SecurityRolesTests.java | 2 +- .../privileges}/PrivilegesEvaluatorTest.java | 2 +- .../test/GenericIntegrationTest.java | 74 ------------------- .../cluster/ContextHeaderDecoratorClient.java | 9 +++ ...inimumSecuritySettingsSupplierFactory.java | 2 - .../test/framework/cluster/NodeType.java | 10 +++ .../cluster/RestClientException.java | 9 +++ .../test/framework/cluster/StartStage.java | 9 +++ 8 files changed, 39 insertions(+), 78 deletions(-) rename src/integrationTest/java/org/opensearch/{test => security}/SecurityRolesTests.java (98%) rename src/integrationTest/java/org/opensearch/{test => security/privileges}/PrivilegesEvaluatorTest.java (98%) delete mode 100644 src/integrationTest/java/org/opensearch/test/GenericIntegrationTest.java diff --git a/src/integrationTest/java/org/opensearch/test/SecurityRolesTests.java b/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java similarity index 98% rename from src/integrationTest/java/org/opensearch/test/SecurityRolesTests.java rename to src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java index 22742aea0c..51323bfb25 100644 --- a/src/integrationTest/java/org/opensearch/test/SecurityRolesTests.java +++ b/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java @@ -24,7 +24,7 @@ * GitHub history for details. */ -package org.opensearch.test; +package org.opensearch.security; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.apache.http.HttpStatus; diff --git a/src/integrationTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java b/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java similarity index 98% rename from src/integrationTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java rename to src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java index 9bd469cabb..c320028b67 100644 --- a/src/integrationTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java @@ -9,7 +9,7 @@ * GitHub history for details. */ -package org.opensearch.test; +package org.opensearch.security.privileges; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.apache.http.HttpStatus; diff --git a/src/integrationTest/java/org/opensearch/test/GenericIntegrationTest.java b/src/integrationTest/java/org/opensearch/test/GenericIntegrationTest.java deleted file mode 100644 index 05346623e9..0000000000 --- a/src/integrationTest/java/org/opensearch/test/GenericIntegrationTest.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.test; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import org.apache.http.HttpStatus; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.runner.RunWith; - -import org.opensearch.test.framework.TestIndex; -import org.opensearch.test.framework.TestSecurityConfig; -import org.opensearch.test.framework.TestSecurityConfig.Role; -import org.opensearch.test.framework.cluster.ClusterManager; -import org.opensearch.test.framework.cluster.LocalCluster; -import org.opensearch.test.framework.cluster.TestRestClient; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; -import static org.opensearch.test.framework.TestSecurityConfig.User.USER_ADMIN; - -/** - * WIP - * Generic test class that demonstrates how to use the test framework to - * set up a test cluster with users, roles, indices and data, and how to - * implement tests. One main goal here is to make tests self-contained. - */ -@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) -@ThreadLeakScope(ThreadLeakScope.Scope.NONE) -public class GenericIntegrationTest { - - // define indices used in this test - private final static TestIndex INDEX_A = TestIndex.name("index-a").build(); - private final static TestIndex INDEX_B = TestIndex.name("index-b").build(); - - private final static TestSecurityConfig.User INDEX_A_USER = new TestSecurityConfig.User("index_a_user") - .roles(new Role("index_a_role").indexPermissions("*").on(INDEX_A).clusterPermissions("*")); - - - // build our test cluster as a ClassRule - @ClassRule - public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_MASTERS) - .authc(AUTHC_HTTPBASIC_INTERNAL) - .users(USER_ADMIN, INDEX_A_USER) - .indices(INDEX_A, INDEX_B).build(); - - @Test - public void testAdminUserHasAccessToAllIndices() throws Exception { - try (TestRestClient client = cluster.getRestClient(USER_ADMIN)) { - assertThat(client.get("*/_search?pretty").getStatusCode(), equalTo(HttpStatus.SC_OK)); - } - } - - @Test - public void testIndexAUserHasOnlyAccessToIndexA() throws Exception { - try (TestRestClient client = cluster.getRestClient(INDEX_A_USER)) { - assertThat(client.get("index-a/_search?pretty").getStatusCode(), equalTo(HttpStatus.SC_OK)); - // demo: work with JSON response body and check values - assertThat(client.get("index-a/_search?pretty").getIntFromJsonBody("/_source/hits/value"), equalTo(0)); - assertThat(client.get("index-b/_search?pretty").getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN)); - } - } - -} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java index 2c788598d9..727dd3fbec 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java @@ -1,3 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ package org.opensearch.test.framework.cluster; import java.util.Collections; diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java index f31eb2b440..56bd00a479 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java @@ -28,8 +28,6 @@ package org.opensearch.test.framework.cluster; -import java.io.IOException; - import org.opensearch.common.settings.Settings; import org.opensearch.test.framework.certificate.TestCertificates; diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java index 0d7e39c40c..6dc35c81c2 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java @@ -1,3 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ package org.opensearch.test.framework.cluster; enum NodeType { diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/RestClientException.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/RestClientException.java index 78cc158863..55cd0c7f4c 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/RestClientException.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/RestClientException.java @@ -1,3 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ package org.opensearch.test.framework.cluster; class RestClientException extends RuntimeException { diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java index 2a0ee0020f..f10077735e 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java @@ -1,3 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ package org.opensearch.test.framework.cluster; enum StartStage { From ea4251ca1e8e156db8edcae53c7ce9d65ebff6bd Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Wed, 17 Aug 2022 18:02:29 +0200 Subject: [PATCH 28/47] Netty dependencies conflict resolved Signed-off-by: Lukasz Soszynski --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index 27346ab8c9..6da5ddec20 100644 --- a/build.gradle +++ b/build.gradle @@ -242,6 +242,7 @@ configurations { force "io.netty:netty-common:${versions.netty}" force "io.netty:netty-handler:${versions.netty}" force "io.netty:netty-transport:${versions.netty}" + force "io.netty:netty-transport-native-unix-common:${versions.netty}" } } From 237a49b7db6845fff82dec1c3e9e7c24824a8043 Mon Sep 17 00:00:00 2001 From: Kacper Trochimiak Date: Fri, 19 Aug 2022 10:41:50 +0200 Subject: [PATCH 29/47] change version of commons-io:commons-io for test and integrationTest Signed-off-by: Kacper Trochimiak --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 6da5ddec20..ce53ca2de7 100644 --- a/build.gradle +++ b/build.gradle @@ -370,7 +370,7 @@ dependencies { testImplementation "org.opensearch.plugin:parent-join-client:${opensearch_version}" testImplementation "org.opensearch.plugin:aggs-matrix-stats-client:${opensearch_version}" testImplementation 'org.apache.logging.log4j:log4j-core:2.17.1' - testImplementation 'commons-io:commons-io:2.7' + testImplementation 'commons-io:commons-io:2.11.0' testImplementation 'javax.servlet:servlet-api:2.5' testImplementation 'com.unboundid:unboundid-ldapsdk:4.0.9' testImplementation 'com.github.stephenc.jcip:jcip-annotations:1.0-1' @@ -413,7 +413,7 @@ dependencies { integrationTestImplementation 'junit:junit:4.13.2' integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}" integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}" - integrationTestImplementation 'commons-io:commons-io:2.7' + integrationTestImplementation 'commons-io:commons-io:2.11.0' integrationTestImplementation 'org.apache.logging.log4j:log4j-core:2.17.1' integrationTestImplementation 'org.hamcrest:hamcrest:2.1' } From c0b23d2accabda860e2dec49efb408c0ec0a739f Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Fri, 19 Aug 2022 11:50:42 +0200 Subject: [PATCH 30/47] Word master replaced with cluster manager in method and local variables names Signed-off-by: Lukasz Soszynski --- .../security/SecurityRolesTests.java | 2 +- .../privileges/PrivilegesEvaluatorTest.java | 2 +- .../framework/cluster/ClusterManager.java | 34 +++++----- .../test/framework/cluster/LocalCluster.java | 4 +- .../cluster/LocalOpenSearchCluster.java | 68 ++++++++++--------- .../test/framework/cluster/NodeType.java | 2 +- 6 files changed, 59 insertions(+), 53 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java b/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java index 51323bfb25..74718938c9 100644 --- a/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java +++ b/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java @@ -53,7 +53,7 @@ public class SecurityRolesTests { @ClassRule public static LocalCluster cluster = new LocalCluster.Builder() - .clusterManager(ClusterManager.THREE_MASTERS).anonymousAuth(true) + .clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS).anonymousAuth(true) .authc(AUTHC_HTTPBASIC_INTERNAL).users(USER_SR).build(); @Test diff --git a/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java b/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java index c320028b67..e5a2a7df4c 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java @@ -47,7 +47,7 @@ public class PrivilegesEvaluatorTest { @ClassRule public static LocalCluster cluster = new LocalCluster.Builder() - .clusterManager(ClusterManager.THREE_MASTERS).authc(AUTHC_HTTPBASIC_INTERNAL) + .clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS).authc(AUTHC_HTTPBASIC_INTERNAL) .users(NEGATIVE_LOOKAHEAD, NEGATED_REGEX).build(); @Test diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java index a9d4153b32..89d56277da 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java @@ -47,10 +47,10 @@ import static java.util.Collections.unmodifiableList; import static org.opensearch.test.framework.cluster.NodeType.CLIENT; import static org.opensearch.test.framework.cluster.NodeType.DATA; -import static org.opensearch.test.framework.cluster.NodeType.MASTER; +import static org.opensearch.test.framework.cluster.NodeType.CLUSTER_MANAGER; public enum ClusterManager { - //first one needs to be a master + //first one needs to be a cluster manager //HUGE(new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(false, true,false), new NodeSettings(false, true, false)), //3 nodes (1m, 2d) @@ -62,7 +62,7 @@ public enum ClusterManager { //4 node (1m, 2d, 1c) CLIENTNODE(new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true), new NodeSettings(false, false)), - THREE_MASTERS(new NodeSettings(true, false), new NodeSettings(true, false), new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true)); + THREE_CLUSTER_MANAGERS(new NodeSettings(true, false), new NodeSettings(true, false), new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true)); private List nodeSettings = new LinkedList<>(); @@ -74,20 +74,20 @@ public List getNodeSettings() { return unmodifiableList(nodeSettings); } - public List getMasterNodeSettings() { - return unmodifiableList(nodeSettings.stream().filter(a -> a.masterNode).collect(Collectors.toList())); + public List getClusterManagerNodeSettings() { + return unmodifiableList(nodeSettings.stream().filter(a -> a.clusterManagerNode).collect(Collectors.toList())); } - public List getNonMasterNodeSettings() { - return unmodifiableList(nodeSettings.stream().filter(a -> !a.masterNode).collect(Collectors.toList())); + public List getNonClusterManagerNodeSettings() { + return unmodifiableList(nodeSettings.stream().filter(a -> !a.clusterManagerNode).collect(Collectors.toList())); } public int getNodes() { return nodeSettings.size(); } - public int getMasterNodes() { - return (int) nodeSettings.stream().filter(a -> a.masterNode).count(); + public int getClusterManagerNodes() { + return (int) nodeSettings.stream().filter(a -> a.clusterManagerNode).count(); } public int getDataNodes() { @@ -95,30 +95,30 @@ public int getDataNodes() { } public int getClientNodes() { - return (int) nodeSettings.stream().filter(a -> !a.masterNode && !a.dataNode).count(); + return (int) nodeSettings.stream().filter(a -> !a.clusterManagerNode && !a.dataNode).count(); } public static class NodeSettings { private final static List> DEFAULT_PLUGINS = List.of(Netty4Plugin.class, OpenSearchSecurityPlugin.class, MatrixAggregationPlugin.class, ParentJoinPlugin.class, PercolatorPlugin.class, ReindexPlugin.class); - public final boolean masterNode; + public final boolean clusterManagerNode; public final boolean dataNode; public final List> plugins; - public NodeSettings(boolean masterNode, boolean dataNode) { - this(masterNode, dataNode, Collections.emptyList()); + public NodeSettings(boolean clusterManagerNode, boolean dataNode) { + this(clusterManagerNode, dataNode, Collections.emptyList()); } - public NodeSettings(boolean masterNode, boolean dataNode, List> additionalPlugins) { + public NodeSettings(boolean clusterManagerNode, boolean dataNode, List> additionalPlugins) { super(); - this.masterNode = masterNode; + this.clusterManagerNode = clusterManagerNode; this.dataNode = dataNode; this.plugins = mergePlugins(additionalPlugins, DEFAULT_PLUGINS); } NodeType recognizeNodeType() { - if (masterNode) { - return MASTER; + if (clusterManagerNode) { + return CLUSTER_MANAGER; } else if (dataNode) { return DATA; } else { diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 0d003accc0..28504038d4 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -103,7 +103,7 @@ public void before() throws Throwable { for (Map.Entry entry : remotes.entrySet()) { @SuppressWarnings("resource") - InetSocketAddress transportAddress = entry.getValue().localOpenSearchCluster.masterNode().getTransportAddress(); + InetSocketAddress transportAddress = entry.getValue().localOpenSearchCluster.clusterManagerNode().getTransportAddress(); nodeOverride = Settings.builder().put(nodeOverride) .putList("cluster.remote." + entry.getKey() + ".seeds", transportAddress.getHostString() + ":" + transportAddress.getPort()) .build(); @@ -151,7 +151,7 @@ public Client getInternalNodeClient() { } public PluginAwareNode node() { - return this.localOpenSearchCluster.masterNode().esNode(); + return this.localOpenSearchCluster.clusterManagerNode().esNode(); } public List nodes() { diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index fd16698d42..13e680facc 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -41,6 +41,7 @@ import java.util.Locale; import java.util.Objects; import java.util.Random; +import java.util.Set; import java.util.SortedSet; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -69,7 +70,8 @@ import static org.junit.Assert.assertEquals; import static org.opensearch.test.framework.cluster.NodeType.CLIENT; import static org.opensearch.test.framework.cluster.NodeType.DATA; -import static org.opensearch.test.framework.cluster.NodeType.MASTER; +import static org.opensearch.test.framework.cluster.NodeType.CLUSTER_MANAGER; +import static org.opensearch.test.framework.cluster.PortAllocator.TCP; public class LocalOpenSearchCluster { @@ -88,7 +90,7 @@ public class LocalOpenSearchCluster { private File clusterHomeDir; private List seedHosts; - private List initialMasterHosts; + private List initialClusterManagerHosts; private int retry = 0; private boolean started; private Random random = new Random(); @@ -120,32 +122,34 @@ private long countNodesByType(NodeType nodeType) { public void start() throws Exception { log.info("Starting {}", clusterName); - int masterNodeCount = clusterManager.getMasterNodes(); - int nonMasterNodeCount = clusterManager.getDataNodes() + clusterManager.getClientNodes(); + int clusterManagerNodeCount = clusterManager.getClusterManagerNodes(); + int nonClusterManagerNodeCount = clusterManager.getDataNodes() + clusterManager.getClientNodes(); - SortedSet masterNodeTransportPorts = PortAllocator.TCP.allocate(clusterName, Math.max(masterNodeCount, 4), 5000 + 42 * 1000 + 300); - SortedSet masterNodeHttpPorts = PortAllocator.TCP.allocate(clusterName, masterNodeCount, 5000 + 42 * 1000 + 200); + SortedSet clusterManagerNodeTransportPorts = TCP.allocate(clusterName, Math.max(clusterManagerNodeCount, 4), 5000 + 42 * 1000 + 300); + SortedSet clusterManagerNodeHttpPorts = TCP.allocate(clusterName, clusterManagerNodeCount, 5000 + 42 * 1000 + 200); - this.seedHosts = toHostList(masterNodeTransportPorts); - this.initialMasterHosts = toHostList(masterNodeTransportPorts.stream().limit(masterNodeCount).collect(Collectors.toSet())); + this.seedHosts = toHostList(clusterManagerNodeTransportPorts); + Set clusterManagerPorts = clusterManagerNodeTransportPorts + .stream().limit(clusterManagerNodeCount).collect(Collectors.toSet()); + this.initialClusterManagerHosts = toHostList(clusterManagerPorts); started = true; - CompletableFuture masterNodeFuture = startNodes( - clusterManager.getMasterNodeSettings(), masterNodeTransportPorts, - masterNodeHttpPorts); + CompletableFuture clusterManagerNodeFuture = startNodes( + clusterManager.getClusterManagerNodeSettings(), clusterManagerNodeTransportPorts, + clusterManagerNodeHttpPorts); - SortedSet nonMasterNodeTransportPorts = PortAllocator.TCP.allocate(clusterName, nonMasterNodeCount, 5000 + 42 * 1000 + 310); - SortedSet nonMasterNodeHttpPorts = PortAllocator.TCP.allocate(clusterName, nonMasterNodeCount, 5000 + 42 * 1000 + 210); + SortedSet nonClusterManagerNodeTransportPorts = TCP.allocate(clusterName, nonClusterManagerNodeCount, 5000 + 42 * 1000 + 310); + SortedSet nonClusterManagerNodeHttpPorts = TCP.allocate(clusterName, nonClusterManagerNodeCount, 5000 + 42 * 1000 + 210); - CompletableFuture nonMasterNodeFuture = startNodes( - clusterManager.getNonMasterNodeSettings(), nonMasterNodeTransportPorts, - nonMasterNodeHttpPorts); + CompletableFuture nonClusterManagerNodeFuture = startNodes( + clusterManager.getNonClusterManagerNodeSettings(), nonClusterManagerNodeTransportPorts, + nonClusterManagerNodeHttpPorts); - CompletableFuture.allOf(masterNodeFuture, nonMasterNodeFuture).join(); + CompletableFuture.allOf(clusterManagerNodeFuture, nonClusterManagerNodeFuture).join(); if (isNodeFailedWithPortCollision()) { - log.info("Detected port collision for master node. Retrying."); + log.info("Detected port collision for cluster manager node. Retrying."); retry(); return; @@ -177,7 +181,7 @@ public void stop() { stopFutures.add(node.stop(2, TimeUnit.SECONDS)); } - for (Node node : getNodesByType(MASTER)) { + for (Node node : getNodesByType(CLUSTER_MANAGER)) { stopFutures.add(node.stop(2, TimeUnit.SECONDS)); } CompletableFuture.allOf(stopFutures.toArray(size -> new CompletableFuture[size])).join(); @@ -195,11 +199,11 @@ public void destroy() { } public Node clientNode() { - return findRunningNode(getNodesByType(CLIENT), getNodesByType(DATA), getNodesByType(MASTER)); + return findRunningNode(getNodesByType(CLIENT), getNodesByType(DATA), getNodesByType(CLUSTER_MANAGER)); } - public Node masterNode() { - return findRunningNode(getNodesByType(MASTER)); + public Node clusterManagerNode() { + return findRunningNode(getNodesByType(CLUSTER_MANAGER)); } public List getNodes() { @@ -219,14 +223,14 @@ private void retry() throws Exception { retry++; if (retry > 10) { - throw new RuntimeException("Detected port collisions for master node. Giving up."); + throw new RuntimeException("Detected port collisions for cluster manager node. Giving up."); } stop(); this.nodes.clear(); this.seedHosts = null; - this.initialMasterHosts = null; + this.initialClusterManagerHosts = null; this.clusterHomeDir = Files.createTempDirectory("local_cluster_" + clusterName + "_retry_" + retry).toFile(); start(); @@ -293,11 +297,13 @@ public void waitForCluster(ClusterHealthStatus status, TimeValue timeout, int ex @Override public String toString() { - String masterNodes = nodeByTypeToString(MASTER); + String clusterManagerNodes = nodeByTypeToString(CLUSTER_MANAGER); String dataNodes = nodeByTypeToString(DATA); String clientNodes = nodeByTypeToString(CLIENT); - return "\nES Cluster " + clusterName + "\nmaster nodes: " + masterNodes + "\n data nodes: " + dataNodes + "\nclient nodes: " + clientNodes - + "\n"; + return "\nES Cluster " + clusterName + + "\ncluster manager nodes: " + clusterManagerNodes + + "\n data nodes: " + dataNodes + + "\nclient nodes: " + clientNodes + "\n"; } private String nodeByTypeToString(NodeType type) { @@ -353,7 +359,7 @@ boolean hasAssignedType(NodeType type) { CompletableFuture start() { CompletableFuture completableFuture = new CompletableFuture<>(); Class[] mergedPlugins = nodeSettings.pluginsWithAddition(additionalPlugins); - this.node = new PluginAwareNode(nodeSettings.masterNode, getOpenSearchSettings(), mergedPlugins); + this.node = new PluginAwareNode(nodeSettings.clusterManagerNode, getOpenSearchSettings(), mergedPlugins); new Thread(new Runnable() { @@ -373,7 +379,7 @@ public void run() { } node = null; - PortAllocator.TCP.reserve(transportPort, httpPort); + TCP.reserve(transportPort, httpPort); completableFuture.complete(StartStage.RETRY); @@ -464,9 +470,9 @@ private Settings getOpenSearchSettings() { } private Settings getMinimalOpenSearchSettings() { - return Settings.builder().put("node.name", nodeName).put("node.data", nodeSettings.dataNode).put("node.master", nodeSettings.masterNode) + return Settings.builder().put("node.name", nodeName).put("node.data", nodeSettings.dataNode).put("node.master", nodeSettings.clusterManagerNode) .put("cluster.name", clusterName).put("path.home", nodeHomeDir.toPath()).put("path.data", dataDir.toPath()) - .put("path.logs", logsDir.toPath()).putList("cluster.initial_master_nodes", initialMasterHosts) + .put("path.logs", logsDir.toPath()).putList("cluster.initial_master_nodes", initialClusterManagerHosts) .put("discovery.initial_state_timeout", "8s").putList("discovery.seed_hosts", seedHosts).put("transport.tcp.port", transportPort) .put("http.port", httpPort).put("cluster.routing.allocation.disk.threshold_enabled", false) .put("discovery.probe.connect_timeout", "10s").put("discovery.probe.handshake_timeout", "10s").put("http.cors.enabled", true) diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java index 6dc35c81c2..ad00b143c6 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java @@ -11,5 +11,5 @@ package org.opensearch.test.framework.cluster; enum NodeType { - CLIENT, DATA, MASTER + CLIENT, DATA, CLUSTER_MANAGER } From 9f6a60cb3d097ad206ad0d6f609a6c773021d3f0 Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Fri, 19 Aug 2022 12:15:19 +0200 Subject: [PATCH 31/47] Method LocalOpenSearchCluster#stop simplified. Signed-off-by: Lukasz Soszynski --- .../test/framework/cluster/ClusterManager.java | 2 +- .../framework/cluster/LocalOpenSearchCluster.java | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java index 89d56277da..2457e60658 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java @@ -46,8 +46,8 @@ import static java.util.Collections.unmodifiableList; import static org.opensearch.test.framework.cluster.NodeType.CLIENT; -import static org.opensearch.test.framework.cluster.NodeType.DATA; import static org.opensearch.test.framework.cluster.NodeType.CLUSTER_MANAGER; +import static org.opensearch.test.framework.cluster.NodeType.DATA; public enum ClusterManager { //first one needs to be a cluster manager diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index 13e680facc..5610a9fa2f 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -69,8 +69,8 @@ import static java.util.Objects.requireNonNull; import static org.junit.Assert.assertEquals; import static org.opensearch.test.framework.cluster.NodeType.CLIENT; -import static org.opensearch.test.framework.cluster.NodeType.DATA; import static org.opensearch.test.framework.cluster.NodeType.CLUSTER_MANAGER; +import static org.opensearch.test.framework.cluster.NodeType.DATA; import static org.opensearch.test.framework.cluster.PortAllocator.TCP; public class LocalOpenSearchCluster { @@ -173,15 +173,7 @@ public boolean isStarted() { public void stop() { List> stopFutures = new ArrayList<>(); - for (Node node : getNodesByType(CLIENT)) { - stopFutures.add(node.stop(2, TimeUnit.SECONDS)); - } - - for (Node node : getNodesByType(DATA)) { - stopFutures.add(node.stop(2, TimeUnit.SECONDS)); - } - - for (Node node : getNodesByType(CLUSTER_MANAGER)) { + for (Node node : nodes) { stopFutures.add(node.stop(2, TimeUnit.SECONDS)); } CompletableFuture.allOf(stopFutures.toArray(size -> new CompletableFuture[size])).join(); From a418b6469c33741d52192b771f5811f4abd112d6 Mon Sep 17 00:00:00 2001 From: Kacper Trochimiak Date: Tue, 23 Aug 2022 12:46:49 +0200 Subject: [PATCH 32/47] integrationTest - upgrade org.hamcrest:hamcrest to 2.2 Signed-off-by: Kacper Trochimiak --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index ce53ca2de7..f23eb10913 100644 --- a/build.gradle +++ b/build.gradle @@ -415,7 +415,7 @@ dependencies { integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}" integrationTestImplementation 'commons-io:commons-io:2.11.0' integrationTestImplementation 'org.apache.logging.log4j:log4j-core:2.17.1' - integrationTestImplementation 'org.hamcrest:hamcrest:2.1' + integrationTestImplementation 'org.hamcrest:hamcrest:2.2' } jar { From 5340d9ae08ee1c86ea8bc1f86132566b0e9154f2 Mon Sep 17 00:00:00 2001 From: Kacper Trochimiak Date: Tue, 23 Aug 2022 15:02:41 +0200 Subject: [PATCH 33/47] add tests for SocketUtils class Signed-off-by: Kacper Trochimiak --- .../framework/cluster/SocketUtilsTests.java | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java new file mode 100644 index 0000000000..da6d730762 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java @@ -0,0 +1,212 @@ +package org.opensearch.test.framework.cluster; + +/* + * Copyright 2002-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.util.SortedSet; + +import javax.net.ServerSocketFactory; + +import org.junit.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.startsWith; +import static org.junit.Assert.assertThrows; +import static org.opensearch.test.framework.cluster.SocketUtils.PORT_RANGE_MAX; +import static org.opensearch.test.framework.cluster.SocketUtils.PORT_RANGE_MIN; + +/** + * Unit tests for {@link SocketUtils}. + * + * @author Sam Brannen + * @author Gary Russell + */ +public class SocketUtilsTests { + + // TCP + + @Test + public void findAvailableTcpPort() { + int port = SocketUtils.findAvailableTcpPort(); + assertPortInRange(port, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + @Test + public void findAvailableTcpPortWithMinPortEqualToMaxPort() { + int minMaxPort = SocketUtils.findAvailableTcpPort(); + int port = SocketUtils.findAvailableTcpPort(minMaxPort, minMaxPort); + assertThat(port, equalTo(minMaxPort)); + } + + @Test + public void findAvailableTcpPortWhenPortOnLoopbackInterfaceIsNotAvailable() throws Exception { + int port = SocketUtils.findAvailableTcpPort(); + try (ServerSocket socket = ServerSocketFactory.getDefault().createServerSocket(port, 1, InetAddress.getByName("localhost"))) { + assertThat(socket, notNullValue()); + // will only look for the exact port + IllegalStateException exception = assertThrows( + IllegalStateException.class, + () -> SocketUtils.findAvailableTcpPort(port, port) + ); + assertThat(exception.getMessage(), startsWith("Could not find an available TCP port")); + assertThat(exception.getMessage(), endsWith("after 1 attempts")); + } + } + + @Test + public void findAvailableTcpPortWithMin() { + int port = SocketUtils.findAvailableTcpPort(50000); + assertPortInRange(port, 50000, PORT_RANGE_MAX); + } + + @Test + public void findAvailableTcpPortInRange() { + int minPort = 20000; + int maxPort = minPort + 1000; + int port = SocketUtils.findAvailableTcpPort(minPort, maxPort); + assertPortInRange(port, minPort, maxPort); + } + + @Test + public void find4AvailableTcpPorts() { + findAvailableTcpPorts(4); + } + + @Test + public void find50AvailableTcpPorts() { + findAvailableTcpPorts(50); + } + + @Test + public void find4AvailableTcpPortsInRange() { + findAvailableTcpPorts(4, 30000, 35000); + } + + @Test + public void find50AvailableTcpPortsInRange() { + findAvailableTcpPorts(50, 40000, 45000); + } + + // UDP + + @Test + public void findAvailableUdpPort() { + int port = SocketUtils.findAvailableUdpPort(); + assertPortInRange(port, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + @Test + public void findAvailableUdpPortWhenPortOnLoopbackInterfaceIsNotAvailable() throws Exception { + int port = SocketUtils.findAvailableUdpPort(); + try (DatagramSocket socket = new DatagramSocket(port, InetAddress.getByName("localhost"))) { + assertThat(socket, notNullValue()); + // will only look for the exact port + IllegalStateException exception = assertThrows( + IllegalStateException.class, + () -> SocketUtils.findAvailableUdpPort(port, port) + ); + assertThat(exception.getMessage(), startsWith("Could not find an available UDP port")); + assertThat(exception.getMessage(), endsWith("after 1 attempts")); + } + } + + @Test + public void findAvailableUdpPortWithMin() { + int port = SocketUtils.findAvailableUdpPort(50000); + assertPortInRange(port, 50000, PORT_RANGE_MAX); + } + + @Test + public void findAvailableUdpPortInRange() { + int minPort = 20000; + int maxPort = minPort + 1000; + int port = SocketUtils.findAvailableUdpPort(minPort, maxPort); + assertPortInRange(port, minPort, maxPort); + } + + @Test + public void find4AvailableUdpPorts() { + findAvailableUdpPorts(4); + } + + @Test + public void find50AvailableUdpPorts() { + findAvailableUdpPorts(50); + } + + @Test + public void find4AvailableUdpPortsInRange() { + findAvailableUdpPorts(4, 30000, 35000); + } + + @Test + public void find50AvailableUdpPortsInRange() { + findAvailableUdpPorts(50, 40000, 45000); + } + + // Helpers + + private void findAvailableTcpPorts(int numRequested) { + SortedSet ports = SocketUtils.findAvailableTcpPorts(numRequested); + assertAvailablePorts(ports, numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + private void findAvailableTcpPorts(int numRequested, int minPort, int maxPort) { + SortedSet ports = SocketUtils.findAvailableTcpPorts(numRequested, minPort, maxPort); + assertAvailablePorts(ports, numRequested, minPort, maxPort); + } + + private void findAvailableUdpPorts(int numRequested) { + SortedSet ports = SocketUtils.findAvailableUdpPorts(numRequested); + assertAvailablePorts(ports, numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + private void findAvailableUdpPorts(int numRequested, int minPort, int maxPort) { + SortedSet ports = SocketUtils.findAvailableUdpPorts(numRequested, minPort, maxPort); + assertAvailablePorts(ports, numRequested, minPort, maxPort); + } + private void assertPortInRange(int port, int minPort, int maxPort) { + assertThat("port [" + port + "] >= " + minPort, port, greaterThanOrEqualTo(minPort)); + assertThat("port [" + port + "] <= " + maxPort, port, lessThanOrEqualTo(maxPort)); + } + + private void assertAvailablePorts(SortedSet ports, int numRequested, int minPort, int maxPort) { + assertThat("number of ports requested", ports.size(), equalTo(numRequested)); + for (int port : ports) { + assertPortInRange(port, minPort, maxPort); + } + } + +} \ No newline at end of file From e2c66c939575828c8ada4aa925472179065dcaf4 Mon Sep 17 00:00:00 2001 From: Kacper Trochimiak Date: Tue, 23 Aug 2022 17:21:39 +0200 Subject: [PATCH 34/47] add new line to SocketUtilsTests class Signed-off-by: Kacper Trochimiak --- .../org/opensearch/test/framework/cluster/SocketUtilsTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java index da6d730762..b5d788180d 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java @@ -209,4 +209,4 @@ private void assertAvailablePorts(SortedSet ports, int numRequested, in } } -} \ No newline at end of file +} From 777096e6a5dfd8ddf56ce71700caeefb26d52fae Mon Sep 17 00:00:00 2001 From: Kacper Trochimiak Date: Wed, 24 Aug 2022 13:48:24 +0200 Subject: [PATCH 35/47] add separate spotless configuration for integrationTest Signed-off-by: Kacper Trochimiak --- build.gradle | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/build.gradle b/build.gradle index f23eb10913..9579bc29eb 100644 --- a/build.gradle +++ b/build.gradle @@ -9,6 +9,8 @@ * GitHub history for details. */ + +import com.diffplug.gradle.spotless.JavaExtension import org.opensearch.gradle.test.RestIntegTestTask buildscript { @@ -73,6 +75,12 @@ spotless { java { // note: you can use an empty string for all the imports you didn't specify explicitly, and '\\#` prefix for static imports importOrder('java', 'javax', '', 'com.amazon', 'org.opensearch', '\\#') + targetExclude('src/integrationTest/**') + } + format("integrationTest", JavaExtension) { + target('src/integrationTest/java/**/*.java') + importOrder('java', 'javax', '', 'com.amazon', 'org.opensearch', '\\#') + indentWithTabs(4) } } From 652af5b540223a14665efc9d0d9c789abaf8ad6b Mon Sep 17 00:00:00 2001 From: Kacper Trochimiak Date: Wed, 24 Aug 2022 13:55:20 +0200 Subject: [PATCH 36/47] integrationTest - apply spotless formatting Signed-off-by: Kacper Trochimiak --- .../org/opensearch/node/PluginAwareNode.java | 66 +- .../security/SecurityRolesTests.java | 44 +- .../privileges/PrivilegesEvaluatorTest.java | 26 +- .../opensearch/test/framework/TestIndex.java | 116 +-- .../test/framework/TestSecurityConfig.java | 376 ++++---- .../framework/certificate/Certificates.java | 144 +-- .../certificate/TestCertificates.java | 94 +- .../framework/cluster/ClusterManager.java | 234 ++--- .../cluster/ContextHeaderDecoratorClient.java | 48 +- .../test/framework/cluster/LocalCluster.java | 586 ++++++------ .../cluster/LocalOpenSearchCluster.java | 838 ++++++++-------- ...inimumSecuritySettingsSupplierFactory.java | 48 +- .../framework/cluster/NestedValueMap.java | 902 +++++++++--------- .../cluster/NodeSettingsSupplier.java | 46 +- .../test/framework/cluster/NodeType.java | 20 +- .../cluster/OpenSearchClientProvider.java | 170 ++-- .../test/framework/cluster/PortAllocator.java | 216 ++--- .../cluster/RestClientException.java | 22 +- .../test/framework/cluster/SocketUtils.java | 598 ++++++------ .../framework/cluster/SocketUtilsTests.java | 362 +++---- .../test/framework/cluster/StartStage.java | 20 +- .../framework/cluster/TestRestClient.java | 542 +++++------ 22 files changed, 2759 insertions(+), 2759 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/node/PluginAwareNode.java b/src/integrationTest/java/org/opensearch/node/PluginAwareNode.java index 39c93e6b8e..1599cd2a37 100644 --- a/src/integrationTest/java/org/opensearch/node/PluginAwareNode.java +++ b/src/integrationTest/java/org/opensearch/node/PluginAwareNode.java @@ -1,28 +1,28 @@ /* - * Copyright 2015-2018 _floragunn_ GmbH - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Copyright 2015-2018 _floragunn_ GmbH +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.node; @@ -33,17 +33,17 @@ import org.opensearch.plugins.Plugin; public class PluginAwareNode extends Node { - - private final boolean clusterManagerEligible; + + private final boolean clusterManagerEligible; - @SafeVarargs - public PluginAwareNode(boolean clusterManagerEligible, final Settings preparedSettings, final Class... plugins) { - super(InternalSettingsPreparer.prepareEnvironment(preparedSettings, Collections.emptyMap(), null, () -> System.getenv("HOSTNAME")), Arrays.asList(plugins), true); - this.clusterManagerEligible = clusterManagerEligible; - } - + @SafeVarargs + public PluginAwareNode(boolean clusterManagerEligible, final Settings preparedSettings, final Class... plugins) { + super(InternalSettingsPreparer.prepareEnvironment(preparedSettings, Collections.emptyMap(), null, () -> System.getenv("HOSTNAME")), Arrays.asList(plugins), true); + this.clusterManagerEligible = clusterManagerEligible; + } + - public boolean isClusterManagerEligible() { - return clusterManagerEligible; - } + public boolean isClusterManagerEligible() { + return clusterManagerEligible; + } } diff --git a/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java b/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java index 74718938c9..c1b5549051 100644 --- a/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java +++ b/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java @@ -1,28 +1,28 @@ /* - * Copyright 2015-2018 _floragunn_ GmbH - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Copyright 2015-2018 _floragunn_ GmbH +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.security; diff --git a/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java b/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java index e5a2a7df4c..c3ea872537 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java @@ -1,13 +1,13 @@ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.security.privileges; @@ -28,10 +28,10 @@ import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; /** - * This is a port for the test - * org.opensearch.security.privileges.PrivilegesEvaluatorTest to the new test - * framework for direct comparison - */ +* This is a port for the test +* org.opensearch.security.privileges.PrivilegesEvaluatorTest to the new test +* framework for direct comparison +*/ @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) public class PrivilegesEvaluatorTest { diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java b/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java index 97ace5e4b8..9d5feb9eee 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java @@ -1,30 +1,30 @@ /* - * Copyright 2021-2022 floragunn GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ +* Copyright 2021-2022 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.test.framework; @@ -34,51 +34,51 @@ public class TestIndex { - private final String name; - private final Settings settings; - - public TestIndex(String name, Settings settings) { - this.name = name; - this.settings = settings; - - } + private final String name; + private final Settings settings; + + public TestIndex(String name, Settings settings) { + this.name = name; + this.settings = settings; + + } public void create(Client client) { client.admin().indices().create(new CreateIndexRequest(name).settings(settings)).actionGet(); } - public String getName() { - return name; - } + public String getName() { + return name; + } - public static Builder name(String name) { - return new Builder().name(name); - } + public static Builder name(String name) { + return new Builder().name(name); + } - public static class Builder { - private String name; - private Settings.Builder settings = Settings.builder(); + public static class Builder { + private String name; + private Settings.Builder settings = Settings.builder(); - public Builder name(String name) { - this.name = name; - return this; - } + public Builder name(String name) { + this.name = name; + return this; + } - public Builder setting(String name, int value) { - settings.put(name, value); - return this; - } + public Builder setting(String name, int value) { + settings.put(name, value); + return this; + } - public Builder shards(int value) { - settings.put("index.number_of_shards", 5); - return this; - } - - public TestIndex build() { - return new TestIndex(name, settings.build()); - } + public Builder shards(int value) { + settings.put("index.number_of_shards", 5); + return this; + } + + public TestIndex build() { + return new TestIndex(name, settings.build()); + } - } + } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java index ef8afc33c3..454f73b84e 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -1,30 +1,30 @@ /* - * Copyright 2021 floragunn GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ +* Copyright 2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.test.framework; @@ -104,26 +104,26 @@ public TestSecurityConfig xff(String proxies) { return this; } - public TestSecurityConfig anonymousAuth(boolean anonymousAuthEnabled) { - if (overrideSecurityConfigSettings == null) { - overrideSecurityConfigSettings = new NestedValueMap(); - } + public TestSecurityConfig anonymousAuth(boolean anonymousAuthEnabled) { + if (overrideSecurityConfigSettings == null) { + overrideSecurityConfigSettings = new NestedValueMap(); + } - overrideSecurityConfigSettings.put(new NestedValueMap.Path("config", "dynamic", "http"), - NestedValueMap.of("anonymous_auth_enabled", anonymousAuthEnabled)); + overrideSecurityConfigSettings.put(new NestedValueMap.Path("config", "dynamic", "http"), + NestedValueMap.of("anonymous_auth_enabled", anonymousAuthEnabled)); - return this; - } - - public TestSecurityConfig authc(AuthcDomain authcDomain) { - if (overrideSecurityConfigSettings == null) { - overrideSecurityConfigSettings = new NestedValueMap(); - } + return this; + } + + public TestSecurityConfig authc(AuthcDomain authcDomain) { + if (overrideSecurityConfigSettings == null) { + overrideSecurityConfigSettings = new NestedValueMap(); + } - overrideSecurityConfigSettings.put(new NestedValueMap.Path("config", "dynamic", "authc"), authcDomain.toMap()); + overrideSecurityConfigSettings.put(new NestedValueMap.Path("config", "dynamic", "authc"), authcDomain.toMap()); - return this; - } + return this; + } public TestSecurityConfig user(User user) { if (user.roleNames != null) { @@ -407,145 +407,145 @@ public NestedValueMap toJsonMap() { } - public static class AuthcDomain { - - public final static AuthcDomain AUTHC_HTTPBASIC_INTERNAL = new TestSecurityConfig.AuthcDomain("basic", 0) - .httpAuthenticator("basic").backend("internal"); - - private final String id; - private boolean enabled = true; - private boolean transportEnabled = true; - private int order; - private List skipUsers = new ArrayList<>(); - private HttpAuthenticator httpAuthenticator; - private AuthenticationBackend authenticationBackend; - - public AuthcDomain(String id, int order) { - this.id = id; - this.order = order; - } - - public AuthcDomain httpAuthenticator(String type) { - this.httpAuthenticator = new HttpAuthenticator(type); - return this; - } - - public AuthcDomain challengingAuthenticator(String type) { - this.httpAuthenticator = new HttpAuthenticator(type).challenge(true); - return this; - } - - public AuthcDomain httpAuthenticator(HttpAuthenticator httpAuthenticator) { - this.httpAuthenticator = httpAuthenticator; - return this; - } - - public AuthcDomain backend(String type) { - this.authenticationBackend = new AuthenticationBackend(type); - return this; - } - - public AuthcDomain backend(AuthenticationBackend authenticationBackend) { - this.authenticationBackend = authenticationBackend; - return this; - } - - public AuthcDomain skipUsers(String... users) { - this.skipUsers.addAll(Arrays.asList(users)); - return this; - } - - NestedValueMap toMap() { - NestedValueMap result = new NestedValueMap(); - result.put(new NestedValueMap.Path(id, "http_enabled"), enabled); - result.put(new NestedValueMap.Path(id, "transport_enabled"), transportEnabled); - result.put(new NestedValueMap.Path(id, "order"), order); - - if (httpAuthenticator != null) { - result.put(new NestedValueMap.Path(id, "http_authenticator"), httpAuthenticator.toMap()); - } - - if (authenticationBackend != null) { - result.put(new NestedValueMap.Path(id, "authentication_backend"), authenticationBackend.toMap()); - } - - - if (skipUsers != null && skipUsers.size() > 0) { - result.put(new NestedValueMap.Path(id, "skip_users"), skipUsers); - } - - return result; - } - - public static class HttpAuthenticator { - private final String type; - private boolean challenge; - private NestedValueMap config = new NestedValueMap(); - - public HttpAuthenticator(String type) { - this.type = type; - } - - public HttpAuthenticator challenge(boolean challenge) { - this.challenge = challenge; - return this; - } - - public HttpAuthenticator config(Map config) { - this.config.putAllFromAnyMap(config); - return this; - } - - public HttpAuthenticator config(String key, Object value) { - this.config.put(Path.parse(key), value); - return this; - } - - NestedValueMap toMap() { - NestedValueMap result = new NestedValueMap(); - result.put("type", type); - result.put("challenge", challenge); - result.put("config", config); - return result; - } - } - - public static class AuthenticationBackend { - private final String type; - private NestedValueMap config = new NestedValueMap(); - - public AuthenticationBackend(String type) { - this.type = type; - } - - public AuthenticationBackend config(Map config) { - this.config.putAllFromAnyMap(config); - return this; - } - - public AuthenticationBackend config(String key, Object value) { - this.config.put(Path.parse(key), value); - return this; - } - - NestedValueMap toMap() { - NestedValueMap result = new NestedValueMap(); - result.put("type", type); - result.put("config", config); - return result; - } - } - } - - public TestSecurityConfig clone() { - TestSecurityConfig result = new TestSecurityConfig(); - result.indexName = indexName; - result.overrideRoleSettings = overrideRoleSettings != null ? overrideRoleSettings.clone() : null; - result.overrideSecurityConfigSettings = overrideSecurityConfigSettings != null ? overrideSecurityConfigSettings.clone() : null; - result.overrideUserSettings = overrideUserSettings != null ? overrideUserSettings.clone() : null; - - return result; - } + public static class AuthcDomain { + + public final static AuthcDomain AUTHC_HTTPBASIC_INTERNAL = new TestSecurityConfig.AuthcDomain("basic", 0) + .httpAuthenticator("basic").backend("internal"); + + private final String id; + private boolean enabled = true; + private boolean transportEnabled = true; + private int order; + private List skipUsers = new ArrayList<>(); + private HttpAuthenticator httpAuthenticator; + private AuthenticationBackend authenticationBackend; + + public AuthcDomain(String id, int order) { + this.id = id; + this.order = order; + } + + public AuthcDomain httpAuthenticator(String type) { + this.httpAuthenticator = new HttpAuthenticator(type); + return this; + } + + public AuthcDomain challengingAuthenticator(String type) { + this.httpAuthenticator = new HttpAuthenticator(type).challenge(true); + return this; + } + + public AuthcDomain httpAuthenticator(HttpAuthenticator httpAuthenticator) { + this.httpAuthenticator = httpAuthenticator; + return this; + } + + public AuthcDomain backend(String type) { + this.authenticationBackend = new AuthenticationBackend(type); + return this; + } + + public AuthcDomain backend(AuthenticationBackend authenticationBackend) { + this.authenticationBackend = authenticationBackend; + return this; + } + + public AuthcDomain skipUsers(String... users) { + this.skipUsers.addAll(Arrays.asList(users)); + return this; + } + + NestedValueMap toMap() { + NestedValueMap result = new NestedValueMap(); + result.put(new NestedValueMap.Path(id, "http_enabled"), enabled); + result.put(new NestedValueMap.Path(id, "transport_enabled"), transportEnabled); + result.put(new NestedValueMap.Path(id, "order"), order); + + if (httpAuthenticator != null) { + result.put(new NestedValueMap.Path(id, "http_authenticator"), httpAuthenticator.toMap()); + } + + if (authenticationBackend != null) { + result.put(new NestedValueMap.Path(id, "authentication_backend"), authenticationBackend.toMap()); + } + + + if (skipUsers != null && skipUsers.size() > 0) { + result.put(new NestedValueMap.Path(id, "skip_users"), skipUsers); + } + + return result; + } + + public static class HttpAuthenticator { + private final String type; + private boolean challenge; + private NestedValueMap config = new NestedValueMap(); + + public HttpAuthenticator(String type) { + this.type = type; + } + + public HttpAuthenticator challenge(boolean challenge) { + this.challenge = challenge; + return this; + } + + public HttpAuthenticator config(Map config) { + this.config.putAllFromAnyMap(config); + return this; + } + + public HttpAuthenticator config(String key, Object value) { + this.config.put(Path.parse(key), value); + return this; + } + + NestedValueMap toMap() { + NestedValueMap result = new NestedValueMap(); + result.put("type", type); + result.put("challenge", challenge); + result.put("config", config); + return result; + } + } + + public static class AuthenticationBackend { + private final String type; + private NestedValueMap config = new NestedValueMap(); + + public AuthenticationBackend(String type) { + this.type = type; + } + + public AuthenticationBackend config(Map config) { + this.config.putAllFromAnyMap(config); + return this; + } + + public AuthenticationBackend config(String key, Object value) { + this.config.put(Path.parse(key), value); + return this; + } + + NestedValueMap toMap() { + NestedValueMap result = new NestedValueMap(); + result.put("type", type); + result.put("config", config); + return result; + } + } + } + + public TestSecurityConfig clone() { + TestSecurityConfig result = new TestSecurityConfig(); + result.indexName = indexName; + result.overrideRoleSettings = overrideRoleSettings != null ? overrideRoleSettings.clone() : null; + result.overrideSecurityConfigSettings = overrideSecurityConfigSettings != null ? overrideSecurityConfigSettings.clone() : null; + result.overrideUserSettings = overrideUserSettings != null ? overrideUserSettings.clone() : null; + + return result; + } public void initIndex(Client client) { Map settings = new HashMap<>(); @@ -554,13 +554,13 @@ public void initIndex(Client client) { } client.admin().indices().create(new CreateIndexRequest(indexName).settings(settings)).actionGet(); - writeConfigToIndex(client, CType.CONFIG, overrideSecurityConfigSettings); - writeConfigToIndex(client, CType.ROLES, overrideRoleSettings); + writeConfigToIndex(client, CType.CONFIG, overrideSecurityConfigSettings); + writeConfigToIndex(client, CType.ROLES, overrideRoleSettings); writeConfigToIndex(client, CType.INTERNALUSERS, overrideUserSettings); - writeConfigToIndex(client, CType.ROLESMAPPING, overrideRoleMappingSettings); - writeConfigToIndex(client, CType.ACTIONGROUPS); - writeConfigToIndex(client, CType.TENANTS); - + writeConfigToIndex(client, CType.ROLESMAPPING, overrideRoleMappingSettings); + writeConfigToIndex(client, CType.ACTIONGROUPS); + writeConfigToIndex(client, CType.TENANTS); + ConfigUpdateResponse configUpdateResponse = client.execute(ConfigUpdateAction.INSTANCE, new ConfigUpdateRequest(CType.lcStringValues().toArray(new String[0]))).actionGet(); @@ -580,9 +580,9 @@ private static String hash(final char[] clearTextPassword) { } - private void writeConfigToIndex(Client client, CType configType) { - writeConfigToIndex(client, configType, NestedValueMap.createNonCloningMap()); - } + private void writeConfigToIndex(Client client, CType configType) { + writeConfigToIndex(client, configType, NestedValueMap.createNonCloningMap()); + } private void writeConfigToIndex(Client client, CType configType, NestedValueMap overrides) { try { diff --git a/src/integrationTest/java/org/opensearch/test/framework/certificate/Certificates.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/Certificates.java index c24eebd1b7..9895fc1484 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/certificate/Certificates.java +++ b/src/integrationTest/java/org/opensearch/test/framework/certificate/Certificates.java @@ -1,22 +1,22 @@ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.test.framework.certificate; /** - * Contains static certificates for the test cluster. - * Note: This is WIP and will be replaced by classes - * that can generate certificates on the fly. This - * class will be removed after that. - */ +* Contains static certificates for the test cluster. +* Note: This is WIP and will be replaced by classes +* that can generate certificates on the fly. This +* class will be removed after that. +*/ public class Certificates { final static String ROOT_CA_CERTIFICATE = "-----BEGIN CERTIFICATE-----\n" @@ -45,65 +45,65 @@ public class Certificates { + "-----END CERTIFICATE-----\n" + ""; - final static String NODE_CERTIFICATE = "-----BEGIN CERTIFICATE-----\n" - + "MIIEyTCCA7GgAwIBAgIGAWLrc1O2MA0GCSqGSIb3DQEBCwUAMIGPMRMwEQYKCZIm\n" - + "iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ\n" - + "RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290\n" - + "IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwHhcNMTgwNDIy\n" - + "MDM0MzQ3WhcNMjgwNDE5MDM0MzQ3WjBeMRIwEAYKCZImiZPyLGQBGRYCZGUxDTAL\n" - + "BgNVBAcMBHRlc3QxDTALBgNVBAoMBG5vZGUxDTALBgNVBAsMBG5vZGUxGzAZBgNV\n" - + "BAMMEm5vZGUtMC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC\n" - + "AQoCggEBAJa+f476vLB+AwK53biYByUwN+40D8jMIovGXm6wgT8+9Sbs899dDXgt\n" - + "9CE1Beo65oP1+JUz4c7UHMrCY3ePiDt4cidHVzEQ2g0YoVrQWv0RedS/yx/DKhs8\n" - + "Pw1O715oftP53p/2ijD5DifFv1eKfkhFH+lwny/vMSNxellpl6NxJTiJVnQ9HYOL\n" - + "gf2t971ITJHnAuuxUF48HcuNovW4rhtkXef8kaAN7cE3LU+A9T474ULNCKkEFPIl\n" - + "ZAKN3iJNFdVsxrTU+CUBHzk73Do1cCkEvJZ0ZFjp0Z3y8wLY/gqWGfGVyA9l2CUq\n" - + "eIZNf55PNPtGzOrvvONiui48vBKH1LsCAwEAAaOCAVkwggFVMIG8BgNVHSMEgbQw\n" - + "gbGAFJI1DOAPHitF9k0583tfouYSl0BzoYGVpIGSMIGPMRMwEQYKCZImiZPyLGQB\n" - + "GRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhhbXBs\n" - + "ZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENBMSEw\n" - + "HwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCAQEwHQYDVR0OBBYEFKyv\n" - + "78ZmFjVKM9g7pMConYH7FVBHMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgXg\n" - + "MCAGA1UdJQEB/wQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA1BgNVHREELjAsiAUq\n" - + "AwQFBYISbm9kZS0wLmV4YW1wbGUuY29tgglsb2NhbGhvc3SHBH8AAAEwDQYJKoZI\n" - + "hvcNAQELBQADggEBAIOKuyXsFfGv1hI/Lkpd/73QNqjqJdxQclX57GOMWNbOM5H0\n" - + "5/9AOIZ5JQsWULNKN77aHjLRr4owq2jGbpc/Z6kAd+eiatkcpnbtbGrhKpOtoEZy\n" - + "8KuslwkeixpzLDNISSbkeLpXz4xJI1ETMN/VG8ZZP1bjzlHziHHDu0JNZ6TnNzKr\n" - + "XzCGMCohFfem8vnKNnKUneMQMvXd3rzUaAgvtf7Hc2LTBlf4fZzZF1EkwdSXhaMA\n" - + "1lkfHiqOBxtgeDLxCHESZ2fqgVqsWX+t3qHQfivcPW6txtDyrFPRdJOGhiMGzT/t\n" - + "e/9kkAtQRgpTb3skYdIOOUOV0WGQ60kJlFhAzIs=\n" - + "-----END CERTIFICATE-----\n" - + ""; + final static String NODE_CERTIFICATE = "-----BEGIN CERTIFICATE-----\n" + + "MIIEyTCCA7GgAwIBAgIGAWLrc1O2MA0GCSqGSIb3DQEBCwUAMIGPMRMwEQYKCZIm\n" + + "iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ\n" + + "RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290\n" + + "IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwHhcNMTgwNDIy\n" + + "MDM0MzQ3WhcNMjgwNDE5MDM0MzQ3WjBeMRIwEAYKCZImiZPyLGQBGRYCZGUxDTAL\n" + + "BgNVBAcMBHRlc3QxDTALBgNVBAoMBG5vZGUxDTALBgNVBAsMBG5vZGUxGzAZBgNV\n" + + "BAMMEm5vZGUtMC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC\n" + + "AQoCggEBAJa+f476vLB+AwK53biYByUwN+40D8jMIovGXm6wgT8+9Sbs899dDXgt\n" + + "9CE1Beo65oP1+JUz4c7UHMrCY3ePiDt4cidHVzEQ2g0YoVrQWv0RedS/yx/DKhs8\n" + + "Pw1O715oftP53p/2ijD5DifFv1eKfkhFH+lwny/vMSNxellpl6NxJTiJVnQ9HYOL\n" + + "gf2t971ITJHnAuuxUF48HcuNovW4rhtkXef8kaAN7cE3LU+A9T474ULNCKkEFPIl\n" + + "ZAKN3iJNFdVsxrTU+CUBHzk73Do1cCkEvJZ0ZFjp0Z3y8wLY/gqWGfGVyA9l2CUq\n" + + "eIZNf55PNPtGzOrvvONiui48vBKH1LsCAwEAAaOCAVkwggFVMIG8BgNVHSMEgbQw\n" + + "gbGAFJI1DOAPHitF9k0583tfouYSl0BzoYGVpIGSMIGPMRMwEQYKCZImiZPyLGQB\n" + + "GRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhhbXBs\n" + + "ZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENBMSEw\n" + + "HwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCAQEwHQYDVR0OBBYEFKyv\n" + + "78ZmFjVKM9g7pMConYH7FVBHMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgXg\n" + + "MCAGA1UdJQEB/wQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA1BgNVHREELjAsiAUq\n" + + "AwQFBYISbm9kZS0wLmV4YW1wbGUuY29tgglsb2NhbGhvc3SHBH8AAAEwDQYJKoZI\n" + + "hvcNAQELBQADggEBAIOKuyXsFfGv1hI/Lkpd/73QNqjqJdxQclX57GOMWNbOM5H0\n" + + "5/9AOIZ5JQsWULNKN77aHjLRr4owq2jGbpc/Z6kAd+eiatkcpnbtbGrhKpOtoEZy\n" + + "8KuslwkeixpzLDNISSbkeLpXz4xJI1ETMN/VG8ZZP1bjzlHziHHDu0JNZ6TnNzKr\n" + + "XzCGMCohFfem8vnKNnKUneMQMvXd3rzUaAgvtf7Hc2LTBlf4fZzZF1EkwdSXhaMA\n" + + "1lkfHiqOBxtgeDLxCHESZ2fqgVqsWX+t3qHQfivcPW6txtDyrFPRdJOGhiMGzT/t\n" + + "e/9kkAtQRgpTb3skYdIOOUOV0WGQ60kJlFhAzIs=\n" + + "-----END CERTIFICATE-----\n" + + ""; - final static String NODE_KEY = "-----BEGIN PRIVATE KEY-----\n" - + "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCWvn+O+rywfgMC\n" - + "ud24mAclMDfuNA/IzCKLxl5usIE/PvUm7PPfXQ14LfQhNQXqOuaD9fiVM+HO1BzK\n" - + "wmN3j4g7eHInR1cxENoNGKFa0Fr9EXnUv8sfwyobPD8NTu9eaH7T+d6f9oow+Q4n\n" - + "xb9Xin5IRR/pcJ8v7zEjcXpZaZejcSU4iVZ0PR2Di4H9rfe9SEyR5wLrsVBePB3L\n" - + "jaL1uK4bZF3n/JGgDe3BNy1PgPU+O+FCzQipBBTyJWQCjd4iTRXVbMa01PglAR85\n" - + "O9w6NXApBLyWdGRY6dGd8vMC2P4KlhnxlcgPZdglKniGTX+eTzT7Rszq77zjYrou\n" - + "PLwSh9S7AgMBAAECggEABwiohxFoEIwws8XcdKqTWsbfNTw0qFfuHLuK2Htf7IWR\n" - + "htlzn66F3F+4jnwc5IsPCoVFriCXnsEC/usHHSMTZkL+gJqxlNaGdin6DXS/aiOQ\n" - + "nb69SaQfqNmsz4ApZyxVDqsQGkK0vAhDAtQVU45gyhp/nLLmmqP8lPzMirOEodmp\n" - + "U9bA8t/ttrzng7SVAER42f6IVpW0iTKTLyFii0WZbq+ObViyqib9hVFrI6NJuQS+\n" - + "IelcZB0KsSi6rqIjXg1XXyMiIUcSlhq+GfEa18AYgmsbPwMbExate7/8Ci7ZtCbh\n" - + "lx9bves2+eeqq5EMm3sMHyhdcg61yzd5UYXeZhwJkQKBgQDS9YqrAtztvLY2gMgv\n" - + "d+wOjb9awWxYbQTBjx33kf66W+pJ+2j8bI/XX2CpZ98w/oq8VhMqbr9j5b8MfsrF\n" - + "EoQvedA4joUo8sXd4j1mR2qKF4/KLmkgy6YYusNP2UrVSw7sh77bzce+YaVVoO/e\n" - + "0wIVTHuD/QZ6fG6MasOqcbl6hwKBgQC27cQruaHFEXR/16LrMVAX+HyEEv44KOCZ\n" - + "ij5OE4P7F0twb+okngG26+OJV3BtqXf0ULlXJ+YGwXCRf6zUZkld3NMy3bbKPgH6\n" - + "H/nf3BxqS2tudj7+DV52jKtisBghdvtlKs56oc9AAuwOs37DvhptBKUPdzDDqfys\n" - + "Qchv5JQdLQKBgERev+pcqy2Bk6xmYHrB6wdseS/4sByYeIoi0BuEfYH4eB4yFPx6\n" - + "UsQCbVl6CKPgWyZe3ydJbU37D8gE78KfFagtWoZ56j4zMF2RDUUwsB7BNCDamce/\n" - + "OL2bCeG/Erm98cBG3lxufOX+z47I8fTNfkdY2k8UmhzoZwurLm73HJ3RAoGBAKsp\n" - + "6yamuXF2FbYRhUXgjHsBbTD/vJO72/yO2CGiLRpi/5mjfkjo99269trp0C8sJSub\n" - + "5PBiSuADXFsoRgUv+HI1UAEGaCTwxFTQWrRWdtgW3d0sE2EQDVWL5kmfT9TwSeat\n" - + "mSoyAYR5t3tCBNkPJhbgA7pm4mASzHQ50VyxWs25AoGBAKPFx9X2oKhYQa+mW541\n" - + "bbqRuGFMoXIIcr/aeM3LayfLETi48o5NDr2NDP11j4yYuz26YLH0Dj8aKpWuehuH\n" - + "uB27n6j6qu0SVhQi6mMJBe1JrKbzhqMKQjYOoy8VsC2gdj5pCUP/kLQPW7zm9diX\n" - + "CiKTtKgPIeYdigor7V3AHcVT\n" - + "-----END PRIVATE KEY-----\n" - + ""; + final static String NODE_KEY = "-----BEGIN PRIVATE KEY-----\n" + + "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCWvn+O+rywfgMC\n" + + "ud24mAclMDfuNA/IzCKLxl5usIE/PvUm7PPfXQ14LfQhNQXqOuaD9fiVM+HO1BzK\n" + + "wmN3j4g7eHInR1cxENoNGKFa0Fr9EXnUv8sfwyobPD8NTu9eaH7T+d6f9oow+Q4n\n" + + "xb9Xin5IRR/pcJ8v7zEjcXpZaZejcSU4iVZ0PR2Di4H9rfe9SEyR5wLrsVBePB3L\n" + + "jaL1uK4bZF3n/JGgDe3BNy1PgPU+O+FCzQipBBTyJWQCjd4iTRXVbMa01PglAR85\n" + + "O9w6NXApBLyWdGRY6dGd8vMC2P4KlhnxlcgPZdglKniGTX+eTzT7Rszq77zjYrou\n" + + "PLwSh9S7AgMBAAECggEABwiohxFoEIwws8XcdKqTWsbfNTw0qFfuHLuK2Htf7IWR\n" + + "htlzn66F3F+4jnwc5IsPCoVFriCXnsEC/usHHSMTZkL+gJqxlNaGdin6DXS/aiOQ\n" + + "nb69SaQfqNmsz4ApZyxVDqsQGkK0vAhDAtQVU45gyhp/nLLmmqP8lPzMirOEodmp\n" + + "U9bA8t/ttrzng7SVAER42f6IVpW0iTKTLyFii0WZbq+ObViyqib9hVFrI6NJuQS+\n" + + "IelcZB0KsSi6rqIjXg1XXyMiIUcSlhq+GfEa18AYgmsbPwMbExate7/8Ci7ZtCbh\n" + + "lx9bves2+eeqq5EMm3sMHyhdcg61yzd5UYXeZhwJkQKBgQDS9YqrAtztvLY2gMgv\n" + + "d+wOjb9awWxYbQTBjx33kf66W+pJ+2j8bI/XX2CpZ98w/oq8VhMqbr9j5b8MfsrF\n" + + "EoQvedA4joUo8sXd4j1mR2qKF4/KLmkgy6YYusNP2UrVSw7sh77bzce+YaVVoO/e\n" + + "0wIVTHuD/QZ6fG6MasOqcbl6hwKBgQC27cQruaHFEXR/16LrMVAX+HyEEv44KOCZ\n" + + "ij5OE4P7F0twb+okngG26+OJV3BtqXf0ULlXJ+YGwXCRf6zUZkld3NMy3bbKPgH6\n" + + "H/nf3BxqS2tudj7+DV52jKtisBghdvtlKs56oc9AAuwOs37DvhptBKUPdzDDqfys\n" + + "Qchv5JQdLQKBgERev+pcqy2Bk6xmYHrB6wdseS/4sByYeIoi0BuEfYH4eB4yFPx6\n" + + "UsQCbVl6CKPgWyZe3ydJbU37D8gE78KfFagtWoZ56j4zMF2RDUUwsB7BNCDamce/\n" + + "OL2bCeG/Erm98cBG3lxufOX+z47I8fTNfkdY2k8UmhzoZwurLm73HJ3RAoGBAKsp\n" + + "6yamuXF2FbYRhUXgjHsBbTD/vJO72/yO2CGiLRpi/5mjfkjo99269trp0C8sJSub\n" + + "5PBiSuADXFsoRgUv+HI1UAEGaCTwxFTQWrRWdtgW3d0sE2EQDVWL5kmfT9TwSeat\n" + + "mSoyAYR5t3tCBNkPJhbgA7pm4mASzHQ50VyxWs25AoGBAKPFx9X2oKhYQa+mW541\n" + + "bbqRuGFMoXIIcr/aeM3LayfLETi48o5NDr2NDP11j4yYuz26YLH0Dj8aKpWuehuH\n" + + "uB27n6j6qu0SVhQi6mMJBe1JrKbzhqMKQjYOoy8VsC2gdj5pCUP/kLQPW7zm9diX\n" + + "CiKTtKgPIeYdigor7V3AHcVT\n" + + "-----END PRIVATE KEY-----\n" + + ""; final static String ADMIN_CERTIFICATE = "-----BEGIN CERTIFICATE-----\n" + "MIIEdzCCA1+gAwIBAgIGAWLrc1O4MA0GCSqGSIb3DQEBCwUAMIGPMRMwEQYKCZIm\n" diff --git a/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java index d5fe9a527c..0f132f8a09 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java +++ b/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java @@ -1,19 +1,19 @@ /* - * Copyright 2021 floragunn GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ +* Copyright 2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ package org.opensearch.test.framework.certificate; @@ -23,44 +23,44 @@ import java.nio.file.Path; /** - * Provides TLS certificates required in test cases. - * WIP At the moment the certificates are hard coded. - * This will be replaced by classes - * that can generate certificates on the fly. - */ +* Provides TLS certificates required in test cases. +* WIP At the moment the certificates are hard coded. +* This will be replaced by classes +* that can generate certificates on the fly. +*/ public class TestCertificates { - public File getRootCertificate() { - return createTempFile("root", ".cert", Certificates.ROOT_CA_CERTIFICATE); - } + public File getRootCertificate() { + return createTempFile("root", ".cert", Certificates.ROOT_CA_CERTIFICATE); + } - public File getNodeCertificate(int node) { - return createTempFile("node-" + node, ".cert", Certificates.NODE_CERTIFICATE); - } + public File getNodeCertificate(int node) { + return createTempFile("node-" + node, ".cert", Certificates.NODE_CERTIFICATE); + } - public File getNodeKey(int node) { - return createTempFile("node-" + node, ".key", Certificates.NODE_KEY); - } + public File getNodeKey(int node) { + return createTempFile("node-" + node, ".key", Certificates.NODE_KEY); + } - public File getAdminCertificate() { - return createTempFile("admin", ".cert", Certificates.ADMIN_CERTIFICATE); - } + public File getAdminCertificate() { + return createTempFile("admin", ".cert", Certificates.ADMIN_CERTIFICATE); + } - public File getAdminKey() { - return createTempFile("admin", ".key", Certificates.ADMIN_KEY); - } + public File getAdminKey() { + return createTempFile("admin", ".key", Certificates.ADMIN_KEY); + } - public String[] getAdminDNs() { - return new String[] {"CN=kirk,OU=client,O=client,L=test,C=de"}; - } + public String[] getAdminDNs() { + return new String[] {"CN=kirk,OU=client,O=client,L=test,C=de"}; + } - private File createTempFile(String name, String suffix, String contents) { - try { - Path path = Files.createTempFile(name, suffix); - Files.writeString(path, contents); - return path.toFile(); - } catch (IOException e) { - throw new RuntimeException(String.format("Error while creating a temp file: %s%s", name, suffix), e); - } - } + private File createTempFile(String name, String suffix, String contents) { + try { + Path path = Files.createTempFile(name, suffix); + Files.writeString(path, contents); + return path.toFile(); + } catch (IOException e) { + throw new RuntimeException(String.format("Error while creating a temp file: %s%s", name, suffix), e); + } + } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java index 2457e60658..b135df3620 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java @@ -1,30 +1,30 @@ /* - * Copyright 2015-2017 floragunn GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ +* Copyright 2015-2017 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.test.framework.cluster; @@ -50,97 +50,97 @@ import static org.opensearch.test.framework.cluster.NodeType.DATA; public enum ClusterManager { - //first one needs to be a cluster manager - //HUGE(new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(false, true,false), new NodeSettings(false, true, false)), - - //3 nodes (1m, 2d) - DEFAULT(new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true)), - - //1 node (1md) - SINGLENODE(new NodeSettings(true, true)), - - //4 node (1m, 2d, 1c) - CLIENTNODE(new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true), new NodeSettings(false, false)), - - THREE_CLUSTER_MANAGERS(new NodeSettings(true, false), new NodeSettings(true, false), new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true)); - - private List nodeSettings = new LinkedList<>(); - - private ClusterManager(NodeSettings... settings) { - nodeSettings.addAll(Arrays.asList(settings)); - } - - public List getNodeSettings() { - return unmodifiableList(nodeSettings); - } - - public List getClusterManagerNodeSettings() { - return unmodifiableList(nodeSettings.stream().filter(a -> a.clusterManagerNode).collect(Collectors.toList())); - } - - public List getNonClusterManagerNodeSettings() { - return unmodifiableList(nodeSettings.stream().filter(a -> !a.clusterManagerNode).collect(Collectors.toList())); - } - - public int getNodes() { - return nodeSettings.size(); - } - - public int getClusterManagerNodes() { - return (int) nodeSettings.stream().filter(a -> a.clusterManagerNode).count(); - } - - public int getDataNodes() { - return (int) nodeSettings.stream().filter(a -> a.dataNode).count(); - } - - public int getClientNodes() { - return (int) nodeSettings.stream().filter(a -> !a.clusterManagerNode && !a.dataNode).count(); - } - - public static class NodeSettings { - - private final static List> DEFAULT_PLUGINS = List.of(Netty4Plugin.class, OpenSearchSecurityPlugin.class, - MatrixAggregationPlugin.class, ParentJoinPlugin.class, PercolatorPlugin.class, ReindexPlugin.class); - public final boolean clusterManagerNode; - public final boolean dataNode; - public final List> plugins; - - public NodeSettings(boolean clusterManagerNode, boolean dataNode) { - this(clusterManagerNode, dataNode, Collections.emptyList()); - } - - public NodeSettings(boolean clusterManagerNode, boolean dataNode, List> additionalPlugins) { - super(); - this.clusterManagerNode = clusterManagerNode; - this.dataNode = dataNode; - this.plugins = mergePlugins(additionalPlugins, DEFAULT_PLUGINS); - } - NodeType recognizeNodeType() { - if (clusterManagerNode) { - return CLUSTER_MANAGER; - } else if (dataNode) { - return DATA; - } else { - return CLIENT; - } - } - - private List> mergePlugins(Collection>...plugins) { - List> mergedPlugins = Arrays.stream(plugins) - .filter(Objects::nonNull) - .flatMap(Collection::stream) - .collect(Collectors.toList()); - return unmodifiableList(mergedPlugins); - } - - @SuppressWarnings("unchecked") - public Class[] getPlugins() { - return plugins.toArray(new Class[0]); - } - - public Class[] pluginsWithAddition(List> additionalPlugins) { - return mergePlugins(plugins, additionalPlugins).toArray(Class[]::new); - } - } + //first one needs to be a cluster manager + //HUGE(new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(false, true,false), new NodeSettings(false, true, false)), + + //3 nodes (1m, 2d) + DEFAULT(new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true)), + + //1 node (1md) + SINGLENODE(new NodeSettings(true, true)), + + //4 node (1m, 2d, 1c) + CLIENTNODE(new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true), new NodeSettings(false, false)), + + THREE_CLUSTER_MANAGERS(new NodeSettings(true, false), new NodeSettings(true, false), new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true)); + + private List nodeSettings = new LinkedList<>(); + + private ClusterManager(NodeSettings... settings) { + nodeSettings.addAll(Arrays.asList(settings)); + } + + public List getNodeSettings() { + return unmodifiableList(nodeSettings); + } + + public List getClusterManagerNodeSettings() { + return unmodifiableList(nodeSettings.stream().filter(a -> a.clusterManagerNode).collect(Collectors.toList())); + } + + public List getNonClusterManagerNodeSettings() { + return unmodifiableList(nodeSettings.stream().filter(a -> !a.clusterManagerNode).collect(Collectors.toList())); + } + + public int getNodes() { + return nodeSettings.size(); + } + + public int getClusterManagerNodes() { + return (int) nodeSettings.stream().filter(a -> a.clusterManagerNode).count(); + } + + public int getDataNodes() { + return (int) nodeSettings.stream().filter(a -> a.dataNode).count(); + } + + public int getClientNodes() { + return (int) nodeSettings.stream().filter(a -> !a.clusterManagerNode && !a.dataNode).count(); + } + + public static class NodeSettings { + + private final static List> DEFAULT_PLUGINS = List.of(Netty4Plugin.class, OpenSearchSecurityPlugin.class, + MatrixAggregationPlugin.class, ParentJoinPlugin.class, PercolatorPlugin.class, ReindexPlugin.class); + public final boolean clusterManagerNode; + public final boolean dataNode; + public final List> plugins; + + public NodeSettings(boolean clusterManagerNode, boolean dataNode) { + this(clusterManagerNode, dataNode, Collections.emptyList()); + } + + public NodeSettings(boolean clusterManagerNode, boolean dataNode, List> additionalPlugins) { + super(); + this.clusterManagerNode = clusterManagerNode; + this.dataNode = dataNode; + this.plugins = mergePlugins(additionalPlugins, DEFAULT_PLUGINS); + } + NodeType recognizeNodeType() { + if (clusterManagerNode) { + return CLUSTER_MANAGER; + } else if (dataNode) { + return DATA; + } else { + return CLIENT; + } + } + + private List> mergePlugins(Collection>...plugins) { + List> mergedPlugins = Arrays.stream(plugins) + .filter(Objects::nonNull) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + return unmodifiableList(mergedPlugins); + } + + @SuppressWarnings("unchecked") + public Class[] getPlugins() { + return plugins.toArray(new Class[0]); + } + + public Class[] pluginsWithAddition(List> additionalPlugins) { + return mergePlugins(plugins, additionalPlugins).toArray(Class[]::new); + } + } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java index 727dd3fbec..f5cd4ffc2f 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java @@ -1,12 +1,12 @@ /* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ package org.opensearch.test.framework.cluster; import java.util.Collections; @@ -24,23 +24,23 @@ public class ContextHeaderDecoratorClient extends FilterClient { - private Map headers; + private Map headers; - public ContextHeaderDecoratorClient(Client in, Map headers) { - super(in); - this.headers = headers != null ? headers : Collections.emptyMap(); - } + public ContextHeaderDecoratorClient(Client in, Map headers) { + super(in); + this.headers = headers != null ? headers : Collections.emptyMap(); + } - @Override - protected void doExecute(ActionType action, Request request, - ActionListener listener) { + @Override + protected void doExecute(ActionType action, Request request, + ActionListener listener) { - ThreadContext threadContext = threadPool().getThreadContext(); - ContextPreservingActionListener wrappedListener = new ContextPreservingActionListener<>(threadContext.newRestorableContext(true), listener); - - try (StoredContext ctx = threadContext.stashContext()) { - threadContext.putHeader(this.headers); - super.doExecute(action, request, wrappedListener); - } - } + ThreadContext threadContext = threadPool().getThreadContext(); + ContextPreservingActionListener wrappedListener = new ContextPreservingActionListener<>(threadContext.newRestorableContext(true), listener); + + try (StoredContext ctx = threadContext.stashContext()) { + threadContext.putHeader(this.headers); + super.doExecute(action, request, wrappedListener); + } + } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 28504038d4..50e9582199 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -1,30 +1,30 @@ /* - * Copyright 2015-2021 floragunn GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ +* Copyright 2015-2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.test.framework.cluster; @@ -57,279 +57,279 @@ public class LocalCluster extends ExternalResource implements AutoCloseable, OpenSearchClientProvider { - private static final Logger log = LogManager.getLogger(LocalCluster.class); + private static final Logger log = LogManager.getLogger(LocalCluster.class); - static { - System.setProperty("security.default_init.dir", new File("./securityconfig").getAbsolutePath()); - } - - protected static final AtomicLong num = new AtomicLong(); + static { + System.setProperty("security.default_init.dir", new File("./securityconfig").getAbsolutePath()); + } - private final List> plugins; - private final ClusterManager clusterManager; - private final TestSecurityConfig testSecurityConfig; - private Settings nodeOverride; - private final String clusterName; - private final MinimumSecuritySettingsSupplierFactory minimumOpenSearchSettingsSupplierFactory; - private final TestCertificates testCertificates; - private final List clusterDependencies; - private final Map remotes; - private volatile LocalOpenSearchCluster localOpenSearchCluster; + protected static final AtomicLong num = new AtomicLong(); + + private final List> plugins; + private final ClusterManager clusterManager; + private final TestSecurityConfig testSecurityConfig; + private Settings nodeOverride; + private final String clusterName; + private final MinimumSecuritySettingsSupplierFactory minimumOpenSearchSettingsSupplierFactory; + private final TestCertificates testCertificates; + private final List clusterDependencies; + private final Map remotes; + private volatile LocalOpenSearchCluster localOpenSearchCluster; private final List testIndices; - private LocalCluster(String clusterName, TestSecurityConfig testSgConfig, Settings nodeOverride, - ClusterManager clusterManager, List> plugins, TestCertificates testCertificates, - List clusterDependencies, Map remotes, List testIndices) { - this.plugins = plugins; - this.testCertificates = testCertificates; - this.clusterManager = clusterManager; - this.testSecurityConfig = testSgConfig; - this.nodeOverride = nodeOverride; - this.clusterName = clusterName; - this.minimumOpenSearchSettingsSupplierFactory = new MinimumSecuritySettingsSupplierFactory(testCertificates); - this.remotes = remotes; - this.clusterDependencies = clusterDependencies; - this.testIndices = testIndices; - } - - @Override - public void before() throws Throwable { - if (localOpenSearchCluster == null) { - for (LocalCluster dependency : clusterDependencies) { - if (!dependency.isStarted()) { - dependency.before(); - } - } - - for (Map.Entry entry : remotes.entrySet()) { - @SuppressWarnings("resource") - InetSocketAddress transportAddress = entry.getValue().localOpenSearchCluster.clusterManagerNode().getTransportAddress(); - nodeOverride = Settings.builder().put(nodeOverride) - .putList("cluster.remote." + entry.getKey() + ".seeds", transportAddress.getHostString() + ":" + transportAddress.getPort()) - .build(); - } - - start(); - } - } - - @Override - protected void after() { - close(); - } - - @Override - public void close() { - if (localOpenSearchCluster != null && localOpenSearchCluster.isStarted()) { - try { - localOpenSearchCluster.destroy(); - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - localOpenSearchCluster = null; - } - } - } - - @Override - public String getClusterName() { - return clusterName; - } - - @Override - public InetSocketAddress getHttpAddress() { - return localOpenSearchCluster.clientNode().getHttpAddress(); - } - - @Override - public InetSocketAddress getTransportAddress() { - return localOpenSearchCluster.clientNode().getTransportAddress(); - } - - public Client getInternalNodeClient() { - return localOpenSearchCluster.clientNode().getInternalNodeClient(); - } - - public PluginAwareNode node() { - return this.localOpenSearchCluster.clusterManagerNode().esNode(); - } - - public List nodes() { - return this.localOpenSearchCluster.getNodes(); - } - - public LocalOpenSearchCluster.Node getNodeByName(String name) { - return this.localOpenSearchCluster.getNodeByName(name); - } - - public boolean isStarted() { - return localOpenSearchCluster != null; - } - - public Random getRandom() { - return localOpenSearchCluster.getRandom(); - } - - private void start() { - try { - localOpenSearchCluster = new LocalOpenSearchCluster(clusterName, clusterManager, - minimumOpenSearchSettingsSupplierFactory.minimumOpenSearchSettings(nodeOverride), plugins, testCertificates); - - localOpenSearchCluster.start(); - - - if (testSecurityConfig != null) { - initSecurityIndex(testSecurityConfig); - } - - try (Client client = getInternalNodeClient()) { - for (TestIndex index : this.testIndices) { - index.create(client); - } - } - - } catch (Exception e) { - log.error("Local ES cluster start failed", e); - throw new RuntimeException(e); - } - } - - private void initSecurityIndex(TestSecurityConfig testSecurityConfig) { - log.info("Initializing OpenSearch Security index"); - try(Client client = new ContextHeaderDecoratorClient(this.getInternalNodeClient(), Map.of(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER , "true"))) { - testSecurityConfig.initIndex(client); - } - } - - public static class Builder { - - private final Settings.Builder nodeOverrideSettingsBuilder = Settings.builder(); - private final List> plugins = new ArrayList<>(); - private Map remoteClusters = new HashMap<>(); - private List clusterDependencies = new ArrayList<>(); - private List testIndices = new ArrayList<>(); - private ClusterManager clusterManager = ClusterManager.DEFAULT; - private TestSecurityConfig testSecurityConfig = new TestSecurityConfig(); - private String clusterName = "local_cluster"; - private TestCertificates testCertificates; - - public Builder() { - this.testCertificates = new TestCertificates(); - } - - public Builder dependsOn(Object object) { - // We just want to make sure that the object is already done - if (object == null) { - throw new IllegalStateException("Dependency not fulfilled"); - } - return this; - } - - public Builder clusterManager(ClusterManager clusterManager) { - this.clusterManager = clusterManager; - return this; - } - - public Builder singleNode() { - this.clusterManager = ClusterManager.SINGLENODE; - return this; - } - - public Builder sgConfig(TestSecurityConfig testSgConfig) { - this.testSecurityConfig = testSgConfig; - return this; - } - - public Builder nodeSettings(Map settings) { - settings.forEach((key, value) -> { - if (value instanceof List) { - List values = ((List) value).stream().map(String::valueOf).collect(Collectors.toList()); - nodeOverrideSettingsBuilder.putList(key, values); - } else { - nodeOverrideSettingsBuilder.put(key, String.valueOf(value)); - } - }); - - return this; - } - - public Builder plugin(Class plugin) { - this.plugins.add(plugin); - - return this; - } - - public Builder remote(String name, LocalCluster anotherCluster) { - remoteClusters.put(name, anotherCluster); - - clusterDependencies.add(anotherCluster); - - return this; - } - - public Builder indices(TestIndex... indices) { - this.testIndices.addAll(Arrays.asList(indices)); - return this; - } - - public Builder users(TestSecurityConfig.User... users) { - for (TestSecurityConfig.User user : users) { - testSecurityConfig.user(user); - } - return this; - } - - public Builder roles(Role... roles) { - testSecurityConfig.roles(roles); - return this; - } - - public Builder roleMapping(RoleMapping... mappings) { - testSecurityConfig.roleMapping(mappings); - return this; - } - - public Builder roleToRoleMapping(Role role, String... backendRoles) { - testSecurityConfig.roleToRoleMapping(role, backendRoles); - return this; - } - - public Builder authc(TestSecurityConfig.AuthcDomain authc) { - testSecurityConfig.authc(authc); - return this; - } - - public Builder clusterName(String clusterName) { - this.clusterName = clusterName; - return this; - } - - public Builder configIndexName(String configIndexName) { - testSecurityConfig.configIndexName(configIndexName); - return this; - } - - public Builder anonymousAuth(boolean anonAuthEnabled) { - testSecurityConfig.anonymousAuth(anonAuthEnabled); - return this; - } - - public LocalCluster build() { - try { - - clusterName += "_" + num.incrementAndGet(); - Settings settings = nodeOverrideSettingsBuilder - .put(ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, false) - .build(); - return new LocalCluster(clusterName, testSecurityConfig, settings, clusterManager, plugins, - testCertificates, clusterDependencies, remoteClusters, testIndices); - } catch (Exception e) { - log.error("Failed to build LocalCluster", e); - throw new RuntimeException(e); - } - } - - } + private LocalCluster(String clusterName, TestSecurityConfig testSgConfig, Settings nodeOverride, + ClusterManager clusterManager, List> plugins, TestCertificates testCertificates, + List clusterDependencies, Map remotes, List testIndices) { + this.plugins = plugins; + this.testCertificates = testCertificates; + this.clusterManager = clusterManager; + this.testSecurityConfig = testSgConfig; + this.nodeOverride = nodeOverride; + this.clusterName = clusterName; + this.minimumOpenSearchSettingsSupplierFactory = new MinimumSecuritySettingsSupplierFactory(testCertificates); + this.remotes = remotes; + this.clusterDependencies = clusterDependencies; + this.testIndices = testIndices; + } + + @Override + public void before() throws Throwable { + if (localOpenSearchCluster == null) { + for (LocalCluster dependency : clusterDependencies) { + if (!dependency.isStarted()) { + dependency.before(); + } + } + + for (Map.Entry entry : remotes.entrySet()) { + @SuppressWarnings("resource") + InetSocketAddress transportAddress = entry.getValue().localOpenSearchCluster.clusterManagerNode().getTransportAddress(); + nodeOverride = Settings.builder().put(nodeOverride) + .putList("cluster.remote." + entry.getKey() + ".seeds", transportAddress.getHostString() + ":" + transportAddress.getPort()) + .build(); + } + + start(); + } + } + + @Override + protected void after() { + close(); + } + + @Override + public void close() { + if (localOpenSearchCluster != null && localOpenSearchCluster.isStarted()) { + try { + localOpenSearchCluster.destroy(); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + localOpenSearchCluster = null; + } + } + } + + @Override + public String getClusterName() { + return clusterName; + } + + @Override + public InetSocketAddress getHttpAddress() { + return localOpenSearchCluster.clientNode().getHttpAddress(); + } + + @Override + public InetSocketAddress getTransportAddress() { + return localOpenSearchCluster.clientNode().getTransportAddress(); + } + + public Client getInternalNodeClient() { + return localOpenSearchCluster.clientNode().getInternalNodeClient(); + } + + public PluginAwareNode node() { + return this.localOpenSearchCluster.clusterManagerNode().esNode(); + } + + public List nodes() { + return this.localOpenSearchCluster.getNodes(); + } + + public LocalOpenSearchCluster.Node getNodeByName(String name) { + return this.localOpenSearchCluster.getNodeByName(name); + } + + public boolean isStarted() { + return localOpenSearchCluster != null; + } + + public Random getRandom() { + return localOpenSearchCluster.getRandom(); + } + + private void start() { + try { + localOpenSearchCluster = new LocalOpenSearchCluster(clusterName, clusterManager, + minimumOpenSearchSettingsSupplierFactory.minimumOpenSearchSettings(nodeOverride), plugins, testCertificates); + + localOpenSearchCluster.start(); + + + if (testSecurityConfig != null) { + initSecurityIndex(testSecurityConfig); + } + + try (Client client = getInternalNodeClient()) { + for (TestIndex index : this.testIndices) { + index.create(client); + } + } + + } catch (Exception e) { + log.error("Local ES cluster start failed", e); + throw new RuntimeException(e); + } + } + + private void initSecurityIndex(TestSecurityConfig testSecurityConfig) { + log.info("Initializing OpenSearch Security index"); + try(Client client = new ContextHeaderDecoratorClient(this.getInternalNodeClient(), Map.of(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER , "true"))) { + testSecurityConfig.initIndex(client); + } + } + + public static class Builder { + + private final Settings.Builder nodeOverrideSettingsBuilder = Settings.builder(); + private final List> plugins = new ArrayList<>(); + private Map remoteClusters = new HashMap<>(); + private List clusterDependencies = new ArrayList<>(); + private List testIndices = new ArrayList<>(); + private ClusterManager clusterManager = ClusterManager.DEFAULT; + private TestSecurityConfig testSecurityConfig = new TestSecurityConfig(); + private String clusterName = "local_cluster"; + private TestCertificates testCertificates; + + public Builder() { + this.testCertificates = new TestCertificates(); + } + + public Builder dependsOn(Object object) { + // We just want to make sure that the object is already done + if (object == null) { + throw new IllegalStateException("Dependency not fulfilled"); + } + return this; + } + + public Builder clusterManager(ClusterManager clusterManager) { + this.clusterManager = clusterManager; + return this; + } + + public Builder singleNode() { + this.clusterManager = ClusterManager.SINGLENODE; + return this; + } + + public Builder sgConfig(TestSecurityConfig testSgConfig) { + this.testSecurityConfig = testSgConfig; + return this; + } + + public Builder nodeSettings(Map settings) { + settings.forEach((key, value) -> { + if (value instanceof List) { + List values = ((List) value).stream().map(String::valueOf).collect(Collectors.toList()); + nodeOverrideSettingsBuilder.putList(key, values); + } else { + nodeOverrideSettingsBuilder.put(key, String.valueOf(value)); + } + }); + + return this; + } + + public Builder plugin(Class plugin) { + this.plugins.add(plugin); + + return this; + } + + public Builder remote(String name, LocalCluster anotherCluster) { + remoteClusters.put(name, anotherCluster); + + clusterDependencies.add(anotherCluster); + + return this; + } + + public Builder indices(TestIndex... indices) { + this.testIndices.addAll(Arrays.asList(indices)); + return this; + } + + public Builder users(TestSecurityConfig.User... users) { + for (TestSecurityConfig.User user : users) { + testSecurityConfig.user(user); + } + return this; + } + + public Builder roles(Role... roles) { + testSecurityConfig.roles(roles); + return this; + } + + public Builder roleMapping(RoleMapping... mappings) { + testSecurityConfig.roleMapping(mappings); + return this; + } + + public Builder roleToRoleMapping(Role role, String... backendRoles) { + testSecurityConfig.roleToRoleMapping(role, backendRoles); + return this; + } + + public Builder authc(TestSecurityConfig.AuthcDomain authc) { + testSecurityConfig.authc(authc); + return this; + } + + public Builder clusterName(String clusterName) { + this.clusterName = clusterName; + return this; + } + + public Builder configIndexName(String configIndexName) { + testSecurityConfig.configIndexName(configIndexName); + return this; + } + + public Builder anonymousAuth(boolean anonAuthEnabled) { + testSecurityConfig.anonymousAuth(anonAuthEnabled); + return this; + } + + public LocalCluster build() { + try { + + clusterName += "_" + num.incrementAndGet(); + Settings settings = nodeOverrideSettingsBuilder + .put(ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, false) + .build(); + return new LocalCluster(clusterName, testSecurityConfig, settings, clusterManager, plugins, + testCertificates, clusterDependencies, remoteClusters, testIndices); + } catch (Exception e) { + log.error("Failed to build LocalCluster", e); + throw new RuntimeException(e); + } + } + + } @Override public TestCertificates getTestCertificates() { diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index 5610a9fa2f..565537bd15 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -1,30 +1,30 @@ /* - * Copyright 2015-2021 floragunn GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ +* Copyright 2015-2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.test.framework.cluster; @@ -75,415 +75,415 @@ public class LocalOpenSearchCluster { - static { - System.setProperty("opensearch.enforce.bootstrap.checks", "true"); - } - - private static final Logger log = LogManager.getLogger(LocalOpenSearchCluster.class); - - private final String clusterName; - private final ClusterManager clusterManager; - private final NodeSettingsSupplier nodeSettingsSupplier; - private final List> additionalPlugins; - private final List nodes = new ArrayList<>(); - private final TestCertificates testCertificates; - - private File clusterHomeDir; - private List seedHosts; - private List initialClusterManagerHosts; - private int retry = 0; - private boolean started; - private Random random = new Random(); - - public LocalOpenSearchCluster(String clusterName, ClusterManager clusterManager, NodeSettingsSupplier nodeSettingsSupplier, - List> additionalPlugins, TestCertificates testCertificates) { - this.clusterName = clusterName; - this.clusterManager = clusterManager; - this.nodeSettingsSupplier = nodeSettingsSupplier; - this.additionalPlugins = additionalPlugins; - this.testCertificates = testCertificates; - try { + static { + System.setProperty("opensearch.enforce.bootstrap.checks", "true"); + } + + private static final Logger log = LogManager.getLogger(LocalOpenSearchCluster.class); + + private final String clusterName; + private final ClusterManager clusterManager; + private final NodeSettingsSupplier nodeSettingsSupplier; + private final List> additionalPlugins; + private final List nodes = new ArrayList<>(); + private final TestCertificates testCertificates; + + private File clusterHomeDir; + private List seedHosts; + private List initialClusterManagerHosts; + private int retry = 0; + private boolean started; + private Random random = new Random(); + + public LocalOpenSearchCluster(String clusterName, ClusterManager clusterManager, NodeSettingsSupplier nodeSettingsSupplier, + List> additionalPlugins, TestCertificates testCertificates) { + this.clusterName = clusterName; + this.clusterManager = clusterManager; + this.nodeSettingsSupplier = nodeSettingsSupplier; + this.additionalPlugins = additionalPlugins; + this.testCertificates = testCertificates; + try { this.clusterHomeDir = Files.createTempDirectory("local_cluster_" + clusterName).toFile(); } catch (IOException e) { throw new IllegalStateException(e); } - } + } - private List getNodesByType(NodeType nodeType) { - return nodes.stream() - .filter(currentNode -> currentNode.hasAssignedType(nodeType)) - .collect(Collectors.toList()); - } + private List getNodesByType(NodeType nodeType) { + return nodes.stream() + .filter(currentNode -> currentNode.hasAssignedType(nodeType)) + .collect(Collectors.toList()); + } - private long countNodesByType(NodeType nodeType) { - return getNodesByType(nodeType).stream().count(); - } + private long countNodesByType(NodeType nodeType) { + return getNodesByType(nodeType).stream().count(); + } - public void start() throws Exception { - log.info("Starting {}", clusterName); + public void start() throws Exception { + log.info("Starting {}", clusterName); - int clusterManagerNodeCount = clusterManager.getClusterManagerNodes(); - int nonClusterManagerNodeCount = clusterManager.getDataNodes() + clusterManager.getClientNodes(); + int clusterManagerNodeCount = clusterManager.getClusterManagerNodes(); + int nonClusterManagerNodeCount = clusterManager.getDataNodes() + clusterManager.getClientNodes(); - SortedSet clusterManagerNodeTransportPorts = TCP.allocate(clusterName, Math.max(clusterManagerNodeCount, 4), 5000 + 42 * 1000 + 300); - SortedSet clusterManagerNodeHttpPorts = TCP.allocate(clusterName, clusterManagerNodeCount, 5000 + 42 * 1000 + 200); + SortedSet clusterManagerNodeTransportPorts = TCP.allocate(clusterName, Math.max(clusterManagerNodeCount, 4), 5000 + 42 * 1000 + 300); + SortedSet clusterManagerNodeHttpPorts = TCP.allocate(clusterName, clusterManagerNodeCount, 5000 + 42 * 1000 + 200); - this.seedHosts = toHostList(clusterManagerNodeTransportPorts); - Set clusterManagerPorts = clusterManagerNodeTransportPorts - .stream().limit(clusterManagerNodeCount).collect(Collectors.toSet()); - this.initialClusterManagerHosts = toHostList(clusterManagerPorts); + this.seedHosts = toHostList(clusterManagerNodeTransportPorts); + Set clusterManagerPorts = clusterManagerNodeTransportPorts + .stream().limit(clusterManagerNodeCount).collect(Collectors.toSet()); + this.initialClusterManagerHosts = toHostList(clusterManagerPorts); - started = true; + started = true; - CompletableFuture clusterManagerNodeFuture = startNodes( - clusterManager.getClusterManagerNodeSettings(), clusterManagerNodeTransportPorts, - clusterManagerNodeHttpPorts); + CompletableFuture clusterManagerNodeFuture = startNodes( + clusterManager.getClusterManagerNodeSettings(), clusterManagerNodeTransportPorts, + clusterManagerNodeHttpPorts); - SortedSet nonClusterManagerNodeTransportPorts = TCP.allocate(clusterName, nonClusterManagerNodeCount, 5000 + 42 * 1000 + 310); - SortedSet nonClusterManagerNodeHttpPorts = TCP.allocate(clusterName, nonClusterManagerNodeCount, 5000 + 42 * 1000 + 210); + SortedSet nonClusterManagerNodeTransportPorts = TCP.allocate(clusterName, nonClusterManagerNodeCount, 5000 + 42 * 1000 + 310); + SortedSet nonClusterManagerNodeHttpPorts = TCP.allocate(clusterName, nonClusterManagerNodeCount, 5000 + 42 * 1000 + 210); - CompletableFuture nonClusterManagerNodeFuture = startNodes( - clusterManager.getNonClusterManagerNodeSettings(), nonClusterManagerNodeTransportPorts, - nonClusterManagerNodeHttpPorts); + CompletableFuture nonClusterManagerNodeFuture = startNodes( + clusterManager.getNonClusterManagerNodeSettings(), nonClusterManagerNodeTransportPorts, + nonClusterManagerNodeHttpPorts); - CompletableFuture.allOf(clusterManagerNodeFuture, nonClusterManagerNodeFuture).join(); - - if (isNodeFailedWithPortCollision()) { - log.info("Detected port collision for cluster manager node. Retrying."); - - retry(); - return; - } - - log.info("Startup finished. Waiting for GREEN"); - - waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(10), nodes.size()); - - log.info("Started: {}", this); - - } - - public String getClusterName() { - return clusterName; - } - - public boolean isStarted() { - return started; - } - - public void stop() { - List> stopFutures = new ArrayList<>(); - for (Node node : nodes) { - stopFutures.add(node.stop(2, TimeUnit.SECONDS)); - } - CompletableFuture.allOf(stopFutures.toArray(size -> new CompletableFuture[size])).join(); - } - - public void destroy() { - stop(); - nodes.clear(); - - try { - FileUtils.deleteDirectory(clusterHomeDir); - } catch (IOException e) { - log.warn("Error while deleting " + clusterHomeDir, e); - } - } - - public Node clientNode() { - return findRunningNode(getNodesByType(CLIENT), getNodesByType(DATA), getNodesByType(CLUSTER_MANAGER)); - } - - public Node clusterManagerNode() { - return findRunningNode(getNodesByType(CLUSTER_MANAGER)); - } - - public List getNodes() { - return Collections.unmodifiableList(nodes); - } - - public Node getNodeByName(String name) { - return nodes.stream().filter(node -> node.getNodeName().equals(name)).findAny().orElseThrow(() -> new RuntimeException( - "No such node with name: " + name + "; available: " + nodes.stream().map(Node::getNodeName).collect(Collectors.toList()))); - } - - private boolean isNodeFailedWithPortCollision() { - return nodes.stream().anyMatch(Node::isPortCollision); - } - - private void retry() throws Exception { - retry++; - - if (retry > 10) { - throw new RuntimeException("Detected port collisions for cluster manager node. Giving up."); - } - - stop(); - - this.nodes.clear(); - this.seedHosts = null; - this.initialClusterManagerHosts = null; - this.clusterHomeDir = Files.createTempDirectory("local_cluster_" + clusterName + "_retry_" + retry).toFile(); - - start(); - } - - @SafeVarargs - private final Node findRunningNode(List nodes, List... moreNodes) { - for (Node node : nodes) { - if (node.isRunning()) { - return node; - } - } - - if (moreNodes != null && moreNodes.length > 0) { - for (List nodesList : moreNodes) { - for (Node node : nodesList) { - if (node.isRunning()) { - return node; - } - } - } - } - - return null; - } - - private CompletableFuture startNodes(List nodeSettingList, SortedSet transportPorts, SortedSet httpPorts) { - Iterator transportPortIterator = transportPorts.iterator(); - Iterator httpPortIterator = httpPorts.iterator(); - List> futures = new ArrayList<>(); - - for (NodeSettings nodeSettings : nodeSettingList) { - Node node = new Node(nodeSettings, transportPortIterator.next(), httpPortIterator.next()); - futures.add(node.start()); - } - - return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); - } - - public void waitForCluster(ClusterHealthStatus status, TimeValue timeout, int expectedNodeCount) throws IOException { - Client client = clientNode().getInternalNodeClient(); - - - log.debug("waiting for cluster state {} and {} nodes", status.name(), expectedNodeCount); - AdminClient adminClient = client.admin(); - - final ClusterHealthResponse healthResponse = adminClient.cluster().prepareHealth().setWaitForStatus(status).setTimeout(timeout) - .setMasterNodeTimeout(timeout).setWaitForNodes("" + expectedNodeCount).execute().actionGet(); - - if (log.isDebugEnabled()) { - log.debug("Current ClusterState:\n{}", Strings.toString(healthResponse)); - } - - if (healthResponse.isTimedOut()) { - throw new IOException( - "cluster state is " + healthResponse.getStatus().name() + " with " + healthResponse.getNumberOfNodes() + " nodes"); - } else { - log.debug("... cluster state ok {} with {} nodes", healthResponse.getStatus().name(), healthResponse.getNumberOfNodes()); - } - - assertEquals(expectedNodeCount, healthResponse.getNumberOfNodes()); - - } - - @Override - public String toString() { - String clusterManagerNodes = nodeByTypeToString(CLUSTER_MANAGER); - String dataNodes = nodeByTypeToString(DATA); - String clientNodes = nodeByTypeToString(CLIENT); - return "\nES Cluster " + clusterName - + "\ncluster manager nodes: " + clusterManagerNodes - + "\n data nodes: " + dataNodes - + "\nclient nodes: " + clientNodes + "\n"; - } - - private String nodeByTypeToString(NodeType type) { - return getNodesByType(type).stream().map(Objects::toString).collect(Collectors.joining(", ")); - } - - private static List toHostList(Collection ports) { - return ports.stream().map(port -> "127.0.0.1:" + port).collect(Collectors.toList()); - } - - private String createNextNodeName(NodeSettings nodeSettings) { - NodeType type = nodeSettings.recognizeNodeType(); - long nodeTypeCount = countNodesByType(type); - String nodeType = type.name().toLowerCase(Locale.ROOT); - return nodeType + "_" + nodeTypeCount; - } - - public class Node implements OpenSearchClientProvider { - private final NodeType nodeType; - private final String nodeName; - private final NodeSettings nodeSettings; - private final File nodeHomeDir; - private final File dataDir; - private final File logsDir; - private final int transportPort; - private final int httpPort; - private final InetSocketAddress httpAddress; - private final InetSocketAddress transportAddress; - private PluginAwareNode node; - private boolean running = false; - private boolean portCollision = false; - - Node(NodeSettings nodeSettings, int transportPort, int httpPort) { - this.nodeName = createNextNodeName(requireNonNull(nodeSettings, "Node settings are required.")); - this.nodeSettings = nodeSettings; - this.nodeHomeDir = new File(clusterHomeDir, nodeName); - this.dataDir = new File(this.nodeHomeDir, "data"); - this.logsDir = new File(this.nodeHomeDir, "logs"); - this.transportPort = transportPort; - this.httpPort = httpPort; - InetAddress hostAddress = InetAddresses.forString("127.0.0.1"); - this.httpAddress = new InetSocketAddress(hostAddress, httpPort); - this.transportAddress = new InetSocketAddress(hostAddress, transportPort); - - this.nodeType = nodeSettings.recognizeNodeType(); - nodes.add(this); - } - - boolean hasAssignedType(NodeType type) { - return requireNonNull(type, "Node type is required.").equals(this.nodeType); - } - - CompletableFuture start() { - CompletableFuture completableFuture = new CompletableFuture<>(); - Class[] mergedPlugins = nodeSettings.pluginsWithAddition(additionalPlugins); - this.node = new PluginAwareNode(nodeSettings.clusterManagerNode, getOpenSearchSettings(), mergedPlugins); - - new Thread(new Runnable() { - - @Override - public void run() { - try { - node.start(); - running = true; - completableFuture.complete(StartStage.INITIALIZED); - } catch (BindTransportException | BindHttpException e) { - log.warn("Port collision detected for {}", this, e); - portCollision = true; - try { - node.close(); - } catch (IOException e1) { - log.error(e1); - } - - node = null; - TCP.reserve(transportPort, httpPort); - - completableFuture.complete(StartStage.RETRY); - - } catch (Throwable e) { - log.error("Unable to start {}", this, e); - node = null; - completableFuture.completeExceptionally(e); - } - } - }).start(); - - return completableFuture; - } - - public Client getInternalNodeClient() { - return node.client(); - } - - public PluginAwareNode esNode() { - return node; - } - - public boolean isRunning() { - return running; - } - - public X getInjectable(Class clazz) { - return node.injector().getInstance(clazz); - } - - public CompletableFuture stop(long timeout, TimeUnit timeUnit) { - return CompletableFuture.supplyAsync(() -> { - try { - log.info("Stopping {}", this); - - running = false; - - if (node != null) { - node.close(); - boolean stopped = node.awaitClose(timeout, timeUnit); - node = null; - return stopped; - } else { - return false; - } - } catch (Throwable e) { - String message = "Error while stopping " + this; - log.warn(message, e); - throw new RuntimeException(message, e); - } - }); - } - - @Override - public String toString() { - String state = running ? "RUNNING" : node != null ? "INITIALIZING" : "STOPPED"; - - return nodeName + " " + state + " [" + transportPort + ", " + httpPort + "]"; - } - - public boolean isPortCollision() { - return portCollision; - } - - public String getNodeName() { - return nodeName; - } - - @Override - public InetSocketAddress getHttpAddress() { - return httpAddress; - } - - @Override - public InetSocketAddress getTransportAddress() { - return transportAddress; - } - - private Settings getOpenSearchSettings() { - Settings settings = getMinimalOpenSearchSettings(); - - if (nodeSettingsSupplier != null) { - // TODO node number - return Settings.builder().put(settings).put(nodeSettingsSupplier.get(0)).build(); - } - - return settings; - } - - private Settings getMinimalOpenSearchSettings() { - return Settings.builder().put("node.name", nodeName).put("node.data", nodeSettings.dataNode).put("node.master", nodeSettings.clusterManagerNode) - .put("cluster.name", clusterName).put("path.home", nodeHomeDir.toPath()).put("path.data", dataDir.toPath()) - .put("path.logs", logsDir.toPath()).putList("cluster.initial_master_nodes", initialClusterManagerHosts) - .put("discovery.initial_state_timeout", "8s").putList("discovery.seed_hosts", seedHosts).put("transport.tcp.port", transportPort) - .put("http.port", httpPort).put("cluster.routing.allocation.disk.threshold_enabled", false) - .put("discovery.probe.connect_timeout", "10s").put("discovery.probe.handshake_timeout", "10s").put("http.cors.enabled", true) - .build(); - } - - @Override - public String getClusterName() { - return clusterName; - } - - @Override - public TestCertificates getTestCertificates() { - return testCertificates; - } - } - - public Random getRandom() { - return random; - } + CompletableFuture.allOf(clusterManagerNodeFuture, nonClusterManagerNodeFuture).join(); + + if (isNodeFailedWithPortCollision()) { + log.info("Detected port collision for cluster manager node. Retrying."); + + retry(); + return; + } + + log.info("Startup finished. Waiting for GREEN"); + + waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(10), nodes.size()); + + log.info("Started: {}", this); + + } + + public String getClusterName() { + return clusterName; + } + + public boolean isStarted() { + return started; + } + + public void stop() { + List> stopFutures = new ArrayList<>(); + for (Node node : nodes) { + stopFutures.add(node.stop(2, TimeUnit.SECONDS)); + } + CompletableFuture.allOf(stopFutures.toArray(size -> new CompletableFuture[size])).join(); + } + + public void destroy() { + stop(); + nodes.clear(); + + try { + FileUtils.deleteDirectory(clusterHomeDir); + } catch (IOException e) { + log.warn("Error while deleting " + clusterHomeDir, e); + } + } + + public Node clientNode() { + return findRunningNode(getNodesByType(CLIENT), getNodesByType(DATA), getNodesByType(CLUSTER_MANAGER)); + } + + public Node clusterManagerNode() { + return findRunningNode(getNodesByType(CLUSTER_MANAGER)); + } + + public List getNodes() { + return Collections.unmodifiableList(nodes); + } + + public Node getNodeByName(String name) { + return nodes.stream().filter(node -> node.getNodeName().equals(name)).findAny().orElseThrow(() -> new RuntimeException( + "No such node with name: " + name + "; available: " + nodes.stream().map(Node::getNodeName).collect(Collectors.toList()))); + } + + private boolean isNodeFailedWithPortCollision() { + return nodes.stream().anyMatch(Node::isPortCollision); + } + + private void retry() throws Exception { + retry++; + + if (retry > 10) { + throw new RuntimeException("Detected port collisions for cluster manager node. Giving up."); + } + + stop(); + + this.nodes.clear(); + this.seedHosts = null; + this.initialClusterManagerHosts = null; + this.clusterHomeDir = Files.createTempDirectory("local_cluster_" + clusterName + "_retry_" + retry).toFile(); + + start(); + } + + @SafeVarargs + private final Node findRunningNode(List nodes, List... moreNodes) { + for (Node node : nodes) { + if (node.isRunning()) { + return node; + } + } + + if (moreNodes != null && moreNodes.length > 0) { + for (List nodesList : moreNodes) { + for (Node node : nodesList) { + if (node.isRunning()) { + return node; + } + } + } + } + + return null; + } + + private CompletableFuture startNodes(List nodeSettingList, SortedSet transportPorts, SortedSet httpPorts) { + Iterator transportPortIterator = transportPorts.iterator(); + Iterator httpPortIterator = httpPorts.iterator(); + List> futures = new ArrayList<>(); + + for (NodeSettings nodeSettings : nodeSettingList) { + Node node = new Node(nodeSettings, transportPortIterator.next(), httpPortIterator.next()); + futures.add(node.start()); + } + + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + } + + public void waitForCluster(ClusterHealthStatus status, TimeValue timeout, int expectedNodeCount) throws IOException { + Client client = clientNode().getInternalNodeClient(); + + + log.debug("waiting for cluster state {} and {} nodes", status.name(), expectedNodeCount); + AdminClient adminClient = client.admin(); + + final ClusterHealthResponse healthResponse = adminClient.cluster().prepareHealth().setWaitForStatus(status).setTimeout(timeout) + .setMasterNodeTimeout(timeout).setWaitForNodes("" + expectedNodeCount).execute().actionGet(); + + if (log.isDebugEnabled()) { + log.debug("Current ClusterState:\n{}", Strings.toString(healthResponse)); + } + + if (healthResponse.isTimedOut()) { + throw new IOException( + "cluster state is " + healthResponse.getStatus().name() + " with " + healthResponse.getNumberOfNodes() + " nodes"); + } else { + log.debug("... cluster state ok {} with {} nodes", healthResponse.getStatus().name(), healthResponse.getNumberOfNodes()); + } + + assertEquals(expectedNodeCount, healthResponse.getNumberOfNodes()); + + } + + @Override + public String toString() { + String clusterManagerNodes = nodeByTypeToString(CLUSTER_MANAGER); + String dataNodes = nodeByTypeToString(DATA); + String clientNodes = nodeByTypeToString(CLIENT); + return "\nES Cluster " + clusterName + + "\ncluster manager nodes: " + clusterManagerNodes + + "\n data nodes: " + dataNodes + + "\nclient nodes: " + clientNodes + "\n"; + } + + private String nodeByTypeToString(NodeType type) { + return getNodesByType(type).stream().map(Objects::toString).collect(Collectors.joining(", ")); + } + + private static List toHostList(Collection ports) { + return ports.stream().map(port -> "127.0.0.1:" + port).collect(Collectors.toList()); + } + + private String createNextNodeName(NodeSettings nodeSettings) { + NodeType type = nodeSettings.recognizeNodeType(); + long nodeTypeCount = countNodesByType(type); + String nodeType = type.name().toLowerCase(Locale.ROOT); + return nodeType + "_" + nodeTypeCount; + } + + public class Node implements OpenSearchClientProvider { + private final NodeType nodeType; + private final String nodeName; + private final NodeSettings nodeSettings; + private final File nodeHomeDir; + private final File dataDir; + private final File logsDir; + private final int transportPort; + private final int httpPort; + private final InetSocketAddress httpAddress; + private final InetSocketAddress transportAddress; + private PluginAwareNode node; + private boolean running = false; + private boolean portCollision = false; + + Node(NodeSettings nodeSettings, int transportPort, int httpPort) { + this.nodeName = createNextNodeName(requireNonNull(nodeSettings, "Node settings are required.")); + this.nodeSettings = nodeSettings; + this.nodeHomeDir = new File(clusterHomeDir, nodeName); + this.dataDir = new File(this.nodeHomeDir, "data"); + this.logsDir = new File(this.nodeHomeDir, "logs"); + this.transportPort = transportPort; + this.httpPort = httpPort; + InetAddress hostAddress = InetAddresses.forString("127.0.0.1"); + this.httpAddress = new InetSocketAddress(hostAddress, httpPort); + this.transportAddress = new InetSocketAddress(hostAddress, transportPort); + + this.nodeType = nodeSettings.recognizeNodeType(); + nodes.add(this); + } + + boolean hasAssignedType(NodeType type) { + return requireNonNull(type, "Node type is required.").equals(this.nodeType); + } + + CompletableFuture start() { + CompletableFuture completableFuture = new CompletableFuture<>(); + Class[] mergedPlugins = nodeSettings.pluginsWithAddition(additionalPlugins); + this.node = new PluginAwareNode(nodeSettings.clusterManagerNode, getOpenSearchSettings(), mergedPlugins); + + new Thread(new Runnable() { + + @Override + public void run() { + try { + node.start(); + running = true; + completableFuture.complete(StartStage.INITIALIZED); + } catch (BindTransportException | BindHttpException e) { + log.warn("Port collision detected for {}", this, e); + portCollision = true; + try { + node.close(); + } catch (IOException e1) { + log.error(e1); + } + + node = null; + TCP.reserve(transportPort, httpPort); + + completableFuture.complete(StartStage.RETRY); + + } catch (Throwable e) { + log.error("Unable to start {}", this, e); + node = null; + completableFuture.completeExceptionally(e); + } + } + }).start(); + + return completableFuture; + } + + public Client getInternalNodeClient() { + return node.client(); + } + + public PluginAwareNode esNode() { + return node; + } + + public boolean isRunning() { + return running; + } + + public X getInjectable(Class clazz) { + return node.injector().getInstance(clazz); + } + + public CompletableFuture stop(long timeout, TimeUnit timeUnit) { + return CompletableFuture.supplyAsync(() -> { + try { + log.info("Stopping {}", this); + + running = false; + + if (node != null) { + node.close(); + boolean stopped = node.awaitClose(timeout, timeUnit); + node = null; + return stopped; + } else { + return false; + } + } catch (Throwable e) { + String message = "Error while stopping " + this; + log.warn(message, e); + throw new RuntimeException(message, e); + } + }); + } + + @Override + public String toString() { + String state = running ? "RUNNING" : node != null ? "INITIALIZING" : "STOPPED"; + + return nodeName + " " + state + " [" + transportPort + ", " + httpPort + "]"; + } + + public boolean isPortCollision() { + return portCollision; + } + + public String getNodeName() { + return nodeName; + } + + @Override + public InetSocketAddress getHttpAddress() { + return httpAddress; + } + + @Override + public InetSocketAddress getTransportAddress() { + return transportAddress; + } + + private Settings getOpenSearchSettings() { + Settings settings = getMinimalOpenSearchSettings(); + + if (nodeSettingsSupplier != null) { + // TODO node number + return Settings.builder().put(settings).put(nodeSettingsSupplier.get(0)).build(); + } + + return settings; + } + + private Settings getMinimalOpenSearchSettings() { + return Settings.builder().put("node.name", nodeName).put("node.data", nodeSettings.dataNode).put("node.master", nodeSettings.clusterManagerNode) + .put("cluster.name", clusterName).put("path.home", nodeHomeDir.toPath()).put("path.data", dataDir.toPath()) + .put("path.logs", logsDir.toPath()).putList("cluster.initial_master_nodes", initialClusterManagerHosts) + .put("discovery.initial_state_timeout", "8s").putList("discovery.seed_hosts", seedHosts).put("transport.tcp.port", transportPort) + .put("http.port", httpPort).put("cluster.routing.allocation.disk.threshold_enabled", false) + .put("discovery.probe.connect_timeout", "10s").put("discovery.probe.handshake_timeout", "10s").put("http.cors.enabled", true) + .build(); + } + + @Override + public String getClusterName() { + return clusterName; + } + + @Override + public TestCertificates getTestCertificates() { + return testCertificates; + } + } + + public Random getRandom() { + return random; + } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java index 56bd00a479..263ec65b1d 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java @@ -1,30 +1,30 @@ /* - * Copyright 2021 floragunn GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ +* Copyright 2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.test.framework.cluster; diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java index 0ed52e203f..caa5cb26f9 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java @@ -1,30 +1,30 @@ /* - * Copyright 2021-2022 floragunn GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ +* Copyright 2021-2022 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.test.framework.cluster; @@ -45,89 +45,89 @@ public class NestedValueMap extends HashMap { - private static final long serialVersionUID = 2953312818482932741L; + private static final long serialVersionUID = 2953312818482932741L; - private Map originalToCloneMap; - private final boolean cloneWhilePut; - private boolean writable = true; + private Map originalToCloneMap; + private final boolean cloneWhilePut; + private boolean writable = true; - public NestedValueMap() { - originalToCloneMap = new MapMaker().weakKeys().makeMap(); - cloneWhilePut = true; - } + public NestedValueMap() { + originalToCloneMap = new MapMaker().weakKeys().makeMap(); + cloneWhilePut = true; + } - public NestedValueMap(int initialCapacity) { - super(initialCapacity); - originalToCloneMap = new MapMaker().weakKeys().makeMap(); - cloneWhilePut = true; - } + public NestedValueMap(int initialCapacity) { + super(initialCapacity); + originalToCloneMap = new MapMaker().weakKeys().makeMap(); + cloneWhilePut = true; + } - NestedValueMap(Map originalToCloneMap, boolean cloneWhilePut) { - this.originalToCloneMap = originalToCloneMap; - this.cloneWhilePut = cloneWhilePut; - } + NestedValueMap(Map originalToCloneMap, boolean cloneWhilePut) { + this.originalToCloneMap = originalToCloneMap; + this.cloneWhilePut = cloneWhilePut; + } - NestedValueMap(int initialCapacity, Map originalToCloneMap, boolean cloneWhilePut) { - super(initialCapacity); - this.originalToCloneMap = originalToCloneMap; - this.cloneWhilePut = cloneWhilePut; - } + NestedValueMap(int initialCapacity, Map originalToCloneMap, boolean cloneWhilePut) { + super(initialCapacity); + this.originalToCloneMap = originalToCloneMap; + this.cloneWhilePut = cloneWhilePut; + } - @Override - public NestedValueMap clone() { - NestedValueMap result = new NestedValueMap(Math.max(this.size(), 10), - this.originalToCloneMap != null ? new MapMaker().weakKeys().makeMap() : null, this.cloneWhilePut); + @Override + public NestedValueMap clone() { + NestedValueMap result = new NestedValueMap(Math.max(this.size(), 10), + this.originalToCloneMap != null ? new MapMaker().weakKeys().makeMap() : null, this.cloneWhilePut); - result.putAll(this); + result.putAll(this); - return result; - } + return result; + } - public NestedValueMap without(String... keys) { - NestedValueMap result = new NestedValueMap(Math.max(this.size(), 10), - this.originalToCloneMap != null ? new MapMaker().weakKeys().makeMap() : null, this.cloneWhilePut); + public NestedValueMap without(String... keys) { + NestedValueMap result = new NestedValueMap(Math.max(this.size(), 10), + this.originalToCloneMap != null ? new MapMaker().weakKeys().makeMap() : null, this.cloneWhilePut); - Set withoutKeySet = new HashSet<>(Arrays.asList(keys)); + Set withoutKeySet = new HashSet<>(Arrays.asList(keys)); - for (Map.Entry entry : this.entrySet()) { - if (!withoutKeySet.contains(entry.getKey())) { - result.put(entry.getKey(), entry.getValue()); - } - } + for (Map.Entry entry : this.entrySet()) { + if (!withoutKeySet.contains(entry.getKey())) { + result.put(entry.getKey(), entry.getValue()); + } + } - return result; - } + return result; + } - public static NestedValueMap copy(Map data) { - NestedValueMap result = new NestedValueMap(data.size()); + public static NestedValueMap copy(Map data) { + NestedValueMap result = new NestedValueMap(data.size()); - result.putAllFromAnyMap(data); + result.putAllFromAnyMap(data); - return result; - } + return result; + } - public static NestedValueMap copy(Object data) { - if (data instanceof Map) { - return copy((Map) data); - } else { - NestedValueMap result = new NestedValueMap(); - result.put("_value", data); - return result; - } - } + public static NestedValueMap copy(Object data) { + if (data instanceof Map) { + return copy((Map) data); + } else { + NestedValueMap result = new NestedValueMap(); + result.put("_value", data); + return result; + } + } - public static NestedValueMap createNonCloningMap() { - return new NestedValueMap(null, false); - } + public static NestedValueMap createNonCloningMap() { + return new NestedValueMap(null, false); + } - public static NestedValueMap createUnmodifieableMap(Map data) { - NestedValueMap result = new NestedValueMap(data.size()); + public static NestedValueMap createUnmodifieableMap(Map data) { + NestedValueMap result = new NestedValueMap(data.size()); - result.putAllFromAnyMap(data); - result.seal(); + result.putAllFromAnyMap(data); + result.seal(); - return result; - } + return result; + } // public static NestedValueMap fromJsonString(String jsonString) throws IOException, DocumentParseException, UnexpectedDocumentStructureException { // return NestedValueMap.copy(DocReader.json().readObject(jsonString)); @@ -141,184 +141,184 @@ public static NestedValueMap createUnmodifieableMap(Map data) { // return NestedValueMap.copy(DocReader.yaml().read(inputSteam)); // } - public static NestedValueMap of(String key1, Object value1) { - NestedValueMap result = new NestedValueMap(1); - result.put(key1, value1); - return result; - } - - public static NestedValueMap of(String key1, Object value1, String key2, Object value2) { - NestedValueMap result = new NestedValueMap(2); - result.put(key1, value1); - result.put(key2, value2); - return result; - } - - public static NestedValueMap of(String key1, Object value1, String key2, Object value2, String key3, Object value3) { - NestedValueMap result = new NestedValueMap(3); - result.put(key1, value1); - result.put(key2, value2); - result.put(key3, value3); - - return result; - } - - public static NestedValueMap of(String key1, Object value1, String key2, Object value2, String key3, Object value3, Object... furtherEntries) { - NestedValueMap result = new NestedValueMap(3 + furtherEntries.length); - result.put(key1, value1); - result.put(key2, value2); - result.put(key3, value3); - - for (int i = 0; i < furtherEntries.length - 1; i += 2) { - result.put(String.valueOf(furtherEntries[i]), furtherEntries[i + 1]); - } - - return result; - } - - public static NestedValueMap of(Path key1, Object value1) { - NestedValueMap result = new NestedValueMap(1); - result.put(key1, value1); - return result; - } - - public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2) { - NestedValueMap result = new NestedValueMap(2); - result.put(key1, value1); - result.put(key2, value2); - return result; - } - - public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2, Path key3, Object value3) { - NestedValueMap result = new NestedValueMap(3); - result.put(key1, value1); - result.put(key2, value2); - result.put(key3, value3); - - return result; - } - - public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2, Path key3, Object value3, Object... furtherEntries) { - NestedValueMap result = new NestedValueMap(3 + furtherEntries.length); - result.put(key1, value1); - result.put(key2, value2); - result.put(key3, value3); - - for (int i = 0; i < furtherEntries.length - 1; i += 2) { - result.put(Path.parse(String.valueOf(furtherEntries[i])), furtherEntries[i + 1]); - } - - return result; - } - - public Object put(String key, Map data) { - checkWritable(); - - Object result = this.get(key); - NestedValueMap subMap = this.getOrCreateSubMapAt(key, data.size()); - - subMap.putAllFromAnyMap(data); - return result; - } - - public void putAll(Map map) { - checkWritable(); - - for (Map.Entry entry : map.entrySet()) { - String key = String.valueOf(entry.getKey()); - put(key, entry.getValue()); - } - } - - public void putAllFromAnyMap(Map map) { - checkWritable(); - - for (Map.Entry entry : map.entrySet()) { - String key = String.valueOf(entry.getKey()); - put(key, entry.getValue()); - } - } - - public void overrideLeafs(NestedValueMap map) { - checkWritable(); - - for (Map.Entry entry : map.entrySet()) { - String key = String.valueOf(entry.getKey()); - - if (entry.getValue() instanceof NestedValueMap) { - NestedValueMap subMap = (NestedValueMap) entry.getValue(); - - getOrCreateSubMapAt(key, subMap.size()).overrideLeafs(subMap); - } else { - put(key, entry.getValue()); - } - } - } - - public Object put(String key, Object object) { - checkWritable(); - - if (object instanceof Map) { - return put(key, (Map) object); - } - - return super.put(key, deepCloneObject(object)); - } - - public void put(Path path, Object object) { - checkWritable(); - - if (path.isEmpty()) { - if (object instanceof Map) { - putAllFromAnyMap((Map) object); - } else { - throw new IllegalArgumentException("put([], " + object + "): If an empty path is given, the object must be of type map"); - } - - } else { - NestedValueMap subMap = getOrCreateSubMapAtPath(path.withoutLast()); - subMap.put(path.getLast(), object); - } - } - - public Object get(Path path) { - if (path.isEmpty()) { - return this; - } else if (path.length() == 1) { - return this.get(path.getFirst()); - } else { - Object subObject = this.get(path.getFirst()); - - if (subObject instanceof NestedValueMap) { - return ((NestedValueMap) subObject).get(path.withoutFirst()); - } else { - return null; - } - } - } - - public void seal() { - if (!this.writable) { - return; - } - - this.writable = false; - this.originalToCloneMap = null; - - for (Object value : this.values()) { - if (value instanceof NestedValueMap) { - NestedValueMap subMap = (NestedValueMap) value; - subMap.seal(); - } else if (value instanceof Iterable) { - for (Object subValue : ((Iterable) value)) { - if (subValue instanceof NestedValueMap) { - NestedValueMap subMap = (NestedValueMap) subValue; - subMap.seal(); - } - } - } - } - } + public static NestedValueMap of(String key1, Object value1) { + NestedValueMap result = new NestedValueMap(1); + result.put(key1, value1); + return result; + } + + public static NestedValueMap of(String key1, Object value1, String key2, Object value2) { + NestedValueMap result = new NestedValueMap(2); + result.put(key1, value1); + result.put(key2, value2); + return result; + } + + public static NestedValueMap of(String key1, Object value1, String key2, Object value2, String key3, Object value3) { + NestedValueMap result = new NestedValueMap(3); + result.put(key1, value1); + result.put(key2, value2); + result.put(key3, value3); + + return result; + } + + public static NestedValueMap of(String key1, Object value1, String key2, Object value2, String key3, Object value3, Object... furtherEntries) { + NestedValueMap result = new NestedValueMap(3 + furtherEntries.length); + result.put(key1, value1); + result.put(key2, value2); + result.put(key3, value3); + + for (int i = 0; i < furtherEntries.length - 1; i += 2) { + result.put(String.valueOf(furtherEntries[i]), furtherEntries[i + 1]); + } + + return result; + } + + public static NestedValueMap of(Path key1, Object value1) { + NestedValueMap result = new NestedValueMap(1); + result.put(key1, value1); + return result; + } + + public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2) { + NestedValueMap result = new NestedValueMap(2); + result.put(key1, value1); + result.put(key2, value2); + return result; + } + + public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2, Path key3, Object value3) { + NestedValueMap result = new NestedValueMap(3); + result.put(key1, value1); + result.put(key2, value2); + result.put(key3, value3); + + return result; + } + + public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2, Path key3, Object value3, Object... furtherEntries) { + NestedValueMap result = new NestedValueMap(3 + furtherEntries.length); + result.put(key1, value1); + result.put(key2, value2); + result.put(key3, value3); + + for (int i = 0; i < furtherEntries.length - 1; i += 2) { + result.put(Path.parse(String.valueOf(furtherEntries[i])), furtherEntries[i + 1]); + } + + return result; + } + + public Object put(String key, Map data) { + checkWritable(); + + Object result = this.get(key); + NestedValueMap subMap = this.getOrCreateSubMapAt(key, data.size()); + + subMap.putAllFromAnyMap(data); + return result; + } + + public void putAll(Map map) { + checkWritable(); + + for (Map.Entry entry : map.entrySet()) { + String key = String.valueOf(entry.getKey()); + put(key, entry.getValue()); + } + } + + public void putAllFromAnyMap(Map map) { + checkWritable(); + + for (Map.Entry entry : map.entrySet()) { + String key = String.valueOf(entry.getKey()); + put(key, entry.getValue()); + } + } + + public void overrideLeafs(NestedValueMap map) { + checkWritable(); + + for (Map.Entry entry : map.entrySet()) { + String key = String.valueOf(entry.getKey()); + + if (entry.getValue() instanceof NestedValueMap) { + NestedValueMap subMap = (NestedValueMap) entry.getValue(); + + getOrCreateSubMapAt(key, subMap.size()).overrideLeafs(subMap); + } else { + put(key, entry.getValue()); + } + } + } + + public Object put(String key, Object object) { + checkWritable(); + + if (object instanceof Map) { + return put(key, (Map) object); + } + + return super.put(key, deepCloneObject(object)); + } + + public void put(Path path, Object object) { + checkWritable(); + + if (path.isEmpty()) { + if (object instanceof Map) { + putAllFromAnyMap((Map) object); + } else { + throw new IllegalArgumentException("put([], " + object + "): If an empty path is given, the object must be of type map"); + } + + } else { + NestedValueMap subMap = getOrCreateSubMapAtPath(path.withoutLast()); + subMap.put(path.getLast(), object); + } + } + + public Object get(Path path) { + if (path.isEmpty()) { + return this; + } else if (path.length() == 1) { + return this.get(path.getFirst()); + } else { + Object subObject = this.get(path.getFirst()); + + if (subObject instanceof NestedValueMap) { + return ((NestedValueMap) subObject).get(path.withoutFirst()); + } else { + return null; + } + } + } + + public void seal() { + if (!this.writable) { + return; + } + + this.writable = false; + this.originalToCloneMap = null; + + for (Object value : this.values()) { + if (value instanceof NestedValueMap) { + NestedValueMap subMap = (NestedValueMap) value; + subMap.seal(); + } else if (value instanceof Iterable) { + for (Object subValue : ((Iterable) value)) { + if (subValue instanceof NestedValueMap) { + NestedValueMap subMap = (NestedValueMap) subValue; + subMap.seal(); + } + } + } + } + } // public String toJsonString() { // return DocWriter.json().writeAsString(this); @@ -328,191 +328,191 @@ public void seal() { // return DocWriter.yaml().writeAsString(this); // } - private Object deepCloneObject(Object object) { - if (!cloneWhilePut || object == null || isImmutable(object)) { - return object; - } - - Object clone = this.originalToCloneMap.get(object); - - if (clone != null) { - return clone; - } - - if (object instanceof Set) { - Set set = (Set) object; - Set copy = new HashSet<>(set.size()); - this.originalToCloneMap.put(object, copy); - - for (Object element : set) { - copy.add(deepCloneObject(element)); - } - - return copy; - } else if (object instanceof Map) { - Map map = (Map) object; - NestedValueMap copy = new NestedValueMap(map.size(), this.originalToCloneMap, this.cloneWhilePut); - this.originalToCloneMap.put(object, copy); - - for (Map.Entry entry : map.entrySet()) { - copy.put((String) deepCloneObject(String.valueOf(entry.getKey())), deepCloneObject(entry.getValue())); - } - - return copy; - } else if (object instanceof Collection) { - Collection collection = (Collection) object; - ArrayList copy = new ArrayList<>(collection.size()); - this.originalToCloneMap.put(object, copy); - - for (Object element : collection) { - copy.add(deepCloneObject(element)); - } - - return copy; - } else if (object.getClass().isArray()) { - int length = Array.getLength(object); - Object copy = Array.newInstance(object.getClass().getComponentType(), length); - this.originalToCloneMap.put(object, copy); - - for (int i = 0; i < length; i++) { - Array.set(copy, i, deepCloneObject(Array.get(object, i))); - } - - return copy; - } else { - // Hope the best - - return object; - } - } - - private boolean isImmutable(Object object) { - return object instanceof String || object instanceof Number || object instanceof Boolean || object instanceof Void || object instanceof Class - || object instanceof Character || object instanceof Enum || object instanceof File || object instanceof UUID || object instanceof URL - || object instanceof URI; - } - - private NestedValueMap getOrCreateSubMapAt(String key, int capacity) { - Object value = this.get(key); - - if (value instanceof NestedValueMap) { - return (NestedValueMap) value; - } else { - if (value instanceof Map) { - capacity = Math.max(capacity, ((Map) value).size()); - } - - NestedValueMap mapValue = new NestedValueMap(capacity, this.originalToCloneMap, this.cloneWhilePut); - - if (value instanceof Map) { - mapValue.putAllFromAnyMap((Map) value); - } - - super.put(key, mapValue); - return mapValue; - } - - } - - private NestedValueMap getOrCreateSubMapAtPath(Path path) { - if (path.isEmpty()) { - return this; - } - - String pathElement = path.getFirst(); - Path remainingPath = path.withoutFirst(); - - Object value = this.get(pathElement); - - if (value instanceof NestedValueMap) { - NestedValueMap mapValue = (NestedValueMap) value; - if (remainingPath.isEmpty()) { - return mapValue; - } else { - return mapValue.getOrCreateSubMapAtPath(remainingPath); - } - } else { - NestedValueMap mapValue = new NestedValueMap(this.originalToCloneMap, this.cloneWhilePut); - super.put(pathElement, mapValue); - - if (remainingPath.isEmpty()) { - return mapValue; - } else { - return mapValue.getOrCreateSubMapAtPath(remainingPath); - } - } - } - - private void checkWritable() { - if (!writable) { - throw new UnsupportedOperationException("Map is not writable"); - } - } - - public static class Path { - private String[] elements; - private int start; - private int end; - - public Path(String... elements) { - this.elements = elements; - this.start = 0; - this.end = elements.length; - } - - private Path(String[] elements, int start, int end) { - this.elements = elements; - this.start = start; - this.end = end; - } - - public String getFirst() { - if (this.start >= this.end) { - return null; - } - - return this.elements[start]; - } - - public String getLast() { - if (this.start >= this.end) { - return null; - } - - return this.elements[end - 1]; - } - - public Path withoutFirst() { - if (this.start >= this.end - 1) { - return new Path(null, 0, 0); - } - - return new Path(elements, start + 1, end); - } - - public Path withoutLast() { - if (this.start >= this.end - 1) { - return new Path(null, 0, 0); - } - - return new Path(elements, start, end - 1); - } - - public int length() { - return this.end - this.start; - } - - public boolean isEmpty() { - return this.start == this.end; - } - - public static Path parse(String path) { - if (path.length() == 0) { - return new Path(new String [0]); - } else { - return new Path(path.split("\\.")); - } - } - } + private Object deepCloneObject(Object object) { + if (!cloneWhilePut || object == null || isImmutable(object)) { + return object; + } + + Object clone = this.originalToCloneMap.get(object); + + if (clone != null) { + return clone; + } + + if (object instanceof Set) { + Set set = (Set) object; + Set copy = new HashSet<>(set.size()); + this.originalToCloneMap.put(object, copy); + + for (Object element : set) { + copy.add(deepCloneObject(element)); + } + + return copy; + } else if (object instanceof Map) { + Map map = (Map) object; + NestedValueMap copy = new NestedValueMap(map.size(), this.originalToCloneMap, this.cloneWhilePut); + this.originalToCloneMap.put(object, copy); + + for (Map.Entry entry : map.entrySet()) { + copy.put((String) deepCloneObject(String.valueOf(entry.getKey())), deepCloneObject(entry.getValue())); + } + + return copy; + } else if (object instanceof Collection) { + Collection collection = (Collection) object; + ArrayList copy = new ArrayList<>(collection.size()); + this.originalToCloneMap.put(object, copy); + + for (Object element : collection) { + copy.add(deepCloneObject(element)); + } + + return copy; + } else if (object.getClass().isArray()) { + int length = Array.getLength(object); + Object copy = Array.newInstance(object.getClass().getComponentType(), length); + this.originalToCloneMap.put(object, copy); + + for (int i = 0; i < length; i++) { + Array.set(copy, i, deepCloneObject(Array.get(object, i))); + } + + return copy; + } else { + // Hope the best + + return object; + } + } + + private boolean isImmutable(Object object) { + return object instanceof String || object instanceof Number || object instanceof Boolean || object instanceof Void || object instanceof Class + || object instanceof Character || object instanceof Enum || object instanceof File || object instanceof UUID || object instanceof URL + || object instanceof URI; + } + + private NestedValueMap getOrCreateSubMapAt(String key, int capacity) { + Object value = this.get(key); + + if (value instanceof NestedValueMap) { + return (NestedValueMap) value; + } else { + if (value instanceof Map) { + capacity = Math.max(capacity, ((Map) value).size()); + } + + NestedValueMap mapValue = new NestedValueMap(capacity, this.originalToCloneMap, this.cloneWhilePut); + + if (value instanceof Map) { + mapValue.putAllFromAnyMap((Map) value); + } + + super.put(key, mapValue); + return mapValue; + } + + } + + private NestedValueMap getOrCreateSubMapAtPath(Path path) { + if (path.isEmpty()) { + return this; + } + + String pathElement = path.getFirst(); + Path remainingPath = path.withoutFirst(); + + Object value = this.get(pathElement); + + if (value instanceof NestedValueMap) { + NestedValueMap mapValue = (NestedValueMap) value; + if (remainingPath.isEmpty()) { + return mapValue; + } else { + return mapValue.getOrCreateSubMapAtPath(remainingPath); + } + } else { + NestedValueMap mapValue = new NestedValueMap(this.originalToCloneMap, this.cloneWhilePut); + super.put(pathElement, mapValue); + + if (remainingPath.isEmpty()) { + return mapValue; + } else { + return mapValue.getOrCreateSubMapAtPath(remainingPath); + } + } + } + + private void checkWritable() { + if (!writable) { + throw new UnsupportedOperationException("Map is not writable"); + } + } + + public static class Path { + private String[] elements; + private int start; + private int end; + + public Path(String... elements) { + this.elements = elements; + this.start = 0; + this.end = elements.length; + } + + private Path(String[] elements, int start, int end) { + this.elements = elements; + this.start = start; + this.end = end; + } + + public String getFirst() { + if (this.start >= this.end) { + return null; + } + + return this.elements[start]; + } + + public String getLast() { + if (this.start >= this.end) { + return null; + } + + return this.elements[end - 1]; + } + + public Path withoutFirst() { + if (this.start >= this.end - 1) { + return new Path(null, 0, 0); + } + + return new Path(elements, start + 1, end); + } + + public Path withoutLast() { + if (this.start >= this.end - 1) { + return new Path(null, 0, 0); + } + + return new Path(elements, start, end - 1); + } + + public int length() { + return this.end - this.start; + } + + public boolean isEmpty() { + return this.start == this.end; + } + + public static Path parse(String path) { + if (path.length() == 0) { + return new Path(new String [0]); + } else { + return new Path(path.split("\\.")); + } + } + } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java index 828eb0f05a..75c728287b 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeSettingsSupplier.java @@ -1,28 +1,28 @@ /* - * Copyright 2015-2018 _floragunn_ GmbH - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Copyright 2015-2018 _floragunn_ GmbH +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.test.framework.cluster; @@ -30,5 +30,5 @@ @FunctionalInterface public interface NodeSettingsSupplier { - Settings get(int i); + Settings get(int i); } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java index ad00b143c6..915f99daa8 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeType.java @@ -1,15 +1,15 @@ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.test.framework.cluster; enum NodeType { - CLIENT, DATA, CLUSTER_MANAGER + CLIENT, DATA, CLUSTER_MANAGER } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java index 02817a1dfa..6142e43db8 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java @@ -1,30 +1,30 @@ /* - * Copyright 2020 floragunn GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ +* Copyright 2020 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.test.framework.cluster; @@ -52,82 +52,82 @@ public interface OpenSearchClientProvider { - String getClusterName(); + String getClusterName(); - TestCertificates getTestCertificates(); + TestCertificates getTestCertificates(); - InetSocketAddress getHttpAddress(); + InetSocketAddress getHttpAddress(); - InetSocketAddress getTransportAddress(); + InetSocketAddress getTransportAddress(); - default URI getHttpAddressAsURI() { - InetSocketAddress address = getHttpAddress(); - return URI.create("https://" + address.getHostString() + ":" + address.getPort()); - } + default URI getHttpAddressAsURI() { + InetSocketAddress address = getHttpAddress(); + return URI.create("https://" + address.getHostString() + ":" + address.getPort()); + } - default TestRestClient getRestClient(UserCredentialsHolder user, Header... headers) { - return getRestClient(user.getName(), user.getPassword(), headers); - } + default TestRestClient getRestClient(UserCredentialsHolder user, Header... headers) { + return getRestClient(user.getName(), user.getPassword(), headers); + } - default TestRestClient getRestClient(String user, String password, String tenant) { - return getRestClient(user, password, new BasicHeader("sgtenant", tenant)); - } + default TestRestClient getRestClient(String user, String password, String tenant) { + return getRestClient(user, password, new BasicHeader("sgtenant", tenant)); + } - default TestRestClient getRestClient(String user, String password, Header... headers) { - BasicHeader basicAuthHeader = getBasicAuthHeader(user, password); - if (headers != null && headers.length > 0) { - List
concatenatedHeaders = Stream.concat(Stream.of(basicAuthHeader), Stream.of(headers)).collect(Collectors.toList()); - return getRestClient(concatenatedHeaders); - } - return getRestClient(basicAuthHeader); - } + default TestRestClient getRestClient(String user, String password, Header... headers) { + BasicHeader basicAuthHeader = getBasicAuthHeader(user, password); + if (headers != null && headers.length > 0) { + List
concatenatedHeaders = Stream.concat(Stream.of(basicAuthHeader), Stream.of(headers)).collect(Collectors.toList()); + return getRestClient(concatenatedHeaders); + } + return getRestClient(basicAuthHeader); + } - default TestRestClient getRestClient(Header... headers) { - return getRestClient(Arrays.asList(headers)); - } + default TestRestClient getRestClient(Header... headers) { + return getRestClient(Arrays.asList(headers)); + } - default TestRestClient getRestClient(List
headers) { - return createGenericClientRestClient(headers); - } + default TestRestClient getRestClient(List
headers) { + return createGenericClientRestClient(headers); + } - default TestRestClient createGenericClientRestClient(List
headers) { - return new TestRestClient(getHttpAddress(), headers, getSSLContext()); - } + default TestRestClient createGenericClientRestClient(List
headers) { + return new TestRestClient(getHttpAddress(), headers, getSSLContext()); + } - default BasicHeader getBasicAuthHeader(String user, String password) { - return new BasicHeader("Authorization", - "Basic " + Base64.getEncoder().encodeToString((user + ":" + Objects.requireNonNull(password)).getBytes(StandardCharsets.UTF_8))); - } + default BasicHeader getBasicAuthHeader(String user, String password) { + return new BasicHeader("Authorization", + "Basic " + Base64.getEncoder().encodeToString((user + ":" + Objects.requireNonNull(password)).getBytes(StandardCharsets.UTF_8))); + } - private SSLContext getSSLContext() { - X509Certificate[] trustCertificates; - - try { - trustCertificates = PemKeyReader.loadCertificatesFromFile(getTestCertificates().getRootCertificate() ); + private SSLContext getSSLContext() { + X509Certificate[] trustCertificates; + + try { + trustCertificates = PemKeyReader.loadCertificatesFromFile(getTestCertificates().getRootCertificate() ); - TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); + TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); - ks.load(null); - - for (int i = 0; i < trustCertificates.length; i++) { - ks.setCertificateEntry("caCert-" + i, trustCertificates[i]); + ks.load(null); + + for (int i = 0; i < trustCertificates.length; i++) { + ks.setCertificateEntry("caCert-" + i, trustCertificates[i]); } - tmf.init(ks); - - SSLContext sslContext = SSLContext.getInstance("TLS"); - sslContext.init(null, tmf.getTrustManagers(), null); - return sslContext; - - } catch (Exception e) { - throw new RuntimeException("Error loading root CA ", e); - } - } - - public interface UserCredentialsHolder { - String getName(); - String getPassword(); - } + tmf.init(ks); + + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, tmf.getTrustManagers(), null); + return sslContext; + + } catch (Exception e) { + throw new RuntimeException("Error loading root CA ", e); + } + } + + public interface UserCredentialsHolder { + String getName(); + String getPassword(); + } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java index 04aba5211d..83ebddd4b4 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java @@ -1,30 +1,30 @@ /* - * Copyright 2021 floragunn GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ +* Copyright 2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.test.framework.cluster; @@ -40,118 +40,118 @@ public class PortAllocator { - public static final PortAllocator TCP = new PortAllocator(SocketType.TCP, Duration.ofSeconds(100)); - public static final PortAllocator UDP = new PortAllocator(SocketType.UDP, Duration.ofSeconds(100)); + public static final PortAllocator TCP = new PortAllocator(SocketType.TCP, Duration.ofSeconds(100)); + public static final PortAllocator UDP = new PortAllocator(SocketType.UDP, Duration.ofSeconds(100)); - private final SocketType socketType; - private final Duration timeoutDuration; - private final Map allocatedPorts = new HashMap<>(); + private final SocketType socketType; + private final Duration timeoutDuration; + private final Map allocatedPorts = new HashMap<>(); - PortAllocator(SocketType socketType, Duration timeoutDuration) { - this.socketType = socketType; - this.timeoutDuration = timeoutDuration; - } + PortAllocator(SocketType socketType, Duration timeoutDuration) { + this.socketType = socketType; + this.timeoutDuration = timeoutDuration; + } - public SortedSet allocate(String clientName, int numRequested, int minPort) { + public SortedSet allocate(String clientName, int numRequested, int minPort) { - int startPort = minPort; + int startPort = minPort; - while (!isAvailable(startPort)) { - startPort += 10; - } + while (!isAvailable(startPort)) { + startPort += 10; + } - SortedSet foundPorts = new TreeSet<>(); + SortedSet foundPorts = new TreeSet<>(); - for (int currentPort = startPort; foundPorts.size() < numRequested && currentPort < SocketUtils.PORT_RANGE_MAX - && (currentPort - startPort) < 10000; currentPort++) { - if (allocate(clientName, currentPort)) { - foundPorts.add(currentPort); - } - } + for (int currentPort = startPort; foundPorts.size() < numRequested && currentPort < SocketUtils.PORT_RANGE_MAX + && (currentPort - startPort) < 10000; currentPort++) { + if (allocate(clientName, currentPort)) { + foundPorts.add(currentPort); + } + } - if (foundPorts.size() < numRequested) { - throw new IllegalStateException("Could not find " + numRequested + " free ports starting at " + minPort + " for " + clientName); - } + if (foundPorts.size() < numRequested) { + throw new IllegalStateException("Could not find " + numRequested + " free ports starting at " + minPort + " for " + clientName); + } - return foundPorts; - } + return foundPorts; + } - public int allocateSingle(String clientName, int minPort) { + public int allocateSingle(String clientName, int minPort) { - int startPort = minPort; + int startPort = minPort; - for (int currentPort = startPort; currentPort < SocketUtils.PORT_RANGE_MAX && (currentPort - startPort) < 10000; currentPort++) { - if (allocate(clientName, currentPort)) { - return currentPort; - } - } + for (int currentPort = startPort; currentPort < SocketUtils.PORT_RANGE_MAX && (currentPort - startPort) < 10000; currentPort++) { + if (allocate(clientName, currentPort)) { + return currentPort; + } + } - throw new IllegalStateException("Could not find free port starting at " + minPort + " for " + clientName); + throw new IllegalStateException("Could not find free port starting at " + minPort + " for " + clientName); - } + } - public void reserve(int... ports) { + public void reserve(int... ports) { - for (int port : ports) { - allocate("reserved", port); - } - } + for (int port : ports) { + allocate("reserved", port); + } + } - private boolean isInUse(int port) { - boolean result = !this.socketType.isPortAvailable(port); + private boolean isInUse(int port) { + boolean result = !this.socketType.isPortAvailable(port); - if (result) { - synchronized (this) { - allocatedPorts.put(port, new AllocatedPort("external")); - } - } + if (result) { + synchronized (this) { + allocatedPorts.put(port, new AllocatedPort("external")); + } + } - return result; - } + return result; + } - private boolean isAvailable(int port) { - return !isAllocated(port) && !isInUse(port); - } + private boolean isAvailable(int port) { + return !isAllocated(port) && !isInUse(port); + } - private synchronized boolean isAllocated(int port) { - AllocatedPort allocatedPort = this.allocatedPorts.get(port); + private synchronized boolean isAllocated(int port) { + AllocatedPort allocatedPort = this.allocatedPorts.get(port); - return allocatedPort != null && !allocatedPort.isTimedOut(); - } + return allocatedPort != null && !allocatedPort.isTimedOut(); + } - private synchronized boolean allocate(String clientName, int port) { + private synchronized boolean allocate(String clientName, int port) { - AllocatedPort allocatedPort = allocatedPorts.get(port); + AllocatedPort allocatedPort = allocatedPorts.get(port); - if (allocatedPort != null && allocatedPort.isTimedOut()) { - allocatedPort = null; - allocatedPorts.remove(port); - } + if (allocatedPort != null && allocatedPort.isTimedOut()) { + allocatedPort = null; + allocatedPorts.remove(port); + } - if (allocatedPort == null && !isInUse(port)) { - allocatedPorts.put(port, new AllocatedPort(clientName)); - return true; - } else { - return false; - } - } + if (allocatedPort == null && !isInUse(port)) { + allocatedPorts.put(port, new AllocatedPort(clientName)); + return true; + } else { + return false; + } + } - private class AllocatedPort { - final String client; - final Instant allocatedAt; + private class AllocatedPort { + final String client; + final Instant allocatedAt; - AllocatedPort(String client) { - this.client = client; - this.allocatedAt = Instant.now(); - } + AllocatedPort(String client) { + this.client = client; + this.allocatedAt = Instant.now(); + } - boolean isTimedOut() { - return allocatedAt.plus(timeoutDuration).isBefore(Instant.now()); - } + boolean isTimedOut() { + return allocatedAt.plus(timeoutDuration).isBefore(Instant.now()); + } - @Override - public String toString() { - return "AllocatedPort [client=" + client + ", allocatedAt=" + allocatedAt + "]"; - } - } + @Override + public String toString() { + return "AllocatedPort [client=" + client + ", allocatedAt=" + allocatedAt + "]"; + } + } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/RestClientException.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/RestClientException.java index 55cd0c7f4c..527fe1cb2f 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/RestClientException.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/RestClientException.java @@ -1,16 +1,16 @@ /* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ package org.opensearch.test.framework.cluster; class RestClientException extends RuntimeException { - public RestClientException(String message, Throwable cause) { - super(message, cause); - } + public RestClientException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtils.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtils.java index 35ef48f022..273e71026e 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtils.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtils.java @@ -1,29 +1,29 @@ /* - * Copyright 2002-2017 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Copyright 2002-2017 the original author or authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.test.framework.cluster; @@ -37,283 +37,283 @@ import javax.net.ServerSocketFactory; /** - * Simple utility methods for working with network sockets — for example, - * for finding available ports on {@code localhost}. - * - *

Within this class, a TCP port refers to a port for a {@link ServerSocket}; - * whereas, a UDP port refers to a port for a {@link DatagramSocket}. - * - * @author Sam Brannen - * @author Ben Hale - * @author Arjen Poutsma - * @author Gunnar Hillert - * @author Gary Russell - * @since 4.0 - */ +* Simple utility methods for working with network sockets — for example, +* for finding available ports on {@code localhost}. +* +*

Within this class, a TCP port refers to a port for a {@link ServerSocket}; +* whereas, a UDP port refers to a port for a {@link DatagramSocket}. +* +* @author Sam Brannen +* @author Ben Hale +* @author Arjen Poutsma +* @author Gunnar Hillert +* @author Gary Russell +* @since 4.0 +*/ public class SocketUtils { - /** - * The default minimum value for port ranges used when finding an available - * socket port. - */ - public static final int PORT_RANGE_MIN = 1024; - - /** - * The default maximum value for port ranges used when finding an available - * socket port. - */ - public static final int PORT_RANGE_MAX = 65535; - - - private static final Random random = new Random(System.currentTimeMillis()); - - - /** - * Although {@code SocketUtils} consists solely of static utility methods, - * this constructor is intentionally {@code public}. - *

Rationale

- *

Static methods from this class may be invoked from within XML - * configuration files using the Spring Expression Language (SpEL) and the - * following syntax. - * 

<bean id="bean1" ... p:port="#{T(org.springframework.util.SocketUtils).findAvailableTcpPort(12000)}" />
- * If this constructor were {@code private}, you would be required to supply - * the fully qualified class name to SpEL's {@code T()} function for each usage. - * Thus, the fact that this constructor is {@code public} allows you to reduce - * boilerplate configuration with SpEL as can be seen in the following example. - * 
<bean id="socketUtils" class="org.springframework.util.SocketUtils" />
-     * <bean id="bean1" ... p:port="#{socketUtils.findAvailableTcpPort(12000)}" />
-     * <bean id="bean2" ... p:port="#{socketUtils.findAvailableTcpPort(30000)}" />
- */ - public SocketUtils() { - /* no-op */ - } - - - /** - * Find an available TCP port randomly selected from the range - * [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. - * @return an available TCP port number - * @throws IllegalStateException if no available port could be found - */ - public static int findAvailableTcpPort() { - return findAvailableTcpPort(PORT_RANGE_MIN); - } - - /** - * Find an available TCP port randomly selected from the range - * [{@code minPort}, {@value #PORT_RANGE_MAX}]. - * @param minPort the minimum port number - * @return an available TCP port number - * @throws IllegalStateException if no available port could be found - */ - public static int findAvailableTcpPort(int minPort) { - return findAvailableTcpPort(minPort, PORT_RANGE_MAX); - } - - /** - * Find an available TCP port randomly selected from the range - * [{@code minPort}, {@code maxPort}]. - * @param minPort the minimum port number - * @param maxPort the maximum port number - * @return an available TCP port number - * @throws IllegalStateException if no available port could be found - */ - public static int findAvailableTcpPort(int minPort, int maxPort) { - return SocketType.TCP.findAvailablePort(minPort, maxPort); - } - - /** - * Find the requested number of available TCP ports, each randomly selected - * from the range [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. - * @param numRequested the number of available ports to find - * @return a sorted set of available TCP port numbers - * @throws IllegalStateException if the requested number of available ports could not be found - */ - public static SortedSet findAvailableTcpPorts(int numRequested) { - return findAvailableTcpPorts(numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); - } - - /** - * Find the requested number of available TCP ports, each randomly selected - * from the range [{@code minPort}, {@code maxPort}]. - * @param numRequested the number of available ports to find - * @param minPort the minimum port number - * @param maxPort the maximum port number - * @return a sorted set of available TCP port numbers - * @throws IllegalStateException if the requested number of available ports could not be found - */ - public static SortedSet findAvailableTcpPorts(int numRequested, int minPort, int maxPort) { - return SocketType.TCP.findAvailablePorts(numRequested, minPort, maxPort); - } - - /** - * Find an available UDP port randomly selected from the range - * [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. - * @return an available UDP port number - * @throws IllegalStateException if no available port could be found - */ - public static int findAvailableUdpPort() { - return findAvailableUdpPort(PORT_RANGE_MIN); - } - - /** - * Find an available UDP port randomly selected from the range - * [{@code minPort}, {@value #PORT_RANGE_MAX}]. - * @param minPort the minimum port number - * @return an available UDP port number - * @throws IllegalStateException if no available port could be found - */ - public static int findAvailableUdpPort(int minPort) { - return findAvailableUdpPort(minPort, PORT_RANGE_MAX); - } - - /** - * Find an available UDP port randomly selected from the range - * [{@code minPort}, {@code maxPort}]. - * @param minPort the minimum port number - * @param maxPort the maximum port number - * @return an available UDP port number - * @throws IllegalStateException if no available port could be found - */ - public static int findAvailableUdpPort(int minPort, int maxPort) { - return SocketType.UDP.findAvailablePort(minPort, maxPort); - } - - /** - * Find the requested number of available UDP ports, each randomly selected - * from the range [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. - * @param numRequested the number of available ports to find - * @return a sorted set of available UDP port numbers - * @throws IllegalStateException if the requested number of available ports could not be found - */ - public static SortedSet findAvailableUdpPorts(int numRequested) { - return findAvailableUdpPorts(numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); - } - - /** - * Find the requested number of available UDP ports, each randomly selected - * from the range [{@code minPort}, {@code maxPort}]. - * @param numRequested the number of available ports to find - * @param minPort the minimum port number - * @param maxPort the maximum port number - * @return a sorted set of available UDP port numbers - * @throws IllegalStateException if the requested number of available ports could not be found - */ - public static SortedSet findAvailableUdpPorts(int numRequested, int minPort, int maxPort) { - return SocketType.UDP.findAvailablePorts(numRequested, minPort, maxPort); - } - - - public enum SocketType { - - TCP { - @Override - protected boolean isPortAvailable(int port) { - try { - ServerSocket serverSocket = ServerSocketFactory.getDefault().createServerSocket( - port, 1, InetAddress.getByName("localhost")); - serverSocket.close(); - return true; - } - catch (Exception ex) { - return false; - } - } - }, - - UDP { - @Override - protected boolean isPortAvailable(int port) { - try { - DatagramSocket socket = new DatagramSocket(port, InetAddress.getByName("localhost")); - socket.close(); - return true; - } - catch (Exception ex) { - return false; - } - } - }; - - /** - * Determine if the specified port for this {@code SocketType} is - * currently available on {@code localhost}. - */ - protected abstract boolean isPortAvailable(int port); - - /** - * Find a pseudo-random port number within the range - * [{@code minPort}, {@code maxPort}]. - * @param minPort the minimum port number - * @param maxPort the maximum port number - * @return a random port number within the specified range - */ - private int findRandomPort(int minPort, int maxPort) { - int portRange = maxPort - minPort; - return minPort + random.nextInt(portRange + 1); - } - - /** - * Find an available port for this {@code SocketType}, randomly selected - * from the range [{@code minPort}, {@code maxPort}]. - * @param minPort the minimum port number - * @param maxPort the maximum port number - * @return an available port number for this socket type - * @throws IllegalStateException if no available port could be found - */ - int findAvailablePort(int minPort, int maxPort) { - //Assert.assertTrue(minPort > 0, "'minPort' must be greater than 0"); - //Assert.isTrue(maxPort >= minPort, "'maxPort' must be greater than or equal to 'minPort'"); - //Assert.isTrue(maxPort <= PORT_RANGE_MAX, "'maxPort' must be less than or equal to " + PORT_RANGE_MAX); - - int portRange = maxPort - minPort; - int candidatePort; - int searchCounter = 0; - do { - if (searchCounter > portRange) { - throw new IllegalStateException(String.format( - "Could not find an available %s port in the range [%d, %d] after %d attempts", - name(), minPort, maxPort, searchCounter)); - } - candidatePort = findRandomPort(minPort, maxPort); - searchCounter++; - } - while (!isPortAvailable(candidatePort)); - - return candidatePort; - } - - /** - * Find the requested number of available ports for this {@code SocketType}, - * each randomly selected from the range [{@code minPort}, {@code maxPort}]. - * @param numRequested the number of available ports to find - * @param minPort the minimum port number - * @param maxPort the maximum port number - * @return a sorted set of available port numbers for this socket type - * @throws IllegalStateException if the requested number of available ports could not be found - */ - SortedSet findAvailablePorts(int numRequested, int minPort, int maxPort) { - //Assert.isTrue(minPort > 0, "'minPort' must be greater than 0"); - //Assert.isTrue(maxPort > minPort, "'maxPort' must be greater than 'minPort'"); - //Assert.isTrue(maxPort <= PORT_RANGE_MAX, "'maxPort' must be less than or equal to " + PORT_RANGE_MAX); - //Assert.isTrue(numRequested > 0, "'numRequested' must be greater than 0"); - //Assert.isTrue((maxPort - minPort) >= numRequested, - // "'numRequested' must not be greater than 'maxPort' - 'minPort'"); - - SortedSet availablePorts = new TreeSet<>(); - int attemptCount = 0; - while ((++attemptCount <= numRequested + 100) && availablePorts.size() < numRequested) { - availablePorts.add(findAvailablePort(minPort, maxPort)); - } - - if (availablePorts.size() != numRequested) { - throw new IllegalStateException(String.format( - "Could not find %d available %s ports in the range [%d, %d]", - numRequested, name(), minPort, maxPort)); - } - - return availablePorts; - } - } + /** + * The default minimum value for port ranges used when finding an available + * socket port. + */ + public static final int PORT_RANGE_MIN = 1024; + + /** + * The default maximum value for port ranges used when finding an available + * socket port. + */ + public static final int PORT_RANGE_MAX = 65535; + + + private static final Random random = new Random(System.currentTimeMillis()); + + + /** + * Although {@code SocketUtils} consists solely of static utility methods, + * this constructor is intentionally {@code public}. + *

Rationale

+ *

Static methods from this class may be invoked from within XML + * configuration files using the Spring Expression Language (SpEL) and the + * following syntax. + * 

<bean id="bean1" ... p:port="#{T(org.springframework.util.SocketUtils).findAvailableTcpPort(12000)}" />
+ * If this constructor were {@code private}, you would be required to supply + * the fully qualified class name to SpEL's {@code T()} function for each usage. + * Thus, the fact that this constructor is {@code public} allows you to reduce + * boilerplate configuration with SpEL as can be seen in the following example. + * 
<bean id="socketUtils" class="org.springframework.util.SocketUtils" />
+	* <bean id="bean1" ... p:port="#{socketUtils.findAvailableTcpPort(12000)}" />
+	* <bean id="bean2" ... p:port="#{socketUtils.findAvailableTcpPort(30000)}" />
+ */ + public SocketUtils() { + /* no-op */ + } + + + /** + * Find an available TCP port randomly selected from the range + * [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. + * @return an available TCP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableTcpPort() { + return findAvailableTcpPort(PORT_RANGE_MIN); + } + + /** + * Find an available TCP port randomly selected from the range + * [{@code minPort}, {@value #PORT_RANGE_MAX}]. + * @param minPort the minimum port number + * @return an available TCP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableTcpPort(int minPort) { + return findAvailableTcpPort(minPort, PORT_RANGE_MAX); + } + + /** + * Find an available TCP port randomly selected from the range + * [{@code minPort}, {@code maxPort}]. + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return an available TCP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableTcpPort(int minPort, int maxPort) { + return SocketType.TCP.findAvailablePort(minPort, maxPort); + } + + /** + * Find the requested number of available TCP ports, each randomly selected + * from the range [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. + * @param numRequested the number of available ports to find + * @return a sorted set of available TCP port numbers + * @throws IllegalStateException if the requested number of available ports could not be found + */ + public static SortedSet findAvailableTcpPorts(int numRequested) { + return findAvailableTcpPorts(numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + /** + * Find the requested number of available TCP ports, each randomly selected + * from the range [{@code minPort}, {@code maxPort}]. + * @param numRequested the number of available ports to find + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return a sorted set of available TCP port numbers + * @throws IllegalStateException if the requested number of available ports could not be found + */ + public static SortedSet findAvailableTcpPorts(int numRequested, int minPort, int maxPort) { + return SocketType.TCP.findAvailablePorts(numRequested, minPort, maxPort); + } + + /** + * Find an available UDP port randomly selected from the range + * [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. + * @return an available UDP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableUdpPort() { + return findAvailableUdpPort(PORT_RANGE_MIN); + } + + /** + * Find an available UDP port randomly selected from the range + * [{@code minPort}, {@value #PORT_RANGE_MAX}]. + * @param minPort the minimum port number + * @return an available UDP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableUdpPort(int minPort) { + return findAvailableUdpPort(minPort, PORT_RANGE_MAX); + } + + /** + * Find an available UDP port randomly selected from the range + * [{@code minPort}, {@code maxPort}]. + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return an available UDP port number + * @throws IllegalStateException if no available port could be found + */ + public static int findAvailableUdpPort(int minPort, int maxPort) { + return SocketType.UDP.findAvailablePort(minPort, maxPort); + } + + /** + * Find the requested number of available UDP ports, each randomly selected + * from the range [{@value #PORT_RANGE_MIN}, {@value #PORT_RANGE_MAX}]. + * @param numRequested the number of available ports to find + * @return a sorted set of available UDP port numbers + * @throws IllegalStateException if the requested number of available ports could not be found + */ + public static SortedSet findAvailableUdpPorts(int numRequested) { + return findAvailableUdpPorts(numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + /** + * Find the requested number of available UDP ports, each randomly selected + * from the range [{@code minPort}, {@code maxPort}]. + * @param numRequested the number of available ports to find + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return a sorted set of available UDP port numbers + * @throws IllegalStateException if the requested number of available ports could not be found + */ + public static SortedSet findAvailableUdpPorts(int numRequested, int minPort, int maxPort) { + return SocketType.UDP.findAvailablePorts(numRequested, minPort, maxPort); + } + + + public enum SocketType { + + TCP { + @Override + protected boolean isPortAvailable(int port) { + try { + ServerSocket serverSocket = ServerSocketFactory.getDefault().createServerSocket( + port, 1, InetAddress.getByName("localhost")); + serverSocket.close(); + return true; + } + catch (Exception ex) { + return false; + } + } + }, + + UDP { + @Override + protected boolean isPortAvailable(int port) { + try { + DatagramSocket socket = new DatagramSocket(port, InetAddress.getByName("localhost")); + socket.close(); + return true; + } + catch (Exception ex) { + return false; + } + } + }; + + /** + * Determine if the specified port for this {@code SocketType} is + * currently available on {@code localhost}. + */ + protected abstract boolean isPortAvailable(int port); + + /** + * Find a pseudo-random port number within the range + * [{@code minPort}, {@code maxPort}]. + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return a random port number within the specified range + */ + private int findRandomPort(int minPort, int maxPort) { + int portRange = maxPort - minPort; + return minPort + random.nextInt(portRange + 1); + } + + /** + * Find an available port for this {@code SocketType}, randomly selected + * from the range [{@code minPort}, {@code maxPort}]. + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return an available port number for this socket type + * @throws IllegalStateException if no available port could be found + */ + int findAvailablePort(int minPort, int maxPort) { + //Assert.assertTrue(minPort > 0, "'minPort' must be greater than 0"); + //Assert.isTrue(maxPort >= minPort, "'maxPort' must be greater than or equal to 'minPort'"); + //Assert.isTrue(maxPort <= PORT_RANGE_MAX, "'maxPort' must be less than or equal to " + PORT_RANGE_MAX); + + int portRange = maxPort - minPort; + int candidatePort; + int searchCounter = 0; + do { + if (searchCounter > portRange) { + throw new IllegalStateException(String.format( + "Could not find an available %s port in the range [%d, %d] after %d attempts", + name(), minPort, maxPort, searchCounter)); + } + candidatePort = findRandomPort(minPort, maxPort); + searchCounter++; + } + while (!isPortAvailable(candidatePort)); + + return candidatePort; + } + + /** + * Find the requested number of available ports for this {@code SocketType}, + * each randomly selected from the range [{@code minPort}, {@code maxPort}]. + * @param numRequested the number of available ports to find + * @param minPort the minimum port number + * @param maxPort the maximum port number + * @return a sorted set of available port numbers for this socket type + * @throws IllegalStateException if the requested number of available ports could not be found + */ + SortedSet findAvailablePorts(int numRequested, int minPort, int maxPort) { + //Assert.isTrue(minPort > 0, "'minPort' must be greater than 0"); + //Assert.isTrue(maxPort > minPort, "'maxPort' must be greater than 'minPort'"); + //Assert.isTrue(maxPort <= PORT_RANGE_MAX, "'maxPort' must be less than or equal to " + PORT_RANGE_MAX); + //Assert.isTrue(numRequested > 0, "'numRequested' must be greater than 0"); + //Assert.isTrue((maxPort - minPort) >= numRequested, + // "'numRequested' must not be greater than 'maxPort' - 'minPort'"); + + SortedSet availablePorts = new TreeSet<>(); + int attemptCount = 0; + while ((++attemptCount <= numRequested + 100) && availablePorts.size() < numRequested) { + availablePorts.add(findAvailablePort(minPort, maxPort)); + } + + if (availablePorts.size() != numRequested) { + throw new IllegalStateException(String.format( + "Could not find %d available %s ports in the range [%d, %d]", + numRequested, name(), minPort, maxPort)); + } + + return availablePorts; + } + } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java index b5d788180d..d04bfa8024 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java @@ -1,31 +1,31 @@ package org.opensearch.test.framework.cluster; /* - * Copyright 2002-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Copyright 2002-2020 the original author or authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* https://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ import java.net.DatagramSocket; import java.net.InetAddress; @@ -48,165 +48,165 @@ import static org.opensearch.test.framework.cluster.SocketUtils.PORT_RANGE_MIN; /** - * Unit tests for {@link SocketUtils}. - * - * @author Sam Brannen - * @author Gary Russell - */ +* Unit tests for {@link SocketUtils}. +* +* @author Sam Brannen +* @author Gary Russell +*/ public class SocketUtilsTests { - // TCP - - @Test - public void findAvailableTcpPort() { - int port = SocketUtils.findAvailableTcpPort(); - assertPortInRange(port, PORT_RANGE_MIN, PORT_RANGE_MAX); - } - - @Test - public void findAvailableTcpPortWithMinPortEqualToMaxPort() { - int minMaxPort = SocketUtils.findAvailableTcpPort(); - int port = SocketUtils.findAvailableTcpPort(minMaxPort, minMaxPort); - assertThat(port, equalTo(minMaxPort)); - } - - @Test - public void findAvailableTcpPortWhenPortOnLoopbackInterfaceIsNotAvailable() throws Exception { - int port = SocketUtils.findAvailableTcpPort(); - try (ServerSocket socket = ServerSocketFactory.getDefault().createServerSocket(port, 1, InetAddress.getByName("localhost"))) { - assertThat(socket, notNullValue()); - // will only look for the exact port - IllegalStateException exception = assertThrows( - IllegalStateException.class, - () -> SocketUtils.findAvailableTcpPort(port, port) - ); - assertThat(exception.getMessage(), startsWith("Could not find an available TCP port")); - assertThat(exception.getMessage(), endsWith("after 1 attempts")); - } - } - - @Test - public void findAvailableTcpPortWithMin() { - int port = SocketUtils.findAvailableTcpPort(50000); - assertPortInRange(port, 50000, PORT_RANGE_MAX); - } - - @Test - public void findAvailableTcpPortInRange() { - int minPort = 20000; - int maxPort = minPort + 1000; - int port = SocketUtils.findAvailableTcpPort(minPort, maxPort); - assertPortInRange(port, minPort, maxPort); - } - - @Test - public void find4AvailableTcpPorts() { - findAvailableTcpPorts(4); - } - - @Test - public void find50AvailableTcpPorts() { - findAvailableTcpPorts(50); - } - - @Test - public void find4AvailableTcpPortsInRange() { - findAvailableTcpPorts(4, 30000, 35000); - } - - @Test - public void find50AvailableTcpPortsInRange() { - findAvailableTcpPorts(50, 40000, 45000); - } - - // UDP - - @Test - public void findAvailableUdpPort() { - int port = SocketUtils.findAvailableUdpPort(); - assertPortInRange(port, PORT_RANGE_MIN, PORT_RANGE_MAX); - } - - @Test - public void findAvailableUdpPortWhenPortOnLoopbackInterfaceIsNotAvailable() throws Exception { - int port = SocketUtils.findAvailableUdpPort(); - try (DatagramSocket socket = new DatagramSocket(port, InetAddress.getByName("localhost"))) { - assertThat(socket, notNullValue()); - // will only look for the exact port - IllegalStateException exception = assertThrows( - IllegalStateException.class, - () -> SocketUtils.findAvailableUdpPort(port, port) - ); - assertThat(exception.getMessage(), startsWith("Could not find an available UDP port")); - assertThat(exception.getMessage(), endsWith("after 1 attempts")); - } - } - - @Test - public void findAvailableUdpPortWithMin() { - int port = SocketUtils.findAvailableUdpPort(50000); - assertPortInRange(port, 50000, PORT_RANGE_MAX); - } - - @Test - public void findAvailableUdpPortInRange() { - int minPort = 20000; - int maxPort = minPort + 1000; - int port = SocketUtils.findAvailableUdpPort(minPort, maxPort); - assertPortInRange(port, minPort, maxPort); - } - - @Test - public void find4AvailableUdpPorts() { - findAvailableUdpPorts(4); - } - - @Test - public void find50AvailableUdpPorts() { - findAvailableUdpPorts(50); - } - - @Test - public void find4AvailableUdpPortsInRange() { - findAvailableUdpPorts(4, 30000, 35000); - } - - @Test - public void find50AvailableUdpPortsInRange() { - findAvailableUdpPorts(50, 40000, 45000); - } - - // Helpers - - private void findAvailableTcpPorts(int numRequested) { - SortedSet ports = SocketUtils.findAvailableTcpPorts(numRequested); - assertAvailablePorts(ports, numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); - } - - private void findAvailableTcpPorts(int numRequested, int minPort, int maxPort) { - SortedSet ports = SocketUtils.findAvailableTcpPorts(numRequested, minPort, maxPort); - assertAvailablePorts(ports, numRequested, minPort, maxPort); - } - - private void findAvailableUdpPorts(int numRequested) { - SortedSet ports = SocketUtils.findAvailableUdpPorts(numRequested); - assertAvailablePorts(ports, numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); - } - - private void findAvailableUdpPorts(int numRequested, int minPort, int maxPort) { - SortedSet ports = SocketUtils.findAvailableUdpPorts(numRequested, minPort, maxPort); - assertAvailablePorts(ports, numRequested, minPort, maxPort); - } - private void assertPortInRange(int port, int minPort, int maxPort) { - assertThat("port [" + port + "] >= " + minPort, port, greaterThanOrEqualTo(minPort)); - assertThat("port [" + port + "] <= " + maxPort, port, lessThanOrEqualTo(maxPort)); - } - - private void assertAvailablePorts(SortedSet ports, int numRequested, int minPort, int maxPort) { - assertThat("number of ports requested", ports.size(), equalTo(numRequested)); - for (int port : ports) { - assertPortInRange(port, minPort, maxPort); - } - } + // TCP + + @Test + public void findAvailableTcpPort() { + int port = SocketUtils.findAvailableTcpPort(); + assertPortInRange(port, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + @Test + public void findAvailableTcpPortWithMinPortEqualToMaxPort() { + int minMaxPort = SocketUtils.findAvailableTcpPort(); + int port = SocketUtils.findAvailableTcpPort(minMaxPort, minMaxPort); + assertThat(port, equalTo(minMaxPort)); + } + + @Test + public void findAvailableTcpPortWhenPortOnLoopbackInterfaceIsNotAvailable() throws Exception { + int port = SocketUtils.findAvailableTcpPort(); + try (ServerSocket socket = ServerSocketFactory.getDefault().createServerSocket(port, 1, InetAddress.getByName("localhost"))) { + assertThat(socket, notNullValue()); + // will only look for the exact port + IllegalStateException exception = assertThrows( + IllegalStateException.class, + () -> SocketUtils.findAvailableTcpPort(port, port) + ); + assertThat(exception.getMessage(), startsWith("Could not find an available TCP port")); + assertThat(exception.getMessage(), endsWith("after 1 attempts")); + } + } + + @Test + public void findAvailableTcpPortWithMin() { + int port = SocketUtils.findAvailableTcpPort(50000); + assertPortInRange(port, 50000, PORT_RANGE_MAX); + } + + @Test + public void findAvailableTcpPortInRange() { + int minPort = 20000; + int maxPort = minPort + 1000; + int port = SocketUtils.findAvailableTcpPort(minPort, maxPort); + assertPortInRange(port, minPort, maxPort); + } + + @Test + public void find4AvailableTcpPorts() { + findAvailableTcpPorts(4); + } + + @Test + public void find50AvailableTcpPorts() { + findAvailableTcpPorts(50); + } + + @Test + public void find4AvailableTcpPortsInRange() { + findAvailableTcpPorts(4, 30000, 35000); + } + + @Test + public void find50AvailableTcpPortsInRange() { + findAvailableTcpPorts(50, 40000, 45000); + } + + // UDP + + @Test + public void findAvailableUdpPort() { + int port = SocketUtils.findAvailableUdpPort(); + assertPortInRange(port, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + @Test + public void findAvailableUdpPortWhenPortOnLoopbackInterfaceIsNotAvailable() throws Exception { + int port = SocketUtils.findAvailableUdpPort(); + try (DatagramSocket socket = new DatagramSocket(port, InetAddress.getByName("localhost"))) { + assertThat(socket, notNullValue()); + // will only look for the exact port + IllegalStateException exception = assertThrows( + IllegalStateException.class, + () -> SocketUtils.findAvailableUdpPort(port, port) + ); + assertThat(exception.getMessage(), startsWith("Could not find an available UDP port")); + assertThat(exception.getMessage(), endsWith("after 1 attempts")); + } + } + + @Test + public void findAvailableUdpPortWithMin() { + int port = SocketUtils.findAvailableUdpPort(50000); + assertPortInRange(port, 50000, PORT_RANGE_MAX); + } + + @Test + public void findAvailableUdpPortInRange() { + int minPort = 20000; + int maxPort = minPort + 1000; + int port = SocketUtils.findAvailableUdpPort(minPort, maxPort); + assertPortInRange(port, minPort, maxPort); + } + + @Test + public void find4AvailableUdpPorts() { + findAvailableUdpPorts(4); + } + + @Test + public void find50AvailableUdpPorts() { + findAvailableUdpPorts(50); + } + + @Test + public void find4AvailableUdpPortsInRange() { + findAvailableUdpPorts(4, 30000, 35000); + } + + @Test + public void find50AvailableUdpPortsInRange() { + findAvailableUdpPorts(50, 40000, 45000); + } + + // Helpers + + private void findAvailableTcpPorts(int numRequested) { + SortedSet ports = SocketUtils.findAvailableTcpPorts(numRequested); + assertAvailablePorts(ports, numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + private void findAvailableTcpPorts(int numRequested, int minPort, int maxPort) { + SortedSet ports = SocketUtils.findAvailableTcpPorts(numRequested, minPort, maxPort); + assertAvailablePorts(ports, numRequested, minPort, maxPort); + } + + private void findAvailableUdpPorts(int numRequested) { + SortedSet ports = SocketUtils.findAvailableUdpPorts(numRequested); + assertAvailablePorts(ports, numRequested, PORT_RANGE_MIN, PORT_RANGE_MAX); + } + + private void findAvailableUdpPorts(int numRequested, int minPort, int maxPort) { + SortedSet ports = SocketUtils.findAvailableUdpPorts(numRequested, minPort, maxPort); + assertAvailablePorts(ports, numRequested, minPort, maxPort); + } + private void assertPortInRange(int port, int minPort, int maxPort) { + assertThat("port [" + port + "] >= " + minPort, port, greaterThanOrEqualTo(minPort)); + assertThat("port [" + port + "] <= " + maxPort, port, lessThanOrEqualTo(maxPort)); + } + + private void assertAvailablePorts(SortedSet ports, int numRequested, int minPort, int maxPort) { + assertThat("number of ports requested", ports.size(), equalTo(numRequested)); + for (int port : ports) { + assertPortInRange(port, minPort, maxPort); + } + } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java index f10077735e..80db4ba87a 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/StartStage.java @@ -1,15 +1,15 @@ /* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ package org.opensearch.test.framework.cluster; enum StartStage { - INITIALIZED, - RETRY + INITIALIZED, + RETRY } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index 7704794059..c32a19ca64 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -1,30 +1,30 @@ /* - * Copyright 2021 floragunn GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ +* Copyright 2021 floragunn GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*/ /* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ package org.opensearch.test.framework.cluster; @@ -72,113 +72,113 @@ public class TestRestClient implements AutoCloseable { - private static final Logger log = LogManager.getLogger(TestRestClient.class); - - private boolean enableHTTPClientSSL = true; - private boolean sendHTTPClientCertificate = false; - private InetSocketAddress nodeHttpAddress; - private RequestConfig requestConfig; - private List
headers = new ArrayList<>(); - private Header CONTENT_TYPE_JSON = new BasicHeader("Content-Type", "application/json"); - private SSLContext sslContext; - - public TestRestClient(InetSocketAddress nodeHttpAddress, List
headers, SSLContext sslContext) { - this.nodeHttpAddress = nodeHttpAddress; - this.headers.addAll(headers); - this.sslContext = sslContext; - } - - public HttpResponse get(String path, Header... headers) { - return executeRequest(new HttpGet(getHttpServerUri() + "/" + path), headers); - } - - public HttpResponse getAuthInfo( Header... headers) { - return executeRequest(new HttpGet(getHttpServerUri() + "/_opendistro/_security/authinfo?pretty"), headers); - } - - public HttpResponse head(String path, Header... headers) { - return executeRequest(new HttpHead(getHttpServerUri() + "/" + path), headers); - } - - public HttpResponse options(String path, Header... headers) { - return executeRequest(new HttpOptions(getHttpServerUri() + "/" + path), headers); - } - - public HttpResponse putJson(String path, String body, Header... headers) { - HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); - uriRequest.setEntity(toStringEntity(body)); - return executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); - } - - private StringEntity toStringEntity(String body) { - try { - return new StringEntity(body); - } catch (UnsupportedEncodingException e) { - throw new RestClientException("Cannot create string entity", e); - } - } - - public HttpResponse putJson(String path, ToXContentObject body) { - return putJson(path, Strings.toString(body)); - } - - public HttpResponse put(String path) { - HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); - return executeRequest(uriRequest); - } - - public HttpResponse delete(String path, Header... headers) { - return executeRequest(new HttpDelete(getHttpServerUri() + "/" + path), headers); - } - - public HttpResponse postJson(String path, String body, Header... headers) { - HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); - uriRequest.setEntity(toStringEntity(body)); - return executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); - } - - public HttpResponse postJson(String path, ToXContentObject body) { - return postJson(path, Strings.toString(body)); - } - - public HttpResponse post(String path) { - HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); - return executeRequest(uriRequest); - } - - public HttpResponse patch(String path, String body) { - HttpPatch uriRequest = new HttpPatch(getHttpServerUri() + "/" + path); - uriRequest.setEntity(toStringEntity(body)); - return executeRequest(uriRequest, CONTENT_TYPE_JSON); - } - - public HttpResponse executeRequest(HttpUriRequest uriRequest, Header... requestSpecificHeaders) { - - try(CloseableHttpClient httpClient = getHTTPClient()) { - - - if (requestSpecificHeaders != null && requestSpecificHeaders.length > 0) { - for (int i = 0; i < requestSpecificHeaders.length; i++) { - Header h = requestSpecificHeaders[i]; - uriRequest.addHeader(h); - } - } - - for (Header header : headers) { - uriRequest.addHeader(header); - } - - HttpResponse res = new HttpResponse(httpClient.execute(uriRequest)); - log.debug(res.getBody()); - return res; - } catch (IOException e) { - throw new RestClientException("Error occured during HTTP request execution", e); - } - } - - protected final String getHttpServerUri() { - return "http" + (enableHTTPClientSSL ? "s" : "") + "://" + nodeHttpAddress.getHostString() + ":" + nodeHttpAddress.getPort(); - } + private static final Logger log = LogManager.getLogger(TestRestClient.class); + + private boolean enableHTTPClientSSL = true; + private boolean sendHTTPClientCertificate = false; + private InetSocketAddress nodeHttpAddress; + private RequestConfig requestConfig; + private List
headers = new ArrayList<>(); + private Header CONTENT_TYPE_JSON = new BasicHeader("Content-Type", "application/json"); + private SSLContext sslContext; + + public TestRestClient(InetSocketAddress nodeHttpAddress, List
headers, SSLContext sslContext) { + this.nodeHttpAddress = nodeHttpAddress; + this.headers.addAll(headers); + this.sslContext = sslContext; + } + + public HttpResponse get(String path, Header... headers) { + return executeRequest(new HttpGet(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse getAuthInfo( Header... headers) { + return executeRequest(new HttpGet(getHttpServerUri() + "/_opendistro/_security/authinfo?pretty"), headers); + } + + public HttpResponse head(String path, Header... headers) { + return executeRequest(new HttpHead(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse options(String path, Header... headers) { + return executeRequest(new HttpOptions(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse putJson(String path, String body, Header... headers) { + HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); + uriRequest.setEntity(toStringEntity(body)); + return executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); + } + + private StringEntity toStringEntity(String body) { + try { + return new StringEntity(body); + } catch (UnsupportedEncodingException e) { + throw new RestClientException("Cannot create string entity", e); + } + } + + public HttpResponse putJson(String path, ToXContentObject body) { + return putJson(path, Strings.toString(body)); + } + + public HttpResponse put(String path) { + HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); + return executeRequest(uriRequest); + } + + public HttpResponse delete(String path, Header... headers) { + return executeRequest(new HttpDelete(getHttpServerUri() + "/" + path), headers); + } + + public HttpResponse postJson(String path, String body, Header... headers) { + HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); + uriRequest.setEntity(toStringEntity(body)); + return executeRequest(uriRequest, mergeHeaders(CONTENT_TYPE_JSON, headers)); + } + + public HttpResponse postJson(String path, ToXContentObject body) { + return postJson(path, Strings.toString(body)); + } + + public HttpResponse post(String path) { + HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); + return executeRequest(uriRequest); + } + + public HttpResponse patch(String path, String body) { + HttpPatch uriRequest = new HttpPatch(getHttpServerUri() + "/" + path); + uriRequest.setEntity(toStringEntity(body)); + return executeRequest(uriRequest, CONTENT_TYPE_JSON); + } + + public HttpResponse executeRequest(HttpUriRequest uriRequest, Header... requestSpecificHeaders) { + + try(CloseableHttpClient httpClient = getHTTPClient()) { + + + if (requestSpecificHeaders != null && requestSpecificHeaders.length > 0) { + for (int i = 0; i < requestSpecificHeaders.length; i++) { + Header h = requestSpecificHeaders[i]; + uriRequest.addHeader(h); + } + } + + for (Header header : headers) { + uriRequest.addHeader(header); + } + + HttpResponse res = new HttpResponse(httpClient.execute(uriRequest)); + log.debug(res.getBody()); + return res; + } catch (IOException e) { + throw new RestClientException("Error occured during HTTP request execution", e); + } + } + + protected final String getHttpServerUri() { + return "http" + (enableHTTPClientSSL ? "s" : "") + "://" + nodeHttpAddress.getHostString() + ":" + nodeHttpAddress.getPort(); + } protected final CloseableHttpClient getHTTPClient() { @@ -200,150 +200,150 @@ protected final CloseableHttpClient getHTTPClient() { return hcb.build(); } - private Header[] mergeHeaders(Header header, Header... headers) { - - if (headers == null || headers.length == 0) { - return new Header[] { header }; - } else { - Header[] result = new Header[headers.length + 1]; - result[0] = header; - System.arraycopy(headers, 0, result, 1, headers.length); - return result; - } - } - - public static class HttpResponse { - private final CloseableHttpResponse inner; - private final String body; - private final Header[] header; - private final int statusCode; - private final String statusReason; - - public HttpResponse(CloseableHttpResponse inner) throws IllegalStateException, IOException { - super(); - this.inner = inner; - final HttpEntity entity = inner.getEntity(); - if (entity == null) { //head request does not have a entity - this.body = ""; - } else { - this.body = IOUtils.toString(entity.getContent(), StandardCharsets.UTF_8); - } - this.header = inner.getAllHeaders(); - this.statusCode = inner.getStatusLine().getStatusCode(); - this.statusReason = inner.getStatusLine().getReasonPhrase(); - inner.close(); - } - - public String getContentType() { - Header h = getInner().getFirstHeader("content-type"); - if (h != null) { - return h.getValue(); - } - return null; - } - - public boolean isJsonContentType() { - String ct = getContentType(); - if (ct == null) { - return false; - } - return ct.contains("application/json"); - } - - public CloseableHttpResponse getInner() { - return inner; - } - - public String getBody() { - return body; - } - - public Header[] getHeader() { - return header; - } - - public int getStatusCode() { - return statusCode; - } - - public String getStatusReason() { - return statusReason; - } - - public List
getHeaders() { - return header == null ? Collections.emptyList() : Arrays.asList(header); - } - - public String getTextFromJsonBody(String jsonPointer) { - return getJsonNodeAt(jsonPointer).asText(); - } - - public int getIntFromJsonBody(String jsonPointer) { - return getJsonNodeAt(jsonPointer).asInt(); - } - - public Boolean getBooleanFromJsonBody(String jsonPointer) { - return getJsonNodeAt(jsonPointer).asBoolean(); - } - - public Double getDoubleFromJsonBody(String jsonPointer) { - return getJsonNodeAt(jsonPointer).asDouble(); - } - - public Long getLongFromJsonBody(String jsonPointer) { - return getJsonNodeAt(jsonPointer).asLong(); - } - - private JsonNode getJsonNodeAt(String jsonPointer) { - try { + private Header[] mergeHeaders(Header header, Header... headers) { + + if (headers == null || headers.length == 0) { + return new Header[] { header }; + } else { + Header[] result = new Header[headers.length + 1]; + result[0] = header; + System.arraycopy(headers, 0, result, 1, headers.length); + return result; + } + } + + public static class HttpResponse { + private final CloseableHttpResponse inner; + private final String body; + private final Header[] header; + private final int statusCode; + private final String statusReason; + + public HttpResponse(CloseableHttpResponse inner) throws IllegalStateException, IOException { + super(); + this.inner = inner; + final HttpEntity entity = inner.getEntity(); + if (entity == null) { //head request does not have a entity + this.body = ""; + } else { + this.body = IOUtils.toString(entity.getContent(), StandardCharsets.UTF_8); + } + this.header = inner.getAllHeaders(); + this.statusCode = inner.getStatusLine().getStatusCode(); + this.statusReason = inner.getStatusLine().getReasonPhrase(); + inner.close(); + } + + public String getContentType() { + Header h = getInner().getFirstHeader("content-type"); + if (h != null) { + return h.getValue(); + } + return null; + } + + public boolean isJsonContentType() { + String ct = getContentType(); + if (ct == null) { + return false; + } + return ct.contains("application/json"); + } + + public CloseableHttpResponse getInner() { + return inner; + } + + public String getBody() { + return body; + } + + public Header[] getHeader() { + return header; + } + + public int getStatusCode() { + return statusCode; + } + + public String getStatusReason() { + return statusReason; + } + + public List
getHeaders() { + return header == null ? Collections.emptyList() : Arrays.asList(header); + } + + public String getTextFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asText(); + } + + public int getIntFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asInt(); + } + + public Boolean getBooleanFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asBoolean(); + } + + public Double getDoubleFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asDouble(); + } + + public Long getLongFromJsonBody(String jsonPointer) { + return getJsonNodeAt(jsonPointer).asLong(); + } + + private JsonNode getJsonNodeAt(String jsonPointer) { + try { return toJsonNode().at(jsonPointer); } catch (IOException e) { throw new IllegalArgumentException("Cound not convert response body to JSON node ",e); } - } - - private JsonNode toJsonNode() throws JsonProcessingException, IOException { - return DefaultObjectMapper.objectMapper.readTree(getBody()); - } - - - - @Override - public String toString() { - return "HttpResponse [inner=" + inner + ", body=" + body + ", header=" + Arrays.toString(header) + ", statusCode=" + statusCode - + ", statusReason=" + statusReason + "]"; - } - - } - - @Override - public String toString() { - return "TestRestClient [server=" + getHttpServerUri() + ", node=" + nodeHttpAddress + "]"; - } - - public RequestConfig getRequestConfig() { - return requestConfig; - } - - public void setRequestConfig(RequestConfig requestConfig) { - this.requestConfig = requestConfig; - } - - public void setLocalAddress(InetAddress inetAddress) { - if (requestConfig == null) { - requestConfig = RequestConfig.custom().setLocalAddress(inetAddress).build(); - } else { - requestConfig = RequestConfig.copy(requestConfig).setLocalAddress(inetAddress).build(); - } - } - - public boolean isSendHTTPClientCertificate() { - return sendHTTPClientCertificate; - } - - public void setSendHTTPClientCertificate(boolean sendHTTPClientCertificate) { - this.sendHTTPClientCertificate = sendHTTPClientCertificate; - } + } + + private JsonNode toJsonNode() throws JsonProcessingException, IOException { + return DefaultObjectMapper.objectMapper.readTree(getBody()); + } + + + + @Override + public String toString() { + return "HttpResponse [inner=" + inner + ", body=" + body + ", header=" + Arrays.toString(header) + ", statusCode=" + statusCode + + ", statusReason=" + statusReason + "]"; + } + + } + + @Override + public String toString() { + return "TestRestClient [server=" + getHttpServerUri() + ", node=" + nodeHttpAddress + "]"; + } + + public RequestConfig getRequestConfig() { + return requestConfig; + } + + public void setRequestConfig(RequestConfig requestConfig) { + this.requestConfig = requestConfig; + } + + public void setLocalAddress(InetAddress inetAddress) { + if (requestConfig == null) { + requestConfig = RequestConfig.custom().setLocalAddress(inetAddress).build(); + } else { + requestConfig = RequestConfig.copy(requestConfig).setLocalAddress(inetAddress).build(); + } + } + + public boolean isSendHTTPClientCertificate() { + return sendHTTPClientCertificate; + } + + public void setSendHTTPClientCertificate(boolean sendHTTPClientCertificate) { + this.sendHTTPClientCertificate = sendHTTPClientCertificate; + } @Override public void close() { From df0da3c3478437fffdc26667825d7caec94617de Mon Sep 17 00:00:00 2001 From: Kacper Trochimiak Date: Wed, 24 Aug 2022 15:01:43 +0200 Subject: [PATCH 37/47] CI/CD - run integrationTest task only when test task is executed Signed-off-by: Kacper Trochimiak --- .github/workflows/cd.yml | 4 ++-- .github/workflows/ci.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index f2bc154d3f..03d5d6bd9b 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -34,9 +34,9 @@ jobs: - name: Build run: | - ./gradlew clean build -Dbuild.snapshot=false -x test + ./gradlew clean build -Dbuild.snapshot=false -x test -x integrationTest artifact_zip=`ls $(pwd)/build/distributions/opensearch-security-*.zip | grep -v admin-standalone` - ./gradlew build buildDeb buildRpm -ParchivePath=$artifact_zip -Dbuild.snapshot=false -x test + ./gradlew build buildDeb buildRpm -ParchivePath=$artifact_zip -Dbuild.snapshot=false -x test -x integrationTest mkdir artifacts cp $artifact_zip artifacts/ cp build/distributions/*.deb artifacts/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdbcb45ab2..f00e5bef68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,10 +36,10 @@ jobs: ${{ runner.os }}-gradle- - name: Package - run: ./gradlew clean build -Dbuild.snapshot=false -x test + run: ./gradlew clean build -Dbuild.snapshot=false -x test -x integrationTest - name: Test - run: OPENDISTRO_SECURITY_TEST_OPENSSL_OPT=true ./gradlew test -i + run: OPENDISTRO_SECURITY_TEST_OPENSSL_OPT=true ./gradlew test integrationTest -i - name: Coverage uses: codecov/codecov-action@v1 @@ -65,7 +65,7 @@ jobs: - uses: actions/setup-java@v1 with: java-version: 11 - - run: ./gradlew clean build -Dbuild.snapshot=false -x test + - run: ./gradlew clean build -Dbuild.snapshot=false -x test -x integrationTest - run: | echo "Running backwards compatibility tests ..." security_plugin_version_no_snapshot=$(./gradlew properties -q | grep -E '^version:' | awk '{print $2}' | sed 's/-SNAPSHOT//g') @@ -88,7 +88,7 @@ jobs: - uses: github/codeql-action/init@v1 with: languages: java - - run: ./gradlew clean build -Dbuild.snapshot=false -x test + - run: ./gradlew clean build -Dbuild.snapshot=false -x test -x integrationTest - uses: github/codeql-action/analyze@v1 build-artifact-names: From 9f4d7153ede69c7288d2c875bc3a59b0500799b8 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Thu, 25 Aug 2022 09:43:07 +0200 Subject: [PATCH 38/47] Removed uses of NestedValueMap and replaced by ToXContent Signed-Off-By: Nils Bandener --- .../test/framework/TestSecurityConfig.java | 429 ++++++--------- .../test/framework/cluster/LocalCluster.java | 25 +- .../framework/cluster/NestedValueMap.java | 518 ------------------ 3 files changed, 184 insertions(+), 788 deletions(-) delete mode 100644 src/integrationTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java index 454f73b84e..83a5685124 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -28,16 +28,10 @@ package org.opensearch.test.framework; +import java.io.IOException; import java.nio.ByteBuffer; import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; @@ -50,26 +44,23 @@ import org.opensearch.client.Client; import org.opensearch.common.Strings; import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.*; import org.opensearch.security.action.configupdate.ConfigUpdateAction; import org.opensearch.security.action.configupdate.ConfigUpdateRequest; import org.opensearch.security.action.configupdate.ConfigUpdateResponse; import org.opensearch.security.securityconf.impl.CType; -import org.opensearch.test.framework.cluster.NestedValueMap; -import org.opensearch.test.framework.cluster.NestedValueMap.Path; import org.opensearch.test.framework.cluster.OpenSearchClientProvider.UserCredentialsHolder; public class TestSecurityConfig { private static final Logger log = LogManager.getLogger(TestSecurityConfig.class); - private NestedValueMap overrideSecurityConfigSettings; - private NestedValueMap overrideUserSettings; - private NestedValueMap overrideRoleSettings; - private NestedValueMap overrideRoleMappingSettings; + private Config config = new Config(); + private Map internalUsers = new LinkedHashMap<>(); + private Map roles = new LinkedHashMap<>(); + private String indexName = ".opendistro_security"; - + public TestSecurityConfig() { } @@ -79,173 +70,74 @@ public TestSecurityConfig configIndexName(String configIndexName) { return this; } - public TestSecurityConfig securityConfigSettings(String keyPath, Object value, Object... more) { - if (overrideSecurityConfigSettings == null) { - overrideSecurityConfigSettings = new NestedValueMap(); - } - - overrideSecurityConfigSettings.put(NestedValueMap.Path.parse(keyPath), value); - - for (int i = 0; i < more.length - 1; i += 2) { - overrideSecurityConfigSettings.put(NestedValueMap.Path.parse(String.valueOf(more[i])), more[i + 1]); - } - - return this; - } - - public TestSecurityConfig xff(String proxies) { - if (overrideSecurityConfigSettings == null) { - overrideSecurityConfigSettings = new NestedValueMap(); - } - - overrideSecurityConfigSettings.put(new NestedValueMap.Path("config", "dynamic", "http", "xff"), - NestedValueMap.of("enabled", true, "internalProxies", proxies)); - - return this; - } - public TestSecurityConfig anonymousAuth(boolean anonymousAuthEnabled) { - if (overrideSecurityConfigSettings == null) { - overrideSecurityConfigSettings = new NestedValueMap(); - } - - overrideSecurityConfigSettings.put(new NestedValueMap.Path("config", "dynamic", "http"), - NestedValueMap.of("anonymous_auth_enabled", anonymousAuthEnabled)); - + config.anonymousAuth(anonymousAuthEnabled); return this; } public TestSecurityConfig authc(AuthcDomain authcDomain) { - if (overrideSecurityConfigSettings == null) { - overrideSecurityConfigSettings = new NestedValueMap(); - } - - overrideSecurityConfigSettings.put(new NestedValueMap.Path("config", "dynamic", "authc"), authcDomain.toMap()); - + config.authc(authcDomain); return this; } - public TestSecurityConfig user(User user) { - if (user.roleNames != null) { - return this.user(user.name, user.password, user.attributes, user.roleNames); - } else { - return this.user(user.name, user.password, user.attributes, user.roles); - } - } - - public TestSecurityConfig user(String name, String password, String... sgRoles) { - return user(name, password, null, sgRoles); - } - - public TestSecurityConfig user(String name, String password, Map attributes, String... securityRoles) { - if (overrideUserSettings == null) { - overrideUserSettings = new NestedValueMap(); - } - - overrideUserSettings.put(new NestedValueMap.Path(name, "hash"), hash(password.toCharArray())); + this.internalUsers.put(user.name, user); - if (securityRoles != null && securityRoles.length > 0) { - overrideUserSettings.put(new NestedValueMap.Path(name, "opensearch_security_roles"), securityRoles); - } - - if (attributes != null && attributes.size() != 0) { - for (Map.Entry attr : attributes.entrySet()) { - overrideUserSettings.put(new NestedValueMap.Path(name, "attributes", attr.getKey()), attr.getValue()); - } - } - - return this; - } - - public TestSecurityConfig user(String name, String password, Role... sgRoles) { - return user(name, password, null, sgRoles); - } - - public TestSecurityConfig user(String name, String password, Map attributes, Role... sgRoles) { - if (overrideUserSettings == null) { - overrideUserSettings = new NestedValueMap(); - } - - overrideUserSettings.put(new NestedValueMap.Path(name, "hash"), hash(password.toCharArray())); - - if (sgRoles != null && sgRoles.length > 0) { - String roleNamePrefix = "user_" + name + "__"; - - overrideUserSettings.put(new NestedValueMap.Path(name, "opendistro_security_roles"), - Arrays.asList(sgRoles).stream().map((r) -> roleNamePrefix + r.name).collect(Collectors.toList())); - roles(roleNamePrefix, sgRoles); - } - - if (attributes != null && attributes.size() != 0) { - for (Map.Entry attr : attributes.entrySet()) { - overrideUserSettings.put(new NestedValueMap.Path(name, "attributes", attr.getKey()), attr.getValue()); - } + for (Role role : user.roles) { + this.roles.put(role.name, role); } return this; } public TestSecurityConfig roles(Role... roles) { - return roles("", roles); - } - - public TestSecurityConfig roles(String roleNamePrefix, Role... roles) { - if (overrideRoleSettings == null) { - overrideRoleSettings = new NestedValueMap(); - } - for (Role role : roles) { - - String name = roleNamePrefix + role.name; - - if (role.clusterPermissions.size() > 0) { - overrideRoleSettings.put(new NestedValueMap.Path(name, "cluster_permissions"), role.clusterPermissions); - } - - if (role.indexPermissions.size() > 0) { - overrideRoleSettings.put(new NestedValueMap.Path(name, "index_permissions"), - role.indexPermissions.stream().map((p) -> p.toJsonMap()).collect(Collectors.toList())); - } + this.roles.put(role.name, role); } return this; } - public TestSecurityConfig roleMapping(RoleMapping... roleMappings) { - if (overrideRoleMappingSettings == null) { - overrideRoleMappingSettings = new NestedValueMap(); - } + public static class Config implements ToXContentObject { + private boolean anonymousAuth; + private Map authcDomainMap = new LinkedHashMap<>(); - for (RoleMapping roleMapping : roleMappings) { + public Config anonymousAuth(boolean anonymousAuth) { + this.anonymousAuth = anonymousAuth; + return this; + } - String name = roleMapping.name; + public Config authc(AuthcDomain authcDomain) { + authcDomainMap.put(authcDomain.id, authcDomain); + return this; + } - if (roleMapping.backendRoles.size() > 0) { - overrideRoleMappingSettings.put(new NestedValueMap.Path(name, "backend_roles"), - roleMapping.backendRoles); - } + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + xContentBuilder.startObject("dynamic"); - if (roleMapping.users.size() > 0) { - overrideRoleMappingSettings.put(new NestedValueMap.Path(name, "users"), roleMapping.users); + if (anonymousAuth) { + xContentBuilder.startObject("http"); + xContentBuilder.field("anonymous_auth_enabled", true); + xContentBuilder.endObject(); } - } - return this; - } + xContentBuilder.field("authc", authcDomainMap); - public TestSecurityConfig roleToRoleMapping(Role role, String... backendRoles) { - return this.roleMapping(new RoleMapping(role.name).backendRoles(backendRoles)); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + return xContentBuilder; + } } - public static class User implements UserCredentialsHolder { + public static class User implements UserCredentialsHolder, ToXContentObject { public final static TestSecurityConfig.User USER_ADMIN = new TestSecurityConfig.User("admin") .roles(new Role("allaccess").indexPermissions("*").on("*").clusterPermissions("*")); private String name; private String password; - private Role[] roles; - private String[] roleNames; + private List roles = new ArrayList<>(); private Map attributes = new HashMap<>(); public User(String name) { @@ -259,12 +151,9 @@ public User password(String password) { } public User roles(Role... roles) { - this.roles = roles; - return this; - } - - public User roles(String... roles) { - this.roleNames = roles; + // We scope the role names by user to keep tests free of potential side effects + String roleNamePrefix = "user_" + this.name + "__"; + this.roles.addAll(Arrays.asList(roles).stream().map((r) -> r.clone().name(roleNamePrefix + r.name)).collect(Collectors.toSet())); return this; } @@ -282,22 +171,31 @@ public String getPassword() { } public Set getRoleNames() { - Set result = new HashSet(); + return roles.stream().map(Role::getName).collect(Collectors.toSet()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + + xContentBuilder.field("hash", hash(password.toCharArray())); + + Set roleNames = getRoleNames(); - if (roleNames != null) { - result.addAll(Arrays.asList(roleNames)); + if (!roleNames.isEmpty()) { + xContentBuilder.field("opendistro_security_roles", roleNames); } - if (roles != null) { - result.addAll(Arrays.asList(roles).stream().map(Role::getName).collect(Collectors.toSet())); + if (attributes != null && attributes.size() != 0) { + xContentBuilder.field("attributes", attributes); } - return result; + xContentBuilder.endObject(); + return xContentBuilder; } - } - public static class Role { + public static class Role implements ToXContentObject { public static Role ALL_ACCESS = new Role("all_access").clusterPermissions("*").indexPermissions("*").on("*"); private String name; @@ -318,33 +216,40 @@ public IndexPermission indexPermissions(String... indexPermissions) { return new IndexPermission(this, indexPermissions); } + public Role name(String name) { + this.name = name; + return this; + } + public String getName() { return name; } - } - - public static class RoleMapping { - private String name; - private List backendRoles = new ArrayList<>(); - private List users = new ArrayList<>(); - public RoleMapping(String name) { - this.name = name; + public Role clone() { + Role role = new Role(this.name); + role.clusterPermissions.addAll(this.clusterPermissions); + role.indexPermissions.addAll(this.indexPermissions); + return role; } - public RoleMapping backendRoles(String... backendRoles) { - this.backendRoles.addAll(Arrays.asList(backendRoles)); - return this; - } + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); - public RoleMapping users(String... users) { - this.users.addAll(Arrays.asList(users)); - return this; - } + if (!clusterPermissions.isEmpty()) { + xContentBuilder.field("cluster_permissions", clusterPermissions); + } + + if (!indexPermissions.isEmpty()) { + xContentBuilder.field("index_permissions", indexPermissions); + } + xContentBuilder.endObject(); + return xContentBuilder; + } } - public static class IndexPermission { + public static class IndexPermission implements ToXContentObject { private List allowedActions; private List indexPatterns; private Role role; @@ -383,38 +288,38 @@ public Role on(TestIndex... testindices) { this.role.indexPermissions.add(this); return this.role; } - - public NestedValueMap toJsonMap() { - NestedValueMap result = new NestedValueMap(); - result.put("index_patterns", indexPatterns); - result.put("allowed_actions", allowedActions); + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + + xContentBuilder.field("index_patterns", indexPatterns); + xContentBuilder.field("allowed_actions", allowedActions); if (dlsQuery != null) { - result.put("dls", dlsQuery); + xContentBuilder.field("dls", dlsQuery); } if (fls != null) { - result.put("fls", fls); + xContentBuilder.field("fls", fls); } if (maskedFields != null) { - result.put("masked_fields", maskedFields); + xContentBuilder.field("masked_fields", maskedFields); } - return result; + xContentBuilder.endObject(); + return xContentBuilder; } - } - public static class AuthcDomain { + public static class AuthcDomain implements ToXContentObject { public final static AuthcDomain AUTHC_HTTPBASIC_INTERNAL = new TestSecurityConfig.AuthcDomain("basic", 0) .httpAuthenticator("basic").backend("internal"); private final String id; private boolean enabled = true; - private boolean transportEnabled = true; private int order; private List skipUsers = new ArrayList<>(); private HttpAuthenticator httpAuthenticator; @@ -455,32 +360,33 @@ public AuthcDomain skipUsers(String... users) { return this; } - NestedValueMap toMap() { - NestedValueMap result = new NestedValueMap(); - result.put(new NestedValueMap.Path(id, "http_enabled"), enabled); - result.put(new NestedValueMap.Path(id, "transport_enabled"), transportEnabled); - result.put(new NestedValueMap.Path(id, "order"), order); + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + + xContentBuilder.field("http_enabled", enabled); + xContentBuilder.field("order", order); if (httpAuthenticator != null) { - result.put(new NestedValueMap.Path(id, "http_authenticator"), httpAuthenticator.toMap()); + xContentBuilder.field("http_authenticator", httpAuthenticator); } if (authenticationBackend != null) { - result.put(new NestedValueMap.Path(id, "authentication_backend"), authenticationBackend.toMap()); + xContentBuilder.field("authentication_backend", authenticationBackend); } - if (skipUsers != null && skipUsers.size() > 0) { - result.put(new NestedValueMap.Path(id, "skip_users"), skipUsers); + xContentBuilder.field("skip_users", skipUsers); } - return result; + xContentBuilder.endObject(); + return xContentBuilder; } - public static class HttpAuthenticator { + public static class HttpAuthenticator implements ToXContentObject { private final String type; private boolean challenge; - private NestedValueMap config = new NestedValueMap(); + private Map config = new HashMap(); public HttpAuthenticator(String type) { this.type = type; @@ -492,61 +398,49 @@ public HttpAuthenticator challenge(boolean challenge) { } public HttpAuthenticator config(Map config) { - this.config.putAllFromAnyMap(config); + this.config.putAll(config); return this; } - public HttpAuthenticator config(String key, Object value) { - this.config.put(Path.parse(key), value); - return this; - } + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); + + xContentBuilder.field("type", type); + xContentBuilder.field("challenge", challenge); + xContentBuilder.field("config", config); - NestedValueMap toMap() { - NestedValueMap result = new NestedValueMap(); - result.put("type", type); - result.put("challenge", challenge); - result.put("config", config); - return result; + xContentBuilder.endObject(); + return xContentBuilder; } } - public static class AuthenticationBackend { + public static class AuthenticationBackend implements ToXContentObject { private final String type; - private NestedValueMap config = new NestedValueMap(); + private Map config = new HashMap(); public AuthenticationBackend(String type) { this.type = type; } public AuthenticationBackend config(Map config) { - this.config.putAllFromAnyMap(config); + this.config.putAll(config); return this; } - public AuthenticationBackend config(String key, Object value) { - this.config.put(Path.parse(key), value); - return this; - } + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + xContentBuilder.startObject(); - NestedValueMap toMap() { - NestedValueMap result = new NestedValueMap(); - result.put("type", type); - result.put("config", config); - return result; + xContentBuilder.field("type", type); + xContentBuilder.field("config", config); + + xContentBuilder.endObject(); + return xContentBuilder; } } } - public TestSecurityConfig clone() { - TestSecurityConfig result = new TestSecurityConfig(); - result.indexName = indexName; - result.overrideRoleSettings = overrideRoleSettings != null ? overrideRoleSettings.clone() : null; - result.overrideSecurityConfigSettings = overrideSecurityConfigSettings != null ? overrideSecurityConfigSettings.clone() : null; - result.overrideUserSettings = overrideUserSettings != null ? overrideUserSettings.clone() : null; - - return result; - } - public void initIndex(Client client) { Map settings = new HashMap<>(); if (indexName.startsWith(".")) { @@ -554,12 +448,12 @@ public void initIndex(Client client) { } client.admin().indices().create(new CreateIndexRequest(indexName).settings(settings)).actionGet(); - writeConfigToIndex(client, CType.CONFIG, overrideSecurityConfigSettings); - writeConfigToIndex(client, CType.ROLES, overrideRoleSettings); - writeConfigToIndex(client, CType.INTERNALUSERS, overrideUserSettings); - writeConfigToIndex(client, CType.ROLESMAPPING, overrideRoleMappingSettings); - writeConfigToIndex(client, CType.ACTIONGROUPS); - writeConfigToIndex(client, CType.TENANTS); + writeSingleEntryConfigToIndex(client, CType.CONFIG, config); + writeConfigToIndex(client, CType.ROLES, roles); + writeConfigToIndex(client, CType.INTERNALUSERS, internalUsers); + writeEmptyConfigToIndex(client, CType.ROLESMAPPING); + writeEmptyConfigToIndex(client, CType.ACTIONGROUPS); + writeEmptyConfigToIndex(client, CType.TENANTS); ConfigUpdateResponse configUpdateResponse = client.execute(ConfigUpdateAction.INSTANCE, new ConfigUpdateRequest(CType.lcStringValues().toArray(new String[0]))).actionGet(); @@ -579,24 +473,28 @@ private static String hash(final char[] clearTextPassword) { return hash; } - - private void writeConfigToIndex(Client client, CType configType) { - writeConfigToIndex(client, configType, NestedValueMap.createNonCloningMap()); + private void writeEmptyConfigToIndex(Client client, CType configType) { + writeConfigToIndex(client, configType, Collections.emptyMap()); } - private void writeConfigToIndex(Client client, CType configType, NestedValueMap overrides) { + private void writeConfigToIndex(Client client, CType configType, Map config) { try { + XContentBuilder builder = XContentFactory.jsonBuilder(); - NestedValueMap config = NestedValueMap.of(new NestedValueMap.Path("_meta", "type"), configType.toLCString(), - new NestedValueMap.Path("_meta", "config_version"), 2); + builder.startObject(); + builder.startObject("_meta"); + builder.field("type", configType.toLCString()); + builder.field("config_version", 2); + builder.endObject(); - if (overrides != null) { - config.overrideLeafs(overrides); + for (Map.Entry entry : config.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); } - XContentBuilder builder = XContentFactory.jsonBuilder().map(config); + builder.endObject(); + String json = Strings.toString(builder); - + log.info("Writing " + configType + ":\n" + json); client.index(new IndexRequest(indexName).id(configType.toLCString()) @@ -607,4 +505,31 @@ private void writeConfigToIndex(Client client, CType configType, NestedValueMap throw new RuntimeException("Error while initializing config for " + indexName, e); } } + + private void writeSingleEntryConfigToIndex(Client client, CType configType, ToXContentObject config) { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + + builder.startObject(); + builder.startObject("_meta"); + builder.field("type", configType.toLCString()); + builder.field("config_version", 2); + builder.endObject(); + + builder.field(configType.toLCString(), config); + + builder.endObject(); + + String json = Strings.toString(builder); + + log.info("Writing " + configType + ":\n" + json); + + client.index(new IndexRequest(indexName).id(configType.toLCString()) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE).source(configType.toLCString(), + BytesReference.fromByteBuffer(ByteBuffer.wrap(json.getBytes("utf-8"))))) + .actionGet(); + } catch (Exception e) { + throw new RuntimeException("Error while initializing config for " + indexName, e); + } + } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 50e9582199..d016a89045 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -51,7 +51,6 @@ import org.opensearch.test.framework.TestIndex; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; -import org.opensearch.test.framework.TestSecurityConfig.RoleMapping; import org.opensearch.test.framework.certificate.TestCertificates; @@ -186,7 +185,7 @@ private void start() { for (TestIndex index : this.testIndices) { index.create(client); } - } + } } catch (Exception e) { log.error("Local ES cluster start failed", e); @@ -195,12 +194,12 @@ private void start() { } private void initSecurityIndex(TestSecurityConfig testSecurityConfig) { - log.info("Initializing OpenSearch Security index"); + log.info("Initializing OpenSearch Security index"); try(Client client = new ContextHeaderDecoratorClient(this.getInternalNodeClient(), Map.of(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER , "true"))) { testSecurityConfig.initIndex(client); } } - + public static class Builder { private final Settings.Builder nodeOverrideSettingsBuilder = Settings.builder(); @@ -212,7 +211,7 @@ public static class Builder { private TestSecurityConfig testSecurityConfig = new TestSecurityConfig(); private String clusterName = "local_cluster"; private TestCertificates testCertificates; - + public Builder() { this.testCertificates = new TestCertificates(); } @@ -235,8 +234,8 @@ public Builder singleNode() { return this; } - public Builder sgConfig(TestSecurityConfig testSgConfig) { - this.testSecurityConfig = testSgConfig; + public Builder config(TestSecurityConfig testSecurityConfig) { + this.testSecurityConfig = testSecurityConfig; return this; } @@ -271,7 +270,7 @@ public Builder indices(TestIndex... indices) { this.testIndices.addAll(Arrays.asList(indices)); return this; } - + public Builder users(TestSecurityConfig.User... users) { for (TestSecurityConfig.User user : users) { testSecurityConfig.user(user); @@ -284,16 +283,6 @@ public Builder roles(Role... roles) { return this; } - public Builder roleMapping(RoleMapping... mappings) { - testSecurityConfig.roleMapping(mappings); - return this; - } - - public Builder roleToRoleMapping(Role role, String... backendRoles) { - testSecurityConfig.roleToRoleMapping(role, backendRoles); - return this; - } - public Builder authc(TestSecurityConfig.AuthcDomain authc) { testSecurityConfig.authc(authc); return this; diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java deleted file mode 100644 index caa5cb26f9..0000000000 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/NestedValueMap.java +++ /dev/null @@ -1,518 +0,0 @@ -/* -* Copyright 2021-2022 floragunn GmbH -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -*/ - -/* -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -* -* Modifications Copyright OpenSearch Contributors. See -* GitHub history for details. -*/ - -package org.opensearch.test.framework.cluster; - -import java.io.File; -import java.lang.reflect.Array; -import java.net.URI; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - -import com.google.common.collect.MapMaker; - -public class NestedValueMap extends HashMap { - - private static final long serialVersionUID = 2953312818482932741L; - - private Map originalToCloneMap; - private final boolean cloneWhilePut; - private boolean writable = true; - - public NestedValueMap() { - originalToCloneMap = new MapMaker().weakKeys().makeMap(); - cloneWhilePut = true; - } - - public NestedValueMap(int initialCapacity) { - super(initialCapacity); - originalToCloneMap = new MapMaker().weakKeys().makeMap(); - cloneWhilePut = true; - } - - NestedValueMap(Map originalToCloneMap, boolean cloneWhilePut) { - this.originalToCloneMap = originalToCloneMap; - this.cloneWhilePut = cloneWhilePut; - } - - NestedValueMap(int initialCapacity, Map originalToCloneMap, boolean cloneWhilePut) { - super(initialCapacity); - this.originalToCloneMap = originalToCloneMap; - this.cloneWhilePut = cloneWhilePut; - } - - @Override - public NestedValueMap clone() { - NestedValueMap result = new NestedValueMap(Math.max(this.size(), 10), - this.originalToCloneMap != null ? new MapMaker().weakKeys().makeMap() : null, this.cloneWhilePut); - - result.putAll(this); - - return result; - } - - public NestedValueMap without(String... keys) { - NestedValueMap result = new NestedValueMap(Math.max(this.size(), 10), - this.originalToCloneMap != null ? new MapMaker().weakKeys().makeMap() : null, this.cloneWhilePut); - - Set withoutKeySet = new HashSet<>(Arrays.asList(keys)); - - for (Map.Entry entry : this.entrySet()) { - if (!withoutKeySet.contains(entry.getKey())) { - result.put(entry.getKey(), entry.getValue()); - } - } - - return result; - } - - public static NestedValueMap copy(Map data) { - NestedValueMap result = new NestedValueMap(data.size()); - - result.putAllFromAnyMap(data); - - return result; - } - - public static NestedValueMap copy(Object data) { - if (data instanceof Map) { - return copy((Map) data); - } else { - NestedValueMap result = new NestedValueMap(); - result.put("_value", data); - return result; - } - } - - public static NestedValueMap createNonCloningMap() { - return new NestedValueMap(null, false); - } - - public static NestedValueMap createUnmodifieableMap(Map data) { - NestedValueMap result = new NestedValueMap(data.size()); - - result.putAllFromAnyMap(data); - result.seal(); - - return result; - } - -// public static NestedValueMap fromJsonString(String jsonString) throws IOException, DocumentParseException, UnexpectedDocumentStructureException { -// return NestedValueMap.copy(DocReader.json().readObject(jsonString)); -// } -// -// public static NestedValueMap fromYaml(String yamlString) throws IOException, DocumentParseException { -// return NestedValueMap.copy(DocReader.yaml().read(yamlString)); -// } -// -// public static NestedValueMap fromYaml(InputStream inputSteam) throws DocumentParseException, IOException { -// return NestedValueMap.copy(DocReader.yaml().read(inputSteam)); -// } - - public static NestedValueMap of(String key1, Object value1) { - NestedValueMap result = new NestedValueMap(1); - result.put(key1, value1); - return result; - } - - public static NestedValueMap of(String key1, Object value1, String key2, Object value2) { - NestedValueMap result = new NestedValueMap(2); - result.put(key1, value1); - result.put(key2, value2); - return result; - } - - public static NestedValueMap of(String key1, Object value1, String key2, Object value2, String key3, Object value3) { - NestedValueMap result = new NestedValueMap(3); - result.put(key1, value1); - result.put(key2, value2); - result.put(key3, value3); - - return result; - } - - public static NestedValueMap of(String key1, Object value1, String key2, Object value2, String key3, Object value3, Object... furtherEntries) { - NestedValueMap result = new NestedValueMap(3 + furtherEntries.length); - result.put(key1, value1); - result.put(key2, value2); - result.put(key3, value3); - - for (int i = 0; i < furtherEntries.length - 1; i += 2) { - result.put(String.valueOf(furtherEntries[i]), furtherEntries[i + 1]); - } - - return result; - } - - public static NestedValueMap of(Path key1, Object value1) { - NestedValueMap result = new NestedValueMap(1); - result.put(key1, value1); - return result; - } - - public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2) { - NestedValueMap result = new NestedValueMap(2); - result.put(key1, value1); - result.put(key2, value2); - return result; - } - - public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2, Path key3, Object value3) { - NestedValueMap result = new NestedValueMap(3); - result.put(key1, value1); - result.put(key2, value2); - result.put(key3, value3); - - return result; - } - - public static NestedValueMap of(Path key1, Object value1, Path key2, Object value2, Path key3, Object value3, Object... furtherEntries) { - NestedValueMap result = new NestedValueMap(3 + furtherEntries.length); - result.put(key1, value1); - result.put(key2, value2); - result.put(key3, value3); - - for (int i = 0; i < furtherEntries.length - 1; i += 2) { - result.put(Path.parse(String.valueOf(furtherEntries[i])), furtherEntries[i + 1]); - } - - return result; - } - - public Object put(String key, Map data) { - checkWritable(); - - Object result = this.get(key); - NestedValueMap subMap = this.getOrCreateSubMapAt(key, data.size()); - - subMap.putAllFromAnyMap(data); - return result; - } - - public void putAll(Map map) { - checkWritable(); - - for (Map.Entry entry : map.entrySet()) { - String key = String.valueOf(entry.getKey()); - put(key, entry.getValue()); - } - } - - public void putAllFromAnyMap(Map map) { - checkWritable(); - - for (Map.Entry entry : map.entrySet()) { - String key = String.valueOf(entry.getKey()); - put(key, entry.getValue()); - } - } - - public void overrideLeafs(NestedValueMap map) { - checkWritable(); - - for (Map.Entry entry : map.entrySet()) { - String key = String.valueOf(entry.getKey()); - - if (entry.getValue() instanceof NestedValueMap) { - NestedValueMap subMap = (NestedValueMap) entry.getValue(); - - getOrCreateSubMapAt(key, subMap.size()).overrideLeafs(subMap); - } else { - put(key, entry.getValue()); - } - } - } - - public Object put(String key, Object object) { - checkWritable(); - - if (object instanceof Map) { - return put(key, (Map) object); - } - - return super.put(key, deepCloneObject(object)); - } - - public void put(Path path, Object object) { - checkWritable(); - - if (path.isEmpty()) { - if (object instanceof Map) { - putAllFromAnyMap((Map) object); - } else { - throw new IllegalArgumentException("put([], " + object + "): If an empty path is given, the object must be of type map"); - } - - } else { - NestedValueMap subMap = getOrCreateSubMapAtPath(path.withoutLast()); - subMap.put(path.getLast(), object); - } - } - - public Object get(Path path) { - if (path.isEmpty()) { - return this; - } else if (path.length() == 1) { - return this.get(path.getFirst()); - } else { - Object subObject = this.get(path.getFirst()); - - if (subObject instanceof NestedValueMap) { - return ((NestedValueMap) subObject).get(path.withoutFirst()); - } else { - return null; - } - } - } - - public void seal() { - if (!this.writable) { - return; - } - - this.writable = false; - this.originalToCloneMap = null; - - for (Object value : this.values()) { - if (value instanceof NestedValueMap) { - NestedValueMap subMap = (NestedValueMap) value; - subMap.seal(); - } else if (value instanceof Iterable) { - for (Object subValue : ((Iterable) value)) { - if (subValue instanceof NestedValueMap) { - NestedValueMap subMap = (NestedValueMap) subValue; - subMap.seal(); - } - } - } - } - } - -// public String toJsonString() { -// return DocWriter.json().writeAsString(this); -// } -// -// public String toYamlString() { -// return DocWriter.yaml().writeAsString(this); -// } - - private Object deepCloneObject(Object object) { - if (!cloneWhilePut || object == null || isImmutable(object)) { - return object; - } - - Object clone = this.originalToCloneMap.get(object); - - if (clone != null) { - return clone; - } - - if (object instanceof Set) { - Set set = (Set) object; - Set copy = new HashSet<>(set.size()); - this.originalToCloneMap.put(object, copy); - - for (Object element : set) { - copy.add(deepCloneObject(element)); - } - - return copy; - } else if (object instanceof Map) { - Map map = (Map) object; - NestedValueMap copy = new NestedValueMap(map.size(), this.originalToCloneMap, this.cloneWhilePut); - this.originalToCloneMap.put(object, copy); - - for (Map.Entry entry : map.entrySet()) { - copy.put((String) deepCloneObject(String.valueOf(entry.getKey())), deepCloneObject(entry.getValue())); - } - - return copy; - } else if (object instanceof Collection) { - Collection collection = (Collection) object; - ArrayList copy = new ArrayList<>(collection.size()); - this.originalToCloneMap.put(object, copy); - - for (Object element : collection) { - copy.add(deepCloneObject(element)); - } - - return copy; - } else if (object.getClass().isArray()) { - int length = Array.getLength(object); - Object copy = Array.newInstance(object.getClass().getComponentType(), length); - this.originalToCloneMap.put(object, copy); - - for (int i = 0; i < length; i++) { - Array.set(copy, i, deepCloneObject(Array.get(object, i))); - } - - return copy; - } else { - // Hope the best - - return object; - } - } - - private boolean isImmutable(Object object) { - return object instanceof String || object instanceof Number || object instanceof Boolean || object instanceof Void || object instanceof Class - || object instanceof Character || object instanceof Enum || object instanceof File || object instanceof UUID || object instanceof URL - || object instanceof URI; - } - - private NestedValueMap getOrCreateSubMapAt(String key, int capacity) { - Object value = this.get(key); - - if (value instanceof NestedValueMap) { - return (NestedValueMap) value; - } else { - if (value instanceof Map) { - capacity = Math.max(capacity, ((Map) value).size()); - } - - NestedValueMap mapValue = new NestedValueMap(capacity, this.originalToCloneMap, this.cloneWhilePut); - - if (value instanceof Map) { - mapValue.putAllFromAnyMap((Map) value); - } - - super.put(key, mapValue); - return mapValue; - } - - } - - private NestedValueMap getOrCreateSubMapAtPath(Path path) { - if (path.isEmpty()) { - return this; - } - - String pathElement = path.getFirst(); - Path remainingPath = path.withoutFirst(); - - Object value = this.get(pathElement); - - if (value instanceof NestedValueMap) { - NestedValueMap mapValue = (NestedValueMap) value; - if (remainingPath.isEmpty()) { - return mapValue; - } else { - return mapValue.getOrCreateSubMapAtPath(remainingPath); - } - } else { - NestedValueMap mapValue = new NestedValueMap(this.originalToCloneMap, this.cloneWhilePut); - super.put(pathElement, mapValue); - - if (remainingPath.isEmpty()) { - return mapValue; - } else { - return mapValue.getOrCreateSubMapAtPath(remainingPath); - } - } - } - - private void checkWritable() { - if (!writable) { - throw new UnsupportedOperationException("Map is not writable"); - } - } - - public static class Path { - private String[] elements; - private int start; - private int end; - - public Path(String... elements) { - this.elements = elements; - this.start = 0; - this.end = elements.length; - } - - private Path(String[] elements, int start, int end) { - this.elements = elements; - this.start = start; - this.end = end; - } - - public String getFirst() { - if (this.start >= this.end) { - return null; - } - - return this.elements[start]; - } - - public String getLast() { - if (this.start >= this.end) { - return null; - } - - return this.elements[end - 1]; - } - - public Path withoutFirst() { - if (this.start >= this.end - 1) { - return new Path(null, 0, 0); - } - - return new Path(elements, start + 1, end); - } - - public Path withoutLast() { - if (this.start >= this.end - 1) { - return new Path(null, 0, 0); - } - - return new Path(elements, start, end - 1); - } - - public int length() { - return this.end - this.start; - } - - public boolean isEmpty() { - return this.start == this.end; - } - - public static Path parse(String path) { - if (path.length() == 0) { - return new Path(new String [0]); - } else { - return new Path(path.split("\\.")); - } - } - } - -} From 33f35f55ecfc033117e5608e511af452b4c5b82c Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Thu, 25 Aug 2022 11:49:54 +0200 Subject: [PATCH 39/47] Fixed imports Signed-Off-By: Nils Bandener --- .../test/framework/TestSecurityConfig.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java index 83a5685124..577914fbc5 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -31,7 +31,15 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.security.SecureRandom; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; @@ -44,7 +52,9 @@ import org.opensearch.client.Client; import org.opensearch.common.Strings; import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.xcontent.*; +import org.opensearch.common.xcontent.ToXContentObject; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.security.action.configupdate.ConfigUpdateAction; import org.opensearch.security.action.configupdate.ConfigUpdateRequest; import org.opensearch.security.action.configupdate.ConfigUpdateResponse; From f1bce59fa290fe1ef31139e6d3211a2083701b63 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 25 Aug 2022 15:56:21 +0000 Subject: [PATCH 40/47] Modify defaults to prevent warnings from being added to the test output Removes the following messages from the output: 2022-08-25 12:51:21 SUITE-SecurityRolesTests-seed#[BB25A03A663EB2F7] WARN Salt:48 - If you plan to use field masking pls configure compliance salt e1ukloTsQlOgPquJ to be a random string of 16 chars length identical on all nodes 2022-08-25 12:51:21 SUITE-SecurityRolesTests-seed#[BB25A03A663EB2F7] ERROR SinkProvider:64 - Default endpoint could not be created, auditlog will not work properly. 2022-08-25 12:51:21 SUITE-SecurityRolesTests-seed#[BB25A03A663EB2F7] WARN AuditMessageRouter:61 - No default storage available, audit log may not work properly. Please check configuration. Signed-off-by: Peter Nied --- .../test/framework/cluster/LocalOpenSearchCluster.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index 565537bd15..e9c0480536 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -468,6 +468,8 @@ private Settings getMinimalOpenSearchSettings() { .put("discovery.initial_state_timeout", "8s").putList("discovery.seed_hosts", seedHosts).put("transport.tcp.port", transportPort) .put("http.port", httpPort).put("cluster.routing.allocation.disk.threshold_enabled", false) .put("discovery.probe.connect_timeout", "10s").put("discovery.probe.handshake_timeout", "10s").put("http.cors.enabled", true) + .put("plugins.security.compliance.salt", "1234567890123456") + .put("plugins.security.audit.type", "noop") .build(); } From 33b78b3da0b4d10d16b7895f5eb2abae0fa98206 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 25 Aug 2022 15:54:18 +0000 Subject: [PATCH 41/47] Replace non-inclusive terms from configuration settings Signed-off-by: Peter Nied --- .../cluster/LocalOpenSearchCluster.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index e9c0480536..f026e0ecce 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -47,6 +47,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import com.google.common.collect.ImmutableList; import com.google.common.net.InetAddresses; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; @@ -270,7 +271,7 @@ public void waitForCluster(ClusterHealthStatus status, TimeValue timeout, int ex AdminClient adminClient = client.admin(); final ClusterHealthResponse healthResponse = adminClient.cluster().prepareHealth().setWaitForStatus(status).setTimeout(timeout) - .setMasterNodeTimeout(timeout).setWaitForNodes("" + expectedNodeCount).execute().actionGet(); + .setClusterManagerNodeTimeout(timeout).setWaitForNodes("" + expectedNodeCount).execute().actionGet(); if (log.isDebugEnabled()) { log.debug("Current ClusterState:\n{}", Strings.toString(healthResponse)); @@ -462,9 +463,9 @@ private Settings getOpenSearchSettings() { } private Settings getMinimalOpenSearchSettings() { - return Settings.builder().put("node.name", nodeName).put("node.data", nodeSettings.dataNode).put("node.master", nodeSettings.clusterManagerNode) + return Settings.builder().put("node.name", nodeName).putList("node.roles", createNodeRolesSettings()) .put("cluster.name", clusterName).put("path.home", nodeHomeDir.toPath()).put("path.data", dataDir.toPath()) - .put("path.logs", logsDir.toPath()).putList("cluster.initial_master_nodes", initialClusterManagerHosts) + .put("path.logs", logsDir.toPath()).putList("cluster.initial_cluster_manager_nodes", initialClusterManagerHosts) .put("discovery.initial_state_timeout", "8s").putList("discovery.seed_hosts", seedHosts).put("transport.tcp.port", transportPort) .put("http.port", httpPort).put("cluster.routing.allocation.disk.threshold_enabled", false) .put("discovery.probe.connect_timeout", "10s").put("discovery.probe.handshake_timeout", "10s").put("http.cors.enabled", true) @@ -473,6 +474,17 @@ private Settings getMinimalOpenSearchSettings() { .build(); } + private List createNodeRolesSettings() { + final ImmutableList.Builder nodeRolesBuilder = ImmutableList.builder(); + if (nodeSettings.dataNode) { + nodeRolesBuilder.add("data"); + } + if (nodeSettings.clusterManagerNode) { + nodeRolesBuilder.add("cluster_manager"); + } + return nodeRolesBuilder.build(); + } + @Override public String getClusterName() { return clusterName; From 674be407d9f587df36a3f29de539448edd7da194 Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Tue, 30 Aug 2022 15:16:30 +0200 Subject: [PATCH 42/47] Output of integration tests cleaned. Signed-off-by: Lukasz Soszynski --- build.gradle | 4 ++++ .../java/org/opensearch/security/SecurityRolesTests.java | 1 - .../test/framework/cluster/LocalOpenSearchCluster.java | 1 + src/integrationTest/resources/log4j2-test.properties | 6 ++++++ .../security/securityconf/DynamicConfigFactory.java | 3 ++- 5 files changed, 13 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 9579bc29eb..eff3814bc4 100644 --- a/build.gradle +++ b/build.gradle @@ -280,6 +280,8 @@ sourceSets { task integrationTest(type: Test) { description = 'Run integration tests.' group = 'verification' + systemProperty "org.opensearch.common.logging.NodeAndClusterIdConverter.enabled", "false" + systemProperty "java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager" testClassesDirs = sourceSets.integrationTest.output.classesDirs classpath = sourceSets.integrationTest.runtimeClasspath @@ -423,6 +425,8 @@ dependencies { integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}" integrationTestImplementation 'commons-io:commons-io:2.11.0' integrationTestImplementation 'org.apache.logging.log4j:log4j-core:2.17.1' + integrationTestImplementation 'org.apache.logging.log4j:log4j-jul:2.17.1' + integrationTestImplementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.17.1' integrationTestImplementation 'org.hamcrest:hamcrest:2.2' } diff --git a/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java b/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java index c1b5549051..fc0107bafd 100644 --- a/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java +++ b/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java @@ -58,7 +58,6 @@ public class SecurityRolesTests { @Test public void testSecurityRoles() throws Exception { - try (TestRestClient client = cluster.getRestClient(USER_SR)) { HttpResponse response = client.getAuthInfo(); assertThat(response.getStatusCode(), equalTo(HttpStatus.SC_OK)); diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index f026e0ecce..351816d9eb 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -471,6 +471,7 @@ private Settings getMinimalOpenSearchSettings() { .put("discovery.probe.connect_timeout", "10s").put("discovery.probe.handshake_timeout", "10s").put("http.cors.enabled", true) .put("plugins.security.compliance.salt", "1234567890123456") .put("plugins.security.audit.type", "noop") + .put("gateway.auto_import_dangling_indices", "true") .build(); } diff --git a/src/integrationTest/resources/log4j2-test.properties b/src/integrationTest/resources/log4j2-test.properties index aba0156605..b98cc92b78 100644 --- a/src/integrationTest/resources/log4j2-test.properties +++ b/src/integrationTest/resources/log4j2-test.properties @@ -1,10 +1,16 @@ status = info name = Integration test logging configuration + + appender.console.type = Console appender.console.name = consoleLogger appender.console.layout.type = PatternLayout appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %threadName %-5p %c{1}:%L - %m%n +appender.console.filter.prerelease.type=RegexFilter +appender.console.filter.prerelease.regex=.+\\Qis a pre-release version of OpenSearch and is not suitable for production\\E +appender.console.filter.prerelease.onMatch=DENY +appender.console.filter.prerelease.onMismatch=NEUTRAL rootLogger.level = warn rootLogger.appenderRef.stdout.ref = consoleLogger diff --git a/src/main/java/org/opensearch/security/securityconf/DynamicConfigFactory.java b/src/main/java/org/opensearch/security/securityconf/DynamicConfigFactory.java index d812bd7bbb..cdf0eaf366 100644 --- a/src/main/java/org/opensearch/security/securityconf/DynamicConfigFactory.java +++ b/src/main/java/org/opensearch/security/securityconf/DynamicConfigFactory.java @@ -42,6 +42,7 @@ import org.apache.logging.log4j.Logger; import org.greenrobot.eventbus.EventBus; import org.greenrobot.eventbus.EventBusBuilder; +import org.greenrobot.eventbus.Logger.JavaLogger; import org.opensearch.client.Client; import org.opensearch.common.settings.Settings; @@ -120,7 +121,7 @@ public final static SecurityDynamicConfiguration addStatics(SecurityDynamicCo protected final Logger log = LogManager.getLogger(this.getClass()); private final ConfigurationRepository cr; private final AtomicBoolean initialized = new AtomicBoolean(); - private final EventBus eventBus = EVENT_BUS_BUILDER.build(); + private final EventBus eventBus = EVENT_BUS_BUILDER.logger(new JavaLogger(DynamicConfigFactory.class.getCanonicalName())).build(); private final Settings opensearchSettings; private final Path configPath; private final InternalAuthenticationBackend iab = new InternalAuthenticationBackend(); From 0757c080f181a82a22e27de44c6410e1d6f8324a Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Tue, 6 Sep 2022 16:12:46 +0200 Subject: [PATCH 43/47] Added API docs Signed-Off-By: Nils Bandener --- .../test/framework/TestSecurityConfig.java | 11 ++++++ .../test/framework/cluster/LocalCluster.java | 38 ++++++++++++++++++- .../cluster/LocalOpenSearchCluster.java | 6 +++ .../cluster/OpenSearchClientProvider.java | 30 ++++++++++++--- .../test/framework/cluster/PortAllocator.java | 9 ++++- .../framework/cluster/TestRestClient.java | 7 ++++ 6 files changed, 94 insertions(+), 7 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java index 577914fbc5..fc91797625 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -61,6 +61,17 @@ import org.opensearch.security.securityconf.impl.CType; import org.opensearch.test.framework.cluster.OpenSearchClientProvider.UserCredentialsHolder; +/** + * This class allows the declarative specification of the security configuration; in particular: + * + * - config.yml + * - internal_users.yml + * - roles.yml + * - roles_mapping.yml + * + * The class does the whole round-trip, i.e., the configuration is serialized to YAML/JSON and then written to + * the configuration index of the security plugin. + */ public class TestSecurityConfig { private static final Logger log = LogManager.getLogger(TestSecurityConfig.class); diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index d016a89045..2a847eac47 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -53,7 +53,14 @@ import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.certificate.TestCertificates; - +/** + * This class allows to you start and manage a local cluster in an integration test. In contrast to the + * OpenSearchIntegTestCase class, this class can be used in a composite way and allows the specification + * of the security plugin configuration. + * + * This class can be both used as a JUnit @ClassRule (preferred) or in a try-with-resources block. The latter way should + * be only sparingly used, as starting a cluster is not a particularly fast operation. + */ public class LocalCluster extends ExternalResource implements AutoCloseable, OpenSearchClientProvider { private static final Logger log = LogManager.getLogger(LocalCluster.class); @@ -145,14 +152,25 @@ public InetSocketAddress getTransportAddress() { return localOpenSearchCluster.clientNode().getTransportAddress(); } + /** + * Returns a Client object that performs cluster-internal requests. As these requests are regard as cluster-internal, + * no authentication is performed and no user-information is attached to these requests. Thus, this client should + * be only used for preparing test environments, but not as a test subject. + */ public Client getInternalNodeClient() { return localOpenSearchCluster.clientNode().getInternalNodeClient(); } + /** + * Returns a random node of this cluster. + */ public PluginAwareNode node() { return this.localOpenSearchCluster.clusterManagerNode().esNode(); } + /** + * Returns all nodes of this cluster. + */ public List nodes() { return this.localOpenSearchCluster.getNodes(); } @@ -229,11 +247,19 @@ public Builder clusterManager(ClusterManager clusterManager) { return this; } + /** + * Starts a cluster with only one node and thus saves some resources during startup. This shall be only used + * for tests where the node interactions are not relevant to the test. An example for this would be + * authentication tests, as authentication is always done on the directly connected node. + */ public Builder singleNode() { this.clusterManager = ClusterManager.SINGLENODE; return this; } + /** + * Specifies the configuration of the security plugin that shall be used by this cluster. + */ public Builder config(TestSecurityConfig testSecurityConfig) { this.testSecurityConfig = testSecurityConfig; return this; @@ -252,12 +278,19 @@ public Builder nodeSettings(Map settings) { return this; } + /** + * Adds additional plugins to the cluster + */ public Builder plugin(Class plugin) { this.plugins.add(plugin); return this; } + /** + * Specifies a remote cluster and its name. The remote cluster can be then used in Cross Cluster Search + * operations with the specified name. + */ public Builder remote(String name, LocalCluster anotherCluster) { remoteClusters.put(name, anotherCluster); @@ -266,6 +299,9 @@ public Builder remote(String name, LocalCluster anotherCluster) { return this; } + /** + * Specifies test indices that shall be created upon startup of the cluster. + */ public Builder indices(TestIndex... indices) { this.testIndices.addAll(Arrays.asList(indices)); return this; diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index 351816d9eb..67c4923685 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -74,6 +74,12 @@ import static org.opensearch.test.framework.cluster.NodeType.DATA; import static org.opensearch.test.framework.cluster.PortAllocator.TCP; +/** + * Encapsulates all the logic to start a local OpenSearch cluster - without any configuration of the security plugin. + * + * The security plugin configuration is the job of LocalCluster, which uses this class under the hood. Thus, test code + * for the security plugin should always use LocalCluster. + */ public class LocalOpenSearchCluster { static { diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java index 6142e43db8..cb89076dd5 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java @@ -49,7 +49,13 @@ import org.opensearch.security.support.PemKeyReader; import org.opensearch.test.framework.certificate.TestCertificates; - +/** + * OpenSearchClientProvider provides methods to get a REST client for an underlying cluster or node. + * + * This interface is implemented by both LocalCluster and LocalOpenSearchCluster.Node. Thus, it is possible to get a + * REST client for a whole cluster (without choosing the node it is operating on) or to get a REST client for a specific + * node. + */ public interface OpenSearchClientProvider { String getClusterName(); @@ -65,14 +71,24 @@ default URI getHttpAddressAsURI() { return URI.create("https://" + address.getHostString() + ":" + address.getPort()); } + /** + * Returns a REST client that sends requests with basic authentication for the specified User object. Optionally, + * additional HTTP headers can be specified which will be sent with each request. + * + * This method should be usually preferred. The other getRestClient() methods shall be only used for specific + * situations. + */ default TestRestClient getRestClient(UserCredentialsHolder user, Header... headers) { return getRestClient(user.getName(), user.getPassword(), headers); } - default TestRestClient getRestClient(String user, String password, String tenant) { - return getRestClient(user, password, new BasicHeader("sgtenant", tenant)); - } - + /** + * Returns a REST client that sends requests with basic authentication for the specified user name and password. Optionally, + * additional HTTP headers can be specified which will be sent with each request. + * + * Normally, you should use the method with the User object argument instead. Use this only if you need more + * control over username and password - for example, when you want to send a wrong password. + */ default TestRestClient getRestClient(String user, String password, Header... headers) { BasicHeader basicAuthHeader = getBasicAuthHeader(user, password); if (headers != null && headers.length > 0) { @@ -82,6 +98,10 @@ default TestRestClient getRestClient(String user, String password, Header... hea return getRestClient(basicAuthHeader); } + /** + * Returns a REST client. You can specify additional HTTP headers that will be sent with each request. Use this + * method to test non-basic authentication, such as JWT bearer authentication. + */ default TestRestClient getRestClient(Header... headers) { return getRestClient(Arrays.asList(headers)); } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java index 83ebddd4b4..93f3e03596 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java @@ -37,7 +37,14 @@ import org.opensearch.test.framework.cluster.SocketUtils.SocketType; - +/** + * Helper class that allows you to allocate ports. This helps with avoiding port conflicts when running tests. + * + * NOTE: This class shall be only considered as a heuristic; ports allocated by this class are just likely to be unused; + * however, there is no guarantee that these will be unused. Thus, you still need to be prepared for port-conflicts + * and retry the procedure in such a case. If you notice a port conflict, you can use the method reserve() to mark the + * port as used. + */ public class PortAllocator { public static final PortAllocator TCP = new PortAllocator(SocketType.TCP, Duration.ofSeconds(100)); diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index c32a19ca64..1fd8624a3f 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -70,6 +70,13 @@ import org.opensearch.common.xcontent.ToXContentObject; import org.opensearch.security.DefaultObjectMapper; +/** + * A OpenSearch REST client, which is tailored towards use in integration tests. Instances of this class can be + * obtained via the OpenSearchClientProvider interface, which is implemented by LocalCluster and Node. + * + * Usually, an instance of this class sends constant authentication headers which are defined when obtaining the + * instance from OpenSearchClientProvider. + */ public class TestRestClient implements AutoCloseable { private static final Logger log = LogManager.getLogger(TestRestClient.class); From 08d6500ac86dd77ff39013833f45f0f8afc8acd7 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Tue, 6 Sep 2022 16:38:14 +0200 Subject: [PATCH 44/47] Applied spotless rules Signed-Off-By: Nils Bandener --- .../test/framework/TestSecurityConfig.java | 20 +++---- .../test/framework/cluster/LocalCluster.java | 56 +++++++++---------- .../cluster/LocalOpenSearchCluster.java | 10 ++-- .../cluster/OpenSearchClientProvider.java | 42 +++++++------- .../test/framework/cluster/PortAllocator.java | 14 ++--- .../framework/cluster/TestRestClient.java | 12 ++-- 6 files changed, 77 insertions(+), 77 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java index fc91797625..f220df3eb6 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -62,16 +62,16 @@ import org.opensearch.test.framework.cluster.OpenSearchClientProvider.UserCredentialsHolder; /** - * This class allows the declarative specification of the security configuration; in particular: - * - * - config.yml - * - internal_users.yml - * - roles.yml - * - roles_mapping.yml - * - * The class does the whole round-trip, i.e., the configuration is serialized to YAML/JSON and then written to - * the configuration index of the security plugin. - */ +* This class allows the declarative specification of the security configuration; in particular: +* +* - config.yml +* - internal_users.yml +* - roles.yml +* - roles_mapping.yml +* +* The class does the whole round-trip, i.e., the configuration is serialized to YAML/JSON and then written to +* the configuration index of the security plugin. +*/ public class TestSecurityConfig { private static final Logger log = LogManager.getLogger(TestSecurityConfig.class); diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index 2a847eac47..1170a25906 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -54,13 +54,13 @@ import org.opensearch.test.framework.certificate.TestCertificates; /** - * This class allows to you start and manage a local cluster in an integration test. In contrast to the - * OpenSearchIntegTestCase class, this class can be used in a composite way and allows the specification - * of the security plugin configuration. - * - * This class can be both used as a JUnit @ClassRule (preferred) or in a try-with-resources block. The latter way should - * be only sparingly used, as starting a cluster is not a particularly fast operation. - */ +* This class allows to you start and manage a local cluster in an integration test. In contrast to the +* OpenSearchIntegTestCase class, this class can be used in a composite way and allows the specification +* of the security plugin configuration. +* +* This class can be both used as a JUnit @ClassRule (preferred) or in a try-with-resources block. The latter way should +* be only sparingly used, as starting a cluster is not a particularly fast operation. +*/ public class LocalCluster extends ExternalResource implements AutoCloseable, OpenSearchClientProvider { private static final Logger log = LogManager.getLogger(LocalCluster.class); @@ -153,24 +153,24 @@ public InetSocketAddress getTransportAddress() { } /** - * Returns a Client object that performs cluster-internal requests. As these requests are regard as cluster-internal, - * no authentication is performed and no user-information is attached to these requests. Thus, this client should - * be only used for preparing test environments, but not as a test subject. - */ + * Returns a Client object that performs cluster-internal requests. As these requests are regard as cluster-internal, + * no authentication is performed and no user-information is attached to these requests. Thus, this client should + * be only used for preparing test environments, but not as a test subject. + */ public Client getInternalNodeClient() { return localOpenSearchCluster.clientNode().getInternalNodeClient(); } /** - * Returns a random node of this cluster. - */ + * Returns a random node of this cluster. + */ public PluginAwareNode node() { return this.localOpenSearchCluster.clusterManagerNode().esNode(); } /** - * Returns all nodes of this cluster. - */ + * Returns all nodes of this cluster. + */ public List nodes() { return this.localOpenSearchCluster.getNodes(); } @@ -248,18 +248,18 @@ public Builder clusterManager(ClusterManager clusterManager) { } /** - * Starts a cluster with only one node and thus saves some resources during startup. This shall be only used - * for tests where the node interactions are not relevant to the test. An example for this would be - * authentication tests, as authentication is always done on the directly connected node. - */ + * Starts a cluster with only one node and thus saves some resources during startup. This shall be only used + * for tests where the node interactions are not relevant to the test. An example for this would be + * authentication tests, as authentication is always done on the directly connected node. + */ public Builder singleNode() { this.clusterManager = ClusterManager.SINGLENODE; return this; } /** - * Specifies the configuration of the security plugin that shall be used by this cluster. - */ + * Specifies the configuration of the security plugin that shall be used by this cluster. + */ public Builder config(TestSecurityConfig testSecurityConfig) { this.testSecurityConfig = testSecurityConfig; return this; @@ -279,8 +279,8 @@ public Builder nodeSettings(Map settings) { } /** - * Adds additional plugins to the cluster - */ + * Adds additional plugins to the cluster + */ public Builder plugin(Class plugin) { this.plugins.add(plugin); @@ -288,9 +288,9 @@ public Builder plugin(Class plugin) { } /** - * Specifies a remote cluster and its name. The remote cluster can be then used in Cross Cluster Search - * operations with the specified name. - */ + * Specifies a remote cluster and its name. The remote cluster can be then used in Cross Cluster Search + * operations with the specified name. + */ public Builder remote(String name, LocalCluster anotherCluster) { remoteClusters.put(name, anotherCluster); @@ -300,8 +300,8 @@ public Builder remote(String name, LocalCluster anotherCluster) { } /** - * Specifies test indices that shall be created upon startup of the cluster. - */ + * Specifies test indices that shall be created upon startup of the cluster. + */ public Builder indices(TestIndex... indices) { this.testIndices.addAll(Arrays.asList(indices)); return this; diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index 67c4923685..2750080225 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -75,11 +75,11 @@ import static org.opensearch.test.framework.cluster.PortAllocator.TCP; /** - * Encapsulates all the logic to start a local OpenSearch cluster - without any configuration of the security plugin. - * - * The security plugin configuration is the job of LocalCluster, which uses this class under the hood. Thus, test code - * for the security plugin should always use LocalCluster. - */ +* Encapsulates all the logic to start a local OpenSearch cluster - without any configuration of the security plugin. +* +* The security plugin configuration is the job of LocalCluster, which uses this class under the hood. Thus, test code +* for the security plugin should always use LocalCluster. +*/ public class LocalOpenSearchCluster { static { diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java index cb89076dd5..d70afbaa1a 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java @@ -50,12 +50,12 @@ import org.opensearch.test.framework.certificate.TestCertificates; /** - * OpenSearchClientProvider provides methods to get a REST client for an underlying cluster or node. - * - * This interface is implemented by both LocalCluster and LocalOpenSearchCluster.Node. Thus, it is possible to get a - * REST client for a whole cluster (without choosing the node it is operating on) or to get a REST client for a specific - * node. - */ +* OpenSearchClientProvider provides methods to get a REST client for an underlying cluster or node. +* +* This interface is implemented by both LocalCluster and LocalOpenSearchCluster.Node. Thus, it is possible to get a +* REST client for a whole cluster (without choosing the node it is operating on) or to get a REST client for a specific +* node. +*/ public interface OpenSearchClientProvider { String getClusterName(); @@ -72,23 +72,23 @@ default URI getHttpAddressAsURI() { } /** - * Returns a REST client that sends requests with basic authentication for the specified User object. Optionally, - * additional HTTP headers can be specified which will be sent with each request. - * - * This method should be usually preferred. The other getRestClient() methods shall be only used for specific - * situations. - */ + * Returns a REST client that sends requests with basic authentication for the specified User object. Optionally, + * additional HTTP headers can be specified which will be sent with each request. + * + * This method should be usually preferred. The other getRestClient() methods shall be only used for specific + * situations. + */ default TestRestClient getRestClient(UserCredentialsHolder user, Header... headers) { return getRestClient(user.getName(), user.getPassword(), headers); } /** - * Returns a REST client that sends requests with basic authentication for the specified user name and password. Optionally, - * additional HTTP headers can be specified which will be sent with each request. - * - * Normally, you should use the method with the User object argument instead. Use this only if you need more - * control over username and password - for example, when you want to send a wrong password. - */ + * Returns a REST client that sends requests with basic authentication for the specified user name and password. Optionally, + * additional HTTP headers can be specified which will be sent with each request. + * + * Normally, you should use the method with the User object argument instead. Use this only if you need more + * control over username and password - for example, when you want to send a wrong password. + */ default TestRestClient getRestClient(String user, String password, Header... headers) { BasicHeader basicAuthHeader = getBasicAuthHeader(user, password); if (headers != null && headers.length > 0) { @@ -99,9 +99,9 @@ default TestRestClient getRestClient(String user, String password, Header... hea } /** - * Returns a REST client. You can specify additional HTTP headers that will be sent with each request. Use this - * method to test non-basic authentication, such as JWT bearer authentication. - */ + * Returns a REST client. You can specify additional HTTP headers that will be sent with each request. Use this + * method to test non-basic authentication, such as JWT bearer authentication. + */ default TestRestClient getRestClient(Header... headers) { return getRestClient(Arrays.asList(headers)); } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java index 93f3e03596..ed72fae91f 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/PortAllocator.java @@ -38,13 +38,13 @@ import org.opensearch.test.framework.cluster.SocketUtils.SocketType; /** - * Helper class that allows you to allocate ports. This helps with avoiding port conflicts when running tests. - * - * NOTE: This class shall be only considered as a heuristic; ports allocated by this class are just likely to be unused; - * however, there is no guarantee that these will be unused. Thus, you still need to be prepared for port-conflicts - * and retry the procedure in such a case. If you notice a port conflict, you can use the method reserve() to mark the - * port as used. - */ +* Helper class that allows you to allocate ports. This helps with avoiding port conflicts when running tests. +* +* NOTE: This class shall be only considered as a heuristic; ports allocated by this class are just likely to be unused; +* however, there is no guarantee that these will be unused. Thus, you still need to be prepared for port-conflicts +* and retry the procedure in such a case. If you notice a port conflict, you can use the method reserve() to mark the +* port as used. +*/ public class PortAllocator { public static final PortAllocator TCP = new PortAllocator(SocketType.TCP, Duration.ofSeconds(100)); diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index 1fd8624a3f..70ad600283 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -71,12 +71,12 @@ import org.opensearch.security.DefaultObjectMapper; /** - * A OpenSearch REST client, which is tailored towards use in integration tests. Instances of this class can be - * obtained via the OpenSearchClientProvider interface, which is implemented by LocalCluster and Node. - * - * Usually, an instance of this class sends constant authentication headers which are defined when obtaining the - * instance from OpenSearchClientProvider. - */ +* A OpenSearch REST client, which is tailored towards use in integration tests. Instances of this class can be +* obtained via the OpenSearchClientProvider interface, which is implemented by LocalCluster and Node. +* +* Usually, an instance of this class sends constant authentication headers which are defined when obtaining the +* instance from OpenSearchClientProvider. +*/ public class TestRestClient implements AutoCloseable { private static final Logger log = LogManager.getLogger(TestRestClient.class); From 67b5446ff05eff692d73d652aea80f4d4326f3cc Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Tue, 13 Sep 2022 17:57:35 +0200 Subject: [PATCH 45/47] A temporary solution to limit the number of warning messages caused by class NodeAndClusterIdConverter during tests execution Signed-off-by: Lukasz Soszynski --- build.gradle | 3 +- .../logging/NodeAndClusterIdConverter.java | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 src/integrationTest/java/org/opensearch/common/logging/NodeAndClusterIdConverter.java diff --git a/build.gradle b/build.gradle index eff3814bc4..726dd06d6f 100644 --- a/build.gradle +++ b/build.gradle @@ -263,9 +263,9 @@ configurations { sourceSets { integrationTest { java { + srcDir file ('src/integrationTest/java') compileClasspath += sourceSets.main.output runtimeClasspath += sourceSets.main.output - srcDir file ('src/integrationTest/java') } resources { srcDir file('src/integrationTest/resources') @@ -280,7 +280,6 @@ sourceSets { task integrationTest(type: Test) { description = 'Run integration tests.' group = 'verification' - systemProperty "org.opensearch.common.logging.NodeAndClusterIdConverter.enabled", "false" systemProperty "java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager" testClassesDirs = sourceSets.integrationTest.output.classesDirs classpath = sourceSets.integrationTest.runtimeClasspath diff --git a/src/integrationTest/java/org/opensearch/common/logging/NodeAndClusterIdConverter.java b/src/integrationTest/java/org/opensearch/common/logging/NodeAndClusterIdConverter.java new file mode 100644 index 0000000000..94242ecc28 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/common/logging/NodeAndClusterIdConverter.java @@ -0,0 +1,33 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.common.logging; + +/** +* Class uses to override OpenSearch NodeAndClusterIdConverter Log4j2 plugin in order to disable plugin and limit number of +* warn messages like "...ApplierService#updateTask][T#1] WARN ClusterApplierService:628 - failed to notify ClusterStateListener..." +* during tests execution. +* +* The class is rather a temporary solution and the real one should be developed in scope of: +* https://github.com/opensearch-project/OpenSearch/pull/4322 +*/ +import org.apache.logging.log4j.core.LogEvent; + +class NodeAndClusterIdConverter { + + + public NodeAndClusterIdConverter() { + } + + public static void setNodeIdAndClusterId(String nodeId, String clusterUUID) { + } + + public void format(LogEvent event, StringBuilder toAppendTo) { + } +} From 6e5e885fad2841bad69cbd10775be9892aaf7f00 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Tue, 13 Sep 2022 18:16:36 +0200 Subject: [PATCH 46/47] Fixed license headers Signed-Off-By: Nils Bandener --- .../opensearch/security/SecurityRolesTests.java | 15 --------------- .../framework/certificate/TestCertificates.java | 11 +++++++++++ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java b/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java index fc0107bafd..df20cded48 100644 --- a/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java +++ b/src/integrationTest/java/org/opensearch/security/SecurityRolesTests.java @@ -1,18 +1,3 @@ -/* -* Copyright 2015-2018 _floragunn_ GmbH -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - /* * SPDX-License-Identifier: Apache-2.0 * diff --git a/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java index 0f132f8a09..d810f0b32f 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java +++ b/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java @@ -15,6 +15,17 @@ * */ +/* +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +*/ + package org.opensearch.test.framework.certificate; import java.io.File; From 9fdb73ae4fa6b3e188bf0688f8ce76d935aa6fe3 Mon Sep 17 00:00:00 2001 From: Lukasz Soszynski Date: Mon, 19 Sep 2022 11:16:43 +0200 Subject: [PATCH 47/47] Correction after code review, removed some comments and one javadoc added. Signed-off-by: Lukasz Soszynski --- .../opensearch/test/framework/cluster/ClusterManager.java | 2 -- .../framework/cluster/ContextHeaderDecoratorClient.java | 3 +++ .../cluster/MinimumSecuritySettingsSupplierFactory.java | 4 +--- .../org/opensearch/test/framework/cluster/SocketUtils.java | 7 ------- .../test/framework/cluster/SocketUtilsTests.java | 4 ++-- 5 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java index b135df3620..005321b24c 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java @@ -50,8 +50,6 @@ import static org.opensearch.test.framework.cluster.NodeType.DATA; public enum ClusterManager { - //first one needs to be a cluster manager - //HUGE(new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(true, false, false), new NodeSettings(false, true,false), new NodeSettings(false, true, false)), //3 nodes (1m, 2d) DEFAULT(new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true)), diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java index f5cd4ffc2f..890de49ed7 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/ContextHeaderDecoratorClient.java @@ -22,6 +22,9 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.concurrent.ThreadContext.StoredContext; +/** +* The class adds provided headers into context before sending request via wrapped {@link Client} +*/ public class ContextHeaderDecoratorClient extends FilterClient { private Map headers; diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java index 263ec65b1d..37cf69b266 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/MinimumSecuritySettingsSupplierFactory.java @@ -54,9 +54,7 @@ public NodeSettingsSupplier minimumOpenSearchSettingsSslOnly(Settings other) { private Settings.Builder minimumOpenSearchSettingsBuilder(int node, boolean sslOnly) { Settings.Builder builder = Settings.builder(); - - // TODO: At the moment the test node certificates have an OID set, so we do not need to - // specify any node_dns here. Once we make generating and specifying + builder.put("plugins.security.ssl.transport.pemtrustedcas_filepath", testCertificates.getRootCertificate().getAbsolutePath()); builder.put("plugins.security.ssl.transport.pemcert_filepath", testCertificates.getNodeCertificate(node).getAbsolutePath()); builder.put("plugins.security.ssl.transport.pemkey_filepath", testCertificates.getNodeKey(node).getAbsolutePath()); diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtils.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtils.java index 273e71026e..92ec47d658 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtils.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtils.java @@ -293,13 +293,6 @@ int findAvailablePort(int minPort, int maxPort) { * @throws IllegalStateException if the requested number of available ports could not be found */ SortedSet findAvailablePorts(int numRequested, int minPort, int maxPort) { - //Assert.isTrue(minPort > 0, "'minPort' must be greater than 0"); - //Assert.isTrue(maxPort > minPort, "'maxPort' must be greater than 'minPort'"); - //Assert.isTrue(maxPort <= PORT_RANGE_MAX, "'maxPort' must be less than or equal to " + PORT_RANGE_MAX); - //Assert.isTrue(numRequested > 0, "'numRequested' must be greater than 0"); - //Assert.isTrue((maxPort - minPort) >= numRequested, - // "'numRequested' must not be greater than 'maxPort' - 'minPort'"); - SortedSet availablePorts = new TreeSet<>(); int attemptCount = 0; while ((++attemptCount <= numRequested + 100) && availablePorts.size() < numRequested) { diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java index d04bfa8024..548bedbfa6 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/SocketUtilsTests.java @@ -1,5 +1,3 @@ -package org.opensearch.test.framework.cluster; - /* * Copyright 2002-2020 the original author or authors. * @@ -27,6 +25,8 @@ * GitHub history for details. */ +package org.opensearch.test.framework.cluster; + import java.net.DatagramSocket; import java.net.InetAddress; import java.net.ServerSocket;