From d91df69c11d4411be6cb297772a006b310cd43f7 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn Date: Thu, 17 Nov 2022 10:47:08 -0800 Subject: [PATCH] Unify test clean logic (#609) * Unify wipe indices logic after tests Signed-off-by: bowenlan-amzn * Enhance wipeAllIndices function Signed-off-by: bowenlan-amzn * Customize cleanup for multi node test Signed-off-by: bowenlan-amzn Signed-off-by: bowenlan-amzn --- .github/workflows/bwc-test-workflow.yml | 4 +- .../workflows/multi-node-test-workflow.yml | 4 +- .../ManagedIndexRunner.kt | 8 +- .../action/explain/TransportExplainAction.kt | 2 +- .../indexmanagement/util/JobSchedulerUtils.kt | 2 +- .../IndexManagementIndicesIT.kt | 3 +- .../IndexManagementRestTestCase.kt | 122 +++++++++++++++++- .../IndexStateManagementSecurityBehaviorIT.kt | 4 - .../indexmanagement/ODFERestTestCase.kt | 105 --------------- .../RollupSecurityBehaviorIT.kt | 4 - .../indexmanagement/SecurityBehaviorIT.kt | 4 - .../TransformSecurityBehaviorIT.kt | 4 - ...IndexManagementBackwardsCompatibilityIT.kt | 2 - .../IndexStateManagementIntegTestCase.kt | 8 ++ .../IndexStateManagementRestTestCase.kt | 6 + .../action/ActionTimeoutIT.kt | 4 +- .../RefreshSearchAnalyzerActionIT.kt | 7 +- .../RestRefreshSearchAnalyzerActionIT.kt | 8 ++ .../rollup/RollupRestTestCase.kt | 12 +- .../rollup/runner/RollupRunnerIT.kt | 4 + .../SnapshotManagementRestTestCase.kt | 6 + .../transform/TransformRestTestCase.kt | 5 +- 22 files changed, 186 insertions(+), 142 deletions(-) diff --git a/.github/workflows/bwc-test-workflow.yml b/.github/workflows/bwc-test-workflow.yml index 839163a96..692880cdd 100644 --- a/.github/workflows/bwc-test-workflow.yml +++ b/.github/workflows/bwc-test-workflow.yml @@ -13,10 +13,10 @@ jobs: runs-on: ubuntu-latest steps: # This step uses the setup-java Github action: https://github.com/actions/setup-java - - name: Set Up JDK 11 + - name: Set Up JDK uses: actions/setup-java@v1 with: - java-version: 11 + java-version: 17 # index-management - name: Checkout Branch uses: actions/checkout@v2 diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index f2c6a433b..aaa37dc98 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -14,10 +14,10 @@ jobs: runs-on: ubuntu-latest steps: # This step uses the setup-java Github action: https://github.com/actions/setup-java - - name: Set Up JDK 11 + - name: Set Up JDK uses: actions/setup-java@v1 with: - java-version: 11 + java-version: 17 # index-management - name: Checkout Branch uses: actions/checkout@v2 diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt index 8fb3d6687..668012a25 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt @@ -403,7 +403,13 @@ object ManagedIndexRunner : @Suppress("ComplexCondition", "MaxLineLength") if (updateResult.metadataSaved && state != null && action != null && step != null && currentActionMetaData != null) { if (validationServiceEnabled) { - val validationResult = actionValidation.validate(action.type, stepContext.metadata.index) + val validationResult = withClosableContext( + IndexManagementSecurityContext( + managedIndexConfig.id, settings, threadPool.threadContext, managedIndexConfig.policy.user + ) + ) { + actionValidation.validate(action.type, stepContext.metadata.index) + } if (validationResult.validationStatus == Validate.ValidationStatus.RE_VALIDATING) { logger.warn("Validation Status is: RE_VALIDATING. The action is {}, state is {}, step is {}.\", action.type, state.name, step.name") publishErrorNotification(policy, managedIndexMetaData) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt index ac7caab1f..f3d079a53 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt @@ -383,7 +383,7 @@ class TransportExplainAction @Inject constructor( filteredIndices.add(indexNames[i]) filteredMetadata.add(indexMetadatas[i]) filteredPolicies.add(indexPolicyIDs[i]) - validationResults[i]?.let { filteredValidationResult.add(it) } + validationResults[i].let { filteredValidationResult.add(it) } enabledState[indexNames[i]]?.let { enabledStatus[indexNames[i]] = it } appliedPolicies[indexNames[i]]?.let { filteredAppliedPolicies[indexNames[i]] = it } } catch (e: OpenSearchSecurityException) { diff --git a/src/main/kotlin/org/opensearch/indexmanagement/util/JobSchedulerUtils.kt b/src/main/kotlin/org/opensearch/indexmanagement/util/JobSchedulerUtils.kt index ad6023f2d..3157f7631 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/util/JobSchedulerUtils.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/util/JobSchedulerUtils.kt @@ -13,7 +13,7 @@ import org.opensearch.jobscheduler.spi.JobExecutionContext import org.opensearch.jobscheduler.spi.LockModel import org.opensearch.jobscheduler.spi.ScheduledJobParameter -private val logger = LogManager.getLogger("JobSchedulerUtils") +private val logger = LogManager.getLogger("o.o.i.u.JobSchedulerUtils") /** * Util method to attempt to get the lock on the requested scheduled job using the backoff policy. diff --git a/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementIndicesIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementIndicesIT.kt index 34e68cfa9..123c89007 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementIndicesIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementIndicesIT.kt @@ -71,8 +71,7 @@ class IndexManagementIndicesIT : IndexStateManagementRestTestCase() { } fun `test update management index mapping with new schema version`() { - wipeAllODFEIndices() - waitForPendingTasks(adminClient()) + wipeAllIndices() assertIndexDoesNotExist(INDEX_MANAGEMENT_INDEX) val mapping = indexManagementMappings.trim().trimStart('{').trimEnd('}') diff --git a/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt index 0e37d9f15..eba7db734 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt @@ -10,20 +10,31 @@ import org.apache.http.entity.StringEntity import org.junit.AfterClass import org.junit.Before import org.junit.rules.DisableOnDebug +import org.opensearch.action.admin.cluster.node.tasks.list.ListTasksAction import org.opensearch.client.Request -import org.opensearch.client.RequestOptions import org.opensearch.client.Response import org.opensearch.client.RestClient +import org.opensearch.client.RequestOptions +import org.opensearch.client.WarningsHandler +import org.opensearch.client.ResponseException import org.opensearch.common.Strings +import org.opensearch.common.collect.Set import org.opensearch.common.io.PathUtils import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.DeprecationHandler +import org.opensearch.common.xcontent.NamedXContentRegistry +import org.opensearch.common.xcontent.XContentType import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_HIDDEN import org.opensearch.rest.RestStatus +import java.io.IOException import java.nio.file.Files +import java.util.* import javax.management.MBeanServerInvocationHandler import javax.management.ObjectName import javax.management.remote.JMXConnectorFactory import javax.management.remote.JMXServiceURL +import kotlin.collections.ArrayList +import kotlin.collections.HashSet abstract class IndexManagementRestTestCase : ODFERestTestCase() { @@ -57,7 +68,6 @@ abstract class IndexManagementRestTestCase : ODFERestTestCase() { protected val isDebuggingTest = DisableOnDebug(null).isDebugging protected val isDebuggingRemoteCluster = System.getProperty("cluster.debug", "false")!!.toBoolean() - protected val isMultiNode = System.getProperty("cluster.number_of_nodes", "1").toInt() > 1 protected val isLocalTest = clusterName() == "integTest" private fun clusterName(): String { @@ -153,7 +163,115 @@ abstract class IndexManagementRestTestCase : ODFERestTestCase() { } } + override fun preserveIndicesUponCompletion(): Boolean = true companion object { + @JvmStatic + protected val isMultiNode = System.getProperty("cluster.number_of_nodes", "1").toInt() > 1 + protected val defaultKeepIndexSet = setOf(".opendistro_security") + /** + * We override preserveIndicesUponCompletion to true and use this function to clean up indices + * Meant to be used in @After or @AfterClass of your feature test suite + */ + fun wipeAllIndices(client: RestClient = adminClient(), keepIndex: kotlin.collections.Set = defaultKeepIndexSet) { + try { + client.performRequest(Request("DELETE", "_data_stream/*")) + } catch (e: ResponseException) { + // We hit a version of ES that doesn't serialize DeleteDataStreamAction.Request#wildcardExpressionsOriginallySpecified field or + // that doesn't support data streams so it's safe to ignore + val statusCode = e.response.statusLine.statusCode + if (!Set.of(404, 405, 500).contains(statusCode)) { + throw e + } + } + + val response = client.performRequest(Request("GET", "/_cat/indices?format=json&expand_wildcards=all")) + val xContentType = XContentType.fromMediaType(response.entity.contentType.value) + xContentType.xContent().createParser( + NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + response.entity.content + ).use { parser -> + for (index in parser.list()) { + val jsonObject: Map<*, *> = index as java.util.HashMap<*, *> + val indexName: String = jsonObject["index"] as String + // .opendistro_security isn't allowed to delete from cluster + if (!keepIndex.contains(indexName)) { + val request = Request("DELETE", "/$indexName") + // TODO: remove PERMISSIVE option after moving system index access to REST API call + val options = RequestOptions.DEFAULT.toBuilder() + options.setWarningsHandler(WarningsHandler.PERMISSIVE) + request.options = options.build() + client.performRequest(request) + } + } + } + + waitFor { + if (!isMultiNode) { + waitForRunningTasks(client) + waitForPendingTasks(client) + waitForThreadPools(client) + } else { + // Multi node test is not suitable to waitFor + // We have seen long-running write task that fails the waitFor + // probably because of cluster manager - data node task not in sync + // So instead we just sleep 1s after wiping indices + Thread.sleep(1_000) + } + } + } + + @JvmStatic + @Throws(IOException::class) + protected fun waitForRunningTasks(client: RestClient) { + val runningTasks: MutableSet = runningTasks(client.performRequest(Request("GET", "/_tasks?detailed"))) + if (runningTasks.isEmpty()) { + return + } + val stillRunning = ArrayList(runningTasks) + fail("${Date()}: There are still tasks running after this test that might break subsequent tests: \n${stillRunning.joinToString("\n")}.") + } + + @Suppress("UNCHECKED_CAST") + @Throws(IOException::class) + private fun runningTasks(response: Response): MutableSet { + val runningTasks: MutableSet = HashSet() + val nodes = entityAsMap(response)["nodes"] as Map? + for ((_, value) in nodes!!) { + val nodeInfo = value as Map + val nodeTasks = nodeInfo["tasks"] as Map? + for ((_, value1) in nodeTasks!!) { + val task = value1 as Map + // Ignore the task list API - it doesn't count against us + if (task["action"] == ListTasksAction.NAME || task["action"] == ListTasksAction.NAME + "[n]") continue + // runningTasks.add(task["action"].toString() + " | " + task["description"].toString()) + runningTasks.add(task.toString()) + } + } + return runningTasks + } + + @JvmStatic + protected fun waitForThreadPools(client: RestClient) { + val response = client.performRequest(Request("GET", "/_cat/thread_pool?format=json")) + + val xContentType = XContentType.fromMediaType(response.entity.contentType.value) + xContentType.xContent().createParser( + NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + response.entity.content + ).use { parser -> + for (index in parser.list()) { + val jsonObject: Map<*, *> = index as java.util.HashMap<*, *> + val active = (jsonObject["active"] as String).toInt() + val queue = (jsonObject["queue"] as String).toInt() + val name = jsonObject["name"] + val trueActive = if (name == "management") active - 1 else active + if (trueActive > 0 || queue > 0) { + fail("Still active threadpools in cluster: $jsonObject") + } + } + } + } + internal interface IProxy { val version: String? var sessionId: String? diff --git a/src/test/kotlin/org/opensearch/indexmanagement/IndexStateManagementSecurityBehaviorIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/IndexStateManagementSecurityBehaviorIT.kt index 9fc774551..523dfe0e4 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/IndexStateManagementSecurityBehaviorIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/IndexStateManagementSecurityBehaviorIT.kt @@ -52,10 +52,6 @@ class IndexStateManagementSecurityBehaviorIT : SecurityRestTestCase() { private val testRole = "test_role" var testClient: RestClient? = null - override fun preserveIndicesUponCompletion(): Boolean { - return true - } - @Before fun setupUsersAndRoles() { updateClusterSetting(ManagedIndexSettings.JITTER.key, "0.0", false) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/ODFERestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/ODFERestTestCase.kt index 7de7e994a..fa1a77a92 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/ODFERestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/ODFERestTestCase.kt @@ -6,18 +6,9 @@ package org.opensearch.indexmanagement import org.apache.http.HttpHost -import org.junit.After -import org.opensearch.action.admin.cluster.node.tasks.list.ListTasksAction -import org.opensearch.client.Request -import org.opensearch.client.RequestOptions -import org.opensearch.client.Response import org.opensearch.client.RestClient -import org.opensearch.client.WarningsHandler import org.opensearch.common.io.PathUtils import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.DeprecationHandler -import org.opensearch.common.xcontent.NamedXContentRegistry -import org.opensearch.common.xcontent.XContentType import org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_ENABLED import org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH import org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD @@ -26,7 +17,6 @@ import org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_PEMCE import org.opensearch.commons.rest.SecureRestClientBuilder import org.opensearch.test.rest.OpenSearchRestTestCase import java.io.IOException -import java.time.Instant abstract class ODFERestTestCase : OpenSearchRestTestCase() { @@ -36,101 +26,6 @@ abstract class ODFERestTestCase : OpenSearchRestTestCase() { override fun getProtocol(): String = if (isHttps()) "https" else "http" - @Suppress("UNCHECKED_CAST") - @Throws(IOException::class) - private fun runningTasks(response: Response): MutableSet { - val runningTasks: MutableSet = HashSet() - val nodes = entityAsMap(response)["nodes"] as Map? - for ((_, value) in nodes!!) { - val nodeInfo = value as Map - val nodeTasks = nodeInfo["tasks"] as Map? - for ((_, value1) in nodeTasks!!) { - val task = value1 as Map - runningTasks.add(task["action"].toString()) - } - } - return runningTasks - } - - @After - fun waitForCleanup() { - waitFor { - waitForRunningTasks() - waitForThreadPools() - waitForPendingTasks(adminClient()) - } - } - - @Throws(IOException::class) - private fun waitForRunningTasks() { - waitFor(timeout = Instant.ofEpochSecond(5)) { - val runningTasks: MutableSet = runningTasks(adminClient().performRequest(Request("GET", "/_tasks"))) - // Ignore the task list API - it doesn't count against us - runningTasks.remove(ListTasksAction.NAME) - runningTasks.remove(ListTasksAction.NAME + "[n]") - if (runningTasks.isEmpty()) { - return@waitFor - } - val stillRunning = ArrayList(runningTasks) - fail("There are still tasks running after this test that might break subsequent tests $stillRunning.") - } - } - - private fun waitForThreadPools() { - waitFor { - val response = client().performRequest(Request("GET", "/_cat/thread_pool?format=json")) - - val xContentType = XContentType.fromMediaType(response.entity.contentType.value) - xContentType.xContent().createParser( - NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - response.entity.content - ).use { parser -> - for (index in parser.list()) { - val jsonObject: Map<*, *> = index as java.util.HashMap<*, *> - val active = (jsonObject["active"] as String).toInt() - val queue = (jsonObject["queue"] as String).toInt() - val name = jsonObject["name"] - val trueActive = if (name == "management") active - 1 else active - if (trueActive > 0 || queue > 0) { - fail("Still active threadpools in cluster: $jsonObject") - } - } - } - } - } - - open fun preserveODFEIndicesAfterTest(): Boolean = false - - @Throws(IOException::class) - open fun wipeAllODFEIndices() { - if (preserveODFEIndicesAfterTest()) return - - // Delete all data stream indices - client().performRequest(Request("DELETE", "/_data_stream/*")) - - // Delete all indices - val response = client().performRequest(Request("GET", "/_cat/indices?format=json&expand_wildcards=all")) - - val xContentType = XContentType.fromMediaType(response.entity.contentType.value) - xContentType.xContent().createParser( - NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - response.entity.content - ).use { parser -> - for (index in parser.list()) { - val jsonObject: Map<*, *> = index as java.util.HashMap<*, *> - val indexName: String = jsonObject["index"] as String - // .opendistro_security isn't allowed to delete from cluster - if (".opendistro_security" != indexName) { - val request = Request("DELETE", "/$indexName") - // TODO: remove PERMISSIVE option after moving system index access to REST API call - val options = RequestOptions.DEFAULT.toBuilder() - options.setWarningsHandler(WarningsHandler.PERMISSIVE) - request.options = options.build() - adminClient().performRequest(request) - } - } - } - } /** * Returns the REST client settings used for super-admin actions like cleaning up after the test has completed. */ diff --git a/src/test/kotlin/org/opensearch/indexmanagement/RollupSecurityBehaviorIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/RollupSecurityBehaviorIT.kt index 79d66cd3c..edcff6b18 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/RollupSecurityBehaviorIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/RollupSecurityBehaviorIT.kt @@ -43,10 +43,6 @@ class RollupSecurityBehaviorIT : SecurityRestTestCase() { private val testRole = "test_role" var testUserClient: RestClient? = null - override fun preserveIndicesUponCompletion(): Boolean { - return true - } - @Before fun setupUsersAndRoles() { updateClusterSetting(ManagedIndexSettings.JITTER.key, "0.0", false) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/SecurityBehaviorIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/SecurityBehaviorIT.kt index 8beff5aad..fc203442a 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/SecurityBehaviorIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/SecurityBehaviorIT.kt @@ -26,10 +26,6 @@ class SecurityBehaviorIT : SecurityRestTestCase() { private val john = "john" private var johnClient: RestClient? = null - override fun preserveIndicesUponCompletion(): Boolean { - return true - } - @Before fun setupUsersAndRoles() { updateClusterSetting(ManagedIndexSettings.JITTER.key, "0.0", false) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/TransformSecurityBehaviorIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/TransformSecurityBehaviorIT.kt index e46add04d..77d84323e 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/TransformSecurityBehaviorIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/TransformSecurityBehaviorIT.kt @@ -37,10 +37,6 @@ class TransformSecurityBehaviorIT : SecurityRestTestCase() { private val testRole = "test_role" var testUserClient: RestClient? = null - override fun preserveIndicesUponCompletion(): Boolean { - return true - } - @Before fun setupUsersAndRoles() { updateClusterSetting(ManagedIndexSettings.JITTER.key, "0.0", false) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/bwc/IndexManagementBackwardsCompatibilityIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/bwc/IndexManagementBackwardsCompatibilityIT.kt index 630fcca42..84344d181 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/bwc/IndexManagementBackwardsCompatibilityIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/bwc/IndexManagementBackwardsCompatibilityIT.kt @@ -36,8 +36,6 @@ class IndexManagementBackwardsCompatibilityIT : IndexManagementRestTestCase() { override fun preserveTemplatesUponCompletion(): Boolean = true - override fun preserveODFEIndicesAfterTest(): Boolean = true - override fun restClientSettings(): Settings { return Settings.builder() .put(super.restClientSettings()) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementIntegTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementIntegTestCase.kt index 2bc393693..798c8eb1a 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementIntegTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementIntegTestCase.kt @@ -7,6 +7,7 @@ package org.opensearch.indexmanagement.indexstatemanagement import org.apache.http.entity.ContentType import org.apache.http.entity.StringEntity +import org.junit.After import org.junit.Before import org.opensearch.OpenSearchParseException import org.opensearch.action.ActionRequest @@ -28,6 +29,7 @@ import org.opensearch.common.xcontent.XContentParserUtils import org.opensearch.common.xcontent.XContentType import org.opensearch.common.xcontent.json.JsonXContent import org.opensearch.indexmanagement.IndexManagementPlugin +import org.opensearch.indexmanagement.IndexManagementRestTestCase.Companion.wipeAllIndices import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.resthandler.RestExplainAction @@ -55,6 +57,12 @@ import java.time.Duration import java.time.Instant abstract class IndexStateManagementIntegTestCase : OpenSearchIntegTestCase() { + + @After + fun clearIndicesAfterEachTest() { + wipeAllIndices(getRestClient()) + } + @Before fun disableIndexStateManagementJitter() { // jitter would add a test-breaking delay to the integration tests diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index c3ddd9385..17d0c9cce 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -10,6 +10,7 @@ import org.apache.http.HttpHeaders import org.apache.http.entity.ContentType.APPLICATION_JSON import org.apache.http.entity.StringEntity import org.apache.http.message.BasicHeader +import org.junit.After import org.junit.Before import org.opensearch.OpenSearchParseException import org.opensearch.action.get.GetResponse @@ -74,6 +75,11 @@ import java.util.Locale abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() { + @After + fun clearIndicesAfterEachTest() { + wipeAllIndices() + } + val explainResponseOpendistroPolicyIdSetting = "index.opendistro.index_state_management.policy_id" val explainResponseOpenSearchPolicyIdSetting = "index.plugins.index_state_management.policy_id" diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionTimeoutIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionTimeoutIT.kt index 23b16ad0a..551132009 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionTimeoutIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionTimeoutIT.kt @@ -71,8 +71,8 @@ class ActionTimeoutIT : IndexStateManagementRestTestCase() { // https://github.com/opendistro-for-elasticsearch/index-management/issues/130 fun `test action timeout doesn't bleed over into next action`() { - val indexName = "${testIndexName}_index_1" - val policyID = "${testIndexName}_testPolicyName_1" + val indexName = "${testIndexName}_index_2" + val policyID = "${testIndexName}_testPolicyName_2" val testPolicy = """ {"policy":{"description":"Default policy","default_state":"rolloverstate","states":[ {"name":"rolloverstate","actions":[{"timeout": "5s","open":{}},{"timeout":"1s","rollover":{"min_doc_count":100}}], diff --git a/src/test/kotlin/org/opensearch/indexmanagement/refreshanalyzer/RefreshSearchAnalyzerActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/refreshanalyzer/RefreshSearchAnalyzerActionIT.kt index f3d8c243c..a16dbc48f 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/refreshanalyzer/RefreshSearchAnalyzerActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/refreshanalyzer/RefreshSearchAnalyzerActionIT.kt @@ -5,6 +5,7 @@ package org.opensearch.indexmanagement.refreshanalyzer +import org.junit.After import org.junit.Assume import org.junit.Before import org.opensearch.client.Request @@ -20,6 +21,11 @@ import java.nio.file.Files class RefreshSearchAnalyzerActionIT : IndexManagementRestTestCase() { + @After + fun clearIndicesAfterEachTest() { + wipeAllIndices() + } + @Before fun checkIfLocalCluster() { Assume.assumeTrue(isLocalTest) @@ -165,7 +171,6 @@ class RefreshSearchAnalyzerActionIT : IndexManagementRestTestCase() { } companion object { - fun writeToFile(filePath: String, contents: String) { val path = org.opensearch.common.io.PathUtils.get(filePath) Files.newBufferedWriter(path, Charset.forName("UTF-8")).use { writer -> writer.write(contents) } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/refreshanalyzer/RestRefreshSearchAnalyzerActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/refreshanalyzer/RestRefreshSearchAnalyzerActionIT.kt index c12a3df5a..f58ff4d43 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/refreshanalyzer/RestRefreshSearchAnalyzerActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/refreshanalyzer/RestRefreshSearchAnalyzerActionIT.kt @@ -5,6 +5,7 @@ package org.opensearch.indexmanagement.refreshanalyzer +import org.junit.AfterClass import org.opensearch.client.ResponseException import org.opensearch.common.settings.Settings import org.opensearch.indexmanagement.IndexManagementRestTestCase @@ -15,6 +16,13 @@ import org.opensearch.rest.RestStatus class RestRefreshSearchAnalyzerActionIT : IndexManagementRestTestCase() { + companion object { + @AfterClass + @JvmStatic fun clearIndicesAfterClass() { + wipeAllIndices() + } + } + fun `test missing indices`() { try { client().makeRequest(POST.toString(), REFRESH_SEARCH_ANALYZER_BASE_URI) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt index 3bd41b433..150d935fb 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt @@ -11,6 +11,7 @@ import org.apache.http.entity.ContentType.APPLICATION_JSON import org.apache.http.entity.StringEntity import org.apache.http.message.BasicHeader import org.junit.AfterClass +import org.junit.Before import org.opensearch.client.Response import org.opensearch.client.ResponseException import org.opensearch.client.RestClient @@ -43,12 +44,19 @@ import java.time.Instant abstract class RollupRestTestCase : IndexManagementRestTestCase() { companion object { - @AfterClass @JvmStatic fun clearIndicesAfterClassCompletion() { + @AfterClass + @JvmStatic fun clearIndicesAfterClass() { wipeAllIndices() } } - override fun preserveIndicesUponCompletion(): Boolean = true + @Before + fun setDebugLogLevel() { + client().makeRequest( + "PUT", "_cluster/settings", + StringEntity("""{"transient":{"logger.org.opensearch.indexmanagement.rollup":"DEBUG"}}""", APPLICATION_JSON) + ) + } protected fun createRollup( rollup: Rollup, diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/runner/RollupRunnerIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/runner/RollupRunnerIT.kt index 90334b431..72727fffe 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/runner/RollupRunnerIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/runner/RollupRunnerIT.kt @@ -867,6 +867,10 @@ class RollupRunnerIT : RollupRestTestCase() { var rollupMetadata = getRollupMetadata(rollupMetadataID) assertTrue("Did not process any doc during rollup", rollupMetadata.stats.documentsProcessed > 0) + // TODO Flaky: version conflict could happen here + // From log diving, it seems to be a race condition coming from RollupRunner + // (need more dive to understand rollup business logic) + // There are indexRollup happened between get and enable // restart job client().makeRequest( "PUT", diff --git a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/SnapshotManagementRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/SnapshotManagementRestTestCase.kt index c460c2439..8b694b1d0 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/SnapshotManagementRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/SnapshotManagementRestTestCase.kt @@ -10,6 +10,7 @@ import org.apache.http.HttpHeaders import org.apache.http.entity.ContentType.APPLICATION_JSON import org.apache.http.entity.StringEntity import org.apache.http.message.BasicHeader +import org.junit.After import org.junit.Before import org.opensearch.client.Response import org.opensearch.client.ResponseException @@ -36,6 +37,11 @@ import java.time.Instant.now abstract class SnapshotManagementRestTestCase : IndexManagementRestTestCase() { + @After + fun clearIndicesAfterEachTest() { + wipeAllIndices() + } + var timeout: Instant = Instant.ofEpochSecond(20) /** diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRestTestCase.kt index 286a3cd19..7e26fa263 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRestTestCase.kt @@ -40,13 +40,12 @@ import java.time.Instant abstract class TransformRestTestCase : IndexManagementRestTestCase() { companion object { - @AfterClass @JvmStatic fun clearIndicesAfterClassCompletion() { + @AfterClass + @JvmStatic fun clearIndicesAfterClass() { wipeAllIndices() } } - override fun preserveIndicesUponCompletion(): Boolean = true - protected fun createTransform( transform: Transform, transformId: String = randomAlphaOfLength(10),