diff --git a/.github/workflows/docker-security-test-workflow.yml b/.github/workflows/docker-security-test-workflow.yml new file mode 100644 index 000000000..96dd01e86 --- /dev/null +++ b/.github/workflows/docker-security-test-workflow.yml @@ -0,0 +1,83 @@ +name: Docker Security Test Workflow +on: + pull_request: + branches: + - "*" + push: + branches: + - "*" + +jobs: + test: + # This job runs on Linux + runs-on: ubuntu-latest + steps: + - name: Set Up JDK + uses: actions/setup-java@v1 + with: + java-version: 17 + - name: Checkout Branch + uses: actions/checkout@v2 + - name: Build Index Management + run: ./gradlew assemble -Dbuild.snapshot=false + - name: Pull and Run Docker + run: | + plugin=`basename $(ls build/distributions/*.zip)` + list_of_files=`ls` + list_of_all_files=`ls build/distributions/` + version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3` + plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4` + qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1` + candidate_version=`echo $plugin|awk -F- '{print $5}'| cut -d. -f 1-1` + if qualifier + then + docker_version=$version-$qualifier + else + docker_version=$version + fi + + [[ -z $candidate_version ]] && candidate_version=$qualifier && qualifier="" + + echo plugin version plugin_version qualifier candidate_version docker_version + echo "($plugin) ($version) ($plugin_version) ($qualifier) ($candidate_version) ($docker_version)" + echo $ls $list_of_all_files + + if docker pull opensearchstaging/opensearch:$docker_version + then + echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile + echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-index-management ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-index-management; fi" >> Dockerfile + echo "ADD build/distributions/$plugin /tmp/" >> Dockerfile + echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile + echo "RUN echo 'path.repo: ["/usr/share/opensearch/data/repo"]' >> /usr/share/opensearch/config/opensearch.yml" >> Dockerfile + + docker build -t opensearch-index-management:test . + echo "imagePresent=true" >> $GITHUB_ENV + else + echo "imagePresent=false" >> $GITHUB_ENV + fi + - name: Run Docker Image + if: env.imagePresent == 'true' + run: | + cd .. + docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-index-management:test + sleep 120 + - name: Run Index Management Test for security enabled test cases + if: env.imagePresent == 'true' + run: | + cluster_running=`curl -XGET https://localhost:9200/_cat/plugins -u admin:admin --insecure` + echo $cluster_running + security=`curl -XGET https://localhost:9200/_cat/plugins -u admin:admin --insecure |grep opensearch-security|wc -l` + echo $security + if [ $security -gt 0 ] + then + echo "Security plugin is available" + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity=true -Dhttps=true -Duser=admin -Dpassword=admin + else + echo "Security plugin is NOT available skipping this run as tests without security have already been run" + fi + - name: Upload failed logs + uses: actions/upload-artifact@v2 + if: failure() + with: + name: logs + path: build/testclusters/integTest-*/logs/* diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml new file mode 100644 index 000000000..7e66f057d --- /dev/null +++ b/.github/workflows/security-test-workflow.yml @@ -0,0 +1,36 @@ +name: Security test workflow +# This workflow is triggered on pull requests to main +on: + pull_request: + branches: + - "*" + push: + branches: + - "*" + +jobs: + test: + # This job runs on Linux + runs-on: ubuntu-latest + steps: + # This step uses the setup-java Github action: https://github.com/actions/setup-java + - name: Set Up JDK + uses: actions/setup-java@v1 + with: + java-version: 17 + # index-management + - name: Checkout Branch + uses: actions/checkout@v2 + - name: Start cluster with security plugin + run: | + ./gradlew run -Dsecurity=true & + sleep 120 + - name: Run integration tests + run: | + ./gradlew integTestRemote -Dsecurity=true -Dhttps=true -Dtests.rest.cluster="localhost:9200" -Dtests.cluster="localhost:9200" -Dtests.clustername="integTest" -Duser=admin -Dpassword=admin + - name: Upload failed logs + uses: actions/upload-artifact@v2 + if: failure() + with: + name: logs + path: build/testclusters/integTest-*/logs/* diff --git a/.gitignore b/.gitignore index 67a223b7a..3ea494d86 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ build/ out/ .idea/* !.idea/copyright +.vscode *.ipr *.iws .DS_Store diff --git a/build.gradle b/build.gradle index 8e8f6ca89..1afac94f8 100644 --- a/build.gradle +++ b/build.gradle @@ -3,12 +3,24 @@ * SPDX-License-Identifier: Apache-2.0 */ -import org.opensearch.gradle.testclusters.TestClusterConfiguration + +import org.opensearch.gradle.testclusters.OpenSearchCluster import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask +import javax.net.ssl.HostnameVerifier +import javax.net.ssl.HttpsURLConnection +import javax.net.ssl.SSLContext +import javax.net.ssl.SSLSession +import javax.net.ssl.TrustManager +import javax.net.ssl.X509TrustManager +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.security.GeneralSecurityException +import java.security.cert.X509Certificate import java.util.concurrent.Callable -import java.util.concurrent.TimeUnit import java.util.function.Predicate +import java.util.stream.Collectors +import java.util.concurrent.TimeUnit buildscript { @@ -21,6 +33,7 @@ buildscript { opensearch_build = version_tokens[0] + '.0' job_scheduler_no_snapshot = opensearch_build notifications_no_snapshot = opensearch_build + security_no_snapshot = opensearch_build if (buildVersionQualifier) { opensearch_build += "-${buildVersionQualifier}" job_scheduler_no_snapshot += "-${buildVersionQualifier}" @@ -34,7 +47,6 @@ buildscript { notifications_resource_folder = "src/test/resources/notifications" notifications_core_resource_folder = "src/test/resources/notifications-core" - // notification_version = System.getProperty("notification.version", opensearch_build) common_utils_version = System.getProperty("common_utils.version", opensearch_build) job_scheduler_version = System.getProperty("job_scheduler_version.version", opensearch_build) job_scheduler_build_download = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot + @@ -46,6 +58,8 @@ buildscript { '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-notifications-core-' + notifications_no_snapshot + '.zip' kotlin_version = System.getProperty("kotlin.version", "1.6.10") + + security_plugin_version = System.getProperty("security.version", security_no_snapshot) } repositories { @@ -171,6 +185,9 @@ allprojects { version = "${opensearch_build}" } +configurations { + opensearchPlugin +} dependencies { compileOnly "org.opensearch:opensearch:${opensearch_version}" compileOnly "org.opensearch:opensearch-job-scheduler-spi:${job_scheduler_version}" @@ -196,6 +213,9 @@ dependencies { attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling, Bundling.EXTERNAL)) } } + + // https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/plugin/ + opensearchPlugin "org.opensearch.plugin:opensearch-security:${security_plugin_version}@zip" } repositories { @@ -246,6 +266,24 @@ validateNebulaPom.enabled = false def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile opensearch_tmp_dir.mkdirs() +// === Setup security test === +ext.resolvePluginFile = { pluginId -> + return new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.opensearchPlugin.resolvedConfiguration.resolvedArtifacts + .find { ResolvedArtifact f -> f.name.contains(pluginId) } + .file + } + } + } + } +} +def securityPluginFile = resolvePluginFile("opensearch-security") +// This flag indicates the existence of security plugin def securityEnabled = System.getProperty("security", "false") == "true" afterEvaluate { testClusters.integTest.nodes.each { node -> @@ -270,7 +308,7 @@ afterEvaluate { node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem") node.setting("plugins.security.allow_unsafe_democertificates", "true") node.setting("plugins.security.allow_default_init_securityindex", "true") - node.setting("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,L=test,C=de") + node.setting("plugins.security.authcz.admin_dn", "\n - CN=kirk,OU=client,O=client,L=test,C=de") node.setting("plugins.security.audit.type", "internal_elasticsearch") node.setting("plugins.security.enable_snapshot_restore_privilege", "true") node.setting("plugins.security.check_snapshot_restore_write_privileges", "true") @@ -293,7 +331,6 @@ ext.getPluginResource = { download_to_folder, download_from_src -> return fileTree(download_to_folder).getSingleFile() } - File repo = file("$buildDir/testclusters/repo") def _numNodes = findProperty('numNodes') as Integer ?: 1 testClusters.integTest { @@ -370,16 +407,139 @@ testClusters.integTest { })) if (securityEnabled) { - plugin(provider({ - new RegularFile() { - @Override - File getAsFile() { fileTree("src/test/resources/security") { include "opensearch-security*" }.getSingleFile() } - } - })) + plugin(provider(securityPluginFile)) } setting 'path.repo', repo.absolutePath } +// Re-write WaitForHttpResource with updated code to support security plugin use case +class WaitForClusterYellow { + + private URL url + private String username + private String password + Set validResponseCodes = Collections.singleton(200) + + WaitForClusterYellow(String protocol, String host, int numberOfNodes) throws MalformedURLException { + this(new URL(protocol + "://" + host + "/_cluster/health?wait_for_nodes=>=" + numberOfNodes + "&wait_for_status=yellow")) + } + + WaitForClusterYellow(URL url) { + this.url = url + } + + boolean wait(int durationInMs) throws GeneralSecurityException, InterruptedException, IOException { + final long waitUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(durationInMs) + final long sleep = 100 + + IOException failure = null + while (true) { + try { + checkResource() + return true + } catch (IOException e) { + failure = e + } + if (System.nanoTime() < waitUntil) { + Thread.sleep(sleep) + } else { + throw failure + } + } + } + + void setUsername(String username) { + this.username = username + } + + void setPassword(String password) { + this.password = password + } + + void checkResource() throws IOException { + final HttpURLConnection connection = buildConnection() + connection.connect() + final Integer response = connection.getResponseCode() + if (validResponseCodes.contains(response)) { + return + } else { + throw new IOException(response + " " + connection.getResponseMessage()) + } + } + + HttpURLConnection buildConnection() throws IOException { + final HttpURLConnection connection = (HttpURLConnection) this.@url.openConnection() + + if (connection instanceof HttpsURLConnection) { + TrustManager[] trustAllCerts = [new X509TrustManager() { + X509Certificate[] getAcceptedIssuers() { + return null + } + + void checkClientTrusted(X509Certificate[] certs, String authType) { + } + + void checkServerTrusted(X509Certificate[] certs, String authType) { + } + } + ] as TrustManager[] + SSLContext sc = SSLContext.getInstance("SSL") + sc.init(null, trustAllCerts, new java.security.SecureRandom()) + connection.setSSLSocketFactory(sc.getSocketFactory()) + // Create all-trusting host name verifier + HostnameVerifier allHostsValid = new HostnameVerifier() { + boolean verify(String hostname, SSLSession session) { + return true + } + } + // Install the all-trusting host verifier + connection.setHostnameVerifier(allHostsValid) + } + + configureBasicAuth(connection) + connection.setRequestMethod("GET") + return connection + } + + void configureBasicAuth(HttpURLConnection connection) { + if (username != null) { + if (password == null) { + throw new IllegalStateException("Basic Auth user [" + username + "] has been set, but no password has been configured") + } + connection.setRequestProperty( + "Authorization", + "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes(StandardCharsets.UTF_8)) + ) + } + } + +} + +def waitForClusterSetup(OpenSearchCluster cluster, Boolean securityEnabled) { + cluster.@waitConditions.clear() + String unicastUris = cluster.nodes.stream().flatMap { node -> + node.getAllTransportPortURI().stream() + }.collect(Collectors.joining("\n")) + cluster.nodes.forEach {node -> + try { + Files.write(node.getConfigDir().resolve("unicast_hosts.txt"), unicastUris.getBytes(StandardCharsets.UTF_8)) + } catch (IOException e) { + throw new java.io.UncheckedIOException("Failed to write configuation files for " + this, e) + } + } + + Predicate pred = { + String protocol = securityEnabled ? "https" : "http" + WaitForClusterYellow wait = new WaitForClusterYellow(protocol, cluster.getFirstNode().getHttpSocketURI(), cluster.nodes.size()) + wait.setUsername(System.getProperty("user", "admin")) + wait.setPassword(System.getProperty("password", "admin")) + return wait.wait(500) + } + + cluster.@waitConditions.put("cluster health yellow", pred) + cluster.waitForAllConditions() +} + integTest { systemProperty 'tests.security.manager', 'false' systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath @@ -397,11 +557,11 @@ integTest { // There seems to be an issue when running multi node run or integ tasks with unicast_hosts // not being written, the waitForAllConditions ensures it's written getClusters().forEach { cluster -> - cluster.waitForAllConditions() + waitForClusterSetup(cluster, securityEnabled) } } - // The -Dcluster.debug option makes the cluster debuggable; this makes the tests debuggable + // The -Dcluster.debug option makes the cluster debuggable, this makes the tests debuggable if (System.getProperty("test.debug") != null) { jvmArgs '-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=8000' } @@ -423,6 +583,7 @@ integTest { filter { excludeTestsMatching "org.opensearch.indexmanagement.indexstatemanagement.action.NotificationActionIT" } + exclude 'org/opensearch/indexmanagement/indexstatemanagement/MetadataRegressionIT.class' } // TODO: raise issue in Core, this is because of the test framework @@ -439,6 +600,7 @@ task integTestRemote(type: RestIntegTestTask) { systemProperty "https", System.getProperty("https") systemProperty "user", System.getProperty("user") systemProperty "password", System.getProperty("password") + systemProperty 'buildDir', buildDir.path if (System.getProperty("tests.rest.bwcsuite") == null) { filter { @@ -649,13 +811,7 @@ run { // There seems to be an issue when running multi node run or integ tasks with unicast_hosts // not being written, the waitForAllConditions ensures it's written getClusters().forEach { cluster -> - if (securityEnabled) { - // TODO: This is a bit of a hack - LinkedHashMap> waitConditions = new LinkedHashMap<>() - cluster.waitForConditions(waitConditions, System.currentTimeMillis(), 40, TimeUnit.SECONDS, cluster) - } else { - cluster.waitForAllConditions() - } + waitForClusterSetup(cluster, securityEnabled) } } } @@ -678,7 +834,6 @@ import org.opensearch.gradle.test.RestIntegTestTask def mixedClusterTest = project.tasks.create('mixedCluster', RestIntegTestTask.class) def mixedClusterFlag = findProperty('mixed') as Boolean ?: false -println("mixed cluster flag: $mixedClusterFlag") mixedClusterTest.dependsOn(bundlePlugin) testClusters.mixedCluster { diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/index/TransportIndexRollupAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/index/TransportIndexRollupAction.kt index baad1f24b..57940f7c5 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/index/TransportIndexRollupAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/index/TransportIndexRollupAction.kt @@ -191,7 +191,7 @@ class TransportIndexRollupAction @Inject constructor( private fun validateTargetIndexName(): Boolean { val targetIndexResolvedName = RollupFieldValueExpressionResolver.resolve(request.rollup, request.rollup.targetIndex) - return targetIndexResolvedName.contains("*") == false && targetIndexResolvedName.contains("?") == false + return !targetIndexResolvedName.contains("*") && !targetIndexResolvedName.contains("?") } } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/ODFERestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/ODFERestTestCase.kt index fa1a77a92..a9c86c1af 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/ODFERestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/ODFERestTestCase.kt @@ -48,15 +48,17 @@ abstract class ODFERestTestCase : OpenSearchRestTestCase() { return when (keystore != null) { true -> { // create adminDN (super-admin) client - val uri = javaClass.classLoader.getResource("security/sample.pem").toURI() + val uri = javaClass.classLoader.getResource("security/sample.pem")?.toURI() val configPath = PathUtils.get(uri).parent.toAbsolutePath() - SecureRestClientBuilder(settings, configPath).setSocketTimeout(60000).build() + // TODO once common utils is updated in maven, we can use this method to define hosts + // SecureRestClientBuilder(settings, configPath, hosts).setSocketTimeout(5000).build() + SecureRestClientBuilder(settings, configPath).setSocketTimeout(5000).build() } false -> { // create client with passed user val userName = System.getProperty("user") val password = System.getProperty("password") - SecureRestClientBuilder(hosts, isHttps(), userName, password).setSocketTimeout(60000).build() + SecureRestClientBuilder(hosts, isHttps(), userName, password).setSocketTimeout(5000).build() } } } else { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionIT.kt index 59d408f74..b0eb1f86d 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionIT.kt @@ -281,11 +281,14 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { insertSampleData(index = firstIndex, docCount = 20, delay = 0, jsonString = "{ \"test_field\": \"${OpenSearchTestCase.randomAlphaOfLength(7000)}\" }", routing = "custom_routing") flush(firstIndex, true) forceMerge(firstIndex, "1") - val primaryShards = (cat("shards/$firstIndex?format=json&bytes=b") as List>).filter { it["prirep"] == "p" } - // TODO seeing flakyness of multiple shards over 100kb, log out shards to further debug - logger.info("cat shards result: $primaryShards") - val primaryShardsOver100KB = primaryShards.filter { (it["store"] as String).toInt() > 100000 } - assertTrue("Found multiple shards over 100kb", primaryShardsOver100KB.size == 1) + val primaryShards = waitFor { + val primaryShards = (cat("shards/$firstIndex?format=json&bytes=b") as List>).filter { it["prirep"] == "p" } + // TODO seeing flakyness of multiple shards over 100kb, log out shards to further debug + logger.info("cat shards result: $primaryShards") + val primaryShardsOver100KB = primaryShards.filter { (it["store"] as String).toInt() > 100000 } + assertTrue("Shard over 100kb is not exactly 1", primaryShardsOver100KB.size == 1) + primaryShards + } primaryShardSizeBytes = primaryShards.maxOf { (it["store"] as String).toInt() } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/refreshanalyzer/RefreshSearchAnalyzerActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/refreshanalyzer/RefreshSearchAnalyzerActionIT.kt index a16dbc48f..3ecbe9cd1 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/refreshanalyzer/RefreshSearchAnalyzerActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/refreshanalyzer/RefreshSearchAnalyzerActionIT.kt @@ -18,8 +18,10 @@ import java.io.InputStreamReader import java.nio.charset.Charset import java.nio.charset.StandardCharsets import java.nio.file.Files +import java.util.Locale class RefreshSearchAnalyzerActionIT : IndexManagementRestTestCase() { + private val testIndexName = javaClass.simpleName.lowercase(Locale.ROOT) @After fun clearIndicesAfterEachTest() { @@ -34,7 +36,7 @@ class RefreshSearchAnalyzerActionIT : IndexManagementRestTestCase() { fun `test index time analyzer`() { val buildDir = System.getProperty("buildDir") val numNodes = System.getProperty("cluster.number_of_nodes", "1").toInt() - val indexName = "testindex" + val indexName = "${testIndexName}_index_1" for (i in 0 until numNodes) { writeToFile("$buildDir/testclusters/integTest-$i/config/pacman_synonyms.txt", "hello, hola") @@ -80,7 +82,7 @@ class RefreshSearchAnalyzerActionIT : IndexManagementRestTestCase() { fun `test search time analyzer`() { val buildDir = System.getProperty("buildDir") val numNodes = System.getProperty("cluster.number_of_nodes", "1").toInt() - val indexName = "testindex" + val indexName = "${testIndexName}_index_2" for (i in 0 until numNodes) { writeToFile("$buildDir/testclusters/integTest-$i/config/pacman_synonyms.txt", "hello, hola") @@ -124,7 +126,7 @@ class RefreshSearchAnalyzerActionIT : IndexManagementRestTestCase() { } fun `test alias`() { - val indexName = "testindex" + val indexName = "${testIndexName}_index_3" val numNodes = System.getProperty("cluster.number_of_nodes", "1").toInt() val buildDir = System.getProperty("buildDir") val aliasName = "test" diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt index 150d935fb..f680a3a17 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt @@ -54,7 +54,17 @@ abstract class RollupRestTestCase : IndexManagementRestTestCase() { fun setDebugLogLevel() { client().makeRequest( "PUT", "_cluster/settings", - StringEntity("""{"transient":{"logger.org.opensearch.indexmanagement.rollup":"DEBUG"}}""", APPLICATION_JSON) + StringEntity( + """ + { + "transient": { + "logger.org.opensearch.indexmanagement.rollup":"DEBUG", + "logger.org.opensearch.jobscheduler":"DEBUG" + } + } + """.trimIndent(), + APPLICATION_JSON + ) ) } @@ -233,10 +243,16 @@ abstract class RollupRestTestCase : IndexManagementRestTestCase() { } val intervalSchedule = (update.jobSchedule as IntervalSchedule) val millis = Duration.of(intervalSchedule.interval.toLong(), intervalSchedule.unit).minusSeconds(2).toMillis() - val startTimeMillis = desiredStartTimeMillis ?: Instant.now().toEpochMilli() - millis + val startTimeMillis = desiredStartTimeMillis ?: (Instant.now().toEpochMilli() - millis) val waitForActiveShards = if (isMultiNode) "all" else "1" + // TODO flaky: Add this log to confirm this update is missed by job scheduler + // This miss is because shard remove, job scheduler deschedule on the original node and reschedule on another node + // However the shard comes back, and job scheduler deschedule on the another node and reschedule on the original node + // During this period, this update got missed + // Since from the log, this happens very fast (within 0.1~0.2s), the above cluster explain may not have the granularity to catch this. + logger.info("Update rollup start time to $startTimeMillis") val response = client().makeRequest( - "POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.id}?wait_for_active_shards=$waitForActiveShards", + "POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.id}?wait_for_active_shards=$waitForActiveShards&refresh=true", StringEntity( "{\"doc\":{\"rollup\":{\"schedule\":{\"interval\":{\"start_time\":" + "\"$startTimeMillis\"}}}}}", diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStartRollupActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStartRollupActionIT.kt index 6a35a617c..ddb008dc7 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStartRollupActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStartRollupActionIT.kt @@ -21,12 +21,9 @@ import org.opensearch.indexmanagement.rollup.randomRollup import org.opensearch.indexmanagement.waitFor import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule import org.opensearch.rest.RestStatus -import org.opensearch.test.junit.annotations.TestLogging import java.time.Instant import java.time.temporal.ChronoUnit -@TestLogging(value = "level:DEBUG", reason = "Debugging tests") -@Suppress("UNCHECKED_CAST") class RestStartRollupActionIT : RollupRestTestCase() { @Throws(Exception::class) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStopRollupActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStopRollupActionIT.kt index 118a0df74..e29b55a15 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStopRollupActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStopRollupActionIT.kt @@ -5,6 +5,7 @@ package org.opensearch.indexmanagement.rollup.resthandler +import org.junit.After import org.opensearch.client.ResponseException import org.opensearch.common.settings.Settings import org.opensearch.indexmanagement.IndexManagementIndices @@ -23,14 +24,19 @@ import org.opensearch.indexmanagement.rollup.randomRollup import org.opensearch.indexmanagement.waitFor import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule import org.opensearch.rest.RestStatus -import org.opensearch.test.junit.annotations.TestLogging import java.time.Instant import java.time.temporal.ChronoUnit -@TestLogging(value = "level:DEBUG", reason = "Debugging tests") -@Suppress("UNCHECKED_CAST") class RestStopRollupActionIT : RollupRestTestCase() { + @After + fun clearIndicesAfterEachTest() { + // Flaky could happen if config index not deleted + // metadata creation could cause the mapping to be auto set to + // a wrong type, namely, [rollup_metadata.continuous.next_window_end_time] to long + wipeAllIndices() + } + @Throws(Exception::class) fun `test stopping a started rollup`() { val rollup = createRollup(randomRollup().copy(enabled = true, jobEnabledTime = randomInstant(), metadataID = null))