diff --git a/.editorconfig b/.editorconfig index 8314e0243..a9c874b1c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,3 +7,5 @@ root = true # See https://github.com/arturbosch/detekt max_line_length=off + +disabled_rules=import-ordering \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 3571f6a0b..29eddb95e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,33 +1,31 @@ --- -name: Bug report +name: 🐛 Bug report about: Create a report to help us improve -title: "[BUG]" -labels: 'bug, untriaged, Beta' +title: '[BUG]' +labels: 'bug, untriaged' assignees: '' --- -**Describe the bug** -A clear and concise description of what the bug is. +**What is the bug?** +A clear and concise description of the bug. -**To Reproduce** +**How can one reproduce the bug?** Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error -**Expected behavior** +**What is the expected behavior?** A clear and concise description of what you expected to happen. -**Plugins** -Please list all plugins currently enabled. - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Desktop (please complete the following information):** +**What is your host/environment?** - OS: [e.g. iOS] - Version [e.g. 22] + - Plugins + +**Do you have any screenshots?** +If applicable, add screenshots to help explain your problem. -**Additional context** -Add any other context about the problem here. +**Do you have any additional context?** +Add any other context about the problem. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..a8199a104 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,7 @@ +contact_links: + - name: OpenSearch Community Support + url: https://discuss.opendistrocommunity.dev/ + about: Please ask and answer questions here. + - name: AWS/Amazon Security + url: https://aws.amazon.com/security/vulnerability-reporting/ + about: Please report security vulnerabilities here. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 5713d7aa9..6198f3383 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,19 +1,18 @@ --- name: 🎆 Feature request -about: Suggest an idea for this project -title: '' -labels: enhancement +about: Request a feature in this project +title: '[FEATURE]' +labels: 'enhancement, untriaged' assignees: '' --- +**Is your feature request related to a problem?** +A clear and concise description of what the problem is, e.g. _I'm always frustrated when [...]_ -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -**Describe the solution you'd like** +**What solution would you like?** A clear and concise description of what you want to happen. -**Describe alternatives you've considered** +**What alternatives have you considered?** A clear and concise description of any alternative solutions or features you've considered. -**Additional context** -Add any other context or screenshots about the feature request here. +**Do you have any additional context?** +Add any other context or screenshots about the feature request here. \ No newline at end of file diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 000000000..e3f96a44f --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,29 @@ + +name: Backport +on: + pull_request_target: + types: + - closed + - labeled + +jobs: + backport: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + name: Backport + steps: + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.5.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + + - name: Backport + uses: VachaShah/backport@v1.1.4 + with: + github_token: ${{ steps.github_app_token.outputs.token }} + branch_name: backport/backport-${{ github.event.number }} diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml new file mode 100644 index 000000000..f24f022b0 --- /dev/null +++ b/.github/workflows/delete_backport_branch.yml @@ -0,0 +1,15 @@ +name: Delete merged branch of the backport PRs +on: + pull_request: + types: + - closed + +jobs: + delete-branch: + runs-on: ubuntu-latest + if: startsWith(github.event.pull_request.head.ref,'backport/') + steps: + - name: Delete merged branch + uses: SvanBoxel/delete-merged-branch@main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index c0ebfa57f..8f634003a 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -16,15 +16,15 @@ jobs: runs-on: ubuntu-latest steps: # This step uses the setup-java Github action: https://github.com/actions/setup-java - - name: Set Up JDK 14 + - name: Set Up JDK 11 uses: actions/setup-java@v1 with: - java-version: 14 + java-version: 11 # index-management - name: Checkout Branch uses: actions/checkout@v2 - name: Run integration tests with multi node config - run: ./gradlew integTest -PnumNodes=3 -Dopensearch.version=1.3.0-SNAPSHOT + run: ./gradlew integTest -PnumNodes=3 - name: Upload failed logs uses: actions/upload-artifact@v2 if: failure() @@ -37,10 +37,10 @@ jobs: runs-on: ubuntu-latest steps: # This step uses the setup-java Github action: https://github.com/actions/setup-java - - name: Set Up JDK 14 + - name: Set Up JDK 11 uses: actions/setup-java@v1 with: - java-version: 14 + java-version: 11 # index-management - name: Checkout Branch uses: actions/checkout@v2 diff --git a/.github/workflows/release-workflow.yml b/.github/workflows/release-workflow.yml index 8182af1cd..b6343f9f1 100644 --- a/.github/workflows/release-workflow.yml +++ b/.github/workflows/release-workflow.yml @@ -9,7 +9,7 @@ jobs: build: strategy: matrix: - java: [14] + java: [11] # Job name name: Build Index Management with JDK ${{ matrix.java }} # This job runs on Linux diff --git a/.github/workflows/test-and-build-workflow.yml b/.github/workflows/test-and-build-workflow.yml index 03838f37a..b3dd0ded9 100644 --- a/.github/workflows/test-and-build-workflow.yml +++ b/.github/workflows/test-and-build-workflow.yml @@ -12,15 +12,16 @@ jobs: # Job name name: Build Index Management env: - BUILD_ARGS: -D"opensearch.version=1.3.0-SNAPSHOT" ${{ matrix.os_build_args }} + BUILD_ARGS: ${{ matrix.os_build_args }} WORKING_DIR: ${{ matrix.working_directory }}. strategy: # This setting says that all jobs should finish, even if one fails fail-fast: false # This starts three jobs, setting these environment variables uniquely for the different jobs matrix: + java: [11, 17] + os: [ubuntu-latest, windows-latest, macos-latest] include: - - os: ubuntu-latest - os: windows-latest os_build_args: -x integTest -x jacocoTestReport working_directory: X:\ @@ -30,10 +31,10 @@ jobs: runs-on: ${{ matrix.os }} steps: # This step uses the setup-java Github action: https://github.com/actions/setup-java - - name: Set Up JDK 14 + - name: Set Up JDK ${{ matrix.java }} uses: actions/setup-java@v1 with: - java-version: 14 + java-version: ${{ matrix.java }} # build index management - name: Checkout Branch uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index 037f7a3e3..2cbb4064d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,6 @@ out/ *.log http .project -.settings \ No newline at end of file +.settings +src/test/resources/job-scheduler/ +src/test/resources/bwc/ \ No newline at end of file diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 9ed69a1eb..61ba3f9f9 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1,7 +1,7 @@ - [Developer Guide](#developer-guide) - [Forking and Cloning](#forking-and-cloning) - [Install Prerequisites](#install-prerequisites) - - [JDK 14](#jdk-14) + - [JDK 11](#jdk-11) - [Setup](#setup) - [Build](#build) - [Building from the command line](#building-from-the-command-line) @@ -19,15 +19,17 @@ Fork this repository on GitHub, and clone locally with `git clone`. ### Install Prerequisites -#### JDK 14 +#### JDK 11 -OpenSearch components build using Java 14 at a minimum. This means you must have a JDK 14 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 14 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-14`. +OpenSearch components build using Java 11 at a minimum. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`. + +Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11). ## Setup 1. Check out this package from version control. 2. Launch Intellij IDEA, choose **Import Project**, and select the `settings.gradle` file in the root of this package. -3. To build from the command line, set `JAVA_HOME` to point to a JDK >= 14 before running `./gradlew`. +3. To build from the command line, set `JAVA_HOME` to point to a JDK >= 11 before running `./gradlew`. - Unix System 1. `export JAVA_HOME=jdk-install-dir`: Replace `jdk-install-dir` with the JAVA_HOME directory of your system. 2. `export PATH=$JAVA_HOME/bin:$PATH` @@ -52,11 +54,12 @@ However, to build the `index management` plugin project, we also use the OpenSea 4. `./gradlew integTest` launches a single node cluster with the index management (and job-scheduler) plugin installed and runs all integ tests. 5. `./gradlew integTest -PnumNodes=3` launches a multi-node cluster with the index management (and job-scheduler) plugin installed and runs all integ tests. 6. `./gradlew integTest -Dtests.class=*RestChangePolicyActionIT` runs a single integ class -7. `./gradlew integTest -Dtests.class=*RestChangePolicyActionIT -Dtests.method="test missing index"` runs a single integ test method (remember to quote the test method name if it contains spaces) +7. `./gradlew integTest -Dtests.class=*RestChangePolicyActionIT -Dtests.method="test missing index"` runs a single integ test method (remember to quote the test method name if it contains spaces) 8. `./gradlew indexmanagementBwcCluster#mixedClusterTask -Dtests.security.manager=false` launches a cluster of three nodes of bwc version of OpenSearch with index management and tests backwards compatibility by performing rolling upgrade of each node with the current version of OpenSearch with index management. 9. `./gradlew indexmanagementBwcCluster#rollingUpgradeClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with index management and tests backwards compatibility by performing rolling upgrade of each node with the current version of OpenSearch with index management. 10. `./gradlew indexmanagementBwcCluster#fullRestartClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with index management and tests backwards compatibility by performing a full restart on the cluster upgrading all the nodes with the current version of OpenSearch with index management. 11. `./gradlew bwcTestSuite -Dtests.security.manager=false` runs all the above bwc tests combined. +12. `./gradlew integTestRemote -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin` launches integration tests against a local cluster and run tests with security When launching a cluster using one of the above commands, logs are placed in `build/testclusters/integTest-0/logs`. Though the logs are teed to the console, in practices it's best to check the actual log file. @@ -101,3 +104,7 @@ Launch Intellij IDEA, choose **Import Project**, and select the `settings.gradle ### Submitting Changes See [CONTRIBUTING](CONTRIBUTING.md). + +### Backport + +- [Link to backport documentation](https://github.com/opensearch-project/opensearch-plugins/blob/main/BACKPORT.md) diff --git a/README.md b/README.md index a98c73bfa..baaacf99b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Test and Build Workflow](https://github.com/opensearch-project/index-management/workflows/Test%20and%20Build%20Workflow/badge.svg)](https://github.com/opensearch-project/index-management/actions) [![codecov](https://codecov.io/gh/opensearch-project/index-management/branch/main/graph/badge.svg)](https://codecov.io/gh/opensearch-project/index-management) +[![Roadmap](https://img.shields.io/badge/roadmap-checkout-ff69b4)](https://github.com/opensearch-project/index-management/projects/1) [![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://opensearch.org/docs/im-plugin/index/) [![Chat](https://img.shields.io/badge/chat-on%20forums-blue)](https://discuss.opendistrocommunity.dev/c/index-management/) ![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success) diff --git a/build.gradle b/build.gradle index 524d2f69d..0a01e7643 100644 --- a/build.gradle +++ b/build.gradle @@ -13,13 +13,28 @@ import java.util.function.Predicate buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "1.3.0-SNAPSHOT") - // 1.1.0 -> 1.1.0.0, and 1.1.0-SNAPSHOT -> 1.1.0.0-SNAPSHOT - opensearch_build = opensearch_version.replaceAll(/(\.\d)([^\d]*)$/, '$1.0$2') - notification_version = System.getProperty("notification.version", opensearch_build) + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + opensearch_version = System.getProperty("opensearch.version", "2.0.0-rc1-SNAPSHOT") + buildVersionQualifier = System.getProperty("build.version_qualifier", "rc1") + // 2.0.0-rc1-SNAPSHOT -> 2.0.0.0-rc1-SNAPSHOT + version_tokens = opensearch_version.tokenize('-') + opensearch_build = version_tokens[0] + '.0' + job_scheduler_no_snapshot = opensearch_build + if (buildVersionQualifier) { + opensearch_build += "-${buildVersionQualifier}" + job_scheduler_no_snapshot += "-${buildVersionQualifier}" + } + if (isSnapshot) { + opensearch_build += "-SNAPSHOT" + } + opensearch_no_snapshot = opensearch_version.replace("-SNAPSHOT","") + job_scheduler_resource_folder = "src/test/resources/job-scheduler" + // notification_version = System.getProperty("notification.version", opensearch_build) common_utils_version = System.getProperty("common_utils.version", opensearch_build) job_scheduler_version = System.getProperty("job_scheduler_version.version", opensearch_build) - kotlin_version = System.getProperty("kotlin.version", "1.4.0") + job_scheduler_build_download = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot + + '/latest/linux/x64/builds/opensearch/plugins/opensearch-job-scheduler-' + job_scheduler_no_snapshot + '.zip' + kotlin_version = System.getProperty("kotlin.version", "1.6.10") } repositories { @@ -34,7 +49,7 @@ buildscript { classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${kotlin_version}" classpath "org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}" classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.17.1" - classpath "org.jacoco:org.jacoco.agent:0.8.5" + classpath "org.jacoco:org.jacoco.agent:0.8.7" } } @@ -104,7 +119,7 @@ task ktlint(type: JavaExec, group: "verification") { description = "Check Kotlin code style." main = "com.pinterest.ktlint.Main" classpath = configurations.ktlint - args "src/**/*.kt" + args "src/**/*.kt", "spi/src/main/**/*.kt" } check.dependsOn ktlint @@ -113,7 +128,7 @@ task ktlintFormat(type: JavaExec, group: "formatting") { description = "Fix Kotlin code style deviations." main = "com.pinterest.ktlint.Main" classpath = configurations.ktlint - args "-F", "src/**/*.kt" + args "-F", "src/**/*.kt", "spi/src/main/**/*.kt" } detekt { @@ -121,43 +136,41 @@ detekt { buildUponDefaultConfig = true } -configurations.testCompile { +configurations.testImplementation { exclude module: "securemock" } ext { projectSubstitutions = [:] - isSnapshot = "true" == System.getProperty("build.snapshot", "true") licenseFile = rootProject.file('LICENSE') noticeFile = rootProject.file('NOTICE') } allprojects { group = "org.opensearch" - version = "${opensearch_version}" - "-SNAPSHOT" + ".0" - if (isSnapshot) { - version += "-SNAPSHOT" - } + version = "${opensearch_build}" } dependencies { compileOnly "org.opensearch:opensearch:${opensearch_version}" compileOnly "org.opensearch:opensearch-job-scheduler-spi:${job_scheduler_version}" - compile "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" - compile "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}" - compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${kotlin_version}" - compile 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.9' - compile "org.jetbrains:annotations:13.0" - compile "org.opensearch:notification:${notification_version}" - compile "org.opensearch:common-utils:${common_utils_version}" - compile "com.github.seancfoley:ipaddress:5.3.3" - - testCompile "org.opensearch.test:framework:${opensearch_version}" - testCompile "org.jetbrains.kotlin:kotlin-test:${kotlin_version}" + implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" + implementation "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}" + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:${kotlin_version}" + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.9' + implementation "org.jetbrains:annotations:13.0" + implementation project(path: ":${rootProject.name}-spi", configuration: 'shadow') + // implementation "org.opensearch:notification:${notification_version}" + implementation "org.opensearch:common-utils:${common_utils_version}" + implementation "com.github.seancfoley:ipaddress:5.3.3" + implementation "commons-codec:commons-codec:1.13" + + testImplementation "org.opensearch.test:framework:${opensearch_version}" + testImplementation "org.jetbrains.kotlin:kotlin-test:${kotlin_version}" testImplementation "com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0" - testCompile "org.mockito:mockito-core:3.12.4" + testImplementation "org.mockito:mockito-core:4.3.1" - add("ktlint", "com.pinterest:ktlint:0.41.0") { + add("ktlint", "com.pinterest:ktlint:0.45.1") { attributes { attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling, Bundling.EXTERNAL)) } @@ -170,11 +183,11 @@ repositories { } plugins.withId('java') { - sourceCompatibility = targetCompatibility = "1.8" + sourceCompatibility = targetCompatibility = JavaVersion.VERSION_11 } plugins.withId('org.jetbrains.kotlin.jvm') { - compileKotlin.kotlinOptions.jvmTarget = compileTestKotlin.kotlinOptions.jvmTarget = "1.8" + compileKotlin.kotlinOptions.jvmTarget = compileTestKotlin.kotlinOptions.jvmTarget = JavaVersion.VERSION_11 compileKotlin.dependsOn ktlint } @@ -227,6 +240,15 @@ test { systemProperty 'tests.security.manager', 'false' } +ext.getPluginResource = { download_to_folder, download_from_src -> + project.mkdir download_to_folder + ant.get(src: download_from_src, + dest: download_to_folder, + httpusecaches: false) + return fileTree(download_to_folder).getSingleFile() +} + + File repo = file("$buildDir/testclusters/repo") def _numNodes = findProperty('numNodes') as Integer ?: 1 testClusters.integTest { @@ -244,10 +266,23 @@ testClusters.integTest { debugPort += 1 } } - plugin(provider({ - new RegularFile() { - @Override - File getAsFile() { fileTree("src/test/resources/job-scheduler").getSingleFile() } + + plugin(provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + if (new File("$project.rootDir/$job_scheduler_resource_folder").exists()) { + project.delete(files("$project.rootDir/$job_scheduler_resource_folder")) + } + project.mkdir job_scheduler_resource_folder + ant.get(src: job_scheduler_build_download, + dest: job_scheduler_resource_folder, + httpusecaches: false) + return fileTree(job_scheduler_resource_folder).getSingleFile() + } + } } })) @@ -307,16 +342,57 @@ integTest { } } +task integTestRemote(type: RestIntegTestTask) { + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + systemProperty 'tests.security.manager', 'false' + systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath + systemProperty "https", System.getProperty("https") + systemProperty "user", System.getProperty("user") + systemProperty "password", System.getProperty("password") + + if (System.getProperty("tests.rest.bwcsuite") == null) { + filter { + excludeTestsMatching "org.opensearch.indexmanagement.bwc.*IT" + } + } + + // Only rest case can run with remote cluster + if (System.getProperty("tests.rest.cluster") != null) { + exclude 'org/opensearch/indexmanagement/indexstatemanagement/MetadataRegressionIT.class' + } + // TODO: Fix running notification test against remote cluster with security plugin installed + if (System.getProperty("https") != null) { + filter { + excludeTestsMatching "org.opensearch.indexmanagement.indexstatemanagement.action.NotificationActionIT" + } + } + // Snapshot action integration tests rely on node level setting path.repo which we can't set remotely + exclude 'org/opensearch/indexmanagement/indexstatemanagement/action/SnapshotActionIT.class' +} + String bwcVersion = "1.13.2.0" String bwcJobSchedulerVersion = "1.13.0.0" String baseName = "indexmanagementBwcCluster" String bwcFilePath = "src/test/resources/bwc/" +String bwc_js_resource_location = bwcFilePath + "job-scheduler/" + bwcJobSchedulerVersion +String bwc_im_resource_location = bwcFilePath + "indexmanagement/" + bwcVersion + +// Downloads the bwc job scheduler version +String bwc_js_download_url = "https://github.com/opendistro-for-elasticsearch/job-scheduler/releases/download/v" + + bwcJobSchedulerVersion + "/job-scheduler-artifacts.zip" +getPluginResource(bwc_js_resource_location, bwc_js_download_url) + +// Downloads the bwc index management version +String bwc_im_download_url = "https://github.com/opendistro-for-elasticsearch/index-management/releases/download/v" + + bwcVersion + "/index-management-artifacts.zip" +getPluginResource(bwc_im_resource_location, bwc_im_download_url) 2.times {i -> testClusters { "${baseName}$i" { testDistribution = "ARCHIVE" - versions = ["7.10.2", "1.3.0-SNAPSHOT"] + versions = ["7.10.2", opensearch_version] numberOfNodes = 3 plugin(provider(new Callable(){ @Override @@ -324,7 +400,7 @@ String bwcFilePath = "src/test/resources/bwc/" return new RegularFile() { @Override File getAsFile() { - return fileTree(bwcFilePath + "job-scheduler/" + bwcJobSchedulerVersion).getSingleFile() + return fileTree(bwc_js_resource_location).getSingleFile() } } } @@ -336,7 +412,7 @@ String bwcFilePath = "src/test/resources/bwc/" return new RegularFile() { @Override File getAsFile() { - return fileTree(bwcFilePath + "indexmanagement/" + bwcVersion).getSingleFile() + return fileTree(bwc_im_resource_location).getSingleFile() } } } @@ -354,6 +430,8 @@ List> plugins = [] task prepareBwcTests { dependsOn bundlePlugin doLast { + // Download the job scheduler test dependency + getPluginResource(job_scheduler_resource_folder, job_scheduler_build_download) plugins = [ provider(new Callable(){ @Override @@ -361,7 +439,7 @@ task prepareBwcTests { return new RegularFile() { @Override File getAsFile() { - return fileTree("src/test/resources/job-scheduler").getSingleFile() + return fileTree(job_scheduler_resource_folder).getSingleFile() } } } @@ -509,7 +587,7 @@ testClusters.mixedCluster { node.plugin(provider({ new RegularFile() { @Override - File getAsFile() { fileTree("src/test/resources/job-scheduler").getSingleFile() } + File getAsFile() { fileTree(job_scheduler_resource_folder).getSingleFile() } } })) @@ -523,7 +601,6 @@ testClusters.mixedCluster { } else { node.plugin(project.tasks.bundlePlugin.archiveFile) } - node.plugins.each { println("plugin in the node: ${it.get()}") } } setting 'path.repo', repo.absolutePath } diff --git a/detekt.yml b/detekt.yml index 1929d8197..47b9d163c 100644 --- a/detekt.yml +++ b/detekt.yml @@ -1,6 +1,6 @@ # TODO: Remove this before initial release, only for developmental purposes build: - maxIssues: 10 + maxIssues: 20 exceptions: TooGenericExceptionCaught: diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index e708b1c02..7454180f2 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 14e30f741..2e6e5897b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.7-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 4f906e0c8..1b6c78733 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,67 +17,101 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` +APP_BASE_NAME=${0##*/} # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +121,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" + JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the @@ -106,80 +140,95 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/release-notes/opensearch-index-management.release-notes-1.3.0.0.md b/release-notes/opensearch-index-management.release-notes-1.3.0.0.md new file mode 100644 index 000000000..dab63a5d6 --- /dev/null +++ b/release-notes/opensearch-index-management.release-notes-1.3.0.0.md @@ -0,0 +1,45 @@ +## Version 1.3.0.0 2022-03-08 + +Compatible with OpenSearch 1.3.0 + +### Features +* Continuous transforms ([#206](https://github.com/opensearch-project/index-management/pull/206)) +* Refactor IndexManagement to support custom actions ([#288](https://github.com/opensearch-project/index-management/pull/288)) + +### Enhancements +* Adds default action retries ([#212](https://github.com/opensearch-project/index-management/pull/212)) +* Adds min rollover age as a transition condition ([#215](https://github.com/opensearch-project/index-management/pull/215)) +* Adds min primary shard size rollover condition to the ISM rollover action ([#220](https://github.com/opensearch-project/index-management/pull/220)) +* Not managing indices when matched certain pattern ([#255](https://github.com/opensearch-project/index-management/pull/255/files)) +* Show applied policy in explain API ([#251](https://github.com/opensearch-project/index-management/pull/251)) + +### Bug Fixes +* Successful deletes of an index still adds history document ([#160](https://github.com/opensearch-project/index-management/pull/160)) +* Porting missing bugfixes ([#232](https://github.com/opensearch-project/index-management/pull/181)) +* ISM Template Migration ([#237](https://github.com/opensearch-project/index-management/pull/237)) +* Fixes flaky tests ([#211](https://github.com/opensearch-project/index-management/pull/211)) +* Fixes flaky rollup/transform explain IT ([#247](https://github.com/opensearch-project/index-management/pull/247)) +* Avoids restricted index warning check in blocked index pattern test ([#263](https://github.com/opensearch-project/index-management/pull/263)) +* Porting missing logic ([#240](https://github.com/opensearch-project/index-management/pull/240)) +* Fixes flaky continuous transforms test ([#276](https://github.com/opensearch-project/index-management/pull/276)) +* Porting additional missing logic ([#275](https://github.com/opensearch-project/index-management/pull/275)) +* Fixes test failures with security enabled ([#292](https://github.com/opensearch-project/index-management/pull/292)) +* Enforces extension action parsers have custom flag ([#306](https://github.com/opensearch-project/index-management/pull/306)) + +### Infrastructure +* Add support for codeowners to repo ([#195](https://github.com/opensearch-project/index-management/pull/195)) +* Adds test and build workflow for mac and windows ([#210](https://github.com/opensearch-project/index-management/pull/210)) +* Adding debug log to log the user object for all user callable transport actions ([#166](https://github.com/opensearch-project/index-management/pull/166)) +* Added ISM policy backwards compatibility test ([#181](https://github.com/opensearch-project/index-management/pull/181)) +* Add backport and auto delete workflow ([#283](https://github.com/opensearch-project/index-management/pull/283)) +* Updates integTest gradle scripts to run via remote cluster independently ([#291](https://github.com/opensearch-project/index-management/pull/291)) + +### Documentation +* Add roadmap badge in README ([#295](https://github.com/opensearch-project/index-management/pull/295)) + +### Maintenance +* Updating license headers ([#196](https://github.com/opensearch-project/index-management/pull/196)) +* Configure WhiteSource for GitHub.com ([#244](https://github.com/opensearch-project/index-management/pull/244)) +* Upgrades detekt version to 1.17.1 ([#252](https://github.com/opensearch-project/index-management/pull/252)) +* Changes integ test java version from 14 to 11 ([#284](https://github.com/opensearch-project/index-management/pull/284)) + diff --git a/settings.gradle b/settings.gradle index 4a4d049c1..be34bde5c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -4,3 +4,6 @@ */ rootProject.name = 'opensearch-index-management' + +include "spi" +project(":spi").name = rootProject.name + "-spi" diff --git a/spi/build.gradle b/spi/build.gradle new file mode 100644 index 000000000..a9806df4f --- /dev/null +++ b/spi/build.gradle @@ -0,0 +1,85 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +import org.opensearch.gradle.test.RestIntegTestTask + +plugins { + id 'com.github.johnrengelman.shadow' + id 'jacoco' +} + +apply plugin: 'opensearch.java' +apply plugin: 'opensearch.testclusters' +apply plugin: 'opensearch.java-rest-test' +apply plugin: 'kotlin' +apply plugin: 'org.jetbrains.kotlin.jvm' +apply plugin: 'org.jetbrains.kotlin.plugin.allopen' + +ext { + projectSubstitutions = [:] + licenseFile = rootProject.file('LICENSE.txt') + noticeFile = rootProject.file('NOTICE') +} + +jacoco { + toolVersion = '0.8.7' + reportsDir = file("$buildDir/JacocoReport") +} + +jacocoTestReport { + reports { + xml.enabled false + csv.enabled false + html.destination file("${buildDir}/jacoco/") + } +} +check.dependsOn jacocoTestReport + +repositories { + mavenLocal() + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } +} + +configurations.all { + if (it.state != Configuration.State.UNRESOLVED) return + resolutionStrategy { + force "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" + force "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}" + } +} + +dependencies { + compileOnly "org.opensearch:opensearch:${opensearch_version}" + compileOnly "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" + compileOnly "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}" + compileOnly "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${kotlin_version}" + compileOnly "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.9" + compileOnly "org.opensearch:common-utils:${common_utils_version}" + + testImplementation "org.opensearch.test:framework:${opensearch_version}" + testImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}" +} + +test { + doFirst { + test.classpath -= project.files(project.tasks.named('shadowJar')) + test.classpath -= project.configurations.getByName(ShadowBasePlugin.CONFIGURATION_NAME) + test.classpath += project.extensions.getByType(SourceSetContainer).getByName(SourceSet.MAIN_SOURCE_SET_NAME).runtimeClasspath + } + systemProperty 'tests.security.manager', 'false' +} + +task integTest(type: RestIntegTestTask) { + description 'Run integ test with opensearch test framework' + group 'verification' + systemProperty 'tests.security.manager', 'false' + dependsOn test +} +check.dependsOn integTest + +testClusters.javaRestTest { + testDistribution = 'INTEG_TEST' +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/IndexManagementExtension.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/IndexManagementExtension.kt new file mode 100644 index 000000000..0d2891581 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/IndexManagementExtension.kt @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi + +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser +import org.opensearch.indexmanagement.spi.indexstatemanagement.DefaultStatusChecker +import org.opensearch.indexmanagement.spi.indexstatemanagement.IndexMetadataService +import org.opensearch.indexmanagement.spi.indexstatemanagement.StatusChecker + +/** + * SPI for IndexManagement + */ +interface IndexManagementExtension { + + /** + * List of action parsers that are supported by the extension, each of the action parser will parse the policy action into the defined action. + * The ActionParser provides the ability to parse the action + */ + fun getISMActionParsers(): List + + /** + * Status checker is used by IndexManagement to check the status of the extension before executing the actions registered by the extension. + * Actions registered by the plugin can only be executed if in enabled, otherwise the action fails without retries. The status returned + * should represent if the extension is enabled or disabled, and should not represent extension health or the availability of some extension + * dependency. + */ + fun statusChecker(): StatusChecker { + return DefaultStatusChecker() + } + + /** + * Name of the extension + */ + fun getExtensionName(): String + + /** + * Not Required to override but if extension moves the index metadata outside of cluster state and requires IndexManagement to manage these + * indices provide the metadata service that can provide the index metadata for these indices. An extension need to label the metadata service + * with a type string which is used to distinguish indices in IndexManagement plugin + */ + fun getIndexMetadataService(): Map { + return mapOf() + } + + /** + * Caution: Experimental and can be removed in future + * + * If extension wants IndexManagement to determine cluster state indices UUID based on custom index setting if + * present of cluster state override this method. + */ + fun overrideClusterStateIndexUuidSetting(): String? { + return null + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Action.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Action.kt new file mode 100644 index 000000000..7b127fd96 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Action.kt @@ -0,0 +1,121 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement + +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.io.stream.Writeable +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.ToXContentObject +import org.opensearch.common.xcontent.XContentBuilder +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionRetry +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionTimeout +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import java.time.Instant + +abstract class Action( + val type: String, + val actionIndex: Int +) : ToXContentObject, Writeable { + + var configTimeout: ActionTimeout? = null + var configRetry: ActionRetry? = ActionRetry(DEFAULT_RETRIES) + var customAction: Boolean = false + + final override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject() + configTimeout?.toXContent(builder, params) + configRetry?.toXContent(builder, params) + // Include a "custom" object wrapper for custom actions to allow extensions to put arbitrary action configs in the config + // index. The EXCLUDE_CUSTOM_FIELD_PARAM is used to not include this wrapper in api responses + if (customAction && !params.paramAsBoolean(EXCLUDE_CUSTOM_FIELD_PARAM, false)) builder.startObject(CUSTOM_ACTION_FIELD) + populateAction(builder, params) + if (customAction && !params.paramAsBoolean(EXCLUDE_CUSTOM_FIELD_PARAM, false)) builder.endObject() + return builder.endObject() + } + + /** + * The implementer of Action can change this method to correctly serialize the internals of the action + * when stored internally or returned as response + */ + open fun populateAction(builder: XContentBuilder, params: ToXContent.Params) { + builder.startObject(type).endObject() + } + + final override fun writeTo(out: StreamOutput) { + out.writeString(type) + out.writeOptionalWriteable(configTimeout) + out.writeOptionalWriteable(configRetry) + populateAction(out) + } + + fun getUpdatedActionMetadata(managedIndexMetaData: ManagedIndexMetaData, stateName: String): ActionMetaData { + val stateMetaData = managedIndexMetaData.stateMetaData + val actionMetaData = managedIndexMetaData.actionMetaData + + return when { + // start a new action + stateMetaData?.name != stateName -> + ActionMetaData(this.type, Instant.now().toEpochMilli(), this.actionIndex, false, 0, 0, null) + actionMetaData?.index != this.actionIndex -> + ActionMetaData(this.type, Instant.now().toEpochMilli(), this.actionIndex, false, 0, 0, null) + // RetryAPI will reset startTime to null for actionMetaData and we'll reset it to "now" here + else -> actionMetaData.copy(startTime = actionMetaData.startTime ?: Instant.now().toEpochMilli()) + } + } + + /** + * The implementer of Action can change this method to correctly serialize the internals of the action + * when data is shared between nodes + */ + open fun populateAction(out: StreamOutput) { + out.writeInt(actionIndex) + } + + /** + * Get all the steps associated with the action + */ + abstract fun getSteps(): List + + /** + * Get the current step to execute in the action + */ + abstract fun getStepToExecute(context: StepContext): Step + + final fun isLastStep(stepName: String): Boolean = getSteps().last().name == stepName + + final fun isFirstStep(stepName: String): Boolean = getSteps().first().name == stepName + + /* + * Gets if the managedIndexMetaData reflects a state in which this action has completed successfully. Used in the + * runner when determining if the index metadata should be deleted. If the action isFinishedSuccessfully and + * deleteIndexMetadataAfterFinish is set to true, then we issue a request to delete the managedIndexConfig and its + * managedIndexMetadata. + */ + final fun isFinishedSuccessfully(managedIndexMetaData: ManagedIndexMetaData): Boolean { + val policyRetryInfo = managedIndexMetaData.policyRetryInfo + if (policyRetryInfo?.failed == true) return false + val actionMetaData = managedIndexMetaData.actionMetaData + if (actionMetaData == null || actionMetaData.failed || actionMetaData.name != this.type) return false + val stepMetaData = managedIndexMetaData.stepMetaData + if (stepMetaData == null || !isLastStep(stepMetaData.name) || stepMetaData.stepStatus != Step.StepStatus.COMPLETED) return false + return true + } + + /* + * Denotes if the index metadata in the config index should be deleted for the index this action has just + * successfully finished running on. This may be used by custom actions which delete some off-cluster index, + * and following the action's success, the managed index config and metadata need to be deleted. + */ + open fun deleteIndexMetadataAfterFinish(): Boolean = false + + companion object { + const val DEFAULT_RETRIES = 3L + const val CUSTOM_ACTION_FIELD = "custom" + const val EXCLUDE_CUSTOM_FIELD_PARAM = "exclude_custom" + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/ActionParser.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/ActionParser.kt new file mode 100644 index 000000000..292d7dcb6 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/ActionParser.kt @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.XContentParser + +abstract class ActionParser(var customAction: Boolean = false) { + + /** + * The action type parser will parse + */ + abstract fun getActionType(): String + + /** + * Deserialize Action from stream input + */ + abstract fun fromStreamInput(sin: StreamInput): Action + + /** + * Deserialize Action from xContent + */ + abstract fun fromXContent(xcp: XContentParser, index: Int): Action +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/IndexMetadataService.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/IndexMetadataService.kt new file mode 100644 index 000000000..f7506d53c --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/IndexMetadataService.kt @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement + +import org.opensearch.client.Client +import org.opensearch.cluster.service.ClusterService +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ISMIndexMetadata + +/** + * ISM by default considers all the index metadata to be part of the cluster state, + * if that doesn't hold true and indices metadata is present in some other place and + * ISM still need to manage these indices the following interface provides a mechanism + * for ISM extensions to register the metadata service for the type so ISM can get the + * index metadata for these special type of indices. + * + * ISM Rest APIs allows support for type param which determines the type of index, if there + * is a registered metadata service for the type - ISM will use the service to get the metadata + * else uses the default i.e cluster state + */ +interface IndexMetadataService { + + /** + * Returns the index metadata needed for ISM + */ + suspend fun getMetadata(indices: List, client: Client, clusterService: ClusterService): Map + + /** + * Returns all the indices metadata + */ + suspend fun getMetadataForAllIndices(client: Client, clusterService: ClusterService): Map + + /** + * Returns an optional setting path which, when set to true in the index settings, overrides a cluster level metadata write block. + */ + fun getIndexMetadataWriteOverrideSetting(): String? = null +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/StatusChecker.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/StatusChecker.kt new file mode 100644 index 000000000..d2adf9ad3 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/StatusChecker.kt @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement + +import org.opensearch.cluster.ClusterState + +interface StatusChecker { + + /** + * checks and returns the status of the extension + */ + fun check(clusterState: ClusterState): Status { + return Status.ENABLED + } +} + +enum class Status(private val value: String) { + ENABLED("enabled"), + DISABLED("disabled"); + + override fun toString(): String { + return value + } +} + +class DefaultStatusChecker : StatusChecker diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Step.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Step.kt new file mode 100644 index 000000000..1a4b2f971 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Step.kt @@ -0,0 +1,75 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement + +import org.apache.logging.log4j.Logger +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.io.stream.Writeable +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import java.time.Instant +import java.util.Locale + +abstract class Step(val name: String, val isSafeToDisableOn: Boolean = true) { + + var context: StepContext? = null + private set + + fun preExecute(logger: Logger, context: StepContext): Step { + logger.info("Executing $name for ${context.metadata.index}") + this.context = context + return this + } + + abstract suspend fun execute(): Step + + fun postExecute(logger: Logger): Step { + logger.info("Finished executing $name for ${context?.metadata?.index}") + this.context = null + return this + } + + abstract fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData + + abstract fun isIdempotent(): Boolean + + final fun getStepStartTime(metadata: ManagedIndexMetaData): Instant { + return when { + metadata.stepMetaData == null -> Instant.now() + metadata.stepMetaData.name != this.name -> Instant.now() + // The managed index metadata is a historical snapshot of the metadata and refers to what has happened from the previous + // execution, so if we ever see it as COMPLETED it means we are always going to be in a new step, this specifically + // helps with the Transition -> Transition (empty state) sequence which the above do not capture + metadata.stepMetaData.stepStatus == StepStatus.COMPLETED -> Instant.now() + else -> Instant.ofEpochMilli(metadata.stepMetaData.startTime) + } + } + + final fun getStartingStepMetaData(metadata: ManagedIndexMetaData): StepMetaData = StepMetaData(name, getStepStartTime(metadata).toEpochMilli(), StepStatus.STARTING) + + enum class StepStatus(val status: String) : Writeable { + STARTING("starting"), + CONDITION_NOT_MET("condition_not_met"), + FAILED("failed"), + COMPLETED("completed"); + + override fun toString(): String { + return status + } + + override fun writeTo(out: StreamOutput) { + out.writeString(status) + } + + companion object { + fun read(streamInput: StreamInput): StepStatus { + return valueOf(streamInput.readString().uppercase(Locale.ROOT)) + } + } + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Utils.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Utils.kt new file mode 100644 index 000000000..6854973c7 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Utils.kt @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement + +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.ToXContentFragment +import org.opensearch.common.xcontent.XContentBuilder + +// forIndex means saving to config index, distinguish from Explain and History, +// which only show meaningful partial metadata +@Suppress("ReturnCount") +fun XContentBuilder.addObject(name: String, metadata: ToXContentFragment?, params: ToXContent.Params, forIndex: Boolean = false): XContentBuilder { + if (metadata != null) return this.buildMetadata(name, metadata, params) + return if (forIndex) nullField(name) else this +} + +fun XContentBuilder.buildMetadata(name: String, metadata: ToXContentFragment, params: ToXContent.Params): XContentBuilder { + this.startObject(name) + metadata.toXContent(this, params) + this.endObject() + return this +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/ActionMetaData.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionMetaData.kt similarity index 87% rename from src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/ActionMetaData.kt rename to spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionMetaData.kt index 933abef29..9e82ba412 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/ActionMetaData.kt +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionMetaData.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata +package org.opensearch.indexmanagement.spi.indexstatemanagement.model import org.opensearch.common.Strings import org.opensearch.common.io.stream.StreamInput @@ -15,11 +15,10 @@ import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.ToXContentFragment import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.common.xcontent.XContentParserUtils import org.opensearch.common.xcontent.XContentType -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData.Companion.NAME -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData.Companion.START_TIME +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData.Companion.NAME +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData.Companion.START_TIME import java.io.ByteArrayInputStream import java.nio.charset.StandardCharsets @@ -116,20 +115,20 @@ data class ActionMetaData( var lastRetryTime: Long? = null var actionProperties: ActionProperties? = null - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { val fieldName = xcp.currentName() xcp.nextToken() when (fieldName) { NAME -> name = xcp.text() - START_TIME -> startTime = if (xcp.currentToken() == Token.VALUE_NULL) null else xcp.longValue() + START_TIME -> startTime = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) null else xcp.longValue() INDEX -> index = xcp.intValue() FAILED -> failed = xcp.booleanValue() CONSUMED_RETRIES -> consumedRetries = xcp.intValue() - LAST_RETRY_TIME -> lastRetryTime = if (xcp.currentToken() == Token.VALUE_NULL) null else xcp.longValue() + LAST_RETRY_TIME -> lastRetryTime = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) null else xcp.longValue() ActionProperties.ACTION_PROPERTIES -> - actionProperties = if (xcp.currentToken() == Token.VALUE_NULL) null else ActionProperties.parse(xcp) + actionProperties = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) null else ActionProperties.parse(xcp) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/ActionProperties.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionProperties.kt similarity index 89% rename from src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/ActionProperties.kt rename to spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionProperties.kt index 4dc511b88..175dc447d 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/ActionProperties.kt +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionProperties.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata +package org.opensearch.indexmanagement.spi.indexstatemanagement.model import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.io.stream.StreamOutput @@ -12,8 +12,7 @@ import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.ToXContentFragment import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.common.xcontent.XContentParserUtils /** Properties that will persist across steps of a single Action. Will be stored in the [ActionMetaData]. */ // TODO: Create namespaces to group properties together @@ -57,8 +56,8 @@ data class ActionProperties( var rollupId: String? = null var hasRollupFailed: Boolean? = null - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { val fieldName = xcp.currentName() xcp.nextToken() diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ActionRetry.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionRetry.kt similarity index 89% rename from src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ActionRetry.kt rename to spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionRetry.kt index be73d5483..045fdbb9b 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ActionRetry.kt +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionRetry.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.indexmanagement.indexstatemanagement.model.action +package org.opensearch.indexmanagement.spi.indexstatemanagement.model import org.apache.logging.log4j.LogManager import org.opensearch.common.io.stream.StreamInput @@ -14,9 +14,7 @@ import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.ToXContentFragment import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData +import org.opensearch.common.xcontent.XContentParserUtils import java.io.IOException import java.time.Instant import java.util.Locale @@ -67,14 +65,14 @@ data class ActionRetry( var backoff: Backoff = Backoff.EXPONENTIAL var delay: TimeValue = TimeValue.timeValueMinutes(1) - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { val fieldName = xcp.currentName() xcp.nextToken() when (fieldName) { COUNT_FIELD -> count = xcp.longValue() - BACKOFF_FIELD -> backoff = Backoff.valueOf(xcp.text().toUpperCase(Locale.ROOT)) + BACKOFF_FIELD -> backoff = Backoff.valueOf(xcp.text().uppercase(Locale.ROOT)) DELAY_FIELD -> delay = TimeValue.parseTimeValue(xcp.text(), DELAY_FIELD) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ActionTimeout.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionTimeout.kt similarity index 88% rename from src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ActionTimeout.kt rename to spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionTimeout.kt index dcd05ed59..a21055402 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ActionTimeout.kt +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionTimeout.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.indexmanagement.indexstatemanagement.model.action +package org.opensearch.indexmanagement.spi.indexstatemanagement.model import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.io.stream.StreamOutput @@ -13,7 +13,6 @@ import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.ToXContentFragment import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token import java.io.IOException data class ActionTimeout(val timeout: TimeValue) : ToXContentFragment, Writeable { @@ -38,7 +37,7 @@ data class ActionTimeout(val timeout: TimeValue) : ToXContentFragment, Writeable @JvmStatic @Throws(IOException::class) fun parse(xcp: XContentParser): ActionTimeout { - if (xcp.currentToken() == Token.VALUE_STRING) { + if (xcp.currentToken() == XContentParser.Token.VALUE_STRING) { return ActionTimeout(TimeValue.parseTimeValue(xcp.text(), TIMEOUT_FIELD)) } else { throw IllegalArgumentException("Invalid token: [${xcp.currentToken()}] for ActionTimeout") diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ISMIndexMetadata.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ISMIndexMetadata.kt new file mode 100644 index 000000000..a00681dc3 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ISMIndexMetadata.kt @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.model + +data class ISMIndexMetadata( + val indexUuid: String, + val indexCreationDate: Long, + val documentCount: Long, +) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ManagedIndexMetaData.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ManagedIndexMetaData.kt similarity index 82% rename from src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ManagedIndexMetaData.kt rename to spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ManagedIndexMetaData.kt index b3ca7a4e5..28e6c4051 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ManagedIndexMetaData.kt +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ManagedIndexMetaData.kt @@ -3,30 +3,22 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.indexmanagement.indexstatemanagement.model +package org.opensearch.indexmanagement.spi.indexstatemanagement.model import org.opensearch.common.Strings import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.io.stream.StreamOutput import org.opensearch.common.io.stream.Writeable -import org.opensearch.common.util.concurrent.ThreadContext import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.ToXContentFragment import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentFactory import org.opensearch.common.xcontent.XContentHelper import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.common.xcontent.XContentParserUtils import org.opensearch.common.xcontent.json.JsonXContent -import org.opensearch.commons.authuser.User import org.opensearch.index.seqno.SequenceNumbers -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.PolicyRetryInfoMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StateMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.addObject +import org.opensearch.indexmanagement.spi.indexstatemanagement.addObject import java.io.IOException data class ManagedIndexMetaData( @@ -37,6 +29,7 @@ data class ManagedIndexMetaData( val policyPrimaryTerm: Long?, val policyCompleted: Boolean?, val rolledOver: Boolean?, + val indexCreationDate: Long?, val transitionTo: String?, val stateMetaData: StateMetaData?, val actionMetaData: ActionMetaData?, @@ -45,13 +38,7 @@ data class ManagedIndexMetaData( val info: Map?, val id: String = NO_ID, val seqNo: Long = SequenceNumbers.UNASSIGNED_SEQ_NO, - val primaryTerm: Long = SequenceNumbers.UNASSIGNED_PRIMARY_TERM, - // TODO: Remove this once the step interface is updated to pass in user information. - // The user information is not being stored/written anywhere, this is only intended to be used during the step execution. - val user: User? = null, - // TODO: Remove this once the step interface is updated to pass in thread context information. - // This information is not being stored/written anywhere, this is only intended to be used during the step execution. - val threadContext: ThreadContext? = null + val primaryTerm: Long = SequenceNumbers.UNASSIGNED_PRIMARY_TERM ) : Writeable, ToXContentFragment { @Suppress("ComplexMethod") @@ -64,6 +51,7 @@ data class ManagedIndexMetaData( if (policyPrimaryTerm != null) resultMap[POLICY_PRIMARY_TERM] = policyPrimaryTerm.toString() if (policyCompleted != null) resultMap[POLICY_COMPLETED] = policyCompleted.toString() if (rolledOver != null) resultMap[ROLLED_OVER] = rolledOver.toString() + if (indexCreationDate != null) resultMap[INDEX_CREATION_DATE] = indexCreationDate.toString() if (transitionTo != null) resultMap[TRANSITION_TO] = transitionTo if (stateMetaData != null) resultMap[StateMetaData.STATE] = stateMetaData.getMapValueString() if (actionMetaData != null) resultMap[ActionMetaData.ACTION] = actionMetaData.getMapValueString() @@ -88,6 +76,7 @@ data class ManagedIndexMetaData( .field(POLICY_PRIMARY_TERM, policyPrimaryTerm) .field(POLICY_COMPLETED, policyCompleted) .field(ROLLED_OVER, rolledOver) + .field(INDEX_CREATION_DATE, indexCreationDate) .field(TRANSITION_TO, transitionTo) .addObject(StateMetaData.STATE, stateMetaData, params, true) .addObject(ActionMetaData.ACTION, actionMetaData, params, true) @@ -99,6 +88,14 @@ data class ManagedIndexMetaData( return builder } + fun isFailed(): Boolean { + // If PolicyRetryInfo is failed then the ManagedIndex has failed. + if (this.policyRetryInfo?.failed == true) return true + // If ActionMetaData is not null and some action is failed. Then the ManagedIndex has failed. + if (this.actionMetaData?.failed == true) return true + return false + } + @Suppress("ComplexMethod") override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { // The order we check values matters here as we are only trying to show what is needed for the customer @@ -111,10 +108,12 @@ data class ManagedIndexMetaData( if (policyPrimaryTerm != null) builder.field(POLICY_PRIMARY_TERM, policyPrimaryTerm) // Only show rolled_over if we have rolled over or we are in the rollover action - if (rolledOver == true || (actionMetaData != null && actionMetaData.name == ActionConfig.ActionType.ROLLOVER.type)) { + if (rolledOver == true || (actionMetaData != null && actionMetaData.name == "rollover")) { builder.field(ROLLED_OVER, rolledOver) } + if (indexCreationDate != null) builder.field(INDEX_CREATION_DATE, indexCreationDate) + if (policyCompleted == true) { builder.field(POLICY_COMPLETED, policyCompleted) return builder @@ -143,6 +142,7 @@ data class ManagedIndexMetaData( streamOutput.writeOptionalLong(policyPrimaryTerm) streamOutput.writeOptionalBoolean(policyCompleted) streamOutput.writeOptionalBoolean(rolledOver) + streamOutput.writeOptionalLong(indexCreationDate) streamOutput.writeOptionalString(transitionTo) streamOutput.writeOptionalWriteable(stateMetaData) @@ -172,8 +172,10 @@ data class ManagedIndexMetaData( const val POLICY_PRIMARY_TERM = "policy_primary_term" const val POLICY_COMPLETED = "policy_completed" const val ROLLED_OVER = "rolled_over" + const val INDEX_CREATION_DATE = "index_creation_date" const val TRANSITION_TO = "transition_to" const val INFO = "info" + const val ENABLED = "enabled" fun fromStreamInput(si: StreamInput): ManagedIndexMetaData { val index: String? = si.readString() @@ -183,6 +185,7 @@ data class ManagedIndexMetaData( val policyPrimaryTerm: Long? = si.readOptionalLong() val policyCompleted: Boolean? = si.readOptionalBoolean() val rolledOver: Boolean? = si.readOptionalBoolean() + val indexCreationDate: Long? = si.readOptionalLong() val transitionTo: String? = si.readOptionalString() val state: StateMetaData? = si.readOptionalWriteable { StateMetaData.fromStreamInput(it) } @@ -204,6 +207,7 @@ data class ManagedIndexMetaData( policyPrimaryTerm = policyPrimaryTerm, policyCompleted = policyCompleted, rolledOver = rolledOver, + indexCreationDate = indexCreationDate, transitionTo = transitionTo, stateMetaData = state, actionMetaData = action, @@ -230,6 +234,7 @@ data class ManagedIndexMetaData( var policyPrimaryTerm: Long? = null var policyCompleted: Boolean? = null var rolledOver: Boolean? = null + var indexCreationDate: Long? = null var transitionTo: String? = null var state: StateMetaData? = null @@ -239,8 +244,8 @@ data class ManagedIndexMetaData( var info: Map? = null - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { val fieldName = xcp.currentName() xcp.nextToken() @@ -248,19 +253,20 @@ data class ManagedIndexMetaData( INDEX -> index = xcp.text() INDEX_UUID -> indexUuid = xcp.text() POLICY_ID -> policyID = xcp.text() - POLICY_SEQ_NO -> policySeqNo = if (xcp.currentToken() == Token.VALUE_NULL) null else xcp.longValue() - POLICY_PRIMARY_TERM -> policyPrimaryTerm = if (xcp.currentToken() == Token.VALUE_NULL) null else xcp.longValue() - POLICY_COMPLETED -> policyCompleted = if (xcp.currentToken() == Token.VALUE_NULL) null else xcp.booleanValue() - ROLLED_OVER -> rolledOver = if (xcp.currentToken() == Token.VALUE_NULL) null else xcp.booleanValue() - TRANSITION_TO -> transitionTo = if (xcp.currentToken() == Token.VALUE_NULL) null else xcp.text() + POLICY_SEQ_NO -> policySeqNo = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) null else xcp.longValue() + POLICY_PRIMARY_TERM -> policyPrimaryTerm = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) null else xcp.longValue() + POLICY_COMPLETED -> policyCompleted = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) null else xcp.booleanValue() + ROLLED_OVER -> rolledOver = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) null else xcp.booleanValue() + INDEX_CREATION_DATE -> indexCreationDate = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) null else xcp.longValue() + TRANSITION_TO -> transitionTo = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) null else xcp.text() StateMetaData.STATE -> { - state = if (xcp.currentToken() == Token.VALUE_NULL) null else StateMetaData.parse(xcp) + state = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) null else StateMetaData.parse(xcp) } ActionMetaData.ACTION -> { - action = if (xcp.currentToken() == Token.VALUE_NULL) null else ActionMetaData.parse(xcp) + action = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) null else ActionMetaData.parse(xcp) } StepMetaData.STEP -> { - step = if (xcp.currentToken() == Token.VALUE_NULL) null else StepMetaData.parse(xcp) + step = if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) null else StepMetaData.parse(xcp) } PolicyRetryInfoMetaData.RETRY_INFO -> { retryInfo = PolicyRetryInfoMetaData.parse(xcp) @@ -279,6 +285,7 @@ data class ManagedIndexMetaData( policyPrimaryTerm, policyCompleted, rolledOver, + indexCreationDate, transitionTo, state, action, @@ -300,11 +307,11 @@ data class ManagedIndexMetaData( seqNo: Long = SequenceNumbers.UNASSIGNED_SEQ_NO, primaryTerm: Long = SequenceNumbers.UNASSIGNED_PRIMARY_TERM ): ManagedIndexMetaData { - ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) - ensureExpectedToken(Token.FIELD_NAME, xcp.nextToken(), xcp) - ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, xcp.nextToken(), xcp) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) val managedIndexMetaData = parse(xcp, id, seqNo, primaryTerm) - ensureExpectedToken(Token.END_OBJECT, xcp.nextToken(), xcp) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, xcp.nextToken(), xcp) return managedIndexMetaData } @@ -317,6 +324,7 @@ data class ManagedIndexMetaData( policyPrimaryTerm = map[POLICY_PRIMARY_TERM]?.toLong(), policyCompleted = map[POLICY_COMPLETED]?.toBoolean(), rolledOver = map[ROLLED_OVER]?.toBoolean(), + indexCreationDate = map[INDEX_CREATION_DATE]?.toLong(), transitionTo = map[TRANSITION_TO], stateMetaData = StateMetaData.fromManagedIndexMetaDataMap(map), actionMetaData = ActionMetaData.fromManagedIndexMetaDataMap(map), diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/PolicyRetryInfoMetaData.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/PolicyRetryInfoMetaData.kt similarity index 89% rename from src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/PolicyRetryInfoMetaData.kt rename to spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/PolicyRetryInfoMetaData.kt index d9318e296..79b5dbe89 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/PolicyRetryInfoMetaData.kt +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/PolicyRetryInfoMetaData.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata +package org.opensearch.indexmanagement.spi.indexstatemanagement.model import org.opensearch.common.Strings import org.opensearch.common.io.stream.StreamInput @@ -15,8 +15,7 @@ import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.ToXContentFragment import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.common.xcontent.XContentParserUtils import org.opensearch.common.xcontent.XContentType import java.io.ByteArrayInputStream import java.nio.charset.StandardCharsets @@ -70,8 +69,8 @@ data class PolicyRetryInfoMetaData( var failed: Boolean? = null var consumedRetries: Int? = null - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { val fieldName = xcp.currentName() xcp.nextToken() diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/StateMetaData.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt similarity index 83% rename from src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/StateMetaData.kt rename to spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt index 0e8e72db2..614f9e25a 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/StateMetaData.kt +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata +package org.opensearch.indexmanagement.spi.indexstatemanagement.model import org.opensearch.common.Strings import org.opensearch.common.io.stream.StreamInput @@ -15,11 +15,10 @@ import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.ToXContentFragment import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.common.xcontent.XContentParserUtils import org.opensearch.common.xcontent.XContentType -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData.Companion.NAME -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData.Companion.START_TIME +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData.Companion.NAME +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData.Companion.START_TIME import java.io.ByteArrayInputStream import java.nio.charset.StandardCharsets @@ -70,8 +69,8 @@ data class StateMetaData( var name: String? = null var startTime: Long? = null - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { val fieldName = xcp.currentName() xcp.nextToken() diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StepContext.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StepContext.kt new file mode 100644 index 000000000..6773d08f4 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StepContext.kt @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.model + +import org.opensearch.client.Client +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.settings.Settings +import org.opensearch.common.util.concurrent.ThreadContext +import org.opensearch.commons.authuser.User +import org.opensearch.script.ScriptService + +class StepContext( + val metadata: ManagedIndexMetaData, + val clusterService: ClusterService, + val client: Client, + val threadContext: ThreadContext?, + val user: User?, + val scriptService: ScriptService, + val settings: Settings, +) { + fun getUpdatedContext(metadata: ManagedIndexMetaData): StepContext { + return StepContext(metadata, this.clusterService, this.client, this.threadContext, this.user, this.scriptService, this.settings) + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/StepMetaData.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StepMetaData.kt similarity index 83% rename from src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/StepMetaData.kt rename to spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StepMetaData.kt index d9e0bc20f..9b9ca40f1 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/managedindexmetadata/StepMetaData.kt +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StepMetaData.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata +package org.opensearch.indexmanagement.spi.indexstatemanagement.model import org.opensearch.common.Strings import org.opensearch.common.io.stream.StreamInput @@ -15,12 +15,11 @@ import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.ToXContentFragment import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.common.xcontent.XContentParserUtils import org.opensearch.common.xcontent.XContentType -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData.Companion.NAME -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData.Companion.START_TIME -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData.Companion.NAME +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData.Companion.START_TIME import java.io.ByteArrayInputStream import java.nio.charset.StandardCharsets import java.util.Locale @@ -83,15 +82,15 @@ data class StepMetaData( var startTime: Long? = null var stepStatus: Step.StepStatus? = null - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { val fieldName = xcp.currentName() xcp.nextToken() when (fieldName) { NAME -> name = xcp.text() START_TIME -> startTime = xcp.longValue() - STEP_STATUS -> stepStatus = Step.StepStatus.valueOf(xcp.text().toUpperCase(Locale.ROOT)) + STEP_STATUS -> stepStatus = Step.StepStatus.valueOf(xcp.text().uppercase(Locale.ROOT)) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementIndices.kt b/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementIndices.kt index 2b3d67356..54a2f1cf7 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementIndices.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementIndices.kt @@ -19,7 +19,6 @@ import org.opensearch.client.Client import org.opensearch.client.IndicesAdminClient import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.XContentType import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_HIDDEN @@ -28,7 +27,6 @@ import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_NUMBER_OF_ import org.opensearch.indexmanagement.opensearchapi.suspendUntil import org.opensearch.indexmanagement.util.IndexUtils import org.opensearch.indexmanagement.util.OpenForTesting -import org.opensearch.indexmanagement.util._DOC @OpenForTesting class IndexManagementIndices( @@ -54,7 +52,7 @@ class IndexManagementIndices( fun checkAndUpdateIMConfigIndex(actionListener: ActionListener) { if (!indexManagementIndexExists()) { val indexRequest = CreateIndexRequest(INDEX_MANAGEMENT_INDEX) - .mapping(_DOC, indexManagementMappings, XContentType.JSON) + .mapping(indexManagementMappings) .settings(Settings.builder().put(INDEX_HIDDEN, true).build()) client.create( indexRequest, @@ -141,7 +139,7 @@ class IndexManagementIndices( if (existsResponse.isExists) return true val request = CreateIndexRequest(index) - .mapping(_DOC, indexStateManagementHistoryMappings, XContentType.JSON) + .mapping(indexStateManagementHistoryMappings) .settings( Settings.builder() .put(INDEX_HIDDEN, true) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt b/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt index b34b87eac..609f87330 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt @@ -29,13 +29,16 @@ import org.opensearch.common.xcontent.XContentParser.Token import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken import org.opensearch.env.Environment import org.opensearch.env.NodeEnvironment +import org.opensearch.indexmanagement.indexstatemanagement.DefaultIndexMetadataService +import org.opensearch.indexmanagement.indexstatemanagement.ExtensionStatusChecker +import org.opensearch.indexmanagement.indexstatemanagement.ISMActionsParser +import org.opensearch.indexmanagement.indexstatemanagement.IndexMetadataProvider import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementHistory import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexCoordinator import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexRunner import org.opensearch.indexmanagement.indexstatemanagement.MetadataService import org.opensearch.indexmanagement.indexstatemanagement.SkipExecution import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.resthandler.RestAddPolicyAction import org.opensearch.indexmanagement.indexstatemanagement.resthandler.RestChangePolicyAction @@ -69,7 +72,7 @@ import org.opensearch.indexmanagement.indexstatemanagement.transport.action.retr import org.opensearch.indexmanagement.indexstatemanagement.transport.action.retryfailedmanagedindex.TransportRetryFailedManagedIndexAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.TransportUpdateManagedIndexMetaDataAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataAction -import org.opensearch.indexmanagement.indexstatemanagement.util.IndexEvaluator +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.indexmanagement.migration.ISMTemplateService import org.opensearch.indexmanagement.refreshanalyzer.RefreshSearchAnalyzerAction import org.opensearch.indexmanagement.refreshanalyzer.RestRefreshSearchAnalyzerAction @@ -108,6 +111,10 @@ import org.opensearch.indexmanagement.rollup.resthandler.RestStopRollupAction import org.opensearch.indexmanagement.rollup.settings.LegacyOpenDistroRollupSettings import org.opensearch.indexmanagement.rollup.settings.RollupSettings import org.opensearch.indexmanagement.settings.IndexManagementSettings +import org.opensearch.indexmanagement.spi.IndexManagementExtension +import org.opensearch.indexmanagement.spi.indexstatemanagement.IndexMetadataService +import org.opensearch.indexmanagement.spi.indexstatemanagement.StatusChecker +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.transform.TransformRunner import org.opensearch.indexmanagement.transform.action.delete.DeleteTransformsAction import org.opensearch.indexmanagement.transform.action.delete.TransportDeleteTransformsAction @@ -140,6 +147,7 @@ import org.opensearch.jobscheduler.spi.ScheduledJobParser import org.opensearch.jobscheduler.spi.ScheduledJobRunner import org.opensearch.monitor.jvm.JvmService import org.opensearch.plugins.ActionPlugin +import org.opensearch.plugins.ExtensiblePlugin import org.opensearch.plugins.NetworkPlugin import org.opensearch.plugins.Plugin import org.opensearch.repositories.RepositoriesService @@ -154,7 +162,7 @@ import org.opensearch.watcher.ResourceWatcherService import java.util.function.Supplier @Suppress("TooManyFunctions") -class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin, Plugin() { +class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin, ExtensiblePlugin, Plugin() { private val logger = LogManager.getLogger(javaClass) lateinit var indexManagementIndices: IndexManagementIndices @@ -162,6 +170,11 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin lateinit var indexNameExpressionResolver: IndexNameExpressionResolver lateinit var rollupInterceptor: RollupInterceptor lateinit var fieldCapsFilter: FieldCapsFilter + lateinit var indexMetadataProvider: IndexMetadataProvider + private val indexMetadataServices: MutableList> = mutableListOf() + private var customIndexUUIDSetting: String? = null + private val extensions = mutableSetOf() + private val extensionCheckerMap = mutableMapOf() companion object { const val PLUGINS_BASE_URI = "/_plugins" @@ -232,6 +245,30 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin } } + override fun loadExtensions(loader: ExtensiblePlugin.ExtensionLoader) { + val indexManagementExtensions = loader.loadExtensions(IndexManagementExtension::class.java) + + indexManagementExtensions.forEach { extension -> + val extensionName = extension.getExtensionName() + if (extensionName in extensions) { + throw IllegalStateException("Multiple extensions of IndexManagement have same name $extensionName - not supported") + } + extension.getISMActionParsers().forEach { parser -> + ISMActionsParser.instance.addParser(parser, extensionName) + } + indexMetadataServices.add(extension.getIndexMetadataService()) + extension.overrideClusterStateIndexUuidSetting()?.let { + if (customIndexUUIDSetting != null) { + throw IllegalStateException( + "Multiple extensions of IndexManagement plugin overriding ClusterStateIndexUUIDSetting - not supported" + ) + } + customIndexUUIDSetting = extension.overrideClusterStateIndexUuidSetting() + } + extensionCheckerMap[extensionName] = extension.statusChecker() + } + } + override fun getRestHandlers( settings: Settings, restController: RestController, @@ -297,7 +334,6 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin fieldCapsFilter = FieldCapsFilter(clusterService, settings, indexNameExpressionResolver) this.indexNameExpressionResolver = indexNameExpressionResolver - val indexEvaluator = IndexEvaluator(settings, clusterService) val skipFlag = SkipExecution(client, clusterService) val rollupRunner = RollupRunner .registerClient(client) @@ -322,6 +358,15 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin indexManagementIndices ) + indexMetadataProvider = IndexMetadataProvider( + settings, client, clusterService, + hashMapOf( + DEFAULT_INDEX_TYPE to DefaultIndexMetadataService(customIndexUUIDSetting) + ) + ) + indexMetadataServices.forEach { indexMetadataProvider.addMetadataServices(it) } + + val extensionChecker = ExtensionStatusChecker(extensionCheckerMap, clusterService) val managedIndexRunner = ManagedIndexRunner .registerClient(client) .registerClusterService(clusterService) @@ -333,17 +378,25 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin .registerHistoryIndex(indexStateManagementHistory) .registerSkipFlag(skipFlag) .registerThreadPool(threadPool) + .registerExtensionChecker(extensionChecker) + .registerIndexMetadataProvider(indexMetadataProvider) val metadataService = MetadataService(client, clusterService, skipFlag, indexManagementIndices) val templateService = ISMTemplateService(client, clusterService, xContentRegistry, indexManagementIndices) val managedIndexCoordinator = ManagedIndexCoordinator( environment.settings(), - client, clusterService, threadPool, indexManagementIndices, metadataService, templateService, indexEvaluator + client, clusterService, threadPool, indexManagementIndices, metadataService, templateService, indexMetadataProvider ) return listOf( - managedIndexRunner, rollupRunner, transformRunner, indexManagementIndices, managedIndexCoordinator, indexStateManagementHistory, indexEvaluator + managedIndexRunner, + rollupRunner, + transformRunner, + indexManagementIndices, + managedIndexCoordinator, + indexStateManagementHistory, + indexMetadataProvider ) } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/DateHistogram.kt b/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/DateHistogram.kt index 5294a9639..4ed1bc2d4 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/DateHistogram.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/DateHistogram.kt @@ -12,6 +12,8 @@ import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentParser.Token import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.index.query.AbstractQueryBuilder +import org.opensearch.index.query.RangeQueryBuilder import org.opensearch.indexmanagement.util.IndexUtils.Companion.getFieldFromMappings import org.opensearch.search.aggregations.AggregatorFactories import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder @@ -83,6 +85,18 @@ data class DateHistogram( } } + override fun toBucketQuery(bucketKey: Any): AbstractQueryBuilder<*> { + if (bucketKey !is Long) { + throw IllegalArgumentException("Received invalid date histogram bucket key type [${bucketKey::class}] when Long is expected.") + } + val interval: Long = DateHistogramInterval(fixedInterval ?: calendarInterval).estimateMillis() + return RangeQueryBuilder(sourceField) + .from(bucketKey, true) + .to(bucketKey + interval, false) + .timeZone(timezone.toString()) + .format("epoch_millis") + } + override fun canBeRealizedInMappings(mappings: Map): Boolean { val fieldType = getFieldFromMappings(sourceField, mappings)?.get("type") ?: return false return "date" == fieldType diff --git a/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/Dimension.kt b/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/Dimension.kt index b8837341d..2d48018bc 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/Dimension.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/Dimension.kt @@ -10,6 +10,7 @@ import org.opensearch.common.xcontent.ToXContentObject import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentParser.Token import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.index.query.AbstractQueryBuilder import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder import java.io.IOException @@ -30,6 +31,13 @@ abstract class Dimension( abstract fun toSourceBuilder(appendType: Boolean = false): CompositeValuesSourceBuilder<*> + /** + * Helper method to get a query which specifies the documents contained within the bucket determined by this dimension. + * + * e.g. a terms dimension would return a TermsQueryBuilder specifying just the bucketKey term + */ + abstract fun toBucketQuery(bucketKey: Any): AbstractQueryBuilder<*> + /** * Helper method that evaluates if the dimension can be realized using mappings provided. * diff --git a/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/Histogram.kt b/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/Histogram.kt index f06291cc0..94f1a70bf 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/Histogram.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/Histogram.kt @@ -13,6 +13,8 @@ import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentParser.Token import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken import org.opensearch.index.mapper.NumberFieldMapper +import org.opensearch.index.query.AbstractQueryBuilder +import org.opensearch.index.query.RangeQueryBuilder import org.opensearch.indexmanagement.util.IndexUtils.Companion.getFieldFromMappings import org.opensearch.search.aggregations.AggregatorFactories import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder @@ -63,6 +65,15 @@ data class Histogram( .interval(this.interval) } + override fun toBucketQuery(bucketKey: Any): AbstractQueryBuilder<*> { + if (bucketKey !is Double) { + throw IllegalArgumentException("Received invalid histogram bucket key type [${bucketKey::class}] when Double is expected.") + } + return RangeQueryBuilder(sourceField) + .from(bucketKey - Companion.bucketError, true) + .to(bucketKey + interval + Companion.bucketError, true) + } + override fun canBeRealizedInMappings(mappings: Map): Boolean { val fieldType = getFieldFromMappings(sourceField, mappings)?.get("type") ?: return false @@ -98,6 +109,9 @@ data class Histogram( companion object { const val HISTOGRAM_INTERVAL_FIELD = "interval" + // There can be rounding issues with small intervals where the range query will select documents differently than the Histogram + // so add an error to the range query and then limit the buckets indexed later. + private const val bucketError = 0.00005 @Suppress("ComplexMethod", "LongMethod") @JvmStatic diff --git a/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/Terms.kt b/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/Terms.kt index 1d7856134..6ed627c3c 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/Terms.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/Terms.kt @@ -12,6 +12,8 @@ import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentParser.Token import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.index.query.AbstractQueryBuilder +import org.opensearch.index.query.TermsQueryBuilder import org.opensearch.indexmanagement.util.IndexUtils.Companion.getFieldFromMappings import org.opensearch.search.aggregations.AggregatorFactories import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder @@ -55,6 +57,10 @@ data class Terms( .field(this.sourceField) } + override fun toBucketQuery(bucketKey: Any): AbstractQueryBuilder<*> { + return TermsQueryBuilder(sourceField, bucketKey) + } + override fun canBeRealizedInMappings(mappings: Map): Boolean { val fieldType = getFieldFromMappings(sourceField, mappings)?.get("type") ?: return false diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/DefaultIndexMetadataService.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/DefaultIndexMetadataService.kt new file mode 100644 index 000000000..fb6fa43c8 --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/DefaultIndexMetadataService.kt @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement + +import org.opensearch.action.admin.cluster.state.ClusterStateRequest +import org.opensearch.action.admin.cluster.state.ClusterStateResponse +import org.opensearch.action.support.IndicesOptions +import org.opensearch.client.Client +import org.opensearch.cluster.metadata.IndexMetadata +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.unit.TimeValue +import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.IndexMetadataService +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ISMIndexMetadata + +class DefaultIndexMetadataService(val customUUIDSetting: String? = null) : IndexMetadataService { + + /** + * Returns the default index metadata needed for ISM + */ + @Suppress("SpreadOperator") + override suspend fun getMetadata(indices: List, client: Client, clusterService: ClusterService): Map { + val indexNameToMetadata: MutableMap = HashMap() + + // We want to go through all cluster indices - open/closed/hidden + val lenientExpandOptions = IndicesOptions.lenientExpandHidden() + val clusterStateRequest = ClusterStateRequest() + .clear() + .indices(*indices.toTypedArray()) + .metadata(true) + .local(false) + .waitForTimeout(TimeValue.timeValueMillis(DEFAULT_GET_METADATA_TIMEOUT_IN_MILLIS)) + .indicesOptions(lenientExpandOptions) + + val response: ClusterStateResponse = client.suspendUntil { client.admin().cluster().state(clusterStateRequest, it) } + + response.state.metadata.indices.forEach { + // TODO waiting to add document count until it is definitely needed + val uuid = getCustomIndexUUID(it.value) + val indexMetadata = ISMIndexMetadata(uuid, it.value.creationDate, -1) + indexNameToMetadata[it.key] = indexMetadata + } + + return indexNameToMetadata + } + + /* + * If an extension wants Index Management to determine cluster state indices UUID based on a custom index setting if + * present of cluster state, the extension will override this customUUID setting. This allows an index to migrate off + * cluster and back while using this persistent uuid. + */ + fun getCustomIndexUUID(indexMetadata: IndexMetadata): String { + return if (customUUIDSetting != null) { + indexMetadata.settings.get(customUUIDSetting, indexMetadata.indexUUID) + } else { + indexMetadata.indexUUID + } + } + + override suspend fun getMetadataForAllIndices(client: Client, clusterService: ClusterService): Map { + return getMetadata(listOf("*"), client, clusterService) + } + + companion object { + const val DEFAULT_GET_METADATA_TIMEOUT_IN_MILLIS = 30000L + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ExtensionStatusChecker.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ExtensionStatusChecker.kt new file mode 100644 index 000000000..aa1d681de --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ExtensionStatusChecker.kt @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement + +import org.opensearch.cluster.service.ClusterService +import org.opensearch.indexmanagement.spi.indexstatemanagement.Status +import org.opensearch.indexmanagement.spi.indexstatemanagement.StatusChecker + +/** + * Check the extension status check. The extension status should be used to represent if the extension is turned on or off, + * not as a health check denoting availability. + */ +class ExtensionStatusChecker(private val extensionCheckers: Map, val clusterService: ClusterService) { + + fun isEnabled(extensionName: String?): Boolean { + val checker = extensionCheckers[extensionName] ?: return true + val clusterState = clusterService.state() + return checker.check(clusterState) == Status.ENABLED + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ISMActionsParser.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ISMActionsParser.kt new file mode 100644 index 000000000..9b0160658 --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ISMActionsParser.kt @@ -0,0 +1,131 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils +import org.opensearch.indexmanagement.indexstatemanagement.action.AllocationActionParser +import org.opensearch.indexmanagement.indexstatemanagement.action.CloseActionParser +import org.opensearch.indexmanagement.indexstatemanagement.action.DeleteActionParser +import org.opensearch.indexmanagement.indexstatemanagement.action.ForceMergeActionParser +import org.opensearch.indexmanagement.indexstatemanagement.action.IndexPriorityActionParser +import org.opensearch.indexmanagement.indexstatemanagement.action.NotificationActionParser +import org.opensearch.indexmanagement.indexstatemanagement.action.OpenActionParser +import org.opensearch.indexmanagement.indexstatemanagement.action.ReadOnlyActionParser +import org.opensearch.indexmanagement.indexstatemanagement.action.ReadWriteActionParser +import org.opensearch.indexmanagement.indexstatemanagement.action.ReplicaCountActionParser +import org.opensearch.indexmanagement.indexstatemanagement.action.RolloverActionParser +import org.opensearch.indexmanagement.indexstatemanagement.action.RollupActionParser +import org.opensearch.indexmanagement.indexstatemanagement.action.SnapshotActionParser +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionRetry +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionTimeout + +class ISMActionsParser private constructor() { + + private object HOLDER { + val instance = ISMActionsParser() + } + + val parsers = mutableListOf( + AllocationActionParser(), + CloseActionParser(), + DeleteActionParser(), + ForceMergeActionParser(), + IndexPriorityActionParser(), + NotificationActionParser(), + OpenActionParser(), + ReadOnlyActionParser(), + ReadWriteActionParser(), + ReplicaCountActionParser(), + RollupActionParser(), + RolloverActionParser(), + SnapshotActionParser() + ) + + val customActionExtensionMap = mutableMapOf() + + /* + * This method is used for adding custom action parsers. Action parsers from ISM should be added directly + * to the parsers list. + */ + fun addParser(parser: ActionParser, extensionName: String) { + if (parsers.map { it.getActionType() }.contains(parser.getActionType())) { + throw IllegalArgumentException(getDuplicateActionTypesMessage(parser.getActionType())) + } + // Set the parser as custom to make sure that the custom actions are written with the "custom" wrapper + parser.customAction = true + parsers.add(parser) + customActionExtensionMap[parser.getActionType()] = extensionName + } + + fun fromStreamInput(sin: StreamInput): Action { + val type: String = sin.readString() + val configTimeout = sin.readOptionalWriteable(::ActionTimeout) + val configRetry = sin.readOptionalWriteable(::ActionRetry) + val parser = parsers.firstOrNull { it.getActionType() == type } + val action: Action = parser?.fromStreamInput(sin) ?: throw IllegalArgumentException("Invalid field: [$type] found in Actions.") + + action.configTimeout = configTimeout + action.configRetry = configRetry + + return action + } + + fun parse(xcp: XContentParser, totalActions: Int): Action { + var action: Action? = null + var timeout: ActionTimeout? = null + var retry: ActionRetry? = ActionRetry(Action.DEFAULT_RETRIES) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val type = xcp.currentName() + xcp.nextToken() + when (type) { + ActionTimeout.TIMEOUT_FIELD -> timeout = ActionTimeout.parse(xcp) + ActionRetry.RETRY_FIELD -> retry = ActionRetry.parse(xcp) + Action.CUSTOM_ACTION_FIELD -> { + // The "custom" wrapper allows extensions to create arbitrary actions without updating the config mappings + // We consume the full custom wrapper and parse the action in this step + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, xcp.nextToken(), xcp) + val customActionType = xcp.currentName() + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) + action = parseAction(xcp, totalActions, customActionType) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, xcp.nextToken(), xcp) + } + else -> { + action = parseAction(xcp, totalActions, type) + } + } + } + + requireNotNull(action) { "Action inside state is null" } + + action.configTimeout = timeout + action.configRetry = retry + + return action + } + + private fun parseAction(xcp: XContentParser, totalActions: Int, type: String): Action { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + val action: Action? + val parser = parsers.firstOrNull { it.getActionType() == type } + if (parser != null) { + action = parser.fromXContent(xcp, totalActions) + action.customAction = parser.customAction + } else { + throw IllegalArgumentException("Invalid field: [$type] found in Actions.") + } + return action + } + + companion object { + val instance: ISMActionsParser by lazy { HOLDER.instance } + fun getDuplicateActionTypesMessage(actionType: String) = "Multiple action parsers attempted to register the same action type [$actionType]" + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexMetadataProvider.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexMetadataProvider.kt new file mode 100644 index 000000000..471742c87 --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexMetadataProvider.kt @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement + +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import org.opensearch.client.Client +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.settings.Settings +import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE +import org.opensearch.indexmanagement.spi.indexstatemanagement.IndexMetadataService +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ISMIndexMetadata + +/** + * Consolidates IndexMetadataServices from all extensions to delegate the index metadata provider based on index type. + * Provides index metadata of specified index names using the implementation denoted by the index type. + */ +class IndexMetadataProvider( + val settings: Settings, + val client: Client, + val clusterService: ClusterService, + val services: MutableMap, +) { + + @Volatile private var restrictedIndexPattern = ManagedIndexSettings.RESTRICTED_INDEX_PATTERN.get(settings) + + init { + clusterService.clusterSettings.addSettingsUpdateConsumer(ManagedIndexSettings.RESTRICTED_INDEX_PATTERN) { + restrictedIndexPattern = it + } + } + + fun isUnManageableIndex(index: String): Boolean { + return Regex(restrictedIndexPattern).matches(index) + } + + suspend fun getISMIndexMetadataByType(type: String = DEFAULT_INDEX_TYPE, indexNames: List): Map { + val service = services[type] ?: throw IllegalArgumentException(getTypeNotRecognizedMessage(type)) + if (type != DEFAULT_INDEX_TYPE && indexNames.size > 1) throw IllegalArgumentException(MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR) + return service.getMetadata(indexNames, client, clusterService) + } + + suspend fun getAllISMIndexMetadataByType(type: String = DEFAULT_INDEX_TYPE): Map { + val service = services[type] ?: throw IllegalArgumentException(getTypeNotRecognizedMessage(type)) + return service.getMetadataForAllIndices(client, clusterService) + } + + /* + * Attempts to get the index metadata for of all indexNames for each of the index types designated in the types parameter. + * Returns a map of > + */ + suspend fun getMultiTypeISMIndexMetadata( + types: List = services.keys.toList(), + indexNames: List + ): Map> = coroutineScope { + if (types.any { it != DEFAULT_INDEX_TYPE } && indexNames.size > 1) throw IllegalArgumentException(MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR) + val requests = ArrayList>>>() + // Start all index metadata requests at the same time + types.forEach { type -> + requests.add(async { type to getISMIndexMetadataByType(type, indexNames) }) + } + // Wait for all index metadata responses, and return + requests.awaitAll().toMap() + } + + fun addMetadataServices(newServices: Map) { + val duplicateIndexType = newServices.keys.firstOrNull { services.containsKey(it) } + if (duplicateIndexType != null) { + throw IllegalArgumentException(getDuplicateServicesMessage(duplicateIndexType)) + } + services.putAll(newServices) + } + + suspend fun getAllISMIndexMetadata(): Set = coroutineScope { + val metadata = mutableSetOf() + val requests = ArrayList>>() + services.forEach { (_, service) -> + requests.add(async { service.getMetadataForAllIndices(client, clusterService) }) + } + + requests.awaitAll().forEach { metadata.addAll(it.values) } + + metadata + } + + fun getIndexMetadataWriteOverrideSettings(): List { + return services.values.mapNotNull { it.getIndexMetadataWriteOverrideSetting() } + } + + companion object { + const val EVALUATION_FAILURE_MESSAGE = "Matches restricted index pattern defined in the cluster setting" + const val MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR = "Cannot get metadata for more than one index name/pattern when using a custom index type" + fun getTypeNotRecognizedMessage(indexType: String) = "Index type [type=$indexType] was not recognized when trying to get index metadata" + fun getDuplicateServicesMessage(indexType: String) = "Multiple metadata services attempted to assign a service to the index type [$indexType]" + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementHistory.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementHistory.kt index 5a2bb3f48..763603ecc 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementHistory.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementHistory.kt @@ -24,18 +24,17 @@ import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.XContentFactory -import org.opensearch.common.xcontent.XContentType import org.opensearch.indexmanagement.IndexManagementIndices import org.opensearch.indexmanagement.IndexManagementPlugin -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_HIDDEN import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_NUMBER_OF_REPLICAS import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_NUMBER_OF_SHARDS +import org.opensearch.indexmanagement.opensearchapi.OPENDISTRO_SECURITY_PROTECTED_INDICES_CONF_REQUEST import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.util.OpenForTesting -import org.opensearch.indexmanagement.util._DOC import org.opensearch.threadpool.Scheduler import org.opensearch.threadpool.ThreadPool import java.time.Instant @@ -113,8 +112,16 @@ class IndexStateManagementHistory( } private fun rolloverAndDeleteHistoryIndex() { - if (historyEnabled) rolloverHistoryIndex() - deleteOldHistoryIndex() + val ctx = threadPool.threadContext.stashContext() + try { + if (threadPool.threadContext.getTransient(OPENDISTRO_SECURITY_PROTECTED_INDICES_CONF_REQUEST) == null) { + threadPool.threadContext.putTransient(OPENDISTRO_SECURITY_PROTECTED_INDICES_CONF_REQUEST, "true") + } + if (historyEnabled) rolloverHistoryIndex() + deleteOldHistoryIndex() + } finally { + ctx.close() + } } private fun rolloverHistoryIndex() { @@ -125,7 +132,7 @@ class IndexStateManagementHistory( // We have to pass null for newIndexName in order to get Elastic to increment the index count. val request = RolloverRequest(IndexManagementIndices.HISTORY_WRITE_INDEX_ALIAS, null) request.createIndexRequest.index(IndexManagementIndices.HISTORY_INDEX_PATTERN) - .mapping(_DOC, IndexManagementIndices.indexStateManagementHistoryMappings, XContentType.JSON) + .mapping(IndexManagementIndices.indexStateManagementHistoryMappings) .settings( Settings.builder() .put(INDEX_HIDDEN, true) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt index 81eb724d7..738214784 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt @@ -20,9 +20,13 @@ import org.opensearch.action.bulk.BulkResponse import org.opensearch.action.get.MultiGetRequest import org.opensearch.action.get.MultiGetResponse import org.opensearch.action.index.IndexRequest +import org.opensearch.action.search.ClearScrollAction +import org.opensearch.action.search.ClearScrollRequest +import org.opensearch.action.search.ClearScrollResponse import org.opensearch.action.search.SearchPhaseExecutionException import org.opensearch.action.search.SearchRequest import org.opensearch.action.search.SearchResponse +import org.opensearch.action.search.SearchScrollRequest import org.opensearch.action.support.master.AcknowledgedResponse import org.opensearch.action.update.UpdateRequest import org.opensearch.client.Client @@ -44,10 +48,8 @@ import org.opensearch.indexmanagement.IndexManagementIndices import org.opensearch.indexmanagement.IndexManagementPlugin import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.coordinator.ClusterStateManagedIndexConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.coordinator.SweptManagedIndexConfig import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.mgetManagedIndexMetadata import org.opensearch.indexmanagement.indexstatemanagement.settings.LegacyOpenDistroManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.AUTO_MANAGE @@ -63,10 +65,8 @@ import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndex import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexRequest import org.opensearch.indexmanagement.indexstatemanagement.util.ISM_TEMPLATE_FIELD -import org.opensearch.indexmanagement.indexstatemanagement.util.IndexEvaluator import org.opensearch.indexmanagement.indexstatemanagement.util.deleteManagedIndexMetadataRequest import org.opensearch.indexmanagement.indexstatemanagement.util.deleteManagedIndexRequest -import org.opensearch.indexmanagement.indexstatemanagement.util.getDeleteManagedIndexRequests import org.opensearch.indexmanagement.indexstatemanagement.util.getManagedIndicesToDelete import org.opensearch.indexmanagement.indexstatemanagement.util.getSweptManagedIndexSearchRequest import org.opensearch.indexmanagement.indexstatemanagement.util.isFailed @@ -81,7 +81,8 @@ import org.opensearch.indexmanagement.opensearchapi.parseWithType import org.opensearch.indexmanagement.opensearchapi.retry import org.opensearch.indexmanagement.opensearchapi.suspendUntil import org.opensearch.indexmanagement.opensearchapi.withClosableContext -import org.opensearch.indexmanagement.util.NO_ID +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ISMIndexMetadata +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.util.OpenForTesting import org.opensearch.rest.RestStatus import org.opensearch.search.builder.SearchSourceBuilder @@ -95,7 +96,7 @@ import java.time.Instant * * This class listens for [ClusterChangedEvent] to pick up on [ManagedIndexConfig] to create or delete. * Also sets up a background process that sweeps the cluster state for [ClusterStateManagedIndexConfig] - * and the [INDEX_MANAGEMENT_INDEX] for [SweptManagedIndexConfig]. It will then compare these + * and the [INDEX_MANAGEMENT_INDEX] for current managed index jobs. It will then compare these * ManagedIndices to appropriately create or delete each [ManagedIndexConfig]. Each node that has * the [IndexManagementPlugin] installed will have an instance of this class, but only the elected * master node will set up the background sweep process and listen for [ClusterChangedEvent]. @@ -114,7 +115,7 @@ class ManagedIndexCoordinator( indexManagementIndices: IndexManagementIndices, private val metadataService: MetadataService, private val templateService: ISMTemplateService, - private val indexEvaluator: IndexEvaluator = IndexEvaluator(settings, clusterService) + private val indexMetadataProvider: IndexMetadataProvider ) : ClusterStateListener, CoroutineScope by CoroutineScope(SupervisorJob() + Dispatchers.Default + CoroutineName("ManagedIndexCoordinator")), LifecycleListener() { @@ -167,8 +168,7 @@ class ManagedIndexCoordinator( if (!templateMigrationEnabled) scheduledTemplateMigration?.cancel() else initTemplateMigration(it) } - clusterService.clusterSettings.addSettingsUpdateConsumer(COORDINATOR_BACKOFF_MILLIS, COORDINATOR_BACKOFF_COUNT) { - millis, count -> + clusterService.clusterSettings.addSettingsUpdateConsumer(COORDINATOR_BACKOFF_MILLIS, COORDINATOR_BACKOFF_COUNT) { millis, count -> retryPolicy = BackoffPolicy.constantBackoff(millis, count) } } @@ -267,8 +267,10 @@ class ManagedIndexCoordinator( * 2. Does not have a completed Policy * 3. Does not have a failed Policy */ - val currentManagedIndices = sweepManagedIndexJobs(client, ismIndices.indexManagementIndexExists()) - val metadataList = client.mgetManagedIndexMetadata(currentManagedIndices.map { Index(it.key, it.value.uuid) }) + // If IM config doesn't exist skip + if (!ismIndices.indexManagementIndexExists()) return + val currentManagedIndexUuids = sweepManagedIndexJobs(client) + val metadataList = client.mgetManagedIndexMetadata(currentManagedIndexUuids) val managedIndicesToEnableReq = mutableListOf() metadataList.forEach { val metadata = it?.first @@ -277,7 +279,7 @@ class ManagedIndexCoordinator( } } - updateManagedIndices(managedIndicesToEnableReq, false) + updateManagedIndices(managedIndicesToEnableReq) } private fun isIndexStateManagementEnabled(): Boolean = indexStateManagementEnabled == true @@ -292,16 +294,18 @@ class ManagedIndexCoordinator( var indicesToClean = emptyList() if (event.indicesDeleted().isNotEmpty()) { val managedIndices = getManagedIndices(event.indicesDeleted().map { it.uuid }) - indicesToClean = event.indicesDeleted().filter { it.uuid in managedIndices.keys } + val deletedIndices = event.indicesDeleted().map { it.name } + val allIndicesUuid = indexMetadataProvider.getMultiTypeISMIndexMetadata(indexNames = deletedIndices).map { (_, metadataMapForType) -> + metadataMapForType.values.map { it.indexUuid } + }.flatten() + // Check if the deleted index uuid is still part of any metadata service in the cluster and has an existing managed index job + indicesToClean = event.indicesDeleted().filter { it.uuid in managedIndices.keys && !allIndicesUuid.contains(it.uuid) } removeManagedIndexReq = indicesToClean.map { deleteManagedIndexRequest(it.uuid) } } - // check if newly created indices match with any ISM templates - var updateMatchingIndexReq = emptyList>() - if (event.indicesCreated().isNotEmpty()) - updateMatchingIndexReq = getMatchingIndicesUpdateReq(event.state(), event.indicesCreated()) + val updateMatchingIndexReq = createManagedIndexRequests(event.state(), event.indicesCreated()) - updateManagedIndices(updateMatchingIndexReq + removeManagedIndexReq, updateMatchingIndexReq.isNotEmpty()) + updateManagedIndices(updateMatchingIndexReq + removeManagedIndexReq) clearManagedIndexMetaData(indicesToClean.map { deleteManagedIndexMetadataRequest(it.uuid) }) } @@ -310,22 +314,24 @@ class ManagedIndexCoordinator( * build requests to create jobs for indices matching ISM templates */ @Suppress("NestedBlockDepth") - suspend fun getMatchingIndicesUpdateReq( + private suspend fun createManagedIndexRequests( clusterState: ClusterState, indexNames: List ): List> { val updateManagedIndexReqs = mutableListOf>() - if (indexNames.isEmpty()) return updateManagedIndexReqs.toList() + if (indexNames.isEmpty()) return updateManagedIndexReqs val policiesWithTemplates = getPoliciesWithISMTemplates() - + // We use the metadata provider to give us the correct uuid for the index when creating managed index for the index + val ismIndicesMetadata: Map = indexMetadataProvider.getISMIndexMetadataByType(indexNames = indexNames) // Iterate over each unmanaged hot/warm index and if it matches an ISM template add a managed index config index request indexNames.forEach { indexName -> + val ismIndexMetadata = ismIndicesMetadata[indexName] + // We try to find lookup name instead of using index name as datastream indices need the alias to match policy val lookupName = findIndexLookupName(indexName, clusterState) - if (lookupName != null && !indexEvaluator.isUnManageableIndex(lookupName)) { - val indexMetadata = clusterState.metadata.index(indexName) - val creationDate = indexMetadata.creationDate - val indexUuid = indexMetadata.indexUUID + if (lookupName != null && !indexMetadataProvider.isUnManageableIndex(lookupName) && ismIndexMetadata != null) { + val creationDate = ismIndexMetadata.indexCreationDate + val indexUuid = ismIndexMetadata.indexUuid findMatchingPolicy(lookupName, creationDate, policiesWithTemplates) ?.let { policy -> logger.info("Index [$indexName] matched ISM policy template and will be managed by ${policy.id}") @@ -343,7 +349,7 @@ class ManagedIndexCoordinator( } } - return updateManagedIndexReqs.toList() + return updateManagedIndexReqs } private fun findIndexLookupName(indexName: String, clusterState: ClusterState): String? { @@ -411,7 +417,7 @@ class ManagedIndexCoordinator( return null } - suspend fun canPolicyManagedIndex(policy: Policy, indexName: String): Boolean { + private suspend fun canPolicyManagedIndex(policy: Policy, indexName: String): Boolean { if (policy.user != null) { try { val request = ManagedIndexRequest().indices(indexName) @@ -429,7 +435,7 @@ class ManagedIndexCoordinator( return true } - suspend fun getPoliciesWithISMTemplates(): List { + private suspend fun getPoliciesWithISMTemplates(): List { val errorMessage = "Failed to get ISM policies with templates" val searchRequest = SearchRequest() .source( @@ -577,58 +583,78 @@ class ManagedIndexCoordinator( */ @OpenForTesting suspend fun sweep() { - // get all indices in the cluster state - val currentIndices = clusterService.state().metadata.indices.values().map { it.value } - .distinct().filterNotNull() + // If IM config doesn't exist skip + if (!ismIndices.indexManagementIndexExists()) return - val currentManagedIndices = sweepManagedIndexJobs(client, ismIndices.indexManagementIndexExists()) + // Get all current managed indices uuids + val currentManagedIndexUuids = sweepManagedIndexJobs(client) - // check all un-managed indices that are manageable, if its name matches any template - val unManagedIndices = currentIndices - .filter { it.index.uuid !in currentManagedIndices.keys && !indexEvaluator.isUnManageableIndex(it.index.name) } - .map { it.index }.distinct() - val updateMatchingIndicesReqs = getMatchingIndicesUpdateReq(clusterService.state(), unManagedIndices.map { it.name }) + // get all indices in the cluster state + val currentIndices = indexMetadataProvider.getISMIndexMetadataByType(indexNames = listOf("*")) + val unManagedIndices = getUnManagedIndices(currentIndices, currentManagedIndexUuids.toHashSet()) + + // Get the matching policyIds for applicable indices + val updateMatchingIndicesReqs = createManagedIndexRequests( + clusterService.state(), unManagedIndices.map { (indexName, _) -> indexName } + ) // check all managed indices, if the index has already been deleted - val deleteManagedIndexRequests = - getDeleteManagedIndexRequests(currentIndices, currentManagedIndices) + val allIndicesUuids = indexMetadataProvider.getAllISMIndexMetadata().map { it.indexUuid } + val managedIndicesToDelete = getManagedIndicesToDelete(allIndicesUuids, currentManagedIndexUuids) + val deleteManagedIndexRequests = managedIndicesToDelete.map { deleteManagedIndexRequest(it) } - updateManagedIndices( - updateMatchingIndicesReqs + deleteManagedIndexRequests, - updateMatchingIndicesReqs.isNotEmpty() - ) + updateManagedIndices(updateMatchingIndicesReqs + deleteManagedIndexRequests) // clean metadata of un-managed index - val indicesToDeleteMetadataFrom = - unManagedIndices + getManagedIndicesToDelete(currentIndices, currentManagedIndices) - clearManagedIndexMetaData(indicesToDeleteMetadataFrom.map { deleteManagedIndexMetadataRequest(it.uuid) }) + val indicesToDeleteMetadataFrom = unManagedIndices.map { (_, ismMetadata) -> ismMetadata.indexUuid } + managedIndicesToDelete + clearManagedIndexMetaData(indicesToDeleteMetadataFrom.map { deleteManagedIndexMetadataRequest(it) }) lastFullSweepTimeNano = System.nanoTime() } + private fun getUnManagedIndices(allIndices: Map, managedIndexUuids: Set): Map { + val unManagedIndices = mutableMapOf() + allIndices.forEach { (indexName, ismMetadata) -> + if (ismMetadata.indexUuid !in managedIndexUuids) { + unManagedIndices[indexName] = ismMetadata + } + } + return unManagedIndices + } + /** * Sweeps the [INDEX_MANAGEMENT_INDEX] for ManagedIndices. * - * Sweeps the [INDEX_MANAGEMENT_INDEX] for ManagedIndices and only fetches the index, index_uuid, - * policy_id, and change_policy fields to convert into a [SweptManagedIndexConfig]. + * Sweeps the [INDEX_MANAGEMENT_INDEX] for ManagedIndices and only fetches the index_uuid * - * @return map of IndexUuid to [SweptManagedIndexConfig]. + * @return list of IndexUuid. */ @OpenForTesting - suspend fun sweepManagedIndexJobs( - client: Client, - indexManagementIndexExists: Boolean - ): Map { - if (!indexManagementIndexExists) return mapOf() + suspend fun sweepManagedIndexJobs(client: Client): List { + val managedIndexUuids = mutableListOf() val managedIndexSearchRequest = getSweptManagedIndexSearchRequest() - val response: SearchResponse = client.suspendUntil { search(managedIndexSearchRequest, it) } - return response.hits.map { - it.id to contentParser(it.sourceRef).parseWithType( - NO_ID, it.seqNo, - it.primaryTerm, SweptManagedIndexConfig.Companion::parse - ) - }.toMap() + var response: SearchResponse = client.suspendUntil { search(managedIndexSearchRequest, it) } + var uuids = response.hits.map { it.id } + val scrollIDsToClear = mutableSetOf() + + while (uuids.isNotEmpty()) { + managedIndexUuids.addAll(uuids) + val scrollID = response.scrollId + scrollIDsToClear.add(scrollID) + val scrollRequest = SearchScrollRequest().scrollId(scrollID).scroll(TimeValue.timeValueMinutes(1)) + response = client.suspendUntil { searchScroll(scrollRequest, it) } + uuids = response.hits.map { it.id } + } + + if (scrollIDsToClear.isNotEmpty()) { + val clearScrollRequest = ClearScrollRequest() + clearScrollRequest.scrollIds(scrollIDsToClear.toList()) + val clearScrollResponse: ClearScrollResponse = + client.suspendUntil { execute(ClearScrollAction.INSTANCE, clearScrollRequest, it) } + } + + return managedIndexUuids } /** @@ -636,7 +662,7 @@ class ManagedIndexCoordinator( * * @return map of IndexUuid to [ManagedIndexConfig] */ - suspend fun getManagedIndices(indexUuids: List): Map { + private suspend fun getManagedIndices(indexUuids: List): Map { if (indexUuids.isEmpty()) return emptyMap() val result: MutableMap = mutableMapOf() @@ -661,24 +687,10 @@ class ManagedIndexCoordinator( } @OpenForTesting - suspend fun updateManagedIndices(requests: List>, hasCreateRequests: Boolean = false) { + private suspend fun updateManagedIndices(requests: List>) { var requestsToRetry = requests if (requestsToRetry.isEmpty()) return - if (hasCreateRequests) { - val created = ismIndices.attemptInitStateManagementIndex(client) - if (!created) { - logger.error("Failed to create $INDEX_MANAGEMENT_INDEX") - return - } - - val updated = ismIndices.attemptUpdateConfigIndexMapping() - if (!updated) { - logger.error("Failed to update mapping for $INDEX_MANAGEMENT_INDEX") - return - } - } - retryPolicy.retry(logger, listOf(RestStatus.TOO_MANY_REQUESTS)) { val bulkRequest = BulkRequest().add(requestsToRetry) val bulkResponse: BulkResponse = client.suspendUntil { bulk(bulkRequest, it) } @@ -693,21 +705,6 @@ class ManagedIndexCoordinator( } } - /** - * Returns [Index]es not being managed by ISM - * but still has ISM metadata - */ - suspend fun getIndicesToRemoveMetadataFrom(unManagedIndices: List): List { - val indicesToRemoveManagedIndexMetaDataFrom = mutableListOf() - val metadataList = client.mgetManagedIndexMetadata(unManagedIndices) - metadataList.forEach { - val metadata = it?.first - if (metadata != null) - indicesToRemoveManagedIndexMetaDataFrom.add(Index(metadata.index, metadata.indexUuid)) - } - return indicesToRemoveManagedIndexMetaDataFrom - } - /** * Removes the [ManagedIndexMetaData] from the given list of [Index]es. */ diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt index 8ff4d3888..c4cdce159 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt @@ -16,6 +16,8 @@ import org.apache.logging.log4j.LogManager import org.opensearch.action.admin.cluster.state.ClusterStateRequest import org.opensearch.action.admin.cluster.state.ClusterStateResponse import org.opensearch.action.bulk.BackoffPolicy +import org.opensearch.action.bulk.BulkRequest +import org.opensearch.action.bulk.BulkResponse import org.opensearch.action.get.GetRequest import org.opensearch.action.get.GetResponse import org.opensearch.action.index.IndexResponse @@ -40,14 +42,9 @@ import org.opensearch.index.engine.VersionConflictEngineException import org.opensearch.index.seqno.SequenceNumbers import org.opensearch.indexmanagement.IndexManagementIndices import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX -import org.opensearch.indexmanagement.indexstatemanagement.action.Action +import org.opensearch.indexmanagement.indexstatemanagement.action.TransitionsAction import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.indexstatemanagement.model.Policy -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.PolicyRetryInfoMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StateMetaData import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.getManagedIndexMetadata import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.ALLOW_LIST @@ -56,18 +53,18 @@ import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndex import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.DEFAULT_JOB_INTERVAL import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.INDEX_STATE_MANAGEMENT_ENABLED import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.JOB_INTERVAL -import org.opensearch.indexmanagement.indexstatemanagement.step.Step -import org.opensearch.indexmanagement.indexstatemanagement.util.getActionToExecute +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE +import org.opensearch.indexmanagement.indexstatemanagement.util.MetadataCheck +import org.opensearch.indexmanagement.indexstatemanagement.util.checkMetadata +import org.opensearch.indexmanagement.indexstatemanagement.util.deleteManagedIndexMetadataRequest +import org.opensearch.indexmanagement.indexstatemanagement.util.deleteManagedIndexRequest import org.opensearch.indexmanagement.indexstatemanagement.util.getCompletedManagedIndexMetaData import org.opensearch.indexmanagement.indexstatemanagement.util.getStartingManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.util.getStateToExecute -import org.opensearch.indexmanagement.indexstatemanagement.util.getUpdatedActionMetaData import org.opensearch.indexmanagement.indexstatemanagement.util.hasDifferentJobInterval import org.opensearch.indexmanagement.indexstatemanagement.util.hasTimedOut import org.opensearch.indexmanagement.indexstatemanagement.util.hasVersionConflict import org.opensearch.indexmanagement.indexstatemanagement.util.isAllowed import org.opensearch.indexmanagement.indexstatemanagement.util.isFailed -import org.opensearch.indexmanagement.indexstatemanagement.util.isMetadataMoved import org.opensearch.indexmanagement.indexstatemanagement.util.isSafeToChange import org.opensearch.indexmanagement.indexstatemanagement.util.isSuccessfulDelete import org.opensearch.indexmanagement.indexstatemanagement.util.managedIndexConfigIndexRequest @@ -82,6 +79,13 @@ import org.opensearch.indexmanagement.opensearchapi.retry import org.opensearch.indexmanagement.opensearchapi.string import org.opensearch.indexmanagement.opensearchapi.suspendUntil import org.opensearch.indexmanagement.opensearchapi.withClosableContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext import org.opensearch.jobscheduler.spi.JobExecutionContext import org.opensearch.jobscheduler.spi.LockModel import org.opensearch.jobscheduler.spi.ScheduledJobParameter @@ -95,7 +99,7 @@ import org.opensearch.threadpool.ThreadPool import java.time.Instant import java.time.temporal.ChronoUnit -@Suppress("TooManyFunctions") +@Suppress("TooManyFunctions", "LargeClass") object ManagedIndexRunner : ScheduledJobRunner, CoroutineScope by CoroutineScope(SupervisorJob() + Dispatchers.Default + CoroutineName("ManagedIndexRunner")) { @@ -111,6 +115,8 @@ object ManagedIndexRunner : private lateinit var ismHistory: IndexStateManagementHistory private lateinit var skipExecFlag: SkipExecution private lateinit var threadPool: ThreadPool + private lateinit var extensionStatusChecker: ExtensionStatusChecker + private lateinit var indexMetadataProvider: IndexMetadataProvider private var indexStateManagementEnabled: Boolean = DEFAULT_ISM_ENABLED @Suppress("MagicNumber") private val savePolicyRetryPolicy = BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(250), 3) @@ -188,6 +194,16 @@ object ManagedIndexRunner : return this } + fun registerIndexMetadataProvider(indexMetadataProvider: IndexMetadataProvider): ManagedIndexRunner { + this.indexMetadataProvider = indexMetadataProvider + return this + } + + fun registerExtensionChecker(extensionStatusChecker: ExtensionStatusChecker): ManagedIndexRunner { + this.extensionStatusChecker = extensionStatusChecker + return this + } + override fun runJob(job: ScheduledJobParameter, context: JobExecutionContext) { if (job !is ManagedIndexConfig) { throw IllegalArgumentException("Invalid job type, found ${job.javaClass.simpleName} with id: ${context.jobId}") @@ -214,7 +230,7 @@ object ManagedIndexRunner : } } - @Suppress("ReturnCount", "ComplexMethod", "LongMethod", "ComplexCondition") + @Suppress("ReturnCount", "ComplexMethod", "LongMethod", "ComplexCondition", "NestedBlockDepth") private suspend fun runManagedIndexConfig(managedIndexConfig: ManagedIndexConfig) { logger.debug("Run job for index ${managedIndexConfig.index}") // doing a check of local cluster health as we do not want to overload master node with potentially a lot of calls @@ -223,23 +239,44 @@ object ManagedIndexRunner : return } - // Get current IndexMetaData and ManagedIndexMetaData - val indexMetaData = getIndexMetadata(managedIndexConfig.index) - if (indexMetaData == null) { - logger.warn("Failed to retrieve IndexMetadata.") + val (managedIndexMetaData, getMetadataSuccess) = client.getManagedIndexMetadata(managedIndexConfig.indexUuid) + if (!getMetadataSuccess) { + logger.info("Failed to retrieve managed index metadata of index [${managedIndexConfig.index}] from config index, abort this run.") return } - val managedIndexMetaData = indexMetaData.getManagedIndexMetadata(client) - val clusterStateMetadata = indexMetaData.getManagedIndexMetadata() - if (!isMetadataMoved(clusterStateMetadata, managedIndexMetaData, logger)) { - logger.info("Skipping execution while pending migration of metadata for ${managedIndexConfig.jobName}") - return + // Check the cluster state for the index metadata + var clusterStateIndexMetadata = getIndexMetadata(managedIndexConfig.index) + val defaultIndexMetadataService = indexMetadataProvider.services[DEFAULT_INDEX_TYPE] as DefaultIndexMetadataService + val clusterStateIndexUUID = clusterStateIndexMetadata?.let { defaultIndexMetadataService.getCustomIndexUUID(it) } + // If the index metadata is null, the index is not in the cluster state. If the index metadata is not null, but + // the cluster state index uuid differs from the one in the managed index config then the config is referring + // to a different index which does not exist in the cluster. We need to check all of the extensions to confirm an index exists + if (clusterStateIndexMetadata == null || clusterStateIndexUUID != managedIndexConfig.indexUuid) { + clusterStateIndexMetadata = null + // If the cluster state/default index type didn't have an index with a matching name and uuid combination, try all other index types + val nonDefaultIndexTypes = indexMetadataProvider.services.keys.filter { it != DEFAULT_INDEX_TYPE } + val multiTypeIndexNameToMetaData = + indexMetadataProvider.getMultiTypeISMIndexMetadata(nonDefaultIndexTypes, listOf(managedIndexConfig.index)) + val someTypeMatchedUuid = multiTypeIndexNameToMetaData.values.any { + it[managedIndexConfig.index]?.indexUuid == managedIndexConfig.indexUuid + } + // If no index types had an index with a matching name and uuid combination, return + if (!someTypeMatchedUuid) { + logger.warn("Failed to find IndexMetadata for ${managedIndexConfig.index}.") + return + } + } else { + val clusterStateMetadata = clusterStateIndexMetadata.getManagedIndexMetadata() + val metadataCheck = checkMetadata(clusterStateMetadata, managedIndexMetaData, managedIndexConfig.indexUuid, logger) + if (metadataCheck != MetadataCheck.SUCCESS) { + logger.info("Skipping execution while metadata status is $metadataCheck") + return + } } // If policy or managedIndexMetaData is null then initialize val policy = managedIndexConfig.policy - if (policy == null || managedIndexMetaData == null) { initManagedIndex(managedIndexConfig, managedIndexMetaData) return @@ -266,11 +303,10 @@ object ManagedIndexRunner : } val state = policy.getStateToExecute(managedIndexMetaData) - val action: Action? = state?.getActionToExecute( - clusterService, scriptService, client, settings, managedIndexMetaData.copy(user = policy.user, threadContext = threadPool.threadContext) - ) - val step: Step? = action?.getStepToExecute() - val currentActionMetaData = action?.getUpdatedActionMetaData(managedIndexMetaData, state) + val action: Action? = state?.getActionToExecute(managedIndexMetaData, indexMetadataProvider) + val stepContext = StepContext(managedIndexMetaData, clusterService, client, threadPool.threadContext, policy.user, scriptService, settings) + val step: Step? = action?.getStepToExecute(stepContext) + val currentActionMetaData = action?.getUpdatedActionMetadata(managedIndexMetaData, state.name) // If Index State Management is disabled and the current step is not null and safe to disable on // then disable the job and return early @@ -281,7 +317,7 @@ object ManagedIndexRunner : if (action?.hasTimedOut(currentActionMetaData) == true) { val info = mapOf("message" to "Action timed out") - logger.error("Action=${action.type.type} has timed out") + logger.error("Action=${action.type} has timed out") val updated = updateManagedIndexMetaData( managedIndexMetaData .copy(actionMetaData = currentActionMetaData?.copy(failed = true), info = info) @@ -295,7 +331,7 @@ object ManagedIndexRunner : return } - val shouldBackOff = action?.shouldBackoff(currentActionMetaData, action.config.configRetry) + val shouldBackOff = action?.shouldBackoff(currentActionMetaData, action.configRetry) if (shouldBackOff?.first == true) { // If we should back off then exit early. logger.info("Backoff for retrying. Remaining time ${shouldBackOff.second}") @@ -317,10 +353,23 @@ object ManagedIndexRunner : } } + // If the extension from which action is registered is not enabled then the action will fail + val actionExtensionName = ISMActionsParser.instance.customActionExtensionMap[action?.type] + if (!extensionStatusChecker.isEnabled(actionExtensionName)) { + val info = mapOf("message" to "Failed to execute action=${action?.type} as extension [$actionExtensionName] is not enabled.") + val updated = updateManagedIndexMetaData( + managedIndexMetaData.copy( + policyRetryInfo = PolicyRetryInfoMetaData(true, 0), info = info + ) + ) + if (updated.metadataSaved) disableManagedIndexConfig(managedIndexConfig) + return + } + // If this action is not allowed and the step to be executed is the first step in the action then we will fail // as this action has been removed from the AllowList, but if its not the first step we will let it finish as it's already inflight - if (action?.isAllowed(allowList) == false && action.isFirstStep(step?.name)) { - val info = mapOf("message" to "Attempted to execute action=${action.type.type} which is not allowed.") + if (action?.isAllowed(allowList) == false && step != null && action.isFirstStep(step.name) && action.type != TransitionsAction.name) { + val info = mapOf("message" to "Attempted to execute action=${action?.type} which is not allowed.") val updated = updateManagedIndexMetaData( managedIndexMetaData.copy( policyRetryInfo = PolicyRetryInfoMetaData(true, 0), info = info @@ -342,14 +391,14 @@ object ManagedIndexRunner : managedIndexConfig.id, settings, threadPool.threadContext, managedIndexConfig.policy.user ) ) { - step.preExecute(logger).execute().postExecute(logger) + step.preExecute(logger, stepContext.getUpdatedContext(startingManagedIndexMetaData)).execute().postExecute(logger) } var executedManagedIndexMetaData = startingManagedIndexMetaData.getCompletedManagedIndexMetaData(action, step) if (executedManagedIndexMetaData.isFailed) { try { // if the policy has no error_notification this will do nothing otherwise it will try to send the configured error message - publishErrorNotification(policy, executedManagedIndexMetaData) + // publishErrorNotification(policy, executedManagedIndexMetaData) } catch (e: Exception) { logger.error("Failed to publish error notification", e) val errorMessage = e.message ?: "Failed to publish error notification" @@ -366,6 +415,16 @@ object ManagedIndexRunner : return } + // If a custom action deletes some off-cluster index and has deleteIndexMetadataAfterFinish set to true, + // then when the action successfully finishes, we will delete the managed index config and metadata. We do not + // need to do this for the standard delete action as the coordinator picks up the index deletion and removes the config + if (action.isFinishedSuccessfully(executedManagedIndexMetaData)) { + if (action.deleteIndexMetadataAfterFinish()) { + deleteFromManagedIndex(managedIndexConfig, action.type) + return + } + } + if (!updateManagedIndexMetaData(executedManagedIndexMetaData, updateResult).metadataSaved) { logger.error("Failed to update ManagedIndexMetaData after executing the Step : ${step.name}") } @@ -401,7 +460,7 @@ object ManagedIndexRunner : getInitializedManagedIndexMetaData(managedIndexMetaData, managedIndexConfig, policy) } - updateManagedIndexMetaData(updatedManagedIndexMetaData) + updateManagedIndexMetaData(updatedManagedIndexMetaData, create = managedIndexMetaData == null) } @Suppress("ReturnCount", "BlockingMethodInNonBlockingContext") @@ -480,7 +539,7 @@ object ManagedIndexRunner : } } - private fun getFailedInitializedManagedIndexMetaData( + private suspend fun getFailedInitializedManagedIndexMetaData( managedIndexMetaData: ManagedIndexMetaData?, managedIndexConfig: ManagedIndexConfig, policyID: String @@ -497,6 +556,7 @@ object ManagedIndexRunner : policyPrimaryTerm = null, policyCompleted = false, rolledOver = false, + indexCreationDate = getIndexCreationDate(managedIndexConfig), transitionTo = null, stateMetaData = null, actionMetaData = null, @@ -524,6 +584,7 @@ object ManagedIndexRunner : policyPrimaryTerm = policy.primaryTerm, policyCompleted = false, rolledOver = false, + indexCreationDate = getIndexCreationDate(managedIndexConfig), transitionTo = null, stateMetaData = stateMetaData, actionMetaData = null, @@ -544,8 +605,9 @@ object ManagedIndexRunner : // this is an edge case where a user deletes the job config or index and we already have a policySeqNo/primaryTerm // in the metadata, in this case we just want to say we successfully initialized the policy again but we will not // modify the state, action, etc. so it can resume where it left off - managedIndexMetaData.policySeqNo == policy.seqNo && managedIndexMetaData.policyPrimaryTerm == policy.primaryTerm - && managedIndexMetaData.policyID == policy.id -> + managedIndexMetaData.policySeqNo == policy.seqNo && + managedIndexMetaData.policyPrimaryTerm == policy.primaryTerm && + managedIndexMetaData.policyID == policy.id -> // If existing PolicySeqNo and PolicyPrimaryTerm is equal to cached Policy then no issue. managedIndexMetaData.copy( policyRetryInfo = PolicyRetryInfoMetaData(failed = false, consumedRetries = 0), @@ -572,7 +634,8 @@ object ManagedIndexRunner : */ private suspend fun updateManagedIndexMetaData( managedIndexMetaData: ManagedIndexMetaData, - lastUpdateResult: UpdateMetadataResult? = null + lastUpdateResult: UpdateMetadataResult? = null, + create: Boolean = false ): UpdateMetadataResult { var result = UpdateMetadataResult() if (!imIndices.attemptUpdateConfigIndexMapping()) { @@ -585,7 +648,7 @@ object ManagedIndexRunner : metadata = managedIndexMetaData.copy(seqNo = lastUpdateResult.seqNo, primaryTerm = lastUpdateResult.primaryTerm) } - val indexRequest = managedIndexMetadataIndexRequest(metadata) + val indexRequest = managedIndexMetadataIndexRequest(metadata, create = create) try { updateMetaDataRetryPolicy.retry(logger) { val indexResponse: IndexResponse = client.suspendUntil { index(indexRequest, it) } @@ -647,10 +710,10 @@ object ManagedIndexRunner : // if the action to execute is transition then set the actionMetaData to a new transition metadata to reflect we are // in transition (in case we triggered change policy from entering transition) or to reflect this is a new policy transition phase val newTransitionMetaData = ActionMetaData( - ActionConfig.ActionType.TRANSITION.type, Instant.now().toEpochMilli(), -1, + TransitionsAction.name, Instant.now().toEpochMilli(), -1, false, 0, 0, null ) - val actionMetaData = if (actionToExecute?.type == ActionConfig.ActionType.TRANSITION) { + val actionMetaData = if (actionToExecute?.type == TransitionsAction.name) { newTransitionMetaData } else { managedIndexMetaData.actionMetaData @@ -714,7 +777,7 @@ object ManagedIndexRunner : policy.errorNotification?.run { errorNotificationRetryPolicy.retry(logger) { withContext(Dispatchers.IO) { - destination.publish(null, compileTemplate(messageTemplate, managedIndexMetaData), hostDenyList) + // destination.publish(null, compileTemplate(messageTemplate, managedIndexMetaData), hostDenyList) } } } @@ -747,9 +810,6 @@ object ManagedIndexRunner : val response: ClusterStateResponse = client.admin().cluster().suspendUntil { state(clusterStateRequest, it) } indexMetaData = response.state.metadata.indices.firstOrNull()?.value - if (indexMetaData == null) { - logger.error("Could not find IndexMetaData in master cluster state for $index") - } } catch (e: Exception) { logger.error("Failed to get IndexMetaData from master cluster state for index=$index", e) } @@ -778,4 +838,49 @@ object ManagedIndexRunner : Instant.ofEpochMilli(requireNotNull(startTime)) } } + + /** + * Get the index creation date for the first time to cache it on the ManagedIndexMetadata + */ + @Suppress("ReturnCount") + private suspend fun getIndexCreationDate(managedIndexConfig: ManagedIndexConfig): Long? { + try { + val multiTypeIndexNameToMetaData = indexMetadataProvider.getMultiTypeISMIndexMetadata(indexNames = listOf(managedIndexConfig.index)) + // the managedIndexConfig.indexUuid should be unique across all index types + val indexCreationDate = multiTypeIndexNameToMetaData.values.firstOrNull { + it[managedIndexConfig.index]?.indexUuid == managedIndexConfig.indexUuid + }?.get(managedIndexConfig.index)?.indexCreationDate + return indexCreationDate + } catch (e: Exception) { + logger.error("Failed to get the index creation date", e) + } + return null + } + + /** + * Deletes a managedIndexConfig and its managedIndexMetadata. Used after an action has successfully completely + * and the action has deleteIndexMetadataAfterFinish set to true. This should only be set to true for custom + * actions added by extensions, as when indices in the cluster are deleted,the deletion should get picked up + * and the ism config metadata for the index will be removed, but if the index is not in the cluster, as it + * might be for some other index type, we have to do this manually + */ + private suspend fun deleteFromManagedIndex(managedIndexConfig: ManagedIndexConfig, actionType: String) { + try { + val bulkRequest = BulkRequest() + .add(deleteManagedIndexRequest(managedIndexConfig.indexUuid)) + .add(deleteManagedIndexMetadataRequest(managedIndexConfig.indexUuid)) + + val bulkResponse: BulkResponse = client.suspendUntil { bulk(bulkRequest, it) } + for (bulkItemResponse in bulkResponse) { + if (bulkItemResponse.isFailed) { + logger.warn( + "Failed to delete managed index job/metadata [id=${bulkItemResponse.id}] for ${managedIndexConfig.index}" + + " after a successful $actionType [result=${bulkItemResponse.failureMessage}]" + ) + } + } + } catch (e: Exception) { + logger.warn("Failed to delete managed index job for for ${managedIndexConfig.index} after a successful $actionType", e) + } + } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/MetadataService.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/MetadataService.kt index 182fbfebd..b69fcac5c 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/MetadataService.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/MetadataService.kt @@ -51,12 +51,13 @@ class MetadataService( @Volatile private var runningLock = false // in case 2 moveMetadata() process running - private val successfullyIndexedIndices = mutableSetOf() - private var failedToIndexIndices = mutableMapOf() + private val successfullyIndexedIndices = mutableSetOf() + private var failedToIndexIndices = mutableMapOf() private var failedToCleanIndices = mutableSetOf() private var counter = 0 - private var runTimeCounter = 0 + final var runTimeCounter = 1 + private set private val maxRunTime = 10 // used in coordinator sweep to cancel scheduled process @@ -84,12 +85,18 @@ class MetadataService( return } - if (runTimeCounter >= maxRunTime) { + if (!imIndices.indexManagementIndexExists()) { + logger.info("ISM config index not exist, so we cancel the metadata migration job.") + finishFlag = true; runningLock = false; runTimeCounter = 0 + return + } + + if (runTimeCounter > maxRunTime) { updateStatusSetting(-1) finishFlag = true; runningLock = false; runTimeCounter = 0 return } - logger.info("Doing metadata migration ${++runTimeCounter} time.") + logger.info("Doing metadata migration $runTimeCounter time.") val indicesMetadata = clusterService.state().metadata.indices var clusterStateManagedIndexMetadata = indicesMetadata.map { @@ -98,21 +105,37 @@ class MetadataService( // filter out previous failedToClean indices which already been indexed clusterStateManagedIndexMetadata = clusterStateManagedIndexMetadata.filter { it.key !in failedToCleanIndices.map { index -> index.name } } - val indexUuidMap = clusterStateManagedIndexMetadata.map { indicesMetadata[it.key].indexUUID to it.key }.toMap() + + // filter out cluster state metadata with outdated index uuid + val corruptManagedIndices = mutableListOf() + val indexUuidMap = mutableMapOf() + clusterStateManagedIndexMetadata.forEach { (indexName, metadata) -> + val indexMetadata = indicesMetadata[indexName] + val currentIndexUuid = indexMetadata.indexUUID + if (currentIndexUuid != metadata?.indexUuid) { + corruptManagedIndices.add(indexMetadata.index) + } else { + indexUuidMap[currentIndexUuid] = indexName + } + } + logger.info("Corrupt managed indices with outdated index uuid in metadata: $corruptManagedIndices") + clusterStateManagedIndexMetadata = clusterStateManagedIndexMetadata.filter { (indexName, _) -> + indexName !in corruptManagedIndices.map { it.name } + } if (clusterStateManagedIndexMetadata.isEmpty()) { + if (counter++ > 2 && corruptManagedIndices.isEmpty()) { + logger.info("Move Metadata succeed, set finish flag to true. Indices failed to get indexed: $failedToIndexIndices") + updateStatusSetting(1) + finishFlag = true; runningLock = false; runTimeCounter = 0 + return + } if (failedToCleanIndices.isNotEmpty()) { logger.info("Failed to clean indices: $failedToCleanIndices. Only clean cluster state metadata in this run.") cleanMetadatas(failedToCleanIndices.toList()) finishFlag = false; runningLock = false return } - if (counter++ > 2) { - logger.info("Move Metadata succeed, set finish flag to true. Indices failed to get indexed: $failedToIndexIndices") - updateStatusSetting(1) - finishFlag = true; runningLock = false - return - } } else { counter = 0; finishFlag = false // index metadata for indices which metadata hasn't been indexed val bulkIndexReq = @@ -138,10 +161,14 @@ class MetadataService( // clean metadata for indices which metadata already been indexed val indicesToCleanMetadata = indexUuidMap.filter { it.key in successfullyIndexedIndices }.map { Index(it.value, it.key) } - .toList() + failedToCleanIndices + .toList() + failedToCleanIndices + corruptManagedIndices cleanMetadatas(indicesToCleanMetadata) - logger.info("Failed to clean cluster metadata for: ${failedToCleanIndices.map { it.name }}") + if (failedToCleanIndices.isNotEmpty()) { + logger.info("Failed to clean cluster metadata for: ${failedToCleanIndices.map { it.name }}") + } + + runTimeCounter++ } finally { runningLock = false } @@ -237,4 +264,6 @@ class MetadataService( } } -typealias metadataDocID = String +typealias MetadataDocID = String +typealias IndexUuid = String +typealias IndexName = String diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/Action.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/Action.kt deleted file mode 100644 index 7142fdd3d..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/Action.kt +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.action - -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig.ActionType -import org.opensearch.indexmanagement.indexstatemanagement.step.Step - -abstract class Action(val type: ActionType, val config: ActionConfig, val managedIndexMetaData: ManagedIndexMetaData) { - - abstract fun getSteps(): List - - abstract fun getStepToExecute(): Step - - fun isLastStep(stepName: String): Boolean = getSteps().last().name == stepName - - fun isFirstStep(stepName: String?): Boolean = getSteps().first().name == stepName -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/AllocationAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/AllocationAction.kt index a5a878c33..70e9a9cf3 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/AllocationAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/AllocationAction.kt @@ -5,28 +5,58 @@ package org.opensearch.indexmanagement.indexstatemanagement.action -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.AllocationActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.indexmanagement.indexstatemanagement.step.allocation.AttemptAllocationStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext class AllocationAction( - clusterService: ClusterService, - client: Client, - managedIndexMetaData: ManagedIndexMetaData, - config: AllocationActionConfig -) : Action(ActionConfig.ActionType.ALLOCATION, config, managedIndexMetaData) { + val require: Map, + val include: Map, + val exclude: Map, + val waitFor: Boolean = false, + index: Int +) : Action(name, index) { - private val attemptAllocationStep = AttemptAllocationStep(clusterService, client, config, managedIndexMetaData) + init { + require(require.isNotEmpty() || include.isNotEmpty() || exclude.isNotEmpty()) { "At least one allocation parameter need to be specified." } + } + + private val attemptAllocationStep = AttemptAllocationStep(this) private val steps = listOf(attemptAllocationStep) + override fun getStepToExecute(context: StepContext): Step { + return attemptAllocationStep + } + override fun getSteps(): List = steps - override fun getStepToExecute(): Step { - return attemptAllocationStep + override fun populateAction(builder: XContentBuilder, params: ToXContent.Params) { + builder.startObject(type) + builder.field(REQUIRE, require) + builder.field(INCLUDE, include) + builder.field(EXCLUDE, exclude) + builder.field(WAIT_FOR, waitFor) + builder.endObject() + } + + override fun populateAction(out: StreamOutput) { + out.writeMap(require) + out.writeMap(include) + out.writeMap(exclude) + out.writeBoolean(waitFor) + out.writeInt(actionIndex) + } + + companion object { + const val name = "allocation" + const val REQUIRE = "require" + const val INCLUDE = "include" + const val EXCLUDE = "exclude" + const val WAIT_FOR = "wait_for" } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/AllocationActionParser.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/AllocationActionParser.kt new file mode 100644 index 000000000..76aeacc5e --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/AllocationActionParser.kt @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.action + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParser.Token +import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.indexmanagement.indexstatemanagement.action.AllocationAction.Companion.EXCLUDE +import org.opensearch.indexmanagement.indexstatemanagement.action.AllocationAction.Companion.INCLUDE +import org.opensearch.indexmanagement.indexstatemanagement.action.AllocationAction.Companion.REQUIRE +import org.opensearch.indexmanagement.indexstatemanagement.action.AllocationAction.Companion.WAIT_FOR +import org.opensearch.indexmanagement.indexstatemanagement.model.destination.CustomWebhook.Companion.suppressWarning +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser + +class AllocationActionParser : ActionParser() { + override fun fromStreamInput(sin: StreamInput): Action { + val require = suppressWarning(sin.readMap()) + val include = suppressWarning(sin.readMap()) + val exclude = suppressWarning(sin.readMap()) + val waitFor = sin.readBoolean() + val index = sin.readInt() + + return AllocationAction(require, include, exclude, waitFor, index) + } + + override fun fromXContent(xcp: XContentParser, index: Int): Action { + val require: MutableMap = mutableMapOf() + val include: MutableMap = mutableMapOf() + val exclude: MutableMap = mutableMapOf() + var waitFor = false + + ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + when (fieldName) { + REQUIRE -> assignObject(xcp, require) + INCLUDE -> assignObject(xcp, include) + EXCLUDE -> assignObject(xcp, exclude) + WAIT_FOR -> waitFor = xcp.booleanValue() + else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in AllocationAction.") + } + } + return AllocationAction(require, include, exclude, waitFor, index) + } + + override fun getActionType(): String { + return AllocationAction.name + } + + private fun assignObject(xcp: XContentParser, objectMap: MutableMap) { + ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + objectMap[fieldName] = xcp.text() + } + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/CloseAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/CloseAction.kt index 5c6420b76..59d08ba68 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/CloseAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/CloseAction.kt @@ -5,28 +5,25 @@ package org.opensearch.indexmanagement.indexstatemanagement.action -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig.ActionType -import org.opensearch.indexmanagement.indexstatemanagement.model.action.CloseActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.indexstatemanagement.step.close.AttemptCloseStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext class CloseAction( - clusterService: ClusterService, - client: Client, - managedIndexMetaData: ManagedIndexMetaData, - config: CloseActionConfig -) : Action(ActionType.CLOSE, config, managedIndexMetaData) { + index: Int +) : Action(name, index) { - private val attemptCloseStep = AttemptCloseStep(clusterService, client, config, managedIndexMetaData) + companion object { + const val name = "close" + } + private val attemptCloseStep = AttemptCloseStep() private val steps = listOf(attemptCloseStep) - override fun getSteps(): List = steps - - override fun getStepToExecute(): Step { + override fun getStepToExecute(context: StepContext): Step { return attemptCloseStep } + + override fun getSteps(): List = steps } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/CloseActionParser.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/CloseActionParser.kt new file mode 100644 index 000000000..c8ec9d9fd --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/CloseActionParser.kt @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.action + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser + +class CloseActionParser : ActionParser() { + override fun fromStreamInput(sin: StreamInput): Action { + val index = sin.readInt() + return CloseAction(index) + } + + override fun fromXContent(xcp: XContentParser, index: Int): Action { + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + ensureExpectedToken(XContentParser.Token.END_OBJECT, xcp.nextToken(), xcp) + + return CloseAction(index) + } + + override fun getActionType(): String { + return CloseAction.name + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/DeleteAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/DeleteAction.kt index bceeb8c96..fdd325e81 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/DeleteAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/DeleteAction.kt @@ -5,27 +5,25 @@ package org.opensearch.indexmanagement.indexstatemanagement.action -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig.ActionType -import org.opensearch.indexmanagement.indexstatemanagement.model.action.DeleteActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.indexstatemanagement.step.delete.AttemptDeleteStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext class DeleteAction( - clusterService: ClusterService, - client: Client, - managedIndexMetaData: ManagedIndexMetaData, - config: DeleteActionConfig -) : Action(ActionType.DELETE, config, managedIndexMetaData) { + index: Int +) : Action(name, index) { - private val attemptDeleteStep = AttemptDeleteStep(clusterService, client, config, managedIndexMetaData) - private val steps = listOf(attemptDeleteStep) + companion object { + const val name = "delete" + } - override fun getSteps(): List = steps + private val attemptDeleteStep = AttemptDeleteStep() + private val steps = listOf(attemptDeleteStep) - override fun getStepToExecute(): Step { + override fun getStepToExecute(context: StepContext): Step { return attemptDeleteStep } + + override fun getSteps(): List = steps } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/DeleteActionParser.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/DeleteActionParser.kt new file mode 100644 index 000000000..1b2c116f7 --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/DeleteActionParser.kt @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.action + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser + +class DeleteActionParser : ActionParser() { + override fun fromStreamInput(sin: StreamInput): Action { + val index = sin.readInt() + return DeleteAction(index) + } + + override fun fromXContent(xcp: XContentParser, index: Int): Action { + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + ensureExpectedToken(XContentParser.Token.END_OBJECT, xcp.nextToken(), xcp) + + return DeleteAction(index) + } + + override fun getActionType(): String { + return DeleteAction.name + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ForceMergeAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ForceMergeAction.kt index 1e3314bcc..95d3c914f 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ForceMergeAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ForceMergeAction.kt @@ -5,26 +5,28 @@ package org.opensearch.indexmanagement.indexstatemanagement.action -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig.ActionType -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ForceMergeActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.indexmanagement.indexstatemanagement.step.forcemerge.AttemptCallForceMergeStep import org.opensearch.indexmanagement.indexstatemanagement.step.forcemerge.AttemptSetReadOnlyStep import org.opensearch.indexmanagement.indexstatemanagement.step.forcemerge.WaitForForceMergeStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext class ForceMergeAction( - clusterService: ClusterService, - client: Client, - managedIndexMetaData: ManagedIndexMetaData, - config: ForceMergeActionConfig -) : Action(ActionType.FORCE_MERGE, config, managedIndexMetaData) { + val maxNumSegments: Int, + index: Int +) : Action(name, index) { - private val attemptSetReadOnlyStep = AttemptSetReadOnlyStep(clusterService, client, config, managedIndexMetaData) - private val attemptCallForceMergeStep = AttemptCallForceMergeStep(clusterService, client, config, managedIndexMetaData) - private val waitForForceMergeStep = WaitForForceMergeStep(clusterService, client, config, managedIndexMetaData) + init { + require(maxNumSegments > 0) { "Force merge {$MAX_NUM_SEGMENTS_FIELD} must be greater than 0" } + } + + private val attemptSetReadOnlyStep = AttemptSetReadOnlyStep(this) + private val attemptCallForceMergeStep = AttemptCallForceMergeStep(this) + private val waitForForceMergeStep = WaitForForceMergeStep(this) // Using a LinkedHashMap here to maintain order of steps for getSteps() while providing a convenient way to // get the current Step object using the current step's name in getStepToExecute() @@ -34,10 +36,9 @@ class ForceMergeAction( WaitForForceMergeStep.name to waitForForceMergeStep ) - override fun getSteps(): List = stepNameToStep.values.toList() - @Suppress("ReturnCount") - override fun getStepToExecute(): Step { + override fun getStepToExecute(context: StepContext): Step { + val managedIndexMetaData = context.metadata // If stepMetaData is null, return the first step in ForceMergeAction val stepMetaData = managedIndexMetaData.stepMetaData ?: return attemptSetReadOnlyStep val currentStep = stepMetaData.name @@ -61,4 +62,22 @@ class ForceMergeAction( // If the current step has not completed, return it return stepNameToStep[currentStep]!! } + + override fun getSteps(): List = stepNameToStep.values.toList() + + override fun populateAction(builder: XContentBuilder, params: ToXContent.Params) { + builder.startObject(type) + builder.field(MAX_NUM_SEGMENTS_FIELD, maxNumSegments) + builder.endObject() + } + + override fun populateAction(out: StreamOutput) { + out.writeInt(maxNumSegments) + out.writeInt(actionIndex) + } + + companion object { + const val name = "force_merge" + const val MAX_NUM_SEGMENTS_FIELD = "max_num_segments" + } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ForceMergeActionParser.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ForceMergeActionParser.kt new file mode 100644 index 000000000..860a65648 --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ForceMergeActionParser.kt @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.action + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.indexmanagement.indexstatemanagement.action.ForceMergeAction.Companion.MAX_NUM_SEGMENTS_FIELD +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser + +class ForceMergeActionParser : ActionParser() { + override fun fromStreamInput(sin: StreamInput): Action { + val maxNumSegments = sin.readInt() + val index = sin.readInt() + return ForceMergeAction(maxNumSegments, index) + } + + override fun fromXContent(xcp: XContentParser, index: Int): Action { + var maxNumSegments: Int? = null + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + MAX_NUM_SEGMENTS_FIELD -> maxNumSegments = xcp.intValue() + else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in ForceMergeActionConfig.") + } + } + + return ForceMergeAction( + requireNotNull(maxNumSegments) { "ForceMergeActionConfig maxNumSegments is null" }, + index + ) + } + + override fun getActionType(): String { + return ForceMergeAction.name + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexPriorityAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexPriorityAction.kt index fcd6eab41..a879e626d 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexPriorityAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexPriorityAction.kt @@ -5,25 +5,43 @@ package org.opensearch.indexmanagement.indexstatemanagement.action -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig.ActionType -import org.opensearch.indexmanagement.indexstatemanagement.model.action.IndexPriorityActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.indexmanagement.indexstatemanagement.step.indexpriority.AttemptSetIndexPriorityStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext class IndexPriorityAction( - clusterService: ClusterService, - client: Client, - managedIndexMetaData: ManagedIndexMetaData, - config: IndexPriorityActionConfig -) : Action(ActionType.INDEX_PRIORITY, config, managedIndexMetaData) { + val indexPriority: Int, + index: Int +) : Action(name, index) { - private val attemptSetIndexPriorityStep = AttemptSetIndexPriorityStep(clusterService, client, config, managedIndexMetaData) + init { + require(indexPriority >= 0) { "IndexPriorityAction index_priority value must be a non-negative number" } + } + + private val attemptSetIndexPriorityStep = AttemptSetIndexPriorityStep(this) private val steps = listOf(attemptSetIndexPriorityStep) + override fun getStepToExecute(context: StepContext): Step = attemptSetIndexPriorityStep + override fun getSteps(): List = steps - override fun getStepToExecute(): Step = attemptSetIndexPriorityStep + override fun populateAction(builder: XContentBuilder, params: ToXContent.Params) { + builder.startObject(type) + builder.field(INDEX_PRIORITY_FIELD, indexPriority) + builder.endObject() + } + + override fun populateAction(out: StreamOutput) { + out.writeInt(indexPriority) + out.writeInt(actionIndex) + } + + companion object { + const val name = "index_priority" + const val INDEX_PRIORITY_FIELD = "priority" + } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexPriorityActionParser.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexPriorityActionParser.kt new file mode 100644 index 000000000..3bbf349bf --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexPriorityActionParser.kt @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.action + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.indexmanagement.indexstatemanagement.action.IndexPriorityAction.Companion.INDEX_PRIORITY_FIELD +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser + +class IndexPriorityActionParser : ActionParser() { + override fun fromStreamInput(sin: StreamInput): Action { + val indexPriority = sin.readInt() + val index = sin.readInt() + return IndexPriorityAction(indexPriority, index) + } + + override fun fromXContent(xcp: XContentParser, index: Int): Action { + var indexPriority: Int? = null + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + INDEX_PRIORITY_FIELD -> indexPriority = xcp.intValue() + else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in IndexPriorityActionConfig.") + } + } + + return IndexPriorityAction( + indexPriority = requireNotNull(indexPriority) { "$INDEX_PRIORITY_FIELD is null" }, + index = index + ) + } + + override fun getActionType(): String { + return IndexPriorityAction.name + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/NotificationAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/NotificationAction.kt index e6e0a4f14..7437cd24f 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/NotificationAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/NotificationAction.kt @@ -5,29 +5,52 @@ package org.opensearch.indexmanagement.indexstatemanagement.action -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Settings -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.NotificationActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder +import org.opensearch.indexmanagement.indexstatemanagement.model.destination.Destination import org.opensearch.indexmanagement.indexstatemanagement.step.notification.AttemptNotificationStep -import org.opensearch.script.ScriptService +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.script.Script class NotificationAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData, - config: NotificationActionConfig -) : Action(ActionConfig.ActionType.NOTIFICATION, config, managedIndexMetaData) { - - private val attemptNotificationStep = AttemptNotificationStep(clusterService, scriptService, client, settings, config, managedIndexMetaData) + val destination: Destination, + val messageTemplate: Script, + index: Int +) : Action(name, index) { + + init { + require(messageTemplate.lang == MUSTACHE) { "Notification message template must be a mustache script" } + } + + private val attemptNotificationStep = AttemptNotificationStep(this) private val steps = listOf(attemptNotificationStep) + override fun getStepToExecute(context: StepContext): Step { + return attemptNotificationStep + } + override fun getSteps(): List = steps - override fun getStepToExecute(): Step = attemptNotificationStep + override fun populateAction(builder: XContentBuilder, params: ToXContent.Params) { + builder.startObject(type) + builder.field(DESTINATION_FIELD, destination) + builder.field(MESSAGE_TEMPLATE_FIELD, messageTemplate) + builder.endObject() + } + + override fun populateAction(out: StreamOutput) { + destination.writeTo(out) + messageTemplate.writeTo(out) + out.writeInt(actionIndex) + } + + companion object { + const val name = "notification" + const val DESTINATION_FIELD = "destination" + const val MESSAGE_TEMPLATE_FIELD = "message_template" + const val MUSTACHE = "mustache" + } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/NotificationActionParser.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/NotificationActionParser.kt new file mode 100644 index 000000000..e5493cca7 --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/NotificationActionParser.kt @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.action + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParser.Token +import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.indexmanagement.indexstatemanagement.action.NotificationAction.Companion.DESTINATION_FIELD +import org.opensearch.indexmanagement.indexstatemanagement.action.NotificationAction.Companion.MESSAGE_TEMPLATE_FIELD +import org.opensearch.indexmanagement.indexstatemanagement.model.destination.Destination +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser +import org.opensearch.script.Script + +class NotificationActionParser : ActionParser() { + override fun fromStreamInput(sin: StreamInput): Action { + val destination = Destination(sin) + val messageTemplate = Script(sin) + val index = sin.readInt() + + return NotificationAction(destination, messageTemplate, index) + } + + override fun fromXContent(xcp: XContentParser, index: Int): Action { + var destination: Destination? = null + var messageTemplate: Script? = null + + ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + DESTINATION_FIELD -> destination = Destination.parse(xcp) + MESSAGE_TEMPLATE_FIELD -> messageTemplate = Script.parse(xcp, Script.DEFAULT_TEMPLATE_LANG) + else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in NotificationAction.") + } + } + + return NotificationAction( + destination = requireNotNull(destination) { "NotificationAction destination is null" }, + messageTemplate = requireNotNull(messageTemplate) { "NotificationAction message template is null" }, + index = index + ) + } + + override fun getActionType(): String { + return NotificationAction.name + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/OpenAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/OpenAction.kt index e3fce5e8f..de3247521 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/OpenAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/OpenAction.kt @@ -5,28 +5,24 @@ package org.opensearch.indexmanagement.indexstatemanagement.action -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig.ActionType -import org.opensearch.indexmanagement.indexstatemanagement.model.action.OpenActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.indexstatemanagement.step.open.AttemptOpenStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext class OpenAction( - clusterService: ClusterService, - client: Client, - managedIndexMetaData: ManagedIndexMetaData, - config: OpenActionConfig -) : Action(ActionType.OPEN, config, managedIndexMetaData) { - - private val attemptOpenStep = AttemptOpenStep(clusterService, client, config, managedIndexMetaData) + index: Int +) : Action(name, index) { + companion object { + const val name = "open" + } + private val attemptOpenStep = AttemptOpenStep() private val steps = listOf(attemptOpenStep) - override fun getSteps(): List = steps - - override fun getStepToExecute(): Step { + override fun getStepToExecute(context: StepContext): Step { return attemptOpenStep } + + override fun getSteps(): List = steps } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/OpenActionParser.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/OpenActionParser.kt new file mode 100644 index 000000000..f22701922 --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/OpenActionParser.kt @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.action + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser + +class OpenActionParser : ActionParser() { + override fun fromStreamInput(sin: StreamInput): Action { + val index = sin.readInt() + return OpenAction(index) + } + + override fun fromXContent(xcp: XContentParser, index: Int): Action { + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + ensureExpectedToken(XContentParser.Token.END_OBJECT, xcp.nextToken(), xcp) + + return OpenAction(index) + } + + override fun getActionType(): String { + return OpenAction.name + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadOnlyAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadOnlyAction.kt index e9b736b5c..e17e1ae5c 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadOnlyAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadOnlyAction.kt @@ -5,27 +5,24 @@ package org.opensearch.indexmanagement.indexstatemanagement.action -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig.ActionType -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReadOnlyActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.indexstatemanagement.step.readonly.SetReadOnlyStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext class ReadOnlyAction( - clusterService: ClusterService, - client: Client, - managedIndexMetaData: ManagedIndexMetaData, - config: ReadOnlyActionConfig -) : Action(ActionType.READ_ONLY, config, managedIndexMetaData) { + index: Int +) : Action(name, index) { - private val setReadOnlyStep = SetReadOnlyStep(clusterService, client, config, managedIndexMetaData) + companion object { + const val name = "read_only" + } + private val setReadOnlyStep = SetReadOnlyStep() private val steps = listOf(setReadOnlyStep) - override fun getSteps(): List = steps - - override fun getStepToExecute(): Step { + override fun getStepToExecute(context: StepContext): Step { return setReadOnlyStep } + + override fun getSteps(): List = steps } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadOnlyActionParser.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadOnlyActionParser.kt new file mode 100644 index 000000000..43858a12f --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadOnlyActionParser.kt @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.action + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser + +class ReadOnlyActionParser : ActionParser() { + override fun fromStreamInput(sin: StreamInput): Action { + val index = sin.readInt() + return ReadOnlyAction(index) + } + + override fun fromXContent(xcp: XContentParser, index: Int): Action { + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + ensureExpectedToken(XContentParser.Token.END_OBJECT, xcp.nextToken(), xcp) + + return ReadOnlyAction(index) + } + + override fun getActionType(): String { + return ReadOnlyAction.name + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadWriteAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadWriteAction.kt index 19b5a9885..3da520302 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadWriteAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadWriteAction.kt @@ -5,27 +5,25 @@ package org.opensearch.indexmanagement.indexstatemanagement.action -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig.ActionType -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReadWriteActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.indexstatemanagement.step.readwrite.SetReadWriteStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext class ReadWriteAction( - clusterService: ClusterService, - client: Client, - managedIndexMetaData: ManagedIndexMetaData, - config: ReadWriteActionConfig -) : Action(ActionType.READ_WRITE, config, managedIndexMetaData) { + index: Int +) : Action(name, index) { - private val setReadWriteStep = SetReadWriteStep(clusterService, client, config, managedIndexMetaData) - private val steps = listOf(setReadWriteStep) + companion object { + const val name = "read_write" + } - override fun getSteps(): List = steps + private val setReadWriteStep = SetReadWriteStep() + private val steps = listOf(setReadWriteStep) - override fun getStepToExecute(): Step { + override fun getStepToExecute(context: StepContext): Step { return setReadWriteStep } + + override fun getSteps(): List = steps } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadWriteActionParser.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadWriteActionParser.kt new file mode 100644 index 000000000..1093c028c --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadWriteActionParser.kt @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.action + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser + +class ReadWriteActionParser : ActionParser() { + override fun fromStreamInput(sin: StreamInput): Action { + val index = sin.readInt() + return ReadWriteAction(index) + } + + override fun fromXContent(xcp: XContentParser, index: Int): Action { + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + ensureExpectedToken(XContentParser.Token.END_OBJECT, xcp.nextToken(), xcp) + + return ReadWriteAction(index) + } + + override fun getActionType(): String { + return ReadWriteAction.name + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReplicaCountAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReplicaCountAction.kt index 9693ea582..d6e8721c3 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReplicaCountAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReplicaCountAction.kt @@ -5,25 +5,45 @@ package org.opensearch.indexmanagement.indexstatemanagement.action -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig.ActionType -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReplicaCountActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.Step -import org.opensearch.indexmanagement.indexstatemanagement.step.replicacount.AttemptSetReplicaCountStep +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder +import org.opensearch.indexmanagement.indexstatemanagement.step.replicacount.AttemptReplicaCountStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext class ReplicaCountAction( - clusterService: ClusterService, - client: Client, - managedIndexMetaData: ManagedIndexMetaData, - config: ReplicaCountActionConfig -) : Action(ActionType.REPLICA_COUNT, config, managedIndexMetaData) { + val numOfReplicas: Int, + index: Int +) : Action(name, index) { - private val attemptSetReplicaCountStep = AttemptSetReplicaCountStep(clusterService, client, config, managedIndexMetaData) - private val steps = listOf(attemptSetReplicaCountStep) + init { + require(numOfReplicas >= 0) { "ReplicaCountAction number_of_replicas value must be a non-negative number" } + } + + private val attemptReplicaCountStep = AttemptReplicaCountStep(this) + private val steps = listOf(attemptReplicaCountStep) + + override fun getStepToExecute(context: StepContext): Step { + return attemptReplicaCountStep + } override fun getSteps(): List = steps - override fun getStepToExecute(): Step = attemptSetReplicaCountStep + override fun populateAction(builder: XContentBuilder, params: ToXContent.Params) { + builder.startObject(type) + builder.field(NUMBER_OF_REPLICAS_FIELD, numOfReplicas) + builder.endObject() + } + + override fun populateAction(out: StreamOutput) { + out.writeInt(numOfReplicas) + out.writeInt(actionIndex) + } + + companion object { + const val NUMBER_OF_REPLICAS_FIELD = "number_of_replicas" + const val name = "replica_count" + } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReplicaCountActionParser.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReplicaCountActionParser.kt new file mode 100644 index 000000000..89ffa01c8 --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReplicaCountActionParser.kt @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.action + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser + +class ReplicaCountActionParser : ActionParser() { + override fun fromStreamInput(sin: StreamInput): Action { + val numOfReplicas = sin.readInt() + val index = sin.readInt() + return ReplicaCountAction(numOfReplicas, index) + } + + override fun fromXContent(xcp: XContentParser, index: Int): Action { + var numOfReplicas: Int? = null + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + ReplicaCountAction.NUMBER_OF_REPLICAS_FIELD -> numOfReplicas = xcp.intValue() + else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in ReplicaCountActionConfig.") + } + } + + return ReplicaCountAction( + numOfReplicas = requireNotNull(numOfReplicas) { "$ReplicaCountAction.NUMBER_OF_REPLICAS_FIELD is null" }, + index = index + ) + } + + override fun getActionType(): String { + return ReplicaCountAction.name + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverAction.kt index 477f599d5..784e9bd23 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverAction.kt @@ -5,27 +5,64 @@ package org.opensearch.indexmanagement.indexstatemanagement.action -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig.ActionType -import org.opensearch.indexmanagement.indexstatemanagement.model.action.RolloverActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.unit.ByteSizeValue +import org.opensearch.common.unit.TimeValue +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.indexmanagement.indexstatemanagement.step.rollover.AttemptRolloverStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext class RolloverAction( - clusterService: ClusterService, - client: Client, - managedIndexMetaData: ManagedIndexMetaData, - config: RolloverActionConfig -) : Action(ActionType.ROLLOVER, config, managedIndexMetaData) { + val minSize: ByteSizeValue?, + val minDocs: Long?, + val minAge: TimeValue?, + val minPrimaryShardSize: ByteSizeValue?, + index: Int +) : Action(name, index) { - private val attemptRolloverStep = AttemptRolloverStep(clusterService, client, config, managedIndexMetaData) + init { + if (minSize != null) require(minSize.bytes > 0) { "RolloverAction minSize value must be greater than 0" } + + if (minPrimaryShardSize != null) require(minPrimaryShardSize.bytes > 0) { + "RolloverActionConfig minPrimaryShardSize value must be greater than 0" + } + if (minDocs != null) require(minDocs > 0) { "RolloverAction minDocs value must be greater than 0" } + } + + private val attemptRolloverStep = AttemptRolloverStep(this) private val steps = listOf(attemptRolloverStep) + override fun getStepToExecute(context: StepContext): Step { + return attemptRolloverStep + } + override fun getSteps(): List = steps - override fun getStepToExecute(): Step { - return attemptRolloverStep + override fun populateAction(builder: XContentBuilder, params: ToXContent.Params) { + builder.startObject(type) + if (minSize != null) builder.field(MIN_SIZE_FIELD, minSize.stringRep) + if (minDocs != null) builder.field(MIN_DOC_COUNT_FIELD, minDocs) + if (minAge != null) builder.field(MIN_INDEX_AGE_FIELD, minAge.stringRep) + if (minPrimaryShardSize != null) builder.field(MIN_PRIMARY_SHARD_SIZE_FIELD, minPrimaryShardSize.stringRep) + builder.endObject() + } + + override fun populateAction(out: StreamOutput) { + out.writeOptionalWriteable(minSize) + out.writeOptionalLong(minDocs) + out.writeOptionalTimeValue(minAge) + out.writeOptionalWriteable(minPrimaryShardSize) + out.writeInt(actionIndex) + } + + companion object { + const val name = "rollover" + const val MIN_SIZE_FIELD = "min_size" + const val MIN_DOC_COUNT_FIELD = "min_doc_count" + const val MIN_INDEX_AGE_FIELD = "min_index_age" + const val MIN_PRIMARY_SHARD_SIZE_FIELD = "min_primary_shard_size" } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionParser.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionParser.kt new file mode 100644 index 000000000..1b22a786a --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionParser.kt @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.action + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.unit.ByteSizeValue +import org.opensearch.common.unit.TimeValue +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser + +class RolloverActionParser : ActionParser() { + override fun fromStreamInput(sin: StreamInput): Action { + val minSize = sin.readOptionalWriteable(::ByteSizeValue) + val minDocs = sin.readOptionalLong() + val minAge = sin.readOptionalTimeValue() + val minPrimaryShardSize = sin.readOptionalWriteable(::ByteSizeValue) + val index = sin.readInt() + + return RolloverAction(minSize, minDocs, minAge, minPrimaryShardSize, index) + } + + override fun fromXContent(xcp: XContentParser, index: Int): Action { + var minSize: ByteSizeValue? = null + var minDocs: Long? = null + var minAge: TimeValue? = null + var minPrimaryShardSize: ByteSizeValue? = null + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + RolloverAction.MIN_SIZE_FIELD -> minSize = ByteSizeValue.parseBytesSizeValue(xcp.text(), RolloverAction.MIN_SIZE_FIELD) + RolloverAction.MIN_DOC_COUNT_FIELD -> minDocs = xcp.longValue() + RolloverAction.MIN_INDEX_AGE_FIELD -> minAge = TimeValue.parseTimeValue(xcp.text(), RolloverAction.MIN_INDEX_AGE_FIELD) + RolloverAction.MIN_PRIMARY_SHARD_SIZE_FIELD -> minPrimaryShardSize = ByteSizeValue.parseBytesSizeValue( + xcp.text(), + RolloverAction + .MIN_PRIMARY_SHARD_SIZE_FIELD + ) + else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in RolloverAction.") + } + } + + return RolloverAction(minSize, minDocs, minAge, minPrimaryShardSize, index) + } + + override fun getActionType(): String { + return RolloverAction.name + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RollupAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RollupAction.kt index 29d587401..54b4c4898 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RollupAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RollupAction.kt @@ -5,31 +5,34 @@ package org.opensearch.indexmanagement.indexstatemanagement.action -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.RollupActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.indexmanagement.indexstatemanagement.step.rollup.AttemptCreateRollupJobStep import org.opensearch.indexmanagement.indexstatemanagement.step.rollup.WaitForRollupCompletionStep +import org.opensearch.indexmanagement.rollup.model.ISMRollup +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext class RollupAction( - clusterService: ClusterService, - client: Client, - managedIndexMetaData: ManagedIndexMetaData, - config: RollupActionConfig -) : Action(ActionConfig.ActionType.ROLLUP, config, managedIndexMetaData) { + val ismRollup: ISMRollup, + index: Int +) : Action(name, index) { - private val attemptCreateRollupJobStep = AttemptCreateRollupJobStep(clusterService, client, config.ismRollup, managedIndexMetaData) - private val waitForRollupCompletionStep = WaitForRollupCompletionStep(clusterService, client, managedIndexMetaData) + companion object { + const val name = "rollup" + const val ISM_ROLLUP_FIELD = "ism_rollup" + } - override fun getSteps(): List = listOf(attemptCreateRollupJobStep, waitForRollupCompletionStep) + private val attemptCreateRollupJobStep = AttemptCreateRollupJobStep(this) + private val waitForRollupCompletionStep = WaitForRollupCompletionStep() + private val steps = listOf(attemptCreateRollupJobStep, waitForRollupCompletionStep) @Suppress("ReturnCount") - override fun getStepToExecute(): Step { + override fun getStepToExecute(context: StepContext): Step { // If stepMetaData is null, return the first step - val stepMetaData = managedIndexMetaData.stepMetaData ?: return attemptCreateRollupJobStep + val stepMetaData = context.metadata.stepMetaData ?: return attemptCreateRollupJobStep // If the current step has completed, return the next step if (stepMetaData.stepStatus == Step.StepStatus.COMPLETED) { @@ -44,4 +47,17 @@ class RollupAction( else -> waitForRollupCompletionStep } } + + override fun getSteps(): List = steps + + override fun populateAction(builder: XContentBuilder, params: ToXContent.Params) { + builder.startObject(type) + builder.field(ISM_ROLLUP_FIELD, ismRollup) + builder.endObject() + } + + override fun populateAction(out: StreamOutput) { + ismRollup.writeTo(out) + out.writeInt(actionIndex) + } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RollupActionParser.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RollupActionParser.kt new file mode 100644 index 000000000..84c77a4ac --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RollupActionParser.kt @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.action + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils +import org.opensearch.indexmanagement.rollup.model.ISMRollup +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser + +class RollupActionParser : ActionParser() { + override fun fromStreamInput(sin: StreamInput): Action { + val ismRollup = ISMRollup(sin) + val index = sin.readInt() + return RollupAction(ismRollup, index) + } + + override fun fromXContent(xcp: XContentParser, index: Int): Action { + var ismRollup: ISMRollup? = null + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + RollupAction.ISM_ROLLUP_FIELD -> ismRollup = ISMRollup.parse(xcp) + else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in RollupAction.") + } + } + + return RollupAction(ismRollup = requireNotNull(ismRollup) { "RollupAction rollup is null" }, index) + } + + override fun getActionType(): String { + return RollupAction.name + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/SnapshotAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/SnapshotAction.kt index 76f2cf12e..de789402a 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/SnapshotAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/SnapshotAction.kt @@ -5,32 +5,35 @@ package org.opensearch.indexmanagement.indexstatemanagement.action -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig.ActionType -import org.opensearch.indexmanagement.indexstatemanagement.model.action.SnapshotActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.indexmanagement.indexstatemanagement.step.snapshot.AttemptSnapshotStep import org.opensearch.indexmanagement.indexstatemanagement.step.snapshot.WaitForSnapshotStep -import org.opensearch.script.ScriptService +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext class SnapshotAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - managedIndexMetaData: ManagedIndexMetaData, - config: SnapshotActionConfig -) : Action(ActionType.SNAPSHOT, config, managedIndexMetaData) { - private val attemptSnapshotStep = AttemptSnapshotStep(clusterService, scriptService, client, config, managedIndexMetaData) - private val waitForSnapshotStep = WaitForSnapshotStep(clusterService, client, config, managedIndexMetaData) + val repository: String, + val snapshot: String, + index: Int +) : Action(name, index) { - override fun getSteps(): List = listOf(attemptSnapshotStep, waitForSnapshotStep) + companion object { + const val name = "snapshot" + const val REPOSITORY_FIELD = "repository" + const val SNAPSHOT_FIELD = "snapshot" + } + + private val attemptSnapshotStep = AttemptSnapshotStep(this) + private val waitForSnapshotStep = WaitForSnapshotStep(this) + private val steps = listOf(attemptSnapshotStep, waitForSnapshotStep) @Suppress("ReturnCount") - override fun getStepToExecute(): Step { + override fun getStepToExecute(context: StepContext): Step { // If stepMetaData is null, return the first step - val stepMetaData = managedIndexMetaData.stepMetaData ?: return attemptSnapshotStep + val stepMetaData = context.metadata.stepMetaData ?: return attemptSnapshotStep // If the current step has completed, return the next step if (stepMetaData.stepStatus == Step.StepStatus.COMPLETED) { @@ -45,4 +48,19 @@ class SnapshotAction( else -> waitForSnapshotStep } } + + override fun getSteps(): List = steps + + override fun populateAction(builder: XContentBuilder, params: ToXContent.Params) { + builder.startObject(type) + builder.field(REPOSITORY_FIELD, repository) + builder.field(SNAPSHOT_FIELD, snapshot) + builder.endObject() + } + + override fun populateAction(out: StreamOutput) { + out.writeString(repository) + out.writeString(snapshot) + out.writeInt(actionIndex) + } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/SnapshotActionParser.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/SnapshotActionParser.kt new file mode 100644 index 000000000..abf7a58ed --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/SnapshotActionParser.kt @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.action + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParser.Token +import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.indexmanagement.indexstatemanagement.action.SnapshotAction.Companion.REPOSITORY_FIELD +import org.opensearch.indexmanagement.indexstatemanagement.action.SnapshotAction.Companion.SNAPSHOT_FIELD +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser + +class SnapshotActionParser : ActionParser() { + override fun fromStreamInput(sin: StreamInput): Action { + val repository = sin.readString() + val snapshot = sin.readString() + val index = sin.readInt() + + return SnapshotAction(repository, snapshot, index) + } + + override fun fromXContent(xcp: XContentParser, index: Int): Action { + var repository: String? = null + var snapshot: String? = null + + ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + REPOSITORY_FIELD -> repository = xcp.text() + SNAPSHOT_FIELD -> snapshot = xcp.text() + else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in SnapshotAction.") + } + } + + return SnapshotAction( + repository = requireNotNull(repository) { "SnapshotAction repository must be specified" }, + snapshot = requireNotNull(snapshot) { "SnapshotAction snapshot must be specified" }, + index = index + ) + } + + override fun getActionType(): String { + return SnapshotAction.name + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/TransitionsAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/TransitionsAction.kt index 19778797f..60f3a7929 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/TransitionsAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/TransitionsAction.kt @@ -5,28 +5,28 @@ package org.opensearch.indexmanagement.indexstatemanagement.action -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig.ActionType -import org.opensearch.indexmanagement.indexstatemanagement.model.action.TransitionsActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.indexmanagement.indexstatemanagement.IndexMetadataProvider +import org.opensearch.indexmanagement.indexstatemanagement.model.Transition import org.opensearch.indexmanagement.indexstatemanagement.step.transition.AttemptTransitionStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext class TransitionsAction( - clusterService: ClusterService, - client: Client, - managedIndexMetaData: ManagedIndexMetaData, - config: TransitionsActionConfig -) : Action(ActionType.TRANSITION, config, managedIndexMetaData) { + val transitions: List, + val indexMetadataProvider: IndexMetadataProvider +) : Action(name, -1) { - private val attemptTransitionStep = - AttemptTransitionStep(clusterService, client, config, managedIndexMetaData) + private val attemptTransitionStep = AttemptTransitionStep(this) private val steps = listOf(attemptTransitionStep) override fun getSteps(): List = steps - override fun getStepToExecute(): Step { + override fun getStepToExecute(context: StepContext): Step { return attemptTransitionStep } + + companion object { + const val name = "transition" + } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/migration/MigrationServices.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/migration/MigrationServices.kt index 364e5366e..43845844f 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/migration/MigrationServices.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/migration/MigrationServices.kt @@ -254,7 +254,7 @@ class ISMTemplateService( private fun populateV2ISMTemplateMap(policyID: String, indexPatterns: List, priority: Int) { var v1Increment = 0 - val v1MaxOrder = v1orderToBucketIncrement.keys.max() + val v1MaxOrder = v1orderToBucketIncrement.keys.maxOrNull() if (v1MaxOrder != null) { v1Increment = v1MaxOrder + v1orderToBucketIncrement.values.sum() } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ChangePolicy.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ChangePolicy.kt index 5deb05149..87fd71d6e 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ChangePolicy.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ChangePolicy.kt @@ -15,9 +15,9 @@ import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentParser.Token import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken import org.opensearch.commons.authuser.User -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StateMetaData import org.opensearch.indexmanagement.indexstatemanagement.util.WITH_USER import org.opensearch.indexmanagement.opensearchapi.optionalUserField +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaData import java.io.IOException /** diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/Policy.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/Policy.kt index c8a45170d..432fa8e0d 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/Policy.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/Policy.kt @@ -22,6 +22,7 @@ import org.opensearch.indexmanagement.opensearchapi.instant import org.opensearch.indexmanagement.opensearchapi.optionalISMTemplateField import org.opensearch.indexmanagement.opensearchapi.optionalTimeField import org.opensearch.indexmanagement.opensearchapi.optionalUserField +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.util.IndexUtils import java.io.IOException import java.time.Instant @@ -67,7 +68,9 @@ data class Policy( .field(SCHEMA_VERSION_FIELD, schemaVersion) .field(ERROR_NOTIFICATION_FIELD, errorNotification) .field(DEFAULT_STATE_FIELD, defaultState) - .field(STATES_FIELD, states.toTypedArray()) + .startArray(STATES_FIELD) + .also { states.forEach { state -> state.toXContent(it, params) } } + .endArray() .optionalISMTemplateField(ISM_TEMPLATE, ismTemplate) if (params.paramAsBoolean(WITH_USER, true)) builder.optionalUserField(USER_FIELD, user) if (params.paramAsBoolean(WITH_TYPE, true)) builder.endObject() @@ -114,6 +117,32 @@ data class Policy( user?.writeTo(out) } + /** + * Disallowed actions are ones that are not specified in the [ManagedIndexSettings.ALLOW_LIST] setting. + */ + fun getDisallowedActions(allowList: List): List { + val allowListSet = allowList.toSet() + val disallowedActions = mutableListOf() + this.states.forEach { state -> + state.actions.forEach { actionConfig -> + if (!allowListSet.contains(actionConfig.type)) { + disallowedActions.add(actionConfig.type) + } + } + } + return disallowedActions.distinct() + } + + fun getStateToExecute(managedIndexMetaData: ManagedIndexMetaData): State? { + if (managedIndexMetaData.transitionTo != null) { + return this.states.find { it.name == managedIndexMetaData.transitionTo } + } + return this.states.find { + val stateMetaData = managedIndexMetaData.stateMetaData + stateMetaData != null && it.name == stateMetaData.name + } + } + companion object { const val POLICY_TYPE = "policy" const val POLICY_ID_FIELD = "policy_id" diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/State.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/State.kt index 5f4d60fe3..d20169daa 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/State.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/State.kt @@ -14,22 +14,28 @@ import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentParser.Token import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig +import org.opensearch.indexmanagement.indexstatemanagement.ISMActionsParser +import org.opensearch.indexmanagement.indexstatemanagement.IndexMetadataProvider +import org.opensearch.indexmanagement.indexstatemanagement.action.DeleteAction +import org.opensearch.indexmanagement.indexstatemanagement.action.TransitionsAction +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import java.io.IOException data class State( val name: String, - val actions: List, + val actions: List, val transitions: List ) : ToXContentObject, Writeable { init { require(name.isNotBlank()) { "State must contain a valid name" } var hasDelete = false - actions.forEach { actionConfig -> + actions.forEach { action -> // dont allow actions after delete as they will never happen require(!hasDelete) { "State=$name must not contain an action after a delete action" } - hasDelete = actionConfig.type == ActionConfig.ActionType.DELETE + hasDelete = action.type == DeleteAction.name || action.deleteIndexMetadataAfterFinish() } // dont allow transitions if state contains delete @@ -40,7 +46,9 @@ data class State( builder .startObject() .field(NAME_FIELD, name) - .field(ACTIONS_FIELD, actions.toTypedArray()) + .startArray(ACTIONS_FIELD) + .also { actions.forEach { action -> action.toXContent(it, params) } } + .endArray() .field(TRANSITIONS_FIELD, transitions.toTypedArray()) .endObject() return builder @@ -49,7 +57,7 @@ data class State( @Throws(IOException::class) constructor(sin: StreamInput) : this( sin.readString(), - sin.readList { ActionConfig.fromStreamInput(it) }, + sin.readList { ISMActionsParser.instance.fromStreamInput(it) }, sin.readList(::Transition) ) @@ -60,6 +68,40 @@ data class State( out.writeList(transitions) } + fun getActionToExecute( + managedIndexMetaData: ManagedIndexMetaData, + indexMetadataProvider: IndexMetadataProvider + ): Action? { + var actionConfig: Action? + val actionMetaData = managedIndexMetaData.actionMetaData + // If we are transitioning to this state get the first action in the state + // If the action/actionIndex are null it means we just initialized and should get the first action from the state + if (managedIndexMetaData.transitionTo != null || actionMetaData == null) { + actionConfig = this.actions.firstOrNull() ?: TransitionsAction(this.transitions, indexMetadataProvider) + } else if (actionMetaData.name == TransitionsAction.name) { + // If the current action is transition and we do not have a transitionTo set then we should be in Transition + actionConfig = TransitionsAction(this.transitions, indexMetadataProvider) + } else { + // Get the current actionConfig that is in the ManagedIndexMetaData + actionConfig = this.actions.filterIndexed { index, config -> + index == actionMetaData.index && config.type == actionMetaData.name + }.firstOrNull() + if (actionConfig == null) return null + + val stepMetaData = managedIndexMetaData.stepMetaData + // TODO: Refactor so we can get isLastStep from somewhere besides an instantiated Action class so we can simplify this to a when block + // If stepCompleted is true and this is the last step of the action then we should get the next action + if (stepMetaData != null && stepMetaData.stepStatus == Step.StepStatus.COMPLETED) { + val action = actionConfig + if (action.isLastStep(stepMetaData.name)) { + actionConfig = this.actions.getOrNull(actionMetaData.index + 1) ?: TransitionsAction(this.transitions, indexMetadataProvider) + } + } + } + + return actionConfig + } + companion object { const val NAME_FIELD = "name" const val ACTIONS_FIELD = "actions" @@ -69,7 +111,7 @@ data class State( @Throws(IOException::class) fun parse(xcp: XContentParser): State { var name: String? = null - val actions: MutableList = mutableListOf() + val actions: MutableList = mutableListOf() val transitions: MutableList = mutableListOf() ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) @@ -82,7 +124,7 @@ data class State( ACTIONS_FIELD -> { ensureExpectedToken(Token.START_ARRAY, xcp.currentToken(), xcp) while (xcp.nextToken() != Token.END_ARRAY) { - actions.add(ActionConfig.parse(xcp, actions.size)) + actions.add(ISMActionsParser.instance.parse(xcp, actions.size)) } } TRANSITIONS_FIELD -> { diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ActionConfig.kt deleted file mode 100644 index f9329f58b..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ActionConfig.kt +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.io.stream.StreamInput -import org.opensearch.common.io.stream.StreamOutput -import org.opensearch.common.io.stream.Writeable -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentFragment -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.script.ScriptService -import java.io.IOException - -abstract class ActionConfig( - val type: ActionType, - val actionIndex: Int -) : ToXContentFragment, Writeable { - - var configTimeout: ActionTimeout? = null - var configRetry: ActionRetry? = null - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - configTimeout?.toXContent(builder, params) - configRetry?.toXContent(builder, params) - return builder - } - - abstract fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action - - enum class ActionType(val type: String) { - DELETE("delete"), - TRANSITION("transition"), - ROLLOVER("rollover"), - CLOSE("close"), - OPEN("open"), - READ_ONLY("read_only"), - READ_WRITE("read_write"), - REPLICA_COUNT("replica_count"), - FORCE_MERGE("force_merge"), - NOTIFICATION("notification"), - SNAPSHOT("snapshot"), - INDEX_PRIORITY("index_priority"), - ALLOCATION("allocation"), - ROLLUP("rollup"); - - override fun toString(): String { - return type - } - } - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - out.writeEnum(type) - out.writeInt(actionIndex) - out.writeOptionalWriteable(configTimeout) - out.writeOptionalWriteable(configRetry) - } - - companion object { - private const val DEFAULT_RETRIES = 3L - - // TODO clean up for actionIndex - @JvmStatic - @Throws(IOException::class) - @Suppress("ComplexMethod") - fun fromStreamInput(sin: StreamInput): ActionConfig { - val type = sin.readEnum(ActionType::class.java) - val actionIndex = sin.readInt() - val configTimeout = sin.readOptionalWriteable(::ActionTimeout) - val configRetry = sin.readOptionalWriteable(::ActionRetry) - - val actionConfig: ActionConfig = when (type.type) { - ActionType.DELETE.type -> DeleteActionConfig(actionIndex) - ActionType.OPEN.type -> OpenActionConfig(actionIndex) - ActionType.CLOSE.type -> CloseActionConfig(actionIndex) - ActionType.READ_ONLY.type -> ReadOnlyActionConfig(actionIndex) - ActionType.READ_WRITE.type -> ReadWriteActionConfig(actionIndex) - ActionType.ROLLOVER.type -> RolloverActionConfig(sin) - ActionType.REPLICA_COUNT.type -> ReplicaCountActionConfig(sin) - ActionType.FORCE_MERGE.type -> ForceMergeActionConfig(sin) - ActionType.NOTIFICATION.type -> NotificationActionConfig(sin) - ActionType.SNAPSHOT.type -> SnapshotActionConfig(sin) - ActionType.INDEX_PRIORITY.type -> IndexPriorityActionConfig(sin) - ActionType.ALLOCATION.type -> AllocationActionConfig(sin) - ActionType.ROLLUP.type -> RollupActionConfig(sin) - else -> throw IllegalArgumentException("Invalid field: [${type.type}] found in Action.") - } - - actionConfig.configTimeout = configTimeout - actionConfig.configRetry = configRetry - - return actionConfig - } - - @Suppress("ComplexMethod") - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, index: Int): ActionConfig { - var actionConfig: ActionConfig? = null - var timeout: ActionTimeout? = null - var retry: ActionRetry? = ActionRetry(DEFAULT_RETRIES) - - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - - when (fieldName) { - ActionTimeout.TIMEOUT_FIELD -> timeout = ActionTimeout.parse(xcp) - ActionRetry.RETRY_FIELD -> retry = ActionRetry.parse(xcp) - ActionType.DELETE.type -> actionConfig = DeleteActionConfig.parse(xcp, index) - ActionType.ROLLOVER.type -> actionConfig = RolloverActionConfig.parse(xcp, index) - ActionType.OPEN.type -> actionConfig = OpenActionConfig.parse(xcp, index) - ActionType.CLOSE.type -> actionConfig = CloseActionConfig.parse(xcp, index) - ActionType.READ_ONLY.type -> actionConfig = ReadOnlyActionConfig.parse(xcp, index) - ActionType.READ_WRITE.type -> actionConfig = ReadWriteActionConfig.parse(xcp, index) - ActionType.REPLICA_COUNT.type -> actionConfig = ReplicaCountActionConfig.parse(xcp, index) - ActionType.FORCE_MERGE.type -> actionConfig = ForceMergeActionConfig.parse(xcp, index) - ActionType.NOTIFICATION.type -> actionConfig = NotificationActionConfig.parse(xcp, index) - ActionType.SNAPSHOT.type -> actionConfig = SnapshotActionConfig.parse(xcp, index) - ActionType.INDEX_PRIORITY.type -> actionConfig = IndexPriorityActionConfig.parse(xcp, index) - ActionType.ALLOCATION.type -> actionConfig = AllocationActionConfig.parse(xcp, index) - ActionType.ROLLUP.type -> actionConfig = RollupActionConfig.parse(xcp, index) - else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in Action.") - } - } - - requireNotNull(actionConfig) { "ActionConfig inside state is null" } - - actionConfig.configTimeout = timeout - actionConfig.configRetry = retry - - return actionConfig - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/AllocationActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/AllocationActionConfig.kt deleted file mode 100644 index fbe73594e..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/AllocationActionConfig.kt +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.io.stream.StreamInput -import org.opensearch.common.io.stream.StreamOutput -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentObject -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.action.AllocationAction -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.script.ScriptService -import java.io.IOException - -data class AllocationActionConfig( - val require: Map, - val include: Map, - val exclude: Map, - val waitFor: Boolean = false, - val index: Int -) : ToXContentObject, ActionConfig(ActionType.ALLOCATION, index) { - - init { - require(require.isNotEmpty() || include.isNotEmpty() || exclude.isNotEmpty()) { "At least one allocation parameter need to be specified." } - } - - override fun isFragment(): Boolean = super.isFragment() - - override fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action = AllocationAction(clusterService, client, managedIndexMetaData, this) - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - super.toXContent(builder, params) - .startObject(ActionType.ALLOCATION.type) - if (require.isNotEmpty()) builder.field(REQUIRE, require) - if (include.isNotEmpty()) builder.field(INCLUDE, include) - if (exclude.isNotEmpty()) builder.field(EXCLUDE, exclude) - return builder.field(WAIT_FOR, waitFor) - .endObject() - .endObject() - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - require = suppressWarning(sin.readMap()), - include = suppressWarning(sin.readMap()), - exclude = suppressWarning(sin.readMap()), - waitFor = sin.readBoolean(), - index = sin.readInt() - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - out.writeMap(require) - out.writeMap(include) - out.writeMap(exclude) - out.writeBoolean(waitFor) - out.writeInt(index) - } - - companion object { - const val REQUIRE = "require" - const val INCLUDE = "include" - const val EXCLUDE = "exclude" - const val WAIT_FOR = "wait_for" - - @Suppress("UNCHECKED_CAST") - fun suppressWarning(map: MutableMap?): MutableMap { - return map as MutableMap - } - - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, index: Int): AllocationActionConfig { - val require: MutableMap = mutableMapOf() - val include: MutableMap = mutableMapOf() - val exclude: MutableMap = mutableMapOf() - var waitFor = false - - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - when (fieldName) { - REQUIRE -> assignObject(xcp, require) - INCLUDE -> assignObject(xcp, include) - EXCLUDE -> assignObject(xcp, exclude) - WAIT_FOR -> waitFor = xcp.booleanValue() - else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in AllocationActionConfig.") - } - } - return AllocationActionConfig(require, include, exclude, waitFor, index) - } - - private fun assignObject(xcp: XContentParser, objectMap: MutableMap) { - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - objectMap[fieldName] = xcp.text() - } - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/CloseActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/CloseActionConfig.kt deleted file mode 100644 index 8b02366f9..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/CloseActionConfig.kt +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentObject -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.action.CloseAction -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.script.ScriptService -import java.io.IOException - -data class CloseActionConfig( - val index: Int -) : ToXContentObject, ActionConfig(ActionType.CLOSE, index) { - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - super.toXContent(builder, params) - .startObject(ActionType.CLOSE.type) - return builder.endObject().endObject() - } - - override fun isFragment(): Boolean = super.isFragment() - - override fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action = CloseAction(clusterService, client, managedIndexMetaData, this) - - companion object { - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, index: Int): CloseActionConfig { - - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - ensureExpectedToken(Token.END_OBJECT, xcp.nextToken(), xcp) - - return CloseActionConfig(index) - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/DeleteActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/DeleteActionConfig.kt deleted file mode 100644 index 97a26adf1..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/DeleteActionConfig.kt +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentObject -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.action.DeleteAction -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.script.ScriptService -import java.io.IOException - -data class DeleteActionConfig( - val index: Int -) : ToXContentObject, ActionConfig(ActionType.DELETE, index) { - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - super.toXContent(builder, params) - .startObject(ActionType.DELETE.type) - return builder.endObject().endObject() - } - - override fun isFragment(): Boolean = super.isFragment() - - override fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action = DeleteAction(clusterService, client, managedIndexMetaData, this) - - companion object { - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, index: Int): DeleteActionConfig { - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - ensureExpectedToken(Token.END_OBJECT, xcp.nextToken(), xcp) - - return DeleteActionConfig(index) - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ForceMergeActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ForceMergeActionConfig.kt deleted file mode 100644 index f2cb128cb..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ForceMergeActionConfig.kt +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.io.stream.StreamInput -import org.opensearch.common.io.stream.StreamOutput -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentObject -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.action.ForceMergeAction -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.script.ScriptService -import java.io.IOException - -data class ForceMergeActionConfig( - val maxNumSegments: Int, - val index: Int -) : ToXContentObject, ActionConfig(ActionType.FORCE_MERGE, index) { - - init { - require(maxNumSegments > 0) { "Force merge {$MAX_NUM_SEGMENTS_FIELD} must be greater than 0" } - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - super.toXContent(builder, params) - .startObject(ActionType.FORCE_MERGE.type) - .field(MAX_NUM_SEGMENTS_FIELD, maxNumSegments) - .endObject() - return builder.endObject() - } - - override fun isFragment(): Boolean = super.isFragment() - - override fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action = ForceMergeAction(clusterService, client, managedIndexMetaData, this) - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - maxNumSegments = sin.readInt(), - index = sin.readInt() - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - out.writeInt(maxNumSegments) - out.writeInt(index) - } - - companion object { - const val MAX_NUM_SEGMENTS_FIELD = "max_num_segments" - - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, index: Int): ForceMergeActionConfig { - var maxNumSegments: Int? = null - - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - - when (fieldName) { - MAX_NUM_SEGMENTS_FIELD -> maxNumSegments = xcp.intValue() - else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in ForceMergeActionConfig.") - } - } - - return ForceMergeActionConfig( - requireNotNull(maxNumSegments) { "ForceMergeActionConfig maxNumSegments is null" }, - index - ) - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/IndexPriorityActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/IndexPriorityActionConfig.kt deleted file mode 100644 index 7437743f1..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/IndexPriorityActionConfig.kt +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.io.stream.StreamInput -import org.opensearch.common.io.stream.StreamOutput -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentObject -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.action.IndexPriorityAction -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.script.ScriptService -import java.io.IOException - -data class IndexPriorityActionConfig( - val indexPriority: Int, - val index: Int -) : ToXContentObject, ActionConfig(ActionType.INDEX_PRIORITY, index) { - - init { - require(indexPriority >= 0) { "IndexPriorityActionConfig index_priority value must be a non-negative number" } - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - super.toXContent(builder, params).startObject(ActionType.INDEX_PRIORITY.type) - builder.field(INDEX_PRIORITY_FIELD, indexPriority) - return builder.endObject().endObject() - } - - override fun isFragment(): Boolean = super.isFragment() - - override fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action = IndexPriorityAction(clusterService, client, managedIndexMetaData, this) - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - indexPriority = sin.readInt(), - index = sin.readInt() - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - out.writeInt(indexPriority) - out.writeInt(index) - } - - companion object { - const val INDEX_PRIORITY_FIELD = "priority" - - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, index: Int): IndexPriorityActionConfig { - var indexPriority: Int? = null - - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - - when (fieldName) { - INDEX_PRIORITY_FIELD -> indexPriority = xcp.intValue() - else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in IndexPriorityActionConfig.") - } - } - - return IndexPriorityActionConfig( - indexPriority = requireNotNull(indexPriority) { "$INDEX_PRIORITY_FIELD is null" }, - index = index - ) - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/NotificationActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/NotificationActionConfig.kt deleted file mode 100644 index e0a69c791..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/NotificationActionConfig.kt +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.io.stream.StreamInput -import org.opensearch.common.io.stream.StreamOutput -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentObject -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.action.NotificationAction -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.destination.Destination -import org.opensearch.script.Script -import org.opensearch.script.ScriptService -import java.io.IOException - -data class NotificationActionConfig( - val destination: Destination, - val messageTemplate: Script, - val index: Int -) : ToXContentObject, ActionConfig(ActionType.NOTIFICATION, index) { - - init { - require(messageTemplate.lang == MUSTACHE) { "Notification message template must be a mustache script" } - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - super.toXContent(builder, params).startObject(ActionType.NOTIFICATION.type) - builder.field(DESTINATION_FIELD, destination) - .field(MESSAGE_TEMPLATE_FIELD, messageTemplate) - .endObject() - .endObject() - return builder - } - - override fun isFragment(): Boolean = super.isFragment() - - override fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action = NotificationAction(clusterService, scriptService, client, settings, managedIndexMetaData, this) - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - destination = Destination(sin), - messageTemplate = Script(sin), - index = sin.readInt() - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - destination.writeTo(out) - messageTemplate.writeTo(out) - out.writeInt(index) - } - - companion object { - const val DESTINATION_FIELD = "destination" - const val MESSAGE_TEMPLATE_FIELD = "message_template" - const val MUSTACHE = "mustache" - - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, index: Int): NotificationActionConfig { - var destination: Destination? = null - var messageTemplate: Script? = null - - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - - when (fieldName) { - DESTINATION_FIELD -> destination = Destination.parse(xcp) - MESSAGE_TEMPLATE_FIELD -> messageTemplate = Script.parse(xcp, Script.DEFAULT_TEMPLATE_LANG) - else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in NotificationActionConfig.") - } - } - - return NotificationActionConfig( - destination = requireNotNull(destination) { "NotificationActionConfig destination is null" }, - messageTemplate = requireNotNull(messageTemplate) { "NotificationActionConfig message template is null" }, - index = index - ) - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/OpenActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/OpenActionConfig.kt deleted file mode 100644 index 0bbc678cc..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/OpenActionConfig.kt +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentObject -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.action.OpenAction -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.script.ScriptService -import java.io.IOException - -data class OpenActionConfig( - val index: Int -) : ToXContentObject, ActionConfig(ActionType.OPEN, index) { - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - super.toXContent(builder, params) - .startObject(ActionType.OPEN.type) - return builder.endObject().endObject() - } - - override fun isFragment(): Boolean = super.isFragment() - - override fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action = OpenAction(clusterService, client, managedIndexMetaData, this) - - companion object { - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, index: Int): OpenActionConfig { - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - ensureExpectedToken(Token.END_OBJECT, xcp.nextToken(), xcp) - - return OpenActionConfig(index) - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ReadOnlyActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ReadOnlyActionConfig.kt deleted file mode 100644 index 485151752..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ReadOnlyActionConfig.kt +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentObject -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.action.ReadOnlyAction -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.script.ScriptService -import java.io.IOException - -data class ReadOnlyActionConfig( - val index: Int -) : ToXContentObject, ActionConfig(ActionType.READ_ONLY, index) { - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - super.toXContent(builder, params) - .startObject(ActionType.READ_ONLY.type) - return builder.endObject().endObject() - } - - override fun isFragment(): Boolean = super.isFragment() - - override fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action = ReadOnlyAction(clusterService, client, managedIndexMetaData, this) - - companion object { - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, index: Int): ReadOnlyActionConfig { - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - ensureExpectedToken(Token.END_OBJECT, xcp.nextToken(), xcp) - - return ReadOnlyActionConfig(index) - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ReadWriteActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ReadWriteActionConfig.kt deleted file mode 100644 index 7044ac3a4..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ReadWriteActionConfig.kt +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentObject -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.action.ReadWriteAction -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.script.ScriptService -import java.io.IOException - -data class ReadWriteActionConfig( - val index: Int -) : ToXContentObject, ActionConfig(ActionType.READ_WRITE, index) { - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - super.toXContent(builder, params) - .startObject(ActionType.READ_WRITE.type) - - return builder.endObject().endObject() - } - - override fun isFragment(): Boolean = super.isFragment() - - override fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action = ReadWriteAction(clusterService, client, managedIndexMetaData, this) - - companion object { - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, index: Int): ReadWriteActionConfig { - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - ensureExpectedToken(Token.END_OBJECT, xcp.nextToken(), xcp) - - return ReadWriteActionConfig(index) - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ReplicaCountActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ReplicaCountActionConfig.kt deleted file mode 100644 index 4bfcd1c2e..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/ReplicaCountActionConfig.kt +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.io.stream.StreamInput -import org.opensearch.common.io.stream.StreamOutput -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentObject -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.action.ReplicaCountAction -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.script.ScriptService -import java.io.IOException - -data class ReplicaCountActionConfig( - val numOfReplicas: Int, - val index: Int -) : ToXContentObject, ActionConfig(ActionType.REPLICA_COUNT, index) { - - init { - require(numOfReplicas >= 0) { "ReplicaCountActionConfig number_of_replicas value must be a non-negative number" } - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - super.toXContent(builder, params).startObject(ActionType.REPLICA_COUNT.type) - builder.field(NUMBER_OF_REPLICAS_FIELD, numOfReplicas) - return builder.endObject().endObject() - } - - override fun isFragment(): Boolean = super.isFragment() - - override fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action = ReplicaCountAction(clusterService, client, managedIndexMetaData, this) - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - numOfReplicas = sin.readInt(), - index = sin.readInt() - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - out.writeInt(numOfReplicas) - out.writeInt(index) - } - - companion object { - const val NUMBER_OF_REPLICAS_FIELD = "number_of_replicas" - - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, index: Int): ReplicaCountActionConfig { - var numOfReplicas: Int? = null - - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - - when (fieldName) { - NUMBER_OF_REPLICAS_FIELD -> numOfReplicas = xcp.intValue() - else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in ReplicaCountActionConfig.") - } - } - - return ReplicaCountActionConfig( - numOfReplicas = requireNotNull(numOfReplicas) { "$NUMBER_OF_REPLICAS_FIELD is null" }, - index = index - ) - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/RolloverActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/RolloverActionConfig.kt deleted file mode 100644 index 1b951497d..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/RolloverActionConfig.kt +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.io.stream.StreamInput -import org.opensearch.common.io.stream.StreamOutput -import org.opensearch.common.settings.Settings -import org.opensearch.common.unit.ByteSizeValue -import org.opensearch.common.unit.TimeValue -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentObject -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.action.RolloverAction -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.script.ScriptService -import java.io.IOException - -data class RolloverActionConfig( - val minSize: ByteSizeValue?, - val minDocs: Long?, - val minAge: TimeValue?, - val minPrimaryShardSize: ByteSizeValue?, - val index: Int -) : ToXContentObject, ActionConfig(ActionType.ROLLOVER, index) { - - init { - if (minSize != null) require(minSize.bytes > 0) { "RolloverActionConfig minSize value must be greater than 0" } - if (minPrimaryShardSize != null) require(minPrimaryShardSize.bytes > 0) { - "RolloverActionConfig minPrimaryShardSize value must be greater than 0" - } - if (minDocs != null) require(minDocs > 0) { "RolloverActionConfig minDocs value must be greater than 0" } - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - super.toXContent(builder, params) - .startObject(ActionType.ROLLOVER.type) - if (minSize != null) builder.field(MIN_SIZE_FIELD, minSize.stringRep) - if (minDocs != null) builder.field(MIN_DOC_COUNT_FIELD, minDocs) - if (minAge != null) builder.field(MIN_INDEX_AGE_FIELD, minAge.stringRep) - if (minPrimaryShardSize != null) builder.field(MIN_PRIMARY_SHARD_SIZE_FIELD, minPrimaryShardSize.stringRep) - return builder.endObject().endObject() - } - - override fun isFragment(): Boolean = super.isFragment() - - override fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action = RolloverAction(clusterService, client, managedIndexMetaData, this) - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - minSize = sin.readOptionalWriteable(::ByteSizeValue), - minDocs = sin.readOptionalLong(), - minAge = sin.readOptionalTimeValue(), - minPrimaryShardSize = sin.readOptionalWriteable(::ByteSizeValue), - index = sin.readInt() - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - out.writeOptionalWriteable(minSize) - out.writeOptionalLong(minDocs) - out.writeOptionalTimeValue(minAge) - out.writeOptionalWriteable(minPrimaryShardSize) - out.writeInt(index) - } - - companion object { - const val MIN_SIZE_FIELD = "min_size" - const val MIN_DOC_COUNT_FIELD = "min_doc_count" - const val MIN_INDEX_AGE_FIELD = "min_index_age" - const val MIN_PRIMARY_SHARD_SIZE_FIELD = "min_primary_shard_size" - - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, index: Int): RolloverActionConfig { - var minSize: ByteSizeValue? = null - var minDocs: Long? = null - var minAge: TimeValue? = null - var minPrimaryShardSize: ByteSizeValue? = null - - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - - when (fieldName) { - MIN_SIZE_FIELD -> minSize = ByteSizeValue.parseBytesSizeValue(xcp.text(), MIN_SIZE_FIELD) - MIN_DOC_COUNT_FIELD -> minDocs = xcp.longValue() - MIN_INDEX_AGE_FIELD -> minAge = TimeValue.parseTimeValue(xcp.text(), MIN_INDEX_AGE_FIELD) - MIN_PRIMARY_SHARD_SIZE_FIELD -> minPrimaryShardSize = ByteSizeValue.parseBytesSizeValue(xcp.text(), MIN_PRIMARY_SHARD_SIZE_FIELD) - else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in RolloverActionConfig.") - } - } - - return RolloverActionConfig( - minSize = minSize, - minDocs = minDocs, - minAge = minAge, - minPrimaryShardSize = minPrimaryShardSize, - index = index - ) - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/RollupActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/RollupActionConfig.kt deleted file mode 100644 index 7ebc8ef54..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/RollupActionConfig.kt +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.io.stream.StreamInput -import org.opensearch.common.io.stream.StreamOutput -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentObject -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParserUtils -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.action.RollupAction -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.rollup.model.ISMRollup -import org.opensearch.script.ScriptService -import java.io.IOException - -class RollupActionConfig( - val ismRollup: ISMRollup, - val index: Int -) : ToXContentObject, ActionConfig(ActionType.ROLLUP, index) { - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - super.toXContent(builder, params) - .startObject(ActionType.ROLLUP.type) - .field(ISM_ROLLUP_FIELD, ismRollup) - .endObject() - .endObject() - return builder - } - - override fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action = RollupAction(clusterService, client, managedIndexMetaData, this) - - override fun isFragment(): Boolean = super.isFragment() - - @Throws(IOException::class) - constructor(sin: StreamInput) : this(ismRollup = ISMRollup(sin), index = sin.readInt()) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - ismRollup.writeTo(out) - out.writeInt(actionIndex) - } - - companion object { - const val ISM_ROLLUP_FIELD = "ism_rollup" - var ismRollup: ISMRollup? = null - - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, actionIndex: Int): RollupActionConfig { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - - when (fieldName) { - ISM_ROLLUP_FIELD -> ismRollup = ISMRollup.parse(xcp) - else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in RollupActionConfig.") - } - } - - return RollupActionConfig( - ismRollup = requireNotNull(ismRollup) { "RollupActionConfig rollup is null" }, - index = actionIndex - ) - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/SnapshotActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/SnapshotActionConfig.kt deleted file mode 100644 index bf551fb84..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/SnapshotActionConfig.kt +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.io.stream.StreamInput -import org.opensearch.common.io.stream.StreamOutput -import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentObject -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.common.xcontent.XContentParser -import org.opensearch.common.xcontent.XContentParser.Token -import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.action.SnapshotAction -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.script.ScriptService -import java.io.IOException - -data class SnapshotActionConfig( - val repository: String, - val snapshot: String, - val index: Int -) : ToXContentObject, ActionConfig(ActionType.SNAPSHOT, index) { - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - super.toXContent(builder, params) - .startObject(ActionType.SNAPSHOT.type) - .field(REPOSITORY_FIELD, repository) - .field(SNAPSHOT_FIELD, snapshot) - return builder.endObject().endObject() - } - - override fun isFragment(): Boolean = super.isFragment() - - override fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action = SnapshotAction(clusterService, scriptService, client, managedIndexMetaData, this) - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - repository = sin.readString(), - snapshot = sin.readString(), - index = sin.readInt() - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - out.writeString(repository) - out.writeString(snapshot) - out.writeInt(index) - } - - companion object { - const val REPOSITORY_FIELD = "repository" - const val SNAPSHOT_FIELD = "snapshot" - const val INCLUDE_GLOBAL_STATE = "include_global_state" - - @JvmStatic - @Throws(IOException::class) - fun parse(xcp: XContentParser, index: Int): SnapshotActionConfig { - var repository: String? = null - var snapshot: String? = null - - ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) - while (xcp.nextToken() != Token.END_OBJECT) { - val fieldName = xcp.currentName() - xcp.nextToken() - - when (fieldName) { - REPOSITORY_FIELD -> repository = xcp.text() - SNAPSHOT_FIELD -> snapshot = xcp.text() - else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in SnapshotActionConfig.") - } - } - - return SnapshotActionConfig( - repository = requireNotNull(repository) { "SnapshotActionConfig repository must be specified" }, - snapshot = requireNotNull(snapshot) { "SnapshotActionConfig snapshot must be specified" }, - index = index - ) - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/TransitionsActionConfig.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/TransitionsActionConfig.kt deleted file mode 100644 index 0e73bd1c9..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/action/TransitionsActionConfig.kt +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.model.action - -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Settings -import org.opensearch.indexmanagement.indexstatemanagement.action.Action -import org.opensearch.indexmanagement.indexstatemanagement.action.TransitionsAction -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.Transition -import org.opensearch.script.ScriptService - -data class TransitionsActionConfig( - val transitions: List -) : ActionConfig(ActionType.TRANSITION, -1) { - - override fun toAction( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData - ): Action = TransitionsAction(clusterService, client, managedIndexMetaData, this) -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/destination/Destination.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/destination/Destination.kt index 092cbfec9..b1a20cf46 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/destination/Destination.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/destination/Destination.kt @@ -6,12 +6,12 @@ package org.opensearch.indexmanagement.indexstatemanagement.model.destination import org.apache.logging.log4j.LogManager -import org.opensearch.alerting.destination.Notification -import org.opensearch.alerting.destination.message.BaseMessage -import org.opensearch.alerting.destination.message.ChimeMessage -import org.opensearch.alerting.destination.message.CustomWebhookMessage -import org.opensearch.alerting.destination.message.SlackMessage -import org.opensearch.alerting.destination.response.DestinationResponse +// import org.opensearch.alerting.destination.Notification +// import org.opensearch.alerting.destination.message.BaseMessage +// import org.opensearch.alerting.destination.message.ChimeMessage +// import org.opensearch.alerting.destination.message.CustomWebhookMessage +// import org.opensearch.alerting.destination.message.SlackMessage +// import org.opensearch.alerting.destination.response.DestinationResponse import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.io.stream.StreamOutput import org.opensearch.common.io.stream.Writeable @@ -21,7 +21,7 @@ import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentParser.Token import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken -import org.opensearch.indexmanagement.indexstatemanagement.util.isHostInDenylist +// import org.opensearch.indexmanagement.indexstatemanagement.util.isHostInDenylist import org.opensearch.indexmanagement.opensearchapi.convertToMap import java.io.IOException @@ -105,41 +105,41 @@ data class Destination( } } - @Throws(IOException::class) - fun publish(compiledSubject: String?, compiledMessage: String, denyHostRanges: List): DestinationResponse { - val destinationMessage: BaseMessage - when (type) { - DestinationType.CHIME -> { - val messageContent = chime?.constructMessageContent(compiledSubject, compiledMessage) - destinationMessage = ChimeMessage.Builder("chime_message") - .withUrl(chime?.url) - .withMessage(messageContent) - .build() - } - DestinationType.SLACK -> { - val messageContent = slack?.constructMessageContent(compiledSubject, compiledMessage) - destinationMessage = SlackMessage.Builder("slack_message") - .withUrl(slack?.url) - .withMessage(messageContent) - .build() - } - DestinationType.CUSTOM_WEBHOOK -> { - destinationMessage = CustomWebhookMessage.Builder("custom_webhook") - .withUrl(customWebhook?.url) - .withScheme(customWebhook?.scheme) - .withHost(customWebhook?.host) - .withPort(customWebhook?.port) - .withPath(customWebhook?.path) - .withQueryParams(customWebhook?.queryParams) - .withHeaderParams(customWebhook?.headerParams) - .withMessage(compiledMessage).build() - } - } - validateDestinationUri(destinationMessage, denyHostRanges) - val response = Notification.publish(destinationMessage) as DestinationResponse - logger.info("Message published for action type: $type, messageid: ${response.responseContent}, statuscode: ${response.statusCode}") - return response - } +// @Throws(IOException::class) +// fun publish(compiledSubject: String?, compiledMessage: String, denyHostRanges: List): DestinationResponse { +// val destinationMessage: BaseMessage +// when (type) { +// DestinationType.CHIME -> { +// val messageContent = chime?.constructMessageContent(compiledSubject, compiledMessage) +// destinationMessage = ChimeMessage.Builder("chime_message") +// .withUrl(chime?.url) +// .withMessage(messageContent) +// .build() +// } +// DestinationType.SLACK -> { +// val messageContent = slack?.constructMessageContent(compiledSubject, compiledMessage) +// destinationMessage = SlackMessage.Builder("slack_message") +// .withUrl(slack?.url) +// .withMessage(messageContent) +// .build() +// } +// DestinationType.CUSTOM_WEBHOOK -> { +// destinationMessage = CustomWebhookMessage.Builder("custom_webhook") +// .withUrl(customWebhook?.url) +// .withScheme(customWebhook?.scheme) +// .withHost(customWebhook?.host) +// .withPort(customWebhook?.port) +// .withPath(customWebhook?.path) +// .withQueryParams(customWebhook?.queryParams) +// .withHeaderParams(customWebhook?.headerParams) +// .withMessage(compiledMessage).build() +// } +// } +// validateDestinationUri(destinationMessage, denyHostRanges) +// val response = Notification.publish(destinationMessage) as DestinationResponse +// logger.info("Message published for action type: $type, messageid: ${response.responseContent}, statuscode: ${response.statusCode}") +// return response +// } fun constructResponseForDestinationType(type: DestinationType): Any { var content: Any? = null @@ -154,9 +154,9 @@ data class Destination( return content } - private fun validateDestinationUri(destinationMessage: BaseMessage, denyHostRanges: List) { - if (destinationMessage.isHostInDenylist(denyHostRanges)) { - throw IllegalArgumentException("The destination address is invalid.") - } - } +// private fun validateDestinationUri(destinationMessage: BaseMessage, denyHostRanges: List) { +// if (destinationMessage.isHostInDenylist(denyHostRanges)) { +// throw IllegalArgumentException("The destination address is invalid.") +// } +// } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/opensearchapi/OpenSearchExtensions.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/opensearchapi/OpenSearchExtensions.kt index e02750b36..14106a78f 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/opensearchapi/OpenSearchExtensions.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/opensearchapi/OpenSearchExtensions.kt @@ -26,15 +26,15 @@ import org.opensearch.common.xcontent.ToXContentFragment import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentHelper import org.opensearch.common.xcontent.XContentType -import org.opensearch.index.Index import org.opensearch.index.IndexNotFoundException import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.indexstatemanagement.DefaultIndexMetadataService import org.opensearch.indexmanagement.indexstatemanagement.settings.LegacyOpenDistroManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.util.managedIndexMetadataID import org.opensearch.indexmanagement.opensearchapi.contentParser import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import java.time.Instant private val log = LogManager.getLogger("Index Management Helper") @@ -67,13 +67,13 @@ fun IndexMetadata.getManagedIndexMetadata(): ManagedIndexMetaData? { return null } -fun getUuidsForClosedIndices(state: ClusterState): MutableList { +fun getUuidsForClosedIndices(state: ClusterState, defaultIndexMetadataService: DefaultIndexMetadataService): MutableList { val indexMetadatas = state.metadata.indices val closeList = mutableListOf() indexMetadatas.forEach { // it.key is index name if (it.value.state == IndexMetadata.State.CLOSE) { - closeList.add(it.value.indexUUID) + closeList.add(defaultIndexMetadataService.getCustomIndexUUID(it.value)) } } return closeList @@ -83,18 +83,22 @@ fun getUuidsForClosedIndices(state: ClusterState): MutableList { fun Map.filterNotNullValues(): Map = filterValues { it != null } as Map -// get metadata from config index using doc id +/** + * Get metadata from config index + * + * @return metadata object and get call successful or not + */ @Suppress("ReturnCount") -suspend fun IndexMetadata.getManagedIndexMetadata(client: Client): ManagedIndexMetaData? { +suspend fun Client.getManagedIndexMetadata(indexUUID: String): Pair { try { val getRequest = GetRequest(INDEX_MANAGEMENT_INDEX, managedIndexMetadataID(indexUUID)) - .routing(this.indexUUID) - val getResponse: GetResponse = client.suspendUntil { get(getRequest, it) } + .routing(indexUUID) + val getResponse: GetResponse = this.suspendUntil { get(getRequest, it) } if (!getResponse.isExists || getResponse.isSourceEmpty) { - return null + return Pair(null, true) } - return withContext(Dispatchers.IO) { + val metadata = withContext(Dispatchers.IO) { val xcp = XContentHelper.createParser( NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, @@ -102,6 +106,7 @@ suspend fun IndexMetadata.getManagedIndexMetadata(client: Client): ManagedIndexM ) ManagedIndexMetaData.parseWithType(xcp, getResponse.id, getResponse.seqNo, getResponse.primaryTerm) } + return Pair(metadata, true) } catch (e: Exception) { when (e) { is IndexNotFoundException, is NoShardAvailableActionException -> { @@ -110,7 +115,7 @@ suspend fun IndexMetadata.getManagedIndexMetadata(client: Client): ManagedIndexM else -> log.error("Failed to get metadata", e) } - return null + return Pair(null, false) } } @@ -119,28 +124,28 @@ suspend fun IndexMetadata.getManagedIndexMetadata(client: Client): ManagedIndexM * * @return list of metadata */ -suspend fun Client.mgetManagedIndexMetadata(indices: List): List?> { - log.debug("trying to get back metadata for indices ${indices.map { it.name }}") - - if (indices.isEmpty()) return emptyList() +suspend fun Client.mgetManagedIndexMetadata(indexUuids: List): List?> { + log.debug("trying to get back metadata for index [$indexUuids]") + if (indexUuids.isEmpty()) return emptyList() val mgetRequest = MultiGetRequest() - indices.forEach { + indexUuids.forEach { mgetRequest.add( MultiGetRequest.Item( - INDEX_MANAGEMENT_INDEX, managedIndexMetadataID(it.uuid) - ).routing(it.uuid) + INDEX_MANAGEMENT_INDEX, managedIndexMetadataID(it) + ).routing(it) ) } var mgetMetadataList = listOf?>() try { val response: MultiGetResponse = this.suspendUntil { multiGet(mgetRequest, it) } - mgetMetadataList = mgetResponseToList(response) + mgetMetadataList = mgetResponseToMap(response).map { it.value } } catch (e: ActionRequestValidationException) { - log.info("No managed index metadata for indices [$indices], ${e.message}") + log.info("No managed index metadata for indices [$indexUuids], ${e.message}") } catch (e: Exception) { - log.error("Failed to multi-get managed index metadata for indices [$indices]", e) + log.error("Failed to multi-get managed index metadata for indices [$indexUuids]", e) } + return mgetMetadataList } @@ -150,41 +155,27 @@ suspend fun Client.mgetManagedIndexMetadata(indices: List): List#metadata to Pair of metadata or exception */ -fun mgetResponseToList(mgetResponse: MultiGetResponse): - List?> { - val mgetList = mutableListOf?>() +fun mgetResponseToMap(mgetResponse: MultiGetResponse): Map?> { + val mgetMap = mutableMapOf?>() mgetResponse.responses.forEach { if (it.isFailed) { - mgetList.add(Pair(null, it.failure.failure)) + mgetMap[it.id] = Pair(null, it.failure.failure) } else if (it.response != null && !it.response.isSourceEmpty) { val xcp = contentParser(it.response.sourceAsBytesRef) - mgetList.add( - Pair( - ManagedIndexMetaData.parseWithType( - xcp, it.response.id, it.response.seqNo, it.response.primaryTerm - ), - null - ) - ) + mgetMap[it.id] = Pair(ManagedIndexMetaData.parseWithType(xcp, it.response.id, it.response.seqNo, it.response.primaryTerm), null) } else { - mgetList.add(null) + mgetMap[it.id] = null } } - return mgetList + return mgetMap } -fun buildMgetMetadataRequest(clusterState: ClusterState): MultiGetRequest { +fun buildMgetMetadataRequest(indexUuids: List): MultiGetRequest { val mgetMetadataRequest = MultiGetRequest() - clusterState.metadata.indices.map { it.value.index }.forEach { - mgetMetadataRequest.add( - MultiGetRequest.Item( - INDEX_MANAGEMENT_INDEX, managedIndexMetadataID(it.uuid) - ).routing(it.uuid) - ) - } + indexUuids.forEach { mgetMetadataRequest.add(MultiGetRequest.Item(INDEX_MANAGEMENT_INDEX, managedIndexMetadataID(it)).routing(it)) } return mgetMetadataRequest } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestAddPolicyAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestAddPolicyAction.kt index 41e14905d..da55fb2d0 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestAddPolicyAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestAddPolicyAction.kt @@ -12,6 +12,8 @@ import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ISM_BASE_U import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.LEGACY_ISM_BASE_URI import org.opensearch.indexmanagement.indexstatemanagement.transport.action.addpolicy.AddPolicyAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.addpolicy.AddPolicyRequest +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE +import org.opensearch.indexmanagement.indexstatemanagement.util.TYPE_PARAM_KEY import org.opensearch.rest.BaseRestHandler import org.opensearch.rest.BaseRestHandler.RestChannelConsumer import org.opensearch.rest.RestHandler.ReplacedRoute @@ -57,9 +59,11 @@ class RestAddPolicyAction : BaseRestHandler() { mapOf() } + val indexType = request.param(TYPE_PARAM_KEY, DEFAULT_INDEX_TYPE) + val policyID = requireNotNull(body.getOrDefault("policy_id", null)) { "Missing policy_id" } - val addPolicyRequest = AddPolicyRequest(indices.toList(), policyID as String) + val addPolicyRequest = AddPolicyRequest(indices.toList(), policyID as String, indexType) return RestChannelConsumer { channel -> client.execute(AddPolicyAction.INSTANCE, addPolicyRequest, RestToXContentListener(channel)) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyAction.kt index 11aabebb3..ebca535f5 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyAction.kt @@ -14,6 +14,8 @@ import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.LEGACY_ISM import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy import org.opensearch.indexmanagement.indexstatemanagement.transport.action.changepolicy.ChangePolicyAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.changepolicy.ChangePolicyRequest +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE +import org.opensearch.indexmanagement.indexstatemanagement.util.TYPE_PARAM_KEY import org.opensearch.rest.BaseRestHandler import org.opensearch.rest.BaseRestHandler.RestChannelConsumer import org.opensearch.rest.RestHandler.ReplacedRoute @@ -51,11 +53,13 @@ class RestChangePolicyAction : BaseRestHandler() { throw IllegalArgumentException("Missing index") } + val indexType = request.param(TYPE_PARAM_KEY, DEFAULT_INDEX_TYPE) + val xcp = request.contentParser() ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) val changePolicy = ChangePolicy.parse(xcp) - val changePolicyRequest = ChangePolicyRequest(indices.toList(), changePolicy) + val changePolicyRequest = ChangePolicyRequest(indices.toList(), changePolicy, indexType) return RestChannelConsumer { channel -> client.execute(ChangePolicyAction.INSTANCE, changePolicyRequest, RestToXContentListener(channel)) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt index 873baf565..4db7dc6ed 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt @@ -14,11 +14,15 @@ import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.LEGACY_ISM import org.opensearch.indexmanagement.indexstatemanagement.model.SearchParams import org.opensearch.indexmanagement.indexstatemanagement.transport.action.explain.ExplainAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.explain.ExplainRequest +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_EXPLAIN_SHOW_POLICY +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_JOB_SORT_FIELD import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_PAGINATION_FROM import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_PAGINATION_SIZE import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_QUERY_STRING import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_SORT_ORDER +import org.opensearch.indexmanagement.indexstatemanagement.util.SHOW_POLICY_QUERY_PARAM +import org.opensearch.indexmanagement.indexstatemanagement.util.TYPE_PARAM_KEY import org.opensearch.rest.BaseRestHandler import org.opensearch.rest.BaseRestHandler.RestChannelConsumer import org.opensearch.rest.RestHandler.ReplacedRoute @@ -68,11 +72,15 @@ class RestExplainAction : BaseRestHandler() { val sortOrder = request.param("sortOrder", DEFAULT_SORT_ORDER) val queryString = request.param("queryString", DEFAULT_QUERY_STRING) + val indexType = request.param(TYPE_PARAM_KEY, DEFAULT_INDEX_TYPE) + val explainRequest = ExplainRequest( indices.toList(), request.paramAsBoolean("local", false), request.paramAsTime("master_timeout", MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT), - SearchParams(size, from, sortField, sortOrder, queryString) + SearchParams(size, from, sortField, sortOrder, queryString), + request.paramAsBoolean(SHOW_POLICY_QUERY_PARAM, DEFAULT_EXPLAIN_SHOW_POLICY), + indexType ) return RestChannelConsumer { channel -> diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestIndexPolicyAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestIndexPolicyAction.kt index 7f5c0e08c..dd98093a0 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestIndexPolicyAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestIndexPolicyAction.kt @@ -18,7 +18,6 @@ import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndex import org.opensearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.IndexPolicyAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.IndexPolicyRequest import org.opensearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.IndexPolicyResponse -import org.opensearch.indexmanagement.indexstatemanagement.util.getDisallowedActions import org.opensearch.indexmanagement.opensearchapi.parseWithType import org.opensearch.indexmanagement.util.IF_PRIMARY_TERM import org.opensearch.indexmanagement.util.IF_SEQ_NO diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRemovePolicyAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRemovePolicyAction.kt index 817627f8e..7e7d79307 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRemovePolicyAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRemovePolicyAction.kt @@ -11,6 +11,8 @@ import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ISM_BASE_U import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.LEGACY_ISM_BASE_URI import org.opensearch.indexmanagement.indexstatemanagement.transport.action.removepolicy.RemovePolicyAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.removepolicy.RemovePolicyRequest +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE +import org.opensearch.indexmanagement.indexstatemanagement.util.TYPE_PARAM_KEY import org.opensearch.rest.BaseRestHandler import org.opensearch.rest.RestHandler.ReplacedRoute import org.opensearch.rest.RestHandler.Route @@ -49,7 +51,9 @@ class RestRemovePolicyAction : BaseRestHandler() { throw IllegalArgumentException("Missing indices") } - val removePolicyRequest = RemovePolicyRequest(indices.toList()) + val indexType = request.param(TYPE_PARAM_KEY, DEFAULT_INDEX_TYPE) + + val removePolicyRequest = RemovePolicyRequest(indices.toList(), indexType) return RestChannelConsumer { channel -> client.execute(RemovePolicyAction.INSTANCE, removePolicyRequest, RestToXContentListener(channel)) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexAction.kt index ea8759806..4580dc54d 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexAction.kt @@ -13,6 +13,8 @@ import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ISM_BASE_U import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.LEGACY_ISM_BASE_URI import org.opensearch.indexmanagement.indexstatemanagement.transport.action.retryfailedmanagedindex.RetryFailedManagedIndexAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.retryfailedmanagedindex.RetryFailedManagedIndexRequest +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE +import org.opensearch.indexmanagement.indexstatemanagement.util.TYPE_PARAM_KEY import org.opensearch.rest.BaseRestHandler import org.opensearch.rest.BaseRestHandler.RestChannelConsumer import org.opensearch.rest.RestHandler.ReplacedRoute @@ -56,9 +58,12 @@ class RestRetryFailedManagedIndexAction : BaseRestHandler() { mapOf() } + val indexType = request.param(TYPE_PARAM_KEY, DEFAULT_INDEX_TYPE) + val retryFailedRequest = RetryFailedManagedIndexRequest( indices.toList(), body["state"] as String?, - request.paramAsTime("master_timeout", DEFAULT_MASTER_NODE_TIMEOUT) + request.paramAsTime("master_timeout", DEFAULT_MASTER_NODE_TIMEOUT), + indexType ) return RestChannelConsumer { channel -> diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/settings/LegacyOpenDistroManagedIndexSettings.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/settings/LegacyOpenDistroManagedIndexSettings.kt index aa5e0632c..5e6c521d2 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/settings/LegacyOpenDistroManagedIndexSettings.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/settings/LegacyOpenDistroManagedIndexSettings.kt @@ -7,7 +7,7 @@ package org.opensearch.indexmanagement.indexstatemanagement.settings import org.opensearch.common.settings.Setting import org.opensearch.common.unit.TimeValue -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig +import org.opensearch.indexmanagement.indexstatemanagement.ISMActionsParser import java.util.concurrent.TimeUnit import java.util.function.Function @@ -18,7 +18,7 @@ class LegacyOpenDistroManagedIndexSettings { const val DEFAULT_METADATA_SERVICE_STATUS = 0 const val DEFAULT_METADATA_SERVICE_ENABLED = true const val DEFAULT_JOB_INTERVAL = 5 - private val ALLOW_LIST_ALL = ActionConfig.ActionType.values().toList().map { it.type } + private val ALLOW_LIST_ALL = ISMActionsParser.instance.parsers.map { it.getActionType() }.toList() val ALLOW_LIST_NONE = emptyList() val SNAPSHOT_DENY_LIST_NONE = emptyList() const val HOST_DENY_LIST = "opendistro.destination.host.deny_list" diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/Step.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/Step.kt deleted file mode 100644 index 77da5bff0..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/Step.kt +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.step - -import org.apache.logging.log4j.Logger -import org.opensearch.common.io.stream.StreamInput -import org.opensearch.common.io.stream.StreamOutput -import org.opensearch.common.io.stream.Writeable -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import java.time.Instant -import java.util.Locale - -abstract class Step(val name: String, val managedIndexMetaData: ManagedIndexMetaData, val isSafeToDisableOn: Boolean = true) { - - fun preExecute(logger: Logger): Step { - logger.info("Executing $name for ${managedIndexMetaData.index}") - return this - } - - abstract suspend fun execute(): Step - - fun postExecute(logger: Logger): Step { - logger.info("Finished executing $name for ${managedIndexMetaData.index}") - return this - } - - abstract fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData - - /** - * Before every execution of a step, we first update the step_status in cluster state to [StepStatus.STARTING] - * to signal that work is about to be done for the managed index. The step then attempts to do work by - * calling execute, and finally updates the step_status with the results of that work ([StepStatus]). - * - * If we ever start an execution with a step_status of [StepStatus.STARTING] it means we failed to update the step_status - * after calling the execute function. Since we do not know if the execution was a noop, failed, or completed then - * we can't always assume it's safe to just retry it (e.g. calling force merge multiple times in a row). This means - * that final update is a failure point that can't be retried and when multiplied by # of executions it leads to a lot of - * chances over time for random network failures, timeouts, etc. - * - * To get around this every step should have an [isIdempotent] method to signal if it's safe to retry this step for such failures. - */ - abstract fun isIdempotent(): Boolean - - fun getStartingStepMetaData(): StepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), StepStatus.STARTING) - - fun getStepStartTime(): Instant { - return when { - managedIndexMetaData.stepMetaData == null -> Instant.now() - managedIndexMetaData.stepMetaData.name != this.name -> Instant.now() - // The managed index metadata is a historical snapshot of the metadata and refers to what has happened from the previous - // execution, so if we ever see it as COMPLETED it means we are always going to be in a new step, this specifically - // helps with the Transition -> Transition (empty state) sequence which the above do not capture - managedIndexMetaData.stepMetaData.stepStatus == StepStatus.COMPLETED -> Instant.now() - else -> Instant.ofEpochMilli(managedIndexMetaData.stepMetaData.startTime) - } - } - - protected val indexName: String = managedIndexMetaData.index - - enum class StepStatus(val status: String) : Writeable { - STARTING("starting"), - CONDITION_NOT_MET("condition_not_met"), - FAILED("failed"), - COMPLETED("completed"); - - override fun toString(): String { - return status - } - - override fun writeTo(out: StreamOutput) { - out.writeString(status) - } - - companion object { - fun read(streamInput: StreamInput): StepStatus { - return valueOf(streamInput.readString().toUpperCase(Locale.ROOT)) - } - } - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/allocation/AttemptAllocationStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/allocation/AttemptAllocationStep.kt index a38e0fc50..4d18652b1 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/allocation/AttemptAllocationStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/allocation/AttemptAllocationStep.kt @@ -8,35 +8,29 @@ package org.opensearch.indexmanagement.indexstatemanagement.step.allocation import org.apache.logging.log4j.LogManager import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest import org.opensearch.action.support.master.AcknowledgedResponse -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.AllocationActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.indexmanagement.indexstatemanagement.action.AllocationAction import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData + +class AttemptAllocationStep(private val action: AllocationAction) : Step(name) { -class AttemptAllocationStep( - val clusterService: ClusterService, - val client: Client, - val config: AllocationActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step("attempt_allocation", managedIndexMetaData) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - override fun isIdempotent() = true - - override suspend fun execute(): AttemptAllocationStep { + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index try { - val response: AcknowledgedResponse = client.admin() + val response: AcknowledgedResponse = context.client.admin() .indices() - .suspendUntil { updateSettings(UpdateSettingsRequest(buildSettings(), managedIndexMetaData.index), it) } - handleResponse(response) + .suspendUntil { updateSettings(UpdateSettingsRequest(buildSettings(), indexName), it) } + handleResponse(response, indexName) } catch (e: Exception) { - handleException(e) + handleException(e, indexName) } return this @@ -44,13 +38,13 @@ class AttemptAllocationStep( private fun buildSettings(): Settings { val builder = Settings.builder() - config.require.forEach { (key, value) -> builder.put(SETTINGS_PREFIX + AllocationActionConfig.REQUIRE + "." + key, value) } - config.include.forEach { (key, value) -> builder.put(SETTINGS_PREFIX + AllocationActionConfig.INCLUDE + "." + key, value) } - config.exclude.forEach { (key, value) -> builder.put(SETTINGS_PREFIX + AllocationActionConfig.EXCLUDE + "." + key, value) } + action.require.forEach { (key, value) -> builder.put(SETTINGS_PREFIX + AllocationAction.REQUIRE + "." + key, value) } + action.include.forEach { (key, value) -> builder.put(SETTINGS_PREFIX + AllocationAction.INCLUDE + "." + key, value) } + action.exclude.forEach { (key, value) -> builder.put(SETTINGS_PREFIX + AllocationAction.EXCLUDE + "." + key, value) } return builder.build() } - private fun handleException(e: Exception) { + private fun handleException(e: Exception, indexName: String) { val message = getFailedMessage(indexName) logger.error(message, e) stepStatus = StepStatus.FAILED @@ -60,7 +54,7 @@ class AttemptAllocationStep( info = mutableInfo.toMap() } - private fun handleResponse(response: AcknowledgedResponse) { + private fun handleResponse(response: AcknowledgedResponse, indexName: String) { if (response.isAcknowledged) { stepStatus = StepStatus.COMPLETED info = mapOf("message" to getSuccessMessage(indexName)) @@ -70,15 +64,18 @@ class AttemptAllocationStep( } } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - return currentMetaData.copy( - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + return currentMetadata.copy( + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), transitionTo = null, info = info ) } + override fun isIdempotent() = true + companion object { + const val name = "attempt_allocation" private const val SETTINGS_PREFIX = "index.routing.allocation." fun getFailedMessage(index: String) = "Failed to update allocation setting [index=$index]" fun getSuccessMessage(index: String) = "Successfully updated allocation setting [index=$index]" diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/close/AttemptCloseStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/close/AttemptCloseStep.kt index 6c38ac045..bab8423e8 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/close/AttemptCloseStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/close/AttemptCloseStep.kt @@ -9,36 +9,29 @@ import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.action.admin.indices.close.CloseIndexRequest import org.opensearch.action.admin.indices.close.CloseIndexResponse -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.CloseActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.snapshots.SnapshotInProgressException import org.opensearch.transport.RemoteTransportException -class AttemptCloseStep( - val clusterService: ClusterService, - val client: Client, - val config: CloseActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step("attempt_close", managedIndexMetaData) { +class AttemptCloseStep : Step(name) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - override fun isIdempotent() = true - - @Suppress("TooGenericExceptionCaught") - override suspend fun execute(): AttemptCloseStep { + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index try { val closeIndexRequest = CloseIndexRequest() .indices(indexName) - val response: CloseIndexResponse = client.admin().indices().suspendUntil { close(closeIndexRequest, it) } + val response: CloseIndexResponse = context.client.admin().indices() + .suspendUntil { close(closeIndexRequest, it) } + if (response.isAcknowledged) { stepStatus = StepStatus.COMPLETED info = mapOf("message" to getSuccessMessage(indexName)) @@ -51,27 +44,27 @@ class AttemptCloseStep( } catch (e: RemoteTransportException) { val cause = ExceptionsHelper.unwrapCause(e) if (cause is SnapshotInProgressException) { - handleSnapshotException(cause) + handleSnapshotException(indexName, cause as SnapshotInProgressException) } else { - handleException(cause as Exception) + handleException(indexName, cause as Exception) } } catch (e: SnapshotInProgressException) { - handleSnapshotException(e) + handleSnapshotException(indexName, e) } catch (e: Exception) { - handleException(e) + handleException(indexName, e) } return this } - private fun handleSnapshotException(e: SnapshotInProgressException) { + private fun handleSnapshotException(indexName: String, e: SnapshotInProgressException) { val message = getSnapshotMessage(indexName) logger.warn(message, e) stepStatus = StepStatus.CONDITION_NOT_MET info = mapOf("message" to message) } - private fun handleException(e: Exception) { + private fun handleException(indexName: String, e: Exception) { val message = getFailedMessage(indexName) logger.error(message, e) stepStatus = StepStatus.FAILED @@ -81,15 +74,18 @@ class AttemptCloseStep( info = mutableInfo.toMap() } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - return currentMetaData.copy( - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + return currentMetadata.copy( + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), transitionTo = null, info = info ) } + override fun isIdempotent() = true + companion object { + const val name = "attempt_close" fun getFailedMessage(index: String) = "Failed to close index [index=$index]" fun getSuccessMessage(index: String) = "Successfully closed index [index=$index]" fun getSnapshotMessage(index: String) = "Index had snapshot in progress, retrying closing [index=$index]" diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/delete/AttemptDeleteStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/delete/AttemptDeleteStep.kt index 431c2b785..8019130a2 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/delete/AttemptDeleteStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/delete/AttemptDeleteStep.kt @@ -9,34 +9,24 @@ import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.action.admin.indices.delete.DeleteIndexRequest import org.opensearch.action.support.master.AcknowledgedResponse -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.DeleteActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.snapshots.SnapshotInProgressException import org.opensearch.transport.RemoteTransportException -import java.lang.Exception -class AttemptDeleteStep( - val clusterService: ClusterService, - val client: Client, - val config: DeleteActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step(name, managedIndexMetaData) { +class AttemptDeleteStep : Step(name) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - override fun isIdempotent() = true - - @Suppress("TooGenericExceptionCaught") - override suspend fun execute(): AttemptDeleteStep { + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index try { - val response: AcknowledgedResponse = client.admin().indices() + val response: AcknowledgedResponse = context.client.admin().indices() .suspendUntil { delete(DeleteIndexRequest(indexName), it) } if (response.isAcknowledged) { @@ -51,27 +41,27 @@ class AttemptDeleteStep( } catch (e: RemoteTransportException) { val cause = ExceptionsHelper.unwrapCause(e) if (cause is SnapshotInProgressException) { - handleSnapshotException(cause) + handleSnapshotException(indexName, cause) } else { - handleException(cause as Exception) + handleException(indexName, cause as Exception) } } catch (e: SnapshotInProgressException) { - handleSnapshotException(e) + handleSnapshotException(indexName, e) } catch (e: Exception) { - handleException(e) + handleException(indexName, e) } return this } - private fun handleSnapshotException(e: SnapshotInProgressException) { + private fun handleSnapshotException(indexName: String, e: SnapshotInProgressException) { val message = getSnapshotMessage(indexName) logger.warn(message, e) stepStatus = StepStatus.CONDITION_NOT_MET info = mapOf("message" to message) } - private fun handleException(e: Exception) { + private fun handleException(indexName: String, e: Exception) { val message = getFailedMessage(indexName) logger.error(message, e) stepStatus = StepStatus.FAILED @@ -81,18 +71,20 @@ class AttemptDeleteStep( info = mutableInfo.toMap() } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - return currentMetaData.copy( - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + return currentMetadata.copy( + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), transitionTo = null, info = info ) } + override fun isIdempotent() = true + companion object { const val name = "attempt_delete" - fun getFailedMessage(index: String) = "Failed to delete index [index=$index]" - fun getSuccessMessage(index: String) = "Successfully deleted index [index=$index]" - fun getSnapshotMessage(index: String) = "Index had snapshot in progress, retrying deletion [index=$index]" + fun getFailedMessage(indexName: String) = "Failed to delete index [index=$indexName]" + fun getSuccessMessage(indexName: String) = "Successfully deleted index [index=$indexName]" + fun getSnapshotMessage(indexName: String) = "Index had snapshot in progress, retrying deletion [index=$indexName]" } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptCallForceMergeStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptCallForceMergeStep.kt index 8ac8f1dfd..640d2c662 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptCallForceMergeStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptCallForceMergeStep.kt @@ -14,43 +14,36 @@ import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ForceMergeActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionProperties -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.indexmanagement.indexstatemanagement.action.ForceMergeAction import org.opensearch.indexmanagement.opensearchapi.getUsefulCauseString import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.rest.RestStatus import org.opensearch.transport.RemoteTransportException import java.time.Instant -class AttemptCallForceMergeStep( - val clusterService: ClusterService, - val client: Client, - val config: ForceMergeActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step(name, managedIndexMetaData) { +class AttemptCallForceMergeStep(private val action: ForceMergeAction) : Step(name) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - override fun isIdempotent() = false - @Suppress("TooGenericExceptionCaught", "ComplexMethod") override suspend fun execute(): AttemptCallForceMergeStep { + val context = this.context ?: return this + val indexName = context.metadata.index try { val startTime = Instant.now().toEpochMilli() - val request = ForceMergeRequest(indexName).maxNumSegments(config.maxNumSegments) + val request = ForceMergeRequest(indexName).maxNumSegments(action.maxNumSegments) var response: ForceMergeResponse? = null var throwable: Throwable? = null GlobalScope.launch(Dispatchers.IO + CoroutineName("ISM-ForceMerge-$indexName")) { try { - response = client.admin().indices().suspendUntil { forceMerge(request, it) } + response = context.client.admin().indices().suspendUntil { forceMerge(request, it) } if (response?.status == RestStatus.OK) { logger.info(getSuccessMessage(indexName)) } else { @@ -80,15 +73,15 @@ class AttemptCallForceMergeStep( ) } } catch (e: RemoteTransportException) { - handleException(ExceptionsHelper.unwrapCause(e) as Exception) + handleException(indexName, ExceptionsHelper.unwrapCause(e) as Exception) } catch (e: Exception) { - handleException(e) + handleException(indexName, e) } return this } - private fun handleException(e: Exception) { + private fun handleException(indexName: String, e: Exception) { val message = getFailedMessage(indexName) logger.error(message, e) stepStatus = StepStatus.FAILED @@ -98,18 +91,20 @@ class AttemptCallForceMergeStep( info = mutableInfo.toMap() } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { // Saving maxNumSegments in ActionProperties after the force merge operation has begun so that if a ChangePolicy occurred // in between this step and WaitForForceMergeStep, a cached segment count expected from the operation is available - val currentActionMetaData = currentMetaData.actionMetaData - return currentMetaData.copy( - actionMetaData = currentActionMetaData?.copy(actionProperties = ActionProperties(maxNumSegments = config.maxNumSegments)), - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), + val currentActionMetaData = currentMetadata.actionMetaData + return currentMetadata.copy( + actionMetaData = currentActionMetaData?.copy(actionProperties = ActionProperties(maxNumSegments = action.maxNumSegments)), + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), transitionTo = null, info = info ) } + override fun isIdempotent() = false + companion object { const val name = "attempt_call_force_merge" const val FIVE_MINUTES_IN_MILLIS = 1000 * 60 * 5 // how long to wait for the force merge request before moving on diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptSetReadOnlyStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptSetReadOnlyStep.kt index 87d8fdd2e..b1f82f314 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptSetReadOnlyStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptSetReadOnlyStep.kt @@ -9,32 +9,26 @@ import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest import org.opensearch.action.support.master.AcknowledgedResponse -import org.opensearch.client.Client import org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_WRITE -import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ForceMergeActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.indexmanagement.indexstatemanagement.action.ForceMergeAction import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.transport.RemoteTransportException -class AttemptSetReadOnlyStep( - val clusterService: ClusterService, - val client: Client, - val config: ForceMergeActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step(name, managedIndexMetaData) { +class AttemptSetReadOnlyStep(private val action: ForceMergeAction) : Step(name) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - override fun isIdempotent() = true - - override suspend fun execute(): AttemptSetReadOnlyStep { - val indexSetToReadOnly = setIndexToReadOnly(indexName) + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index + val indexSetToReadOnly = setIndexToReadOnly(indexName, context) // If setIndexToReadOnly returns false, updating settings failed and failed info was already updated, can return early if (!indexSetToReadOnly) return this @@ -47,12 +41,12 @@ class AttemptSetReadOnlyStep( } @Suppress("TooGenericExceptionCaught") - private suspend fun setIndexToReadOnly(indexName: String): Boolean { + private suspend fun setIndexToReadOnly(indexName: String, context: StepContext): Boolean { try { val updateSettingsRequest = UpdateSettingsRequest() .indices(indexName) .settings(Settings.builder().put(SETTING_BLOCKS_WRITE, true)) - val response: AcknowledgedResponse = client.admin().indices() + val response: AcknowledgedResponse = context.client.admin().indices() .suspendUntil { updateSettings(updateSettingsRequest, it) } if (response.isAcknowledged) { @@ -65,15 +59,15 @@ class AttemptSetReadOnlyStep( stepStatus = StepStatus.FAILED info = mapOf("message" to message) } catch (e: RemoteTransportException) { - handleException(ExceptionsHelper.unwrapCause(e) as Exception) + handleException(indexName, ExceptionsHelper.unwrapCause(e) as Exception) } catch (e: Exception) { - handleException(e) + handleException(indexName, e) } return false } - private fun handleException(e: Exception) { + private fun handleException(indexName: String, e: Exception) { val message = getFailedMessage(indexName) logger.error(message, e) stepStatus = StepStatus.FAILED @@ -83,8 +77,14 @@ class AttemptSetReadOnlyStep( info = mutableInfo.toMap() } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData = - currentMetaData.copy(stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), transitionTo = null, info = info) + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData = + currentMetadata.copy( + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), + transitionTo = null, + info = info + ) + + override fun isIdempotent() = true companion object { const val name = "attempt_set_read_only" diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/WaitForForceMergeStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/WaitForForceMergeStep.kt index 7747a8d53..83ac0e073 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/WaitForForceMergeStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/WaitForForceMergeStep.kt @@ -8,39 +8,33 @@ package org.opensearch.indexmanagement.indexstatemanagement.step.forcemerge import org.apache.logging.log4j.LogManager import org.opensearch.action.admin.indices.stats.IndicesStatsRequest import org.opensearch.action.admin.indices.stats.IndicesStatsResponse -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ForceMergeActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionProperties -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.indexmanagement.indexstatemanagement.action.ForceMergeAction import org.opensearch.indexmanagement.opensearchapi.getUsefulCauseString import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.rest.RestStatus import java.time.Duration import java.time.Instant -class WaitForForceMergeStep( - val clusterService: ClusterService, - val client: Client, - val config: ForceMergeActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step(name, managedIndexMetaData, false) { +class WaitForForceMergeStep(private val action: ForceMergeAction) : Step(name, false) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - override fun isIdempotent() = true - @Suppress("TooGenericExceptionCaught", "ReturnCount") override suspend fun execute(): WaitForForceMergeStep { + val context = this.context ?: return this + val indexName = context.metadata.index // Retrieve maxNumSegments value from ActionProperties. If ActionProperties is null, update failed info and return early. - val maxNumSegments = getMaxNumSegments() ?: return this + val maxNumSegments = getMaxNumSegments(context) ?: return this // Get the number of shards with a segment count greater than maxNumSegments, meaning they are still merging - val shardsStillMergingSegments = getShardsStillMergingSegments(indexName, maxNumSegments) + val shardsStillMergingSegments = getShardsStillMergingSegments(indexName, maxNumSegments, context) // If shardsStillMergingSegments is null, failed info has already been updated and can return early shardsStillMergingSegments ?: return this @@ -61,11 +55,11 @@ class WaitForForceMergeStep( * is optional, if no timeout is given, the segment count would stop going down as merging would no longer * occur and the managed index would become stuck in this action. */ - val timeWaitingForForceMerge: Duration = Duration.between(getActionStartTime(), Instant.now()) + val timeWaitingForForceMerge: Duration = Duration.between(getActionStartTime(context), Instant.now()) // Get ActionTimeout if given, otherwise use default timeout of 12 hours - val timeoutInSeconds: Long = config.configTimeout?.timeout?.seconds ?: FORCE_MERGE_TIMEOUT_IN_SECONDS + val timeoutInSeconds: Long = action.configTimeout?.timeout?.seconds ?: FORCE_MERGE_TIMEOUT_IN_SECONDS - if (timeWaitingForForceMerge.toSeconds() > timeoutInSeconds) { + if (timeWaitingForForceMerge.seconds > timeoutInSeconds) { logger.error( "Force merge on [$indexName] timed out with" + " [$shardsStillMergingSegments] shards containing unmerged segments" @@ -87,7 +81,8 @@ class WaitForForceMergeStep( return this } - private fun getMaxNumSegments(): Int? { + private fun getMaxNumSegments(context: StepContext): Int? { + val managedIndexMetaData = context.metadata val actionProperties = managedIndexMetaData.actionMetaData?.actionProperties if (actionProperties?.maxNumSegments == null) { @@ -102,10 +97,10 @@ class WaitForForceMergeStep( return actionProperties.maxNumSegments } - private suspend fun getShardsStillMergingSegments(indexName: String, maxNumSegments: Int): Int? { + private suspend fun getShardsStillMergingSegments(indexName: String, maxNumSegments: Int, context: StepContext): Int? { try { val statsRequest = IndicesStatsRequest().indices(indexName) - val statsResponse: IndicesStatsResponse = client.admin().indices().suspendUntil { stats(statsRequest, it) } + val statsResponse: IndicesStatsResponse = context.client.admin().indices().suspendUntil { stats(statsRequest, it) } if (statsResponse.status == RestStatus.OK) { return statsResponse.shards.count { @@ -139,29 +134,30 @@ class WaitForForceMergeStep( return null } - private fun getActionStartTime(): Instant { - if (managedIndexMetaData.actionMetaData?.startTime == null) { - return Instant.now() - } + private fun getActionStartTime(context: StepContext): Instant { + val managedIndexMetaData = context.metadata + val startTime = managedIndexMetaData.actionMetaData?.startTime ?: return Instant.now() - return Instant.ofEpochMilli(managedIndexMetaData.actionMetaData.startTime) + return Instant.ofEpochMilli(startTime) } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { // if the step is completed set actionProperties back to null - val currentActionMetaData = currentMetaData.actionMetaData + val currentActionMetaData = currentMetadata.actionMetaData val updatedActionMetaData = currentActionMetaData?.let { if (stepStatus != StepStatus.COMPLETED) it else currentActionMetaData.copy(actionProperties = null) } - return currentMetaData.copy( + return currentMetadata.copy( actionMetaData = updatedActionMetaData, - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), transitionTo = null, info = info ) } + override fun isIdempotent() = true + companion object { const val name = "wait_for_force_merge" const val FORCE_MERGE_TIMEOUT_IN_SECONDS = 43200L // 12 hours diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/indexpriority/AttemptSetIndexPriorityStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/indexpriority/AttemptSetIndexPriorityStep.kt index 85505fdb5..1ce082aaf 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/indexpriority/AttemptSetIndexPriorityStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/indexpriority/AttemptSetIndexPriorityStep.kt @@ -9,59 +9,53 @@ import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest import org.opensearch.action.support.master.AcknowledgedResponse -import org.opensearch.client.Client import org.opensearch.cluster.metadata.IndexMetadata.SETTING_PRIORITY -import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.IndexPriorityActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.indexmanagement.indexstatemanagement.action.IndexPriorityAction import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.transport.RemoteTransportException -class AttemptSetIndexPriorityStep( - val clusterService: ClusterService, - val client: Client, - val config: IndexPriorityActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step("attempt_set_index_priority", managedIndexMetaData) { +class AttemptSetIndexPriorityStep(private val action: IndexPriorityAction) : Step(name) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - override fun isIdempotent() = true - @Suppress("TooGenericExceptionCaught") - override suspend fun execute(): AttemptSetIndexPriorityStep { + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index + val managedIndexMetaData = context.metadata try { val updateSettingsRequest = UpdateSettingsRequest() .indices(managedIndexMetaData.index) - .settings(Settings.builder().put(SETTING_PRIORITY, config.indexPriority)) - val response: AcknowledgedResponse = client.admin().indices() + .settings(Settings.builder().put(SETTING_PRIORITY, action.indexPriority)) + val response: AcknowledgedResponse = context.client.admin().indices() .suspendUntil { updateSettings(updateSettingsRequest, it) } if (response.isAcknowledged) { stepStatus = StepStatus.COMPLETED - info = mapOf("message" to getSuccessMessage(indexName, config.indexPriority)) + info = mapOf("message" to getSuccessMessage(indexName, action.indexPriority)) } else { - val message = getFailedMessage(indexName, config.indexPriority) + val message = getFailedMessage(indexName, action.indexPriority) logger.warn(message) stepStatus = StepStatus.FAILED info = mapOf("message" to message) } } catch (e: RemoteTransportException) { - handleException(ExceptionsHelper.unwrapCause(e) as Exception) + handleException(indexName, ExceptionsHelper.unwrapCause(e) as Exception) } catch (e: Exception) { - handleException(e) + handleException(indexName, e) } return this } - private fun handleException(e: Exception) { - val message = getFailedMessage(indexName, config.indexPriority) + private fun handleException(indexName: String, e: Exception) { + val message = getFailedMessage(indexName, action.indexPriority) logger.error(message, e) stepStatus = StepStatus.FAILED val mutableInfo = mutableMapOf("message" to message) @@ -70,15 +64,18 @@ class AttemptSetIndexPriorityStep( info = mutableInfo.toMap() } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - return currentMetaData.copy( - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + return currentMetadata.copy( + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), transitionTo = null, info = info ) } + override fun isIdempotent() = true + companion object { + const val name = "attempt_set_index_priority" fun getFailedMessage(index: String, indexPriority: Int) = "Failed to set index priority to $indexPriority [index=$index]" fun getSuccessMessage(index: String, indexPriority: Int) = "Successfully set index priority to $indexPriority [index=$index]" } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/notification/AttemptNotificationStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/notification/AttemptNotificationStep.kt index 42f3585a3..6b48046e7 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/notification/AttemptNotificationStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/notification/AttemptNotificationStep.kt @@ -8,40 +8,30 @@ package org.opensearch.indexmanagement.indexstatemanagement.step.notification import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import org.apache.logging.log4j.LogManager -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Settings -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.NotificationActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData +import org.opensearch.indexmanagement.indexstatemanagement.action.NotificationAction import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.opensearchapi.convertToMap +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.script.Script import org.opensearch.script.ScriptService import org.opensearch.script.TemplateScript -class AttemptNotificationStep( - val clusterService: ClusterService, - val scriptService: ScriptService, - val client: Client, - val settings: Settings, - val config: NotificationActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step("attempt_notification", managedIndexMetaData) { +class AttemptNotificationStep(private val action: NotificationAction) : Step(name) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - private val hostDenyList = settings.getAsList(ManagedIndexSettings.HOST_DENY_LIST) - override fun isIdempotent() = false - - @Suppress("TooGenericExceptionCaught") - override suspend fun execute(): AttemptNotificationStep { + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index + val hostDenyList = context.settings.getAsList(ManagedIndexSettings.HOST_DENY_LIST) + val scriptService = context.scriptService try { withContext(Dispatchers.IO) { - config.destination.publish(null, compileTemplate(config.messageTemplate, managedIndexMetaData), hostDenyList) + // action.destination.publish(null, compileTemplate(scriptService, action.messageTemplate, context.metadata), hostDenyList) } // publish internally throws an error for any invalid responses so its safe to assume if we reach this point it was successful @@ -49,13 +39,13 @@ class AttemptNotificationStep( stepStatus = StepStatus.COMPLETED info = mapOf("message" to getSuccessMessage(indexName)) } catch (e: Exception) { - handleException(e) + handleException(indexName, e) } return this } - private fun handleException(e: Exception) { + private fun handleException(indexName: String, e: Exception) { val message = getFailedMessage(indexName) logger.error(message, e) stepStatus = StepStatus.FAILED @@ -65,21 +55,24 @@ class AttemptNotificationStep( info = mutableInfo.toMap() } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - return currentMetaData.copy( - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + return currentMetadata.copy( + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), transitionTo = null, info = info ) } - private fun compileTemplate(template: Script, managedIndexMetaData: ManagedIndexMetaData): String { + private fun compileTemplate(scriptService: ScriptService, template: Script, managedIndexMetaData: ManagedIndexMetaData): String { return scriptService.compile(template, TemplateScript.CONTEXT) .newInstance(template.params + mapOf("ctx" to managedIndexMetaData.convertToMap())) .execute() } + override fun isIdempotent(): Boolean = false + companion object { + const val name = "attempt_notification" fun getFailedMessage(index: String) = "Failed to send notification [index=$index]" fun getSuccessMessage(index: String) = "Successfully sent notification [index=$index]" } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/open/AttemptOpenStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/open/AttemptOpenStep.kt index 7ed451308..2f9987d6e 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/open/AttemptOpenStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/open/AttemptOpenStep.kt @@ -9,35 +9,25 @@ import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.action.admin.indices.open.OpenIndexRequest import org.opensearch.action.admin.indices.open.OpenIndexResponse -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.OpenActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.transport.RemoteTransportException -class AttemptOpenStep( - val clusterService: ClusterService, - val client: Client, - val config: OpenActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step("attempt_open", managedIndexMetaData) { - +class AttemptOpenStep : Step(name) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - override fun isIdempotent() = true - - @Suppress("TooGenericExceptionCaught") - override suspend fun execute(): AttemptOpenStep { + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index try { val openIndexRequest = OpenIndexRequest() .indices(indexName) - val response: OpenIndexResponse = client.admin().indices().suspendUntil { open(openIndexRequest, it) } + val response: OpenIndexResponse = context.client.admin().indices().suspendUntil { open(openIndexRequest, it) } if (response.isAcknowledged) { stepStatus = StepStatus.COMPLETED info = mapOf("message" to getSuccessMessage(indexName)) @@ -48,15 +38,14 @@ class AttemptOpenStep( info = mapOf("message" to message) } } catch (e: RemoteTransportException) { - handleException(ExceptionsHelper.unwrapCause(e) as Exception) + handleException(indexName, ExceptionsHelper.unwrapCause(e) as Exception) } catch (e: Exception) { - handleException(e) + handleException(indexName, e) } return this } - - private fun handleException(e: Exception) { + private fun handleException(indexName: String, e: Exception) { val message = getFailedMessage(indexName) logger.error(message, e) stepStatus = StepStatus.FAILED @@ -66,16 +55,19 @@ class AttemptOpenStep( info = mutableInfo.toMap() } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - return currentMetaData.copy( - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + return currentMetadata.copy( + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), transitionTo = null, info = info ) } + override fun isIdempotent() = true + companion object { - fun getFailedMessage(index: String) = "Failed to open index [index=$index]" - fun getSuccessMessage(index: String) = "Successfully opened index [index=$index]" + const val name = "attempt_open" + fun getFailedMessage(indexName: String) = "Failed to open index [index=$indexName]" + fun getSuccessMessage(indexName: String) = "Successfully opened index [index=$indexName]" } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/readonly/SetReadOnlyStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/readonly/SetReadOnlyStep.kt index 3a676cc63..b821d5b59 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/readonly/SetReadOnlyStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/readonly/SetReadOnlyStep.kt @@ -9,37 +9,28 @@ import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest import org.opensearch.action.support.master.AcknowledgedResponse -import org.opensearch.client.Client import org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_WRITE -import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReadOnlyActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.transport.RemoteTransportException -class SetReadOnlyStep( - val clusterService: ClusterService, - val client: Client, - val config: ReadOnlyActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step("set_read_only", managedIndexMetaData) { +class SetReadOnlyStep : Step(name) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - override fun isIdempotent() = true - - @Suppress("TooGenericExceptionCaught") - override suspend fun execute(): SetReadOnlyStep { + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index try { val updateSettingsRequest = UpdateSettingsRequest() .indices(indexName) .settings(Settings.builder().put(SETTING_BLOCKS_WRITE, true)) - val response: AcknowledgedResponse = client.admin().indices() + val response: AcknowledgedResponse = context.client.admin().indices() .suspendUntil { updateSettings(updateSettingsRequest, it) } if (response.isAcknowledged) { @@ -52,15 +43,15 @@ class SetReadOnlyStep( info = mapOf("message" to message) } } catch (e: RemoteTransportException) { - handleException(ExceptionsHelper.unwrapCause(e) as Exception) + handleException(indexName, ExceptionsHelper.unwrapCause(e) as Exception) } catch (e: Exception) { - handleException(e) + handleException(indexName, e) } return this } - private fun handleException(e: Exception) { + private fun handleException(indexName: String, e: Exception) { val message = getFailedMessage(indexName) logger.error(message, e) stepStatus = StepStatus.FAILED @@ -70,15 +61,18 @@ class SetReadOnlyStep( info = mutableInfo.toMap() } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - return currentMetaData.copy( - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + return currentMetadata.copy( + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), transitionTo = null, info = info ) } + override fun isIdempotent(): Boolean = true + companion object { + const val name = "set_read_only" fun getFailedMessage(index: String) = "Failed to set index to read-only [index=$index]" fun getSuccessMessage(index: String) = "Successfully set index to read-only [index=$index]" } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/readwrite/SetReadWriteStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/readwrite/SetReadWriteStep.kt index 87080d6ec..91e4d1a54 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/readwrite/SetReadWriteStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/readwrite/SetReadWriteStep.kt @@ -9,39 +9,30 @@ import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest import org.opensearch.action.support.master.AcknowledgedResponse -import org.opensearch.client.Client import org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_WRITE -import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReadWriteActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.transport.RemoteTransportException -class SetReadWriteStep( - val clusterService: ClusterService, - val client: Client, - val config: ReadWriteActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step("set_read_write", managedIndexMetaData) { +class SetReadWriteStep : Step(name) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - override fun isIdempotent() = true - - @Suppress("TooGenericExceptionCaught") - override suspend fun execute(): SetReadWriteStep { + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index try { val updateSettingsRequest = UpdateSettingsRequest() .indices(indexName) .settings( Settings.builder().put(SETTING_BLOCKS_WRITE, false) ) - val response: AcknowledgedResponse = client.admin().indices() + val response: AcknowledgedResponse = context.client.admin().indices() .suspendUntil { updateSettings(updateSettingsRequest, it) } if (response.isAcknowledged) { @@ -54,15 +45,15 @@ class SetReadWriteStep( info = mapOf("message" to message) } } catch (e: RemoteTransportException) { - handleException(ExceptionsHelper.unwrapCause(e) as Exception) + handleException(indexName, ExceptionsHelper.unwrapCause(e) as Exception) } catch (e: Exception) { - handleException(e) + handleException(indexName, e) } return this } - private fun handleException(e: Exception) { + private fun handleException(indexName: String, e: Exception) { val message = getFailedMessage(indexName) logger.error(message, e) stepStatus = StepStatus.FAILED @@ -72,15 +63,18 @@ class SetReadWriteStep( info = mutableInfo.toMap() } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - return currentMetaData.copy( - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + return currentMetadata.copy( + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), transitionTo = null, info = info ) } + override fun isIdempotent(): Boolean = true + companion object { + const val name = "set_read_write" fun getFailedMessage(index: String) = "Failed to set index to read-write [index=$index]" fun getSuccessMessage(index: String) = "Successfully set index to read-write [index=$index]" } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/replicacount/AttemptSetReplicaCountStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/replicacount/AttemptReplicaCountStep.kt similarity index 65% rename from src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/replicacount/AttemptSetReplicaCountStep.kt rename to src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/replicacount/AttemptReplicaCountStep.kt index 9fb0e6f4e..4599668c8 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/replicacount/AttemptSetReplicaCountStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/replicacount/AttemptReplicaCountStep.kt @@ -9,38 +9,30 @@ import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest import org.opensearch.action.support.master.AcknowledgedResponse -import org.opensearch.client.Client import org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS -import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReplicaCountActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.indexmanagement.indexstatemanagement.action.ReplicaCountAction import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.transport.RemoteTransportException -class AttemptSetReplicaCountStep( - val clusterService: ClusterService, - val client: Client, - val config: ReplicaCountActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step("attempt_set_replica_count", managedIndexMetaData) { +class AttemptReplicaCountStep(private val action: ReplicaCountAction) : Step(name) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - private val numOfReplicas = config.numOfReplicas + private val numOfReplicas = action.numOfReplicas - override fun isIdempotent() = true - - @Suppress("TooGenericExceptionCaught") - override suspend fun execute(): AttemptSetReplicaCountStep { + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index try { val updateSettingsRequest = UpdateSettingsRequest() .indices(indexName) .settings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, numOfReplicas)) - val response: AcknowledgedResponse = client.admin().indices() + val response: AcknowledgedResponse = context.client.admin().indices() .suspendUntil { updateSettings(updateSettingsRequest, it) } if (response.isAcknowledged) { @@ -53,15 +45,15 @@ class AttemptSetReplicaCountStep( info = mapOf("message" to message) } } catch (e: RemoteTransportException) { - handleException(ExceptionsHelper.unwrapCause(e) as Exception) + handleException(indexName, numOfReplicas, ExceptionsHelper.unwrapCause(e) as Exception) } catch (e: Exception) { - handleException(e) + handleException(indexName, numOfReplicas, e) } return this } - private fun handleException(e: Exception) { + private fun handleException(indexName: String, numOfReplicas: Int, e: Exception) { val message = getFailedMessage(indexName, numOfReplicas) logger.error(message, e) stepStatus = StepStatus.FAILED @@ -71,15 +63,18 @@ class AttemptSetReplicaCountStep( info = mutableInfo.toMap() } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - return currentMetaData.copy( - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + return currentMetadata.copy( + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), transitionTo = null, info = info ) } + override fun isIdempotent() = true + companion object { + const val name = "attempt_set_replica_count" fun getFailedMessage(index: String, numOfReplicas: Int) = "Failed to set number_of_replicas to $numOfReplicas [index=$index]" fun getSuccessMessage(index: String, numOfReplicas: Int) = "Successfully set number_of_replicas to $numOfReplicas [index=$index]" } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/rollover/AttemptRolloverStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/rollover/AttemptRolloverStep.kt index c15c8c6c8..6657e672a 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/rollover/AttemptRolloverStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/rollover/AttemptRolloverStep.kt @@ -11,39 +11,34 @@ import org.opensearch.action.admin.indices.rollover.RolloverRequest import org.opensearch.action.admin.indices.rollover.RolloverResponse import org.opensearch.action.admin.indices.stats.IndicesStatsRequest import org.opensearch.action.admin.indices.stats.IndicesStatsResponse -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService import org.opensearch.common.unit.ByteSizeValue import org.opensearch.common.unit.TimeValue -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.RolloverActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData +import org.opensearch.indexmanagement.indexstatemanagement.action.RolloverAction import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.getRolloverAlias import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.getRolloverSkip -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.indexstatemanagement.util.evaluateConditions import org.opensearch.indexmanagement.opensearchapi.getUsefulCauseString import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.rest.RestStatus import org.opensearch.transport.RemoteTransportException import java.time.Instant -@Suppress("ReturnCount", "TooGenericExceptionCaught") -class AttemptRolloverStep( - val clusterService: ClusterService, - val client: Client, - val config: RolloverActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step("attempt_rollover", managedIndexMetaData) { +@Suppress("ReturnCount") +class AttemptRolloverStep(private val action: RolloverAction) : Step(name) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - override fun isIdempotent() = false - - @Suppress("ComplexMethod", "LongMethod", "TooGenericExceptionCaught") - override suspend fun execute(): AttemptRolloverStep { + @Suppress("ComplexMethod", "LongMethod") + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index + val clusterService = context.clusterService val skipRollover = clusterService.state().metadata.index(indexName).getRolloverSkip() if (skipRollover) { stepStatus = StepStatus.COMPLETED @@ -51,7 +46,7 @@ class AttemptRolloverStep( return this } - val (rolloverTarget, isDataStream) = getRolloverTargetOrUpdateInfo() + val (rolloverTarget, isDataStream) = getRolloverTargetOrUpdateInfo(context) // If the rolloverTarget is null, we would've already updated the failed info from getRolloverTargetOrUpdateInfo and can return early rolloverTarget ?: return this @@ -61,13 +56,13 @@ class AttemptRolloverStep( return this } - if (!isDataStream && !preCheckIndexAlias(rolloverTarget)) { + if (!isDataStream && !preCheckIndexAlias(context, rolloverTarget)) { stepStatus = StepStatus.FAILED info = mapOf("message" to getFailedPreCheckMessage(indexName)) return this } - val statsResponse = getIndexStatsOrUpdateInfo() + val statsResponse = getIndexStatsOrUpdateInfo(context) // If statsResponse is null we already updated failed info from getIndexStatsOrUpdateInfo and can return early statsResponse ?: return this @@ -83,28 +78,29 @@ class AttemptRolloverStep( val indexSize = ByteSizeValue(statsResponse.primaries.docs?.totalSizeInBytes ?: 0) val largestPrimaryShard = statsResponse.shards.maxByOrNull { it.stats.docs?.totalSizeInBytes ?: 0 } val largestPrimaryShardSize = ByteSizeValue(largestPrimaryShard?.stats?.docs?.totalSizeInBytes ?: 0) + val conditions = listOfNotNull( - config.minAge?.let { - RolloverActionConfig.MIN_INDEX_AGE_FIELD to mapOf( + action.minAge?.let { + RolloverAction.MIN_INDEX_AGE_FIELD to mapOf( "condition" to it.toString(), "current" to indexAgeTimeValue.toString(), "creationDate" to indexCreationDate ) }, - config.minDocs?.let { - RolloverActionConfig.MIN_DOC_COUNT_FIELD to mapOf( + action.minDocs?.let { + RolloverAction.MIN_DOC_COUNT_FIELD to mapOf( "condition" to it, "current" to numDocs ) }, - config.minSize?.let { - RolloverActionConfig.MIN_SIZE_FIELD to mapOf( + action.minSize?.let { + RolloverAction.MIN_SIZE_FIELD to mapOf( "condition" to it.toString(), "current" to indexSize.toString() ) }, - config.minPrimaryShardSize?.let { - RolloverActionConfig.MIN_PRIMARY_SHARD_SIZE_FIELD to mapOf( + action.minPrimaryShardSize?.let { + RolloverAction.MIN_PRIMARY_SHARD_SIZE_FIELD to mapOf( "condition" to it.toString(), "current" to largestPrimaryShardSize.toString(), "shard" to largestPrimaryShard?.shardRouting?.id() @@ -112,12 +108,12 @@ class AttemptRolloverStep( } ).toMap() - if (config.evaluateConditions(indexAgeTimeValue, numDocs, indexSize, largestPrimaryShardSize)) { + if (action.evaluateConditions(indexAgeTimeValue, numDocs, indexSize, largestPrimaryShardSize)) { logger.info( "$indexName rollover conditions evaluated to true [indexCreationDate=$indexCreationDate," + " numDocs=$numDocs, indexSize=${indexSize.bytes}, primaryShardSize=${largestPrimaryShardSize.bytes}]" ) - executeRollover(rolloverTarget, isDataStream, conditions) + executeRollover(context, rolloverTarget, isDataStream, conditions) } else { stepStatus = StepStatus.CONDITION_NOT_MET info = mapOf("message" to getPendingMessage(indexName), "conditions" to conditions) @@ -126,63 +122,41 @@ class AttemptRolloverStep( return this } - @Suppress("ComplexMethod") - private suspend fun executeRollover(rolloverTarget: String, isDataStream: Boolean, conditions: Map>) { - try { - val request = RolloverRequest(rolloverTarget, null) - val response: RolloverResponse = client.admin().indices().suspendUntil { rolloverIndex(request, it) } - - // Do not need to check for isRolledOver as we are not passing any conditions or dryrun=true - // which are the only two ways it comes back as false - - // If the response is acknowledged, then the new index is created and added to one of the following index abstractions: - // 1. IndexAbstraction.Type.DATA_STREAM - the new index is added to the data stream indicated by the 'rolloverTarget' - // 2. IndexAbstraction.Type.ALIAS - the new index is added to the alias indicated by the 'rolloverTarget' - if (response.isAcknowledged) { - val message = when { - isDataStream -> getSuccessDataStreamRolloverMessage(rolloverTarget, indexName) - else -> getSuccessMessage(indexName) - } + private fun getRolloverTargetOrUpdateInfo(context: StepContext): Pair { + val indexName = context.metadata.index + val metadata = context.clusterService.state().metadata() + val indexAbstraction = metadata.indicesLookup[indexName] + val isDataStreamIndex = indexAbstraction?.parentDataStream != null - stepStatus = StepStatus.COMPLETED - info = listOfNotNull( - "message" to message, - if (conditions.isEmpty()) null else "conditions" to conditions // don't show empty conditions object if no conditions specified - ).toMap() - } else { - val message = when { - isDataStream -> getFailedDataStreamRolloverMessage(rolloverTarget) + val rolloverTarget = when { + isDataStreamIndex -> indexAbstraction?.parentDataStream?.name + else -> metadata.index(indexName).getRolloverAlias() + } - // If the alias update response was NOT acknowledged, then the new index was created but we failed to swap the alias - else -> getFailedAliasUpdateMessage(indexName, response.newIndex) - } - logger.warn(message) - stepStatus = StepStatus.FAILED - info = listOfNotNull( - "message" to message, - if (conditions.isEmpty()) null else "conditions" to conditions // don't show empty conditions object if no conditions specified - ).toMap() - } - } catch (e: RemoteTransportException) { - handleException(ExceptionsHelper.unwrapCause(e) as Exception) - } catch (e: Exception) { - handleException(e) + if (rolloverTarget == null) { + val message = getFailedNoValidAliasMessage(indexName) + logger.warn(message) + stepStatus = StepStatus.FAILED + info = mapOf("message" to message) } + + return rolloverTarget to isDataStreamIndex } /** * pre-condition check on managed-index's alias before rollover * * This will block - * when managed index dont have alias + * when managed index doesn't have alias * when managed index has alias but not the write index, * and this alias contains more than one index * User can use skip rollover setting to bypass this * * @param alias user defined ISM rollover alias */ - private fun preCheckIndexAlias(alias: String): Boolean { - val metadata = clusterService.state().metadata + private fun preCheckIndexAlias(context: StepContext, alias: String): Boolean { + val indexName = context.metadata.index + val metadata = context.clusterService.state().metadata val indexAlias = metadata.index(indexName)?.aliases?.get(alias) logger.debug("Index $indexName has aliases $indexAlias") if (indexAlias == null) { @@ -200,31 +174,12 @@ class AttemptRolloverStep( return true } - private fun getRolloverTargetOrUpdateInfo(): Pair { - val metadata = clusterService.state().metadata() - val indexAbstraction = metadata.indicesLookup[indexName] - val isDataStreamIndex = indexAbstraction?.parentDataStream != null - - val rolloverTarget = when { - isDataStreamIndex -> indexAbstraction?.parentDataStream?.name - else -> metadata.index(indexName).getRolloverAlias() - } - - if (rolloverTarget == null) { - val message = getFailedNoValidAliasMessage(indexName) - logger.warn(message) - stepStatus = StepStatus.FAILED - info = mapOf("message" to message) - } - - return rolloverTarget to isDataStreamIndex - } - - private suspend fun getIndexStatsOrUpdateInfo(): IndicesStatsResponse? { + private suspend fun getIndexStatsOrUpdateInfo(context: StepContext): IndicesStatsResponse? { + val indexName = context.metadata.index try { val statsRequest = IndicesStatsRequest() .indices(indexName).clear().docs(true) - val statsResponse: IndicesStatsResponse = client.admin().indices().suspendUntil { stats(statsRequest, it) } + val statsResponse: IndicesStatsResponse = context.client.admin().indices().suspendUntil { stats(statsRequest, it) } if (statsResponse.status == RestStatus.OK) { return statsResponse @@ -238,15 +193,74 @@ class AttemptRolloverStep( "shard_failures" to statsResponse.shardFailures.map { it.getUsefulCauseString() } ) } catch (e: RemoteTransportException) { - handleException(ExceptionsHelper.unwrapCause(e) as Exception) + handleException(indexName, ExceptionsHelper.unwrapCause(e) as Exception) } catch (e: Exception) { - handleException(e, getFailedEvaluateMessage(indexName)) + handleException(indexName, e, getFailedEvaluateMessage(indexName)) } return null } - private fun handleException(e: Exception, message: String = getFailedMessage(indexName)) { + @Suppress("ComplexMethod") + private suspend fun executeRollover( + context: StepContext, + rolloverTarget: String, + isDataStream: Boolean, + conditions: Map> + ) { + val indexName = context.metadata.index + try { + val request = RolloverRequest(rolloverTarget, null) + val response: RolloverResponse = context.client.admin().indices().suspendUntil { rolloverIndex(request, it) } + + // Do not need to check for isRolledOver as we are not passing any conditions or dryrun=true + // which are the only two ways it comes back as false + + // If the response is acknowledged, then the new index is created and added to one of the following index abstractions: + // 1. IndexAbstraction.Type.DATA_STREAM - the new index is added to the data stream indicated by the 'rolloverTarget' + // 2. IndexAbstraction.Type.ALIAS - the new index is added to the alias indicated by the 'rolloverTarget' + if (response.isAcknowledged) { + val message = when { + isDataStream -> getSuccessDataStreamRolloverMessage(rolloverTarget, indexName) + else -> getSuccessMessage(indexName) + } + + stepStatus = StepStatus.COMPLETED + info = listOfNotNull( + "message" to message, + if (conditions.isEmpty()) null else "conditions" to conditions // don't show empty conditions object if no conditions specified + ).toMap() + } else { + val message = when { + isDataStream -> getFailedDataStreamRolloverMessage(rolloverTarget) + + // If the alias update response was NOT acknowledged, then the new index was created but we failed to swap the alias + else -> getFailedAliasUpdateMessage(indexName, response.newIndex) + } + logger.warn(message) + stepStatus = StepStatus.FAILED + info = listOfNotNull( + "message" to message, + if (conditions.isEmpty()) null else "conditions" to conditions // don't show empty conditions object if no conditions specified + ).toMap() + } + } catch (e: RemoteTransportException) { + handleException(indexName, ExceptionsHelper.unwrapCause(e) as Exception) + } catch (e: Exception) { + handleException(indexName, e) + } + } + + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + return currentMetadata.copy( + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), + rolledOver = if (currentMetadata.rolledOver == true) true else stepStatus == StepStatus.COMPLETED, + transitionTo = null, + info = info + ) + } + + private fun handleException(indexName: String, e: Exception, message: String = getFailedMessage(indexName)) { logger.error(message, e) stepStatus = StepStatus.FAILED val mutableInfo = mutableMapOf("message" to message) @@ -255,17 +269,11 @@ class AttemptRolloverStep( info = mutableInfo.toMap() } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - return currentMetaData.copy( - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), - rolledOver = if (currentMetaData.rolledOver == true) true else stepStatus == StepStatus.COMPLETED, - transitionTo = null, - info = info - ) - } + override fun isIdempotent(): Boolean = false @Suppress("TooManyFunctions") companion object { + const val name = "attempt_rollover" fun getFailedMessage(index: String) = "Failed to rollover index [index=$index]" fun getFailedAliasUpdateMessage(index: String, newIndex: String) = "New index created, but failed to update alias [index=$index, newIndex=$newIndex]" diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/rollup/AttemptCreateRollupJobStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/rollup/AttemptCreateRollupJobStep.kt index dc901f673..e4a6c6392 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/rollup/AttemptCreateRollupJobStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/rollup/AttemptCreateRollupJobStep.kt @@ -9,52 +9,44 @@ import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.action.support.WriteRequest import org.opensearch.action.support.master.AcknowledgedResponse -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService import org.opensearch.index.engine.VersionConflictEngineException -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionProperties -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.indexmanagement.indexstatemanagement.action.RollupAction import org.opensearch.indexmanagement.opensearchapi.suspendUntil import org.opensearch.indexmanagement.rollup.action.index.IndexRollupAction import org.opensearch.indexmanagement.rollup.action.index.IndexRollupRequest import org.opensearch.indexmanagement.rollup.action.index.IndexRollupResponse import org.opensearch.indexmanagement.rollup.action.start.StartRollupAction import org.opensearch.indexmanagement.rollup.action.start.StartRollupRequest -import org.opensearch.indexmanagement.rollup.model.ISMRollup +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.transport.RemoteTransportException -import java.lang.Exception -class AttemptCreateRollupJobStep( - val clusterService: ClusterService, - val client: Client, - val ismRollup: ISMRollup, - managedIndexMetaData: ManagedIndexMetaData -) : Step(name, managedIndexMetaData) { +class AttemptCreateRollupJobStep(private val action: RollupAction) : Step(name) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null private var rollupId: String? = null - private var previousRunRollupId: String? = null - private var hasPreviousRollupAttemptFailed: Boolean? = null - - override fun isIdempotent() = true override suspend fun execute(): Step { - previousRunRollupId = managedIndexMetaData.actionMetaData?.actionProperties?.rollupId - hasPreviousRollupAttemptFailed = managedIndexMetaData.actionMetaData?.actionProperties?.hasRollupFailed + val context = this.context ?: return this + val indexName = context.metadata.index + val managedIndexMetadata = context.metadata + val previousRunRollupId = managedIndexMetadata.actionMetaData?.actionProperties?.rollupId + val hasPreviousRollupAttemptFailed = managedIndexMetadata.actionMetaData?.actionProperties?.hasRollupFailed // Creating a rollup job - val rollup = ismRollup.toRollup(indexName, managedIndexMetaData.user) + val rollup = action.ismRollup.toRollup(indexName, context.user) rollupId = rollup.id logger.info("Attempting to create a rollup job $rollupId for index $indexName") val indexRollupRequest = IndexRollupRequest(rollup, WriteRequest.RefreshPolicy.IMMEDIATE) try { - val response: IndexRollupResponse = client.suspendUntil { execute(IndexRollupAction.INSTANCE, indexRollupRequest, it) } + val response: IndexRollupResponse = context.client.suspendUntil { execute(IndexRollupAction.INSTANCE, indexRollupRequest, it) } logger.info("Received status ${response.status.status} on trying to create rollup job $rollupId") stepStatus = StepStatus.COMPLETED @@ -63,38 +55,30 @@ class AttemptCreateRollupJobStep( val message = getFailedJobExistsMessage(rollup.id, indexName) logger.info(message) if (rollupId == previousRunRollupId && hasPreviousRollupAttemptFailed == true) { - startRollupJob(rollup.id) + startRollupJob(rollup.id, context) } else { stepStatus = StepStatus.COMPLETED info = mapOf("info" to message) } } catch (e: RemoteTransportException) { - processFailure(rollup.id, ExceptionsHelper.unwrapCause(e) as Exception) + processFailure(rollup.id, indexName, ExceptionsHelper.unwrapCause(e) as Exception) } catch (e: RemoteTransportException) { - processFailure(rollup.id, e) + processFailure(rollup.id, indexName, e) } return this } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - val currentActionMetaData = currentMetaData.actionMetaData - return currentMetaData.copy( - actionMetaData = currentActionMetaData?.copy(actionProperties = ActionProperties(rollupId = rollupId)), - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), - transitionTo = null, - info = info - ) - } - - fun processFailure(rollupId: String, e: Exception) { + fun processFailure(rollupId: String, indexName: String, e: Exception) { val message = getFailedMessage(rollupId, indexName) logger.error(message, e) stepStatus = StepStatus.FAILED info = mapOf("message" to message, "cause" to "${e.message}") } - private suspend fun startRollupJob(rollupId: String) { + private suspend fun startRollupJob(rollupId: String, context: StepContext) { + val indexName = context.metadata.index + val client = context.client logger.info("Attempting to re-start the job $rollupId") try { val startRollupRequest = StartRollupRequest(rollupId) @@ -109,6 +93,18 @@ class AttemptCreateRollupJobStep( } } + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + val currentActionMetaData = currentMetadata.actionMetaData + return currentMetadata.copy( + actionMetaData = currentActionMetaData?.copy(actionProperties = ActionProperties(rollupId = rollupId)), + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), + transitionTo = null, + info = info + ) + } + + override fun isIdempotent(): Boolean = true + companion object { const val name = "attempt_create_rollup" fun getFailedMessage(rollupId: String, index: String) = "Failed to create the rollup job [$rollupId] [index=$index]" diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/rollup/WaitForRollupCompletionStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/rollup/WaitForRollupCompletionStep.kt index 19c876b4e..d7170c031 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/rollup/WaitForRollupCompletionStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/rollup/WaitForRollupCompletionStep.kt @@ -6,34 +6,28 @@ package org.opensearch.indexmanagement.indexstatemanagement.step.rollup import org.apache.logging.log4j.LogManager -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.opensearchapi.suspendUntil import org.opensearch.indexmanagement.rollup.action.explain.ExplainRollupAction import org.opensearch.indexmanagement.rollup.action.explain.ExplainRollupRequest import org.opensearch.indexmanagement.rollup.action.explain.ExplainRollupResponse import org.opensearch.indexmanagement.rollup.model.RollupMetadata +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.transport.RemoteTransportException -import java.lang.Exception -class WaitForRollupCompletionStep( - val clusterService: ClusterService, - val client: Client, - managedIndexMetaData: ManagedIndexMetaData -) : Step(name, managedIndexMetaData) { +class WaitForRollupCompletionStep : Step(name) { + private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - private val logger = LogManager.getLogger(javaClass) private var hasRollupFailed: Boolean? = null - override fun isIdempotent() = true - - override suspend fun execute(): WaitForRollupCompletionStep { - val rollupJobId = managedIndexMetaData.actionMetaData?.actionProperties?.rollupId + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index + val managedIndexMetadata = context.metadata + val rollupJobId = managedIndexMetadata.actionMetaData?.actionProperties?.rollupId if (rollupJobId == null) { logger.error("No rollup job id passed down") @@ -42,7 +36,7 @@ class WaitForRollupCompletionStep( } else { val explainRollupRequest = ExplainRollupRequest(listOf(rollupJobId)) try { - val response: ExplainRollupResponse = client.suspendUntil { execute(ExplainRollupAction.INSTANCE, explainRollupRequest, it) } + val response: ExplainRollupResponse = context.client.suspendUntil { execute(ExplainRollupAction.INSTANCE, explainRollupRequest, it) } logger.info("Received the status for jobs [${response.getIdsToExplain().keys}]") val metadata = response.getIdsToExplain()[rollupJobId]?.metadata @@ -51,34 +45,19 @@ class WaitForRollupCompletionStep( stepStatus = StepStatus.CONDITION_NOT_MET info = mapOf("message" to getJobProcessingMessage(rollupJobId, indexName)) } else { - processRollupMetadataStatus(rollupJobId, metadata) + processRollupMetadataStatus(rollupJobId, indexName, metadata) } } catch (e: RemoteTransportException) { - processFailure(rollupJobId, e) + processFailure(rollupJobId, indexName, e) } catch (e: Exception) { - processFailure(rollupJobId, e) + processFailure(rollupJobId, indexName, e) } } return this } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - val currentActionMetaData = currentMetaData.actionMetaData - val currentActionProperties = currentActionMetaData?.actionProperties - return currentMetaData.copy( - actionMetaData = currentActionMetaData?.copy( - actionProperties = currentActionProperties?.copy( - hasRollupFailed = hasRollupFailed - ) - ), - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), - transitionTo = null, - info = info - ) - } - - fun processRollupMetadataStatus(rollupJobId: String, rollupMetadata: RollupMetadata) { + fun processRollupMetadataStatus(rollupJobId: String, indexName: String, rollupMetadata: RollupMetadata) { when (rollupMetadata.status) { RollupMetadata.Status.INIT -> { stepStatus = StepStatus.CONDITION_NOT_MET @@ -109,7 +88,7 @@ class WaitForRollupCompletionStep( } } - fun processFailure(rollupJobId: String, e: Exception) { + fun processFailure(rollupJobId: String, indexName: String, e: Exception) { stepStatus = StepStatus.FAILED val message = getFailedMessage(rollupJobId, indexName) logger.error(message, e) @@ -119,6 +98,23 @@ class WaitForRollupCompletionStep( info = mutableInfo.toMap() } + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + val currentActionMetaData = currentMetadata.actionMetaData + val currentActionProperties = currentActionMetaData?.actionProperties + return currentMetadata.copy( + actionMetaData = currentActionMetaData?.copy( + actionProperties = currentActionProperties?.copy( + hasRollupFailed = hasRollupFailed + ) + ), + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), + transitionTo = null, + info = info + ) + } + + override fun isIdempotent(): Boolean = true + companion object { const val name = "wait_for_rollup_completion" const val JOB_STOPPED_MESSAGE = "Rollup job was stopped" diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/snapshot/AttemptSnapshotStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/snapshot/AttemptSnapshotStep.kt index 3d04f816d..f4ffef794 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/snapshot/AttemptSnapshotStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/snapshot/AttemptSnapshotStep.kt @@ -9,17 +9,15 @@ import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService import org.opensearch.common.regex.Regex -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.SnapshotActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionProperties -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData +import org.opensearch.indexmanagement.indexstatemanagement.action.SnapshotAction import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.SNAPSHOT_DENY_LIST -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.opensearchapi.convertToMap import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.rest.RestStatus import org.opensearch.script.Script import org.opensearch.script.ScriptService @@ -32,30 +30,28 @@ import java.time.ZoneId import java.time.format.DateTimeFormatter import java.util.Locale -class AttemptSnapshotStep( - val clusterService: ClusterService, - val scriptService: ScriptService, - val client: Client, - val config: SnapshotActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step(name, managedIndexMetaData) { +class AttemptSnapshotStep(private val action: SnapshotAction) : Step(name) { private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null private var snapshotName: String? = null - private var denyList: List = clusterService.clusterSettings.get(SNAPSHOT_DENY_LIST) - override fun isIdempotent() = false - - @Suppress("TooGenericExceptionCaught", "ComplexMethod") - override suspend fun execute(): AttemptSnapshotStep { + @Suppress("TooGenericExceptionCaught", "ComplexMethod", "ReturnCount", "LongMethod") + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index + val managedIndexMetadata = context.metadata + val scriptService = context.scriptService + val denyList: List = context.clusterService.clusterSettings.get(SNAPSHOT_DENY_LIST) + val repository = action.repository + val snapshot = action.snapshot try { val mutableInfo = mutableMapOf() - if (isDenied(denyList, config.repository)) { + if (isDenied(denyList, repository)) { stepStatus = StepStatus.FAILED - mutableInfo["message"] = getBlockedMessage(denyList, config.repository, indexName) + mutableInfo["message"] = getBlockedMessage(denyList, repository, indexName) info = mutableInfo.toMap() return this } @@ -64,19 +60,19 @@ class AttemptSnapshotStep( .format(DateTimeFormatter.ofPattern("uuuu.MM.dd-HH:mm:ss.SSS", Locale.ROOT)) ) - val snapshotScript = Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, config.snapshot, mapOf()) + val snapshotScript = Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, snapshot, mapOf()) // If user intentionally set the snapshot name empty then we are going to honor it - val defaultSnapshotName = if (config.snapshot.isBlank()) config.snapshot else indexName - snapshotName = compileTemplate(snapshotScript, managedIndexMetaData, defaultSnapshotName).plus(snapshotNameSuffix) + val defaultSnapshotName = if (snapshot.isBlank()) snapshot else indexName + snapshotName = compileTemplate(snapshotScript, managedIndexMetadata, defaultSnapshotName, scriptService).plus(snapshotNameSuffix) val createSnapshotRequest = CreateSnapshotRequest() .userMetadata(mapOf("snapshot_created" to "Open Distro for Elasticsearch Index Management")) .indices(indexName) .snapshot(snapshotName) - .repository(config.repository) + .repository(repository) .waitForCompletion(false) - val response: CreateSnapshotResponse = client.admin().cluster().suspendUntil { createSnapshot(createSnapshotRequest, it) } + val response: CreateSnapshotResponse = context.client.admin().cluster().suspendUntil { createSnapshot(createSnapshotRequest, it) } when (response.status()) { RestStatus.ACCEPTED -> { stepStatus = StepStatus.COMPLETED @@ -98,14 +94,14 @@ class AttemptSnapshotStep( } catch (e: RemoteTransportException) { val cause = ExceptionsHelper.unwrapCause(e) if (cause is ConcurrentSnapshotExecutionException) { - handleSnapshotException(cause) + handleSnapshotException(indexName, cause) } else { - handleException(cause as Exception) + handleException(indexName, cause as Exception) } } catch (e: ConcurrentSnapshotExecutionException) { - handleSnapshotException(e) + handleSnapshotException(indexName, e) } catch (e: Exception) { - handleException(e) + handleException(indexName, e) } return this @@ -116,14 +112,14 @@ class AttemptSnapshotStep( return denyList.stream().anyMatch(predicate) } - private fun handleSnapshotException(e: ConcurrentSnapshotExecutionException) { + private fun handleSnapshotException(indexName: String, e: ConcurrentSnapshotExecutionException) { val message = getFailedConcurrentSnapshotMessage(indexName) logger.debug(message, e) stepStatus = StepStatus.CONDITION_NOT_MET info = mapOf("message" to message) } - private fun handleException(e: Exception) { + private fun handleException(indexName: String, e: Exception) { val message = getFailedMessage(indexName) logger.error(message, e) stepStatus = StepStatus.FAILED @@ -133,7 +129,12 @@ class AttemptSnapshotStep( info = mutableInfo.toMap() } - private fun compileTemplate(template: Script, managedIndexMetaData: ManagedIndexMetaData, defaultValue: String): String { + private fun compileTemplate( + template: Script, + managedIndexMetaData: ManagedIndexMetaData, + defaultValue: String, + scriptService: ScriptService + ): String { val contextMap = managedIndexMetaData.convertToMap().filterKeys { key -> key in validTopContextFields } @@ -143,16 +144,18 @@ class AttemptSnapshotStep( return if (compiledValue.isBlank()) defaultValue else compiledValue } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - val currentActionMetaData = currentMetaData.actionMetaData - return currentMetaData.copy( + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + val currentActionMetaData = currentMetadata.actionMetaData + return currentMetadata.copy( actionMetaData = currentActionMetaData?.copy(actionProperties = ActionProperties(snapshotName = snapshotName)), - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), transitionTo = null, info = info ) } + override fun isIdempotent(): Boolean = false + companion object { val validTopContextFields = setOf("index", "indexUuid") const val name = "attempt_snapshot" diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/snapshot/WaitForSnapshotStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/snapshot/WaitForSnapshotStep.kt index 579277809..6018dd2d3 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/snapshot/WaitForSnapshotStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/snapshot/WaitForSnapshotStep.kt @@ -10,42 +10,38 @@ import org.opensearch.ExceptionsHelper import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse -import org.opensearch.client.Client import org.opensearch.cluster.SnapshotsInProgress.State -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.SnapshotActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionProperties -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.indexmanagement.indexstatemanagement.action.SnapshotAction import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.transport.RemoteTransportException -class WaitForSnapshotStep( - val clusterService: ClusterService, - val client: Client, - val config: SnapshotActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step(name, managedIndexMetaData) { +class WaitForSnapshotStep(private val action: SnapshotAction) : Step(name) { + private val logger = LogManager.getLogger(javaClass) private var stepStatus = StepStatus.STARTING private var info: Map? = null - override fun isIdempotent() = true + @Suppress("ComplexMethod", "ReturnCount") + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index + val managedIndexMetadata = context.metadata + val repository = action.repository - @Suppress("ComplexMethod") - override suspend fun execute(): WaitForSnapshotStep { try { - val snapshotName = getSnapshotName() ?: return this + val snapshotName = getSnapshotName(managedIndexMetadata, indexName) ?: return this val request = SnapshotsStatusRequest() .snapshots(arrayOf(snapshotName)) - .repository(config.repository) - val response: SnapshotsStatusResponse = client.admin().cluster().suspendUntil { snapshotsStatus(request, it) } + .repository(repository) + val response: SnapshotsStatusResponse = context.client.admin().cluster().suspendUntil { snapshotsStatus(request, it) } val status: SnapshotStatus? = response .snapshots .find { snapshotStatus -> - snapshotStatus.snapshot.snapshotId.name == snapshotName && - snapshotStatus.snapshot.repository == config.repository + snapshotStatus.snapshot.snapshotId.name == snapshotName && snapshotStatus.snapshot.repository == repository } if (status != null) { when (status.state) { @@ -71,15 +67,15 @@ class WaitForSnapshotStep( info = mapOf("message" to message) } } catch (e: RemoteTransportException) { - handleException(ExceptionsHelper.unwrapCause(e) as Exception) + handleException(indexName, ExceptionsHelper.unwrapCause(e) as Exception) } catch (e: Exception) { - handleException(e) + handleException(indexName, e) } return this } - private fun handleException(e: Exception) { + private fun handleException(indexName: String, e: Exception) { val message = getFailedMessage(indexName) logger.error(message, e) stepStatus = StepStatus.FAILED @@ -89,8 +85,8 @@ class WaitForSnapshotStep( info = mutableInfo.toMap() } - private fun getSnapshotName(): String? { - val actionProperties = managedIndexMetaData.actionMetaData?.actionProperties + private fun getSnapshotName(managedIndexMetadata: ManagedIndexMetaData, indexName: String): String? { + val actionProperties = managedIndexMetadata.actionMetaData?.actionProperties if (actionProperties?.snapshotName == null) { stepStatus = StepStatus.FAILED @@ -101,14 +97,16 @@ class WaitForSnapshotStep( return actionProperties.snapshotName } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - return currentMetaData.copy( - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + return currentMetadata.copy( + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), transitionTo = null, info = info ) } + override fun isIdempotent(): Boolean = true + companion object { const val name = "wait_for_snapshot" fun getFailedMessage(index: String) = "Failed to get status of snapshot [index=$index]" diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/transition/AttemptTransitionStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/transition/AttemptTransitionStep.kt index 841360399..843987b4e 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/transition/AttemptTransitionStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/transition/AttemptTransitionStep.kt @@ -9,35 +9,24 @@ import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.action.admin.indices.stats.IndicesStatsRequest import org.opensearch.action.admin.indices.stats.IndicesStatsResponse -import org.opensearch.client.Client import org.opensearch.cluster.service.ClusterService import org.opensearch.common.unit.ByteSizeValue -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.TransitionsActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData +import org.opensearch.indexmanagement.indexstatemanagement.IndexMetadataProvider +import org.opensearch.indexmanagement.indexstatemanagement.action.TransitionsAction import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.getOldestRolloverTime -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.indexmanagement.indexstatemanagement.util.evaluateConditions import org.opensearch.indexmanagement.indexstatemanagement.util.hasStatsConditions import org.opensearch.indexmanagement.opensearchapi.getUsefulCauseString import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.rest.RestStatus import org.opensearch.transport.RemoteTransportException import java.time.Instant -import kotlin.Exception -/** - * Attempt to transition to the next state - * - * This step compares the transition conditions configuration with the real time index stats data - * to check if the [ManagedIndexConfig] should move to the next state defined in its policy. - */ -class AttemptTransitionStep( - val clusterService: ClusterService, - val client: Client, - val config: TransitionsActionConfig, - managedIndexMetaData: ManagedIndexMetaData -) : Step("attempt_transition", managedIndexMetaData) { +class AttemptTransitionStep(private val action: TransitionsAction) : Step(name) { private val logger = LogManager.getLogger(javaClass) private var stateName: String? = null @@ -45,30 +34,35 @@ class AttemptTransitionStep( private var policyCompleted: Boolean = false private var info: Map? = null - override fun isIdempotent() = true - - @Suppress("TooGenericExceptionCaught", "ReturnCount", "ComplexMethod", "LongMethod") - override suspend fun execute(): AttemptTransitionStep { + @Suppress("ReturnCount", "ComplexMethod", "LongMethod", "NestedBlockDepth") + override suspend fun execute(): Step { + val context = this.context ?: return this + val indexName = context.metadata.index + val clusterService = context.clusterService + val transitions = action.transitions + val indexMetadataProvider = action.indexMetadataProvider try { - if (config.transitions.isEmpty()) { + if (transitions.isEmpty()) { logger.info("$indexName transitions are empty, completing policy") policyCompleted = true stepStatus = StepStatus.COMPLETED return this } - val indexMetaData = clusterService.state().metadata().index(indexName) - val indexCreationDate = indexMetaData.creationDate + val indexMetadata = clusterService.state().metadata().index(indexName) + val inCluster = clusterService.state().metadata().hasIndex(indexName) && indexMetadata?.indexUUID == context.metadata.indexUuid + + val indexCreationDate = getIndexCreationDate(context.metadata, indexMetadataProvider, clusterService, indexName, inCluster) val indexCreationDateInstant = Instant.ofEpochMilli(indexCreationDate) if (indexCreationDate == -1L) { logger.warn("$indexName had an indexCreationDate=-1L, cannot use for comparison") } - val stepStartTime = getStepStartTime() + val stepStartTime = getStepStartTime(context.metadata) var numDocs: Long? = null var indexSize: ByteSizeValue? = null - val rolloverDate: Instant? = indexMetaData.getOldestRolloverTime() - if (config.transitions.any { it.conditions?.rolloverAge !== null }) { + val rolloverDate: Instant? = if (inCluster) indexMetadata.getOldestRolloverTime() else null + if (transitions.any { it.conditions?.rolloverAge !== null }) { // if we have a transition with rollover age condition, then we must have a rollover date // otherwise fail this transition if (rolloverDate == null) { @@ -80,27 +74,32 @@ class AttemptTransitionStep( } } - if (config.transitions.any { it.hasStatsConditions() }) { - val statsRequest = IndicesStatsRequest() - .indices(indexName).clear().docs(true) - val statsResponse: IndicesStatsResponse = client.admin().indices().suspendUntil { stats(statsRequest, it) } + if (transitions.any { it.hasStatsConditions() }) { + if (inCluster) { + val statsRequest = IndicesStatsRequest() + .indices(indexName).clear().docs(true) + val statsResponse: IndicesStatsResponse = + context.client.admin().indices().suspendUntil { stats(statsRequest, it) } - if (statsResponse.status != RestStatus.OK) { - val message = getFailedStatsMessage(indexName) - logger.warn("$message - ${statsResponse.status}") - stepStatus = StepStatus.FAILED - info = mapOf( - "message" to message, - "shard_failures" to statsResponse.shardFailures.map { it.getUsefulCauseString() } - ) - return this + if (statsResponse.status != RestStatus.OK) { + val message = getFailedStatsMessage(indexName) + logger.warn("$message - ${statsResponse.status}") + stepStatus = StepStatus.FAILED + info = mapOf( + "message" to message, + "shard_failures" to statsResponse.shardFailures.map { it.getUsefulCauseString() } + ) + return this + } + numDocs = statsResponse.primaries.getDocs()?.count ?: 0 + indexSize = ByteSizeValue(statsResponse.primaries.getDocs()?.totalSizeInBytes ?: 0) + } else { + logger.warn("Cannot use index size/doc count transition conditions for index [$indexName] that does not exist in cluster") } - numDocs = statsResponse.primaries.getDocs()?.count ?: 0 - indexSize = ByteSizeValue(statsResponse.primaries.getDocs()?.totalSizeInBytes ?: 0) } // Find the first transition that evaluates to true and get the state to transition to, otherwise return null if none are true - stateName = config.transitions.find { + stateName = transitions.find { it.evaluateConditions(indexCreationDateInstant, numDocs, indexSize, stepStartTime, rolloverDate) }?.stateName val message: String @@ -118,15 +117,15 @@ class AttemptTransitionStep( } info = mapOf("message" to message) } catch (e: RemoteTransportException) { - handleException(ExceptionsHelper.unwrapCause(e) as Exception) + handleException(indexName, ExceptionsHelper.unwrapCause(e) as Exception) } catch (e: Exception) { - handleException(e) + handleException(indexName, e) } return this } - private fun handleException(e: Exception) { + private fun handleException(indexName: String, e: Exception) { val message = getFailedMessage(indexName) logger.error(message, e) stepStatus = StepStatus.FAILED @@ -136,16 +135,51 @@ class AttemptTransitionStep( info = mutableInfo.toMap() } - override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData { - return currentMetaData.copy( + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + return currentMetadata.copy( policyCompleted = policyCompleted, transitionTo = stateName, - stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus), + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), info = info ) } + @Suppress("ReturnCount") + private suspend fun getIndexCreationDate( + metadata: ManagedIndexMetaData, + indexMetadataProvider: IndexMetadataProvider, + clusterService: ClusterService, + indexName: String, + inCluster: Boolean + ): Long { + try { + // If we do have an index creation date cached already then use that + metadata.indexCreationDate?.let { return it } + // Otherwise, check if this index is in cluster state first + return if (inCluster) { + val clusterStateMetadata = clusterService.state().metadata() + if (clusterStateMetadata.hasIndex(indexName)) clusterStateMetadata.index(indexName).creationDate else -1L + } else { + // And then finally check all other index types which may not be in the cluster + val nonDefaultIndexTypes = indexMetadataProvider.services.keys.filter { it != DEFAULT_INDEX_TYPE } + val multiTypeIndexNameToMetaData = indexMetadataProvider.getMultiTypeISMIndexMetadata(nonDefaultIndexTypes, listOf(indexName)) + // the managedIndexConfig.indexUuid should be unique across all index types + val indexCreationDate = multiTypeIndexNameToMetaData.values.firstOrNull { + it[indexName]?.indexUuid == metadata.indexUuid + }?.get(indexName)?.indexCreationDate + indexCreationDate ?: -1 + } + } catch (e: Exception) { + logger.error("Failed to get index creation date for $indexName", e) + } + // -1L index age is ignored during condition checks + return -1L + } + + override fun isIdempotent() = true + companion object { + const val name = "attempt_transition_step" fun getFailedMessage(index: String) = "Failed to transition index [index=$index]" fun getFailedStatsMessage(index: String) = "Failed to get stats information for the index [index=$index]" fun getFailedRolloverDateMessage(index: String) = diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/addpolicy/AddPolicyRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/addpolicy/AddPolicyRequest.kt index 690ce76f5..c9ac3bb51 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/addpolicy/AddPolicyRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/addpolicy/AddPolicyRequest.kt @@ -10,23 +10,31 @@ import org.opensearch.action.ActionRequestValidationException import org.opensearch.action.ValidateActions import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import java.io.IOException class AddPolicyRequest( val indices: List, - val policyID: String + val policyID: String, + val indexType: String ) : ActionRequest() { @Throws(IOException::class) constructor(sin: StreamInput) : this( indices = sin.readStringList(), - policyID = sin.readString() + policyID = sin.readString(), + indexType = sin.readString() ) override fun validate(): ActionRequestValidationException? { var validationException: ActionRequestValidationException? = null if (indices.isEmpty()) { validationException = ValidateActions.addValidationError("Missing indices", validationException) + } else if (indexType != DEFAULT_INDEX_TYPE && indices.size > 1) { + validationException = ValidateActions.addValidationError( + MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR, + validationException + ) } return validationException } @@ -35,5 +43,11 @@ class AddPolicyRequest( override fun writeTo(out: StreamOutput) { out.writeStringCollection(indices) out.writeString(policyID) + out.writeString(indexType) + } + + companion object { + const val MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR = + "Cannot add policy to more than one index name/pattern when using a custom index type" } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/addpolicy/TransportAddPolicyAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/addpolicy/TransportAddPolicyAction.kt index 5ad8dcc56..0efdd0686 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/addpolicy/TransportAddPolicyAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/addpolicy/TransportAddPolicyAction.kt @@ -5,21 +5,9 @@ package org.opensearch.indexmanagement.indexstatemanagement.transport.action.addpolicy -/* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.OpenSearchSecurityException @@ -39,9 +27,7 @@ import org.opensearch.action.support.HandledTransportAction import org.opensearch.action.support.IndicesOptions import org.opensearch.action.support.master.AcknowledgedResponse import org.opensearch.client.node.NodeClient -import org.opensearch.cluster.ClusterState import org.opensearch.cluster.block.ClusterBlockException -import org.opensearch.cluster.metadata.IndexNameExpressionResolver import org.opensearch.cluster.service.ClusterService import org.opensearch.common.inject.Inject import org.opensearch.common.settings.Settings @@ -49,19 +35,24 @@ import org.opensearch.common.unit.TimeValue import org.opensearch.common.xcontent.NamedXContentRegistry import org.opensearch.commons.ConfigConstants import org.opensearch.commons.authuser.User +import org.opensearch.index.Index import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX +import org.opensearch.indexmanagement.indexstatemanagement.DefaultIndexMetadataService +import org.opensearch.indexmanagement.indexstatemanagement.IndexMetadataProvider +import org.opensearch.indexmanagement.indexstatemanagement.IndexMetadataProvider.Companion.EVALUATION_FAILURE_MESSAGE import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.getUuidsForClosedIndices import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.transport.action.ISMStatusResponse import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexRequest +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.indexmanagement.indexstatemanagement.util.FailedIndex -import org.opensearch.indexmanagement.indexstatemanagement.util.IndexEvaluator -import org.opensearch.indexmanagement.indexstatemanagement.util.IndexEvaluator.Companion.EVALUATION_FAILURE_MESSAGE import org.opensearch.indexmanagement.indexstatemanagement.util.managedIndexConfigIndexRequest +import org.opensearch.indexmanagement.indexstatemanagement.util.removeClusterStateMetadatas import org.opensearch.indexmanagement.opensearchapi.parseFromGetResponse import org.opensearch.indexmanagement.settings.IndexManagementSettings +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ISMIndexMetadata import org.opensearch.indexmanagement.util.IndexUtils import org.opensearch.indexmanagement.util.SecurityUtils.Companion.buildUser import org.opensearch.indexmanagement.util.SecurityUtils.Companion.userHasPermissionForResource @@ -84,8 +75,7 @@ class TransportAddPolicyAction @Inject constructor( val settings: Settings, val clusterService: ClusterService, val xContentRegistry: NamedXContentRegistry, - val indexNameExpressionResolver: IndexNameExpressionResolver, - val indexEvaluator: IndexEvaluator + val indexMetadataProvider: IndexMetadataProvider ) : HandledTransportAction( AddPolicyAction.NAME, transportService, actionFilters, ::AddPolicyRequest ) { @@ -110,6 +100,7 @@ class TransportAddPolicyAction @Inject constructor( AddPolicyHandler(client, listener, request).start() } + @Suppress("TooManyFunctions") inner class AddPolicyHandler( private val client: NodeClient, private val actionListener: ActionListener, @@ -118,7 +109,7 @@ class TransportAddPolicyAction @Inject constructor( ) { private lateinit var startTime: Instant private lateinit var policy: Policy - private val resolvedIndices = mutableListOf() + private val permittedIndices = mutableListOf() private val indicesToAdd = mutableMapOf() // uuid: name private val failedIndices: MutableList = mutableListOf() @@ -131,41 +122,47 @@ class TransportAddPolicyAction @Inject constructor( if (!validateUserConfiguration(user, filterByEnabled, actionListener)) { return } - val requestedIndices = mutableListOf() - request.indices.forEach { index -> - requestedIndices.addAll( - indexNameExpressionResolver.concreteIndexNames( - clusterService.state(), - IndicesOptions.lenientExpand(), - true, - index - ) - ) - } - if (requestedIndices.isEmpty()) { - // Nothing to do will ignore since found no matching indices - actionListener.onResponse(ISMStatusResponse(0, failedIndices)) - return - } - if (user == null) { - resolvedIndices.addAll(requestedIndices) - getPolicy() - } else { - validateAndGetPolicy(0, requestedIndices) + getClusterState() + } + + @Suppress("SpreadOperator") + fun getClusterState() { + startTime = Instant.now() + CoroutineScope(Dispatchers.IO).launch { + val indexNameToMetadata: MutableMap = HashMap() + try { + indexNameToMetadata.putAll(indexMetadataProvider.getISMIndexMetadataByType(request.indexType, request.indices)) + } catch (e: Exception) { + actionListener.onFailure(ExceptionsHelper.unwrapCause(e) as Exception) + return@launch + } + indexNameToMetadata.forEach { (indexName, indexMetadata) -> + indicesToAdd.putIfAbsent(indexMetadata.indexUuid, indexName) + } + if (indicesToAdd.isEmpty()) { + // Nothing to do will ignore since found no matching indices + actionListener.onResponse(ISMStatusResponse(0, failedIndices)) + return@launch + } + if (user != null) { + validateIndexPermissions(0, indicesToAdd.values.toList()) + } else { + removeClosedIndices() + } } } /** * We filter the requested indices to the indices user has permission to manage and apply policies only on top of those */ - private fun validateAndGetPolicy(current: Int, indices: List) { + private fun validateIndexPermissions(current: Int, indices: List) { val request = ManagedIndexRequest().indices(indices[current]) client.execute( ManagedIndexAction.INSTANCE, request, object : ActionListener { override fun onResponse(response: AcknowledgedResponse) { - resolvedIndices.add(indices[current]) + permittedIndices.add(indices[current]) proceed(current, indices) } @@ -186,13 +183,54 @@ class TransportAddPolicyAction @Inject constructor( private fun proceed(current: Int, indices: List) { if (current < indices.count() - 1) { - validateAndGetPolicy(current + 1, indices) + validateIndexPermissions(current + 1, indices) } else { // sanity check that there are indices - if none then return - if (resolvedIndices.isEmpty()) { + if (permittedIndices.isEmpty()) { actionListener.onResponse(ISMStatusResponse(0, failedIndices)) return } + // Filter out the indices that the user does not have permissions for + indicesToAdd.values.removeIf { !permittedIndices.contains(it) } + removeClosedIndices() + } + } + + private fun removeClosedIndices() { + // Do another cluster state request to fail closed indices + if (request.indexType == DEFAULT_INDEX_TYPE) { + val strictExpandOptions = IndicesOptions.strictExpand() + val clusterStateRequest = ClusterStateRequest() + .clear() + .indices(*indicesToAdd.values.toTypedArray()) + .metadata(true) + .local(false) + .waitForTimeout(TimeValue.timeValueMillis(ADD_POLICY_TIMEOUT_IN_MILLIS)) + .indicesOptions(strictExpandOptions) + client.admin() + .cluster() + .state( + clusterStateRequest, + object : ActionListener { + override fun onResponse(response: ClusterStateResponse) { + CoroutineScope(Dispatchers.IO).launch { + removeClusterStateMetadatas(client, log, indicesToAdd.map { Index(it.value, it.key) }) + } + + val defaultIndexMetadataService = indexMetadataProvider.services[DEFAULT_INDEX_TYPE] as DefaultIndexMetadataService + getUuidsForClosedIndices(response.state, defaultIndexMetadataService).forEach { + failedIndices.add(FailedIndex(indicesToAdd[it] as String, it, "This index is closed")) + indicesToAdd.remove(it) + } + getPolicy() + } + + override fun onFailure(t: Exception) { + actionListener.onFailure(ExceptionsHelper.unwrapCause(t) as Exception) + } + } + ) + } else { getPolicy() } } @@ -233,7 +271,7 @@ class TransportAddPolicyAction @Inject constructor( private fun onUpdateMapping(response: AcknowledgedResponse) { if (response.isAcknowledged) { log.info("Successfully created or updated $INDEX_MANAGEMENT_INDEX with newest mappings.") - getClusterState() + getExistingManagedIndices() } else { log.error("Unable to create or update $INDEX_MANAGEMENT_INDEX with newest mapping.") @@ -246,55 +284,15 @@ class TransportAddPolicyAction @Inject constructor( } } - @Suppress("SpreadOperator") - fun getClusterState() { - val strictExpandOptions = IndicesOptions.strictExpand() - - val clusterStateRequest = ClusterStateRequest() - .clear() - .indices(*resolvedIndices.toTypedArray()) - .metadata(true) - .local(false) - .waitForTimeout(TimeValue.timeValueMillis(ADD_POLICY_TIMEOUT_IN_MILLIS)) - .indicesOptions(strictExpandOptions) - - startTime = Instant.now() - - client.admin() - .cluster() - .state( - clusterStateRequest, - object : ActionListener { - override fun onResponse(response: ClusterStateResponse) { - response.state.metadata.indices.forEach { - indicesToAdd.putIfAbsent(it.value.indexUUID, it.key) - } - - populateLists(response.state) - } - - override fun onFailure(t: Exception) { - actionListener.onFailure(ExceptionsHelper.unwrapCause(t) as Exception) - } - } - ) - } - - private fun populateLists(state: ClusterState) { - getUuidsForClosedIndices(state).forEach { - failedIndices.add(FailedIndex(indicesToAdd[it] as String, it, "This index is closed")) - indicesToAdd.remove(it) - } - + private fun getExistingManagedIndices() { // Removing all the unmanageable Indices indicesToAdd.entries.removeIf { (uuid, indexName) -> - val shouldRemove = indexEvaluator.isUnManageableIndex(indexName) + val shouldRemove = indexMetadataProvider.isUnManageableIndex(indexName) if (shouldRemove) { failedIndices.add(FailedIndex(indexName, uuid, EVALUATION_FAILURE_MESSAGE)) } shouldRemove } - if (indicesToAdd.isEmpty()) { actionListener.onResponse(ISMStatusResponse(0, failedIndices)) return diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/ChangePolicyRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/ChangePolicyRequest.kt index dfc63b936..fcec1bf74 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/ChangePolicyRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/ChangePolicyRequest.kt @@ -11,23 +11,31 @@ import org.opensearch.action.ValidateActions import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.io.stream.StreamOutput import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import java.io.IOException class ChangePolicyRequest( val indices: List, - val changePolicy: ChangePolicy + val changePolicy: ChangePolicy, + val indexType: String ) : ActionRequest() { @Throws(IOException::class) constructor(sin: StreamInput) : this( indices = sin.readStringList(), - changePolicy = ChangePolicy(sin) + changePolicy = ChangePolicy(sin), + indexType = sin.readString() ) override fun validate(): ActionRequestValidationException? { var validationException: ActionRequestValidationException? = null if (indices.isEmpty()) { validationException = ValidateActions.addValidationError("Missing indices", validationException) + } else if (indexType != DEFAULT_INDEX_TYPE && indices.size > 1) { + validationException = ValidateActions.addValidationError( + MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR, + validationException + ) } return validationException } @@ -36,5 +44,11 @@ class ChangePolicyRequest( override fun writeTo(out: StreamOutput) { out.writeStringCollection(indices) changePolicy.writeTo(out) + out.writeString(indexType) + } + + companion object { + const val MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR = + "Cannot change policy on more than one index name/pattern when using a custom index type" } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/TransportChangePolicyAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/TransportChangePolicyAction.kt index f087c935d..fed9cf828 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/TransportChangePolicyAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/TransportChangePolicyAction.kt @@ -5,6 +5,9 @@ package org.opensearch.indexmanagement.indexstatemanagement.transport.action.changepolicy +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.OpenSearchSecurityException @@ -23,7 +26,7 @@ import org.opensearch.action.support.HandledTransportAction import org.opensearch.action.support.IndicesOptions import org.opensearch.action.support.master.AcknowledgedResponse import org.opensearch.client.node.NodeClient -import org.opensearch.cluster.ClusterState +import org.opensearch.cluster.metadata.IndexMetadata import org.opensearch.cluster.service.ClusterService import org.opensearch.common.inject.Inject import org.opensearch.common.settings.Settings @@ -32,24 +35,29 @@ import org.opensearch.commons.ConfigConstants import org.opensearch.commons.authuser.User import org.opensearch.index.Index import org.opensearch.indexmanagement.IndexManagementPlugin +import org.opensearch.indexmanagement.indexstatemanagement.DefaultIndexMetadataService +import org.opensearch.indexmanagement.indexstatemanagement.IndexMetadataProvider import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.coordinator.SweptManagedIndexConfig import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.buildMgetMetadataRequest import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.getManagedIndexMetadata -import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.mgetResponseToList +import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.mgetResponseToMap import org.opensearch.indexmanagement.indexstatemanagement.resthandler.RestChangePolicyAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.ISMStatusResponse import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexRequest +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.indexmanagement.indexstatemanagement.util.FailedIndex import org.opensearch.indexmanagement.indexstatemanagement.util.isSafeToChange +import org.opensearch.indexmanagement.indexstatemanagement.util.managedIndexMetadataID import org.opensearch.indexmanagement.indexstatemanagement.util.updateManagedIndexRequest import org.opensearch.indexmanagement.opensearchapi.contentParser import org.opensearch.indexmanagement.opensearchapi.parseFromGetResponse import org.opensearch.indexmanagement.opensearchapi.parseWithType import org.opensearch.indexmanagement.settings.IndexManagementSettings +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ISMIndexMetadata +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.util.IndexManagementException import org.opensearch.indexmanagement.util.IndexUtils import org.opensearch.indexmanagement.util.NO_ID @@ -64,14 +72,15 @@ import java.lang.IllegalArgumentException private val log = LogManager.getLogger(TransportChangePolicyAction::class.java) -@Suppress("SpreadOperator", "TooManyFunctions") +@Suppress("SpreadOperator", "TooManyFunctions", "LongParameterList") class TransportChangePolicyAction @Inject constructor( val client: NodeClient, transportService: TransportService, actionFilters: ActionFilters, val clusterService: ClusterService, val settings: Settings, - val xContentRegistry: NamedXContentRegistry + val xContentRegistry: NamedXContentRegistry, + val indexMetadataProvider: IndexMetadataProvider ) : HandledTransportAction( ChangePolicyAction.NAME, transportService, actionFilters, ::ChangePolicyRequest ) { @@ -98,9 +107,10 @@ class TransportChangePolicyAction @Inject constructor( private val failedIndices = mutableListOf() private val managedIndicesToUpdate = mutableListOf>() private val indexUuidToCurrentState = mutableMapOf() + private val indicesToUpdate = mutableMapOf() // uuid -> name + private val indexUuidToIndexMetadata = mutableMapOf() // uuid -> indexmetadata private val changePolicy = request.changePolicy private lateinit var policy: Policy - private lateinit var clusterState: ClusterState private var updated: Int = 0 fun start() { @@ -154,6 +164,7 @@ class TransportChangePolicyAction @Inject constructor( } } + @Suppress("ReturnCount") private fun onGetPolicyResponse(response: GetResponse) { if (!response.isExists || response.isSourceEmpty) { actionListener.onFailure(OpenSearchStatusException("Could not find policy=${request.changePolicy.policyID}", RestStatus.NOT_FOUND)) @@ -187,50 +198,92 @@ class TransportChangePolicyAction @Inject constructor( return } - getClusterState() + getIndicesToUpdate() + } + + private fun getIndicesToUpdate() { + CoroutineScope(Dispatchers.IO).launch { + val indexNameToMetadata: MutableMap = HashMap() + try { + indexNameToMetadata.putAll(indexMetadataProvider.getISMIndexMetadataByType(request.indexType, request.indices)) + } catch (e: Exception) { + actionListener.onFailure(ExceptionsHelper.unwrapCause(e) as Exception) + return@launch + } + indexNameToMetadata.forEach { (indexName, indexMetadata) -> + indicesToUpdate.putIfAbsent(indexMetadata.indexUuid, indexName) + } + if (request.indexType == DEFAULT_INDEX_TYPE) { + getClusterState() + } else { + getManagedIndexMetadata() + } + } } @Suppress("SpreadOperator") private fun getClusterState() { + val strictExpandOptions = IndicesOptions.strictExpand() val clusterStateRequest = ClusterStateRequest() .clear() .indices(*request.indices.toTypedArray()) .metadata(true) .local(false) - .indicesOptions(IndicesOptions.strictExpand()) + .indicesOptions(strictExpandOptions) + client.admin() + .cluster() + .state( + clusterStateRequest, + object : ActionListener { + override fun onResponse(response: ClusterStateResponse) { + val clusterState = response.state + val defaultIndexMetadataService = indexMetadataProvider.services[DEFAULT_INDEX_TYPE] as DefaultIndexMetadataService + clusterState.metadata.indices.forEach { + val indexUUID = defaultIndexMetadataService.getCustomIndexUUID(it.value) + indexUuidToIndexMetadata[indexUUID] = it.value + } + // ISMIndexMetadata from the default index metadata service uses lenient expand, we want to use strict expand, filter + // out the indices which are not also in the strict expand response + indicesToUpdate.filter { indexUuidToIndexMetadata.containsKey(it.key) } + getManagedIndexMetadata() + } - client.admin().cluster().state(clusterStateRequest, ActionListener.wrap(::onClusterStateResponse, ::onFailure)) + override fun onFailure(t: Exception) { + actionListener.onFailure(ExceptionsHelper.unwrapCause(t) as Exception) + } + } + ) } - @Suppress("ComplexMethod") - private fun onClusterStateResponse(response: ClusterStateResponse) { - clusterState = response.state - - // get back managed index metadata - client.multiGet(buildMgetMetadataRequest(clusterState), ActionListener.wrap(::onMgetMetadataResponse, ::onFailure)) + private fun getManagedIndexMetadata() { + client.multiGet( + buildMgetMetadataRequest(indicesToUpdate.toList().map { it.first }), + ActionListener.wrap(::onMgetMetadataResponse, ::onFailure) + ) } @Suppress("ComplexMethod") private fun onMgetMetadataResponse(mgetResponse: MultiGetResponse) { - val metadataList = mgetResponseToList(mgetResponse) + val metadataMap = mgetResponseToMap(mgetResponse) val includedStates = changePolicy.include.map { it.state }.toSet() - clusterState.metadata.indices.forEachIndexed { ind, it -> - val indexMetaData = it.value - val clusterStateMetadata = it.value.getManagedIndexMetadata() - val mgetFailure = metadataList[ind]?.second - val managedIndexMetadata: ManagedIndexMetaData? = metadataList[ind]?.first + indicesToUpdate.forEach { (indexUuid, indexName) -> + // indexMetaData and clusterStateMetadata will be null for non-default index types + val indexMetaData = indexUuidToIndexMetadata[indexUuid] + val clusterStateMetadata = indexMetaData?.getManagedIndexMetadata() + val mgetFailure = metadataMap[indexUuid]?.second + val managedIndexMetadata: ManagedIndexMetaData? = metadataMap[managedIndexMetadataID(indexUuid)]?.first val currentState = managedIndexMetadata?.stateMetaData?.name if (currentState != null) { - indexUuidToCurrentState[indexMetaData.indexUUID] = currentState + indexUuidToCurrentState[indexUuid] = currentState } when { mgetFailure != null -> failedIndices.add( FailedIndex( - indexMetaData.index.name, indexMetaData.index.uuid, + indexName, indexUuid, "Failed to get managed index metadata, $mgetFailure" ) ) @@ -239,7 +292,7 @@ class TransportChangePolicyAction @Inject constructor( managedIndexMetadata?.transitionTo != null -> failedIndices.add( FailedIndex( - indexMetaData.index.name, indexMetaData.index.uuid, + indexName, indexUuid, RestChangePolicyAction.INDEX_IN_TRANSITION ) ) @@ -248,21 +301,21 @@ class TransportChangePolicyAction @Inject constructor( if (clusterStateMetadata != null) { failedIndices.add( FailedIndex( - indexMetaData.index.name, indexMetaData.index.uuid, + indexName, indexUuid, "Cannot change policy until metadata has finished migrating" ) ) } else { - managedIndicesToUpdate.add(indexMetaData.index.name to indexMetaData.index.uuid) + managedIndicesToUpdate.add(indexName to indexUuid) } } // else if the includedStates is empty (i.e. not being used) then we will always try to update the managed index - includedStates.isEmpty() -> managedIndicesToUpdate.add(indexMetaData.index.name to indexMetaData.index.uuid) + includedStates.isEmpty() -> managedIndicesToUpdate.add(indexName to indexUuid) // else only update the managed index if its currently in one of the included states includedStates.contains(managedIndexMetadata.stateMetaData?.name) -> - managedIndicesToUpdate.add(indexMetaData.index.name to indexMetaData.index.uuid) + managedIndicesToUpdate.add(indexName to indexUuid) // else the managed index did not match any of the included state filters and we will not update it - else -> log.debug("Skipping ${indexMetaData.index.name} as it does not match any of the include state filters") + else -> log.debug("Skipping $indexName as it does not match any of the include state filters") } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainAllResponse.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainAllResponse.kt deleted file mode 100644 index 9d6a92396..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainAllResponse.kt +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.transport.action.explain - -import org.opensearch.common.io.stream.StreamInput -import org.opensearch.common.io.stream.StreamOutput -import org.opensearch.common.xcontent.ToXContent -import org.opensearch.common.xcontent.ToXContentObject -import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.settings.LegacyOpenDistroManagedIndexSettings -import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings -import java.io.IOException - -class ExplainAllResponse : ExplainResponse, ToXContentObject { - - val totalManagedIndices: Int - val enabledState: Map - - constructor( - indexNames: List, - indexPolicyIDs: List, - indexMetadatas: List, - totalManagedIndices: Int, - enabledState: Map - ) : super(indexNames, indexPolicyIDs, indexMetadatas) { - this.totalManagedIndices = totalManagedIndices - this.enabledState = enabledState - } - - @Throws(IOException::class) - constructor(sin: StreamInput) : this( - indexNames = sin.readStringList(), - indexPolicyIDs = sin.readStringList(), - indexMetadatas = sin.readList { ManagedIndexMetaData.fromStreamInput(it) }, - totalManagedIndices = sin.readInt(), - enabledState = sin.readMap() as Map - ) - - @Throws(IOException::class) - override fun writeTo(out: StreamOutput) { - super.writeTo(out) - out.writeInt(totalManagedIndices) - out.writeMap(enabledState) - } - - override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { - builder.startObject() - indexNames.forEachIndexed { ind, name -> - builder.startObject(name) - builder.field(LegacyOpenDistroManagedIndexSettings.POLICY_ID.key, indexPolicyIDs[ind]) - builder.field(ManagedIndexSettings.POLICY_ID.key, indexPolicyIDs[ind]) - indexMetadatas[ind]?.toXContent(builder, ToXContent.EMPTY_PARAMS) - builder.field("enabled", enabledState[name]) - builder.endObject() - } - builder.field("total_managed_indices", totalManagedIndices) - return builder.endObject() - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt index 2c0e797b0..8292757c3 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt @@ -7,10 +7,12 @@ package org.opensearch.indexmanagement.indexstatemanagement.transport.action.exp import org.opensearch.action.ActionRequest import org.opensearch.action.ActionRequestValidationException +import org.opensearch.action.ValidateActions import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.io.stream.StreamOutput import org.opensearch.common.unit.TimeValue import org.opensearch.indexmanagement.indexstatemanagement.model.SearchParams +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import java.io.IOException class ExplainRequest : ActionRequest { @@ -19,17 +21,23 @@ class ExplainRequest : ActionRequest { val local: Boolean val masterTimeout: TimeValue val searchParams: SearchParams + val showPolicy: Boolean + val indexType: String constructor( indices: List, local: Boolean, masterTimeout: TimeValue, - searchParams: SearchParams + searchParams: SearchParams, + showPolicy: Boolean, + indexType: String ) : super() { this.indices = indices this.local = local this.masterTimeout = masterTimeout this.searchParams = searchParams + this.showPolicy = showPolicy + this.indexType = indexType } @Throws(IOException::class) @@ -37,11 +45,20 @@ class ExplainRequest : ActionRequest { indices = sin.readStringList(), local = sin.readBoolean(), masterTimeout = sin.readTimeValue(), - searchParams = SearchParams(sin) + searchParams = SearchParams(sin), + showPolicy = sin.readBoolean(), + indexType = sin.readString() ) override fun validate(): ActionRequestValidationException? { - return null + var validationException: ActionRequestValidationException? = null + if (indexType != DEFAULT_INDEX_TYPE && indices.size > 1) { + validationException = ValidateActions.addValidationError( + MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR, + validationException + ) + } + return validationException } @Throws(IOException::class) @@ -50,5 +67,12 @@ class ExplainRequest : ActionRequest { out.writeBoolean(local) out.writeTimeValue(masterTimeout) searchParams.writeTo(out) + out.writeBoolean(showPolicy) + out.writeString(indexType) + } + + companion object { + const val MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR = + "Cannot call explain on more than one index name/pattern when using a custom index type" } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainResponse.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainResponse.kt index c80bb2c41..cda65f548 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainResponse.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainResponse.kt @@ -11,9 +11,12 @@ import org.opensearch.common.io.stream.StreamOutput import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.ToXContentObject import org.opensearch.common.xcontent.XContentBuilder -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.settings.LegacyOpenDistroManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings +import org.opensearch.indexmanagement.indexstatemanagement.util.TOTAL_MANAGED_INDICES +import org.opensearch.indexmanagement.indexstatemanagement.util.XCONTENT_WITHOUT_TYPE_AND_USER +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import java.io.IOException open class ExplainResponse : ActionResponse, ToXContentObject { @@ -22,22 +25,34 @@ open class ExplainResponse : ActionResponse, ToXContentObject { val indexNames: List val indexPolicyIDs: List val indexMetadatas: List + val totalManagedIndices: Int + val enabledState: Map + val policies: Map constructor( indexNames: List, indexPolicyIDs: List, - indexMetadatas: List + indexMetadatas: List, + totalManagedIndices: Int, + enabledState: Map, + policies: Map ) : super() { this.indexNames = indexNames this.indexPolicyIDs = indexPolicyIDs this.indexMetadatas = indexMetadatas + this.totalManagedIndices = totalManagedIndices + this.enabledState = enabledState + this.policies = policies } @Throws(IOException::class) constructor(sin: StreamInput) : this( indexNames = sin.readStringList(), indexPolicyIDs = sin.readStringList(), - indexMetadatas = sin.readList { ManagedIndexMetaData.fromStreamInput(it) } + indexMetadatas = sin.readList { ManagedIndexMetaData.fromStreamInput(it) }, + totalManagedIndices = sin.readInt(), + enabledState = sin.readMap() as Map, + policies = sin.readMap(StreamInput::readString, ::Policy) ) @Throws(IOException::class) @@ -45,17 +60,27 @@ open class ExplainResponse : ActionResponse, ToXContentObject { out.writeStringCollection(indexNames) out.writeStringCollection(indexPolicyIDs) out.writeCollection(indexMetadatas) + out.writeInt(totalManagedIndices) + out.writeMap(enabledState) + out.writeMap( + policies, + { _out, key -> _out.writeString(key) }, + { _out, policy -> policy.writeTo(_out) } + ) } override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { builder.startObject() indexNames.forEachIndexed { ind, name -> builder.startObject(name) - builder.field(LegacyOpenDistroManagedIndexSettings.POLICY_ID.key, indexPolicyIDs[ind]) builder.field(ManagedIndexSettings.POLICY_ID.key, indexPolicyIDs[ind]) + builder.field(LegacyOpenDistroManagedIndexSettings.POLICY_ID.key, indexPolicyIDs[ind]) indexMetadatas[ind]?.toXContent(builder, ToXContent.EMPTY_PARAMS) + builder.field("enabled", enabledState[name]) + policies[name]?.let { builder.field(Policy.POLICY_TYPE, it, XCONTENT_WITHOUT_TYPE_AND_USER) } builder.endObject() } + builder.field(TOTAL_MANAGED_INDICES, totalManagedIndices) return builder.endObject() } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt index 17ba9d595..8219c3799 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt @@ -5,6 +5,9 @@ package org.opensearch.indexmanagement.indexstatemanagement.transport.action.explain +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.OpenSearchSecurityException @@ -18,7 +21,6 @@ import org.opensearch.action.search.SearchRequest import org.opensearch.action.search.SearchResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction -import org.opensearch.action.support.IndicesOptions import org.opensearch.action.support.master.AcknowledgedResponse import org.opensearch.client.node.NodeClient import org.opensearch.cluster.metadata.IndexMetadata @@ -35,14 +37,25 @@ import org.opensearch.index.IndexNotFoundException import org.opensearch.index.query.Operator import org.opensearch.index.query.QueryBuilders import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX +import org.opensearch.indexmanagement.indexstatemanagement.IndexMetadataProvider import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexCoordinator.Companion.MAX_HITS -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig +import org.opensearch.indexmanagement.indexstatemanagement.model.Policy +import org.opensearch.indexmanagement.indexstatemanagement.model.SearchParams import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.getManagedIndexMetadata import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexRequest -import org.opensearch.indexmanagement.indexstatemanagement.util.isMetadataMoved +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE +import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_INDEX_UUID_FIELD +import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_NAME_KEYWORD_FIELD +import org.opensearch.indexmanagement.indexstatemanagement.util.MetadataCheck +import org.opensearch.indexmanagement.indexstatemanagement.util.checkMetadata import org.opensearch.indexmanagement.indexstatemanagement.util.managedIndexMetadataID +import org.opensearch.indexmanagement.opensearchapi.parseWithType +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ISMIndexMetadata +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.util.SecurityUtils.Companion.buildUser +import org.opensearch.search.SearchHit import org.opensearch.search.builder.SearchSourceBuilder import org.opensearch.search.fetch.subphase.FetchSourceContext.FETCH_SOURCE import org.opensearch.search.sort.SortBuilders @@ -52,13 +65,22 @@ import org.opensearch.transport.TransportService private val log = LogManager.getLogger(TransportExplainAction::class.java) -@Suppress("SpreadOperator") +// TODO: Move these to higher level and refactor plugin to make it more readable +typealias IndexUUID = String +typealias PolicyID = String +typealias IndexName = String +typealias ManagedIndexConfigDocUUID = String +typealias ManagedIndexMetadataDocUUID = String // managedIndexMetadataID(indexUuid) -> #metadata +typealias ManagedIndexMetadataMap = Map + +@Suppress("SpreadOperator", "TooManyFunctions") class TransportExplainAction @Inject constructor( val client: NodeClient, transportService: TransportService, actionFilters: ActionFilters, val clusterService: ClusterService, - val xContentRegistry: NamedXContentRegistry + val xContentRegistry: NamedXContentRegistry, + val indexMetadataProvider: IndexMetadataProvider ) : HandledTransportAction( ExplainAction.NAME, transportService, actionFilters, ::ExplainRequest ) { @@ -81,17 +103,21 @@ class TransportExplainAction @Inject constructor( ) { private val indices: List = request.indices private val explainAll: Boolean = indices.isEmpty() - private val wildcard: Boolean = indices.any { it.contains("*") } - - // map of index to index metadata got from config index job - private val managedIndicesMetaDataMap: MutableMap> = mutableMapOf() - private val managedIndices: MutableList = mutableListOf() - - private val indexNames: MutableList = mutableListOf() - private val enabledState: MutableMap = mutableMapOf() - private val indexPolicyIDs = mutableListOf() + private val showPolicy: Boolean = request.showPolicy + + // Map of indexName to index metadata got from config index job which is fake/not a real full metadata document + private val managedIndicesMetaDataMap: MutableMap = mutableMapOf() + private val managedIndices: MutableList = mutableListOf() + + // indexNames are the ones that will be iterated on and shown in the final response + // throughout request they are cleared and rewritten + private val indexNames: MutableList = mutableListOf() + private val indexNamesToUUIDs: MutableMap = mutableMapOf() + private val enabledState: MutableMap = mutableMapOf() + private val indexPolicyIDs = mutableListOf() private val indexMetadatas = mutableListOf() private var totalManagedIndices = 0 + private val appliedPolicies: MutableMap = mutableMapOf() @Suppress("SpreadOperator", "NestedBlockDepth") fun start() { @@ -100,8 +126,30 @@ class TransportExplainAction @Inject constructor( ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT )}" ) - val params = request.searchParams + // Use the indexMetadataProvider to get the index names and uuids corresponding to this index type + CoroutineScope(Dispatchers.IO).launch { + val indexNameToMetadata: MutableMap = HashMap() + try { + if (explainAll) { + indexNameToMetadata.putAll(indexMetadataProvider.getAllISMIndexMetadataByType(request.indexType)) + } else { + indexNameToMetadata.putAll(indexMetadataProvider.getISMIndexMetadataByType(request.indexType, indices)) + } + } catch (e: Exception) { + actionListener.onFailure(ExceptionsHelper.unwrapCause(e) as Exception) + return@launch + } + // These index names are resolved and populated by the indexMetadataProvider specific to the index type + indexNames.addAll(indexNameToMetadata.keys) + indexNamesToUUIDs.putAll(indexNameToMetadata.mapValues { it.value.indexUuid }) + + val params = request.searchParams + val searchRequest = getSearchMetadataRequest(params, indexNamesToUUIDs.values.toList(), if (explainAll) params.size else MAX_HITS) + searchForMetadata(searchRequest) + } + } + private fun getSearchMetadataRequest(params: SearchParams, indexUUIDs: List, searchSize: Int): SearchRequest { val sortBuilder = SortBuilders .fieldSort(params.sortField) .order(SortOrder.fromString(params.sortOrder)) @@ -110,41 +158,25 @@ class TransportExplainAction @Inject constructor( .must( QueryBuilders .queryStringQuery(params.queryString) - .defaultField("managed_index.name.keyword") + .defaultField(MANAGED_INDEX_NAME_KEYWORD_FIELD) .defaultOperator(Operator.AND) - ) + ).filter(QueryBuilders.termsQuery(MANAGED_INDEX_INDEX_UUID_FIELD, indexUUIDs)) - var searchSourceBuilder = SearchSourceBuilder() + val searchSourceBuilder = SearchSourceBuilder() .from(params.from) .fetchSource(FETCH_SOURCE) .seqNoAndPrimaryTerm(true) .version(true) .sort(sortBuilder) + .size(searchSize) + .query(queryBuilder) - if (!explainAll) { - searchSourceBuilder = searchSourceBuilder.size(MAX_HITS) - if (wildcard) { // explain/index* - indices.forEach { - if (it.contains("*")) { - queryBuilder.should(QueryBuilders.wildcardQuery("managed_index.index", it)) - } else { - queryBuilder.should(QueryBuilders.termsQuery("managed_index.index", it)) - } - } - } else { // explain/{index} - queryBuilder.filter(QueryBuilders.termsQuery("managed_index.index", indices)) - } - } else { // explain all - searchSourceBuilder = searchSourceBuilder.size(params.size) - queryBuilder.filter(QueryBuilders.existsQuery("managed_index")) - } - - searchSourceBuilder = searchSourceBuilder.query(queryBuilder) - - val searchRequest = SearchRequest() + return SearchRequest() .indices(INDEX_MANAGEMENT_INDEX) .source(searchSourceBuilder) + } + private fun searchForMetadata(searchRequest: SearchRequest) { client.threadPool().threadContext.stashContext().use { threadContext -> client.search( searchRequest, @@ -155,17 +187,18 @@ class TransportExplainAction @Inject constructor( totalManagedIndices = totalHits.value.toInt() } - response.hits.hits.map { - val hitMap = it.sourceAsMap["managed_index"] as Map - val managedIndex = hitMap["index"] as String - managedIndices.add(managedIndex) - enabledState[managedIndex] = hitMap["enabled"] as Boolean - managedIndicesMetaDataMap[managedIndex] = mapOf( - "index" to hitMap["index"] as String?, - "index_uuid" to hitMap["index_uuid"] as String?, - "policy_id" to hitMap["policy_id"] as String?, - "enabled" to hitMap["enabled"]?.toString() + parseSearchHits(response.hits.hits).forEach { managedIndex -> + managedIndices.add(managedIndex.index) + enabledState[managedIndex.index] = managedIndex.enabled + managedIndicesMetaDataMap[managedIndex.index] = mapOf( + "index" to managedIndex.index, + "index_uuid" to managedIndex.indexUuid, + "policy_id" to managedIndex.policyID, + "enabled" to managedIndex.enabled.toString() ) + if (showPolicy) { + managedIndex.policy?.let { appliedPolicies[managedIndex.index] = it } + } } // explain all only return managed indices @@ -173,30 +206,34 @@ class TransportExplainAction @Inject constructor( if (managedIndices.size == 0) { // edge case: if specify query param pagination size to be 0 // we still show total managed indices - sendResponse() + indexNames.clear() + sendResponse(indexNames, indexMetadatas, indexPolicyIDs, enabledState, totalManagedIndices, appliedPolicies) return } else { + // Clear and add the managedIndices from the response to preserve the sort order and size + indexNames.clear() indexNames.addAll(managedIndices) - getMetadata(managedIndices, threadContext) + // Remove entries in case they were limited due to request size + indexNamesToUUIDs.filterKeys { indexNames.contains(it) } + getMetadata(indexNames, threadContext) return } } // explain/{index} return results for all indices - indexNames.addAll(indices) - getMetadata(indices, threadContext) + getMetadata(indexNames, threadContext) } override fun onFailure(t: Exception) { if (t is IndexNotFoundException) { // config index hasn't been initialized // show all requested indices not managed - if (indices.isNotEmpty()) { - indexNames.addAll(indices) - getMetadata(indices, threadContext) + if (!explainAll) { + getMetadata(indexNames, threadContext) return } - sendResponse() + indexNames.clear() + sendResponse(indexNames, indexMetadatas, indexPolicyIDs, enabledState, totalManagedIndices, appliedPolicies) return } actionListener.onFailure(ExceptionsHelper.unwrapCause(t) as Exception) @@ -207,50 +244,45 @@ class TransportExplainAction @Inject constructor( } @Suppress("SpreadOperator") - fun getMetadata(indices: List, threadContext: ThreadContext.StoredContext) { - val clusterStateRequest = ClusterStateRequest() - val strictExpandIndicesOptions = IndicesOptions.strictExpand() - - clusterStateRequest.clear() - .indices(*indices.toTypedArray()) - .metadata(true) - .local(request.local) - .masterNodeTimeout(request.masterTimeout) - .indicesOptions(strictExpandIndicesOptions) - - client.admin().cluster().state( - clusterStateRequest, - object : ActionListener { - override fun onResponse(response: ClusterStateResponse) { - onClusterStateResponse(response, threadContext) - } + fun getMetadata(indexNames: List, threadContext: ThreadContext.StoredContext) { + if (request.indexType == DEFAULT_INDEX_TYPE) { + val clusterStateRequest = ClusterStateRequest() + clusterStateRequest.clear() + .indices(*indexNames.toTypedArray()) + .metadata(true) + .local(request.local) + .masterNodeTimeout(request.masterTimeout) + + client.admin().cluster().state( + clusterStateRequest, + object : ActionListener { + override fun onResponse(response: ClusterStateResponse) { + val clusterStateIndexMetadatas = response.state.metadata.indices.associate { it.key to it.value } + getMetadataMap(clusterStateIndexMetadatas, threadContext) + } - override fun onFailure(t: Exception) { - actionListener.onFailure(ExceptionsHelper.unwrapCause(t) as Exception) + override fun onFailure(t: Exception) { + actionListener.onFailure(ExceptionsHelper.unwrapCause(t) as Exception) + } } - } - ) - } - - fun onClusterStateResponse(clusterStateResponse: ClusterStateResponse, threadContext: ThreadContext.StoredContext) { - val clusterStateIndexMetadatas = clusterStateResponse.state.metadata.indices.map { it.key to it.value }.toMap() - - if (wildcard) { - indexNames.clear() // clear wildcard (index*) from indexNames - clusterStateIndexMetadatas.forEach { indexNames.add(it.key) } + ) + } else { + getMetadataMap(null, threadContext) } + } - val indices = clusterStateIndexMetadatas.map { it.key to it.value.indexUUID }.toMap() + private fun getMetadataMap(clusterStateIndexMetadatas: Map?, threadContext: ThreadContext.StoredContext) { val mgetMetadataReq = MultiGetRequest() - indices.map { it.value }.forEach { uuid -> + indexNamesToUUIDs.values.forEach { uuid -> mgetMetadataReq.add(MultiGetRequest.Item(INDEX_MANAGEMENT_INDEX, managedIndexMetadataID(uuid)).routing(uuid)) } client.multiGet( mgetMetadataReq, object : ActionListener { override fun onResponse(response: MultiGetResponse) { - val metadataMap = response.responses.map { it.id to getMetadata(it.response)?.toMap() }.toMap() - buildResponse(indices, metadataMap, clusterStateIndexMetadatas, threadContext) + val metadataMap: Map = + response.responses.associate { it.id to getMetadata(it.response)?.toMap() } + buildResponse(indexNamesToUUIDs, metadataMap, clusterStateIndexMetadatas, threadContext) } override fun onFailure(t: Exception) { @@ -260,34 +292,37 @@ class TransportExplainAction @Inject constructor( ) } - // metadataMap: doc id -> metadataMap, doc id for metadata is [managedIndexMetadataID(indexUuid)] - fun buildResponse( - indices: Map, - metadataMap: Map?>, - clusterStateIndexMetadatas: Map, + @Suppress("ComplexMethod", "NestedBlockDepth") + private fun buildResponse( + indices: Map, + metadataMap: Map, + clusterStateIndexMetadatas: Map?, threadContext: ThreadContext.StoredContext ) { - - // cluster state response will not resisting the sort order + // cluster state response will not resist the sort order // so use the order from previous search result saved in indexNames for (indexName in indexNames) { - var managedIndexMetadataMap = managedIndicesMetaDataMap[indexName] - indexPolicyIDs.add(managedIndexMetadataMap?.get("policy_id")) // use policyID from metadata + var metadataMapFromManagedIndex = managedIndicesMetaDataMap[indexName] + indexPolicyIDs.add(metadataMapFromManagedIndex?.get("policy_id")) - val clusterStateMetadata = clusterStateIndexMetadatas[indexName]?.getManagedIndexMetadata() var managedIndexMetadata: ManagedIndexMetaData? = null - val configIndexMetadataMap = metadataMap[indices[indexName]?.let { managedIndexMetadataID(it) }] - if (managedIndexMetadataMap != null) { - if (configIndexMetadataMap != null) { // if has metadata saved, use that - managedIndexMetadataMap = configIndexMetadataMap + val managedIndexMetadataDocUUID = indices[indexName]?.let { managedIndexMetadataID(it) } + val configIndexMetadataMap = metadataMap[managedIndexMetadataDocUUID] + if (metadataMapFromManagedIndex != null) { + if (configIndexMetadataMap != null) { + metadataMapFromManagedIndex = configIndexMetadataMap } - if (managedIndexMetadataMap.isNotEmpty()) { - managedIndexMetadata = ManagedIndexMetaData.fromMap(managedIndexMetadataMap) + if (metadataMapFromManagedIndex.isNotEmpty()) { + managedIndexMetadata = ManagedIndexMetaData.fromMap(metadataMapFromManagedIndex) } - if (!isMetadataMoved(clusterStateMetadata, configIndexMetadataMap, log)) { - val info = mapOf("message" to "Metadata is pending migration") - managedIndexMetadata = clusterStateMetadata?.copy(info = info) + // clusterStateIndexMetadatas will not be null only for the default index type + if (clusterStateIndexMetadatas != null) { + val currentIndexUuid = indices[indexName] + val clusterStateMetadata = clusterStateIndexMetadatas[indexName]?.getManagedIndexMetadata() + val metadataCheck = checkMetadata(clusterStateMetadata, configIndexMetadataMap, currentIndexUuid, log) + val info = metadataStatusToInfo[metadataCheck] + info?.let { managedIndexMetadata = clusterStateMetadata?.copy(info = it) } } } indexMetadatas.add(managedIndexMetadata) @@ -295,7 +330,7 @@ class TransportExplainAction @Inject constructor( managedIndicesMetaDataMap.clear() if (user == null || indexNames.isEmpty()) { - sendResponse() + sendResponse(indexNames, indexMetadatas, indexPolicyIDs, enabledState, totalManagedIndices, appliedPolicies) } else { filterAndSendResponse(threadContext) } @@ -305,17 +340,20 @@ class TransportExplainAction @Inject constructor( threadContext.restore() val filteredIndices = mutableListOf() val filteredMetadata = mutableListOf() - val filteredPolicies = mutableListOf() + val filteredPolicies = mutableListOf() val enabledStatus = mutableMapOf() - filter(0, filteredIndices, filteredMetadata, filteredPolicies, enabledStatus) + val filteredAppliedPolicies = mutableMapOf() + filter(0, filteredIndices, filteredMetadata, filteredPolicies, enabledStatus, filteredAppliedPolicies) } + @Suppress("LongParameterList") private fun filter( current: Int, filteredIndices: MutableList, filteredMetadata: MutableList, - filteredPolicies: MutableList, - enabledStatus: MutableMap + filteredPolicies: MutableList, + enabledStatus: MutableMap, + filteredAppliedPolicies: MutableMap ) { val request = ManagedIndexRequest().indices(indexNames[current]) client.execute( @@ -326,12 +364,16 @@ class TransportExplainAction @Inject constructor( filteredIndices.add(indexNames[current]) filteredMetadata.add(indexMetadatas[current]) filteredPolicies.add(indexPolicyIDs[current]) - enabledStatus[indexNames[current]] = enabledState.getOrDefault(indexNames[current], false) + enabledState[indexNames[current]]?.let { enabledStatus[indexNames[current]] = it } + appliedPolicies[indexNames[current]]?.let { filteredAppliedPolicies[indexNames[current]] = it } if (current < indexNames.count() - 1) { // do nothing - skip the index and go to next one - filter(current + 1, filteredIndices, filteredMetadata, filteredPolicies, enabledStatus) + filter(current + 1, filteredIndices, filteredMetadata, filteredPolicies, enabledStatus, filteredAppliedPolicies) } else { - sendResponse(filteredIndices, filteredMetadata, filteredPolicies, enabledStatus) + sendResponse( + filteredIndices, filteredMetadata, filteredPolicies, enabledStatus, + totalManagedIndices, filteredAppliedPolicies + ) } } @@ -341,9 +383,19 @@ class TransportExplainAction @Inject constructor( totalManagedIndices -= 1 if (current < indexNames.count() - 1) { // do nothing - skip the index and go to next one - filter(current + 1, filteredIndices, filteredMetadata, filteredPolicies, enabledStatus) + filter( + current + 1, + filteredIndices, + filteredMetadata, + filteredPolicies, + enabledStatus, + filteredAppliedPolicies + ) } else { - sendResponse(filteredIndices, filteredMetadata, filteredPolicies, enabledStatus) + sendResponse( + filteredIndices, filteredMetadata, filteredPolicies, enabledStatus, + totalManagedIndices, filteredAppliedPolicies + ) } } false -> { @@ -355,34 +407,56 @@ class TransportExplainAction @Inject constructor( ) } + @Suppress("LongParameterList") private fun sendResponse( - indices: List = indexNames, - metadata: List = indexMetadatas, - policies: List = indexPolicyIDs, - enabledStatus: Map = enabledState, - totalIndices: Int = totalManagedIndices + indices: List, + metadata: List, + policyIDs: List, + enabledStatus: Map, + totalIndices: Int, + policies: Map ) { - if (explainAll) { - actionListener.onResponse(ExplainAllResponse(indices, policies, metadata, totalIndices, enabledStatus)) - return - } - actionListener.onResponse(ExplainResponse(indices, policies, metadata)) + actionListener.onResponse(ExplainResponse(indices, policyIDs, metadata, totalIndices, enabledStatus, policies)) } + @Suppress("ReturnCount") private fun getMetadata(response: GetResponse?): ManagedIndexMetaData? { if (response == null || response.sourceAsBytesRef == null) return null - val xcp = XContentHelper.createParser( - xContentRegistry, - LoggingDeprecationHandler.INSTANCE, - response.sourceAsBytesRef, - XContentType.JSON - ) - return ManagedIndexMetaData.parseWithType( - xcp, - response.id, response.seqNo, response.primaryTerm - ) + try { + val xcp = XContentHelper.createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + response.sourceAsBytesRef, + XContentType.JSON + ) + return ManagedIndexMetaData.parseWithType(xcp, response.id, response.seqNo, response.primaryTerm) + } catch (e: Exception) { + log.error("Failed to parse the ManagedIndexMetadata for ${response.id}", e) + } + + return null } + + private fun parseSearchHits(hits: Array): List { + return hits.map { hit -> + XContentHelper.createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + hit.sourceRef, + XContentType.JSON + ).parseWithType(parse = ManagedIndexConfig.Companion::parse) + } + } + } + + companion object { + const val METADATA_MOVING_WARNING = "Managed index's metadata is pending migration." + const val METADATA_CORRUPT_WARNING = "Managed index's metadata is corrupt, please use remove policy API to clean it." + val metadataStatusToInfo = mapOf( + MetadataCheck.PENDING to mapOf("message" to METADATA_MOVING_WARNING), + MetadataCheck.CORRUPT to mapOf("message" to METADATA_CORRUPT_WARNING) + ) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPoliciesResponse.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPoliciesResponse.kt index 300f48356..cf5e038b7 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPoliciesResponse.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPoliciesResponse.kt @@ -12,7 +12,9 @@ import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.ToXContentObject import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.indexmanagement.indexstatemanagement.model.Policy -import org.opensearch.indexmanagement.indexstatemanagement.util.XCONTENT_WITHOUT_TYPE_AND_USER +import org.opensearch.indexmanagement.indexstatemanagement.util.WITH_TYPE +import org.opensearch.indexmanagement.indexstatemanagement.util.WITH_USER +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action import org.opensearch.indexmanagement.util._ID import org.opensearch.indexmanagement.util._PRIMARY_TERM import org.opensearch.indexmanagement.util._SEQ_NO @@ -43,6 +45,7 @@ class GetPoliciesResponse : ActionResponse, ToXContentObject { } override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + val policyParams = ToXContent.MapParams(mapOf(WITH_TYPE to "false", WITH_USER to "false", Action.EXCLUDE_CUSTOM_FIELD_PARAM to "true")) return builder.startObject() .startArray("policies") .apply { @@ -51,7 +54,7 @@ class GetPoliciesResponse : ActionResponse, ToXContentObject { .field(_ID, policy.id) .field(_SEQ_NO, policy.seqNo) .field(_PRIMARY_TERM, policy.primaryTerm) - .field(Policy.POLICY_TYPE, policy, XCONTENT_WITHOUT_TYPE_AND_USER) + .field(Policy.POLICY_TYPE, policy, policyParams) .endObject() } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPolicyResponse.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPolicyResponse.kt index 43eef2d5b..d632c5453 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPolicyResponse.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPolicyResponse.kt @@ -12,7 +12,9 @@ import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.ToXContentObject import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.indexmanagement.indexstatemanagement.model.Policy -import org.opensearch.indexmanagement.indexstatemanagement.util.XCONTENT_WITHOUT_TYPE_AND_USER +import org.opensearch.indexmanagement.indexstatemanagement.util.WITH_TYPE +import org.opensearch.indexmanagement.indexstatemanagement.util.WITH_USER +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action import org.opensearch.indexmanagement.util._ID import org.opensearch.indexmanagement.util._PRIMARY_TERM import org.opensearch.indexmanagement.util._SEQ_NO @@ -65,7 +67,8 @@ class GetPolicyResponse : ActionResponse, ToXContentObject { .field(_SEQ_NO, seqNo) .field(_PRIMARY_TERM, primaryTerm) if (policy != null) { - builder.field(Policy.POLICY_TYPE, policy, XCONTENT_WITHOUT_TYPE_AND_USER) + val policyParams = ToXContent.MapParams(mapOf(WITH_TYPE to "false", WITH_USER to "false", Action.EXCLUDE_CUSTOM_FIELD_PARAM to "true")) + builder.field(Policy.POLICY_TYPE, policy, policyParams) } return builder.endObject() diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/IndexPolicyResponse.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/IndexPolicyResponse.kt index 20546f2dd..226359165 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/IndexPolicyResponse.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/IndexPolicyResponse.kt @@ -12,7 +12,8 @@ import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.ToXContentObject import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.indexmanagement.indexstatemanagement.model.Policy -import org.opensearch.indexmanagement.indexstatemanagement.util.XCONTENT_WITHOUT_USER +import org.opensearch.indexmanagement.indexstatemanagement.util.WITH_USER +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action.Companion.EXCLUDE_CUSTOM_FIELD_PARAM import org.opensearch.indexmanagement.util._ID import org.opensearch.indexmanagement.util._PRIMARY_TERM import org.opensearch.indexmanagement.util._SEQ_NO @@ -66,12 +67,13 @@ class IndexPolicyResponse : ActionResponse, ToXContentObject { } override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + val policyParams = ToXContent.MapParams(mapOf(WITH_USER to "false", EXCLUDE_CUSTOM_FIELD_PARAM to "true")) return builder.startObject() .field(_ID, id) .field(_VERSION, version) .field(_PRIMARY_TERM, primaryTerm) .field(_SEQ_NO, seqNo) - .field(Policy.POLICY_TYPE, policy, XCONTENT_WITHOUT_USER) + .field(Policy.POLICY_TYPE, policy, policyParams) .endObject() } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt index 66f38333d..d16e332c8 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/TransportIndexPolicyAction.kt @@ -236,8 +236,7 @@ class TransportIndexPolicyAction @Inject constructor( private fun checkShardsFailure(response: IndexResponse): String? { val failureReasons = StringBuilder() if (response.shardInfo.failed > 0) { - response.shardInfo.failures.forEach { - entry -> + response.shardInfo.failures.forEach { entry -> failureReasons.append(entry.reason()) } return failureReasons.toString() diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/RemovePolicyRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/RemovePolicyRequest.kt index ef97b290b..38d13a9eb 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/RemovePolicyRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/RemovePolicyRequest.kt @@ -10,27 +10,29 @@ import org.opensearch.action.ActionRequestValidationException import org.opensearch.action.ValidateActions import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import java.io.IOException -class RemovePolicyRequest : ActionRequest { - - val indices: List - - constructor( - indices: List - ) : super() { - this.indices = indices - } +class RemovePolicyRequest( + val indices: List, + val indexType: String +) : ActionRequest() { @Throws(IOException::class) constructor(sin: StreamInput) : this( - indices = sin.readStringList() + indices = sin.readStringList(), + indexType = sin.readString() ) override fun validate(): ActionRequestValidationException? { var validationException: ActionRequestValidationException? = null if (indices.isEmpty()) { validationException = ValidateActions.addValidationError("Missing indices", validationException) + } else if (indexType != DEFAULT_INDEX_TYPE && indices.size > 1) { + validationException = ValidateActions.addValidationError( + MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR, + validationException + ) } return validationException } @@ -38,5 +40,11 @@ class RemovePolicyRequest : ActionRequest { @Throws(IOException::class) override fun writeTo(out: StreamOutput) { out.writeStringCollection(indices) + out.writeString(indexType) + } + + companion object { + const val MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR = + "Cannot remove policy from more than one index name/pattern when using a custom index type" } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/TransportRemovePolicyAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/TransportRemovePolicyAction.kt index 1b079465b..1e14b525a 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/TransportRemovePolicyAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/TransportRemovePolicyAction.kt @@ -5,6 +5,9 @@ package org.opensearch.indexmanagement.indexstatemanagement.transport.action.removepolicy +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.OpenSearchSecurityException @@ -22,13 +25,11 @@ import org.opensearch.action.support.HandledTransportAction import org.opensearch.action.support.IndicesOptions import org.opensearch.action.support.master.AcknowledgedResponse import org.opensearch.client.node.NodeClient -import org.opensearch.cluster.ClusterState import org.opensearch.cluster.block.ClusterBlockException import org.opensearch.cluster.metadata.IndexMetadata.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING import org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING import org.opensearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY import org.opensearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE -import org.opensearch.cluster.service.ClusterService import org.opensearch.common.inject.Inject import org.opensearch.common.settings.Settings import org.opensearch.commons.ConfigConstants @@ -36,14 +37,19 @@ import org.opensearch.commons.authuser.User import org.opensearch.index.Index import org.opensearch.index.IndexNotFoundException import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX +import org.opensearch.indexmanagement.indexstatemanagement.DefaultIndexMetadataService +import org.opensearch.indexmanagement.indexstatemanagement.IndexMetadataProvider import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.getUuidsForClosedIndices import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.transport.action.ISMStatusResponse import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexRequest +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.indexmanagement.indexstatemanagement.util.FailedIndex import org.opensearch.indexmanagement.indexstatemanagement.util.deleteManagedIndexMetadataRequest import org.opensearch.indexmanagement.indexstatemanagement.util.deleteManagedIndexRequest +import org.opensearch.indexmanagement.indexstatemanagement.util.removeClusterStateMetadatas +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ISMIndexMetadata import org.opensearch.indexmanagement.util.IndexManagementException import org.opensearch.indexmanagement.util.SecurityUtils.Companion.buildUser import org.opensearch.rest.RestStatus @@ -55,7 +61,7 @@ class TransportRemovePolicyAction @Inject constructor( val client: NodeClient, transportService: TransportService, actionFilters: ActionFilters, - val clusterService: ClusterService + val indexMetadataProvider: IndexMetadataProvider ) : HandledTransportAction( RemovePolicyAction.NAME, transportService, actionFilters, ::RemovePolicyRequest ) { @@ -75,7 +81,7 @@ class TransportRemovePolicyAction @Inject constructor( private val failedIndices: MutableList = mutableListOf() private val indicesToRemove = mutableMapOf() // uuid: name - private val indicesWithAutoManageBlock = mutableSetOf() + private val indicesWithAutoManageFalseBlock = mutableSetOf() private val indicesWithReadOnlyBlock = mutableSetOf() private val indicesWithReadOnlyAllowDeleteBlock = mutableSetOf() @@ -86,20 +92,20 @@ class TransportRemovePolicyAction @Inject constructor( )}" ) if (user == null) { - getClusterState() + getIndicesToRemove() } else { - validateAndGetClusterState() + validateAndGetIndices() } } - private fun validateAndGetClusterState() { - val request = ManagedIndexRequest().indices(*request.indices.toTypedArray()) + private fun validateAndGetIndices() { + val managedIndexRequest = ManagedIndexRequest().indices(*request.indices.toTypedArray()) client.execute( ManagedIndexAction.INSTANCE, - request, + managedIndexRequest, object : ActionListener { override fun onResponse(response: AcknowledgedResponse) { - getClusterState() + getIndicesToRemove() } override fun onFailure(e: java.lang.Exception) { @@ -119,6 +125,26 @@ class TransportRemovePolicyAction @Inject constructor( ) } + private fun getIndicesToRemove() { + CoroutineScope(Dispatchers.IO).launch { + val indexNameToMetadata: MutableMap = HashMap() + try { + indexNameToMetadata.putAll(indexMetadataProvider.getISMIndexMetadataByType(request.indexType, request.indices)) + } catch (e: Exception) { + actionListener.onFailure(ExceptionsHelper.unwrapCause(e) as Exception) + return@launch + } + indexNameToMetadata.forEach { (indexName, indexMetadata) -> + indicesToRemove.putIfAbsent(indexMetadata.indexUuid, indexName) + } + if (request.indexType == DEFAULT_INDEX_TYPE) { + getClusterState() + } else { + getExistingManagedIndices() + } + } + } + private fun getClusterState() { val strictExpandOptions = IndicesOptions.strictExpand() @@ -138,9 +164,8 @@ class TransportRemovePolicyAction @Inject constructor( override fun onResponse(response: ClusterStateResponse) { val indexMetadatas = response.state.metadata.indices indexMetadatas.forEach { - indicesToRemove.putIfAbsent(it.value.indexUUID, it.key) if (it.value.settings.get(ManagedIndexSettings.AUTO_MANAGE.key) == "false") { - indicesWithAutoManageBlock.add(it.value.indexUUID) + indicesWithAutoManageFalseBlock.add(it.value.indexUUID) } if (it.value.settings.get(SETTING_READ_ONLY) == "true") { indicesWithReadOnlyBlock.add(it.value.indexUUID) @@ -149,7 +174,14 @@ class TransportRemovePolicyAction @Inject constructor( indicesWithReadOnlyAllowDeleteBlock.add(it.value.indexUUID) } } - populateLists(response.state) + + val defaultIndexMetadataService = indexMetadataProvider.services[DEFAULT_INDEX_TYPE] as DefaultIndexMetadataService + getUuidsForClosedIndices(response.state, defaultIndexMetadataService).forEach { + failedIndices.add(FailedIndex(indicesToRemove[it] as String, it, "This index is closed")) + indicesToRemove.remove(it) + } + + getExistingManagedIndices() } override fun onFailure(t: Exception) { @@ -160,11 +192,7 @@ class TransportRemovePolicyAction @Inject constructor( } } - private fun populateLists(state: ClusterState) { - getUuidsForClosedIndices(state).forEach { - failedIndices.add(FailedIndex(indicesToRemove[it] as String, it, "This index is closed")) - indicesToRemove.remove(it) - } + private fun getExistingManagedIndices() { if (indicesToRemove.isEmpty()) { actionListener.onResponse(ISMStatusResponse(0, failedIndices)) return @@ -206,7 +234,11 @@ class TransportRemovePolicyAction @Inject constructor( } } - updateSettings(indicesToRemove) + if (request.indexType == DEFAULT_INDEX_TYPE) { + updateSettings(indicesToRemove) + } else { + removeManagedIndices() + } } override fun onFailure(t: Exception) { @@ -225,7 +257,7 @@ class TransportRemovePolicyAction @Inject constructor( @Suppress("SpreadOperator") fun updateSettings(indices: Map) { // indices divide to read_only, read_only_allow_delete, normal - val indicesUuidsSet = indices.keys.toSet() - indicesWithAutoManageBlock + val indicesUuidsSet = indices.keys.toSet() - indicesWithAutoManageFalseBlock val readOnlyIndices = indicesUuidsSet.filter { it in indicesWithReadOnlyBlock } val readOnlyAllowDeleteIndices = (indicesUuidsSet - readOnlyIndices).filter { it in indicesWithReadOnlyAllowDeleteBlock } val normalIndices = indicesUuidsSet - readOnlyIndices - readOnlyAllowDeleteIndices @@ -320,6 +352,8 @@ class TransportRemovePolicyAction @Inject constructor( // clean metadata for indicesToRemove val indicesToRemoveMetadata = indicesToRemove.map { Index(it.value, it.key) } + // best effort + CoroutineScope(Dispatchers.IO).launch { removeClusterStateMetadatas(client, log, indicesToRemoveMetadata) } removeMetadatas(indicesToRemoveMetadata) } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/retryfailedmanagedindex/RetryFailedManagedIndexRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/retryfailedmanagedindex/RetryFailedManagedIndexRequest.kt index 524b2c065..20750d4e7 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/retryfailedmanagedindex/RetryFailedManagedIndexRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/retryfailedmanagedindex/RetryFailedManagedIndexRequest.kt @@ -11,35 +11,33 @@ import org.opensearch.action.ValidateActions import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.io.stream.StreamOutput import org.opensearch.common.unit.TimeValue +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import java.io.IOException -class RetryFailedManagedIndexRequest : ActionRequest { - - val indices: List - val startState: String? - val masterTimeout: TimeValue - - constructor( - indices: List, - startState: String?, - masterTimeout: TimeValue - ) : super() { - this.indices = indices - this.startState = startState - this.masterTimeout = masterTimeout - } +class RetryFailedManagedIndexRequest( + val indices: List, + val startState: String?, + val masterTimeout: TimeValue, + val indexType: String +) : ActionRequest() { @Throws(IOException::class) constructor(sin: StreamInput) : this( indices = sin.readStringList(), startState = sin.readOptionalString(), - masterTimeout = sin.readTimeValue() + masterTimeout = sin.readTimeValue(), + indexType = sin.readString() ) override fun validate(): ActionRequestValidationException? { var validationException: ActionRequestValidationException? = null if (indices.isEmpty()) { validationException = ValidateActions.addValidationError("Missing indices", validationException) + } else if (indexType != DEFAULT_INDEX_TYPE && indices.size > 1) { + validationException = ValidateActions.addValidationError( + MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR, + validationException + ) } return validationException } @@ -49,5 +47,11 @@ class RetryFailedManagedIndexRequest : ActionRequest { out.writeStringCollection(indices) out.writeOptionalString(startState) out.writeTimeValue(masterTimeout) + out.writeString(indexType) + } + + companion object { + const val MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR = + "Cannot retry on more than one index name/pattern when using a custom index type" } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/retryfailedmanagedindex/TransportRetryFailedManagedIndexAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/retryfailedmanagedindex/TransportRetryFailedManagedIndexAction.kt index 201609f76..975a87622 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/retryfailedmanagedindex/TransportRetryFailedManagedIndexAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/retryfailedmanagedindex/TransportRetryFailedManagedIndexAction.kt @@ -5,6 +5,9 @@ package org.opensearch.indexmanagement.indexstatemanagement.transport.action.retryfailedmanagedindex +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.OpenSearchSecurityException @@ -22,8 +25,8 @@ import org.opensearch.action.support.IndicesOptions import org.opensearch.action.support.master.AcknowledgedResponse import org.opensearch.action.update.UpdateRequest import org.opensearch.client.node.NodeClient -import org.opensearch.cluster.ClusterState import org.opensearch.cluster.block.ClusterBlockException +import org.opensearch.cluster.metadata.IndexMetadata import org.opensearch.common.inject.Inject import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.XContentFactory @@ -32,18 +35,22 @@ import org.opensearch.commons.authuser.User import org.opensearch.index.Index import org.opensearch.index.IndexNotFoundException import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.PolicyRetryInfoMetaData +import org.opensearch.indexmanagement.indexstatemanagement.DefaultIndexMetadataService +import org.opensearch.indexmanagement.indexstatemanagement.IndexMetadataProvider import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.buildMgetMetadataRequest import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.getManagedIndexMetadata -import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.mgetResponseToList +import org.opensearch.indexmanagement.indexstatemanagement.opensearchapi.mgetResponseToMap import org.opensearch.indexmanagement.indexstatemanagement.transport.action.ISMStatusResponse import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexRequest +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.indexmanagement.indexstatemanagement.util.FailedIndex import org.opensearch.indexmanagement.indexstatemanagement.util.isFailed import org.opensearch.indexmanagement.indexstatemanagement.util.managedIndexMetadataID import org.opensearch.indexmanagement.indexstatemanagement.util.updateEnableManagedIndexRequest +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ISMIndexMetadata +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData import org.opensearch.indexmanagement.util.IndexManagementException import org.opensearch.indexmanagement.util.SecurityUtils.Companion.buildUser import org.opensearch.rest.RestStatus @@ -56,7 +63,8 @@ private val log = LogManager.getLogger(TransportRetryFailedManagedIndexAction::c class TransportRetryFailedManagedIndexAction @Inject constructor( val client: NodeClient, transportService: TransportService, - actionFilters: ActionFilters + actionFilters: ActionFilters, + val indexMetadataProvider: IndexMetadataProvider ) : HandledTransportAction( RetryFailedManagedIndexAction.NAME, transportService, actionFilters, ::RetryFailedManagedIndexRequest ) { @@ -74,9 +82,9 @@ class TransportRetryFailedManagedIndexAction @Inject constructor( private val listOfMetadata: MutableList = mutableListOf() private val listOfIndexToMetadata: MutableList> = mutableListOf() private val mapOfItemIdToIndex: MutableMap = mutableMapOf() - private lateinit var clusterState: ClusterState private val indicesManagedState: MutableMap = mutableMapOf() private var indicesToRetry = mutableMapOf() // uuid: name + private val indexUuidToIndexMetadata = mutableMapOf() // uuid -> indexmetadata @Suppress("SpreadOperator") fun start() { @@ -87,20 +95,20 @@ class TransportRetryFailedManagedIndexAction @Inject constructor( ) if (user == null) { // Security plugin is not enabled - getClusterState() + getIndicesToRetry() } else { - validateAndGetClusterState() + validateAndGetIndicesToRetry() } } - fun validateAndGetClusterState() { - val request = ManagedIndexRequest().indices(*request.indices.toTypedArray()) + private fun validateAndGetIndicesToRetry() { + val managedIndexRequest = ManagedIndexRequest().indices(*request.indices.toTypedArray()) client.execute( ManagedIndexAction.INSTANCE, - request, + managedIndexRequest, object : ActionListener { override fun onResponse(response: AcknowledgedResponse) { - getClusterState() + getIndicesToRetry() } override fun onFailure(e: java.lang.Exception) { @@ -120,7 +128,27 @@ class TransportRetryFailedManagedIndexAction @Inject constructor( ) } - fun getClusterState() { + private fun getIndicesToRetry() { + CoroutineScope(Dispatchers.IO).launch { + val indexNameToMetadata: MutableMap = HashMap() + try { + indexNameToMetadata.putAll(indexMetadataProvider.getISMIndexMetadataByType(request.indexType, request.indices)) + } catch (e: Exception) { + actionListener.onFailure(ExceptionsHelper.unwrapCause(e) as Exception) + return@launch + } + indexNameToMetadata.forEach { (indexName, indexMetadata) -> + indicesToRetry.putIfAbsent(indexMetadata.indexUuid, indexName) + } + if (request.indexType == DEFAULT_INDEX_TYPE) { + getClusterState() + } else { + processResponse() + } + } + } + + private fun getClusterState() { val strictExpandIndicesOptions = IndicesOptions.strictExpand() val clusterStateRequest = ClusterStateRequest() @@ -138,12 +166,12 @@ class TransportRetryFailedManagedIndexAction @Inject constructor( clusterStateRequest, object : ActionListener { override fun onResponse(response: ClusterStateResponse) { - clusterState = response.state - val indexMetadatas = response.state.metadata.indices - indexMetadatas.forEach { - indicesToRetry.putIfAbsent(it.value.indexUUID, it.key) + val defaultIndexMetadataService = indexMetadataProvider.services[DEFAULT_INDEX_TYPE] as DefaultIndexMetadataService + response.state.metadata.indices.forEach { + val indexUUID = defaultIndexMetadataService.getCustomIndexUUID(it.value) + indexUuidToIndexMetadata[indexUUID] = it.value } - processResponse(response) + processResponse() } override fun onFailure(t: Exception) { @@ -154,10 +182,9 @@ class TransportRetryFailedManagedIndexAction @Inject constructor( } } - fun processResponse(clusterStateResponse: ClusterStateResponse) { + private fun processResponse() { val mReq = MultiGetRequest() - clusterStateResponse.state.metadata.indices.map { it.value.indexUUID } - .forEach { mReq.add(INDEX_MANAGEMENT_INDEX, it) } + indicesToRetry.map { it.key }.forEach { mReq.add(INDEX_MANAGEMENT_INDEX, it) } client.multiGet( mReq, @@ -178,7 +205,10 @@ class TransportRetryFailedManagedIndexAction @Inject constructor( } // get back metadata from config index - client.multiGet(buildMgetMetadataRequest(clusterState), ActionListener.wrap(::onMgetMetadataResponse, ::onFailure)) + client.multiGet( + buildMgetMetadataRequest(indicesToRetry.toList().map { it.first }), + ActionListener.wrap(::onMgetMetadataResponse, ::onFailure) + ) } override fun onFailure(t: Exception) { @@ -190,46 +220,27 @@ class TransportRetryFailedManagedIndexAction @Inject constructor( @Suppress("ComplexMethod") private fun onMgetMetadataResponse(mgetResponse: MultiGetResponse) { - val metadataList = mgetResponseToList(mgetResponse) - clusterState.metadata.indices.forEachIndexed { ind, it -> - val indexMetaData = it.value - val clusterStateMetadata = it.value.getManagedIndexMetadata() - val mgetFailure = metadataList[ind]?.second - val managedIndexMetadata: ManagedIndexMetaData? = metadataList[ind]?.first + val metadataMap = mgetResponseToMap(mgetResponse) + indicesToRetry.forEach { (indexUuid, indexName) -> + // indexMetaData and clusterStateMetadata will be null for non-default index types + val indexMetaData = indexUuidToIndexMetadata[indexUuid] + val clusterStateMetadata = indexMetaData?.getManagedIndexMetadata() + val mgetFailure = metadataMap[managedIndexMetadataID(indexUuid)]?.second + val managedIndexMetadata: ManagedIndexMetaData? = metadataMap[managedIndexMetadataID(indexUuid)]?.first when { - indicesManagedState[indexMetaData.indexUUID] == false -> - failedIndices.add(FailedIndex(indexMetaData.index.name, indexMetaData.index.uuid, "This index is not being managed.")) + indicesManagedState[indexUuid] == false -> + failedIndices.add(FailedIndex(indexName, indexUuid, "This index is not being managed.")) mgetFailure != null -> - failedIndices.add( - FailedIndex( - indexMetaData.index.name, indexMetaData.index.uuid, - "Failed to get managed index metadata, $mgetFailure" - ) - ) + failedIndices.add(FailedIndex(indexName, indexUuid, "Failed to get managed index metadata, $mgetFailure")) managedIndexMetadata == null -> { if (clusterStateMetadata != null) { - failedIndices.add( - FailedIndex( - indexMetaData.index.name, indexMetaData.index.uuid, - "Cannot retry until metadata has finished migrating" - ) - ) + failedIndices.add(FailedIndex(indexName, indexUuid, "Cannot retry until metadata has finished migrating")) } else { - failedIndices.add( - FailedIndex( - indexMetaData.index.name, indexMetaData.index.uuid, - "This index has no metadata information" - ) - ) + failedIndices.add(FailedIndex(indexName, indexUuid, "This index has no metadata information")) } } !managedIndexMetadata.isFailed -> - failedIndices.add( - FailedIndex( - indexMetaData.index.name, indexMetaData.index.uuid, - "This index is not in failed state." - ) - ) + failedIndices.add(FailedIndex(indexName, indexUuid, "This index is not in failed state.")) else -> listOfMetadata.add(managedIndexMetadata) } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/updateindexmetadata/TransportUpdateManagedIndexMetaDataAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/updateindexmetadata/TransportUpdateManagedIndexMetaDataAction.kt index 9e398142c..cc70132ad 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/updateindexmetadata/TransportUpdateManagedIndexMetaDataAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/updateindexmetadata/TransportUpdateManagedIndexMetaDataAction.kt @@ -27,7 +27,8 @@ import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.io.stream.Writeable import org.opensearch.index.Index import org.opensearch.indexmanagement.IndexManagementPlugin -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.indexstatemanagement.IndexMetadataProvider +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.threadpool.ThreadPool import org.opensearch.transport.TransportService @@ -36,6 +37,7 @@ class TransportUpdateManagedIndexMetaDataAction @Inject constructor( clusterService: ClusterService, transportService: TransportService, actionFilters: ActionFilters, + val indexMetadataProvider: IndexMetadataProvider, indexNameExpressionResolver: IndexNameExpressionResolver ) : TransportMasterNodeAction( UpdateManagedIndexMetaDataAction.INSTANCE.name(), @@ -53,13 +55,27 @@ class TransportUpdateManagedIndexMetaDataAction @Inject constructor( override fun checkBlock(request: UpdateManagedIndexMetaDataRequest, state: ClusterState): ClusterBlockException? { // https://github.com/elastic/elasticsearch/commit/ae14b4e6f96b554ca8f4aaf4039b468f52df0123 // This commit will help us to give each individual index name and the error that is cause it. For now it will be a generic error message. - val indicesToAddTo = request.indicesToAddManagedIndexMetaDataTo.map { it.first.name }.toTypedArray() - val indicesToRemoveFrom = request.indicesToRemoveManagedIndexMetaDataFrom.map { it.name }.toTypedArray() - val indices = indicesToAddTo + indicesToRemoveFrom + val indicesToAddTo = request.indicesToAddManagedIndexMetaDataTo.map { it.first }.toTypedArray() + val indicesToRemoveFrom = request.indicesToRemoveManagedIndexMetaDataFrom.map { it }.toTypedArray() + val indices = checkExtensionsOverrideBlock(indicesToAddTo + indicesToRemoveFrom, state) return state.blocks.indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indices) } + /* + * Index Management extensions may provide an index setting, which, if set to true, overrides the cluster metadata write block + */ + private fun checkExtensionsOverrideBlock(indices: Array, state: ClusterState): Array { + val indexBlockOverrideSettings = indexMetadataProvider.getIndexMetadataWriteOverrideSettings() + val indicesToBlock = indices.toMutableList() + indexBlockOverrideSettings.forEach { indexBlockOverrideSetting -> + indicesToBlock.removeIf { state.metadata.getIndexSafe(it).settings.getAsBoolean(indexBlockOverrideSetting, false) } + } + return indicesToBlock + .map { it.name } + .toTypedArray() + } + override fun masterOperation( request: UpdateManagedIndexMetaDataRequest, state: ClusterState, diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/updateindexmetadata/UpdateManagedIndexMetaDataRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/updateindexmetadata/UpdateManagedIndexMetaDataRequest.kt index eacd0abd9..3fa8e436c 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/updateindexmetadata/UpdateManagedIndexMetaDataRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/updateindexmetadata/UpdateManagedIndexMetaDataRequest.kt @@ -11,7 +11,7 @@ import org.opensearch.action.support.master.AcknowledgedRequest import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.io.stream.StreamOutput import org.opensearch.index.Index -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData class UpdateManagedIndexMetaDataRequest : AcknowledgedRequest { diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/IndexEvaluator.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/IndexEvaluator.kt deleted file mode 100644 index bf22c967d..000000000 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/IndexEvaluator.kt +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.indexmanagement.indexstatemanagement.util - -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Settings -import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings - -class IndexEvaluator(settings: Settings, clusterService: ClusterService) { - - @Volatile private var restrictedIndexPattern = ManagedIndexSettings.RESTRICTED_INDEX_PATTERN.get(settings) - - init { - clusterService.clusterSettings.addSettingsUpdateConsumer(ManagedIndexSettings.RESTRICTED_INDEX_PATTERN) { - restrictedIndexPattern = it - } - } - - fun isUnManageableIndex(index: String): Boolean { - return Regex(restrictedIndexPattern).matches(index) - } - - companion object { - const val EVALUATION_FAILURE_MESSAGE = "Matches restricted index pattern defined in the cluster setting" - } -} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtils.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtils.kt index 6f250b5af..b9c9dc7ee 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtils.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtils.kt @@ -3,58 +3,50 @@ * SPDX-License-Identifier: Apache-2.0 */ -@file:Suppress("TooManyFunctions") +@file:Suppress("TooManyFunctions", "MatchingDeclarationName") @file:JvmName("ManagedIndexUtils") package org.opensearch.indexmanagement.indexstatemanagement.util -import inet.ipaddr.IPAddressString -import org.apache.logging.log4j.LogManager +// import inet.ipaddr.IPAddressString +// import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger -import org.opensearch.action.DocWriteRequest import org.opensearch.action.delete.DeleteRequest import org.opensearch.action.index.IndexRequest import org.opensearch.action.search.SearchRequest import org.opensearch.action.support.WriteRequest import org.opensearch.action.update.UpdateRequest -import org.opensearch.alerting.destination.message.BaseMessage -import org.opensearch.client.Client -import org.opensearch.cluster.metadata.IndexMetadata -import org.opensearch.cluster.service.ClusterService -import org.opensearch.common.settings.Settings +// import org.opensearch.alerting.destination.message.BaseMessage import org.opensearch.common.unit.ByteSizeValue import org.opensearch.common.unit.TimeValue import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.XContentFactory -import org.opensearch.index.Index import org.opensearch.index.query.BoolQueryBuilder import org.opensearch.index.query.QueryBuilders import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexCoordinator -import org.opensearch.indexmanagement.indexstatemanagement.action.Action +import org.opensearch.indexmanagement.indexstatemanagement.action.DeleteAction +import org.opensearch.indexmanagement.indexstatemanagement.action.RolloverAction +import org.opensearch.indexmanagement.indexstatemanagement.action.TransitionsAction import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy import org.opensearch.indexmanagement.indexstatemanagement.model.ISMTemplate import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State import org.opensearch.indexmanagement.indexstatemanagement.model.Transition -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionRetry -import org.opensearch.indexmanagement.indexstatemanagement.model.action.RolloverActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.TransitionsActionConfig import org.opensearch.indexmanagement.indexstatemanagement.model.coordinator.SweptManagedIndexConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.PolicyRetryInfoMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StateMetaData import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings -import org.opensearch.indexmanagement.indexstatemanagement.step.Step -import org.opensearch.indexmanagement.indexstatemanagement.step.delete.AttemptDeleteStep import org.opensearch.indexmanagement.opensearchapi.optionalISMTemplateField import org.opensearch.indexmanagement.opensearchapi.optionalTimeField +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionRetry +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaData import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule -import org.opensearch.script.ScriptService import org.opensearch.search.builder.SearchSourceBuilder -import java.net.InetAddress +// import java.net.InetAddress import java.time.Instant import java.time.temporal.ChronoUnit @@ -159,7 +151,7 @@ fun deleteManagedIndexRequest(uuid: String): DeleteRequest { } fun deleteManagedIndexMetadataRequest(uuid: String): DeleteRequest { - return DeleteRequest(INDEX_MANAGEMENT_INDEX, managedIndexMetadataID(uuid)) + return DeleteRequest(INDEX_MANAGEMENT_INDEX, managedIndexMetadataID(uuid)).routing(uuid) } fun updateManagedIndexRequest(sweptManagedIndexConfig: SweptManagedIndexConfig): UpdateRequest { @@ -170,58 +162,37 @@ fun updateManagedIndexRequest(sweptManagedIndexConfig: SweptManagedIndexConfig): } /** - * Creates DeleteRequests for [ManagedIndexConfig]. - * * Finds ManagedIndices that exist in [INDEX_MANAGEMENT_INDEX] that do not exist in the cluster state * anymore which means we need to delete the [ManagedIndexConfig]. * - * @param currentIndices List of current [IndexMetadata] in cluster state. - * @param currentManagedIndexConfigs map of IndexUuid to [SweptManagedIndexConfig]. - * @return list of [DocWriteRequest]. + * @param currentIndexUuids List of current index uuids in cluster. + * @param currentManagedIndexUuids List of current managed index uuids in cluster. + * @return list of managedIndexUuids to delete. */ -fun getDeleteManagedIndexRequests( - currentIndices: List, - currentManagedIndexConfigs: Map -): List> { - return currentManagedIndexConfigs.filter { currentManagedIndex -> - !currentIndices.map { it.index.uuid }.contains(currentManagedIndex.key) - }.map { deleteManagedIndexRequest(it.value.uuid) } -} - -// if managed index exist but the index is not existing any more -// then we should delete this managed index fun getManagedIndicesToDelete( - currentIndices: List, - currentManagedIndexConfigs: Map -): List { - val currentIndicesSet = currentIndices.map { it.index }.toSet() - val managedIndicesSet = currentManagedIndexConfigs.values.map { Index(it.index, it.uuid) }.toSet() - return (managedIndicesSet - currentIndicesSet).toList() + currentIndexUuids: List, + currentManagedIndexUuids: List +): List { + return currentManagedIndexUuids.filter { currentManagedIndex -> + !currentIndexUuids.contains(currentManagedIndex) + } } fun getSweptManagedIndexSearchRequest(): SearchRequest { val boolQueryBuilder = BoolQueryBuilder().filter(QueryBuilders.existsQuery(ManagedIndexConfig.MANAGED_INDEX_TYPE)) return SearchRequest() .indices(INDEX_MANAGEMENT_INDEX) + .scroll(TimeValue.timeValueMinutes(1)) .source( SearchSourceBuilder.searchSource() - // TODO: Get all ManagedIndices at once or split into searchAfter queries? .size(ManagedIndexCoordinator.MAX_HITS) .seqNoAndPrimaryTerm(true) - .fetchSource( - arrayOf( - "${ManagedIndexConfig.MANAGED_INDEX_TYPE}.${ManagedIndexConfig.INDEX_FIELD}", - "${ManagedIndexConfig.MANAGED_INDEX_TYPE}.${ManagedIndexConfig.INDEX_UUID_FIELD}", - "${ManagedIndexConfig.MANAGED_INDEX_TYPE}.${ManagedIndexConfig.POLICY_ID_FIELD}", - "${ManagedIndexConfig.MANAGED_INDEX_TYPE}.${ManagedIndexConfig.CHANGE_POLICY_FIELD}" - ), - emptyArray() - ) + .fetchSource(emptyArray(), emptyArray()) .query(boolQueryBuilder) ) } -@Suppress("ReturnCount") +@Suppress("ReturnCount", "ComplexCondition") fun Transition.evaluateConditions( indexCreationDate: Instant, numDocs: Long?, @@ -264,8 +235,8 @@ fun Transition.evaluateConditions( fun Transition.hasStatsConditions(): Boolean = this.conditions?.docCount != null || this.conditions?.size != null -@Suppress("ReturnCount", "ComplexMethod", "ComplexCondition") -fun RolloverActionConfig.evaluateConditions( +@Suppress("ReturnCount") +fun RolloverAction.evaluateConditions( indexAgeTimeValue: TimeValue, numDocs: Long, indexSize: ByteSizeValue, @@ -300,49 +271,6 @@ fun RolloverActionConfig.evaluateConditions( return false } -fun Policy.getStateToExecute(managedIndexMetaData: ManagedIndexMetaData): State? { - if (managedIndexMetaData.transitionTo != null) { - return this.states.find { it.name == managedIndexMetaData.transitionTo } - } - return this.states.find { managedIndexMetaData.stateMetaData != null && it.name == managedIndexMetaData.stateMetaData.name } -} - -fun State.getActionToExecute( - clusterService: ClusterService, - scriptService: ScriptService, - client: Client, - settings: Settings, - managedIndexMetaData: ManagedIndexMetaData -): Action? { - var actionConfig: ActionConfig? - - // If we are transitioning to this state get the first action in the state - // If the action/actionIndex are null it means we just initialized and should get the first action from the state - if (managedIndexMetaData.transitionTo != null || managedIndexMetaData.actionMetaData == null) { - actionConfig = this.actions.firstOrNull() ?: TransitionsActionConfig(this.transitions) - } else if (managedIndexMetaData.actionMetaData.name == ActionConfig.ActionType.TRANSITION.type) { - // If the current action is transition and we do not have a transitionTo set then we should be in Transition - actionConfig = TransitionsActionConfig(this.transitions) - } else { - // Get the current actionConfig that is in the ManagedIndexMetaData - actionConfig = this.actions.filterIndexed { index, config -> - index == managedIndexMetaData.actionMetaData.index && config.type.type == managedIndexMetaData.actionMetaData.name - }.firstOrNull() - if (actionConfig == null) return null - - // TODO: Refactor so we can get isLastStep from somewhere besides an instantiated Action class so we can simplify this to a when block - // If stepCompleted is true and this is the last step of the action then we should get the next action - if (managedIndexMetaData.stepMetaData != null && managedIndexMetaData.stepMetaData.stepStatus == Step.StepStatus.COMPLETED) { - val action = actionConfig.toAction(clusterService, scriptService, client, settings, managedIndexMetaData) - if (action.isLastStep(managedIndexMetaData.stepMetaData.name)) { - actionConfig = this.actions.getOrNull(managedIndexMetaData.actionMetaData.index + 1) ?: TransitionsActionConfig(this.transitions) - } - } - } - - return actionConfig.toAction(clusterService, scriptService, client, settings, managedIndexMetaData) -} - fun State.getUpdatedStateMetaData(managedIndexMetaData: ManagedIndexMetaData): StateMetaData { // If the current ManagedIndexMetaData state does not match this state, it means we transitioned and need to update the startStartTime val stateMetaData = managedIndexMetaData.stateMetaData @@ -353,31 +281,16 @@ fun State.getUpdatedStateMetaData(managedIndexMetaData: ManagedIndexMetaData): S } } -fun Action.getUpdatedActionMetaData(managedIndexMetaData: ManagedIndexMetaData, state: State): ActionMetaData { - val stateMetaData = managedIndexMetaData.stateMetaData - val actionMetaData = managedIndexMetaData.actionMetaData - - return when { - // start a new action - stateMetaData?.name != state.name -> - ActionMetaData(this.type.type, Instant.now().toEpochMilli(), this.config.actionIndex, false, 0, 0, null) - actionMetaData?.index != this.config.actionIndex -> - ActionMetaData(this.type.type, Instant.now().toEpochMilli(), this.config.actionIndex, false, 0, 0, null) - // RetryAPI will reset startTime to null for actionMetaData and we'll reset it to "now" here - else -> actionMetaData.copy(startTime = actionMetaData.startTime ?: Instant.now().toEpochMilli()) - } -} - fun Action.shouldBackoff(actionMetaData: ActionMetaData?, actionRetry: ActionRetry?): Pair? { - return this.config.configRetry?.backoff?.shouldBackoff(actionMetaData, actionRetry) + return this.configRetry?.backoff?.shouldBackoff(actionMetaData, actionRetry) } @Suppress("ReturnCount") fun Action.hasTimedOut(actionMetaData: ActionMetaData?): Boolean { - if (actionMetaData?.startTime == null) return false - val configTimeout = this.config.configTimeout - if (configTimeout == null) return false - return (Instant.now().toEpochMilli() - actionMetaData.startTime) > configTimeout.timeout.millis + val startTime = actionMetaData?.startTime + val configTimeout = this.configTimeout + if (startTime == null || configTimeout == null) return false + return (Instant.now().toEpochMilli() - startTime) > configTimeout.timeout.millis } @Suppress("ReturnCount") @@ -404,8 +317,8 @@ fun ManagedIndexMetaData.getStartingManagedIndexMetaData( } val updatedStateMetaData = state.getUpdatedStateMetaData(this) - val updatedActionMetaData = action.getUpdatedActionMetaData(this, state) - val updatedStepMetaData = step.getStartingStepMetaData() + val updatedActionMetaData = action.getUpdatedActionMetadata(this, state.name) + val updatedStepMetaData = step.getStartingStepMetaData(this) return this.copy( stateMetaData = updatedStateMetaData, @@ -420,7 +333,7 @@ fun ManagedIndexMetaData.getCompletedManagedIndexMetaData( action: Action, step: Step ): ManagedIndexMetaData { - val updatedStepMetaData = step.getUpdatedManagedIndexMetaData(this) + val updatedStepMetaData = step.getUpdatedManagedIndexMetadata(this) val actionMetaData = updatedStepMetaData.actionMetaData ?: return this.copy( policyRetryInfo = PolicyRetryInfoMetaData(true, 0), info = mapOf("message" to "Failed due to ActionMetaData being null") @@ -428,8 +341,8 @@ fun ManagedIndexMetaData.getCompletedManagedIndexMetaData( val updatedActionMetaData = if (updatedStepMetaData.stepMetaData?.stepStatus == Step.StepStatus.FAILED) { when { - action.config.configRetry == null -> actionMetaData.copy(failed = true) - actionMetaData.consumedRetries >= action.config.configRetry!!.count -> actionMetaData.copy(failed = true) + action.configRetry == null -> actionMetaData.copy(failed = true) + actionMetaData.consumedRetries >= action.configRetry!!.count -> actionMetaData.copy(failed = true) else -> actionMetaData.copy( failed = false, consumedRetries = actionMetaData.consumedRetries + 1, @@ -452,8 +365,8 @@ fun ManagedIndexMetaData.getCompletedManagedIndexMetaData( } val ManagedIndexMetaData.isSuccessfulDelete: Boolean - get() = (this.actionMetaData?.name == ActionConfig.ActionType.DELETE.type && !this.actionMetaData.failed) && - (this.stepMetaData?.name == AttemptDeleteStep.name && this.stepMetaData.stepStatus == Step.StepStatus.COMPLETED) && + get() = (this.actionMetaData?.name == DeleteAction.name && !this.actionMetaData!!.failed) && + (this.stepMetaData?.name == DeleteAction.name && this.stepMetaData!!.stepStatus == Step.StepStatus.COMPLETED) && (this.policyRetryInfo?.failed != true) val ManagedIndexMetaData.isFailed: Boolean @@ -492,11 +405,11 @@ fun ManagedIndexConfig.shouldChangePolicy(managedIndexMetaData: ManagedIndexMeta // we need this in so that we can change policy before the first transition happens so policy doesnt get completed // before we have a chance to change policy - if (actionToExecute?.type == ActionConfig.ActionType.TRANSITION) { + if (actionToExecute?.type == TransitionsAction.name) { return true } - if (managedIndexMetaData.actionMetaData?.name != ActionConfig.ActionType.TRANSITION.type) { + if (managedIndexMetaData.actionMetaData?.name != TransitionsAction.name) { return false } @@ -554,45 +467,45 @@ fun Policy.isSafeToChange(stateName: String?, newPolicy: Policy, changePolicy: C return true } -/** - * Disallowed actions are ones that are not specified in the [ManagedIndexSettings.ALLOW_LIST] setting. - */ -fun Policy.getDisallowedActions(allowList: List): List { - val allowListSet = allowList.toSet() - val disallowedActions = mutableListOf() - this.states.forEach { state -> - state.actions.forEach { actionConfig -> - if (!allowListSet.contains(actionConfig.type.type)) { - disallowedActions.add(actionConfig.type.type) - } - } - } - return disallowedActions.distinct() -} - /** * Allowed actions are ones that are specified in the [ManagedIndexSettings.ALLOW_LIST] setting. */ -fun Action.isAllowed(allowList: List): Boolean = allowList.contains(this.type.type) +fun Action.isAllowed(allowList: List): Boolean = allowList.contains(this.type) /** * Check if cluster state metadata has been moved to config index * * log warning if remaining cluster state metadata has newer last_updated_time */ -fun isMetadataMoved( +@Suppress("ReturnCount", "ComplexCondition", "ComplexMethod") +fun checkMetadata( clusterStateMetadata: ManagedIndexMetaData?, configIndexMetadata: Any?, + currentIndexUuid: String?, logger: Logger -): Boolean { +): MetadataCheck { + // indexUuid saved in ISM metadata may be outdated + // if an index restored from snapshot + val indexUuid1 = clusterStateMetadata?.indexUuid + val indexUuid2 = when (configIndexMetadata) { + is ManagedIndexMetaData -> configIndexMetadata.indexUuid + is Map<*, *> -> configIndexMetadata["index_uuid"] + else -> null + } as String? + if ((indexUuid1 != null && indexUuid1 != currentIndexUuid) || + (indexUuid2 != null && indexUuid2 != currentIndexUuid) + ) { + return MetadataCheck.CORRUPT + } + if (clusterStateMetadata != null) { - if (configIndexMetadata == null) return false + if (configIndexMetadata == null) return MetadataCheck.PENDING // compare last updated time between 2 metadatas val t1 = clusterStateMetadata.stepMetaData?.startTime val t2 = when (configIndexMetadata) { - is ManagedIndexMetaData? -> configIndexMetadata.stepMetaData?.startTime - is Map<*, *>? -> { + is ManagedIndexMetaData -> configIndexMetadata.stepMetaData?.startTime + is Map<*, *> -> { val stepMetadata = configIndexMetadata["step"] as Map? stepMetadata?.get("start_time") } @@ -602,19 +515,23 @@ fun isMetadataMoved( logger.warn("Cluster state metadata get updates after moved for [${clusterStateMetadata.index}]") } } - return true + return MetadataCheck.SUCCESS } -private val baseMessageLogger = LogManager.getLogger(BaseMessage::class.java) - -fun BaseMessage.isHostInDenylist(networks: List): Boolean { - val ipStr = IPAddressString(this.uri.host) - for (network in networks) { - val netStr = IPAddressString(network) - if (netStr.contains(ipStr)) { - baseMessageLogger.error("Host: {} resolves to: {} which is in denylist: {}.", uri.host, InetAddress.getByName(uri.host), netStr) - return true - } - } - return false +enum class MetadataCheck { + PENDING, CORRUPT, SUCCESS } + +// private val baseMessageLogger = LogManager.getLogger(BaseMessage::class.java) +// +// fun BaseMessage.isHostInDenylist(networks: List): Boolean { +// val ipStr = IPAddressString(this.uri.host) +// for (network in networks) { +// val netStr = IPAddressString(network) +// if (netStr.contains(ipStr)) { +// baseMessageLogger.error("Host: {} resolves to: {} which is in denylist: {}.", uri.host, InetAddress.getByName(uri.host), netStr) +// return true +// } +// } +// return false +// } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt index b5c011392..31b5b4b48 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt @@ -6,6 +6,9 @@ @file:Suppress("TopLevelPropertyNaming", "MatchingDeclarationName") package org.opensearch.indexmanagement.indexstatemanagement.util +import org.apache.logging.log4j.Logger +import org.opensearch.action.support.master.AcknowledgedResponse +import org.opensearch.client.Client import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.io.stream.StreamOutput import org.opensearch.common.io.stream.Writeable @@ -13,9 +16,14 @@ import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.ToXContentFragment import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.index.Index import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig +import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataAction +import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataRequest import org.opensearch.indexmanagement.opensearchapi.optionalTimeField +import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import java.lang.Exception import java.time.Instant const val WITH_TYPE = "with_type" @@ -27,20 +35,31 @@ val XCONTENT_WITHOUT_TYPE_AND_USER = ToXContent.MapParams(mapOf(WITH_TYPE to "fa const val FAILURES = "failures" const val FAILED_INDICES = "failed_indices" const val UPDATED_INDICES = "updated_indices" +const val TOTAL_MANAGED_INDICES = "total_managed_indices" const val ISM_TEMPLATE_FIELD = "policy.ism_template" +const val MANAGED_INDEX_FIELD = "managed_index" +const val MANAGED_INDEX_NAME_KEYWORD_FIELD = "$MANAGED_INDEX_FIELD.name.keyword" +const val MANAGED_INDEX_INDEX_FIELD = "$MANAGED_INDEX_FIELD.index" +const val MANAGED_INDEX_INDEX_UUID_FIELD = "$MANAGED_INDEX_FIELD.index_uuid" const val DEFAULT_PAGINATION_SIZE = 20 const val DEFAULT_PAGINATION_FROM = 0 -const val DEFAULT_JOB_SORT_FIELD = "managed_index.index" +const val DEFAULT_JOB_SORT_FIELD = MANAGED_INDEX_INDEX_FIELD const val DEFAULT_POLICY_SORT_FIELD = "policy.policy_id.keyword" const val DEFAULT_SORT_ORDER = "asc" const val DEFAULT_QUERY_STRING = "*" +const val SHOW_POLICY_QUERY_PARAM = "show_policy" +const val DEFAULT_EXPLAIN_SHOW_POLICY = false + const val INDEX_HIDDEN = "index.hidden" const val INDEX_NUMBER_OF_SHARDS = "index.number_of_shards" const val INDEX_NUMBER_OF_REPLICAS = "index.number_of_replicas" +const val TYPE_PARAM_KEY = "type" +const val DEFAULT_INDEX_TYPE = "_default" + fun buildInvalidIndexResponse(builder: XContentBuilder, failedIndices: List) { if (failedIndices.isNotEmpty()) { builder.field(FAILURES, true) @@ -97,3 +116,17 @@ fun getPartialChangePolicyBuilder( .field(ManagedIndexConfig.CHANGE_POLICY_FIELD, changePolicy) return builder.endObject().endObject() } + +/** + * Removes the managed index metadata from the cluster state for the the provided indices. + */ +suspend fun removeClusterStateMetadatas(client: Client, logger: Logger, indices: List) { + val request = UpdateManagedIndexMetaDataRequest(indicesToRemoveManagedIndexMetaDataFrom = indices) + + try { + val response: AcknowledgedResponse = client.suspendUntil { execute(UpdateManagedIndexMetaDataAction.INSTANCE, request, it) } + logger.debug("Cleaned cluster state metadata for $indices, ${response.isAcknowledged}") + } catch (e: Exception) { + logger.error("Failed to clean cluster state metadata for $indices") + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/opensearchapi/OpenSearchExtensions.kt b/src/main/kotlin/org/opensearch/indexmanagement/opensearchapi/OpenSearchExtensions.kt index 4cd1fa9d8..80e293ba6 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/opensearchapi/OpenSearchExtensions.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/opensearchapi/OpenSearchExtensions.kt @@ -54,6 +54,8 @@ import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException import kotlin.coroutines.suspendCoroutine +const val OPENDISTRO_SECURITY_PROTECTED_INDICES_CONF_REQUEST = "_opendistro_security_protected_indices_conf_request" + fun contentParser(bytesReference: BytesReference): XContentParser { return XContentHelper.createParser( NamedXContentRegistry.EMPTY, @@ -178,6 +180,8 @@ fun OpenSearchException.isRetryable(): Boolean { */ fun XContentBuilder.string(): String = BytesReference.bytes(this).utf8ToString() +fun XContentBuilder.toMap(): Map = XContentHelper.convertToMap(BytesReference.bytes(this), false, XContentType.JSON).v2() + /** * Converts [OpenSearchClient] methods that take a callback into a kotlin suspending function. * diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/RollupIndexer.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/RollupIndexer.kt index 1b9b77963..8c3c5fb93 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/RollupIndexer.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/RollupIndexer.kt @@ -47,8 +47,7 @@ class RollupIndexer( BackoffPolicy.constantBackoff(ROLLUP_INGEST_BACKOFF_MILLIS.get(settings), ROLLUP_INGEST_BACKOFF_COUNT.get(settings)) init { - clusterService.clusterSettings.addSettingsUpdateConsumer(ROLLUP_INGEST_BACKOFF_MILLIS, ROLLUP_INGEST_BACKOFF_COUNT) { - millis, count -> + clusterService.clusterSettings.addSettingsUpdateConsumer(ROLLUP_INGEST_BACKOFF_MILLIS, ROLLUP_INGEST_BACKOFF_COUNT) { millis, count -> retryIngestPolicy = BackoffPolicy.constantBackoff(millis, count) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/RollupMapperService.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/RollupMapperService.kt index 3dbe0b3d5..6763387ee 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/RollupMapperService.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/RollupMapperService.kt @@ -21,7 +21,6 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver import org.opensearch.cluster.metadata.MappingMetadata import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.XContentType import org.opensearch.indexmanagement.IndexManagementIndices import org.opensearch.indexmanagement.common.model.dimension.DateHistogram import org.opensearch.indexmanagement.common.model.dimension.Histogram @@ -36,7 +35,6 @@ import org.opensearch.indexmanagement.rollup.settings.RollupSettings import org.opensearch.indexmanagement.rollup.util.isRollupIndex import org.opensearch.indexmanagement.util.IndexUtils.Companion._META import org.opensearch.indexmanagement.util.IndexUtils.Companion.getFieldFromMappings -import org.opensearch.indexmanagement.util._DOC import org.opensearch.transport.RemoteTransportException // TODO: Validation of fields across source and target indices overwriting existing rollup data @@ -104,7 +102,7 @@ class RollupMapperService( } val request = CreateIndexRequest(job.targetIndex) .settings(settings) - .mapping(_DOC, IndexManagementIndices.rollupTargetMappings, XContentType.JSON) + .mapping(IndexManagementIndices.rollupTargetMappings) // TODO: Perhaps we can do better than this for mappings... as it'll be dynamic for rest // Can we read in the actual mappings from the source index and use that? // Can it have issues with metrics? i.e. an int mapping with 3, 5, 6 added up and divided by 3 for avg is 14/3 = 4.6666 @@ -148,13 +146,11 @@ class RollupMapperService( } val indexTypeMappings = res.mappings[index] - if (indexTypeMappings.isEmpty) { + if (indexTypeMappings == null) { return RollupJobValidationResult.Invalid("Source index [$index] mappings are empty, cannot validate the job.") } - // Starting from 6.0.0 an index can only have one mapping type, but mapping type is still part of the APIs in 7.x, allowing users to - // set a custom mapping type. As a result using first mapping type found instead of _DOC mapping type to validate - val indexMappingSource = indexTypeMappings.first().value.sourceAsMap + val indexMappingSource = indexTypeMappings.sourceAsMap val issues = mutableSetOf() // Validate source fields in dimensions @@ -215,7 +211,7 @@ class RollupMapperService( return RollupJobValidationResult.Failure(getMappingsResult.message, getMappingsResult.cause) } - val indexMapping: MappingMetadata = res.mappings[rollup.targetIndex][_DOC] + val indexMapping: MappingMetadata = res.mappings[rollup.targetIndex] return if (((indexMapping.sourceAsMap?.get(_META) as Map<*, *>?)?.get(ROLLUPS) as Map<*, *>?)?.containsKey(rollup.id) == true) { RollupJobValidationResult.Valid diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/RollupSearchService.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/RollupSearchService.kt index 460b8292d..a69ec735a 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/RollupSearchService.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/RollupSearchService.kt @@ -45,8 +45,7 @@ class RollupSearchService( BackoffPolicy.constantBackoff(ROLLUP_SEARCH_BACKOFF_MILLIS.get(settings), ROLLUP_SEARCH_BACKOFF_COUNT.get(settings)) init { - clusterService.clusterSettings.addSettingsUpdateConsumer(ROLLUP_SEARCH_BACKOFF_MILLIS, ROLLUP_SEARCH_BACKOFF_COUNT) { - millis, count -> + clusterService.clusterSettings.addSettingsUpdateConsumer(ROLLUP_SEARCH_BACKOFF_MILLIS, ROLLUP_SEARCH_BACKOFF_COUNT) { millis, count -> retrySearchPolicy = BackoffPolicy.constantBackoff(millis, count) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/mapping/TransportUpdateRollupMappingAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/mapping/TransportUpdateRollupMappingAction.kt index 076fcecb9..74d94d41f 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/mapping/TransportUpdateRollupMappingAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/mapping/TransportUpdateRollupMappingAction.kt @@ -26,7 +26,6 @@ import org.opensearch.common.xcontent.XContentHelper import org.opensearch.common.xcontent.XContentType import org.opensearch.indexmanagement.indexstatemanagement.util.XCONTENT_WITHOUT_TYPE import org.opensearch.indexmanagement.util.IndexUtils.Companion._META -import org.opensearch.indexmanagement.util._DOC import org.opensearch.threadpool.ThreadPool import org.opensearch.transport.TransportService import java.lang.Exception @@ -114,7 +113,7 @@ class TransportUpdateRollupMappingAction @Inject constructor( } // TODO: Does schema_version get overwritten? - val putMappingRequest = PutMappingRequest(request.rollup.targetIndex).type(_DOC).source(metaMappings) + val putMappingRequest = PutMappingRequest(request.rollup.targetIndex).source(metaMappings) client.admin().indices().putMapping( putMappingRequest, object : ActionListener { diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/actionfilter/FieldCapsFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/actionfilter/FieldCapsFilter.kt index 6dcfd2bae..8d9757b6b 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/actionfilter/FieldCapsFilter.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/actionfilter/FieldCapsFilter.kt @@ -41,8 +41,7 @@ class FieldCapsFilter( @Volatile private var shouldIntercept = RollupSettings.ROLLUP_DASHBOARDS.get(settings) init { - clusterService.clusterSettings.addSettingsUpdateConsumer(RollupSettings.ROLLUP_DASHBOARDS) { - flag -> + clusterService.clusterSettings.addSettingsUpdateConsumer(RollupSettings.ROLLUP_DASHBOARDS) { flag -> shouldIntercept = flag } } @@ -58,8 +57,7 @@ class FieldCapsFilter( val indices = request.indices().map { it.toString() }.toTypedArray() val rollupIndices = mutableSetOf() val nonRollupIndices = mutableSetOf() - val remoteClusterIndices = GuiceHolder.remoteClusterService.groupIndices(request.indicesOptions(), indices) { - idx: String? -> + val remoteClusterIndices = GuiceHolder.remoteClusterService.groupIndices(request.indicesOptions(), indices) { idx: String? -> indexNameExpressionResolver.hasIndexAbstraction(idx, clusterService.state()) } val localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/model/ISMRollup.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/model/ISMRollup.kt index df25eabbf..9604ccbee 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/model/ISMRollup.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/model/ISMRollup.kt @@ -5,6 +5,7 @@ package org.opensearch.indexmanagement.rollup.model +import org.apache.commons.codec.digest.DigestUtils import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.io.stream.StreamOutput import org.opensearch.common.io.stream.Writeable @@ -21,7 +22,6 @@ import org.opensearch.indexmanagement.common.model.dimension.Histogram import org.opensearch.indexmanagement.common.model.dimension.Terms import org.opensearch.indexmanagement.util.IndexUtils import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule -import org.opensearch.notification.repackage.org.apache.commons.codec.digest.DigestUtils import java.io.IOException import java.time.Instant import java.time.temporal.ChronoUnit diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/model/RollupMetadata.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/model/RollupMetadata.kt index 11ee3f2ae..c56a1648c 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/model/RollupMetadata.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/model/RollupMetadata.kt @@ -265,7 +265,7 @@ data class RollupMetadata( AFTER_KEY_FIELD -> afterKey = xcp.map() LAST_UPDATED_FIELD -> lastUpdatedTime = xcp.instant() CONTINUOUS_FIELD -> continuous = ContinuousMetadata.parse(xcp) - STATUS_FIELD -> status = Status.valueOf(xcp.text().toUpperCase(Locale.ROOT)) + STATUS_FIELD -> status = Status.valueOf(xcp.text().uppercase(Locale.ROOT)) FAILURE_REASON -> failureReason = xcp.textOrNull() STATS_FIELD -> stats = RollupStats.parse(xcp) } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformIndexer.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformIndexer.kt index 0c4e5614d..568be5fde 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformIndexer.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformIndexer.kt @@ -19,13 +19,11 @@ import org.opensearch.action.index.IndexRequest import org.opensearch.client.Client import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.XContentType import org.opensearch.indexmanagement.IndexManagementIndices import org.opensearch.indexmanagement.opensearchapi.retry import org.opensearch.indexmanagement.opensearchapi.suspendUntil import org.opensearch.indexmanagement.transform.exceptions.TransformIndexException import org.opensearch.indexmanagement.transform.settings.TransformSettings -import org.opensearch.indexmanagement.util._DOC import org.opensearch.rest.RestStatus import org.opensearch.transport.RemoteTransportException @@ -48,8 +46,7 @@ class TransformIndexer( clusterService.clusterSettings.addSettingsUpdateConsumer( TransformSettings.TRANSFORM_JOB_INDEX_BACKOFF_MILLIS, TransformSettings.TRANSFORM_JOB_INDEX_BACKOFF_COUNT - ) { - millis, count -> + ) { millis, count -> backoffPolicy = BackoffPolicy.constantBackoff(millis, count) } } @@ -57,7 +54,7 @@ class TransformIndexer( private suspend fun createTargetIndex(index: String) { if (!clusterService.state().routingTable.hasIndex(index)) { val request = CreateIndexRequest(index) - .mapping(_DOC, IndexManagementIndices.transformTargetMappings, XContentType.JSON) + .mapping(IndexManagementIndices.transformTargetMappings) // TODO: Read in the actual mappings from the source index and use that val response: CreateIndexResponse = client.admin().indices().suspendUntil { create(request, it) } if (!response.isAcknowledged) { diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformMetadataService.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformMetadataService.kt index 2b1944b17..818a634f9 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformMetadataService.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformMetadataService.kt @@ -26,6 +26,7 @@ import org.opensearch.indexmanagement.IndexManagementPlugin import org.opensearch.indexmanagement.opensearchapi.parseWithType import org.opensearch.indexmanagement.opensearchapi.suspendUntil import org.opensearch.indexmanagement.transform.exceptions.TransformMetadataException +import org.opensearch.indexmanagement.transform.model.ContinuousTransformStats import org.opensearch.indexmanagement.transform.model.Transform import org.opensearch.indexmanagement.transform.model.TransformMetadata import org.opensearch.indexmanagement.transform.model.TransformStats @@ -69,7 +70,8 @@ class TransformMetadataService(private val client: Client, val xContentRegistry: transformId = transform.id, lastUpdatedAt = Instant.now(), status = TransformMetadata.Status.INIT, - stats = TransformStats(0, 0, 0, 0, 0) + stats = TransformStats(0, 0, 0, 0, 0), + continuousStats = if (transform.continuous) ContinuousTransformStats(null, null) else null ) return writeMetadata(metadata) } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformRunner.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformRunner.kt index 4dd5a12a4..07e38fb11 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformRunner.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformRunner.kt @@ -19,14 +19,18 @@ import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings import org.opensearch.common.unit.TimeValue import org.opensearch.common.xcontent.NamedXContentRegistry +import org.opensearch.index.shard.ShardId import org.opensearch.indexmanagement.opensearchapi.IndexManagementSecurityContext import org.opensearch.indexmanagement.opensearchapi.suspendUntil import org.opensearch.indexmanagement.opensearchapi.withClosableContext import org.opensearch.indexmanagement.transform.action.index.IndexTransformAction import org.opensearch.indexmanagement.transform.action.index.IndexTransformRequest import org.opensearch.indexmanagement.transform.action.index.IndexTransformResponse +import org.opensearch.indexmanagement.transform.model.BucketsToTransform +import org.opensearch.indexmanagement.transform.model.ContinuousTransformStats import org.opensearch.indexmanagement.transform.model.Transform import org.opensearch.indexmanagement.transform.model.TransformMetadata +import org.opensearch.indexmanagement.transform.model.initializeShardsToSearch import org.opensearch.indexmanagement.transform.settings.TransformSettings import org.opensearch.indexmanagement.util.acquireLockForScheduledJob import org.opensearch.indexmanagement.util.releaseLockForScheduledJob @@ -38,7 +42,7 @@ import org.opensearch.monitor.jvm.JvmService import org.opensearch.threadpool.ThreadPool import java.time.Instant -@Suppress("LongParameterList") +@Suppress("LongParameterList", "TooManyFunctions") object TransformRunner : ScheduledJobRunner, CoroutineScope by CoroutineScope(SupervisorJob() + Dispatchers.Default + CoroutineName("TransformRunner")) { @@ -99,13 +103,18 @@ object TransformRunner : } // TODO: Add circuit breaker checks - [cluster healthy, utilization within limit] - @Suppress("NestedBlockDepth", "ComplexMethod") + @Suppress("NestedBlockDepth", "ComplexMethod", "LongMethod", "ReturnCount") private suspend fun executeJob(transform: Transform, metadata: TransformMetadata, context: JobExecutionContext) { + var newGlobalCheckpoints: Map? = null + var newGlobalCheckpointTime: Instant? = null var currentMetadata = metadata val backoffPolicy = BackoffPolicy.exponentialBackoff( TimeValue.timeValueMillis(TransformSettings.DEFAULT_RENEW_LOCK_RETRY_DELAY), TransformSettings.DEFAULT_RENEW_LOCK_RETRY_COUNT ) + + var attemptedToIndex = false + var bucketsToTransform = BucketsToTransform(HashSet(), metadata) var lock = acquireLockForScheduledJob(transform, context, backoffPolicy) try { do { @@ -120,7 +129,28 @@ object TransformRunner : return } else -> { - currentMetadata = executeJobIteration(transform, currentMetadata) + val validatedMetadata = validateTransform(transform, currentMetadata) + if (validatedMetadata.status == TransformMetadata.Status.FAILED) { + currentMetadata = validatedMetadata + return + } + if (transform.continuous && (bucketsToTransform.shardsToSearch == null || bucketsToTransform.currentShard != null)) { + // If we have not populated the list of shards to search, do so now + if (bucketsToTransform.shardsToSearch == null) { + // Note the timestamp when we got the shard global checkpoints to the user may know what data is included + newGlobalCheckpointTime = Instant.now() + newGlobalCheckpoints = transformSearchService.getShardsGlobalCheckpoint(transform.sourceIndex) + bucketsToTransform = bucketsToTransform.initializeShardsToSearch( + metadata.shardIDToGlobalCheckpoint, + newGlobalCheckpoints + ) + } + bucketsToTransform = getBucketsToTransformIteration(transform, bucketsToTransform) + currentMetadata = bucketsToTransform.metadata + } else { + currentMetadata = executeTransformIteration(transform, currentMetadata, bucketsToTransform.modifiedBuckets) + attemptedToIndex = true + } // we attempt to renew lock for every loop of transform val renewedLock = renewLockForScheduledJob(context, lock, backoffPolicy) if (renewedLock == null) { @@ -129,7 +159,7 @@ object TransformRunner : lock = renewedLock } } - } while (currentMetadata.afterKey != null) + } while (bucketsToTransform.currentShard != null || currentMetadata.afterKey != null || !attemptedToIndex) } catch (e: Exception) { logger.error("Failed to execute the transform job [${transform.id}] because of exception [${e.localizedMessage}]", e) currentMetadata = currentMetadata.copy( @@ -139,47 +169,103 @@ object TransformRunner : ) } finally { lock?.let { + // Update the global checkpoints only after execution finishes successfully + if (transform.continuous && currentMetadata.status != TransformMetadata.Status.FAILED) { + currentMetadata = currentMetadata.copy( + shardIDToGlobalCheckpoint = newGlobalCheckpoints, + continuousStats = ContinuousTransformStats(newGlobalCheckpointTime, null) + ) + } transformMetadataService.writeMetadata(currentMetadata, true) - logger.info("Disabling the transform job ${transform.id}") - updateTransform(transform.copy(enabled = false, enabledAt = null)) + if (!transform.continuous || currentMetadata.status == TransformMetadata.Status.FAILED) { + logger.info("Disabling the transform job ${transform.id}") + updateTransform(transform.copy(enabled = false, enabledAt = null)) + } releaseLockForScheduledJob(context, it) } } } - private suspend fun executeJobIteration(transform: Transform, metadata: TransformMetadata): TransformMetadata { - val validationResult = withClosableContext( - IndexManagementSecurityContext(transform.id, settings, threadPool.threadContext, transform.user) - ) { + private suspend fun getBucketsToTransformIteration(transform: Transform, bucketsToTransform: BucketsToTransform): BucketsToTransform { + var currentBucketsToTransform = bucketsToTransform + val currentShard = bucketsToTransform.currentShard + + if (currentShard != null) { + val shardLevelModifiedBuckets = withTransformSecurityContext(transform) { + transformSearchService.getShardLevelModifiedBuckets(transform, currentBucketsToTransform.metadata.afterKey, currentShard) + } + currentBucketsToTransform.modifiedBuckets.addAll(shardLevelModifiedBuckets.modifiedBuckets) + val mergedSearchTime = currentBucketsToTransform.metadata.stats.searchTimeInMillis + + shardLevelModifiedBuckets.searchTimeInMillis + currentBucketsToTransform = currentBucketsToTransform.copy( + metadata = currentBucketsToTransform.metadata.copy( + stats = currentBucketsToTransform.metadata.stats.copy( + pagesProcessed = currentBucketsToTransform.metadata.stats.pagesProcessed + 1, + searchTimeInMillis = mergedSearchTime + ), + afterKey = shardLevelModifiedBuckets.afterKey + ), + currentShard = currentShard + ) + } + // If finished with this shard, go to the next + if (currentBucketsToTransform.metadata.afterKey == null) { + val shardsToSearch = currentBucketsToTransform.shardsToSearch + currentBucketsToTransform = if (shardsToSearch?.hasNext() == true) { + currentBucketsToTransform.copy(currentShard = shardsToSearch.next()) + } else { + currentBucketsToTransform.copy(currentShard = null) + } + } + return currentBucketsToTransform + } + + private suspend fun validateTransform(transform: Transform, transformMetadata: TransformMetadata): TransformMetadata { + val validationResult = withTransformSecurityContext(transform) { transformValidator.validate(transform) } - if (validationResult.isValid) { - val transformSearchResult = withClosableContext( - IndexManagementSecurityContext(transform.id, settings, threadPool.threadContext, transform.user) - ) { - transformSearchService.executeCompositeSearch(transform, metadata.afterKey) + return if (!validationResult.isValid) { + val failureMessage = "Failed validation - ${validationResult.issues}" + val failureMetadata = transformMetadata.copy(status = TransformMetadata.Status.FAILED, failureReason = failureMessage) + transformMetadataService.writeMetadata(failureMetadata, true) + } else transformMetadata + } + + /** + * For a continuous transform, we paginate over the set of modified buckets, however, with a histogram grouping and a decimal interval, + * the range query will not precisely specify the modified buckets. As a result, we increase the range for the query and then filter out + * the unintended buckets as part of the composite search step. + */ + private suspend fun executeTransformIteration( + transform: Transform, + metadata: TransformMetadata, + modifiedBuckets: MutableSet> + ): TransformMetadata { + val updatedMetadata = if (!transform.continuous || modifiedBuckets.isNotEmpty()) { + val transformSearchResult = withTransformSecurityContext(transform) { + transformSearchService.executeCompositeSearch(transform, metadata.afterKey, if (transform.continuous) modifiedBuckets else null) } - val indexTimeInMillis = withClosableContext( - IndexManagementSecurityContext(transform.id, settings, threadPool.threadContext, transform.user) - ) { + val indexTimeInMillis = withTransformSecurityContext(transform) { transformIndexer.index(transformSearchResult.docsToIndex) } val afterKey = transformSearchResult.afterKey val stats = transformSearchResult.stats val updatedStats = stats.copy( - indexTimeInMillis = stats.indexTimeInMillis + indexTimeInMillis, documentsIndexed = transformSearchResult.docsToIndex.size.toLong() + pagesProcessed = if (transform.continuous) 0 else stats.pagesProcessed, + indexTimeInMillis = stats.indexTimeInMillis + indexTimeInMillis, + documentsIndexed = transformSearchResult.docsToIndex.size.toLong() ) - val updatedMetadata = metadata.mergeStats(updatedStats).copy( + metadata.mergeStats(updatedStats).copy( afterKey = afterKey, lastUpdatedAt = Instant.now(), - status = if (afterKey == null) TransformMetadata.Status.FINISHED else TransformMetadata.Status.STARTED + status = if (afterKey == null && !transform.continuous) TransformMetadata.Status.FINISHED else TransformMetadata.Status.STARTED ) - return transformMetadataService.writeMetadata(updatedMetadata, true) - } else { - val failureMessage = "Failed validation - ${validationResult.issues}" - val updatedMetadata = metadata.copy(status = TransformMetadata.Status.FAILED, failureReason = failureMessage) - return transformMetadataService.writeMetadata(updatedMetadata, true) - } + } else metadata.copy(lastUpdatedAt = Instant.now(), status = TransformMetadata.Status.STARTED) + return transformMetadataService.writeMetadata(updatedMetadata, true) + } + + private suspend fun withTransformSecurityContext(transform: Transform, block: suspend CoroutineScope.() -> T): T { + return withClosableContext(IndexManagementSecurityContext(transform.id, settings, threadPool.threadContext, transform.user), block) } private suspend fun updateTransform(transform: Transform): Transform { diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformSearchService.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformSearchService.kt index 482e0b299..57090c18c 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformSearchService.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformSearchService.kt @@ -9,18 +9,32 @@ import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.OpenSearchSecurityException import org.opensearch.action.ActionListener +import org.opensearch.action.admin.indices.stats.IndicesStatsAction +import org.opensearch.action.admin.indices.stats.IndicesStatsRequest +import org.opensearch.action.admin.indices.stats.IndicesStatsResponse import org.opensearch.action.bulk.BackoffPolicy import org.opensearch.action.index.IndexRequest import org.opensearch.action.search.SearchRequest import org.opensearch.action.search.SearchResponse import org.opensearch.client.Client +import org.opensearch.cluster.metadata.IndexMetadata import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings import org.opensearch.common.xcontent.XContentType +import org.opensearch.index.Index +import org.opensearch.index.query.BoolQueryBuilder +import org.opensearch.index.query.ExistsQueryBuilder import org.opensearch.index.query.QueryBuilder +import org.opensearch.index.query.QueryBuilders +import org.opensearch.index.query.RangeQueryBuilder +import org.opensearch.index.seqno.SequenceNumbers +import org.opensearch.index.shard.ShardId +import org.opensearch.indexmanagement.common.model.dimension.Dimension import org.opensearch.indexmanagement.opensearchapi.retry import org.opensearch.indexmanagement.opensearchapi.suspendUntil import org.opensearch.indexmanagement.transform.exceptions.TransformSearchServiceException +import org.opensearch.indexmanagement.transform.model.BucketSearchResult +import org.opensearch.indexmanagement.transform.model.ShardNewDocuments import org.opensearch.indexmanagement.transform.model.Transform import org.opensearch.indexmanagement.transform.model.TransformSearchResult import org.opensearch.indexmanagement.transform.model.TransformStats @@ -28,6 +42,7 @@ import org.opensearch.indexmanagement.transform.settings.TransformSettings.Compa import org.opensearch.indexmanagement.transform.settings.TransformSettings.Companion.TRANSFORM_JOB_SEARCH_BACKOFF_MILLIS import org.opensearch.indexmanagement.util.IndexUtils.Companion.ODFE_MAGIC_NULL import org.opensearch.indexmanagement.util.IndexUtils.Companion.hashToFixedSize +import org.opensearch.rest.RestStatus import org.opensearch.search.aggregations.Aggregation import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder @@ -45,7 +60,7 @@ import org.opensearch.transport.RemoteTransportException import kotlin.math.max import kotlin.math.pow -@Suppress("ThrowsCount") +@Suppress("ThrowsCount", "TooManyFunctions") class TransformSearchService( val settings: Settings, val clusterService: ClusterService, @@ -59,14 +74,77 @@ class TransformSearchService( init { clusterService.clusterSettings.addSettingsUpdateConsumer(TRANSFORM_JOB_SEARCH_BACKOFF_MILLIS, TRANSFORM_JOB_SEARCH_BACKOFF_COUNT) { - millis, count -> + millis, count -> backoffPolicy = BackoffPolicy.constantBackoff(millis, count) } } @Suppress("RethrowCaughtException") - suspend fun executeCompositeSearch(transform: Transform, afterKey: Map? = null): TransformSearchResult { - val errorMessage = "Failed to search data in source indices" + suspend fun getShardsGlobalCheckpoint(index: String): Map { + try { + var retryAttempt = 1 + // Retry on standard retry fail statuses plus NOT_FOUND in case a shard routing entry isn't ready yet + val searchResponse: IndicesStatsResponse = backoffPolicy.retry(logger, listOf(RestStatus.NOT_FOUND)) { + val request = IndicesStatsRequest().indices(index).clear() + if (retryAttempt > 1) { + logger.debug(getShardsRetryMessage(retryAttempt)) + } + retryAttempt++ + client.suspendUntil { execute(IndicesStatsAction.INSTANCE, request, it) } + } + if (searchResponse.status == RestStatus.OK) { + return convertIndicesStatsResponse(searchResponse) + } + throw TransformSearchServiceException("$getShardsErrorMessage - ${searchResponse.status}") + } catch (e: TransformSearchServiceException) { + throw e + } catch (e: RemoteTransportException) { + val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception + throw TransformSearchServiceException(getShardsErrorMessage, unwrappedException) + } catch (e: OpenSearchSecurityException) { + throw TransformSearchServiceException("$getShardsErrorMessage - missing required index permissions: ${e.localizedMessage}", e) + } catch (e: Exception) { + throw TransformSearchServiceException(getShardsErrorMessage, e) + } + } + + @Suppress("RethrowCaughtException") + suspend fun getShardLevelModifiedBuckets(transform: Transform, afterKey: Map?, currentShard: ShardNewDocuments): BucketSearchResult { + try { + var retryAttempt = 0 + val searchResponse = backoffPolicy.retry(logger) { + val pageSizeDecay = 2f.pow(retryAttempt++) + client.suspendUntil { listener: ActionListener -> + val pageSize = max(1, transform.pageSize.div(pageSizeDecay.toInt())) + if (retryAttempt > 1) { + logger.debug( + "Attempt [${retryAttempt - 1}] to get modified buckets for transform [${transform.id}]. Attempting " + + "again with reduced page size [$pageSize]" + ) + } + val request = getShardLevelBucketsSearchRequest(transform, afterKey, pageSize, currentShard) + search(request, listener) + } + } + return convertBucketSearchResponse(transform, searchResponse) + } catch (e: TransformSearchServiceException) { + throw e + } catch (e: RemoteTransportException) { + val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception + throw TransformSearchServiceException(modifiedBucketsErrorMessage, unwrappedException) + } catch (e: OpenSearchSecurityException) { + throw TransformSearchServiceException("$modifiedBucketsErrorMessage - missing required index permissions: ${e.localizedMessage}", e) + } catch (e: Exception) { + throw TransformSearchServiceException(modifiedBucketsErrorMessage, e) + } + } + + @Suppress("RethrowCaughtException") + suspend fun executeCompositeSearch( + transform: Transform, + afterKey: Map? = null, + modifiedBuckets: MutableSet>? = null + ): TransformSearchResult { try { var retryAttempt = 0 val searchResponse = backoffPolicy.retry(logger) { @@ -80,32 +158,73 @@ class TransformSearchService( "again with reduced page size [$pageSize]" ) } - val request = getSearchServiceRequest(transform, afterKey, pageSize) + val request = getSearchServiceRequest(transform, afterKey, pageSize, modifiedBuckets) search(request, listener) } } - return convertResponse(transform, searchResponse) + return convertResponse(transform, searchResponse, modifiedBuckets = modifiedBuckets) } catch (e: TransformSearchServiceException) { throw e } catch (e: RemoteTransportException) { val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception - throw TransformSearchServiceException(errorMessage, unwrappedException) + throw TransformSearchServiceException(failedSearchErrorMessage, unwrappedException) } catch (e: OpenSearchSecurityException) { - throw TransformSearchServiceException("$errorMessage - missing required index permissions: ${e.localizedMessage}", e) + throw TransformSearchServiceException("$failedSearchErrorMessage - missing required index permissions: ${e.localizedMessage}", e) } catch (e: Exception) { - throw TransformSearchServiceException(errorMessage, e) + throw TransformSearchServiceException(failedSearchErrorMessage, e) } } companion object { - fun getSearchServiceRequest(transform: Transform, afterKey: Map? = null, pageSize: Int): SearchRequest { + const val failedSearchErrorMessage = "Failed to search data in source indices" + const val modifiedBucketsErrorMessage = "Failed to get the modified buckets in source indices" + const val getShardsErrorMessage = "Failed to get the shards in the source indices" + private fun getShardsRetryMessage(attemptNumber: Int) = "Attempt [$attemptNumber] to get shard global checkpoint numbers" + private fun noTransformGroupErrorMessage(bucketField: String) = "Failed to find a transform group matching the bucket field [$bucketField]" + + fun getSearchServiceRequest( + transform: Transform, + afterKey: Map? = null, + pageSize: Int, + modifiedBuckets: MutableSet>? = null + ): SearchRequest { val sources = mutableListOf>() - transform.groups.forEach { group -> sources.add(group.toSourceBuilder()) } + transform.groups.forEach { group -> sources.add(group.toSourceBuilder().missingBucket(true)) } val aggregationBuilder = CompositeAggregationBuilder(transform.id, sources) .size(pageSize) .subAggregations(transform.aggregations) .apply { afterKey?.let { this.aggregateAfter(it) } } - return getSearchServiceRequest(transform.sourceIndex, transform.dataSelectionQuery, aggregationBuilder) + val query = if (modifiedBuckets == null) { + transform.dataSelectionQuery + } else { + getQueryWithModifiedBuckets(transform.dataSelectionQuery, modifiedBuckets, transform.groups) + } + return getSearchServiceRequest(transform.sourceIndex, query, aggregationBuilder) + } + + private fun getQueryWithModifiedBuckets( + originalQuery: QueryBuilder, + modifiedBuckets: MutableSet>, + groups: List + ): QueryBuilder { + val query: BoolQueryBuilder = QueryBuilders.boolQuery().must(originalQuery).minimumShouldMatch(1) + modifiedBuckets.forEach { bucket -> + val bucketQuery: BoolQueryBuilder = QueryBuilders.boolQuery() + bucket.forEach { group -> + // There should be a transform grouping for each bucket key, if not then throw an error + val transformGroup = groups.find { it.targetField == group.key } + ?: throw TransformSearchServiceException(noTransformGroupErrorMessage(group.key)) + if (group.value as Any? == null) { + val subQuery = ExistsQueryBuilder(transformGroup.sourceField) + bucketQuery.mustNot(subQuery) + } else { + val subQuery = transformGroup.toBucketQuery(group.value) + bucketQuery.filter(subQuery) + } + } + query.should(bucketQuery) + } + return query } private fun getSearchServiceRequest(index: String, query: QueryBuilder, aggregationBuilder: CompositeAggregationBuilder): SearchRequest { @@ -119,15 +238,45 @@ class TransformSearchService( .allowPartialSearchResults(false) } - fun convertResponse(transform: Transform, searchResponse: SearchResponse, waterMarkDocuments: Boolean = true): TransformSearchResult { + fun getShardLevelBucketsSearchRequest( + transform: Transform, + afterKey: Map? = null, + pageSize: Int, + currentShard: ShardNewDocuments + ): SearchRequest { + val rangeQuery = getSeqNoRangeQuery(currentShard.from, currentShard.to) + val query = QueryBuilders.boolQuery().filter(rangeQuery).must(transform.dataSelectionQuery) + val sources = transform.groups.map { it.toSourceBuilder().missingBucket(true) } + val aggregationBuilder = CompositeAggregationBuilder(transform.id, sources) + .size(pageSize) + .apply { afterKey?.let { this.aggregateAfter(it) } } + return getSearchServiceRequest(currentShard.shardId.indexName, query, aggregationBuilder) + .preference("_shards:" + currentShard.shardId.id.toString()) + } + + private fun getSeqNoRangeQuery(from: Long?, to: Long): RangeQueryBuilder { + val rangeQuery = RangeQueryBuilder("_seq_no") + // If to or from is < 0 then the step to get the global checkpoint number failed, and we proceed without bounding the sequence number + if (to >= 0) rangeQuery.to(to, true) + if (from != null && from >= 0) rangeQuery.from(from, false) + return rangeQuery + } + + fun convertResponse( + transform: Transform, + searchResponse: SearchResponse, + waterMarkDocuments: Boolean = true, + modifiedBuckets: MutableSet>? = null + ): TransformSearchResult { val aggs = searchResponse.aggregations.get(transform.id) as CompositeAggregation - val documentsProcessed = aggs.buckets.fold(0L) { sum, it -> sum + it.docCount } + val buckets = if (modifiedBuckets != null) aggs.buckets.filter { modifiedBuckets.contains(it.key) } else aggs.buckets + val documentsProcessed = buckets.fold(0L) { sum, it -> sum + it.docCount } val pagesProcessed = 1L val searchTime = searchResponse.took.millis val stats = TransformStats(pagesProcessed, documentsProcessed, 0, 0, searchTime) val afterKey = aggs.afterKey() val docsToIndex = mutableListOf() - aggs.buckets.forEach { aggregatedBucket -> + buckets.forEach { aggregatedBucket -> val id = transform.id + "#" + aggregatedBucket.key.entries.joinToString(":") { bucket -> bucket.value?.toString() ?: ODFE_MAGIC_NULL } val hashedId = hashToFixedSize(id) @@ -144,6 +293,16 @@ class TransformSearchService( return TransformSearchResult(stats, docsToIndex, afterKey) } + // Gathers and returns from the bucket search response the modified buckets from the query, the afterkey, and the search time + private fun convertBucketSearchResponse( + transform: Transform, + searchResponse: SearchResponse + ): BucketSearchResult { + val aggs = searchResponse.aggregations.get(transform.id) as CompositeAggregation + val modifiedBuckets = aggs.buckets.map { it.key }.toMutableSet() + return BucketSearchResult(modifiedBuckets, aggs.afterKey(), searchResponse.took.millis) + } + private fun getAggregationValue(aggregation: Aggregation): Any { return when (aggregation) { is InternalSum, is InternalMin, is InternalMax, is InternalAvg, is InternalValueCount -> { @@ -165,5 +324,18 @@ class TransformSearchService( ) } } + + fun convertIndicesStatsResponse(response: IndicesStatsResponse): Map { + val shardStats = HashMap() + val shardsToSearch = response.shards.filter { it.shardRouting.primary() && it.shardRouting.active() } + for (shard in shardsToSearch) { + val shardId = shard.shardRouting.shardId() + // Remove uuid as it isn't streamed, so it would break our hashing. We aren't using it anyways + val shardIDNoUUID = ShardId(Index(shardId.index.name, IndexMetadata.INDEX_UUID_NA_VALUE), shardId.id) + // If it is null, we will still run the transform, but without bounding the sequence number + shardStats[shardIDNoUUID] = shard.seqNoStats?.globalCheckpoint ?: SequenceNumbers.UNASSIGNED_SEQ_NO + } + return shardStats + } } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformValidator.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformValidator.kt index 380e94831..bc6c63234 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformValidator.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/TransformValidator.kt @@ -26,7 +26,7 @@ import org.opensearch.monitor.jvm.JvmService import org.opensearch.transport.RemoteTransportException import java.lang.IllegalStateException -@Suppress("SpreadOperator", "ReturnCount") +@Suppress("SpreadOperator", "ReturnCount", "ThrowsCount") class TransformValidator( private val indexNameExpressionResolver: IndexNameExpressionResolver, private val clusterService: ClusterService, @@ -103,14 +103,12 @@ class TransformValidator( fun validateMappingsResponse(index: String, response: GetMappingsResponse, transform: Transform): List { val issues = mutableListOf() val indexTypeMappings = response.mappings[index] - if (indexTypeMappings.isEmpty) { + if (indexTypeMappings == null) { issues.add("Source index [$index] mappings are empty, cannot validate the job.") return issues } - // Starting from 6.0.0 an index can only have one mapping type, but mapping type is still part of the APIs in 7.x, allowing users to - // set a custom mapping type. As a result using first mapping type found instead of _DOC mapping type to validate - val indexMappingSource = indexTypeMappings.first().value.sourceAsMap + val indexMappingSource = indexTypeMappings.sourceAsMap transform.groups.forEach { group -> if (!group.canBeRealizedInMappings(indexMappingSource)) { diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/action/explain/ExplainTransformResponse.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/action/explain/ExplainTransformResponse.kt index 5ff41e4c9..cf3bd3671 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/action/explain/ExplainTransformResponse.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/action/explain/ExplainTransformResponse.kt @@ -14,7 +14,10 @@ import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.indexmanagement.transform.model.ExplainTransform import java.io.IOException -class ExplainTransformResponse(val idsToExplain: Map) : ActionResponse(), ToXContentObject { +class ExplainTransformResponse( + val idsToExplain: Map, + private val failedToExplain: Map +) : ActionResponse(), ToXContentObject { internal fun getIdsToExplain(): Map { return this.idsToExplain @@ -29,7 +32,8 @@ class ExplainTransformResponse(val idsToExplain: Map) idsToExplain[it.readString()] = if (sin.readBoolean()) ExplainTransform(it) else null } idsToExplain.toMap() - } + }, + failedToExplain = sin.readMap({ it.readString() }, { it.readString() }) ) @Throws(IOException::class) @@ -40,6 +44,11 @@ class ExplainTransformResponse(val idsToExplain: Map) out.writeBoolean(metadata != null) metadata?.writeTo(out) } + out.writeMap( + failedToExplain, + { writer, value: String -> writer.writeString(value) }, + { writer, value: String -> writer.writeString(value) } + ) } @Throws(IOException::class) @@ -48,6 +57,9 @@ class ExplainTransformResponse(val idsToExplain: Map) idsToExplain.entries.forEach { (id, explain) -> builder.field(id, explain) } + failedToExplain.entries.forEach { (id, failureReason) -> + builder.field(id, failureReason) + } return builder.endObject() } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/action/explain/TransportExplainTransformAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/action/explain/TransportExplainTransformAction.kt index 4eb94c876..a11aff2e7 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/action/explain/TransportExplainTransformAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/action/explain/TransportExplainTransformAction.kt @@ -5,6 +5,9 @@ package org.opensearch.indexmanagement.transform.action.explain +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper import org.opensearch.ResourceNotFoundException @@ -72,6 +75,7 @@ class TransportExplainTransformAction @Inject constructor( // Instantiate concrete ids to metadata map by removing wildcard matches val idsToExplain: MutableMap = ids.filter { !it.contains("*") } .map { it to null }.toMap(mutableMapOf()) + val failedToExplain: MutableMap = mutableMapOf() val queryBuilder = BoolQueryBuilder().minimumShouldMatch(1).apply { ids.forEach { this.should(WildcardQueryBuilder("${ Transform.TRANSFORM_TYPE}.${Transform.TRANSFORM_ID_FIELD}.keyword", "*$it*")) @@ -87,10 +91,12 @@ class TransportExplainTransformAction @Inject constructor( searchRequest, object : ActionListener { override fun onResponse(response: SearchResponse) { + val metadataIdToTransform: MutableMap = HashMap() try { response.hits.hits.forEach { val transform = contentParser(it.sourceRef).parseWithType(it.id, it.seqNo, it.primaryTerm, Transform.Companion::parse) idsToExplain[transform.id] = ExplainTransform(metadataID = transform.metadataId) + if (transform.metadataId != null) metadataIdToTransform[transform.metadataId] = transform } } catch (e: Exception) { log.error("Failed to parse explain response", e) @@ -105,19 +111,30 @@ class TransportExplainTransformAction @Inject constructor( metadataSearchRequest, object : ActionListener { override fun onResponse(response: SearchResponse) { - try { + CoroutineScope(Dispatchers.IO).launch { response.hits.hits.forEach { - val metadata = contentParser(it.sourceRef) - .parseWithType(it.id, it.seqNo, it.primaryTerm, TransformMetadata.Companion::parse) - idsToExplain.computeIfPresent(metadata.transformId) { _, explainTransform -> - explainTransform.copy(metadata = metadata) + try { + val metadata = contentParser(it.sourceRef) + .parseWithType(it.id, it.seqNo, it.primaryTerm, TransformMetadata.Companion::parse) + + val transform = metadataIdToTransform[metadata.id] + // Only add continuous stats for continuous transforms which have not failed + if (transform?.continuous == true && metadata.status != TransformMetadata.Status.FAILED) { + addContinuousStats(transform, metadata) + } else { + idsToExplain.computeIfPresent(metadata.transformId) { _, explainTransform -> + // Don't provide shardIDToGlobalCheckpoint for a failed or non-continuous transform + explainTransform.copy(metadata = metadata.copy(shardIDToGlobalCheckpoint = null)) + } + } + } catch (e: Exception) { + log.error("Failed to parse transform [${it.id}] metadata", e) + idsToExplain.remove(it.id) + failedToExplain[it.id] = + "Failed to parse transform metadata - ${e.message}" } } - actionListener.onResponse(ExplainTransformResponse(idsToExplain.toMap())) - } catch (e: Exception) { - log.error("Failed to parse transform metadata", e) - actionListener.onFailure(e) - return + actionListener.onResponse(ExplainTransformResponse(idsToExplain.toMap(), failedToExplain)) } } @@ -129,6 +146,25 @@ class TransportExplainTransformAction @Inject constructor( else -> actionListener.onFailure(e) } } + + private suspend fun addContinuousStats(transform: Transform, metadata: TransformMetadata) { + val continuousStats = transform.getContinuousStats(client, metadata) + if (continuousStats == null) { + log.error("Failed to get continuous transform stats for transform [${transform.id}]") + idsToExplain.remove(transform.id) + failedToExplain[transform.id] = + "Failed to get continuous transform stats" + } else { + idsToExplain.computeIfPresent(metadata.transformId) { _, explainTransform -> + explainTransform.copy( + metadata = metadata.copy( + shardIDToGlobalCheckpoint = null, + continuousStats = continuousStats + ) + ) + } + } + } } ) } @@ -137,8 +173,9 @@ class TransportExplainTransformAction @Inject constructor( log.error("Failed to search for transforms", e) when (e) { is ResourceNotFoundException -> { - val nonWildcardIds = ids.filter { !it.contains("*") }.map { it to null }.toMap(mutableMapOf()) - actionListener.onResponse(ExplainTransformResponse(nonWildcardIds)) + val failureReason = "Failed to search transform metadata" + val nonWildcardIds = ids.filter { !it.contains("*") }.map { it to failureReason }.toMap(mutableMapOf()) + actionListener.onResponse(ExplainTransformResponse(mapOf(), nonWildcardIds)) } is RemoteTransportException -> actionListener.onFailure(ExceptionsHelper.unwrapCause(e) as java.lang.Exception) else -> actionListener.onFailure(e) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/action/index/TransportIndexTransformAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/action/index/TransportIndexTransformAction.kt index 32692ebb8..373530386 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/action/index/TransportIndexTransformAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/action/index/TransportIndexTransformAction.kt @@ -147,6 +147,7 @@ class TransportIndexTransformAction @Inject constructor( if (transform.groups != newTransform.groups) modified.add(Transform.GROUPS_FIELD) if (transform.aggregations != newTransform.aggregations) modified.add(Transform.AGGREGATIONS_FIELD) if (transform.roles != newTransform.roles) modified.add(Transform.ROLES_FIELD) + if (transform.continuous != newTransform.continuous) modified.add(Transform.CONTINUOUS_FIELD) return modified.toList() } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/model/ContinuousTransformStats.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/model/ContinuousTransformStats.kt new file mode 100644 index 000000000..f96770b35 --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/model/ContinuousTransformStats.kt @@ -0,0 +1,74 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.transform.model + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.io.stream.Writeable +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.ToXContentObject +import org.opensearch.common.xcontent.XContentBuilder +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils +import org.opensearch.indexmanagement.opensearchapi.instant +import java.io.IOException +import java.time.Instant + +data class ContinuousTransformStats( + val lastTimestamp: Instant?, + val documentsBehind: Map? +) : ToXContentObject, Writeable { + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + lastTimestamp = if (sin.readBoolean()) sin.readInstant() else null, + documentsBehind = if (sin.readBoolean()) sin.readMap({ it.readString() }, { it.readLong() }) else null + ) + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject() + if (lastTimestamp != null) builder.timeField(LAST_TIMESTAMP_FIELD, LAST_TIMESTAMP_FIELD_IN_MILLIS, lastTimestamp.toEpochMilli()) + if (documentsBehind != null) builder.field(DOCUMENTS_BEHIND_FIELD, documentsBehind) + return builder.endObject() + } + + override fun writeTo(out: StreamOutput) { + out.writeBoolean(lastTimestamp != null) + lastTimestamp?.let { out.writeInstant(it) } + out.writeBoolean(documentsBehind != null) + documentsBehind?.let { out.writeMap(it, { writer, k -> writer.writeString(k) }, { writer, v -> writer.writeLong(v) }) } + } + + companion object { + private const val LAST_TIMESTAMP_FIELD = "last_timestamp" + private const val LAST_TIMESTAMP_FIELD_IN_MILLIS = "last_timestamp_in_millis" + private const val DOCUMENTS_BEHIND_FIELD = "documents_behind" + + @Suppress("ComplexMethod, LongMethod") + @JvmStatic + @Throws(IOException::class) + fun parse(xcp: XContentParser): ContinuousTransformStats { + var lastTimestamp: Instant? = null + var documentsBehind: Map? = null + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + LAST_TIMESTAMP_FIELD -> lastTimestamp = xcp.instant() + DOCUMENTS_BEHIND_FIELD -> documentsBehind = xcp.map({ HashMap() }, { parser -> parser.longValue() }) + } + } + + return ContinuousTransformStats( + lastTimestamp = lastTimestamp, + documentsBehind = documentsBehind, + ) + } + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/model/Transform.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/model/Transform.kt index ce815de84..070cf4f5a 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/model/Transform.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/model/Transform.kt @@ -5,6 +5,10 @@ package org.opensearch.indexmanagement.transform.model +import org.opensearch.action.admin.indices.stats.IndicesStatsAction +import org.opensearch.action.admin.indices.stats.IndicesStatsRequest +import org.opensearch.action.admin.indices.stats.IndicesStatsResponse +import org.opensearch.client.Client import org.opensearch.common.bytes.BytesReference import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.io.stream.StreamOutput @@ -22,6 +26,7 @@ import org.opensearch.index.query.AbstractQueryBuilder import org.opensearch.index.query.MatchAllQueryBuilder import org.opensearch.index.query.QueryBuilder import org.opensearch.index.seqno.SequenceNumbers +import org.opensearch.index.shard.ShardId import org.opensearch.indexmanagement.common.model.dimension.DateHistogram import org.opensearch.indexmanagement.common.model.dimension.Dimension import org.opensearch.indexmanagement.common.model.dimension.Histogram @@ -31,16 +36,20 @@ import org.opensearch.indexmanagement.indexstatemanagement.util.WITH_USER import org.opensearch.indexmanagement.opensearchapi.instant import org.opensearch.indexmanagement.opensearchapi.optionalTimeField import org.opensearch.indexmanagement.opensearchapi.optionalUserField +import org.opensearch.indexmanagement.opensearchapi.suspendUntil +import org.opensearch.indexmanagement.transform.TransformSearchService import org.opensearch.indexmanagement.util.IndexUtils import org.opensearch.jobscheduler.spi.ScheduledJobParameter import org.opensearch.jobscheduler.spi.schedule.CronSchedule import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule import org.opensearch.jobscheduler.spi.schedule.Schedule import org.opensearch.jobscheduler.spi.schedule.ScheduleParser +import org.opensearch.rest.RestStatus import org.opensearch.search.aggregations.AggregatorFactories import java.io.IOException import java.time.Instant +@Suppress("TooManyFunctions") data class Transform( val id: String = NO_ID, val seqNo: Long = SequenceNumbers.UNASSIGNED_SEQ_NO, @@ -59,6 +68,7 @@ data class Transform( val pageSize: Int, val groups: List, val aggregations: AggregatorFactories.Builder = AggregatorFactories.builder(), + val continuous: Boolean = false, val user: User? = null ) : ScheduledJobParameter, Writeable { @@ -76,7 +86,11 @@ data class Transform( } require(groups.isNotEmpty()) { "Groupings are Empty" } require(sourceIndex != targetIndex) { "Source and target indices cannot be the same" } - require(pageSize in MINIMUM_PAGE_SIZE..MAXIMUM_PAGE_SIZE) { "Page size must be between 1 and 10,000" } + if (continuous) { + require(pageSize in MINIMUM_PAGE_SIZE..MAXIMUM_PAGE_SIZE_CONTINUOUS) { "Page size must be between 1 and 1,000" } + } else { + require(pageSize in MINIMUM_PAGE_SIZE..MAXIMUM_PAGE_SIZE) { "Page size must be between 1 and 10,000" } + } } override fun getName() = id @@ -108,6 +122,7 @@ data class Transform( .field(PAGE_SIZE_FIELD, pageSize) .field(GROUPS_FIELD, groups.toTypedArray()) .field(AGGREGATIONS_FIELD, aggregations) + .field(CONTINUOUS_FIELD, continuous) if (params.paramAsBoolean(WITH_USER, true)) builder.optionalUserField(USER_FIELD, user) if (params.paramAsBoolean(WITH_TYPE, true)) builder.endObject() builder.endObject() @@ -145,6 +160,7 @@ data class Transform( } } out.writeOptionalWriteable(aggregations) + out.writeBoolean(continuous) out.writeBoolean(user != null) user?.writeTo(out) } @@ -162,6 +178,41 @@ data class Transform( } } + suspend fun getContinuousStats(client: Client, metadata: TransformMetadata): ContinuousTransformStats? { + val indicesStatsRequest = IndicesStatsRequest().indices(sourceIndex).clear() + val response: IndicesStatsResponse = client.suspendUntil { execute(IndicesStatsAction.INSTANCE, indicesStatsRequest, it) } + val shardIDsToGlobalCheckpoint = if (response.status == RestStatus.OK) { + TransformSearchService.convertIndicesStatsResponse(response) + } else return null + return ContinuousTransformStats( + metadata.continuousStats?.lastTimestamp, + getDocumentsBehind( + metadata.shardIDToGlobalCheckpoint, + shardIDsToGlobalCheckpoint + ) + ) + } + + private fun getDocumentsBehind( + oldShardIDsToGlobalCheckpoint: Map?, + newShardIDsToGlobalCheckpoint: Map? + ): MutableMap { + val documentsBehind: MutableMap = HashMap() + if (newShardIDsToGlobalCheckpoint == null) { + return documentsBehind + } + newShardIDsToGlobalCheckpoint.forEach { (shardID, globalCheckpoint) -> + val indexName = shardID.indexName + val newGlobalCheckpoint = java.lang.Long.max(0, globalCheckpoint) + // global checkpoint may be -1 or -2 if not initialized, just set to 0 in those cases + val oldGlobalCheckpoint = java.lang.Long.max(0, oldShardIDsToGlobalCheckpoint?.get(shardID) ?: 0) + val localDocsBehind = newGlobalCheckpoint - oldGlobalCheckpoint + documentsBehind[indexName] = (documentsBehind[indexName] ?: 0) + localDocsBehind + } + + return documentsBehind + } + @Throws(IOException::class) constructor(sin: StreamInput) : this( id = sin.readString(), @@ -200,6 +251,7 @@ data class Transform( dimensionList.toList() }, aggregations = requireNotNull(sin.readOptionalWriteable { AggregatorFactories.Builder(it) }) { "Aggregations cannot be null" }, + continuous = sin.readBoolean(), user = if (sin.readBoolean()) { User(sin) } else null @@ -231,9 +283,11 @@ data class Transform( const val SCHEMA_VERSION_FIELD = "schema_version" const val MINIMUM_PAGE_SIZE = 1 const val MAXIMUM_PAGE_SIZE = 10_000 + const val MAXIMUM_PAGE_SIZE_CONTINUOUS = 1_000 const val MINIMUM_JOB_INTERVAL = 1 const val TRANSFORM_DOC_ID_FIELD = "$TRANSFORM_TYPE._id" const val TRANSFORM_DOC_COUNT_FIELD = "$TRANSFORM_TYPE._doc_count" + const val CONTINUOUS_FIELD = "continuous" const val USER_FIELD = "user" @Suppress("ComplexMethod", "LongMethod") @@ -258,6 +312,7 @@ data class Transform( var pageSize: Int? = null val groups = mutableListOf() var aggregations: AggregatorFactories.Builder = AggregatorFactories.builder() + var continuous = false var user: User? = null ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) @@ -305,6 +360,7 @@ data class Transform( } } AGGREGATIONS_FIELD -> aggregations = AggregatorFactories.parseAggregators(xcp) + CONTINUOUS_FIELD -> continuous = xcp.booleanValue() USER_FIELD -> { user = if (xcp.currentToken() == Token.VALUE_NULL) null else User.parse(xcp) } @@ -346,6 +402,7 @@ data class Transform( pageSize = requireNotNull(pageSize) { "Transform page size is null" }, groups = groups, aggregations = aggregations, + continuous = continuous, user = user ) } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/model/TransformMetadata.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/model/TransformMetadata.kt index ce4dfa44f..6f96cbd9b 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/model/TransformMetadata.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/model/TransformMetadata.kt @@ -14,6 +14,7 @@ import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken import org.opensearch.index.seqno.SequenceNumbers +import org.opensearch.index.shard.ShardId import org.opensearch.indexmanagement.indexstatemanagement.util.WITH_TYPE import org.opensearch.indexmanagement.opensearchapi.instant import org.opensearch.indexmanagement.opensearchapi.optionalTimeField @@ -30,7 +31,9 @@ data class TransformMetadata( val lastUpdatedAt: Instant, val status: Status, val failureReason: String? = null, - val stats: TransformStats + val stats: TransformStats, + val shardIDToGlobalCheckpoint: Map? = null, + val continuousStats: ContinuousTransformStats? = null ) : ToXContentObject, Writeable { enum class Status(val type: String) { @@ -55,7 +58,9 @@ data class TransformMetadata( lastUpdatedAt = sin.readInstant(), status = sin.readEnum(Status::class.java), failureReason = sin.readOptionalString(), - stats = TransformStats(sin) + stats = TransformStats(sin), + shardIDToGlobalCheckpoint = if (sin.readBoolean()) sin.readMap({ ShardId(it) }, { it.readLong() }) else null, + continuousStats = if (sin.readBoolean()) ContinuousTransformStats(sin) else null ) override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { @@ -68,7 +73,10 @@ data class TransformMetadata( builder.field(STATUS_FIELD, status.type) builder.field(FAILURE_REASON, failureReason) builder.field(STATS_FIELD, stats) - + if (shardIDToGlobalCheckpoint != null) { + builder.field(SHARD_ID_TO_GLOBAL_CHECKPOINT_FIELD, shardIDToGlobalCheckpoint.mapKeys { it.key.toString() }) + } + if (continuousStats != null) builder.field(CONTINUOUS_STATS_FIELD, continuousStats) if (params.paramAsBoolean(WITH_TYPE, true)) builder.endObject() return builder.endObject() } @@ -84,6 +92,10 @@ data class TransformMetadata( out.writeEnum(status) out.writeOptionalString(failureReason) stats.writeTo(out) + out.writeBoolean(shardIDToGlobalCheckpoint != null) + shardIDToGlobalCheckpoint?.let { out.writeMap(it, { writer, k -> k.writeTo(writer) }, { writer, v -> writer.writeLong(v) }) } + out.writeBoolean(continuousStats != null) + continuousStats?.let { it.writeTo(out) } } fun mergeStats(stats: TransformStats): TransformMetadata { @@ -105,6 +117,8 @@ data class TransformMetadata( const val LAST_UPDATED_AT_FIELD = "last_updated_at" const val STATUS_FIELD = "status" const val STATS_FIELD = "stats" + const val SHARD_ID_TO_GLOBAL_CHECKPOINT_FIELD = "shard_id_to_global_checkpoint" + const val CONTINUOUS_STATS_FIELD = "continuous_stats" const val FAILURE_REASON = "failure_reason" @Suppress("ComplexMethod", "LongMethod") @@ -122,6 +136,8 @@ data class TransformMetadata( var status: Status? = null var failureReason: String? = null var stats: TransformStats? = null + var shardIDToGlobalCheckpoint: Map? = null + var continuousStats: ContinuousTransformStats? = null ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { @@ -132,9 +148,13 @@ data class TransformMetadata( TRANSFORM_ID_FIELD -> transformId = xcp.text() AFTER_KEY_FIELD -> afterkey = xcp.map() LAST_UPDATED_AT_FIELD -> lastUpdatedAt = xcp.instant() - STATUS_FIELD -> status = Status.valueOf(xcp.text().toUpperCase(Locale.ROOT)) + STATUS_FIELD -> status = Status.valueOf(xcp.text().uppercase(Locale.ROOT)) FAILURE_REASON -> failureReason = xcp.textOrNull() STATS_FIELD -> stats = TransformStats.parse(xcp) + SHARD_ID_TO_GLOBAL_CHECKPOINT_FIELD -> + shardIDToGlobalCheckpoint = xcp.map({ HashMap() }, { parser -> parser.longValue() }) + .mapKeys { ShardId.fromString(it.key) } + CONTINUOUS_STATS_FIELD -> continuousStats = ContinuousTransformStats.parse(xcp) } } @@ -147,7 +167,9 @@ data class TransformMetadata( lastUpdatedAt = requireNotNull(lastUpdatedAt) { "Last updated time must not be null" }, status = requireNotNull(status) { "Status must not be null" }, failureReason = failureReason, - stats = requireNotNull(stats) { "Stats must not be null" } + stats = requireNotNull(stats) { "Stats must not be null" }, + shardIDToGlobalCheckpoint = shardIDToGlobalCheckpoint, + continuousStats = continuousStats ) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/model/TransformSearchResult.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/model/TransformSearchResult.kt index 86636bb17..a920bccd5 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/model/TransformSearchResult.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/model/TransformSearchResult.kt @@ -6,5 +6,44 @@ package org.opensearch.indexmanagement.transform.model import org.opensearch.action.index.IndexRequest +import org.opensearch.index.shard.ShardId data class TransformSearchResult(val stats: TransformStats, val docsToIndex: List, val afterKey: Map? = null) + +data class BucketsToTransform( + val modifiedBuckets: MutableSet>, + val metadata: TransformMetadata, + val shardsToSearch: Iterator? = null, + val currentShard: ShardNewDocuments? = null, +) + +fun BucketsToTransform.initializeShardsToSearch( + originalGlobalCheckpoints: Map?, + currentShardIdToGlobalCheckpoint: Map +): BucketsToTransform { + val shardsToSearch = getShardsToSearch(originalGlobalCheckpoints, currentShardIdToGlobalCheckpoint).iterator() + return this.copy( + shardsToSearch = shardsToSearch, + currentShard = if (shardsToSearch.hasNext()) shardsToSearch.next() else null + ) +} + +// Processes through the old and new maps of sequence numbers to generate a list of objects with the shardId and the seq numbers to search +private fun getShardsToSearch(oldShardIDToMaxSeqNo: Map?, newShardIDToMaxSeqNo: Map): List { + val shardsToSearch: MutableList = ArrayList() + newShardIDToMaxSeqNo.forEach { (shardId, currentMaxSeqNo) -> + // if there are no seq number records, or no records for this shard, or the shard seq number is greater than the record, search it + if ((oldShardIDToMaxSeqNo == null) || oldShardIDToMaxSeqNo[shardId].let { it == null || currentMaxSeqNo > it }) { + shardsToSearch.add(ShardNewDocuments(shardId, oldShardIDToMaxSeqNo?.get(shardId), currentMaxSeqNo)) + } + } + return shardsToSearch +} + +data class BucketSearchResult( + val modifiedBuckets: MutableSet>, + val afterKey: Map? = null, + val searchTimeInMillis: Long = 0 +) + +data class ShardNewDocuments(val shardId: ShardId, val from: Long?, val to: Long) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/util/IndexUtils.kt b/src/main/kotlin/org/opensearch/indexmanagement/util/IndexUtils.kt index 136d3a900..c075bf2b1 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/util/IndexUtils.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/util/IndexUtils.kt @@ -20,7 +20,6 @@ import org.opensearch.common.xcontent.XContentParser.Token import org.opensearch.common.xcontent.XContentType import org.opensearch.indexmanagement.IndexManagementIndices import org.opensearch.indexmanagement.IndexManagementPlugin -import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import java.nio.ByteBuffer import java.util.Base64 @@ -134,7 +133,7 @@ class IndexUtils { ) { if (clusterState.metadata.indices.containsKey(index)) { if (shouldUpdateIndex(clusterState.metadata.indices[index], schemaVersion)) { - val putMappingRequest: PutMappingRequest = PutMappingRequest(index).type(_DOC).source(mapping, XContentType.JSON) + val putMappingRequest: PutMappingRequest = PutMappingRequest(index).source(mapping, XContentType.JSON) client.putMapping(putMappingRequest, actionListener) } else { actionListener.onResponse(AcknowledgedResponse(true)) @@ -167,7 +166,7 @@ class IndexUtils { } else { if (shouldUpdateIndex(writeIndex, schemaVersion)) { val putMappingRequest: PutMappingRequest = PutMappingRequest(writeIndex.index.name) - .type(_DOC).source(mapping, XContentType.JSON) + .source(mapping, XContentType.JSON) client.putMapping(putMappingRequest, actionListener) } else { actionListener.onResponse(AcknowledgedResponse(true)) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/util/RestHandlerUtils.kt b/src/main/kotlin/org/opensearch/indexmanagement/util/RestHandlerUtils.kt index 1400ea405..af4d39af2 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/util/RestHandlerUtils.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/util/RestHandlerUtils.kt @@ -6,7 +6,6 @@ @file:Suppress("TopLevelPropertyNaming", "MatchingDeclarationName") package org.opensearch.indexmanagement.util -const val _DOC = "_doc" const val _ID = "_id" const val NO_ID = "" const val _VERSION = "_version" diff --git a/src/main/resources/mappings/opendistro-ism-config.json b/src/main/resources/mappings/opendistro-ism-config.json index 2144b3ce3..074257bf8 100644 --- a/src/main/resources/mappings/opendistro-ism-config.json +++ b/src/main/resources/mappings/opendistro-ism-config.json @@ -429,6 +429,10 @@ } } } + }, + "custom": { + "enabled": false, + "type": "object" } } }, @@ -658,6 +662,10 @@ "rolled_over": { "type": "boolean" }, + "index_creation_date": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, "transition_to": { "type": "text", "fields": { @@ -1211,6 +1219,9 @@ } } } + }, + "continuous": { + "type": "boolean" } } }, @@ -1242,6 +1253,14 @@ "stats": { "type": "object", "enabled": false + }, + "continuous_stats": { + "type": "object", + "enabled": false + }, + "shard_id_to_global_checkpoint": { + "type": "object", + "enabled": false } } } diff --git a/src/main/resources/mappings/opendistro-ism-history.json b/src/main/resources/mappings/opendistro-ism-history.json index 04d3c46d8..44c7ab896 100644 --- a/src/main/resources/mappings/opendistro-ism-history.json +++ b/src/main/resources/mappings/opendistro-ism-history.json @@ -1,6 +1,6 @@ { "_meta" : { - "schema_version": 3 + "schema_version": 4 }, "dynamic": "strict", "properties": { @@ -37,6 +37,10 @@ "rolled_over": { "type": "boolean" }, + "index_creation_date": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, "transition_to": { "type": "keyword" }, diff --git a/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt index 3ea0b44a2..3cc3a1bba 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt @@ -15,11 +15,11 @@ import org.opensearch.client.RequestOptions import org.opensearch.client.Response import org.opensearch.client.RestClient import org.opensearch.common.Strings +import org.opensearch.common.io.PathUtils import org.opensearch.common.settings.Settings import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_HIDDEN import org.opensearch.rest.RestStatus import java.nio.file.Files -import java.nio.file.Path import javax.management.MBeanServerInvocationHandler import javax.management.ObjectName import javax.management.remote.JMXConnectorFactory @@ -28,7 +28,7 @@ import javax.management.remote.JMXServiceURL abstract class IndexManagementRestTestCase : ODFERestTestCase() { val configSchemaVersion = 13 - val historySchemaVersion = 3 + val historySchemaVersion = 4 // Having issues with tests leaking into other tests and mappings being incorrect and they are not caught by any pending task wait check as // they do not go through the pending task queue. Ideally this should probably be written in a way to wait for the @@ -171,7 +171,7 @@ abstract class IndexManagementRestTestCase : ODFERestTestCase() { false ) proxy.getExecutionData(false)?.let { - val path = Path.of("$jacocoBuildPath/integTest.exec") + val path = PathUtils.get("$jacocoBuildPath/integTest.exec") Files.write(path, it) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/common/model/dimension/DateHistogramTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/common/model/dimension/DateHistogramTests.kt new file mode 100644 index 000000000..e2434f327 --- /dev/null +++ b/src/test/kotlin/org/opensearch/indexmanagement/common/model/dimension/DateHistogramTests.kt @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.common.model.dimension + +import org.junit.Assert +import org.opensearch.index.query.RangeQueryBuilder +import org.opensearch.indexmanagement.randomInstant +import org.opensearch.indexmanagement.rollup.randomCalendarDateHistogram +import org.opensearch.indexmanagement.rollup.randomDateHistogram +import org.opensearch.indexmanagement.rollup.randomFixedDateHistogram +import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval +import org.opensearch.test.OpenSearchTestCase +import kotlin.test.assertFailsWith + +class DateHistogramTests : OpenSearchTestCase() { + fun `test fixed date histogram to bucket query has correct values`() { + val randomTime = randomLong() + val dateHistogram = randomFixedDateHistogram() + val dateHistogramInterval = DateHistogramInterval(dateHistogram.fixedInterval).estimateMillis() + val bucketQuery = dateHistogram.toBucketQuery(randomTime) as RangeQueryBuilder + + assertEquals("Date histogram bucket query did not contain the correct timezone", dateHistogram.timezone.toString(), bucketQuery.timeZone()) + assertEquals("Date histogram bucket query did not contain the correct interval", dateHistogramInterval, bucketQuery.to() as Long - bucketQuery.from() as Long) + assertEquals("Date histogram bucket query did not contain the correct bucket start time", randomTime, bucketQuery.from() as Long) + assertEquals("Date histogram bucket query did not contain the correct format", "epoch_millis", bucketQuery.format()) + assertEquals("Date histogram bucket query did not contain the correct field name", bucketQuery.fieldName(), dateHistogram.sourceField) + Assert.assertTrue("Date histogram bucket query should include the lower bounds", bucketQuery.includeLower()) + Assert.assertFalse("Date histogram bucket query should not include the upper bounds", bucketQuery.includeUpper()) + } + + fun `test calendar histogram to bucket query has correct values`() { + val randomTime = randomLong() + val dateHistogram = randomCalendarDateHistogram() + val dateHistogramInterval = DateHistogramInterval(dateHistogram.calendarInterval).estimateMillis() + val bucketQuery = dateHistogram.toBucketQuery(randomTime) as RangeQueryBuilder + + assertEquals("Date histogram bucket query did not contain the correct timezone", dateHistogram.timezone.toString(), bucketQuery.timeZone()) + assertEquals("Date histogram bucket query did not contain the correct interval", dateHistogramInterval, bucketQuery.to() as Long - bucketQuery.from() as Long) + assertEquals("Date histogram bucket query did not contain the correct bucket start time", randomTime, bucketQuery.from() as Long) + assertEquals("Date histogram bucket query did not contain the correct format", "epoch_millis", bucketQuery.format()) + assertEquals("Date histogram bucket query did not contain the correct field name", bucketQuery.fieldName(), dateHistogram.sourceField) + Assert.assertTrue("Date histogram bucket query should include the lower bounds", bucketQuery.includeLower()) + Assert.assertFalse("Date histogram bucket query should not include the upper bounds", bucketQuery.includeUpper()) + } + + fun `test date histogram to bucket query fails with wrong bucket key type`() { + assertFailsWith(IllegalArgumentException::class, "Expected IllegalArgumentException with type Int when Long is expected") { + randomDateHistogram().toBucketQuery(randomInt()) + } + assertFailsWith(IllegalArgumentException::class, "Expected IllegalArgumentException with type Instant when Long is expected") { + randomDateHistogram().toBucketQuery(randomInstant()) + } + assertFailsWith(IllegalArgumentException::class, "Expected IllegalArgumentException with type Double when Long is expected") { + randomDateHistogram().toBucketQuery(randomDouble()) + } + } +} diff --git a/src/test/kotlin/org/opensearch/indexmanagement/common/model/dimension/HistogramTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/common/model/dimension/HistogramTests.kt new file mode 100644 index 000000000..3e6f93dee --- /dev/null +++ b/src/test/kotlin/org/opensearch/indexmanagement/common/model/dimension/HistogramTests.kt @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.common.model.dimension + +import org.junit.Assert +import org.opensearch.index.query.RangeQueryBuilder +import org.opensearch.indexmanagement.rollup.randomHistogram +import org.opensearch.test.OpenSearchTestCase +import kotlin.test.assertFailsWith + +class HistogramTests : OpenSearchTestCase() { + fun `test histogram to bucket query has correct values`() { + val histogram = randomHistogram() + val randomKey = randomDouble() + val bucketQuery = histogram.toBucketQuery(randomKey) as RangeQueryBuilder + + assertEquals("Histogram bucket query did not contain the correct interval", histogram.interval, bucketQuery.to() as Double - bucketQuery.from() as Double, 0.001) + assertEquals("Histogram bucket query did not contain the correct bucket start", randomKey, bucketQuery.from() as Double, 0.0001) + assertEquals("Histogram bucket query did not contain the correct field name", bucketQuery.fieldName(), histogram.sourceField) + Assert.assertTrue("Histogram bucket query should include the lower bounds", bucketQuery.includeLower()) + Assert.assertTrue("Histogram bucket query should include the upper bounds", bucketQuery.includeUpper()) + } + + fun `test histogram to bucket query fails with wrong bucket key type`() { + assertFailsWith(IllegalArgumentException::class, "Expected IllegalArgumentException with type Int when Double is expected") { + randomHistogram().toBucketQuery(randomInt()) + } + assertFailsWith(IllegalArgumentException::class, "Expected IllegalArgumentException with type Long when Double is expected") { + randomHistogram().toBucketQuery(randomLong()) + } + assertFailsWith(IllegalArgumentException::class, "Expected IllegalArgumentException with type Float when Double is expected") { + randomHistogram().toBucketQuery(randomFloat()) + } + assertFailsWith(IllegalArgumentException::class, "Expected IllegalArgumentException with type String when Double is expected") { + randomHistogram().toBucketQuery(randomAlphaOfLengthBetween(1, 10)) + } + } +} diff --git a/src/test/kotlin/org/opensearch/indexmanagement/common/model/dimension/TermsTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/common/model/dimension/TermsTests.kt new file mode 100644 index 000000000..d815ca531 --- /dev/null +++ b/src/test/kotlin/org/opensearch/indexmanagement/common/model/dimension/TermsTests.kt @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.common.model.dimension + +import org.junit.Assert +import org.opensearch.index.query.TermsQueryBuilder +import org.opensearch.indexmanagement.rollup.randomTerms +import org.opensearch.test.OpenSearchTestCase + +class TermsTests : OpenSearchTestCase() { + fun `test terms to bucket query has correct values`() { + val terms = randomTerms() + val randomKey = randomAlphaOfLengthBetween(1, 10) + val bucketQuery = terms.toBucketQuery(randomKey) as TermsQueryBuilder + + Assert.assertTrue("Terms bucket query did not contain the correct key", bucketQuery.values().contains(randomKey)) + assertEquals("Terms bucket query did not contain the correct field name", bucketQuery.fieldName(), terms.sourceField) + } +} diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/IndexEvaluatorTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexMetadataProviderTests.kt similarity index 82% rename from src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/IndexEvaluatorTests.kt rename to src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexMetadataProviderTests.kt index c2f8ac3cf..757eff33f 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/IndexEvaluatorTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexMetadataProviderTests.kt @@ -3,24 +3,28 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.indexmanagement.indexstatemanagement.util +package org.opensearch.indexmanagement.indexstatemanagement import com.nhaarman.mockitokotlin2.doReturn import com.nhaarman.mockitokotlin2.mock import com.nhaarman.mockitokotlin2.whenever import org.junit.Before +import org.opensearch.client.Client import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.ClusterSettings import org.opensearch.common.settings.Settings import org.opensearch.indexmanagement.IndexManagementPlugin import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings +import org.opensearch.indexmanagement.spi.indexstatemanagement.IndexMetadataService import org.opensearch.test.OpenSearchTestCase import org.opensearch.test.rest.OpenSearchRestTestCase -class IndexEvaluatorTests : OpenSearchTestCase() { +class IndexMetadataProviderTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() + private val client: Client = mock() private val settings: Settings = Settings.EMPTY + private val services = mutableMapOf() @Before fun `setup settings`() { @@ -28,7 +32,7 @@ class IndexEvaluatorTests : OpenSearchTestCase() { } fun `test security index and kibana should not be manageable`() { - val indexEvaluator = IndexEvaluator(settings, clusterService) + val indexEvaluator = IndexMetadataProvider(settings, client, clusterService, services) assertTrue("Should not manage security index", indexEvaluator.isUnManageableIndex(".opendistro_security")) assertTrue("Should not manage kibana index", indexEvaluator.isUnManageableIndex(".kibana_1")) assertTrue("Should not manage kibana index", indexEvaluator.isUnManageableIndex(".kibana")) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementIntegTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementIntegTestCase.kt index b8f6c0a52..540049fe1 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementIntegTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementIntegTestCase.kt @@ -29,18 +29,19 @@ import org.opensearch.common.xcontent.XContentType import org.opensearch.common.xcontent.json.JsonXContent import org.opensearch.indexmanagement.IndexManagementPlugin import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.indexstatemanagement.model.Policy -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.PolicyRetryInfoMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StateMetaData import org.opensearch.indexmanagement.indexstatemanagement.resthandler.RestExplainAction import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.transport.action.explain.ExplainAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.explain.TransportExplainAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.TransportUpdateManagedIndexMetaDataAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataAction +import org.opensearch.indexmanagement.indexstatemanagement.util.TOTAL_MANAGED_INDICES import org.opensearch.indexmanagement.makeRequest import org.opensearch.indexmanagement.opensearchapi.parseWithType +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaData import org.opensearch.indexmanagement.waitFor import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule import org.opensearch.plugins.ActionPlugin @@ -70,6 +71,7 @@ abstract class IndexStateManagementIntegTestCase : OpenSearchIntegTestCase() { policyPrimaryTerm = 1, policyCompleted = false, rolledOver = false, + indexCreationDate = null, transitionTo = null, stateMetaData = StateMetaData("ReplicaCountState", 1234), actionMetaData = null, @@ -94,9 +96,10 @@ abstract class IndexStateManagementIntegTestCase : OpenSearchIntegTestCase() { } } - override fun transportClientPlugins(): Collection> { - return listOf(TestPlugin::class.java) - } + // TODO: ...convert into a test REST plugin that allows us to execute the transport action? +// override fun transportClientPlugins(): Collection> { +// return listOf(TestPlugin::class.java) +// } protected fun getIndexMetadata(indexName: String): IndexMetadata { return client().admin().cluster().prepareState() @@ -272,11 +275,12 @@ abstract class IndexStateManagementIntegTestCase : OpenSearchIntegTestCase() { xcp.nextToken(), xcp ) + var totalManagedIndices = 0 while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { xcp.currentName() xcp.nextToken() - - metadata = ManagedIndexMetaData.parse(xcp) + if (xcp.currentName() == TOTAL_MANAGED_INDICES) totalManagedIndices = xcp.intValue() + else metadata = ManagedIndexMetaData.parse(xcp) } return metadata } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index 08b016a40..be64799ff 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -38,14 +38,9 @@ import org.opensearch.indexmanagement.IndexManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy import org.opensearch.indexmanagement.indexstatemanagement.model.ISMTemplate import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.Policy.Companion.POLICY_TYPE import org.opensearch.indexmanagement.indexstatemanagement.model.StateFilter -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.PolicyRetryInfoMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StateMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData import org.opensearch.indexmanagement.indexstatemanagement.resthandler.RestExplainAction import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.util.FAILED_INDICES @@ -57,6 +52,11 @@ import org.opensearch.indexmanagement.makeRequest import org.opensearch.indexmanagement.opensearchapi.parseWithType import org.opensearch.indexmanagement.rollup.model.Rollup import org.opensearch.indexmanagement.rollup.model.RollupMetadata +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.indexmanagement.util._ID import org.opensearch.indexmanagement.util._PRIMARY_TERM import org.opensearch.indexmanagement.util._SEQ_NO diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/MetadataRegressionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/MetadataRegressionIT.kt index 979f97e6e..3874f79b3 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/MetadataRegressionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/MetadataRegressionIT.kt @@ -14,10 +14,12 @@ import org.opensearch.cluster.metadata.IndexMetadata import org.opensearch.common.settings.Settings import org.opensearch.index.Index import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX +import org.opensearch.indexmanagement.indexstatemanagement.action.ReplicaCountAction import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReplicaCountActionConfig import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings +import org.opensearch.indexmanagement.indexstatemanagement.transport.action.explain.TransportExplainAction.Companion.METADATA_CORRUPT_WARNING +import org.opensearch.indexmanagement.indexstatemanagement.transport.action.explain.TransportExplainAction.Companion.METADATA_MOVING_WARNING import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.updateindexmetadata.UpdateManagedIndexMetaDataRequest import org.opensearch.indexmanagement.waitFor @@ -49,8 +51,7 @@ class MetadataRegressionIT : IndexStateManagementIntegTestCase() { fun `test move metadata service`() { val indexName = "${testIndexName}_index_1" val policyID = "${testIndexName}_testPolicyName_1" - val actionConfig = ReplicaCountActionConfig(10, 0) - + val actionConfig = ReplicaCountAction(10, 0) val states = listOf(State(name = "ReplicaCountState", actions = listOf(actionConfig), transitions = listOf())) val policy = Policy( id = policyID, @@ -89,7 +90,7 @@ class MetadataRegressionIT : IndexStateManagementIntegTestCase() { waitFor { assertEquals( - "Metadata is pending migration", + METADATA_MOVING_WARNING, getExplainManagedIndexMetaData(indexName).info?.get("message") ) } @@ -131,7 +132,7 @@ class MetadataRegressionIT : IndexStateManagementIntegTestCase() { val indexName = "${testIndexName}_index_2" val policyID = "${testIndexName}_testPolicyName_2" - val actionConfig = ReplicaCountActionConfig(10, 0) + val actionConfig = ReplicaCountAction(10, 0) val states = listOf(State(name = "ReplicaCountState", actions = listOf(actionConfig), transitions = listOf())) val policy = Policy( id = policyID, @@ -174,7 +175,7 @@ class MetadataRegressionIT : IndexStateManagementIntegTestCase() { waitFor { assertEquals( - "Metadata is pending migration", + METADATA_MOVING_WARNING, getExplainManagedIndexMetaData(indexName).info?.get("message") ) } @@ -200,6 +201,57 @@ class MetadataRegressionIT : IndexStateManagementIntegTestCase() { } } + fun `test clean corrupt metadata`() { + val indexName = "${testIndexName}_index_3" + val policyID = "${testIndexName}_testPolicyName_3" + val action = ReplicaCountAction(10, 0) + val states = listOf(State(name = "ReplicaCountState", actions = listOf(action), transitions = listOf())) + val policy = Policy( + id = policyID, + description = "$testIndexName description", + schemaVersion = 1L, + lastUpdatedTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), + errorNotification = randomErrorNotification(), + defaultState = states[0].name, + states = states + ) + + createPolicy(policy, policyID) + createIndex(indexName) + + // create a job + addPolicyToIndex(indexName, policyID) + + // put some metadata into cluster state + val indexMetadata = getIndexMetadata(indexName) + metadataToClusterState = metadataToClusterState.copy( + index = indexName, + indexUuid = "randomindexuuid", + policyID = policyID + ) + val request = UpdateManagedIndexMetaDataRequest( + indicesToAddManagedIndexMetaDataTo = listOf( + Pair(Index(indexName, indexMetadata.indexUUID), metadataToClusterState) + ) + ) + client().execute(UpdateManagedIndexMetaDataAction.INSTANCE, request).get() + logger.info("check if metadata is saved in cluster state: ${getIndexMetadata(indexName).getCustomData("managed_index_metadata")}") + + waitFor { + assertEquals( + METADATA_CORRUPT_WARNING, + getExplainManagedIndexMetaData(indexName).info?.get("message") + ) + } + + waitFor(Instant.ofEpochSecond(120)) { + assertEquals(null, getExplainManagedIndexMetaData(indexName).info?.get("message")) + assertEquals(null, getIndexMetadata(indexName).getCustomData("managed_index_metadata")) + } + + logger.info("corrupt metadata has been cleaned") + } + fun `test new node skip execution when old node exist in cluster`() { Assume.assumeTrue(isMixedNodeRegressionTest) @@ -216,7 +268,7 @@ class MetadataRegressionIT : IndexStateManagementIntegTestCase() { val indexName = "${testIndexName}_index_1" val policyID = "${testIndexName}_testPolicyName_1" - val actionConfig = ReplicaCountActionConfig(10, 0) + val actionConfig = ReplicaCountAction(10, 0) val states = listOf(State(name = "ReplicaCountState", actions = listOf(actionConfig), transitions = listOf())) val policy = Policy( id = policyID, diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/MetadataServiceTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/MetadataServiceTests.kt new file mode 100644 index 000000000..314a55b57 --- /dev/null +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/MetadataServiceTests.kt @@ -0,0 +1,116 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement + +import com.nhaarman.mockitokotlin2.any +import com.nhaarman.mockitokotlin2.doAnswer +import com.nhaarman.mockitokotlin2.doReturn +import com.nhaarman.mockitokotlin2.mock +import com.nhaarman.mockitokotlin2.never +import com.nhaarman.mockitokotlin2.verify +import com.nhaarman.mockitokotlin2.whenever +import kotlinx.coroutines.runBlocking +import org.junit.Before +import org.opensearch.action.ActionListener +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse +import org.opensearch.client.AdminClient +import org.opensearch.client.Client +import org.opensearch.client.ClusterAdminClient +import org.opensearch.cluster.ClusterState +import org.opensearch.cluster.metadata.Metadata +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.collect.ImmutableOpenMap +import org.opensearch.indexmanagement.IndexManagementIndices +import org.opensearch.test.OpenSearchTestCase +import kotlin.test.assertFailsWith + +class MetadataServiceTests : OpenSearchTestCase() { + + private val clusterService: ClusterService = mock() + private val clusterState: ClusterState = mock() + private val metadata: Metadata = mock() + private val imIndices: IndexManagementIndices = mock() + + private val ex = Exception() + + @Before + fun setup() { + whenever(clusterService.state()).doReturn(clusterState) + whenever(clusterState.metadata).doReturn(metadata) + whenever(metadata.indices).doReturn(ImmutableOpenMap.of()) + } + + fun `test config index not exists`() = runBlocking { + whenever(imIndices.indexManagementIndexExists()).doReturn(false) + + val client = getClient( + getAdminClient( + getClusterAdminClient( + updateSettingResponse = null, + updateSettingException = ex + ) + ) + ) + val skipFlag = SkipExecution(client, clusterService) + val metadataService = MetadataService(client, clusterService, skipFlag, imIndices) + metadataService.moveMetadata() + + verify(client.admin().cluster(), never()).updateSettings(any(), any()) + assertEquals(metadataService.finishFlag, true) + } + + // If update setting to 1 failed with some exception, runTimeCounter shouldn't be increased + fun `test failed to update setting to 1`() = runBlocking { + whenever(imIndices.indexManagementIndexExists()).doReturn(true) + + val client = getClient( + getAdminClient( + getClusterAdminClient( + updateSettingResponse = null, + updateSettingException = ex + ) + ) + ) + + val skipFlag = SkipExecution(client, clusterService) + val metadataService = MetadataService(client, clusterService, skipFlag, imIndices) + metadataService.moveMetadata() + assertEquals(metadataService.runTimeCounter, 2) + metadataService.moveMetadata() + assertEquals(metadataService.runTimeCounter, 3) + metadataService.moveMetadata() + assertEquals(metadataService.runTimeCounter, 4) + assertFailsWith(Exception::class) { + runBlocking { + metadataService.moveMetadata() + } + } + assertEquals(metadataService.runTimeCounter, 4) + assertEquals(metadataService.finishFlag, false) + } + + private fun getClient(adminClient: AdminClient): Client = mock { on { admin() } doReturn adminClient } + + private fun getAdminClient(clusterAdminClient: ClusterAdminClient): AdminClient = mock { on { cluster() } doReturn clusterAdminClient } + + private fun getClusterAdminClient( + updateSettingResponse: ClusterUpdateSettingsResponse?, + updateSettingException: Exception? + ): ClusterAdminClient { + assertTrue( + "Must provide either a getMappingsResponse or getMappingsException", + (updateSettingResponse != null).xor(updateSettingException != null) + ) + + return mock { + doAnswer { invocationOnMock -> + val listener = invocationOnMock.getArgument>(1) + if (updateSettingResponse != null) listener.onResponse(updateSettingResponse) + else listener.onFailure(updateSettingException) + }.whenever(this.mock).updateSettings(any(), any()) + } + } +} diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/TestHelpers.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/TestHelpers.kt index 68ffb95da..842ded4a5 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/TestHelpers.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/TestHelpers.kt @@ -10,28 +10,28 @@ import org.opensearch.common.unit.TimeValue import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.XContentFactory import org.opensearch.index.seqno.SequenceNumbers +import org.opensearch.indexmanagement.indexstatemanagement.action.AllocationAction +import org.opensearch.indexmanagement.indexstatemanagement.action.CloseAction +import org.opensearch.indexmanagement.indexstatemanagement.action.DeleteAction +import org.opensearch.indexmanagement.indexstatemanagement.action.ForceMergeAction +import org.opensearch.indexmanagement.indexstatemanagement.action.IndexPriorityAction +import org.opensearch.indexmanagement.indexstatemanagement.action.NotificationAction +import org.opensearch.indexmanagement.indexstatemanagement.action.OpenAction +import org.opensearch.indexmanagement.indexstatemanagement.action.ReadOnlyAction +import org.opensearch.indexmanagement.indexstatemanagement.action.ReadWriteAction +import org.opensearch.indexmanagement.indexstatemanagement.action.ReplicaCountAction +import org.opensearch.indexmanagement.indexstatemanagement.action.RolloverAction +import org.opensearch.indexmanagement.indexstatemanagement.action.RollupAction +import org.opensearch.indexmanagement.indexstatemanagement.action.SnapshotAction import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy import org.opensearch.indexmanagement.indexstatemanagement.model.Conditions import org.opensearch.indexmanagement.indexstatemanagement.model.ErrorNotification import org.opensearch.indexmanagement.indexstatemanagement.model.ISMTemplate import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State import org.opensearch.indexmanagement.indexstatemanagement.model.StateFilter import org.opensearch.indexmanagement.indexstatemanagement.model.Transition -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.AllocationActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.DeleteActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ForceMergeActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.IndexPriorityActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.NotificationActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReadOnlyActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReadWriteActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReplicaCountActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.RolloverActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.RollupActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.SnapshotActionConfig import org.opensearch.indexmanagement.indexstatemanagement.model.coordinator.ClusterStateManagedIndexConfig import org.opensearch.indexmanagement.indexstatemanagement.model.coordinator.SweptManagedIndexConfig import org.opensearch.indexmanagement.indexstatemanagement.model.destination.Chime @@ -41,6 +41,8 @@ import org.opensearch.indexmanagement.indexstatemanagement.model.destination.Des import org.opensearch.indexmanagement.indexstatemanagement.model.destination.Slack import org.opensearch.indexmanagement.opensearchapi.string import org.opensearch.indexmanagement.rollup.randomISMRollup +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.jobscheduler.spi.schedule.CronSchedule import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule import org.opensearch.jobscheduler.spi.schedule.Schedule @@ -68,7 +70,7 @@ fun randomPolicy( fun randomState( name: String = OpenSearchRestTestCase.randomAlphaOfLength(10), - actions: List = listOf(), + actions: List = listOf(), transitions: List = listOf() ): State { return State(name = name, actions = actions, transitions = transitions) @@ -110,8 +112,8 @@ fun randomConditions( fun nonNullRandomConditions(): Conditions = randomConditions(OpenSearchRestTestCase.randomFrom(listOf(randomIndexAge(), randomDocCount(), randomSize())))!! -fun randomDeleteActionConfig(): DeleteActionConfig { - return DeleteActionConfig(index = 0) +fun randomDeleteActionConfig(): DeleteAction { + return DeleteAction(index = 0) } fun randomRolloverActionConfig( @@ -119,8 +121,8 @@ fun randomRolloverActionConfig( minDocs: Long = OpenSearchRestTestCase.randomLongBetween(1, 1000), minAge: TimeValue = randomTimeValueObject(), minPrimaryShardSize: ByteSizeValue = randomByteSizeValue() -): RolloverActionConfig { - return RolloverActionConfig( +): RolloverAction { + return RolloverAction( minSize = minSize, minDocs = minDocs, minAge = minAge, @@ -129,42 +131,50 @@ fun randomRolloverActionConfig( ) } -fun randomReadOnlyActionConfig(): ReadOnlyActionConfig { - return ReadOnlyActionConfig(index = 0) +fun randomReadOnlyActionConfig(): ReadOnlyAction { + return ReadOnlyAction(index = 0) } -fun randomReadWriteActionConfig(): ReadWriteActionConfig { - return ReadWriteActionConfig(index = 0) +fun randomReadWriteActionConfig(): ReadWriteAction { + return ReadWriteAction(index = 0) } -fun randomReplicaCountActionConfig(numOfReplicas: Int = OpenSearchRestTestCase.randomIntBetween(0, 200)): ReplicaCountActionConfig { - return ReplicaCountActionConfig(index = 0, numOfReplicas = numOfReplicas) +fun randomReplicaCountActionConfig(numOfReplicas: Int = OpenSearchRestTestCase.randomIntBetween(0, 200)): ReplicaCountAction { + return ReplicaCountAction(index = 0, numOfReplicas = numOfReplicas) } -fun randomIndexPriorityActionConfig(indexPriority: Int = OpenSearchRestTestCase.randomIntBetween(0, 100)): IndexPriorityActionConfig { - return IndexPriorityActionConfig(index = 0, indexPriority = indexPriority) +fun randomIndexPriorityActionConfig(indexPriority: Int = OpenSearchRestTestCase.randomIntBetween(0, 100)): IndexPriorityAction { + return IndexPriorityAction(index = 0, indexPriority = indexPriority) } fun randomForceMergeActionConfig( maxNumSegments: Int = OpenSearchRestTestCase.randomIntBetween(1, 50) -): ForceMergeActionConfig { - return ForceMergeActionConfig(maxNumSegments = maxNumSegments, index = 0) +): ForceMergeAction { + return ForceMergeAction(maxNumSegments = maxNumSegments, index = 0) } fun randomNotificationActionConfig( destination: Destination = randomDestination(), messageTemplate: Script = randomTemplateScript("random message"), index: Int = 0 -): NotificationActionConfig { - return NotificationActionConfig(destination, messageTemplate, index) +): NotificationAction { + return NotificationAction(destination, messageTemplate, index) } -fun randomAllocationActionConfig(require: Map = emptyMap(), exclude: Map = emptyMap(), include: Map = emptyMap()): AllocationActionConfig { - return AllocationActionConfig(require, include, exclude, index = 0) +fun randomAllocationActionConfig(require: Map = emptyMap(), exclude: Map = emptyMap(), include: Map = emptyMap()): AllocationAction { + return AllocationAction(require, include, exclude, index = 0) } -fun randomRollupActionConfig(): RollupActionConfig { - return RollupActionConfig(ismRollup = randomISMRollup(), index = 0) +fun randomRollupActionConfig(): RollupAction { + return RollupAction(ismRollup = randomISMRollup(), index = 0) +} + +fun randomCloseActionConfig(): CloseAction { + return CloseAction(index = 0) +} + +fun randomOpenActionConfig(): OpenAction { + return OpenAction(index = 0) } fun randomDestination(type: DestinationType = randomDestinationType()): Destination { @@ -208,8 +218,8 @@ fun randomTemplateScript( params: Map = emptyMap() ): Script = Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, source, params) -fun randomSnapshotActionConfig(repository: String = "repo", snapshot: String = "sp"): SnapshotActionConfig { - return SnapshotActionConfig(repository, snapshot, index = 0) +fun randomSnapshotActionConfig(repository: String = "repo", snapshot: String = "sp"): SnapshotAction { + return SnapshotAction(repository, snapshot, index = 0) } /** @@ -348,47 +358,47 @@ fun Conditions.toJsonString(): String { return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } -fun DeleteActionConfig.toJsonString(): String { +fun DeleteAction.toJsonString(): String { val builder = XContentFactory.jsonBuilder() return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } -fun RolloverActionConfig.toJsonString(): String { +fun RolloverAction.toJsonString(): String { val builder = XContentFactory.jsonBuilder() return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } -fun ReadOnlyActionConfig.toJsonString(): String { +fun ReadOnlyAction.toJsonString(): String { val builder = XContentFactory.jsonBuilder() return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } -fun ReadWriteActionConfig.toJsonString(): String { +fun ReadWriteAction.toJsonString(): String { val builder = XContentFactory.jsonBuilder() return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } -fun ReplicaCountActionConfig.toJsonString(): String { +fun ReplicaCountAction.toJsonString(): String { val builder = XContentFactory.jsonBuilder() return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } -fun IndexPriorityActionConfig.toJsonString(): String { +fun IndexPriorityAction.toJsonString(): String { val builder = XContentFactory.jsonBuilder() return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } -fun ForceMergeActionConfig.toJsonString(): String { +fun ForceMergeAction.toJsonString(): String { val builder = XContentFactory.jsonBuilder() return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } -fun NotificationActionConfig.toJsonString(): String { +fun NotificationAction.toJsonString(): String { val builder = XContentFactory.jsonBuilder() return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } -fun AllocationActionConfig.toJsonString(): String { +fun AllocationAction.toJsonString(): String { val builder = XContentFactory.jsonBuilder() return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } @@ -408,12 +418,22 @@ fun ManagedIndexMetaData.toJsonString(): String { return this.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject().string() } -fun SnapshotActionConfig.toJsonString(): String { +fun SnapshotAction.toJsonString(): String { + val builder = XContentFactory.jsonBuilder() + return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() +} + +fun RollupAction.toJsonString(): String { + val builder = XContentFactory.jsonBuilder() + return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() +} + +fun CloseAction.toJsonString(): String { val builder = XContentFactory.jsonBuilder() return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } -fun RollupActionConfig.toJsonString(): String { +fun OpenAction.toJsonString(): String { val builder = XContentFactory.jsonBuilder() return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionRetryIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionRetryIT.kt index 3d7ea5fde..cf05dd34d 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionRetryIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionRetryIT.kt @@ -6,13 +6,13 @@ package org.opensearch.indexmanagement.indexstatemanagement.action import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.PolicyRetryInfoMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StateMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.indexstatemanagement.step.rollover.AttemptRolloverStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.indexmanagement.waitFor import java.time.Instant import java.util.Locale @@ -145,6 +145,7 @@ class ActionRetryIT : IndexStateManagementRestTestCase() { ManagedIndexMetaData.POLICY_SEQ_NO to policySeq::equals, ManagedIndexMetaData.POLICY_PRIMARY_TERM to policyPrimaryTerm::equals, ManagedIndexMetaData.ROLLED_OVER to false::equals, + ManagedIndexMetaData.INDEX_CREATION_DATE to fun(indexCreationDate: Any?): Boolean = (indexCreationDate as Long) > 1L, StateMetaData.STATE to fun(stateMetaDataMap: Any?): Boolean = assertStateEquals(StateMetaData("Ingest", Instant.now().toEpochMilli()), stateMetaDataMap), ActionMetaData.ACTION to fun(actionMetaDataMap: Any?): Boolean = @@ -159,7 +160,8 @@ class ActionRetryIT : IndexStateManagementRestTestCase() { ), PolicyRetryInfoMetaData.RETRY_INFO to fun(retryInfoMetaDataMap: Any?): Boolean = assertRetryInfoEquals(PolicyRetryInfoMetaData(false, 0), retryInfoMetaDataMap), - ManagedIndexMetaData.INFO to fun(info: Any?): Boolean = expectedInfoString == info.toString() + ManagedIndexMetaData.INFO to fun(info: Any?): Boolean = expectedInfoString == info.toString(), + ManagedIndexMetaData.ENABLED to true::equals ) ), getExplainMap(indexName) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionTimeoutIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionTimeoutIT.kt index 748c6f2ff..b3d792705 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionTimeoutIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionTimeoutIT.kt @@ -7,11 +7,10 @@ package org.opensearch.indexmanagement.indexstatemanagement.action import org.hamcrest.collection.IsMapContaining import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData import org.opensearch.indexmanagement.indexstatemanagement.step.open.AttemptOpenStep import org.opensearch.indexmanagement.indexstatemanagement.step.rollover.AttemptRolloverStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.waitFor import java.time.Instant import java.util.Locale @@ -58,7 +57,7 @@ class ActionTimeoutIT : IndexStateManagementRestTestCase() { ActionMetaData.ACTION to fun(actionMetaDataMap: Any?): Boolean = assertActionEquals( ActionMetaData( - name = ActionConfig.ActionType.ROLLOVER.type, startTime = Instant.now().toEpochMilli(), index = 0, + name = RolloverAction.name, startTime = Instant.now().toEpochMilli(), index = 0, failed = true, consumedRetries = 0, lastRetryTime = null, actionProperties = null ), actionMetaDataMap diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/AllocationActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/AllocationActionIT.kt index 375816802..295208fdc 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/AllocationActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/AllocationActionIT.kt @@ -9,7 +9,6 @@ import org.junit.Assume import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.AllocationActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.indexstatemanagement.step.allocation.AttemptAllocationStep import org.opensearch.indexmanagement.waitFor @@ -18,13 +17,12 @@ import java.time.temporal.ChronoUnit import java.util.Locale class AllocationActionIT : IndexStateManagementRestTestCase() { - private val testIndexName = javaClass.simpleName.toLowerCase(Locale.ROOT) fun `test basic`() { val indexName = "${testIndexName}_index_1" val policyID = "${testIndexName}_testPolicyName_1" - val actionConfig = AllocationActionConfig(require = mapOf("box_type" to "hot"), exclude = emptyMap(), include = emptyMap(), index = 0) + val actionConfig = AllocationAction(require = mapOf("box_type" to "hot"), exclude = emptyMap(), include = emptyMap(), index = 0) val states = listOf( State("Allocate", listOf(actionConfig), listOf()) ) @@ -69,7 +67,7 @@ class AllocationActionIT : IndexStateManagementRestTestCase() { availableNodes.remove(getIndexShardNodes(indexName)[0]) - val actionConfig = AllocationActionConfig(require = mapOf("_name" to availableNodes.first()), exclude = emptyMap(), include = emptyMap(), index = 0) + val actionConfig = AllocationAction(require = mapOf("_name" to availableNodes.first()), exclude = emptyMap(), include = emptyMap(), index = 0) val states = listOf( State("Allocate", listOf(actionConfig), listOf()) ) @@ -120,7 +118,7 @@ class AllocationActionIT : IndexStateManagementRestTestCase() { val excludedNode = getIndexShardNodes(indexName)[0].toString() - val actionConfig = AllocationActionConfig(require = emptyMap(), exclude = mapOf("_name" to excludedNode), include = emptyMap(), index = 0) + val actionConfig = AllocationAction(require = emptyMap(), exclude = mapOf("_name" to excludedNode), include = emptyMap(), index = 0) val states = listOf( State("Allocate", listOf(actionConfig), listOf()) ) @@ -172,7 +170,7 @@ class AllocationActionIT : IndexStateManagementRestTestCase() { availableNodes.remove(getIndexShardNodes(indexName)[0]) - val actionConfig = AllocationActionConfig(require = emptyMap(), exclude = emptyMap(), include = mapOf("_name" to availableNodes.first()), index = 0) + val actionConfig = AllocationAction(require = emptyMap(), exclude = emptyMap(), include = mapOf("_name" to availableNodes.first()), index = 0) val states = listOf( State("Allocate", listOf(actionConfig), listOf()) ) @@ -216,7 +214,7 @@ class AllocationActionIT : IndexStateManagementRestTestCase() { fun `test fail on illegal key`() { val indexName = "${testIndexName}_illegal_key" val policyID = "${testIndexName}_illegal_key" - val actionConfig = AllocationActionConfig(require = mapOf("..//" to "value"), exclude = emptyMap(), include = emptyMap(), index = 0) + val actionConfig = AllocationAction(require = mapOf("..//" to "value"), exclude = emptyMap(), include = emptyMap(), index = 0) val states = listOf( State("Allocate", listOf(actionConfig), listOf()) ) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/CloseActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/CloseActionIT.kt index 407e195b4..aa8ab9e09 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/CloseActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/CloseActionIT.kt @@ -9,7 +9,6 @@ import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementR import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State import org.opensearch.indexmanagement.indexstatemanagement.model.Transition -import org.opensearch.indexmanagement.indexstatemanagement.model.action.CloseActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.waitFor import java.time.Instant @@ -17,13 +16,12 @@ import java.time.temporal.ChronoUnit import java.util.Locale class CloseActionIT : IndexStateManagementRestTestCase() { - private val testIndexName = javaClass.simpleName.toLowerCase(Locale.ROOT) fun `test basic`() { val indexName = "${testIndexName}_index_1" val policyID = "${testIndexName}_testPolicyName_1" - val actionConfig = CloseActionConfig(0) + val actionConfig = CloseAction(0) val states = listOf( State("CloseState", listOf(actionConfig), listOf()) ) @@ -58,7 +56,7 @@ class CloseActionIT : IndexStateManagementRestTestCase() { fun `test already closed index`() { val indexName = "${testIndexName}_index_2" val policyID = "${testIndexName}_testPolicyName_2" - val actionConfig = CloseActionConfig(0) + val actionConfig = CloseAction(0) val states = listOf( State("CloseState", listOf(actionConfig), listOf()) ) @@ -94,7 +92,7 @@ class CloseActionIT : IndexStateManagementRestTestCase() { fun `test transitioning a closed index`() { val indexName = "${testIndexName}_index_3" val policyID = "${testIndexName}_testPolicyName_3" - val actionConfig = CloseActionConfig(0) + val actionConfig = CloseAction(0) val secondState = State("LastState", emptyList(), emptyList()) val firstState = State("CloseState", listOf(actionConfig), listOf(Transition(stateName = secondState.name, conditions = null))) val states = listOf(firstState, secondState) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/DeleteActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/DeleteActionIT.kt index 9233e3258..22027ec9a 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/DeleteActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/DeleteActionIT.kt @@ -8,7 +8,6 @@ package org.opensearch.indexmanagement.indexstatemanagement.action import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.DeleteActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.waitFor import java.time.Instant @@ -16,13 +15,12 @@ import java.time.temporal.ChronoUnit import java.util.Locale class DeleteActionIT : IndexStateManagementRestTestCase() { - private val testIndexName = javaClass.simpleName.toLowerCase(Locale.ROOT) fun `test basic`() { val indexName = "${testIndexName}_index_1" val policyID = "${testIndexName}_testPolicyName_1" - val actionConfig = DeleteActionConfig(0) + val actionConfig = DeleteAction(0) val states = listOf( State("DeleteState", listOf(actionConfig), listOf()) ) @@ -63,12 +61,16 @@ class DeleteActionIT : IndexStateManagementRestTestCase() { .any { val metadata = it["managed_index_meta_data"] as Map<*, *> val index = metadata["index"] as String - val action = metadata["action"] as Map<*, *> - val actionName = action["name"] as String - val step = metadata["step"] as Map<*, *> - val stepName = step["name"] as String - val stepStatus = step["step_status"] as String - index == indexName && actionName == "delete" && stepName == "attempt_delete" && stepStatus == "completed" + if (metadata.containsKey("action")) { + val action = metadata["action"] as Map<*, *> + val actionName = action["name"] as String + val step = metadata["step"] as Map<*, *> + val stepName = step["name"] as String + val stepStatus = step["step_status"] as String + index == indexName && actionName == "delete" && stepName == "attempt_delete" && stepStatus == "completed" + } else { + false + } } ) } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ForceMergeActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ForceMergeActionIT.kt index 1c316177d..9866f175a 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ForceMergeActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ForceMergeActionIT.kt @@ -10,7 +10,6 @@ import org.opensearch.common.settings.Settings import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ForceMergeActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.indexstatemanagement.step.forcemerge.AttemptCallForceMergeStep import org.opensearch.indexmanagement.indexstatemanagement.step.forcemerge.AttemptSetReadOnlyStep @@ -21,7 +20,6 @@ import java.time.temporal.ChronoUnit import java.util.Locale class ForceMergeActionIT : IndexStateManagementRestTestCase() { - private val testIndexName = javaClass.simpleName.toLowerCase(Locale.ROOT) fun `test basic workflow`() { @@ -29,7 +27,7 @@ class ForceMergeActionIT : IndexStateManagementRestTestCase() { val policyID = "${testIndexName}_testPolicyName_1" // Create a Policy with one State that only preforms a force_merge Action - val forceMergeActionConfig = ForceMergeActionConfig(maxNumSegments = 1, index = 0) + val forceMergeActionConfig = ForceMergeAction(maxNumSegments = 1, index = 0) val states = listOf(State("ForceMergeState", listOf(forceMergeActionConfig), listOf())) val policy = Policy( @@ -90,7 +88,7 @@ class ForceMergeActionIT : IndexStateManagementRestTestCase() { val policyID = "${testIndexName}_testPolicyName_2" // Create a Policy with one State that only preforms a force_merge Action - val forceMergeActionConfig = ForceMergeActionConfig(maxNumSegments = 1, index = 0) + val forceMergeActionConfig = ForceMergeAction(maxNumSegments = 1, index = 0) val states = listOf(State("ForceMergeState", listOf(forceMergeActionConfig), listOf())) val policy = Policy( diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexPriorityActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexPriorityActionIT.kt index db49d96c3..b13bfbab2 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexPriorityActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexPriorityActionIT.kt @@ -8,7 +8,6 @@ package org.opensearch.indexmanagement.indexstatemanagement.action import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.IndexPriorityActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.waitFor import java.time.Instant @@ -16,13 +15,12 @@ import java.time.temporal.ChronoUnit import java.util.Locale class IndexPriorityActionIT : IndexStateManagementRestTestCase() { - private val testIndexName = javaClass.simpleName.toLowerCase(Locale.ROOT) fun `test basic index priority`() { val indexName = "${testIndexName}_index_1" val policyID = "${testIndexName}_testPolicyName_1" - val actionConfig = IndexPriorityActionConfig(50, 0) + val actionConfig = IndexPriorityAction(50, 0) val states = listOf(State(name = "SetPriorityState", actions = listOf(actionConfig), transitions = listOf())) val policy = Policy( id = policyID, diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexStateManagementHistoryIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexStateManagementHistoryIT.kt index c46c6440f..1f80bb245 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexStateManagementHistoryIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/IndexStateManagementHistoryIT.kt @@ -8,32 +8,29 @@ package org.opensearch.indexmanagement.indexstatemanagement.action import org.opensearch.action.search.SearchResponse import org.opensearch.indexmanagement.IndexManagementIndices import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReadOnlyActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.PolicyRetryInfoMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StateMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.indexstatemanagement.step.readonly.SetReadOnlyStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.indexmanagement.waitFor import java.time.Instant import java.time.temporal.ChronoUnit import java.util.Locale class IndexStateManagementHistoryIT : IndexStateManagementRestTestCase() { - private val testIndexName = javaClass.simpleName.toLowerCase(Locale.ROOT) fun `test basic workflow`() { val indexName = "${testIndexName}_index_1" val policyID = "${testIndexName}_testPolicyName_1" - val actionConfig = ReadOnlyActionConfig(0) + val actionConfig = ReadOnlyAction(0) val states = listOf( State("ReadOnlyState", listOf(actionConfig), listOf()) ) @@ -79,9 +76,10 @@ class IndexStateManagementHistoryIT : IndexStateManagementRestTestCase() { policyPrimaryTerm = actualHistory.policyPrimaryTerm, policyCompleted = null, rolledOver = null, + indexCreationDate = actualHistory.indexCreationDate, transitionTo = null, stateMetaData = StateMetaData("ReadOnlyState", actualHistory.stateMetaData!!.startTime), - actionMetaData = ActionMetaData(ActionConfig.ActionType.READ_ONLY.toString(), actualHistory.actionMetaData!!.startTime, 0, false, 0, 0, null), + actionMetaData = ActionMetaData(ReadOnlyAction.name, actualHistory.actionMetaData!!.startTime, 0, false, 0, 0, null), stepMetaData = StepMetaData("set_read_only", actualHistory.stepMetaData!!.startTime, Step.StepStatus.COMPLETED), policyRetryInfo = PolicyRetryInfoMetaData(false, 0), info = mapOf("message" to SetReadOnlyStep.getSuccessMessage(indexName)) @@ -95,7 +93,7 @@ class IndexStateManagementHistoryIT : IndexStateManagementRestTestCase() { fun `test short retention period and history enabled`() { val indexName = "${testIndexName}_index_2" val policyID = "${testIndexName}_testPolicyName_2" - val actionConfig = ReadOnlyActionConfig(0) + val actionConfig = ReadOnlyAction(0) val states = listOf( State("ReadOnlyState", listOf(actionConfig), listOf()) ) @@ -145,9 +143,10 @@ class IndexStateManagementHistoryIT : IndexStateManagementRestTestCase() { policyPrimaryTerm = actualHistory.policyPrimaryTerm, policyCompleted = null, rolledOver = null, + indexCreationDate = actualHistory.indexCreationDate, transitionTo = null, stateMetaData = StateMetaData("ReadOnlyState", actualHistory.stateMetaData!!.startTime), - actionMetaData = ActionMetaData(ActionConfig.ActionType.READ_ONLY.toString(), actualHistory.actionMetaData!!.startTime, 0, false, 0, 0, null), + actionMetaData = ActionMetaData(ReadOnlyAction.name, actualHistory.actionMetaData!!.startTime, 0, false, 0, 0, null), stepMetaData = StepMetaData("set_read_only", actualHistory.stepMetaData!!.startTime, Step.StepStatus.COMPLETED), policyRetryInfo = PolicyRetryInfoMetaData(false, 0), info = mapOf("message" to SetReadOnlyStep.getSuccessMessage(indexName)) @@ -161,7 +160,7 @@ class IndexStateManagementHistoryIT : IndexStateManagementRestTestCase() { fun `test small doc count rolledover index`() { val indexName = "${testIndexName}_index_3" val policyID = "${testIndexName}_testPolicyNam_3" - val actionConfig = ReadOnlyActionConfig(0) + val actionConfig = ReadOnlyAction(0) val states = listOf( State("ReadOnlyState", listOf(actionConfig), listOf()) ) @@ -211,9 +210,10 @@ class IndexStateManagementHistoryIT : IndexStateManagementRestTestCase() { policyPrimaryTerm = actualHistory.policyPrimaryTerm, policyCompleted = null, rolledOver = null, + indexCreationDate = actualHistory.indexCreationDate, transitionTo = null, stateMetaData = StateMetaData("ReadOnlyState", actualHistory.stateMetaData!!.startTime), - actionMetaData = ActionMetaData(ActionConfig.ActionType.READ_ONLY.toString(), actualHistory.actionMetaData!!.startTime, 0, false, 0, 0, null), + actionMetaData = ActionMetaData(ReadOnlyAction.name, actualHistory.actionMetaData!!.startTime, 0, false, 0, 0, null), stepMetaData = StepMetaData("set_read_only", actualHistory.stepMetaData!!.startTime, Step.StepStatus.COMPLETED), policyRetryInfo = PolicyRetryInfoMetaData(false, 0), info = mapOf("message" to SetReadOnlyStep.getSuccessMessage(indexName)) @@ -227,7 +227,7 @@ class IndexStateManagementHistoryIT : IndexStateManagementRestTestCase() { fun `test short retention period and rolledover index`() { val indexName = "${testIndexName}_index_4" val policyID = "${testIndexName}_testPolicyNam_4" - val actionConfig = ReadOnlyActionConfig(0) + val actionConfig = ReadOnlyAction(0) val states = listOf( State("ReadOnlyState", listOf(actionConfig), listOf()) ) @@ -271,6 +271,7 @@ class IndexStateManagementHistoryIT : IndexStateManagementRestTestCase() { policyPrimaryTerm = actualHistory.policyPrimaryTerm, policyCompleted = null, rolledOver = null, + indexCreationDate = actualHistory.indexCreationDate, transitionTo = null, stateMetaData = StateMetaData("ReadOnlyState", actualHistory.stateMetaData!!.startTime), actionMetaData = null, @@ -301,9 +302,10 @@ class IndexStateManagementHistoryIT : IndexStateManagementRestTestCase() { policyPrimaryTerm = actualHistory1.policyPrimaryTerm, policyCompleted = null, rolledOver = null, + indexCreationDate = actualHistory1.indexCreationDate, transitionTo = null, stateMetaData = StateMetaData(states[0].name, actualHistory1.stateMetaData!!.startTime), - actionMetaData = ActionMetaData(ActionConfig.ActionType.READ_ONLY.toString(), actualHistory1.actionMetaData!!.startTime, 0, false, 0, 0, null), + actionMetaData = ActionMetaData(ReadOnlyAction.name, actualHistory1.actionMetaData!!.startTime, 0, false, 0, 0, null), stepMetaData = StepMetaData("set_read_only", actualHistory1.stepMetaData!!.startTime, Step.StepStatus.COMPLETED), policyRetryInfo = PolicyRetryInfoMetaData(false, 0), info = mapOf("message" to SetReadOnlyStep.getSuccessMessage(indexName)) @@ -327,7 +329,7 @@ class IndexStateManagementHistoryIT : IndexStateManagementRestTestCase() { fun `test short retention period and history disabled`() { val indexName = "${testIndexName}_index_5" val policyID = "${testIndexName}_testPolicyName_5" - val actionConfig = ReadOnlyActionConfig(0) + val actionConfig = ReadOnlyAction(0) val states = listOf( State("ReadOnlyState", listOf(actionConfig), listOf()) ) @@ -369,6 +371,7 @@ class IndexStateManagementHistoryIT : IndexStateManagementRestTestCase() { policyPrimaryTerm = actualHistory.policyPrimaryTerm, policyCompleted = null, rolledOver = null, + indexCreationDate = actualHistory.indexCreationDate, transitionTo = null, stateMetaData = StateMetaData(name = states[0].name, startTime = actualHistory.stateMetaData!!.startTime), actionMetaData = null, @@ -394,7 +397,7 @@ class IndexStateManagementHistoryIT : IndexStateManagementRestTestCase() { fun `test history shard settings`() { val indexName = "${testIndexName}_shard_settings" val policyID = "${testIndexName}_shard_settings_1" - val actionConfig = ReadOnlyActionConfig(0) + val actionConfig = ReadOnlyAction(0) val states = listOf(State("ReadOnlyState", listOf(actionConfig), listOf())) val policy = Policy( diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/NotificationActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/NotificationActionIT.kt index d7f7bb125..e90790192 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/NotificationActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/NotificationActionIT.kt @@ -8,7 +8,6 @@ package org.opensearch.indexmanagement.indexstatemanagement.action import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.NotificationActionConfig import org.opensearch.indexmanagement.indexstatemanagement.model.destination.CustomWebhook import org.opensearch.indexmanagement.indexstatemanagement.model.destination.Destination import org.opensearch.indexmanagement.indexstatemanagement.model.destination.DestinationType @@ -22,13 +21,12 @@ import java.time.temporal.ChronoUnit import java.util.Locale class NotificationActionIT : IndexStateManagementRestTestCase() { - private val testIndexName = javaClass.simpleName.toLowerCase(Locale.ROOT) // cannot test chime/slack in integ tests, but can test a custom webhook by // using the POST call to write to the local integTest cluster and verify that index has 1 doc @Suppress("UNCHECKED_CAST") - fun `test custom webhook notification`() { + fun `skip test custom webhook notification`() { val indexName = "${testIndexName}_index" val policyID = "${testIndexName}_testPolicyName" val notificationIndex = "notification_index" @@ -50,7 +48,7 @@ class NotificationActionIT : IndexStateManagementRestTestCase() { ) ) val messageTemplate = Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, "{ \"testing\": 5 }", emptyMap()) - val actionConfig = NotificationActionConfig(destination = destination, messageTemplate = messageTemplate, index = 0) + val actionConfig = NotificationAction(destination = destination, messageTemplate = messageTemplate, index = 0) val states = listOf(State(name = "NotificationState", actions = listOf(actionConfig), transitions = emptyList())) val policy = Policy( id = policyID, diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/OpenActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/OpenActionIT.kt index 11cb90232..4db5127c5 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/OpenActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/OpenActionIT.kt @@ -8,7 +8,6 @@ package org.opensearch.indexmanagement.indexstatemanagement.action import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.OpenActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.waitFor import java.time.Instant @@ -16,13 +15,12 @@ import java.time.temporal.ChronoUnit import java.util.Locale class OpenActionIT : IndexStateManagementRestTestCase() { - private val testIndexName = javaClass.simpleName.toLowerCase(Locale.ROOT) fun `test basic`() { val indexName = "${testIndexName}_index_1" val policyID = "${testIndexName}_testPolicyName_1" - val actionConfig = OpenActionConfig(0) + val actionConfig = OpenAction(0) val states = listOf( State("OpenState", listOf(actionConfig), listOf()) ) @@ -59,7 +57,7 @@ class OpenActionIT : IndexStateManagementRestTestCase() { fun `test already open index`() { val indexName = "${testIndexName}_index_2" val policyID = "${testIndexName}_testPolicyName_2" - val actionConfig = OpenActionConfig(0) + val actionConfig = OpenAction(0) val states = listOf( State("OpenState", listOf(actionConfig), listOf()) ) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadOnlyActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadOnlyActionIT.kt index c159b9c2a..71d329520 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadOnlyActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadOnlyActionIT.kt @@ -8,7 +8,6 @@ package org.opensearch.indexmanagement.indexstatemanagement.action import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReadOnlyActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.waitFor import java.time.Instant @@ -16,13 +15,12 @@ import java.time.temporal.ChronoUnit import java.util.Locale class ReadOnlyActionIT : IndexStateManagementRestTestCase() { - private val testIndexName = javaClass.simpleName.toLowerCase(Locale.ROOT) fun `test basic workflow`() { val indexName = "${testIndexName}_index_1" val policyID = "${testIndexName}_testPolicyName_1" - val actionConfig = ReadOnlyActionConfig(0) + val actionConfig = ReadOnlyAction(0) val states = listOf( State("ReadOnlyState", listOf(actionConfig), listOf()) ) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadWriteActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadWriteActionIT.kt index e828ec2eb..92a971513 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadWriteActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReadWriteActionIT.kt @@ -9,7 +9,6 @@ import org.opensearch.common.settings.Settings import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReadWriteActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.waitFor import java.time.Instant @@ -23,7 +22,7 @@ class ReadWriteActionIT : IndexStateManagementRestTestCase() { fun `test basic workflow`() { val indexName = "${testIndexName}_index_1" val policyID = "${testIndexName}_testPolicyName_1" - val actionConfig = ReadWriteActionConfig(0) + val actionConfig = ReadWriteAction(0) val states = listOf( State("ReadWriteState", listOf(actionConfig), listOf()) ) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReplicaCountActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReplicaCountActionIT.kt index 9a2893c4c..21b8e67ce 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReplicaCountActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ReplicaCountActionIT.kt @@ -8,7 +8,6 @@ package org.opensearch.indexmanagement.indexstatemanagement.action import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReplicaCountActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.waitFor import java.time.Instant @@ -22,7 +21,7 @@ class ReplicaCountActionIT : IndexStateManagementRestTestCase() { fun `test basic replica count`() { val indexName = "${testIndexName}_index_1" val policyID = "${testIndexName}_testPolicyName_1" - val actionConfig = ReplicaCountActionConfig(10, 0) + val actionConfig = ReplicaCountAction(10, 0) val states = listOf(State(name = "ReplicaCountState", actions = listOf(actionConfig), transitions = listOf())) val policy = Policy( id = policyID, diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionIT.kt index c6448564e..99149cdd9 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionIT.kt @@ -18,14 +18,13 @@ import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementR import org.opensearch.indexmanagement.indexstatemanagement.model.ISMTemplate import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionRetry -import org.opensearch.indexmanagement.indexstatemanagement.model.action.RolloverActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.indexstatemanagement.resthandler.RestRetryFailedManagedIndexAction import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.indexstatemanagement.step.rollover.AttemptRolloverStep import org.opensearch.indexmanagement.makeRequest +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionRetry import org.opensearch.indexmanagement.waitFor import org.opensearch.rest.RestRequest import org.opensearch.rest.RestStatus @@ -44,7 +43,7 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val indexNameBase = "${testIndexName}_index" val firstIndex = "$indexNameBase-1" val policyID = "${testIndexName}_testPolicyName_1" - val actionConfig = RolloverActionConfig(null, null, null, null, 0) + val actionConfig = RolloverAction(null, null, null, null, 0) val states = listOf(State(name = "RolloverAction", actions = listOf(actionConfig), transitions = listOf())) val policy = Policy( id = policyID, @@ -98,7 +97,7 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { ) val policyID = "${testIndexName}_bwc" - val actionConfig = RolloverActionConfig(null, null, null, null, 0) + val actionConfig = RolloverAction(null, null, null, null, 0) val states = listOf(State(name = "RolloverAction", actions = listOf(actionConfig), transitions = listOf())) val policy = Policy( id = policyID, @@ -135,7 +134,7 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val indexNameBase = "${testIndexName}_index_byte" val firstIndex = "$indexNameBase-1" val policyID = "${testIndexName}_testPolicyName_byte_1" - val actionConfig = RolloverActionConfig(ByteSizeValue(10, ByteSizeUnit.BYTES), 1000000, null, null, 0) + val actionConfig = RolloverAction(ByteSizeValue(10, ByteSizeUnit.BYTES), 1000000, null, null, 0) val states = listOf(State(name = "RolloverAction", actions = listOf(actionConfig), transitions = listOf())) val policy = Policy( id = policyID, @@ -168,10 +167,10 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val conditions = info["conditions"] as Map assertEquals( "Did not have exclusively min size and min doc count conditions", - setOf(RolloverActionConfig.MIN_SIZE_FIELD, RolloverActionConfig.MIN_DOC_COUNT_FIELD), conditions.keys + setOf(RolloverAction.MIN_SIZE_FIELD, RolloverAction.MIN_DOC_COUNT_FIELD), conditions.keys ) - val minSize = conditions[RolloverActionConfig.MIN_SIZE_FIELD] as Map - val minDocCount = conditions[RolloverActionConfig.MIN_DOC_COUNT_FIELD] as Map + val minSize = conditions[RolloverAction.MIN_SIZE_FIELD] as Map + val minDocCount = conditions[RolloverAction.MIN_DOC_COUNT_FIELD] as Map assertEquals("Did not have min size condition", "10b", minSize["condition"]) assertThat("Did not have min size current", minSize["current"], isA(String::class.java)) assertEquals("Did not have min doc count condition", 1000000, minDocCount["condition"]) @@ -188,10 +187,10 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val conditions = info["conditions"] as Map assertEquals( "Did not have exclusively min size and min doc count conditions", - setOf(RolloverActionConfig.MIN_SIZE_FIELD, RolloverActionConfig.MIN_DOC_COUNT_FIELD), conditions.keys + setOf(RolloverAction.MIN_SIZE_FIELD, RolloverAction.MIN_DOC_COUNT_FIELD), conditions.keys ) - val minSize = conditions[RolloverActionConfig.MIN_SIZE_FIELD] as Map - val minDocCount = conditions[RolloverActionConfig.MIN_DOC_COUNT_FIELD] as Map + val minSize = conditions[RolloverAction.MIN_SIZE_FIELD] as Map + val minDocCount = conditions[RolloverAction.MIN_DOC_COUNT_FIELD] as Map assertEquals("Did not have min size condition", "10b", minSize["condition"]) assertThat("Did not have min size current", minSize["current"], isA(String::class.java)) assertEquals("Did not have min doc count condition", 1000000, minDocCount["condition"]) @@ -207,7 +206,7 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val indexNameBase = "${testIndexName}_index_primary_shard" val firstIndex = "$indexNameBase-1" val policyID = "${testIndexName}_primary_shard_1" - val actionConfig = RolloverActionConfig(null, null, null, ByteSizeValue(100, ByteSizeUnit.KB), 0) + val actionConfig = RolloverAction(null, null, null, ByteSizeValue(100, ByteSizeUnit.KB), 0) val states = listOf(State(name = "RolloverAction", actions = listOf(actionConfig), transitions = listOf())) val policy = Policy( id = policyID, @@ -265,9 +264,9 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val conditions = info["conditions"] as Map assertEquals( "Did not have exclusively min primary shard size condition", - setOf(RolloverActionConfig.MIN_PRIMARY_SHARD_SIZE_FIELD), conditions.keys + setOf(RolloverAction.MIN_PRIMARY_SHARD_SIZE_FIELD), conditions.keys ) - val minPrimarySize = conditions[RolloverActionConfig.MIN_PRIMARY_SHARD_SIZE_FIELD] as Map + val minPrimarySize = conditions[RolloverAction.MIN_PRIMARY_SHARD_SIZE_FIELD] as Map assertEquals("Did not have min size condition", "100kb", minPrimarySize["condition"]) assertThat("Did not have min size current", minPrimarySize["current"], isA(String::class.java)) } @@ -297,9 +296,9 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val conditions = info["conditions"] as Map assertEquals( "Did not have exclusively min primary shard size conditions", - setOf(RolloverActionConfig.MIN_PRIMARY_SHARD_SIZE_FIELD), conditions.keys + setOf(RolloverAction.MIN_PRIMARY_SHARD_SIZE_FIELD), conditions.keys ) - val minPrimaryShardSize = conditions[RolloverActionConfig.MIN_PRIMARY_SHARD_SIZE_FIELD] as Map + val minPrimaryShardSize = conditions[RolloverAction.MIN_PRIMARY_SHARD_SIZE_FIELD] as Map assertEquals("Did not have min primary shard size condition", "100kb", minPrimaryShardSize["condition"]) assertThat("Did not have min primary shard size current", minPrimaryShardSize["current"], isA(String::class.java)) } @@ -312,7 +311,7 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val indexNameBase = "${testIndexName}_index_doc" val firstIndex = "$indexNameBase-1" val policyID = "${testIndexName}_testPolicyName_doc_1" - val actionConfig = RolloverActionConfig(null, 3, TimeValue.timeValueDays(2), null, 0) + val actionConfig = RolloverAction(null, 3, TimeValue.timeValueDays(2), null, 0) val states = listOf(State(name = "RolloverAction", actions = listOf(actionConfig), transitions = listOf())) val policy = Policy( id = policyID, @@ -345,10 +344,10 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val conditions = info["conditions"] as Map assertEquals( "Did not have exclusively min age and min doc count conditions", - setOf(RolloverActionConfig.MIN_INDEX_AGE_FIELD, RolloverActionConfig.MIN_DOC_COUNT_FIELD), conditions.keys + setOf(RolloverAction.MIN_INDEX_AGE_FIELD, RolloverAction.MIN_DOC_COUNT_FIELD), conditions.keys ) - val minAge = conditions[RolloverActionConfig.MIN_INDEX_AGE_FIELD] as Map - val minDocCount = conditions[RolloverActionConfig.MIN_DOC_COUNT_FIELD] as Map + val minAge = conditions[RolloverAction.MIN_INDEX_AGE_FIELD] as Map + val minDocCount = conditions[RolloverAction.MIN_DOC_COUNT_FIELD] as Map assertEquals("Did not have min age condition", "2d", minAge["condition"]) assertThat("Did not have min age current", minAge["current"], isA(String::class.java)) assertEquals("Did not have min doc count condition", 3, minDocCount["condition"]) @@ -365,10 +364,10 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val conditions = info["conditions"] as Map assertEquals( "Did not have exclusively min age and min doc count conditions", - setOf(RolloverActionConfig.MIN_INDEX_AGE_FIELD, RolloverActionConfig.MIN_DOC_COUNT_FIELD), conditions.keys + setOf(RolloverAction.MIN_INDEX_AGE_FIELD, RolloverAction.MIN_DOC_COUNT_FIELD), conditions.keys ) - val minAge = conditions[RolloverActionConfig.MIN_INDEX_AGE_FIELD] as Map - val minDocCount = conditions[RolloverActionConfig.MIN_DOC_COUNT_FIELD] as Map + val minAge = conditions[RolloverAction.MIN_INDEX_AGE_FIELD] as Map + val minDocCount = conditions[RolloverAction.MIN_DOC_COUNT_FIELD] as Map assertEquals("Did not have min age condition", "2d", minAge["condition"]) assertThat("Did not have min age current", minAge["current"], isA(String::class.java)) assertEquals("Did not have min doc count condition", 3, minDocCount["condition"]) @@ -385,7 +384,7 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val index2 = "index-2" val alias1 = "x" val policyID = "${testIndexName}_precheck" - val actionConfig = RolloverActionConfig(null, 3, TimeValue.timeValueDays(2), null, 0) + val actionConfig = RolloverAction(null, 3, TimeValue.timeValueDays(2), null, 0) actionConfig.configRetry = ActionRetry(0) val states = listOf(State(name = "RolloverAction", actions = listOf(actionConfig), transitions = listOf())) val policy = Policy( @@ -444,8 +443,8 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val policyID = "${testIndexName}_rollover_policy" // Create the rollover policy - val rolloverActionConfig = RolloverActionConfig(null, null, null, null, 0) - val states = listOf(State(name = "default", actions = listOf(rolloverActionConfig), transitions = listOf())) + val rolloverAction = RolloverAction(null, null, null, null, 0) + val states = listOf(State(name = "default", actions = listOf(rolloverAction), transitions = listOf())) val policy = Policy( id = policyID, description = "rollover policy description", @@ -500,8 +499,8 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val policyID = "${testIndexName}_rollover_policy_multi" // Create the rollover policy - val rolloverActionConfig = RolloverActionConfig(null, 3, TimeValue.timeValueDays(2), null, 0) - val states = listOf(State(name = "default", actions = listOf(rolloverActionConfig), transitions = listOf())) + val rolloverAction = RolloverAction(null, 3, TimeValue.timeValueDays(2), null, 0) + val states = listOf(State(name = "default", actions = listOf(rolloverAction), transitions = listOf())) val policy = Policy( id = policyID, description = "rollover policy description", @@ -543,12 +542,12 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val conditions = info["conditions"] as Map assertEquals( "Did not have exclusively min age and min doc count conditions", - setOf(RolloverActionConfig.MIN_INDEX_AGE_FIELD, RolloverActionConfig.MIN_DOC_COUNT_FIELD), + setOf(RolloverAction.MIN_INDEX_AGE_FIELD, RolloverAction.MIN_DOC_COUNT_FIELD), conditions.keys ) - val minAge = conditions[RolloverActionConfig.MIN_INDEX_AGE_FIELD] as Map - val minDocCount = conditions[RolloverActionConfig.MIN_DOC_COUNT_FIELD] as Map + val minAge = conditions[RolloverAction.MIN_INDEX_AGE_FIELD] as Map + val minDocCount = conditions[RolloverAction.MIN_DOC_COUNT_FIELD] as Map assertEquals("Incorrect min age condition", "2d", minAge["condition"]) assertEquals("Incorrect min docs condition", 3, minDocCount["condition"]) assertThat("Missing min age current", minAge["current"], isA(String::class.java)) @@ -571,12 +570,12 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val conditions = info["conditions"] as Map assertEquals( "Did not have exclusively min age and min doc count conditions", - setOf(RolloverActionConfig.MIN_INDEX_AGE_FIELD, RolloverActionConfig.MIN_DOC_COUNT_FIELD), + setOf(RolloverAction.MIN_INDEX_AGE_FIELD, RolloverAction.MIN_DOC_COUNT_FIELD), conditions.keys ) - val minAge = conditions[RolloverActionConfig.MIN_INDEX_AGE_FIELD] as Map - val minDocCount = conditions[RolloverActionConfig.MIN_DOC_COUNT_FIELD] as Map + val minAge = conditions[RolloverAction.MIN_INDEX_AGE_FIELD] as Map + val minDocCount = conditions[RolloverAction.MIN_DOC_COUNT_FIELD] as Map assertEquals("Incorrect min age condition", "2d", minAge["condition"]) assertEquals("Incorrect min docs condition", 3, minDocCount["condition"]) assertThat("Missing min age current", minAge["current"], isA(String::class.java)) @@ -593,7 +592,7 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { val indexNameBase = "${testIndexName}_index" val firstIndex = "$indexNameBase-1" val policyID = "${testIndexName}_testPolicyName_1" - val actionConfig = RolloverActionConfig(null, null, null, null, 0) + val actionConfig = RolloverAction(null, null, null, null, 0) val states = listOf(State(name = "RolloverAction", actions = listOf(actionConfig), transitions = listOf())) val policy = Policy( id = policyID, diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RollupActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RollupActionIT.kt index 563e552f1..7b6c06f21 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RollupActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RollupActionIT.kt @@ -14,7 +14,6 @@ import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementR import org.opensearch.indexmanagement.indexstatemanagement.model.ISMTemplate import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.RollupActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.indexstatemanagement.step.rollup.AttemptCreateRollupJobStep import org.opensearch.indexmanagement.indexstatemanagement.step.rollup.WaitForRollupCompletionStep @@ -60,7 +59,7 @@ class RollupActionIT : IndexStateManagementRestTestCase() { RollupMetrics(sourceField = "total_amount", targetField = "total_amount", metrics = listOf(Max(), Min())) ) ) - val actionConfig = RollupActionConfig(rollup, 0) + val actionConfig = RollupAction(rollup, 0) val states = listOf( State("rollup", listOf(actionConfig), listOf()) ) @@ -110,7 +109,7 @@ class RollupActionIT : IndexStateManagementRestTestCase() { ) // Create an ISM policy to rollup backing indices of a data stream. - val actionConfig = RollupActionConfig(rollup, 0) + val actionConfig = RollupAction(rollup, 0) val states = listOf(State("rollup", listOf(actionConfig), listOf())) val policy = Policy( id = policyID, @@ -170,7 +169,7 @@ class RollupActionIT : IndexStateManagementRestTestCase() { ) ) val rollupId = rollup.toRollup(indexName).id - val actionConfig = RollupActionConfig(rollup, 0) + val actionConfig = RollupAction(rollup, 0) val states = listOf( State("rollup", listOf(actionConfig), listOf()) ) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/SnapshotActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/SnapshotActionIT.kt index ea5c1300b..e1bd328d7 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/SnapshotActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/SnapshotActionIT.kt @@ -8,7 +8,6 @@ package org.opensearch.indexmanagement.indexstatemanagement.action import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.SnapshotActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.SNAPSHOT_DENY_LIST import org.opensearch.indexmanagement.indexstatemanagement.step.snapshot.AttemptSnapshotStep @@ -27,7 +26,7 @@ class SnapshotActionIT : IndexStateManagementRestTestCase() { val policyID = "${testIndexName}_policy_basic" val repository = "repository" val snapshot = "snapshot" - val actionConfig = SnapshotActionConfig(repository, snapshot, 0) + val actionConfig = SnapshotAction(repository, snapshot, 0) val states = listOf( State("Snapshot", listOf(actionConfig), listOf()) ) @@ -64,7 +63,7 @@ class SnapshotActionIT : IndexStateManagementRestTestCase() { val indexName = "${testIndexName}_index_basic" val policyID = "${testIndexName}_policy_basic" val repository = "repository" - val actionConfig = SnapshotActionConfig(repository, "{{ctx.index}}", 0) + val actionConfig = SnapshotAction(repository, "{{ctx.index}}", 0) val states = listOf( State("Snapshot", listOf(actionConfig), listOf()) ) @@ -101,7 +100,7 @@ class SnapshotActionIT : IndexStateManagementRestTestCase() { val indexName = "${testIndexName}_index_basic" val policyID = "${testIndexName}_policy_basic" val repository = "repository" - val actionConfig = SnapshotActionConfig(repository, "{{ctx.someField}}", 0) + val actionConfig = SnapshotAction(repository, "{{ctx.someField}}", 0) val states = listOf( State("Snapshot", listOf(actionConfig), listOf()) ) @@ -139,7 +138,7 @@ class SnapshotActionIT : IndexStateManagementRestTestCase() { val policyID = "${testIndexName}_policy_success" val repository = "repository" val snapshot = "snapshot_success_test" - val actionConfig = SnapshotActionConfig(repository, snapshot, 0) + val actionConfig = SnapshotAction(repository, snapshot, 0) val states = listOf( State("Snapshot", listOf(actionConfig), listOf()) ) @@ -188,7 +187,7 @@ class SnapshotActionIT : IndexStateManagementRestTestCase() { val policyID = "${testIndexName}_policy_success" val repository = "repository" val snapshot = "-" - val actionConfig = SnapshotActionConfig(repository, "", 0) + val actionConfig = SnapshotAction(repository, "", 0) val states = listOf( State("Snapshot", listOf(actionConfig), listOf()) ) @@ -237,7 +236,7 @@ class SnapshotActionIT : IndexStateManagementRestTestCase() { val policyID = "${testIndexName}_policy_failed" val repository = "repository" val snapshot = "snapshot_failed_test" - val actionConfig = SnapshotActionConfig(repository, snapshot, 0) + val actionConfig = SnapshotAction(repository, snapshot, 0) val states = listOf( State("Snapshot", listOf(actionConfig), listOf()) ) @@ -290,7 +289,7 @@ class SnapshotActionIT : IndexStateManagementRestTestCase() { val indexName = "${testIndexName}_index_blocked" val policyID = "${testIndexName}_policy_basic" val repository = "hello-world" - val actionConfig = SnapshotActionConfig(repository, "snapshot", 0) + val actionConfig = SnapshotAction(repository, "snapshot", 0) val states = listOf( State("Snapshot", listOf(actionConfig), listOf()) ) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorIT.kt index 3e40aceeb..605e0a1c4 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorIT.kt @@ -5,25 +5,21 @@ package org.opensearch.indexmanagement.indexstatemanagement.coordinator -import org.opensearch.client.ResponseException import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase +import org.opensearch.indexmanagement.indexstatemanagement.action.DeleteAction +import org.opensearch.indexmanagement.indexstatemanagement.action.ForceMergeAction +import org.opensearch.indexmanagement.indexstatemanagement.action.RolloverAction import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State import org.opensearch.indexmanagement.indexstatemanagement.model.Transition -import org.opensearch.indexmanagement.indexstatemanagement.model.action.DeleteActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ForceMergeActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.RolloverActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification -import org.opensearch.indexmanagement.indexstatemanagement.resthandler.RestExplainAction import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.step.forcemerge.WaitForForceMergeStep import org.opensearch.indexmanagement.indexstatemanagement.step.rollover.AttemptRolloverStep -import org.opensearch.indexmanagement.makeRequest +import org.opensearch.indexmanagement.indexstatemanagement.util.TOTAL_MANAGED_INDICES +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.waitFor -import org.opensearch.rest.RestRequest -import org.opensearch.rest.RestStatus import java.time.Instant import java.time.temporal.ChronoUnit import kotlin.test.assertFailsWith @@ -95,7 +91,8 @@ class ManagedIndexCoordinatorIT : IndexStateManagementRestTestCase() { explainResponseOpendistroPolicyIdSetting to fun(policyID: Any?): Boolean = policyID == null, explainResponseOpenSearchPolicyIdSetting to fun(policyID: Any?): Boolean = - policyID == null + policyID == null, + ManagedIndexMetaData.ENABLED to fun(enabled: Any?): Boolean = enabled == null ) ), getExplainMap(index), @@ -127,35 +124,9 @@ class ManagedIndexCoordinatorIT : IndexStateManagementRestTestCase() { // Verify ManagedIndexMetadata has been cleared waitFor { - try { - client().makeRequest(RestRequest.Method.GET.toString(), RestExplainAction.EXPLAIN_BASE_URI) - fail("Expected a failure") - } catch (e: ResponseException) { - assertEquals("Unexpected RestStatus", RestStatus.NOT_FOUND, e.response.restStatus()) - val actualMessage = e.response.asMap() - val expectedErrorMessage = mapOf( - "error" to mapOf( - "root_cause" to listOf( - mapOf( - "type" to "illegal_argument_exception", "reason" to "Missing indices", - "reason" to "no such index [$index]", - "index_uuid" to "_na_", - "index" to index, - "resource.type" to "index_or_alias", - "type" to "index_not_found_exception", - "resource.id" to index - ) - ), - "type" to "index_not_found_exception", - "reason" to "no such index [$index]", - "index_uuid" to "_na_", - "index" to index, - "resource.type" to "index_or_alias", - "resource.id" to index - ), - "status" to 404 - ) - assertEquals(expectedErrorMessage, actualMessage) + val expected = mapOf(TOTAL_MANAGED_INDICES to 0) + waitFor { + assertEquals(expected, getExplainMap(null)) } } } @@ -165,7 +136,7 @@ class ManagedIndexCoordinatorIT : IndexStateManagementRestTestCase() { val policyID = "test_policy_1" // Create a policy with one State that performs rollover - val rolloverActionConfig = RolloverActionConfig(index = 0, minDocs = 5, minAge = null, minSize = null, minPrimaryShardSize = null) + val rolloverActionConfig = RolloverAction(index = 0, minDocs = 5, minAge = null, minSize = null, minPrimaryShardSize = null) val states = listOf(State(name = "RolloverState", actions = listOf(rolloverActionConfig), transitions = listOf())) val policy = Policy( @@ -249,8 +220,8 @@ class ManagedIndexCoordinatorIT : IndexStateManagementRestTestCase() { val policyID = "test_policy_1" // Create a policy with one State that performs force_merge and another State that deletes the index - val forceMergeActionConfig = ForceMergeActionConfig(index = 0, maxNumSegments = 1) - val deleteActionConfig = DeleteActionConfig(index = 0) + val forceMergeActionConfig = ForceMergeAction(index = 0, maxNumSegments = 1) + val deleteActionConfig = DeleteAction(index = 0) val states = listOf( State( name = "ForceMergeState", diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorTests.kt index 6fdd2b6ab..2a92ea6e0 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/coordinator/ManagedIndexCoordinatorTests.kt @@ -17,6 +17,7 @@ import org.opensearch.common.settings.Setting import org.opensearch.common.settings.Settings import org.opensearch.common.xcontent.NamedXContentRegistry import org.opensearch.indexmanagement.IndexManagementIndices +import org.opensearch.indexmanagement.indexstatemanagement.IndexMetadataProvider import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexCoordinator import org.opensearch.indexmanagement.indexstatemanagement.MetadataService import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings @@ -38,6 +39,7 @@ class ManagedIndexCoordinatorTests : OpenSearchAllocationTestCase() { private lateinit var metadataService: MetadataService private lateinit var templateService: ISMTemplateService private lateinit var coordinator: ManagedIndexCoordinator + private lateinit var indexMetadataProvider: IndexMetadataProvider private lateinit var discoveryNode: DiscoveryNode @@ -72,8 +74,11 @@ class ManagedIndexCoordinatorTests : OpenSearchAllocationTestCase() { val clusterSettings = ClusterSettings(settings, settingSet) val originClusterService: ClusterService = ClusterServiceUtils.createClusterService(threadPool, discoveryNode, clusterSettings) clusterService = Mockito.spy(originClusterService) - - coordinator = ManagedIndexCoordinator(settings, client, clusterService, threadPool, indexManagementIndices, metadataService, templateService) + indexMetadataProvider = IndexMetadataProvider(settings, client, clusterService, mutableMapOf()) + coordinator = ManagedIndexCoordinator( + settings, client, clusterService, threadPool, indexManagementIndices, metadataService, + templateService, indexMetadataProvider + ) } fun `test after start`() { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/extension/ISMActionsParserTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/extension/ISMActionsParserTests.kt new file mode 100644 index 000000000..103cd92a9 --- /dev/null +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/extension/ISMActionsParserTests.kt @@ -0,0 +1,90 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.extension + +import org.junit.After +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.common.xcontent.XContentType +import org.opensearch.indexmanagement.indexstatemanagement.ISMActionsParser +import org.opensearch.indexmanagement.opensearchapi.convertToMap +import org.opensearch.indexmanagement.opensearchapi.string +import org.opensearch.test.OpenSearchTestCase +import kotlin.test.assertFailsWith + +class ISMActionsParserTests : OpenSearchTestCase() { + + val extensionName = "testExtension" + + /* + * If any tests added the custom action parser, it should be removed from the static instance to not impact other tests + */ + @After + @Suppress("UnusedPrivateMember") + private fun removeCustomActionParser() { + ISMActionsParser.instance.parsers.removeIf { it.getActionType() == SampleCustomActionParser.SampleCustomAction.name } + } + + fun `test duplicate action names fail`() { + val customActionParser = SampleCustomActionParser() + // Duplicate custom parser names should fail + ISMActionsParser.instance.addParser(customActionParser, extensionName) + assertFailsWith("Expected IllegalArgumentException for duplicate action names") { + ISMActionsParser.instance.addParser(customActionParser, extensionName) + } + // Adding any duplicate parser should fail + assertFailsWith("Expected IllegalArgumentException for duplicate action names") { + val randomExistingParser = ISMActionsParser.instance.parsers.random() + ISMActionsParser.instance.addParser(randomExistingParser, extensionName) + } + } + + fun `test custom action parsing`() { + val customActionParser = SampleCustomActionParser() + ISMActionsParser.instance.addParser(customActionParser, extensionName) + val customAction = SampleCustomActionParser.SampleCustomAction(randomInt(), 0) + val builder = XContentFactory.jsonBuilder() + + val customActionString = customAction.toXContent(builder, ToXContent.EMPTY_PARAMS).string() + val parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, customActionString) + parser.nextToken() + val parsedCustomAction = ISMActionsParser.instance.parse(parser, 1) + assertTrue("Action was not set to be custom after parsing", parsedCustomAction.customAction) + customAction.customAction = true + assertEquals("Round tripping custom action doesn't work", customAction.convertToMap(), parsedCustomAction.convertToMap()) + } + + fun `test parsing custom action without custom flag`() { + val customActionParser = SampleCustomActionParser() + ISMActionsParser.instance.addParser(customActionParser, extensionName) + val customAction = SampleCustomActionParser.SampleCustomAction(randomInt(), 0) + customAction.customAction = true + + val customActionString = "{\"retry\":{\"count\":3,\"backoff\":\"exponential\",\"delay\":\"1m\"},\"some_custom_action\":{\"some_int_field\":${customAction.someInt}}}" + val parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, customActionString) + parser.nextToken() + val parsedCustomAction = ISMActionsParser.instance.parse(parser, 1) + assertTrue("Action was not set to be custom after parsing", parsedCustomAction.customAction) + assertEquals("Round tripping custom action doesn't work", customAction.convertToMap(), parsedCustomAction.convertToMap()) + assertTrue("Custom action did not have custom keyword after parsing", parsedCustomAction.convertToMap().containsKey("custom")) + } + + fun `test parsing custom action with custom flag`() { + val customActionParser = SampleCustomActionParser() + ISMActionsParser.instance.addParser(customActionParser, extensionName) + val customAction = SampleCustomActionParser.SampleCustomAction(randomInt(), 0) + customAction.customAction = true + + val customActionString = "{\"retry\":{\"count\":3,\"backoff\":\"exponential\",\"delay\":\"1m\"},\"custom\": {\"some_custom_action\":{\"some_int_field\":${customAction.someInt}}}}" + val parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, customActionString) + parser.nextToken() + val parsedCustomAction = ISMActionsParser.instance.parse(parser, 1) + assertTrue("Action was not set to be custom after parsing", parsedCustomAction.customAction) + assertEquals("Round tripping custom action doesn't work", customAction.convertToMap(), parsedCustomAction.convertToMap()) + assertTrue("Custom action did not have custom keyword after parsing", parsedCustomAction.convertToMap().containsKey("custom")) + } +} diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/extension/SampleCustomActionParser.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/extension/SampleCustomActionParser.kt new file mode 100644 index 000000000..8da8e0a6f --- /dev/null +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/extension/SampleCustomActionParser.kt @@ -0,0 +1,94 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.extension + +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData + +class SampleCustomActionParser : ActionParser() { + override fun fromStreamInput(sin: StreamInput): Action { + val someInt = sin.readInt() + val index = sin.readInt() + return SampleCustomAction(someInt, index) + } + + override fun fromXContent(xcp: XContentParser, index: Int): Action { + var someInt: Int? = null + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + SampleCustomAction.SOME_INT_FIELD -> someInt = xcp.intValue() + else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in SampleCustomAction.") + } + } + return SampleCustomAction(someInt = requireNotNull(someInt) { "SomeInt field must be specified" }, index) + } + + override fun getActionType(): String { + return SampleCustomAction.name + } + class SampleCustomAction(val someInt: Int, index: Int) : Action(name, index) { + + private val sampleCustomStep = SampleCustomStep() + private val steps = listOf(sampleCustomStep) + + override fun getStepToExecute(context: StepContext): Step = sampleCustomStep + + override fun getSteps(): List = steps + + override fun populateAction(builder: XContentBuilder, params: ToXContent.Params) { + builder.startObject(type) + builder.field(SOME_INT_FIELD, someInt) + builder.endObject() + } + + override fun populateAction(out: StreamOutput) { + out.writeInt(someInt) + out.writeInt(actionIndex) + } + + companion object { + const val name = "some_custom_action" + const val SOME_INT_FIELD = "some_int_field" + } + } + class SampleCustomStep : Step(name) { + private var stepStatus = StepStatus.STARTING + + override suspend fun execute(): Step { + stepStatus = StepStatus.COMPLETED + return this + } + + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { + return currentMetadata.copy( + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), + transitionTo = null, + info = null + ) + } + + override fun isIdempotent(): Boolean = true + + companion object { + const val name = "some_custom_step" + } + } +} diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ActionPropertiesTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ActionPropertiesTests.kt index 44ee4588e..da7a55c88 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ActionPropertiesTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ActionPropertiesTests.kt @@ -6,7 +6,7 @@ package org.opensearch.indexmanagement.indexstatemanagement.model import org.opensearch.common.xcontent.XContentType -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionProperties +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties import org.opensearch.test.OpenSearchTestCase class ActionPropertiesTests : OpenSearchTestCase() { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ActionConfigTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ActionTests.kt similarity index 60% rename from src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ActionConfigTests.kt rename to src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ActionTests.kt index a7654405b..e73e92345 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ActionConfigTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ActionTests.kt @@ -12,27 +12,35 @@ import org.opensearch.common.unit.TimeValue import org.opensearch.common.xcontent.LoggingDeprecationHandler import org.opensearch.common.xcontent.XContentFactory import org.opensearch.common.xcontent.XContentType -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionRetry -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionTimeout -import org.opensearch.indexmanagement.indexstatemanagement.model.action.IndexPriorityActionConfig +import org.opensearch.indexmanagement.indexstatemanagement.ISMActionsParser +import org.opensearch.indexmanagement.indexstatemanagement.action.DeleteAction import org.opensearch.indexmanagement.indexstatemanagement.randomAllocationActionConfig +import org.opensearch.indexmanagement.indexstatemanagement.randomCloseActionConfig +import org.opensearch.indexmanagement.indexstatemanagement.randomDeleteActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomForceMergeActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomIndexPriorityActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomNotificationActionConfig +import org.opensearch.indexmanagement.indexstatemanagement.randomOpenActionConfig +import org.opensearch.indexmanagement.indexstatemanagement.randomReadOnlyActionConfig +import org.opensearch.indexmanagement.indexstatemanagement.randomReadWriteActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomReplicaCountActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomRolloverActionConfig +import org.opensearch.indexmanagement.indexstatemanagement.randomRollupActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomSnapshotActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomTimeValueObject +import org.opensearch.indexmanagement.opensearchapi.convertToMap import org.opensearch.indexmanagement.opensearchapi.string +import org.opensearch.indexmanagement.spi.indexstatemanagement.Action +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionRetry +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionTimeout import org.opensearch.test.OpenSearchTestCase import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import kotlin.test.assertFailsWith -class ActionConfigTests : OpenSearchTestCase() { +class ActionTests : OpenSearchTestCase() { - fun `test invalid timeout for delete action config fails`() { + fun `test invalid timeout for delete action fails`() { assertFailsWith(IllegalArgumentException::class, "Expected IllegalArgumentException for invalid timeout") { ActionTimeout(timeout = TimeValue.parseTimeValue("invalidTimeout", "timeout_test")) } @@ -68,32 +76,63 @@ class ActionConfigTests : OpenSearchTestCase() { } } + fun `test set read write action round trip`() { + roundTripAction(randomReadWriteActionConfig()) + } + + fun `test set read only action round trip`() { + roundTripAction(randomReadOnlyActionConfig()) + } + fun `test rollover action round trip`() { - roundTripActionConfig(randomRolloverActionConfig()) + roundTripAction(randomRolloverActionConfig()) + } + + fun `test rollup action round trip`() { + roundTripAction(randomRollupActionConfig()) } fun `test replica count action round trip`() { - roundTripActionConfig(randomReplicaCountActionConfig()) + roundTripAction(randomReplicaCountActionConfig()) } fun `test force merge action round trip`() { - roundTripActionConfig(randomForceMergeActionConfig()) + roundTripAction(randomForceMergeActionConfig()) } fun `test notification action round trip`() { - roundTripActionConfig(randomNotificationActionConfig()) + roundTripAction(randomNotificationActionConfig()) } fun `test snapshot action round trip`() { - roundTripActionConfig(randomSnapshotActionConfig(snapshot = "snapshot", repository = "repository")) + roundTripAction(randomSnapshotActionConfig(snapshot = "snapshot", repository = "repository")) } fun `test index priority action round trip`() { - roundTripActionConfig(randomIndexPriorityActionConfig()) + roundTripAction(randomIndexPriorityActionConfig()) } fun `test allocation action round trip`() { - roundTripActionConfig(randomAllocationActionConfig(require = mapOf("box_type" to "hot"))) + roundTripAction( + randomAllocationActionConfig + ( + require = mapOf("box_type" to "hot"), + include = mapOf(randomAlphaOfLengthBetween(1, 10) to randomAlphaOfLengthBetween(1, 10)), + exclude = mapOf(randomAlphaOfLengthBetween(1, 10) to randomAlphaOfLengthBetween(1, 10)) + ) + ) + } + + fun `test close action round trip`() { + roundTripAction(randomCloseActionConfig()) + } + + fun `test open action round trip`() { + roundTripAction(randomOpenActionConfig()) + } + + fun `test delete action round trip`() { + roundTripAction(randomDeleteActionConfig()) } fun `test action timeout and retry round trip`() { @@ -105,25 +144,24 @@ class ActionConfigTests : OpenSearchTestCase() { .field(ActionRetry.BACKOFF_FIELD, ActionRetry.Backoff.EXPONENTIAL) .field(ActionRetry.DELAY_FIELD, TimeValue.timeValueMinutes(1)) .endObject() - .startObject(ActionConfig.ActionType.INDEX_PRIORITY.type) - .field(IndexPriorityActionConfig.INDEX_PRIORITY_FIELD, 10) + .startObject(DeleteAction.name) .endObject() .endObject() val parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, builder.string()) parser.nextToken() - val actionConfig = ActionConfig.parse(parser, 1) - roundTripActionConfig(actionConfig) + val action = ISMActionsParser.instance.parse(parser, 1) + roundTripAction(action) } - private fun roundTripActionConfig(expectedActionConfig: ActionConfig) { + private fun roundTripAction(expectedAction: Action) { val baos = ByteArrayOutputStream() val osso = OutputStreamStreamOutput(baos) - expectedActionConfig.writeTo(osso) + expectedAction.writeTo(osso) val input = InputStreamStreamInput(ByteArrayInputStream(baos.toByteArray())) - val actualActionConfig = ActionConfig.fromStreamInput(input) - assertEquals(expectedActionConfig, actualActionConfig) + val actualAction = ISMActionsParser.instance.fromStreamInput(input) + assertEquals(expectedAction.convertToMap(), actualAction.convertToMap()) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ManagedIndexMetaDataTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ManagedIndexMetaDataTests.kt index 16851c003..e16f91319 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ManagedIndexMetaDataTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ManagedIndexMetaDataTests.kt @@ -7,15 +7,17 @@ package org.opensearch.indexmanagement.indexstatemanagement.model import org.opensearch.common.io.stream.InputStreamStreamInput import org.opensearch.common.io.stream.OutputStreamStreamOutput -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionProperties -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.PolicyRetryInfoMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StateMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData -import org.opensearch.indexmanagement.indexstatemanagement.step.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.test.OpenSearchTestCase import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream +import java.time.Instant class ManagedIndexMetaDataTests : OpenSearchTestCase() { @@ -28,6 +30,7 @@ class ManagedIndexMetaDataTests : OpenSearchTestCase() { policyPrimaryTerm = 1, policyCompleted = null, rolledOver = null, + indexCreationDate = Instant.now().toEpochMilli(), transitionTo = null, stateMetaData = StateMetaData("close-index", 1234), actionMetaData = null, @@ -48,6 +51,7 @@ class ManagedIndexMetaDataTests : OpenSearchTestCase() { policyPrimaryTerm = 1, policyCompleted = null, rolledOver = null, + indexCreationDate = null, transitionTo = null, stateMetaData = StateMetaData("close-index", 1234), actionMetaData = ActionMetaData("close", 4321, 0, false, 0, 0, null), @@ -68,6 +72,7 @@ class ManagedIndexMetaDataTests : OpenSearchTestCase() { policyPrimaryTerm = 1, policyCompleted = null, rolledOver = null, + indexCreationDate = null, transitionTo = null, stateMetaData = StateMetaData("close-index", 1234), actionMetaData = ActionMetaData("close", 4321, 0, false, 0, 0, ActionProperties(3)), @@ -88,6 +93,7 @@ class ManagedIndexMetaDataTests : OpenSearchTestCase() { policyPrimaryTerm = 1, policyCompleted = null, rolledOver = false, + indexCreationDate = null, transitionTo = null, stateMetaData = StateMetaData("rollover-index", 1234), actionMetaData = ActionMetaData("rollover", 4321, 0, false, 0, 0, null), diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/XContentTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/XContentTests.kt index f5108da83..7397d4d40 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/XContentTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/XContentTests.kt @@ -8,18 +8,20 @@ package org.opensearch.indexmanagement.indexstatemanagement.model import org.opensearch.common.xcontent.LoggingDeprecationHandler import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentType -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.RollupActionConfig +import org.opensearch.indexmanagement.indexstatemanagement.ISMActionsParser +import org.opensearch.indexmanagement.indexstatemanagement.action.RollupAction import org.opensearch.indexmanagement.indexstatemanagement.model.destination.DestinationType import org.opensearch.indexmanagement.indexstatemanagement.nonNullRandomConditions import org.opensearch.indexmanagement.indexstatemanagement.randomAllocationActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomChangePolicy +import org.opensearch.indexmanagement.indexstatemanagement.randomCloseActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomDeleteActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomDestination import org.opensearch.indexmanagement.indexstatemanagement.randomForceMergeActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomIndexPriorityActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomManagedIndexConfig import org.opensearch.indexmanagement.indexstatemanagement.randomNotificationActionConfig +import org.opensearch.indexmanagement.indexstatemanagement.randomOpenActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomPolicy import org.opensearch.indexmanagement.indexstatemanagement.randomReadOnlyActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomReadWriteActionConfig @@ -30,7 +32,9 @@ import org.opensearch.indexmanagement.indexstatemanagement.randomSnapshotActionC import org.opensearch.indexmanagement.indexstatemanagement.randomState import org.opensearch.indexmanagement.indexstatemanagement.randomTransition import org.opensearch.indexmanagement.indexstatemanagement.toJsonString +import org.opensearch.indexmanagement.opensearchapi.convertToMap import org.opensearch.indexmanagement.opensearchapi.parseWithType +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.test.OpenSearchTestCase class XContentTests : OpenSearchTestCase() { @@ -67,111 +71,110 @@ class XContentTests : OpenSearchTestCase() { assertEquals("Round tripping Conditions doesn't work", conditions, parsedConditions) } - fun `test action config parsing`() { - val deleteActionConfig = randomDeleteActionConfig() + fun `test delete action parsing`() { + val deleteAction = randomDeleteActionConfig() - val deleteActionConfigString = deleteActionConfig.toJsonString() - val parsedActionConfig = ActionConfig.parse((parser(deleteActionConfigString)), 0) - assertEquals("Round tripping ActionConfig doesn't work", deleteActionConfig as ActionConfig, parsedActionConfig) + val deleteActionString = deleteAction.toJsonString() + val parsedDeleteAction = ISMActionsParser.instance.parse(parser(deleteActionString), 0) + assertEquals("Round tripping DeleteAction doesn't work", deleteAction.convertToMap(), parsedDeleteAction.convertToMap()) } - fun `test delete action config parsing`() { - val deleteActionConfig = randomDeleteActionConfig() + fun `test rollover action parsing`() { + val rolloverAction = randomRolloverActionConfig() - val deleteActionConfigString = deleteActionConfig.toJsonString() - val parsedDeleteActionConfig = ActionConfig.parse(parser(deleteActionConfigString), 0) - assertEquals("Round tripping DeleteActionConfig doesn't work", deleteActionConfig, parsedDeleteActionConfig) + val rolloverActionString = rolloverAction.toJsonString() + val parsedRolloverAction = ISMActionsParser.instance.parse(parser(rolloverActionString), 0) + assertEquals("Round tripping RolloverAction doesn't work", rolloverAction.convertToMap(), parsedRolloverAction.convertToMap()) } - fun `test rollover action config parsing`() { - val rolloverActionConfig = randomRolloverActionConfig() + fun `test read_only action parsing`() { + val readOnlyAction = randomReadOnlyActionConfig() - val rolloverActionConfigString = rolloverActionConfig.toJsonString() - val parsedRolloverActionConfig = ActionConfig.parse(parser(rolloverActionConfigString), 0) - assertEquals("Round tripping RolloverActionConfig doesn't work", rolloverActionConfig, parsedRolloverActionConfig) + val readOnlyActionString = readOnlyAction.toJsonString() + val parsedReadOnlyAction = ISMActionsParser.instance.parse(parser(readOnlyActionString), 0) + assertEquals("Round tripping ReadOnlyAction doesn't work", readOnlyAction.convertToMap(), parsedReadOnlyAction.convertToMap()) } - fun `test read_only action config parsing`() { - val readOnlyActionConfig = randomReadOnlyActionConfig() + fun `test read_write action parsing`() { + val readWriteAction = randomReadWriteActionConfig() - val readOnlyActionConfigString = readOnlyActionConfig.toJsonString() - val parsedReadOnlyActionConfig = ActionConfig.parse(parser(readOnlyActionConfigString), 0) - assertEquals("Round tripping ReadOnlyActionConfig doesn't work", readOnlyActionConfig, parsedReadOnlyActionConfig) - } - - fun `test read_write action config parsing`() { - val readWriteActionConfig = randomReadWriteActionConfig() - - val readWriteActionConfigString = readWriteActionConfig.toJsonString() - val parsedReadWriteActionConfig = ActionConfig.parse(parser(readWriteActionConfigString), 0) - assertEquals("Round tripping ReadWriteActionConfig doesn't work", readWriteActionConfig, parsedReadWriteActionConfig) + val readWriteActionString = readWriteAction.toJsonString() + val parsedReadWriteAction = ISMActionsParser.instance.parse(parser(readWriteActionString), 0) + assertEquals("Round tripping ReadWriteAction doesn't work", readWriteAction.convertToMap(), parsedReadWriteAction.convertToMap()) } fun `test replica_count action config parsing`() { - val replicaCountActionConfig = randomReplicaCountActionConfig() + val replicaCountAction = randomReplicaCountActionConfig() - val replicaCountActionConfigString = replicaCountActionConfig.toJsonString() - val parsedReplicaCountActionConfig = ActionConfig.parse(parser(replicaCountActionConfigString), 0) - assertEquals("Round tripping ReplicaCountActionConfig doesn't work", replicaCountActionConfig, parsedReplicaCountActionConfig) + val replicaCountActionString = replicaCountAction.toJsonString() + val parsedReplicaCountAction = ISMActionsParser.instance.parse(parser(replicaCountActionString), 0) + assertEquals("Round tripping ReplicaCountAction doesn't work", replicaCountAction.convertToMap(), parsedReplicaCountAction.convertToMap()) } fun `test set_index_priority action config parsing`() { - val indexPriorityActionConfig = randomIndexPriorityActionConfig() + val indexPriorityAction = randomIndexPriorityActionConfig() - val indexPriorityActionConfigString = indexPriorityActionConfig.toJsonString() - val parsedIndexPriorityActionConfig = ActionConfig.parse(parser(indexPriorityActionConfigString), 0) - assertEquals("Round tripping indexPriorityActionConfig doesn't work", indexPriorityActionConfig, parsedIndexPriorityActionConfig) + val indexPriorityActionString = indexPriorityAction.toJsonString() + val parsedIndexPriorityAction = ISMActionsParser.instance.parse(parser(indexPriorityActionString), 0) + assertEquals("Round tripping indexPriorityAction doesn't work", indexPriorityAction.convertToMap(), parsedIndexPriorityAction.convertToMap()) } fun `test force_merge action config parsing`() { - val forceMergeActionConfig = randomForceMergeActionConfig() + val forceMergeAction = randomForceMergeActionConfig() - val forceMergeActionConfigString = forceMergeActionConfig.toJsonString() - val parsedForceMergeActionConfig = ActionConfig.parse(parser(forceMergeActionConfigString), 0) - assertEquals("Round tripping ForceMergeActionConfig doesn't work", forceMergeActionConfig, parsedForceMergeActionConfig) + val forceMergeActionString = forceMergeAction.toJsonString() + val parsedForceMergeAction = ISMActionsParser.instance.parse(parser(forceMergeActionString), 0) + assertEquals("Round tripping ForceMergeAction doesn't work", forceMergeAction.convertToMap(), parsedForceMergeAction.convertToMap()) } - fun `test notification action config parsing`() { - val chimeNotificationActionConfig = randomNotificationActionConfig(destination = randomDestination(type = DestinationType.CHIME)) - val slackNotificationActionConfig = randomNotificationActionConfig(destination = randomDestination(type = DestinationType.SLACK)) - val customNotificationActionConfig = randomNotificationActionConfig(destination = randomDestination(type = DestinationType.CUSTOM_WEBHOOK)) + fun `test notification action parsing`() { + val chimeNotificationAction = randomNotificationActionConfig(destination = randomDestination(type = DestinationType.CHIME)) + val slackNotificationAction = randomNotificationActionConfig(destination = randomDestination(type = DestinationType.SLACK)) + val customNotificationAction = randomNotificationActionConfig(destination = randomDestination(type = DestinationType.CUSTOM_WEBHOOK)) - val chimeNotificationActionConfigString = chimeNotificationActionConfig.toJsonString() - val chimeParsedNotificationActionConfig = ActionConfig.parse(parser(chimeNotificationActionConfigString), 0) + val chimeNotificationActionString = chimeNotificationAction.toJsonString() + val chimeParsedNotificationAction = ISMActionsParser.instance.parse(parser(chimeNotificationActionString), 0) assertEquals( - "Round tripping chime NotificationActionConfig doesn't work", - chimeNotificationActionConfig, chimeParsedNotificationActionConfig + "Round tripping chime NotificationAction doesn't work", + chimeNotificationAction.convertToMap(), chimeParsedNotificationAction.convertToMap() ) - val slackNotificationActionConfigString = slackNotificationActionConfig.toJsonString() - val slackParsedNotificationActionConfig = ActionConfig.parse(parser(slackNotificationActionConfigString), 0) + val slackNotificationActionString = slackNotificationAction.toJsonString() + val slackParsedNotificationAction = ISMActionsParser.instance.parse(parser(slackNotificationActionString), 0) assertEquals( - "Round tripping slack NotificationActionConfig doesn't work", - slackNotificationActionConfig, slackParsedNotificationActionConfig + "Round tripping slack NotificationAction doesn't work", + slackNotificationAction.convertToMap(), slackParsedNotificationAction.convertToMap() ) - val customNotificationActionConfigString = customNotificationActionConfig.toJsonString() - val customParsedNotificationActionConfig = ActionConfig.parse(parser(customNotificationActionConfigString), 0) + val customNotificationActionString = customNotificationAction.toJsonString() + val customParsedNotificationAction = ISMActionsParser.instance.parse(parser(customNotificationActionString), 0) assertEquals( - "Round tripping custom webhook NotificationActionConfig doesn't work", - customNotificationActionConfig, customParsedNotificationActionConfig + "Round tripping custom webhook NotificationAction doesn't work", + customNotificationAction.convertToMap(), customParsedNotificationAction.convertToMap() ) } fun `test snapshot action config parsing`() { - val snapshotActionConfig = randomSnapshotActionConfig("repository", "snapshot") + val snapshotAction = randomSnapshotActionConfig("repository", "snapshot") - val snapshotActionConfigString = snapshotActionConfig.toJsonString() - val parsedNotificationActionConfig = ActionConfig.parse(parser(snapshotActionConfigString), 0) - assertEquals("Round tripping SnapshotActionConfig doesn't work", snapshotActionConfig, parsedNotificationActionConfig) + val snapshotActionString = snapshotAction.toJsonString() + val parsedSnapshotAction = ISMActionsParser.instance.parse(parser(snapshotActionString), 0) + assertEquals( + "Round tripping SnapshotAction doesn't work", + snapshotAction.convertToMap(), parsedSnapshotAction.convertToMap() + ) } fun `test allocation action config parsing`() { - val allocationActionConfig = randomAllocationActionConfig(require = mapOf("box_type" to "hot")) + val allocationAction = randomAllocationActionConfig( + require = mapOf("box_type" to "hot"), + include = mapOf(randomAlphaOfLengthBetween(1, 10) to randomAlphaOfLengthBetween(1, 10)), + exclude = mapOf(randomAlphaOfLengthBetween(1, 10) to randomAlphaOfLengthBetween(1, 10)) + ) - val allocationActionConfigString = allocationActionConfig.toJsonString() - val parsedAllocationActionConfig = ActionConfig.parse(parser(allocationActionConfigString), 0) - assertEquals("Round tripping AllocationActionConfig doesn't work", allocationActionConfig, parsedAllocationActionConfig) + val allocationActionString = allocationAction.toJsonString() + val parsedAllocationAction = ISMActionsParser.instance.parse(parser(allocationActionString), 0) + assertEquals("Round tripping AllocationAction doesn't work", allocationAction.convertToMap(), parsedAllocationAction.convertToMap()) } fun `test managed index config parsing`() { @@ -196,12 +199,28 @@ class XContentTests : OpenSearchTestCase() { } fun `test rollup action parsing`() { - val rollupActionConfig = randomRollupActionConfig() - val rollupActionConfigString = rollupActionConfig.toJsonString() - val parsedRollupActionConfig = ActionConfig.parse(parser(rollupActionConfigString), 0) as RollupActionConfig + val rollupAction = randomRollupActionConfig() + val rollupActionString = rollupAction.toJsonString() + val parsedRollupAction = ISMActionsParser.instance.parse(parser(rollupActionString), 0) as RollupAction + + assertEquals("Round tripping RollupAction doesn't work", rollupAction.actionIndex, parsedRollupAction.actionIndex) + assertEquals("Round tripping RollupAction doesn't work", rollupAction.ismRollup, parsedRollupAction.ismRollup) + } + + fun `test close action parsing`() { + val closeAction = randomCloseActionConfig() + val closeActionString = closeAction.toJsonString() + val parsedCloseAction = ISMActionsParser.instance.parse(parser(closeActionString), 0) + + assertEquals("Round tripping CloseAction doesn't work", closeAction.convertToMap(), parsedCloseAction.convertToMap()) + } + + fun `test open action parsing`() { + val openAction = randomOpenActionConfig() + val openActionString = openAction.toJsonString() + val parsedOpenAction = ISMActionsParser.instance.parse(parser(openActionString), 0) - assertEquals("Round tripping RollupActionConfig doesn't work", rollupActionConfig.index, parsedRollupActionConfig.index) - assertEquals("Round tripping RollupActionConfig doesn't work", rollupActionConfig.ismRollup, parsedRollupActionConfig.ismRollup) + assertEquals("Round tripping OpenAction doesn't work", openAction.convertToMap(), parsedOpenAction.convertToMap()) } fun `test managed index metadata parsing`() { @@ -213,6 +232,7 @@ class XContentTests : OpenSearchTestCase() { policyPrimaryTerm = randomNonNegativeLong(), policyCompleted = null, rolledOver = null, + indexCreationDate = null, transitionTo = randomAlphaOfLength(10), stateMetaData = null, actionMetaData = null, diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt index 2770f9090..563c3de50 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/ISMTemplateRestAPIIT.kt @@ -8,14 +8,15 @@ package org.opensearch.indexmanagement.indexstatemanagement.resthandler import org.opensearch.client.ResponseException import org.opensearch.common.settings.Settings import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase +import org.opensearch.indexmanagement.indexstatemanagement.action.ReadOnlyAction import org.opensearch.indexmanagement.indexstatemanagement.model.ISMTemplate import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReadOnlyActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.indexstatemanagement.randomPolicy import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_HIDDEN import org.opensearch.indexmanagement.randomInstant +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.waitFor import org.opensearch.rest.RestStatus import java.time.Instant @@ -30,6 +31,7 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { private val policyID2 = "t2" private val policyID3 = "t3" + @Suppress("UNCHECKED_CAST") fun `test add template with invalid index pattern`() { try { val ismTemp = ISMTemplate(listOf(" "), 100, randomInstant()) @@ -43,6 +45,7 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { } } + @Suppress("UNCHECKED_CAST") fun `test add template with self-overlapping index pattern`() { try { val ismTemp = ISMTemplate(listOf("ab*"), 100, randomInstant()) @@ -57,6 +60,7 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { } } + @Suppress("UNCHECKED_CAST") fun `test add template with overlapping index pattern`() { try { val ismTemp = ISMTemplate(listOf("log*"), 100, randomInstant()) @@ -85,9 +89,9 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { val ismTemp = ISMTemplate(listOf("log*"), 100, randomInstant()) - val actionConfig = ReadOnlyActionConfig(0) + val action = ReadOnlyAction(0) val states = listOf( - State("ReadOnlyState", listOf(actionConfig), listOf()) + State("ReadOnlyState", listOf(action), listOf()) ) val policy = Policy( id = policyID, @@ -116,7 +120,8 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { listOf( indexName1 to listOf( explainResponseOpendistroPolicyIdSetting to fun(policyID: Any?): Boolean = policyID == null, - explainResponseOpenSearchPolicyIdSetting to fun(policyID: Any?): Boolean = policyID == null + explainResponseOpenSearchPolicyIdSetting to fun(policyID: Any?): Boolean = policyID == null, + ManagedIndexMetaData.ENABLED to fun(enabled: Any?): Boolean = enabled == null ) ), getExplainMap(indexName1), @@ -129,7 +134,8 @@ class ISMTemplateRestAPIIT : IndexStateManagementRestTestCase() { listOf( indexName1 to listOf( explainResponseOpendistroPolicyIdSetting to fun(policyID: Any?): Boolean = policyID == null, - explainResponseOpenSearchPolicyIdSetting to fun(policyID: Any?): Boolean = policyID == null + explainResponseOpenSearchPolicyIdSetting to fun(policyID: Any?): Boolean = policyID == null, + ManagedIndexMetaData.ENABLED to fun(enabled: Any?): Boolean = enabled == null ) ), getExplainMap(indexName1), diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt index 30c1c3263..cb50d18ed 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt @@ -13,9 +13,10 @@ import org.opensearch.common.xcontent.XContentType import org.opensearch.common.xcontent.json.JsonXContent.jsonXContent import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.POLICY_BASE_URI +import org.opensearch.indexmanagement.indexstatemanagement.ISMActionsParser import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase +import org.opensearch.indexmanagement.indexstatemanagement.action.ReadOnlyAction import org.opensearch.indexmanagement.indexstatemanagement.model.Policy -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomPolicy import org.opensearch.indexmanagement.indexstatemanagement.randomReadOnlyActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomState @@ -93,8 +94,8 @@ class IndexStateManagementRestApiIT : IndexStateManagementRestTestCase() { fun `test creating a policy with a disallowed actions fails`() { try { // remove read_only from the allowlist - val allowedActions = ActionConfig.ActionType.values().toList() - .filter { actionType -> actionType != ActionConfig.ActionType.READ_ONLY } + val allowedActions = ISMActionsParser.instance.parsers.map { it.getActionType() }.toList() + .filter { actionType -> actionType != ReadOnlyAction.name } .joinToString(prefix = "[", postfix = "]") { string -> "\"$string\"" } updateClusterSetting(ManagedIndexSettings.ALLOW_LIST.key, allowedActions, escapeValue = false) val policy = randomPolicy(states = listOf(randomState(actions = listOf(randomReadOnlyActionConfig())))) @@ -109,8 +110,8 @@ class IndexStateManagementRestApiIT : IndexStateManagementRestTestCase() { fun `test updating a policy with a disallowed actions fails`() { try { // remove read_only from the allowlist - val allowedActions = ActionConfig.ActionType.values().toList() - .filter { actionType -> actionType != ActionConfig.ActionType.READ_ONLY } + val allowedActions = ISMActionsParser.instance.parsers.map { it.getActionType() }.toList() + .filter { actionType -> actionType != ReadOnlyAction.name } .joinToString(prefix = "[", postfix = "]") { string -> "\"$string\"" } updateClusterSetting(ManagedIndexSettings.ALLOW_LIST.key, allowedActions, escapeValue = false) // createRandomPolicy currently does not create a random list of actions so it won't accidentally create one with read_only diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyActionIT.kt index ad9af4c0c..22c2f4586 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyActionIT.kt @@ -10,18 +10,15 @@ import org.opensearch.client.ResponseException import org.opensearch.common.settings.Settings import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase +import org.opensearch.indexmanagement.indexstatemanagement.action.DeleteAction +import org.opensearch.indexmanagement.indexstatemanagement.action.OpenAction +import org.opensearch.indexmanagement.indexstatemanagement.action.ReadOnlyAction +import org.opensearch.indexmanagement.indexstatemanagement.action.RolloverAction +import org.opensearch.indexmanagement.indexstatemanagement.action.TransitionsAction import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.indexstatemanagement.model.StateFilter import org.opensearch.indexmanagement.indexstatemanagement.model.Transition -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.DeleteActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.OpenActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReadOnlyActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.RolloverActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StateMetaData import org.opensearch.indexmanagement.indexstatemanagement.randomPolicy import org.opensearch.indexmanagement.indexstatemanagement.randomReplicaCountActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomState @@ -31,6 +28,9 @@ import org.opensearch.indexmanagement.indexstatemanagement.util.FAILED_INDICES import org.opensearch.indexmanagement.indexstatemanagement.util.FAILURES import org.opensearch.indexmanagement.indexstatemanagement.util.UPDATED_INDICES import org.opensearch.indexmanagement.makeRequest +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaData import org.opensearch.indexmanagement.waitFor import org.opensearch.rest.RestRequest import org.opensearch.rest.RestStatus @@ -272,15 +272,15 @@ class RestChangePolicyActionIT : IndexStateManagementRestTestCase() { fun `test changing policy on an index in a state`() { // Creates a policy that has one state with one action (sets index to read only) - val stateWithReadOnlyAction = randomState(actions = listOf(ReadOnlyActionConfig(index = 0))) + val stateWithReadOnlyAction = randomState(actions = listOf(ReadOnlyAction(index = 0))) val randomPolicy = randomPolicy(states = listOf(stateWithReadOnlyAction)) val policy = createPolicy(randomPolicy) // Creates new policy that has two states, same as before except a second state with a delete action and a transition from readonly to delete states // we will also add a new action to readonly state otherwise an immediate change policy is triggered - val stateWithDeleteAction = randomState(actions = listOf(DeleteActionConfig(index = 0))) + val stateWithDeleteAction = randomState(actions = listOf(DeleteAction(index = 0))) val updatedStateWithReadOnlyAction = stateWithReadOnlyAction.copy( - actions = listOf(stateWithReadOnlyAction.actions.first(), OpenActionConfig(index = 1)), + actions = listOf(stateWithReadOnlyAction.actions.first(), OpenAction(index = 1)), transitions = listOf(Transition(stateWithDeleteAction.name, null)) ) val newPolicy = createPolicy(randomPolicy(states = listOf(updatedStateWithReadOnlyAction, stateWithDeleteAction)), "new_policy", true) @@ -373,7 +373,7 @@ class RestChangePolicyActionIT : IndexStateManagementRestTestCase() { ActionMetaData.ACTION to fun(actionMetaDataMap: Any?): Boolean = assertActionEquals( ActionMetaData( - name = ActionConfig.ActionType.READ_ONLY.type, startTime = Instant.now().toEpochMilli(), index = 0, + name = ReadOnlyAction.name, startTime = Instant.now().toEpochMilli(), index = 0, failed = false, consumedRetries = 0, lastRetryTime = null, actionProperties = null ), actionMetaDataMap @@ -412,7 +412,7 @@ class RestChangePolicyActionIT : IndexStateManagementRestTestCase() { ActionMetaData.ACTION to fun(actionMetaDataMap: Any?): Boolean = assertActionEquals( ActionMetaData( - name = ActionConfig.ActionType.TRANSITION.type, startTime = Instant.now().toEpochMilli(), index = 0, + name = TransitionsAction.name, startTime = Instant.now().toEpochMilli(), index = 0, failed = false, consumedRetries = 0, lastRetryTime = null, actionProperties = null ), actionMetaDataMap @@ -551,8 +551,8 @@ class RestChangePolicyActionIT : IndexStateManagementRestTestCase() { fun `test allowing change policy to happen in middle of state if same state structure`() { // Creates a policy that has one state with rollover - val actionConfig = RolloverActionConfig(index = 0, minDocs = 100_000_000, minAge = null, minSize = null, minPrimaryShardSize = null) - val stateWithReadOnlyAction = randomState(actions = listOf(actionConfig)) + val action = RolloverAction(index = 0, minDocs = 100_000_000, minAge = null, minSize = null, minPrimaryShardSize = null) + val stateWithReadOnlyAction = randomState(actions = listOf(action)) val randomPolicy = randomPolicy(states = listOf(stateWithReadOnlyAction)) val policy = createPolicy(randomPolicy) val indexName = "${testIndexName}_safe-000001" @@ -574,14 +574,17 @@ class RestChangePolicyActionIT : IndexStateManagementRestTestCase() { updateManagedIndexConfigStartTime(managedIndexConfig) // verify we are in rollover and have not completed it yet waitFor { - assertEquals(ActionConfig.ActionType.ROLLOVER.type, getExplainManagedIndexMetaData(indexName).actionMetaData?.name) + assertEquals(RolloverAction.name, getExplainManagedIndexMetaData(indexName).actionMetaData?.name) assertEquals( AttemptRolloverStep.getPendingMessage(indexName), getExplainManagedIndexMetaData(indexName).info?.get("message") ) } - val newStateWithReadOnlyAction = randomState(name = stateWithReadOnlyAction.name, actions = listOf(actionConfig.copy(minDocs = 5))) + val newStateWithReadOnlyAction = randomState( + name = stateWithReadOnlyAction.name, + actions = listOf(RolloverAction(index = 0, minDocs = 5, minAge = null, minSize = null, minPrimaryShardSize = null)) + ) val newRandomPolicy = randomPolicy(states = listOf(newStateWithReadOnlyAction)) val newPolicy = createPolicy(newRandomPolicy) val changePolicy = ChangePolicy(newPolicy.id, null, emptyList(), false) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt index 51129c2a6..c7a980932 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt @@ -5,13 +5,18 @@ package org.opensearch.indexmanagement.indexstatemanagement.resthandler +import org.opensearch.common.xcontent.XContentFactory import org.opensearch.indexmanagement.IndexManagementPlugin import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.PolicyRetryInfoMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StateMetaData +import org.opensearch.indexmanagement.indexstatemanagement.util.SHOW_POLICY_QUERY_PARAM +import org.opensearch.indexmanagement.indexstatemanagement.util.TOTAL_MANAGED_INDICES +import org.opensearch.indexmanagement.indexstatemanagement.util.XCONTENT_WITHOUT_TYPE_AND_USER import org.opensearch.indexmanagement.makeRequest +import org.opensearch.indexmanagement.opensearchapi.toMap +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaData import org.opensearch.indexmanagement.waitFor import org.opensearch.rest.RestRequest import org.opensearch.rest.RestStatus @@ -26,9 +31,11 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { val indexName = "${testIndexName}_movies" createIndex(indexName, null) val expected = mapOf( - indexName to mapOf( + TOTAL_MANAGED_INDICES to 0, + indexName to mapOf( explainResponseOpendistroPolicyIdSetting to null, - explainResponseOpenSearchPolicyIdSetting to null + explainResponseOpenSearchPolicyIdSetting to null, + ManagedIndexMetaData.ENABLED to null ) ) assertResponseMap(expected, getExplainMap(indexName)) @@ -52,16 +59,19 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { createIndex(indexName2, null) val expected = mapOf( + TOTAL_MANAGED_INDICES to 1, indexName1 to mapOf( explainResponseOpendistroPolicyIdSetting to policy.id, explainResponseOpenSearchPolicyIdSetting to policy.id, "index" to indexName1, "index_uuid" to getUuid(indexName1), - "policy_id" to policy.id + "policy_id" to policy.id, + ManagedIndexMetaData.ENABLED to true ), indexName2 to mapOf( explainResponseOpendistroPolicyIdSetting to null, - explainResponseOpenSearchPolicyIdSetting to null + explainResponseOpenSearchPolicyIdSetting to null, + ManagedIndexMetaData.ENABLED to null ) ) waitFor { @@ -102,23 +112,27 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { createIndex(indexName2, policyID = policy.id) createIndex(indexName3, null) val expected = mapOf( + TOTAL_MANAGED_INDICES to 2, indexName1 to mapOf( explainResponseOpendistroPolicyIdSetting to policy.id, explainResponseOpenSearchPolicyIdSetting to policy.id, "index" to indexName1, "index_uuid" to getUuid(indexName1), - "policy_id" to policy.id + "policy_id" to policy.id, + ManagedIndexMetaData.ENABLED to true ), indexName2 to mapOf( explainResponseOpendistroPolicyIdSetting to policy.id, explainResponseOpenSearchPolicyIdSetting to policy.id, "index" to indexName2, "index_uuid" to getUuid(indexName2), - "policy_id" to policy.id + "policy_id" to policy.id, + ManagedIndexMetaData.ENABLED to true ), indexName3 to mapOf( explainResponseOpendistroPolicyIdSetting to null, - explainResponseOpenSearchPolicyIdSetting to null + explainResponseOpenSearchPolicyIdSetting to null, + ManagedIndexMetaData.ENABLED to null ) ) waitFor { @@ -256,6 +270,7 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { ManagedIndexMetaData.POLICY_ID to managedIndexConfig.policyID::equals, ManagedIndexMetaData.POLICY_SEQ_NO to policy.seqNo.toInt()::equals, ManagedIndexMetaData.POLICY_PRIMARY_TERM to policy.primaryTerm.toInt()::equals, + ManagedIndexMetaData.INDEX_CREATION_DATE to fun(indexCreationDate: Any?): Boolean = (indexCreationDate as Long) > 1L, StateMetaData.STATE to fun(stateMetaDataMap: Any?): Boolean = assertStateEquals( StateMetaData(policy.defaultState, Instant.now().toEpochMilli()), @@ -263,7 +278,8 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { ), PolicyRetryInfoMetaData.RETRY_INFO to fun(retryInfoMetaDataMap: Any?): Boolean = assertRetryInfoEquals(PolicyRetryInfoMetaData(false, 0), retryInfoMetaDataMap), - ManagedIndexMetaData.INFO to fun(info: Any?): Boolean = expectedInfoString == info.toString() + ManagedIndexMetaData.INFO to fun(info: Any?): Boolean = expectedInfoString == info.toString(), + ManagedIndexMetaData.ENABLED to true::equals ) ), getExplainMap(indexName) @@ -303,7 +319,9 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { ManagedIndexMetaData.POLICY_ID to newPolicy.id::equals, PolicyRetryInfoMetaData.RETRY_INFO to fun(retryInfoMetaDataMap: Any?): Boolean = assertRetryInfoEquals(PolicyRetryInfoMetaData(true, 0), retryInfoMetaDataMap), - ManagedIndexMetaData.INFO to fun(info: Any?): Boolean = expectedInfoString == info.toString() + ManagedIndexMetaData.INFO to fun(info: Any?): Boolean = expectedInfoString == info.toString(), + ManagedIndexMetaData.INDEX_CREATION_DATE to fun(indexCreationDate: Any?): Boolean = (indexCreationDate as Long) > 1L, + ManagedIndexMetaData.ENABLED to true::equals ) ), getExplainMap(indexName) @@ -311,6 +329,29 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { } } + fun `test show_applied_policy query parameter`() { + val indexName = "${testIndexName}_show_applied_policy" + val policy = createRandomPolicy() + createIndex(indexName, policy.id) + + val expectedPolicy = policy.toXContent(XContentFactory.jsonBuilder(), XCONTENT_WITHOUT_TYPE_AND_USER).toMap() + val expected = mapOf( + indexName to mapOf( + explainResponseOpendistroPolicyIdSetting to policy.id, + explainResponseOpenSearchPolicyIdSetting to policy.id, + "index" to indexName, + "index_uuid" to getUuid(indexName), + "policy_id" to policy.id, + ManagedIndexMetaData.ENABLED to true, + "policy" to expectedPolicy + ), + TOTAL_MANAGED_INDICES to 1, + ) + waitFor { + assertResponseMap(expected, getExplainMap(indexName, queryParams = SHOW_POLICY_QUERY_PARAM)) + } + } + @Suppress("UNCHECKED_CAST") // Do assertion of the response map here so we don't have many places to do suppression. private fun assertResponseMap(expected: Map, actual: Map) { assertEquals("Explain Map does not match", expected.size, actual.size) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexActionIT.kt index fcc99c039..07c7fd244 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexActionIT.kt @@ -7,18 +7,18 @@ package org.opensearch.indexmanagement.indexstatemanagement.resthandler import org.opensearch.client.ResponseException import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionRetry -import org.opensearch.indexmanagement.indexstatemanagement.model.action.AllocationActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData +import org.opensearch.indexmanagement.indexstatemanagement.action.AllocationAction import org.opensearch.indexmanagement.indexstatemanagement.randomForceMergeActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomPolicy import org.opensearch.indexmanagement.indexstatemanagement.randomState -import org.opensearch.indexmanagement.indexstatemanagement.step.Step import org.opensearch.indexmanagement.indexstatemanagement.util.FAILED_INDICES import org.opensearch.indexmanagement.indexstatemanagement.util.FAILURES import org.opensearch.indexmanagement.indexstatemanagement.util.UPDATED_INDICES import org.opensearch.indexmanagement.makeRequest +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionRetry +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.waitFor import org.opensearch.rest.RestRequest import org.opensearch.rest.RestStatus @@ -209,7 +209,7 @@ class RestRetryFailedManagedIndexActionIT : IndexStateManagementRestTestCase() { fun `test index failed`() { val indexName = "${testIndexName}_blueberry" - val config = AllocationActionConfig(require = mapOf("..//" to "value"), exclude = emptyMap(), include = emptyMap(), index = 0) + val config = AllocationAction(require = mapOf("..//" to "value"), exclude = emptyMap(), include = emptyMap(), index = 0) config.configRetry = ActionRetry(0) val states = listOf( randomState().copy( @@ -254,9 +254,9 @@ class RestRetryFailedManagedIndexActionIT : IndexStateManagementRestTestCase() { fun `test reset action start time`() { val indexName = "${testIndexName}_drewberry" val policyID = "${testIndexName}_policy_1" - val config = randomForceMergeActionConfig(maxNumSegments = 1) - config.configRetry = ActionRetry(0) - val policy = randomPolicy(states = listOf(randomState(actions = listOf(config)))) + val action = randomForceMergeActionConfig(maxNumSegments = 1) + action.configRetry = ActionRetry(0) + val policy = randomPolicy(states = listOf(randomState(actions = listOf(action)))) createPolicy(policy, policyId = policyID) createIndex(indexName, policyID) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/runner/ManagedIndexRunnerIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/runner/ManagedIndexRunnerIT.kt index 5b3ff36a1..38d013232 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/runner/ManagedIndexRunnerIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/runner/ManagedIndexRunnerIT.kt @@ -5,13 +5,12 @@ package org.opensearch.indexmanagement.indexstatemanagement.runner +import org.opensearch.indexmanagement.indexstatemanagement.ISMActionsParser import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.indexstatemanagement.action.OpenAction +import org.opensearch.indexmanagement.indexstatemanagement.action.ReadOnlyAction import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.OpenActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.PolicyRetryInfoMetaData import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.indexstatemanagement.randomPolicy import org.opensearch.indexmanagement.indexstatemanagement.randomReadOnlyActionConfig @@ -22,6 +21,8 @@ import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndex import org.opensearch.indexmanagement.indexstatemanagement.step.readonly.SetReadOnlyStep import org.opensearch.indexmanagement.indexstatemanagement.step.readwrite.SetReadWriteStep import org.opensearch.indexmanagement.indexstatemanagement.step.transition.AttemptTransitionStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData import org.opensearch.indexmanagement.waitFor import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule import java.time.Instant @@ -32,7 +33,7 @@ class ManagedIndexRunnerIT : IndexStateManagementRestTestCase() { fun `test version conflict fails job`() { val indexName = "version_conflict_index" val policyID = "version_conflict_policy" - val actionConfig = OpenActionConfig(0) + val actionConfig = OpenAction(0) val states = listOf(State("OpenState", listOf(actionConfig), listOf())) val policy = Policy( @@ -92,6 +93,12 @@ class ManagedIndexRunnerIT : IndexStateManagementRestTestCase() { // init policy updateManagedIndexConfigStartTime(managedIndexConfig) + waitFor { assertEquals(createdPolicy.id, getManagedIndexConfigByDocId(managedIndexConfig.id)?.policyID) } + // change cluster job interval setting to 2 (minutes) + updateClusterSetting(ManagedIndexSettings.JOB_INTERVAL.key, "2") + // fast forward to next execution where at the end we should change the job interval time + updateManagedIndexConfigStartTime(managedIndexConfig) + waitFor { (getManagedIndexConfigByDocId(managedIndexConfig.id)?.jobSchedule as? IntervalSchedule)?.interval == 2 } waitFor { assertEquals(createdPolicy.id, getManagedIndexConfigByDocId(managedIndexConfig.id)?.policyID) val currInterval = (getManagedIndexConfigByDocId(managedIndexConfig.id)?.jobSchedule as? IntervalSchedule)?.interval @@ -162,8 +169,8 @@ class ManagedIndexRunnerIT : IndexStateManagementRestTestCase() { waitFor { assertEquals(AttemptTransitionStep.getSuccessMessage(indexName, firstState.name), getExplainManagedIndexMetaData(indexName).info?.get("message")) } // remove read_only from the allowlist - val allowedActions = ActionConfig.ActionType.values().toList() - .filter { actionType -> actionType != ActionConfig.ActionType.READ_ONLY } + val allowedActions = ISMActionsParser.instance.parsers.map { it.getActionType() }.toList() + .filter { actionType -> actionType != ReadOnlyAction.name } .joinToString(prefix = "[", postfix = "]") { string -> "\"$string\"" } updateClusterSetting(ManagedIndexSettings.ALLOW_LIST.key, allowedActions, escapeValue = false) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptCloseStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptCloseStepTests.kt index 4a12d584b..11e01c6a5 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptCloseStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptCloseStepTests.kt @@ -17,9 +17,12 @@ import org.opensearch.client.AdminClient import org.opensearch.client.Client import org.opensearch.client.IndicesAdminClient import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.CloseActionConfig +import org.opensearch.common.settings.Settings import org.opensearch.indexmanagement.indexstatemanagement.step.close.AttemptCloseStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.script.ScriptService import org.opensearch.snapshots.SnapshotInProgressException import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException @@ -28,17 +31,19 @@ import kotlin.IllegalArgumentException class AttemptCloseStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() + private val scriptService: ScriptService = mock() + private val settings: Settings = Settings.EMPTY fun `test close step sets step status to completed when successful`() { val closeIndexResponse = CloseIndexResponse(true, true, listOf()) val client = getClient(getAdminClient(getIndicesAdminClient(closeIndexResponse, null))) runBlocking { - val closeActionConfig = CloseActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptCloseStep = AttemptCloseStep(clusterService, client, closeActionConfig, managedIndexMetaData) - attemptCloseStep.execute() - val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptCloseStep = AttemptCloseStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptCloseStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -48,11 +53,11 @@ class AttemptCloseStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(closeIndexResponse, null))) runBlocking { - val closeActionConfig = CloseActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptCloseStep = AttemptCloseStep(clusterService, client, closeActionConfig, managedIndexMetaData) - attemptCloseStep.execute() - val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptCloseStep = AttemptCloseStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptCloseStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -62,11 +67,11 @@ class AttemptCloseStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val closeActionConfig = CloseActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptCloseStep = AttemptCloseStep(clusterService, client, closeActionConfig, managedIndexMetaData) - attemptCloseStep.execute() - val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptCloseStep = AttemptCloseStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptCloseStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -76,11 +81,11 @@ class AttemptCloseStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val closeActionConfig = CloseActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptCloseStep = AttemptCloseStep(clusterService, client, closeActionConfig, managedIndexMetaData) - attemptCloseStep.execute() - val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptCloseStep = AttemptCloseStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptCloseStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not CONDITION_NOT_MET", Step.StepStatus.CONDITION_NOT_MET, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -90,11 +95,11 @@ class AttemptCloseStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val closeActionConfig = CloseActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptCloseStep = AttemptCloseStep(clusterService, client, closeActionConfig, managedIndexMetaData) - attemptCloseStep.execute() - val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptCloseStep = AttemptCloseStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptCloseStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not CONDITION_NOT_MET", Step.StepStatus.CONDITION_NOT_MET, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -104,11 +109,11 @@ class AttemptCloseStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val closeActionConfig = CloseActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptCloseStep = AttemptCloseStep(clusterService, client, closeActionConfig, managedIndexMetaData) - attemptCloseStep.execute() - val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptCloseStep = AttemptCloseStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptCloseStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "nested", updatedManagedIndexMetaData.info!!["cause"]) } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptCreateRollupJobStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptCreateRollupJobStepTests.kt index 547618153..f41eca7c3 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptCreateRollupJobStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptCreateRollupJobStepTests.kt @@ -5,33 +5,28 @@ package org.opensearch.indexmanagement.indexstatemanagement.step -import com.nhaarman.mockitokotlin2.mock -import org.opensearch.client.Client -import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionProperties import org.opensearch.indexmanagement.indexstatemanagement.randomRollupActionConfig import org.opensearch.indexmanagement.indexstatemanagement.step.rollup.AttemptCreateRollupJobStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.test.OpenSearchTestCase -import java.lang.Exception class AttemptCreateRollupJobStepTests : OpenSearchTestCase() { - private val rollupActionConfig = randomRollupActionConfig() + private val rollupAction = randomRollupActionConfig() private val indexName = "test" - private val rollupId: String = rollupActionConfig.ismRollup.toRollup(indexName).id - private val client: Client = mock() - private val clusterService: ClusterService = mock() + private val rollupId: String = rollupAction.ismRollup.toRollup(indexName).id private val metadata = ManagedIndexMetaData( - indexName, "indexUuid", "policy_id", null, null, null, null, null, null, + indexName, "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(AttemptCreateRollupJobStep.name, 1, 0, false, 0, null, ActionProperties(rollupId = rollupId)), null, null, null ) - private val step = AttemptCreateRollupJobStep(clusterService, client, rollupActionConfig.ismRollup, metadata) + private val step = AttemptCreateRollupJobStep(rollupAction) fun `test process failure`() { - step.processFailure(rollupId, Exception("dummy-error")) - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + step.processFailure(rollupId, indexName, Exception("dummy-error")) + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals( "Error message is not expected", diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptDeleteStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptDeleteStepTests.kt index 655fd18a7..7b6963dba 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptDeleteStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptDeleteStepTests.kt @@ -6,38 +6,42 @@ package org.opensearch.indexmanagement.indexstatemanagement.step import com.nhaarman.mockitokotlin2.any -import com.nhaarman.mockitokotlin2.doAnswer import com.nhaarman.mockitokotlin2.doReturn import com.nhaarman.mockitokotlin2.mock import com.nhaarman.mockitokotlin2.whenever import kotlinx.coroutines.runBlocking +import org.mockito.Mockito.doAnswer import org.opensearch.action.ActionListener import org.opensearch.action.support.master.AcknowledgedResponse import org.opensearch.client.AdminClient import org.opensearch.client.Client import org.opensearch.client.IndicesAdminClient import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.DeleteActionConfig +import org.opensearch.common.settings.Settings import org.opensearch.indexmanagement.indexstatemanagement.step.delete.AttemptDeleteStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.script.ScriptService import org.opensearch.snapshots.SnapshotInProgressException import org.opensearch.test.OpenSearchTestCase -import kotlin.IllegalArgumentException class AttemptDeleteStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() + private val scriptService: ScriptService = mock() + private val settings: Settings = Settings.EMPTY fun `test delete step sets step status to completed when successful`() { val acknowledgedResponse = AcknowledgedResponse(true) val client = getClient(getAdminClient(getIndicesAdminClient(acknowledgedResponse, null))) runBlocking { - val deleteActionConfig = DeleteActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptDeleteStep = AttemptDeleteStep(clusterService, client, deleteActionConfig, managedIndexMetaData) - attemptDeleteStep.execute() - val updatedManagedIndexMetaData = attemptDeleteStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptDeleteStep = AttemptDeleteStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptDeleteStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptDeleteStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -47,11 +51,11 @@ class AttemptDeleteStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(acknowledgedResponse, null))) runBlocking { - val deleteActionConfig = DeleteActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptDeleteStep = AttemptDeleteStep(clusterService, client, deleteActionConfig, managedIndexMetaData) - attemptDeleteStep.execute() - val updatedManagedIndexMetaData = attemptDeleteStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptDeleteStep = AttemptDeleteStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptDeleteStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptDeleteStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -61,11 +65,11 @@ class AttemptDeleteStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val deleteActionConfig = DeleteActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptDeleteStep = AttemptDeleteStep(clusterService, client, deleteActionConfig, managedIndexMetaData) - attemptDeleteStep.execute() - val updatedManagedIndexMetaData = attemptDeleteStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptDeleteStep = AttemptDeleteStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptDeleteStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptDeleteStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) logger.info(updatedManagedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } @@ -76,11 +80,11 @@ class AttemptDeleteStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val deleteActionConfig = DeleteActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptDeleteStep = AttemptDeleteStep(clusterService, client, deleteActionConfig, managedIndexMetaData) - attemptDeleteStep.execute() - val updatedManagedIndexMetaData = attemptDeleteStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptDeleteStep = AttemptDeleteStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptDeleteStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptDeleteStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not CONDITION_NOT_MET", Step.StepStatus.CONDITION_NOT_MET, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptOpenStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptOpenStepTests.kt index d2775c856..2ed8669e2 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptOpenStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptOpenStepTests.kt @@ -17,26 +17,31 @@ import org.opensearch.client.AdminClient import org.opensearch.client.Client import org.opensearch.client.IndicesAdminClient import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.OpenActionConfig +import org.opensearch.common.settings.Settings import org.opensearch.indexmanagement.indexstatemanagement.step.open.AttemptOpenStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.script.ScriptService import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException class AttemptOpenStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() + private val scriptService: ScriptService = mock() + private val settings: Settings = Settings.EMPTY fun `test open step sets step status to failed when not acknowledged`() { val openIndexResponse = OpenIndexResponse(false, false) val client = getClient(getAdminClient(getIndicesAdminClient(openIndexResponse, null))) runBlocking { - val openActionConfig = OpenActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptOpenStep = AttemptOpenStep(clusterService, client, openActionConfig, managedIndexMetaData) - attemptOpenStep.execute() - val updatedManagedIndexMetaData = attemptOpenStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptOpenStep = AttemptOpenStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptOpenStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptOpenStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -46,11 +51,11 @@ class AttemptOpenStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val openActionConfig = OpenActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptOpenStep = AttemptOpenStep(clusterService, client, openActionConfig, managedIndexMetaData) - attemptOpenStep.execute() - val updatedManagedIndexMetaData = attemptOpenStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptOpenStep = AttemptOpenStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptOpenStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptOpenStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -60,11 +65,11 @@ class AttemptOpenStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val openActionConfig = OpenActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptOpenStep = AttemptOpenStep(clusterService, client, openActionConfig, managedIndexMetaData) - attemptOpenStep.execute() - val updatedManagedIndexMetaData = attemptOpenStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptOpenStep = AttemptOpenStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptOpenStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptOpenStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "nested", updatedManagedIndexMetaData.info!!["cause"]) } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetIndexPriorityStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetIndexPriorityStepTests.kt index df78168b1..796baf8ab 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetIndexPriorityStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetIndexPriorityStepTests.kt @@ -17,27 +17,33 @@ import org.opensearch.client.AdminClient import org.opensearch.client.Client import org.opensearch.client.IndicesAdminClient import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.IndexPriorityActionConfig +import org.opensearch.common.settings.Settings +import org.opensearch.indexmanagement.indexstatemanagement.action.IndexPriorityAction import org.opensearch.indexmanagement.indexstatemanagement.step.indexpriority.AttemptSetIndexPriorityStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.script.ScriptService import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException -import kotlin.IllegalArgumentException class AttemptSetIndexPriorityStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() + private val scriptService: ScriptService = mock() + private val settings: Settings = Settings.EMPTY fun `test set priority step sets step status to completed when successful`() { val acknowledgedResponse = AcknowledgedResponse(true) val client = getClient(getAdminClient(getIndicesAdminClient(acknowledgedResponse, null))) runBlocking { - val indexPriorityActionConfig = IndexPriorityActionConfig(50, 0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptSetPriorityStep = AttemptSetIndexPriorityStep(clusterService, client, indexPriorityActionConfig, managedIndexMetaData) - attemptSetPriorityStep.execute() - val updatedManagedIndexMetaData = attemptSetPriorityStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val indexPriorityAction = IndexPriorityAction(50, 0) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptSetPriorityStep = AttemptSetIndexPriorityStep(indexPriorityAction) + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptSetPriorityStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptSetPriorityStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -47,11 +53,12 @@ class AttemptSetIndexPriorityStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(acknowledgedResponse, null))) runBlocking { - val indexPriorityActionConfig = IndexPriorityActionConfig(50, 0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptSetPriorityStep = AttemptSetIndexPriorityStep(clusterService, client, indexPriorityActionConfig, managedIndexMetaData) - attemptSetPriorityStep.execute() - val updatedManagedIndexMetaData = attemptSetPriorityStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val indexPriorityAction = IndexPriorityAction(50, 0) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptSetPriorityStep = AttemptSetIndexPriorityStep(indexPriorityAction) + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptSetPriorityStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptSetPriorityStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -61,11 +68,12 @@ class AttemptSetIndexPriorityStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val indexPriorityActionConfig = IndexPriorityActionConfig(50, 0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptSetPriorityStep = AttemptSetIndexPriorityStep(clusterService, client, indexPriorityActionConfig, managedIndexMetaData) - attemptSetPriorityStep.execute() - val updatedManagedIndexMetaData = attemptSetPriorityStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val indexPriorityAction = IndexPriorityAction(50, 0) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptSetPriorityStep = AttemptSetIndexPriorityStep(indexPriorityAction) + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptSetPriorityStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptSetPriorityStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) logger.info(updatedManagedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } @@ -76,11 +84,12 @@ class AttemptSetIndexPriorityStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val indexPriorityActionConfig = IndexPriorityActionConfig(50, 0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val attemptSetPriorityStep = AttemptSetIndexPriorityStep(clusterService, client, indexPriorityActionConfig, managedIndexMetaData) - attemptSetPriorityStep.execute() - val updatedManagedIndexMetaData = attemptSetPriorityStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val indexPriorityAction = IndexPriorityAction(50, 0) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val attemptSetPriorityStep = AttemptSetIndexPriorityStep(indexPriorityAction) + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + attemptSetPriorityStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptSetPriorityStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) logger.info(updatedManagedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "nested", updatedManagedIndexMetaData.info!!["cause"]) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetReplicaCountStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetReplicaCountStepTests.kt index 47378bf45..c5e589dd9 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetReplicaCountStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetReplicaCountStepTests.kt @@ -6,37 +6,44 @@ package org.opensearch.indexmanagement.indexstatemanagement.step import com.nhaarman.mockitokotlin2.any -import com.nhaarman.mockitokotlin2.doAnswer import com.nhaarman.mockitokotlin2.doReturn import com.nhaarman.mockitokotlin2.mock import com.nhaarman.mockitokotlin2.whenever import kotlinx.coroutines.runBlocking +import org.mockito.Mockito.doAnswer import org.opensearch.action.ActionListener import org.opensearch.action.support.master.AcknowledgedResponse import org.opensearch.client.AdminClient import org.opensearch.client.Client import org.opensearch.client.IndicesAdminClient import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReplicaCountActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.step.replicacount.AttemptSetReplicaCountStep +import org.opensearch.common.settings.Settings +import org.opensearch.indexmanagement.indexstatemanagement.action.ReplicaCountAction +import org.opensearch.indexmanagement.indexstatemanagement.step.replicacount.AttemptReplicaCountStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.script.ScriptService import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException class AttemptSetReplicaCountStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() + private val scriptService: ScriptService = mock() + private val settings: Settings = Settings.EMPTY fun `test replica step sets step status to failed when not acknowledged`() { val replicaCountResponse = AcknowledgedResponse(false) val client = getClient(getAdminClient(getIndicesAdminClient(replicaCountResponse, null))) runBlocking { - val replicaCountActionConfig = ReplicaCountActionConfig(2, 0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val replicaCountStep = AttemptSetReplicaCountStep(clusterService, client, replicaCountActionConfig, managedIndexMetaData) - replicaCountStep.execute() - val updatedManagedIndexMetaData = replicaCountStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val replicaCountAction = ReplicaCountAction(2, 0) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val replicaCountStep = AttemptReplicaCountStep(replicaCountAction) + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + replicaCountStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = replicaCountStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -46,11 +53,12 @@ class AttemptSetReplicaCountStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val replicaCountActionConfig = ReplicaCountActionConfig(2, 0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val replicaCountStep = AttemptSetReplicaCountStep(clusterService, client, replicaCountActionConfig, managedIndexMetaData) - replicaCountStep.execute() - val updatedManagedIndexMetaData = replicaCountStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val replicaCountAction = ReplicaCountAction(2, 0) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val replicaCountStep = AttemptReplicaCountStep(replicaCountAction) + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + replicaCountStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = replicaCountStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -60,11 +68,12 @@ class AttemptSetReplicaCountStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val replicaCountActionConfig = ReplicaCountActionConfig(2, 0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val replicaCountStep = AttemptSetReplicaCountStep(clusterService, client, replicaCountActionConfig, managedIndexMetaData) - replicaCountStep.execute() - val updatedManagedIndexMetaData = replicaCountStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val replicaCountAction = ReplicaCountAction(2, 0) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val replicaCountStep = AttemptReplicaCountStep(replicaCountAction) + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + replicaCountStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = replicaCountStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "nested", updatedManagedIndexMetaData.info!!["cause"]) } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSnapshotStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSnapshotStepTests.kt index 00b528f40..bea3c41ae 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSnapshotStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSnapshotStepTests.kt @@ -21,12 +21,14 @@ import org.opensearch.client.ClusterAdminClient import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.ClusterSettings import org.opensearch.common.settings.Settings -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionProperties import org.opensearch.indexmanagement.indexstatemanagement.randomSnapshotActionConfig import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.SNAPSHOT_DENY_LIST import org.opensearch.indexmanagement.indexstatemanagement.step.snapshot.AttemptSnapshotStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext import org.opensearch.ingest.TestTemplateService.MockTemplateScript import org.opensearch.rest.RestStatus import org.opensearch.script.ScriptService @@ -39,8 +41,9 @@ class AttemptSnapshotStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() private val scriptService: ScriptService = mock() - private val config = randomSnapshotActionConfig("repo", "snapshot-name") - private val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, ActionMetaData(AttemptSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) + private val settings: Settings = Settings.EMPTY + private val snapshotAction = randomSnapshotActionConfig("repo", "snapshot-name") + private val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(AttemptSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) @Before fun settings() { @@ -54,25 +57,28 @@ class AttemptSnapshotStepTests : OpenSearchTestCase() { whenever(response.status()).doReturn(RestStatus.ACCEPTED) runBlocking { - val step = AttemptSnapshotStep(clusterService, scriptService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val step = AttemptSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } whenever(response.status()).doReturn(RestStatus.OK) runBlocking { - val step = AttemptSnapshotStep(clusterService, scriptService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val step = AttemptSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } whenever(response.status()).doReturn(RestStatus.INTERNAL_SERVER_ERROR) runBlocking { - val step = AttemptSnapshotStep(clusterService, scriptService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val step = AttemptSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -81,9 +87,10 @@ class AttemptSnapshotStepTests : OpenSearchTestCase() { val exception = IllegalArgumentException("example") val client = getClient(getAdminClient(getClusterAdminClient(null, exception))) runBlocking { - val step = AttemptSnapshotStep(clusterService, scriptService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val step = AttemptSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "example", updatedManagedIndexMetaData.info!!["cause"]) } @@ -93,9 +100,10 @@ class AttemptSnapshotStepTests : OpenSearchTestCase() { val exception = ConcurrentSnapshotExecutionException("repo", "other-snapshot", "concurrent snapshot in progress") val client = getClient(getAdminClient(getClusterAdminClient(null, exception))) runBlocking { - val step = AttemptSnapshotStep(clusterService, scriptService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val step = AttemptSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not CONDITION_NOT_MET", Step.StepStatus.CONDITION_NOT_MET, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get failed concurrent message", AttemptSnapshotStep.getFailedConcurrentSnapshotMessage("test"), updatedManagedIndexMetaData.info!!["message"]) } @@ -105,9 +113,10 @@ class AttemptSnapshotStepTests : OpenSearchTestCase() { val exception = RemoteTransportException("rte", ConcurrentSnapshotExecutionException("repo", "other-snapshot", "concurrent snapshot in progress")) val client = getClient(getAdminClient(getClusterAdminClient(null, exception))) runBlocking { - val step = AttemptSnapshotStep(clusterService, scriptService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val step = AttemptSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not CONDITION_NOT_MET", Step.StepStatus.CONDITION_NOT_MET, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get failed concurrent message", AttemptSnapshotStep.getFailedConcurrentSnapshotMessage("test"), updatedManagedIndexMetaData.info!!["message"]) } @@ -117,9 +126,10 @@ class AttemptSnapshotStepTests : OpenSearchTestCase() { val exception = RemoteTransportException("rte", IllegalArgumentException("some error")) val client = getClient(getAdminClient(getClusterAdminClient(null, exception))) runBlocking { - val step = AttemptSnapshotStep(clusterService, scriptService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val step = AttemptSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "some error", updatedManagedIndexMetaData.info!!["cause"]) } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptTransitionStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptTransitionStepTests.kt index 5a2952e38..e30ce2e33 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptTransitionStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptTransitionStepTests.kt @@ -11,6 +11,7 @@ import com.nhaarman.mockitokotlin2.doReturn import com.nhaarman.mockitokotlin2.mock import com.nhaarman.mockitokotlin2.whenever import kotlinx.coroutines.runBlocking +import org.junit.Before import org.opensearch.action.ActionListener import org.opensearch.action.admin.indices.rollover.RolloverInfo import org.opensearch.action.admin.indices.stats.CommonStats @@ -23,32 +24,52 @@ import org.opensearch.cluster.metadata.IndexMetadata import org.opensearch.cluster.metadata.Metadata import org.opensearch.cluster.service.ClusterService import org.opensearch.common.collect.ImmutableOpenMap +import org.opensearch.common.settings.ClusterSettings +import org.opensearch.common.settings.Settings import org.opensearch.index.shard.DocsStats +import org.opensearch.indexmanagement.indexstatemanagement.IndexMetadataProvider +import org.opensearch.indexmanagement.indexstatemanagement.action.TransitionsAction import org.opensearch.indexmanagement.indexstatemanagement.model.Conditions -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.indexstatemanagement.model.Transition -import org.opensearch.indexmanagement.indexstatemanagement.model.action.TransitionsActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.StepMetaData +import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.step.transition.AttemptTransitionStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.rest.RestStatus +import org.opensearch.script.ScriptService import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException import java.time.Instant class AttemptTransitionStepTests : OpenSearchTestCase() { + private val indexName: String = "test" + private val indexUUID: String = "indexUuid" @Suppress("UNCHECKED_CAST") private val indexMetadata: IndexMetadata = mock { on { rolloverInfos } doReturn ImmutableOpenMap.builder().build() + on { indexUUID } doReturn indexUUID + } + private val metadata: Metadata = mock { + on { index(any()) } doReturn indexMetadata + on { hasIndex(indexName) } doReturn true } - private val metadata: Metadata = mock { on { index(any()) } doReturn indexMetadata } private val clusterState: ClusterState = mock { on { metadata() } doReturn metadata } private val clusterService: ClusterService = mock { on { state() } doReturn clusterState } + private val scriptService: ScriptService = mock() + private val settings: Settings = Settings.EMPTY private val docsStats: DocsStats = mock() private val primaries: CommonStats = mock { on { getDocs() } doReturn docsStats } private val statsResponse: IndicesStatsResponse = mock { on { primaries } doReturn primaries } + @Before + fun `setup settings`() { + whenever(clusterService.clusterSettings).doReturn(ClusterSettings(Settings.EMPTY, setOf(ManagedIndexSettings.RESTRICTED_INDEX_PATTERN))) + } + fun `test stats response not OK`() { whenever(indexMetadata.creationDate).doReturn(5L) whenever(statsResponse.status).doReturn(RestStatus.INTERNAL_SERVER_ERROR) @@ -56,15 +77,17 @@ class AttemptTransitionStepTests : OpenSearchTestCase() { whenever(docsStats.count).doReturn(6L) whenever(docsStats.totalSizeInBytes).doReturn(2) val client = getClient(getAdminClient(getIndicesAdminClient(statsResponse, null))) + val indexMetadataProvider = IndexMetadataProvider(settings, client, clusterService, mutableMapOf()) runBlocking { - val config = TransitionsActionConfig(listOf(Transition("some_state", Conditions(docCount = 5L)))) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val step = AttemptTransitionStep(clusterService, client, config, managedIndexMetaData) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val transitionsAction = TransitionsAction(listOf(Transition("some_state", Conditions(docCount = 5L))), indexMetadataProvider) + val attemptTransitionStep = AttemptTransitionStep(transitionsAction) + val context = StepContext(managedIndexMetadata, clusterService, client, null, null, scriptService, settings) + attemptTransitionStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptTransitionStep.getUpdatedManagedIndexMetadata(managedIndexMetadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) - assertEquals("Did not get correct failed message", AttemptTransitionStep.getFailedStatsMessage("test"), updatedManagedIndexMetaData.info!!["message"]) + assertEquals("Did not get correct failed message", AttemptTransitionStep.getFailedStatsMessage(indexName), updatedManagedIndexMetaData.info!!["message"]) } } @@ -72,13 +95,15 @@ class AttemptTransitionStepTests : OpenSearchTestCase() { whenever(indexMetadata.creationDate).doReturn(5L) val exception = IllegalArgumentException("example") val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) + val indexMetadataProvider = IndexMetadataProvider(settings, client, clusterService, mutableMapOf()) runBlocking { - val config = TransitionsActionConfig(listOf(Transition("some_state", Conditions(docCount = 5L)))) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val step = AttemptTransitionStep(clusterService, client, config, managedIndexMetaData) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val transitionsAction = TransitionsAction(listOf(Transition("some_state", Conditions(docCount = 5L))), indexMetadataProvider) + val attemptTransitionStep = AttemptTransitionStep(transitionsAction) + val context = StepContext(managedIndexMetadata, clusterService, client, null, null, scriptService, settings) + attemptTransitionStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptTransitionStep.getUpdatedManagedIndexMetadata(managedIndexMetadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "example", updatedManagedIndexMetaData.info!!["cause"]) } @@ -88,28 +113,29 @@ class AttemptTransitionStepTests : OpenSearchTestCase() { whenever(indexMetadata.creationDate).doReturn(5L) val exception = RemoteTransportException("rte", IllegalArgumentException("nested")) val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) + val indexMetadataProvider = IndexMetadataProvider(settings, client, clusterService, mutableMapOf()) runBlocking { - val config = TransitionsActionConfig(listOf(Transition("some_state", Conditions(docCount = 5L)))) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val step = AttemptTransitionStep(clusterService, client, config, managedIndexMetaData) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val transitionsAction = TransitionsAction(listOf(Transition("some_state", Conditions(docCount = 5L))), indexMetadataProvider) + val attemptTransitionStep = AttemptTransitionStep(transitionsAction) + val context = StepContext(managedIndexMetadata, clusterService, client, null, null, scriptService, settings) + attemptTransitionStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = attemptTransitionStep.getUpdatedManagedIndexMetadata(managedIndexMetadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "nested", updatedManagedIndexMetaData.info!!["cause"]) } } fun `test step start time resetting between two transitions`() { - val client = getClient(getAdminClient(getIndicesAdminClient(statsResponse, null))) - + val indexMetadataProvider = IndexMetadataProvider(settings, mock(), clusterService, mutableMapOf()) runBlocking { - val config = TransitionsActionConfig(listOf(Transition("some_state", null))) val completedStartTime = Instant.now() - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, StepMetaData("attempt_transition", completedStartTime.toEpochMilli(), Step.StepStatus.COMPLETED), null, null) - val step = AttemptTransitionStep(clusterService, client, config, managedIndexMetaData) + val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, null, StepMetaData("attempt_transition", completedStartTime.toEpochMilli(), Step.StepStatus.COMPLETED), null, null) + val transitionsAction = TransitionsAction(listOf(Transition("some_state", null)), indexMetadataProvider) + val attemptTransitionStep = AttemptTransitionStep(transitionsAction) Thread.sleep(50) // Make sure we give enough time for the instants to be different - val startTime = step.getStepStartTime() + val startTime = attemptTransitionStep.getStepStartTime(managedIndexMetadata) assertNotEquals("Two separate transitions should not have same start time", completedStartTime.toEpochMilli(), startTime.toEpochMilli()) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/SetReadOnlyStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/SetReadOnlyStepTests.kt index 9717e53c1..cfcc73142 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/SetReadOnlyStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/SetReadOnlyStepTests.kt @@ -17,26 +17,31 @@ import org.opensearch.client.AdminClient import org.opensearch.client.Client import org.opensearch.client.IndicesAdminClient import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReadOnlyActionConfig +import org.opensearch.common.settings.Settings import org.opensearch.indexmanagement.indexstatemanagement.step.readonly.SetReadOnlyStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.script.ScriptService import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException class SetReadOnlyStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() + private val scriptService: ScriptService = mock() + private val settings: Settings = Settings.EMPTY fun `test read only step sets step status to failed when not acknowledged`() { val setReadOnlyResponse = AcknowledgedResponse(false) val client = getClient(getAdminClient(getIndicesAdminClient(setReadOnlyResponse, null))) runBlocking { - val readOnlyActionConfig = ReadOnlyActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val setReadOnlyStep = SetReadOnlyStep(clusterService, client, readOnlyActionConfig, managedIndexMetaData) - setReadOnlyStep.execute() - val updatedManagedIndexMetaData = setReadOnlyStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val setReadOnlyStep = SetReadOnlyStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + setReadOnlyStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = setReadOnlyStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -46,11 +51,11 @@ class SetReadOnlyStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val readOnlyActionConfig = ReadOnlyActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val setReadOnlyStep = SetReadOnlyStep(clusterService, client, readOnlyActionConfig, managedIndexMetaData) - setReadOnlyStep.execute() - val updatedManagedIndexMetaData = setReadOnlyStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val setReadOnlyStep = SetReadOnlyStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + setReadOnlyStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = setReadOnlyStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -60,11 +65,11 @@ class SetReadOnlyStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val readOnlyActionConfig = ReadOnlyActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val setReadOnlyStep = SetReadOnlyStep(clusterService, client, readOnlyActionConfig, managedIndexMetaData) - setReadOnlyStep.execute() - val updatedManagedIndexMetaData = setReadOnlyStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val setReadOnlyStep = SetReadOnlyStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + setReadOnlyStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = setReadOnlyStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "nested", updatedManagedIndexMetaData.info!!["cause"]) } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/SetReadWriteStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/SetReadWriteStepTests.kt index 5235840e2..c332a0c90 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/SetReadWriteStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/SetReadWriteStepTests.kt @@ -17,26 +17,31 @@ import org.opensearch.client.AdminClient import org.opensearch.client.Client import org.opensearch.client.IndicesAdminClient import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.ReadWriteActionConfig +import org.opensearch.common.settings.Settings import org.opensearch.indexmanagement.indexstatemanagement.step.readwrite.SetReadWriteStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.script.ScriptService import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException class SetReadWriteStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() + private val scriptService: ScriptService = mock() + private val settings: Settings = Settings.EMPTY fun `test read write step sets step status to failed when not acknowledged`() { val setReadWriteResponse = AcknowledgedResponse(false) val client = getClient(getAdminClient(getIndicesAdminClient(setReadWriteResponse, null))) runBlocking { - val readWriteActionConfig = ReadWriteActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val setReadWriteStep = SetReadWriteStep(clusterService, client, readWriteActionConfig, managedIndexMetaData) - setReadWriteStep.execute() - val updatedManagedIndexMetaData = setReadWriteStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val setReadWriteStep = SetReadWriteStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + setReadWriteStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = setReadWriteStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -46,11 +51,11 @@ class SetReadWriteStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val readWriteActionConfig = ReadWriteActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val setReadWriteStep = SetReadWriteStep(clusterService, client, readWriteActionConfig, managedIndexMetaData) - setReadWriteStep.execute() - val updatedManagedIndexMetaData = setReadWriteStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val setReadWriteStep = SetReadWriteStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + setReadWriteStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = setReadWriteStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } } @@ -60,11 +65,11 @@ class SetReadWriteStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val readWriteActionConfig = ReadWriteActionConfig(0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null) - val setReadWriteStep = SetReadWriteStep(clusterService, client, readWriteActionConfig, managedIndexMetaData) - setReadWriteStep.execute() - val updatedManagedIndexMetaData = setReadWriteStep.getUpdatedManagedIndexMetaData(managedIndexMetaData) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val setReadWriteStep = SetReadWriteStep() + val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings) + setReadWriteStep.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = setReadWriteStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "nested", updatedManagedIndexMetaData.info!!["cause"]) } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/WaitForRollupCompletionStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/WaitForRollupCompletionStepTests.kt index 8041c963e..83fd61145 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/WaitForRollupCompletionStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/WaitForRollupCompletionStepTests.kt @@ -9,22 +9,28 @@ import com.nhaarman.mockitokotlin2.mock import kotlinx.coroutines.runBlocking import org.opensearch.client.Client import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionProperties +import org.opensearch.common.settings.Settings import org.opensearch.indexmanagement.indexstatemanagement.step.rollup.WaitForRollupCompletionStep import org.opensearch.indexmanagement.rollup.model.RollupMetadata import org.opensearch.indexmanagement.rollup.model.RollupStats +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.script.ScriptService import org.opensearch.test.OpenSearchTestCase import java.time.Instant class WaitForRollupCompletionStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() + private val scriptService: ScriptService = mock() + private val settings: Settings = Settings.EMPTY private val rollupId: String = "dummy-id" private val indexName: String = "test" private val metadata = ManagedIndexMetaData( - indexName, "indexUuid", "policy_id", null, null, null, null, null, null, + indexName, "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData (WaitForRollupCompletionStep.name, 1, 0, false, 0, null, ActionProperties(rollupId = rollupId)), null, null, null @@ -34,18 +40,19 @@ class WaitForRollupCompletionStepTests : OpenSearchTestCase() { stats = RollupStats(1, 1, 1, 1, 1) ) private val client: Client = mock() - private val step = WaitForRollupCompletionStep(clusterService, client, metadata) + private val step = WaitForRollupCompletionStep() fun `test wait for rollup when missing rollup id`() { val actionMetadata = metadata.actionMetaData!!.copy(actionProperties = ActionProperties()) val metadata = metadata.copy(actionMetaData = actionMetadata) - val step = WaitForRollupCompletionStep(clusterService, client, metadata) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + val step = WaitForRollupCompletionStep() runBlocking { - step.execute() + step.preExecute(logger, context).execute() } - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals( "Missing failure message", @@ -56,9 +63,9 @@ class WaitForRollupCompletionStepTests : OpenSearchTestCase() { fun `test process rollup metadata FAILED status`() { val rollupMetadata = rollupMetadata.copy(status = RollupMetadata.Status.FAILED) - step.processRollupMetadataStatus(rollupId, rollupMetadata) + step.processRollupMetadataStatus(rollupId, indexName, rollupMetadata) - val updateManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val updateManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updateManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals( "Missing failure message", @@ -70,9 +77,9 @@ class WaitForRollupCompletionStepTests : OpenSearchTestCase() { fun `test process rollup metadata STOPPED status`() { val rollupMetadata = rollupMetadata.copy(status = RollupMetadata.Status.STOPPED) - step.processRollupMetadataStatus(rollupId, rollupMetadata) + step.processRollupMetadataStatus(rollupId, indexName, rollupMetadata) - val updateManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val updateManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updateManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals( "Missing failure message", @@ -85,9 +92,9 @@ class WaitForRollupCompletionStepTests : OpenSearchTestCase() { fun `test process rollup metadata INIT status`() { val rollupMetadata = rollupMetadata.copy(status = RollupMetadata.Status.INIT) - step.processRollupMetadataStatus(rollupId, rollupMetadata) + step.processRollupMetadataStatus(rollupId, indexName, rollupMetadata) - val updateManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val updateManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not CONDITION_NOT_MET", Step.StepStatus.CONDITION_NOT_MET, updateManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals( "Missing processing message", @@ -99,9 +106,9 @@ class WaitForRollupCompletionStepTests : OpenSearchTestCase() { fun `test process rollup metadata STARTED status`() { val rollupMetadata = rollupMetadata.copy(status = RollupMetadata.Status.STARTED) - step.processRollupMetadataStatus(rollupId, rollupMetadata) + step.processRollupMetadataStatus(rollupId, indexName, rollupMetadata) - val updateManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val updateManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not CONDITION_NOT_MET", Step.StepStatus.CONDITION_NOT_MET, updateManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals( "Missing processing message", @@ -113,9 +120,9 @@ class WaitForRollupCompletionStepTests : OpenSearchTestCase() { fun `test process rollup metadata FINISHED status`() { val rollupMetadata = rollupMetadata.copy(status = RollupMetadata.Status.FINISHED) - step.processRollupMetadataStatus(rollupId, rollupMetadata) + step.processRollupMetadataStatus(rollupId, indexName, rollupMetadata) - val updateManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val updateManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updateManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals( "Missing processing message", @@ -127,9 +134,9 @@ class WaitForRollupCompletionStepTests : OpenSearchTestCase() { fun `test process rollup metadata RETRY status`() { val rollupMetadata = rollupMetadata.copy(status = RollupMetadata.Status.RETRY) - step.processRollupMetadataStatus(rollupId, rollupMetadata) + step.processRollupMetadataStatus(rollupId, indexName, rollupMetadata) - val updateManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val updateManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not CONDITION_NOT_MET", Step.StepStatus.CONDITION_NOT_MET, updateManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals( "Missing processing message", @@ -140,9 +147,9 @@ class WaitForRollupCompletionStepTests : OpenSearchTestCase() { } fun `test process failure`() { - step.processFailure(rollupId, Exception("dummy-exception")) + step.processFailure(rollupId, indexName, Exception("dummy-exception")) - val updateManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val updateManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Mismatch in cause", "dummy-exception", updateManagedIndexMetaData.info?.get("cause")) assertEquals( "Mismatch in message", diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/WaitForSnapshotStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/WaitForSnapshotStepTests.kt index a4dac83db..8af94805f 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/WaitForSnapshotStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/WaitForSnapshotStepTests.kt @@ -19,11 +19,15 @@ import org.opensearch.client.Client import org.opensearch.client.ClusterAdminClient import org.opensearch.cluster.SnapshotsInProgress import org.opensearch.cluster.service.ClusterService -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.action.SnapshotActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionMetaData -import org.opensearch.indexmanagement.indexstatemanagement.model.managedindexmetadata.ActionProperties +import org.opensearch.common.settings.Settings +import org.opensearch.indexmanagement.indexstatemanagement.action.SnapshotAction import org.opensearch.indexmanagement.indexstatemanagement.step.snapshot.WaitForSnapshotStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.script.ScriptService import org.opensearch.snapshots.Snapshot import org.opensearch.snapshots.SnapshotId import org.opensearch.test.OpenSearchTestCase @@ -32,6 +36,8 @@ import org.opensearch.transport.RemoteTransportException class WaitForSnapshotStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() + private val scriptService: ScriptService = mock() + private val settings: Settings = Settings.EMPTY val snapshot = "snapshot-name" fun `test snapshot missing snapshot name in action properties`() { @@ -39,22 +45,24 @@ class WaitForSnapshotStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getClusterAdminClient(null, exception))) runBlocking { val emptyActionProperties = ActionProperties() - val config = SnapshotActionConfig("repo", snapshot, 0) - val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, emptyActionProperties), null, null, null) - val step = WaitForSnapshotStep(clusterService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val snapshotAction = SnapshotAction("repo", snapshot, 0) + val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, emptyActionProperties), null, null, null) + val step = WaitForSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", WaitForSnapshotStep.getFailedActionPropertiesMessage("test", emptyActionProperties), updatedManagedIndexMetaData.info!!["message"]) } runBlocking { val nullActionProperties = null - val config = SnapshotActionConfig("repo", snapshot, 0) - val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, nullActionProperties), null, null, null) - val step = WaitForSnapshotStep(clusterService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val snapshotAction = SnapshotAction("repo", snapshot, 0) + val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, nullActionProperties), null, null, null) + val step = WaitForSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", WaitForSnapshotStep.getFailedActionPropertiesMessage("test", nullActionProperties), updatedManagedIndexMetaData.info!!["message"]) } @@ -69,55 +77,60 @@ class WaitForSnapshotStepTests : OpenSearchTestCase() { whenever(snapshotStatus.state).doReturn(SnapshotsInProgress.State.INIT) runBlocking { - val config = SnapshotActionConfig("repo", snapshot, 0) - val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) - val step = WaitForSnapshotStep(clusterService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val snapshotAction = SnapshotAction("repo", snapshot, 0) + val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) + val step = WaitForSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not CONDITION_NOT_MET", Step.StepStatus.CONDITION_NOT_MET, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get snapshot in progress message", WaitForSnapshotStep.getSnapshotInProgressMessage("test"), updatedManagedIndexMetaData.info!!["message"]) } whenever(snapshotStatus.state).doReturn(SnapshotsInProgress.State.STARTED) runBlocking { - val config = SnapshotActionConfig("repo", snapshot, 0) - val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) - val step = WaitForSnapshotStep(clusterService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val snapshotAction = SnapshotAction("repo", snapshot, 0) + val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) + val step = WaitForSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not CONDITION_NOT_MET", Step.StepStatus.CONDITION_NOT_MET, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get snapshot in progress message", WaitForSnapshotStep.getSnapshotInProgressMessage("test"), updatedManagedIndexMetaData.info!!["message"]) } whenever(snapshotStatus.state).doReturn(SnapshotsInProgress.State.SUCCESS) runBlocking { - val config = SnapshotActionConfig("repo", snapshot, 0) - val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) - val step = WaitForSnapshotStep(clusterService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val snapshotAction = SnapshotAction("repo", snapshot, 0) + val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) + val step = WaitForSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get snapshot completed message", WaitForSnapshotStep.getSuccessMessage("test"), updatedManagedIndexMetaData.info!!["message"]) } whenever(snapshotStatus.state).doReturn(SnapshotsInProgress.State.ABORTED) runBlocking { - val config = SnapshotActionConfig("repo", snapshot, 0) - val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) - val step = WaitForSnapshotStep(clusterService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val snapshotAction = SnapshotAction("repo", snapshot, 0) + val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) + val step = WaitForSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get snapshot failed message", WaitForSnapshotStep.getFailedExistsMessage("test"), updatedManagedIndexMetaData.info!!["message"]) } whenever(snapshotStatus.state).doReturn(SnapshotsInProgress.State.FAILED) runBlocking { - val config = SnapshotActionConfig("repo", snapshot, 0) - val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) - val step = WaitForSnapshotStep(clusterService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val snapshotAction = SnapshotAction("repo", snapshot, 0) + val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) + val step = WaitForSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get snapshot failed message", WaitForSnapshotStep.getFailedExistsMessage("test"), updatedManagedIndexMetaData.info!!["message"]) } @@ -131,11 +144,12 @@ class WaitForSnapshotStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getClusterAdminClient(response, null))) runBlocking { - val config = SnapshotActionConfig("repo", snapshot, 0) - val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) - val step = WaitForSnapshotStep(clusterService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val snapshotAction = SnapshotAction("repo", snapshot, 0) + val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) + val step = WaitForSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get snapshot failed message", WaitForSnapshotStep.getFailedExistsMessage("test"), updatedManagedIndexMetaData.info!!["message"]) } @@ -145,11 +159,12 @@ class WaitForSnapshotStepTests : OpenSearchTestCase() { val exception = IllegalArgumentException("example") val client = getClient(getAdminClient(getClusterAdminClient(null, exception))) runBlocking { - val config = SnapshotActionConfig("repo", snapshot, 0) - val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) - val step = WaitForSnapshotStep(clusterService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val snapshotAction = SnapshotAction("repo", snapshot, 0) + val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) + val step = WaitForSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "example", updatedManagedIndexMetaData.info!!["cause"]) } @@ -159,11 +174,12 @@ class WaitForSnapshotStepTests : OpenSearchTestCase() { val exception = RemoteTransportException("rte", IllegalArgumentException("nested")) val client = getClient(getAdminClient(getClusterAdminClient(null, exception))) runBlocking { - val config = SnapshotActionConfig("repo", snapshot, 0) - val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) - val step = WaitForSnapshotStep(clusterService, client, config, metadata) - step.execute() - val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetaData(metadata) + val snapshotAction = SnapshotAction("repo", snapshot, 0) + val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) + val step = WaitForSnapshotStep(snapshotAction) + val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings) + step.preExecute(logger, context).execute() + val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "nested", updatedManagedIndexMetaData.info!!["cause"]) } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/addpolicy/AddPolicyRequestTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/addpolicy/AddPolicyRequestTests.kt index 884b381fd..704fef4b5 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/addpolicy/AddPolicyRequestTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/addpolicy/AddPolicyRequestTests.kt @@ -7,6 +7,7 @@ package org.opensearch.indexmanagement.indexstatemanagement.transport.action.add import org.opensearch.common.io.stream.BytesStreamOutput import org.opensearch.common.io.stream.StreamInput +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.test.OpenSearchTestCase class AddPolicyRequestTests : OpenSearchTestCase() { @@ -14,7 +15,7 @@ class AddPolicyRequestTests : OpenSearchTestCase() { fun `test add policy request`() { val indices = listOf("index1", "index2") val policyID = "policyID" - val req = AddPolicyRequest(indices, policyID) + val req = AddPolicyRequest(indices, policyID, DEFAULT_INDEX_TYPE) val out = BytesStreamOutput() req.writeTo(out) @@ -23,4 +24,13 @@ class AddPolicyRequestTests : OpenSearchTestCase() { assertEquals(indices, newReq.indices) assertEquals(policyID, newReq.policyID) } + + fun `test add policy request with non default index type and multiple indices fails`() { + val indices = listOf("index1", "index2") + val policyID = "policyID" + val req = AddPolicyRequest(indices, policyID, "non-existent-index-type") + val actualException: String? = req.validate()?.validationErrors()?.firstOrNull() + val expectedException: String = AddPolicyRequest.MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR + assertEquals("Add policy request should have failed validation with specific exception", actualException, expectedException) + } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/ChangePolicyRequestTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/ChangePolicyRequestTests.kt index 3b9575bc8..ac49b2432 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/ChangePolicyRequestTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/changepolicy/ChangePolicyRequestTests.kt @@ -9,6 +9,7 @@ import org.opensearch.common.io.stream.BytesStreamOutput import org.opensearch.common.io.stream.StreamInput import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy import org.opensearch.indexmanagement.indexstatemanagement.model.StateFilter +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.test.OpenSearchTestCase class ChangePolicyRequestTests : OpenSearchTestCase() { @@ -17,7 +18,7 @@ class ChangePolicyRequestTests : OpenSearchTestCase() { val indices = listOf("index1", "index2") val stateFilter = StateFilter("state1") val changePolicy = ChangePolicy("policyID", "state1", listOf(stateFilter), true) - val req = ChangePolicyRequest(indices, changePolicy) + val req = ChangePolicyRequest(indices, changePolicy, DEFAULT_INDEX_TYPE) val out = BytesStreamOutput() req.writeTo(out) @@ -26,4 +27,14 @@ class ChangePolicyRequestTests : OpenSearchTestCase() { assertEquals(indices, newReq.indices) assertEquals(changePolicy, newReq.changePolicy) } + + fun `test change policy request with non default index type and multiple indices fails`() { + val indices = listOf("index1", "index2") + val stateFilter = StateFilter("state1") + val changePolicy = ChangePolicy("policyID", "state1", listOf(stateFilter), true) + val req = ChangePolicyRequest(indices, changePolicy, "non-existent-index-type") + val actualException: String? = req.validate()?.validationErrors()?.firstOrNull() + val expectedException: String = ChangePolicyRequest.MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR + assertEquals("Add policy request should have failed validation with specific exception", actualException, expectedException) + } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt index cc62209c7..70a183984 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt @@ -9,6 +9,7 @@ import org.opensearch.common.io.stream.BytesStreamOutput import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.unit.TimeValue import org.opensearch.indexmanagement.indexstatemanagement.model.SearchParams +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.test.OpenSearchTestCase class ExplainRequestTests : OpenSearchTestCase() { @@ -18,7 +19,8 @@ class ExplainRequestTests : OpenSearchTestCase() { val local = true val masterTimeout = TimeValue.timeValueSeconds(30) val params = SearchParams(0, 20, "sort-field", "asc", "*") - val req = ExplainRequest(indices, local, masterTimeout, params) + val showPolicy = false + val req = ExplainRequest(indices, local, masterTimeout, params, showPolicy, DEFAULT_INDEX_TYPE) val out = BytesStreamOutput() req.writeTo(out) @@ -27,4 +29,17 @@ class ExplainRequestTests : OpenSearchTestCase() { assertEquals(indices, newReq.indices) assertEquals(local, newReq.local) } + + fun `test explain policy request with non default index type and multiple indices fails`() { + val indices = listOf("index1", "index2") + val local = true + val masterTimeout = TimeValue.timeValueSeconds(30) + val params = SearchParams(0, 20, "sort-field", "asc", "*") + val showPolicy = false + val req = ExplainRequest(indices, local, masterTimeout, params, showPolicy, "non-existent-index-type") + + val actualException: String? = req.validate()?.validationErrors()?.firstOrNull() + val expectedException: String = ExplainRequest.MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR + assertEquals("Add policy request should have failed validation with specific exception", actualException, expectedException) + } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainResponseTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainResponseTests.kt index 60e6699f1..647ffd99f 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainResponseTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainResponseTests.kt @@ -7,7 +7,8 @@ package org.opensearch.indexmanagement.indexstatemanagement.transport.action.exp import org.opensearch.common.io.stream.BytesStreamOutput import org.opensearch.common.io.stream.StreamInput -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.indexstatemanagement.randomPolicy +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.test.OpenSearchTestCase class ExplainResponseTests : OpenSearchTestCase() { @@ -23,36 +24,7 @@ class ExplainResponseTests : OpenSearchTestCase() { policyPrimaryTerm = randomNonNegativeLong(), policyCompleted = null, rolledOver = null, - transitionTo = randomAlphaOfLength(10), - stateMetaData = null, - actionMetaData = null, - stepMetaData = null, - policyRetryInfo = null, - info = null - ) - val indexMetadatas = listOf(metadata) - val res = ExplainResponse(indexNames, indexPolicyIDs, indexMetadatas) - - val out = BytesStreamOutput() - res.writeTo(out) - val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newRes = ExplainResponse(sin) - assertEquals(indexNames, newRes.indexNames) - assertEquals(indexPolicyIDs, newRes.indexPolicyIDs) - assertEquals(indexMetadatas, newRes.indexMetadatas) - } - - fun `test explain all response`() { - val indexNames = listOf("index1") - val indexPolicyIDs = listOf("policyID1") - val metadata = ManagedIndexMetaData( - index = "index1", - indexUuid = randomAlphaOfLength(10), - policyID = "policyID1", - policySeqNo = randomNonNegativeLong(), - policyPrimaryTerm = randomNonNegativeLong(), - policyCompleted = null, - rolledOver = null, + indexCreationDate = null, transitionTo = randomAlphaOfLength(10), stateMetaData = null, actionMetaData = null, @@ -63,16 +35,18 @@ class ExplainResponseTests : OpenSearchTestCase() { val indexMetadatas = listOf(metadata) val totalManagedIndices = 1 val enabledState = mapOf("index1" to true) - val res = ExplainAllResponse(indexNames, indexPolicyIDs, indexMetadatas, totalManagedIndices, enabledState) + val appliedPolicies = mapOf("policy" to randomPolicy()) + val res = ExplainResponse(indexNames, indexPolicyIDs, indexMetadatas, totalManagedIndices, enabledState, appliedPolicies) val out = BytesStreamOutput() res.writeTo(out) val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) - val newRes = ExplainAllResponse(sin) + val newRes = ExplainResponse(sin) assertEquals(indexNames, newRes.indexNames) assertEquals(indexPolicyIDs, newRes.indexPolicyIDs) assertEquals(indexMetadatas, newRes.indexMetadatas) assertEquals(totalManagedIndices, newRes.totalManagedIndices) assertEquals(enabledState, newRes.enabledState) + assertEquals(appliedPolicies, newRes.policies) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPoliciesResponseTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPoliciesResponseTests.kt index 94c96a8b9..2a802625e 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPoliciesResponseTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPoliciesResponseTests.kt @@ -7,8 +7,21 @@ package org.opensearch.indexmanagement.indexstatemanagement.transport.action.get import org.opensearch.common.io.stream.BytesStreamOutput import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.json.JsonXContent +import org.opensearch.indexmanagement.indexstatemanagement.ISMActionsParser +import org.opensearch.indexmanagement.indexstatemanagement.extension.SampleCustomActionParser +import org.opensearch.indexmanagement.indexstatemanagement.model.Policy +import org.opensearch.indexmanagement.indexstatemanagement.model.State +import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification import org.opensearch.indexmanagement.indexstatemanagement.randomPolicy +import org.opensearch.indexmanagement.opensearchapi.convertToMap +import org.opensearch.indexmanagement.opensearchapi.string import org.opensearch.test.OpenSearchTestCase +import java.time.Instant +import java.time.temporal.ChronoUnit class GetPoliciesResponseTests : OpenSearchTestCase() { @@ -24,4 +37,35 @@ class GetPoliciesResponseTests : OpenSearchTestCase() { assertEquals(1, newRes.policies.size) assertEquals(policy, newRes.policies[0]) } + + @Suppress("UNCHECKED_CAST") + fun `test get policies response custom action`() { + val customActionParser = SampleCustomActionParser() + customActionParser.customAction = true + val extensionName = "testExtension" + ISMActionsParser.instance.addParser(customActionParser, extensionName) + val policyID = "policyID" + val action = SampleCustomActionParser.SampleCustomAction(someInt = randomInt(), index = 0) + val states = listOf(State(name = "CustomState", actions = listOf(action), transitions = listOf())) + val policy = Policy( + id = policyID, + description = "description", + schemaVersion = 1L, + lastUpdatedTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), + errorNotification = randomErrorNotification(), + defaultState = states[0].name, + states = states + ) + val res = GetPoliciesResponse(listOf(policy), 1) + + val responseString = res.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).string() + val responseMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, responseString, false) + assertEquals("Round tripping custom action doesn't work", res.convertToMap(), responseMap) + assertNotEquals("Get policies response should change the policy output", responseMap, policy.convertToMap()) + val parsedPolicy = (responseMap["policies"] as ArrayList>).first()["policy"] as Map + val parsedStates = parsedPolicy["states"] as ArrayList> + val parsedActions = parsedStates.first()["actions"] as ArrayList> + assertFalse("Get policies response should not contain the custom keyword", parsedActions.first().containsKey("custom")) + ISMActionsParser.instance.parsers.removeIf { it.getActionType() == SampleCustomActionParser.SampleCustomAction.name } + } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPolicyResponseTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPolicyResponseTests.kt index 29b8dc563..9f4de9857 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPolicyResponseTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/GetPolicyResponseTests.kt @@ -7,10 +7,18 @@ package org.opensearch.indexmanagement.indexstatemanagement.transport.action.get import org.opensearch.common.io.stream.BytesStreamOutput import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.json.JsonXContent +import org.opensearch.indexmanagement.indexstatemanagement.ISMActionsParser +import org.opensearch.indexmanagement.indexstatemanagement.action.IndexPriorityAction +import org.opensearch.indexmanagement.indexstatemanagement.extension.SampleCustomActionParser import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.IndexPriorityActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification +import org.opensearch.indexmanagement.opensearchapi.convertToMap +import org.opensearch.indexmanagement.opensearchapi.string import org.opensearch.test.OpenSearchTestCase import java.time.Instant import java.time.temporal.ChronoUnit @@ -22,7 +30,7 @@ class GetPolicyResponseTests : OpenSearchTestCase() { val version: Long = 1 val primaryTerm: Long = 123 val seqNo: Long = 456 - val actionConfig = IndexPriorityActionConfig(50, 0) + val actionConfig = IndexPriorityAction(50, 0) val states = listOf(State(name = "SetPriorityState", actions = listOf(actionConfig), transitions = listOf())) val policy = Policy( id = "policyID", @@ -43,6 +51,40 @@ class GetPolicyResponseTests : OpenSearchTestCase() { assertEquals(version, newRes.version) assertEquals(primaryTerm, newRes.primaryTerm) assertEquals(seqNo, newRes.seqNo) - assertEquals(policy, newRes.policy) + assertEquals(policy.convertToMap(), newRes.policy?.convertToMap()) + } + + @Suppress("UNCHECKED_CAST") + fun `test get policy response custom action`() { + val customActionParser = SampleCustomActionParser() + val extensionName = "testExtension" + ISMActionsParser.instance.addParser(customActionParser, extensionName) + val id = "id" + val version: Long = 1 + val primaryTerm: Long = 123 + val seqNo: Long = 456 + val policyID = "policyID" + val action = SampleCustomActionParser.SampleCustomAction(someInt = randomInt(), index = 0) + val states = listOf(State(name = "CustomState", actions = listOf(action), transitions = listOf())) + val policy = Policy( + id = policyID, + description = "description", + schemaVersion = 1L, + lastUpdatedTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), + errorNotification = randomErrorNotification(), + defaultState = states[0].name, + states = states + ) + val res = GetPolicyResponse(id, version, seqNo, primaryTerm, policy) + + val responseString = res.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).string() + val responseMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, responseString, false) + assertEquals("Round tripping custom action doesn't work", res.convertToMap(), responseMap) + assertNotEquals("Get policy response should change the policy output", responseMap, policy.convertToMap()) + val parsedPolicy = responseMap["policy"] as Map + val parsedStates = parsedPolicy["states"] as ArrayList> + val parsedActions = parsedStates.first()["actions"] as ArrayList> + assertFalse("Get policy response should not contain the custom keyword", parsedActions.first().containsKey("custom")) + ISMActionsParser.instance.parsers.removeIf { it.getActionType() == SampleCustomActionParser.SampleCustomAction.name } } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/IndexPolicyRequestTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/IndexPolicyRequestTests.kt index 26534df9e..5ec01105e 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/IndexPolicyRequestTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/IndexPolicyRequestTests.kt @@ -8,12 +8,15 @@ package org.opensearch.indexmanagement.indexstatemanagement.transport.action.ind import org.opensearch.action.support.WriteRequest import org.opensearch.common.io.stream.BytesStreamOutput import org.opensearch.common.io.stream.StreamInput +import org.opensearch.indexmanagement.indexstatemanagement.ISMActionsParser +import org.opensearch.indexmanagement.indexstatemanagement.action.AllocationAction +import org.opensearch.indexmanagement.indexstatemanagement.action.DeleteAction +import org.opensearch.indexmanagement.indexstatemanagement.action.IndexPriorityAction +import org.opensearch.indexmanagement.indexstatemanagement.extension.SampleCustomActionParser import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.AllocationActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.DeleteActionConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.action.IndexPriorityActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification +import org.opensearch.indexmanagement.opensearchapi.convertToMap import org.opensearch.test.OpenSearchTestCase import java.time.Instant import java.time.temporal.ChronoUnit @@ -22,8 +25,8 @@ class IndexPolicyRequestTests : OpenSearchTestCase() { fun `test index policy request index priority action`() { val policyID = "policyID" - val actionConfig = IndexPriorityActionConfig(50, 0) - val states = listOf(State(name = "SetPriorityState", actions = listOf(actionConfig), transitions = listOf())) + val action = IndexPriorityAction(50, 0) + val states = listOf(State(name = "SetPriorityState", actions = listOf(action), transitions = listOf())) val policy = Policy( id = policyID, description = "description", @@ -43,16 +46,15 @@ class IndexPolicyRequestTests : OpenSearchTestCase() { val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) val newReq = IndexPolicyRequest(sin) assertEquals(policyID, newReq.policyID) - assertEquals(policy, newReq.policy) + assertEquals(policy.convertToMap(), newReq.policy.convertToMap()) assertEquals(seqNo, newReq.seqNo) assertEquals(primaryTerm, newReq.primaryTerm) - assertEquals(policy, newReq.policy) } fun `test index policy request allocation action`() { val policyID = "policyID" - val actionConfig = AllocationActionConfig(require = mapOf("box_type" to "hot"), exclude = emptyMap(), include = emptyMap(), index = 0) - val states = listOf(State("Allocate", listOf(actionConfig), listOf())) + val action = AllocationAction(require = mapOf("box_type" to "hot"), exclude = emptyMap(), include = emptyMap(), index = 0) + val states = listOf(State("Allocate", listOf(action), listOf())) val policy = Policy( id = policyID, @@ -73,16 +75,15 @@ class IndexPolicyRequestTests : OpenSearchTestCase() { val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) val newReq = IndexPolicyRequest(sin) assertEquals(policyID, newReq.policyID) - assertEquals(policy, newReq.policy) + assertEquals(policy.convertToMap(), newReq.policy.convertToMap()) assertEquals(seqNo, newReq.seqNo) assertEquals(primaryTerm, newReq.primaryTerm) - assertEquals(policy, newReq.policy) } fun `test index policy request delete action`() { val policyID = "policyID" - val actionConfig = DeleteActionConfig(index = 0) - val states = listOf(State("Delete", listOf(actionConfig), listOf())) + val action = DeleteAction(index = 0) + val states = listOf(State("Delete", listOf(action), listOf())) val policy = Policy( id = policyID, @@ -103,9 +104,42 @@ class IndexPolicyRequestTests : OpenSearchTestCase() { val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) val newReq = IndexPolicyRequest(sin) assertEquals(policyID, newReq.policyID) - assertEquals(policy, newReq.policy) + assertEquals(policy.convertToMap(), newReq.policy.convertToMap()) assertEquals(seqNo, newReq.seqNo) assertEquals(primaryTerm, newReq.primaryTerm) - assertEquals(policy, newReq.policy) + } + + fun `test index policy request custom action`() { + val customActionParser = SampleCustomActionParser() + val extensionName = "testExtension" + ISMActionsParser.instance.addParser(customActionParser, extensionName) + val policyID = "policyID" + val action = SampleCustomActionParser.SampleCustomAction(someInt = randomInt(), index = 0) + val states = listOf(State("MyState", listOf(action), listOf())) + + val policy = Policy( + id = policyID, + description = "description", + schemaVersion = 1L, + lastUpdatedTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), + errorNotification = randomErrorNotification(), + defaultState = states[0].name, + states = states + ) + val seqNo: Long = 123 + val primaryTerm: Long = 456 + val refreshPolicy = WriteRequest.RefreshPolicy.NONE + val req = IndexPolicyRequest(policyID, policy, seqNo, primaryTerm, refreshPolicy) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = IndexPolicyRequest(sin) + assertEquals(policyID, newReq.policyID) + assertEquals(policy.convertToMap(), newReq.policy.convertToMap()) + assertEquals(seqNo, newReq.seqNo) + assertEquals(primaryTerm, newReq.primaryTerm) + + ISMActionsParser.instance.parsers.removeIf { it.getActionType() == SampleCustomActionParser.SampleCustomAction.name } } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/IndexPolicyResponseTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/IndexPolicyResponseTests.kt index c3f312dca..a20c9e415 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/IndexPolicyResponseTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/indexpolicy/IndexPolicyResponseTests.kt @@ -7,10 +7,18 @@ package org.opensearch.indexmanagement.indexstatemanagement.transport.action.ind import org.opensearch.common.io.stream.BytesStreamOutput import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.json.JsonXContent +import org.opensearch.indexmanagement.indexstatemanagement.ISMActionsParser +import org.opensearch.indexmanagement.indexstatemanagement.action.IndexPriorityAction +import org.opensearch.indexmanagement.indexstatemanagement.extension.SampleCustomActionParser import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.State -import org.opensearch.indexmanagement.indexstatemanagement.model.action.IndexPriorityActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomErrorNotification +import org.opensearch.indexmanagement.opensearchapi.convertToMap +import org.opensearch.indexmanagement.opensearchapi.string import org.opensearch.rest.RestStatus import org.opensearch.test.OpenSearchTestCase import java.time.Instant @@ -24,8 +32,8 @@ class IndexPolicyResponseTests : OpenSearchTestCase() { val primaryTerm: Long = 123 val seqNo: Long = 456 val policyID = "policyID" - val actionConfig = IndexPriorityActionConfig(50, 0) - val states = listOf(State(name = "SetPriorityState", actions = listOf(actionConfig), transitions = listOf())) + val action = IndexPriorityAction(50, 0) + val states = listOf(State(name = "SetPriorityState", actions = listOf(action), transitions = listOf())) val policy = Policy( id = policyID, description = "description", @@ -47,7 +55,42 @@ class IndexPolicyResponseTests : OpenSearchTestCase() { assertEquals(version, newRes.version) assertEquals(primaryTerm, newRes.primaryTerm) assertEquals(seqNo, newRes.seqNo) - assertEquals(policy, newRes.policy) + assertEquals(policy.convertToMap(), newRes.policy.convertToMap()) assertEquals(status, newRes.status) } + + @Suppress("UNCHECKED_CAST") + fun `test index policy response custom action`() { + val customActionParser = SampleCustomActionParser() + val extensionName = "testExtension" + ISMActionsParser.instance.addParser(customActionParser, extensionName) + val id = "id" + val version: Long = 1 + val primaryTerm: Long = 123 + val seqNo: Long = 456 + val policyID = "policyID" + val action = SampleCustomActionParser.SampleCustomAction(someInt = randomInt(), index = 0) + val states = listOf(State(name = "CustomState", actions = listOf(action), transitions = listOf())) + val policy = Policy( + id = policyID, + description = "description", + schemaVersion = 1L, + lastUpdatedTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), + errorNotification = randomErrorNotification(), + defaultState = states[0].name, + states = states + ) + val status = RestStatus.CREATED + + val res = IndexPolicyResponse(id, version, primaryTerm, seqNo, policy, status) + val responseString = res.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).string() + val responseMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, responseString, false) + assertEquals("Round tripping custom action doesn't work", res.convertToMap(), responseMap) + assertNotEquals("Index policy response should change the policy output", responseMap, policy.convertToMap()) + val parsedPolicy = (responseMap["policy"] as Map)["policy"] as Map + val parsedStates = parsedPolicy["states"] as ArrayList> + val parsedActions = parsedStates.first()["actions"] as ArrayList> + assertFalse("Index policy response should not contain the custom keyword", parsedActions.first().containsKey("custom")) + ISMActionsParser.instance.parsers.removeIf { it.getActionType() == SampleCustomActionParser.SampleCustomAction.name } + } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/RemovePolicyRequestTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/RemovePolicyRequestTests.kt index b8e157428..1631066f1 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/RemovePolicyRequestTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/RemovePolicyRequestTests.kt @@ -7,13 +7,14 @@ package org.opensearch.indexmanagement.indexstatemanagement.transport.action.rem import org.opensearch.common.io.stream.BytesStreamOutput import org.opensearch.common.io.stream.StreamInput +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.test.OpenSearchTestCase class RemovePolicyRequestTests : OpenSearchTestCase() { fun `test remove policy request`() { val indices = listOf("index1", "index2") - val req = RemovePolicyRequest(indices) + val req = RemovePolicyRequest(indices, DEFAULT_INDEX_TYPE) val out = BytesStreamOutput() req.writeTo(out) @@ -21,4 +22,12 @@ class RemovePolicyRequestTests : OpenSearchTestCase() { val newReq = RemovePolicyRequest(sin) assertEquals(indices, newReq.indices) } + + fun `test remove policy request with non default index type and multiple indices fails`() { + val indices = listOf("index1", "index2") + val req = RemovePolicyRequest(indices, "non-existent-index-type") + val actualException: String? = req.validate()?.validationErrors()?.firstOrNull() + val expectedException: String = RemovePolicyRequest.MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR + assertEquals("Remove policy request should have failed validation with specific exception", actualException, expectedException) + } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/retryfailedmanagedindex/RetryFailedManagedIndexRequestTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/retryfailedmanagedindex/RetryFailedManagedIndexRequestTests.kt index 611830d09..6124ae4f0 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/retryfailedmanagedindex/RetryFailedManagedIndexRequestTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/retryfailedmanagedindex/RetryFailedManagedIndexRequestTests.kt @@ -8,6 +8,7 @@ package org.opensearch.indexmanagement.indexstatemanagement.transport.action.ret import org.opensearch.common.io.stream.BytesStreamOutput import org.opensearch.common.io.stream.StreamInput import org.opensearch.common.unit.TimeValue +import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.test.OpenSearchTestCase class RetryFailedManagedIndexRequestTests : OpenSearchTestCase() { @@ -16,7 +17,7 @@ class RetryFailedManagedIndexRequestTests : OpenSearchTestCase() { val indices = listOf("index1", "index2") val startState = "state1" val masterTimeout = TimeValue.timeValueSeconds(30) - val req = RetryFailedManagedIndexRequest(indices, startState, masterTimeout) + val req = RetryFailedManagedIndexRequest(indices, startState, masterTimeout, DEFAULT_INDEX_TYPE) val out = BytesStreamOutput() req.writeTo(out) @@ -25,4 +26,15 @@ class RetryFailedManagedIndexRequestTests : OpenSearchTestCase() { assertEquals(indices, newReq.indices) assertEquals(startState, newReq.startState) } + + fun `test retry managed index request with non default index type and multiple indices fails`() { + val indices = listOf("index1", "index2") + val startState = "state1" + val masterTimeout = TimeValue.timeValueSeconds(30) + val req = RetryFailedManagedIndexRequest(indices, startState, masterTimeout, "non-existent-index-type") + + val actualException: String? = req.validate()?.validationErrors()?.firstOrNull() + val expectedException: String = RetryFailedManagedIndexRequest.MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR + assertEquals("Retry failed managed index request should have failed validation with specific exception", actualException, expectedException) + } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtilsTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtilsTests.kt index fa96a8bd4..2e08c057c 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtilsTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/ManagedIndexUtilsTests.kt @@ -5,13 +5,9 @@ package org.opensearch.indexmanagement.indexstatemanagement.util -import com.nhaarman.mockitokotlin2.doReturn -import com.nhaarman.mockitokotlin2.mock -import com.nhaarman.mockitokotlin2.whenever import org.opensearch.action.delete.DeleteRequest -import org.opensearch.alerting.destination.message.BaseMessage -import org.opensearch.alerting.destination.message.CustomWebhookMessage -import org.opensearch.cluster.metadata.IndexMetadata +// import org.opensearch.alerting.destination.message.BaseMessage +// import org.opensearch.alerting.destination.message.CustomWebhookMessage import org.opensearch.common.bytes.BytesReference import org.opensearch.common.unit.ByteSizeValue import org.opensearch.common.unit.TimeValue @@ -19,12 +15,11 @@ import org.opensearch.common.xcontent.LoggingDeprecationHandler import org.opensearch.common.xcontent.XContentHelper import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentType -import org.opensearch.index.Index import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX +import org.opensearch.indexmanagement.indexstatemanagement.action.RolloverAction import org.opensearch.indexmanagement.indexstatemanagement.model.Conditions import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig import org.opensearch.indexmanagement.indexstatemanagement.model.Transition -import org.opensearch.indexmanagement.indexstatemanagement.model.action.RolloverActionConfig import org.opensearch.indexmanagement.indexstatemanagement.model.coordinator.SweptManagedIndexConfig import org.opensearch.indexmanagement.indexstatemanagement.randomChangePolicy import org.opensearch.indexmanagement.indexstatemanagement.randomClusterStateManagedIndexConfig @@ -108,24 +103,11 @@ class ManagedIndexUtilsTests : OpenSearchTestCase() { uuid = clusterConfig.uuid, policyID = clusterConfig.policyID, seqNo = 5, primaryTerm = 17 ) - val indexMetadata1: IndexMetadata = mock() - val indexMetadata2: IndexMetadata = mock() - val indexMetadata3: IndexMetadata = mock() - val indexMetadata4: IndexMetadata = mock() - whenever(indexMetadata1.index).doReturn(Index(clusterConfig.index, clusterConfig.uuid)) - whenever(indexMetadata2.index).doReturn(Index(clusterConfigToUpdate.index, clusterConfigToUpdate.uuid)) - whenever(indexMetadata3.index).doReturn(Index(clusterConfigBeingUpdated.index, clusterConfigBeingUpdated.uuid)) - whenever(indexMetadata4.index).doReturn(Index(clusterConfigToCreate.index, clusterConfigToCreate.uuid)) - - val requests = getDeleteManagedIndexRequests( - listOf(indexMetadata1, indexMetadata2, indexMetadata3, indexMetadata4), - mapOf( - sweptConfig.uuid to sweptConfig, - sweptConfigToDelete.uuid to sweptConfigToDelete, - sweptConfigToBeUpdated.uuid to sweptConfigToBeUpdated, - sweptConfigBeingUpdated.uuid to sweptConfigBeingUpdated - ) + val managedIndicesToDelete = getManagedIndicesToDelete( + listOf(clusterConfig.uuid, clusterConfigToUpdate.uuid, clusterConfigBeingUpdated.uuid, clusterConfigToCreate.uuid), + listOf(sweptConfig.uuid, sweptConfigToDelete.uuid, sweptConfigToBeUpdated.uuid, sweptConfigBeingUpdated.uuid) ) + val requests = managedIndicesToDelete.map { deleteManagedIndexRequest(it) } assertEquals("Too many requests", 1, requests.size) val request = requests.first() @@ -143,7 +125,7 @@ class ManagedIndexUtilsTests : OpenSearchTestCase() { } fun `test rollover action config evaluate conditions`() { - val noConditionsConfig = RolloverActionConfig(minSize = null, minDocs = null, minAge = null, minPrimaryShardSize = null, index = 0) + val noConditionsConfig = RolloverAction(minSize = null, minDocs = null, minAge = null, minPrimaryShardSize = null, index = 0) assertTrue( "No conditions should always pass", noConditionsConfig @@ -160,7 +142,7 @@ class ManagedIndexUtilsTests : OpenSearchTestCase() { .evaluateConditions(indexAgeTimeValue = TimeValue.timeValueMillis(6000), numDocs = 5, indexSize = ByteSizeValue(5), primaryShardSize = ByteSizeValue(5)) ) - val minSizeConfig = RolloverActionConfig(minSize = ByteSizeValue(5), minDocs = null, minAge = null, minPrimaryShardSize = null, index = 0) + val minSizeConfig = RolloverAction(minSize = ByteSizeValue(5), minDocs = null, minAge = null, minPrimaryShardSize = null, index = 0) assertFalse( "Less bytes should not pass", minSizeConfig @@ -177,7 +159,7 @@ class ManagedIndexUtilsTests : OpenSearchTestCase() { .evaluateConditions(indexAgeTimeValue = TimeValue.timeValueMillis(1000), numDocs = 0, indexSize = ByteSizeValue(10), primaryShardSize = ByteSizeValue(10)) ) - val minPrimarySizeConfig = RolloverActionConfig(minSize = null, minDocs = null, minAge = null, minPrimaryShardSize = ByteSizeValue(5), index = 0) + val minPrimarySizeConfig = RolloverAction(minSize = null, minDocs = null, minAge = null, minPrimaryShardSize = ByteSizeValue(5), index = 0) assertFalse( "Less primary bytes should not pass", minPrimarySizeConfig @@ -194,7 +176,7 @@ class ManagedIndexUtilsTests : OpenSearchTestCase() { .evaluateConditions(indexAgeTimeValue = TimeValue.timeValueMillis(1000), numDocs = 0, indexSize = ByteSizeValue(10), primaryShardSize = ByteSizeValue(10)) ) - val minDocsConfig = RolloverActionConfig(minSize = null, minDocs = 5, minAge = null, minPrimaryShardSize = null, index = 0) + val minDocsConfig = RolloverAction(minSize = null, minDocs = 5, minAge = null, minPrimaryShardSize = null, index = 0) assertFalse( "Less docs should not pass", minDocsConfig @@ -211,7 +193,7 @@ class ManagedIndexUtilsTests : OpenSearchTestCase() { .evaluateConditions(indexAgeTimeValue = TimeValue.timeValueMillis(1000), numDocs = 10, indexSize = ByteSizeValue.ZERO, primaryShardSize = ByteSizeValue.ZERO) ) - val minAgeConfig = RolloverActionConfig(minSize = null, minDocs = null, minAge = TimeValue.timeValueSeconds(5), minPrimaryShardSize = null, index = 0) + val minAgeConfig = RolloverAction(minSize = null, minDocs = null, minAge = TimeValue.timeValueSeconds(5), minPrimaryShardSize = null, index = 0) assertFalse( "Index age that is too young should not pass", minAgeConfig @@ -223,7 +205,7 @@ class ManagedIndexUtilsTests : OpenSearchTestCase() { .evaluateConditions(indexAgeTimeValue = TimeValue.timeValueMillis(10000), numDocs = 0, indexSize = ByteSizeValue.ZERO, primaryShardSize = ByteSizeValue.ZERO) ) - val multiConfig = RolloverActionConfig(minSize = ByteSizeValue(1), minDocs = 1, minAge = TimeValue.timeValueSeconds(5), minPrimaryShardSize = ByteSizeValue(1), index = 0) + val multiConfig = RolloverAction(minSize = ByteSizeValue(1), minDocs = 1, minAge = TimeValue.timeValueSeconds(5), minPrimaryShardSize = ByteSizeValue(1), index = 0) assertFalse( "No conditions met should not pass", multiConfig @@ -301,29 +283,29 @@ class ManagedIndexUtilsTests : OpenSearchTestCase() { ) } - fun `test ips in denylist`() { - val ips = listOf( - "127.0.0.1", // 127.0.0.0/8 - "10.0.0.1", // 10.0.0.0/8 - "10.11.12.13", // 10.0.0.0/8 - "172.16.0.1", // "172.16.0.0/12" - "192.168.0.1", // 192.168.0.0/16" - "0.0.0.1", // 0.0.0.0/8 - "9.9.9.9" - ) - for (ip in ips) { - val bm = createMessageWithHost(ip) - assertEquals(true, bm.isHostInDenylist(HOST_DENY_LIST)) - } - } - - fun `test url in denylist`() { - val urls = listOf("https://www.amazon.com", "https://mytest.com", "https://mytest.com") - for (url in urls) { - val bm = createMessageWithURl(url) - assertEquals(false, bm.isHostInDenylist(HOST_DENY_LIST)) - } - } +// fun `test ips in denylist`() { +// val ips = listOf( +// "127.0.0.1", // 127.0.0.0/8 +// "10.0.0.1", // 10.0.0.0/8 +// "10.11.12.13", // 10.0.0.0/8 +// "172.16.0.1", // "172.16.0.0/12" +// "192.168.0.1", // 192.168.0.0/16" +// "0.0.0.1", // 0.0.0.0/8 +// "9.9.9.9" +// ) +// for (ip in ips) { +// val bm = createMessageWithHost(ip) +// assertEquals(true, bm.isHostInDenylist(HOST_DENY_LIST)) +// } +// } + +// fun `test url in denylist`() { +// val urls = listOf("https://www.amazon.com", "https://mytest.com", "https://mytest.com") +// for (url in urls) { +// val bm = createMessageWithURl(url) +// assertEquals(false, bm.isHostInDenylist(HOST_DENY_LIST)) +// } +// } private fun contentParser(bytesReference: BytesReference): XContentParser { return XContentHelper.createParser( @@ -341,21 +323,21 @@ class ManagedIndexUtilsTests : OpenSearchTestCase() { "9.9.9.9" // ip ) - private fun createMessageWithHost(host: String): BaseMessage { - return CustomWebhookMessage.Builder("abc") - .withHost(host) - .withPath("incomingwebhooks/383c0e2b-d028-44f4-8d38-696754bc4574") - .withMessage("{\"Content\":\"Message test\"}") - .withMethod("POST") - .withQueryParams(HashMap()).build() - } - - private fun createMessageWithURl(url: String): BaseMessage { - return CustomWebhookMessage.Builder("abc") - .withUrl(url) - .withPath("incomingwebhooks/383c0e2b-d028-44f4-8d38-696754bc4574") - .withMessage("{\"Content\":\"Message test\"}") - .withMethod("POST") - .withQueryParams(HashMap()).build() - } +// private fun createMessageWithHost(host: String): BaseMessage { +// return CustomWebhookMessage.Builder("abc") +// .withHost(host) +// .withPath("incomingwebhooks/383c0e2b-d028-44f4-8d38-696754bc4574") +// .withMessage("{\"Content\":\"Message test\"}") +// .withMethod("POST") +// .withQueryParams(HashMap()).build() +// } +// +// private fun createMessageWithURl(url: String): BaseMessage { +// return CustomWebhookMessage.Builder("abc") +// .withUrl(url) +// .withPath("incomingwebhooks/383c0e2b-d028-44f4-8d38-696754bc4574") +// .withMessage("{\"Content\":\"Message test\"}") +// .withMethod("POST") +// .withQueryParams(HashMap()).build() +// } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupMapperServiceTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupMapperServiceTests.kt index dd6b8bb0e..f606e207b 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupMapperServiceTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupMapperServiceTests.kt @@ -24,7 +24,6 @@ import org.opensearch.cluster.service.ClusterService import org.opensearch.common.collect.ImmutableOpenMap import org.opensearch.common.xcontent.XContentType import org.opensearch.indexmanagement.rollup.model.RollupJobValidationResult -import org.opensearch.indexmanagement.util._DOC import org.opensearch.test.OpenSearchTestCase import java.time.Instant @@ -92,7 +91,7 @@ class RollupMapperServiceTests : OpenSearchTestCase() { val client = getClient( getAdminClient( getIndicesAdminClient( - getMappingsResponse = getMappingResponse(sourceIndex, "custom_type"), + getMappingsResponse = getMappingResponse(sourceIndex), getMappingsException = null ) ) @@ -130,7 +129,7 @@ class RollupMapperServiceTests : OpenSearchTestCase() { val client = getClient( getAdminClient( getIndicesAdminClient( - getMappingsResponse = getMappingResponse(sourceIndex, "custom_type", true), + getMappingsResponse = getMappingResponse(sourceIndex, true), getMappingsException = null ) ) @@ -291,24 +290,20 @@ class RollupMapperServiceTests : OpenSearchTestCase() { private fun getIndexNameExpressionResolver(concreteIndices: List): IndexNameExpressionResolver = mock { on { concreteIndexNames(any(), any(), anyBoolean(), anyVararg()) } doReturn concreteIndices.toTypedArray() } - private fun getMappingResponse(indexName: String, mappingType: String = _DOC, emptyMapping: Boolean = false): GetMappingsResponse { - val docMappings = if (emptyMapping) { + private fun getMappingResponse(indexName: String, emptyMapping: Boolean = false): GetMappingsResponse { + val mappings = if (emptyMapping) { ImmutableOpenMap.Builder().build() } else { val mappingSourceMap = createParser( XContentType.JSON.xContent(), javaClass.classLoader.getResource("mappings/kibana-sample-data.json").readText() ).map() - val mappingMetadata = MappingMetadata(mappingType, mappingSourceMap) + val mappingMetadata = MappingMetadata("_doc", mappingSourceMap) // it seems it still expects a type, i.e. _doc now ImmutableOpenMap.Builder() - .fPut(mappingType, mappingMetadata) + .fPut(indexName, mappingMetadata) .build() } - val mappings = ImmutableOpenMap.Builder>() - .fPut(indexName, docMappings) - .build() - return GetMappingsResponse(mappings) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/model/ISMRollupTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/model/ISMRollupTests.kt index 6e1c073dd..d0b35ecff 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/model/ISMRollupTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/model/ISMRollupTests.kt @@ -5,13 +5,13 @@ package org.opensearch.indexmanagement.rollup.model +import org.apache.commons.codec.digest.DigestUtils import org.opensearch.index.seqno.SequenceNumbers import org.opensearch.indexmanagement.rollup.randomDateHistogram import org.opensearch.indexmanagement.rollup.randomISMRollup import org.opensearch.indexmanagement.rollup.randomTerms import org.opensearch.indexmanagement.util.IndexUtils import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule -import org.opensearch.notification.repackage.org.apache.commons.codec.digest.DigestUtils import org.opensearch.test.OpenSearchTestCase import java.time.temporal.ChronoUnit import kotlin.test.assertFailsWith diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/TestHelpers.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/TestHelpers.kt index de07b3d14..a08700907 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/TestHelpers.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/TestHelpers.kt @@ -5,12 +5,15 @@ package org.opensearch.indexmanagement.transform +import org.opensearch.cluster.metadata.IndexMetadata import org.opensearch.common.io.stream.BytesStreamOutput import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput import org.opensearch.common.io.stream.NamedWriteableRegistry import org.opensearch.common.settings.Settings import org.opensearch.common.xcontent.ToXContent import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.index.Index +import org.opensearch.index.shard.ShardId import org.opensearch.indexmanagement.common.model.dimension.Dimension import org.opensearch.indexmanagement.opensearchapi.string import org.opensearch.indexmanagement.randomInstant @@ -18,6 +21,7 @@ import org.opensearch.indexmanagement.randomSchedule import org.opensearch.indexmanagement.randomUser import org.opensearch.indexmanagement.rollup.randomAfterKey import org.opensearch.indexmanagement.rollup.randomDimension +import org.opensearch.indexmanagement.transform.model.ContinuousTransformStats import org.opensearch.indexmanagement.transform.model.ExplainTransform import org.opensearch.indexmanagement.transform.model.Transform import org.opensearch.indexmanagement.transform.model.TransformMetadata @@ -68,6 +72,7 @@ fun randomAggregationFactories(): AggregatorFactories.Builder { fun randomTransform(): Transform { val enabled = OpenSearchRestTestCase.randomBoolean() + val isContinuous = OpenSearchRestTestCase.randomBoolean() return Transform( id = OpenSearchRestTestCase.randomAlphaOfLength(10), seqNo = OpenSearchRestTestCase.randomNonNegativeLong(), @@ -82,15 +87,17 @@ fun randomTransform(): Transform { sourceIndex = OpenSearchRestTestCase.randomAlphaOfLength(10).toLowerCase(Locale.ROOT), targetIndex = OpenSearchRestTestCase.randomAlphaOfLength(10).toLowerCase(Locale.ROOT), roles = OpenSearchRestTestCase.randomList(10) { OpenSearchRestTestCase.randomAlphaOfLength(10) }, - pageSize = OpenSearchRestTestCase.randomIntBetween(1, 10000), + pageSize = if (isContinuous) OpenSearchRestTestCase.randomIntBetween(1, 1000) else OpenSearchRestTestCase.randomIntBetween(1, 10000), groups = randomGroups(), aggregations = randomAggregationFactories(), + continuous = isContinuous, user = randomUser() ) } fun randomTransformMetadata(): TransformMetadata { val status = randomTransformMetadataStatus() + val isContinuous = OpenSearchRestTestCase.randomBoolean() return TransformMetadata( id = OpenSearchRestTestCase.randomAlphaOfLength(10), seqNo = OpenSearchRestTestCase.randomNonNegativeLong(), @@ -100,7 +107,9 @@ fun randomTransformMetadata(): TransformMetadata { lastUpdatedAt = randomInstant(), status = status, failureReason = if (status == TransformMetadata.Status.FAILED) OpenSearchRestTestCase.randomAlphaOfLength(10) else null, - stats = randomTransformStats() + stats = randomTransformStats(), + shardIDToGlobalCheckpoint = if (isContinuous) randomShardIDToGlobalCheckpoint() else null, + continuousStats = if (isContinuous) randomContinuousStats() else null ) } @@ -114,6 +123,33 @@ fun randomTransformStats(): TransformStats { ) } +fun randomShardIDToGlobalCheckpoint(): Map { + val numIndices = OpenSearchRestTestCase.randomIntBetween(1, 10) + val randomIndices = (1..numIndices).map { randomShardID() } + return randomIndices.associateWith { OpenSearchRestTestCase.randomNonNegativeLong() } +} + +fun randomShardID(): ShardId { + val indexName: String = OpenSearchRestTestCase.randomAlphaOfLength(10).toLowerCase(Locale.ROOT) + // We lose the index uuid in an XContent round trip, but we don't use it anyways + val testIndex = Index(indexName, IndexMetadata.INDEX_UUID_NA_VALUE) + val shardNumber: Int = OpenSearchRestTestCase.randomIntBetween(0, 100) + return ShardId(testIndex, shardNumber) +} + +fun randomContinuousStats(): ContinuousTransformStats { + return ContinuousTransformStats( + lastTimestamp = randomInstant(), + documentsBehind = randomDocumentsBehind() + ) +} + +fun randomDocumentsBehind(): Map { + val numIndices = OpenSearchRestTestCase.randomIntBetween(1, 10) + val randomIndices = (1..numIndices).map { OpenSearchRestTestCase.randomAlphaOfLength(10).toLowerCase(Locale.ROOT) } + return randomIndices.associateWith { OpenSearchRestTestCase.randomNonNegativeLong() } +} + fun randomTransformMetadataStatus(): TransformMetadata.Status { return OpenSearchRestTestCase.randomFrom(TransformMetadata.Status.values().toList()) } @@ -130,6 +166,6 @@ fun TransformMetadata.toJsonString(params: ToXContent.Params = ToXContent.EMPTY_ // Builds the required stream input for transforms by wrapping the stream input with required NamedWriteableRegistry. fun buildStreamInputForTransforms(out: BytesStreamOutput): NamedWriteableAwareStreamInput { - val namedWriteableRegistry = NamedWriteableRegistry(SearchModule(Settings.EMPTY, false, emptyList()).namedWriteables) + val namedWriteableRegistry = NamedWriteableRegistry(SearchModule(Settings.EMPTY, emptyList()).namedWriteables) return NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry) } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRestTestCase.kt index b337d3172..3941c9bf0 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRestTestCase.kt @@ -76,6 +76,16 @@ abstract class TransformRestTestCase : IndexManagementRestTestCase() { return response } + protected fun disableTransform(transformId: String) { + val response = client() + .makeRequest( + "POST", + "$TRANSFORM_BASE_URI/$transformId/_stop", + emptyMap() + ) + assertEquals("Unable to disable transform $transformId", RestStatus.OK, response.restStatus()) + } + protected fun createRandomTransform(refresh: Boolean = true): Transform { val transform = randomTransform() val transformId = createTransform(transform, refresh = refresh).id @@ -184,6 +194,21 @@ abstract class TransformRestTestCase : IndexManagementRestTestCase() { return metadata } + @Suppress("UNCHECKED_CAST") + protected fun getTransformDocumentsBehind( + transformId: String, + header: BasicHeader = BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ): Map { + val explainResponse = client().makeRequest("GET", "$TRANSFORM_BASE_URI/$transformId/_explain", null, header) + assertEquals(RestStatus.OK, explainResponse.restStatus()) + + val explainResponseMap = explainResponse.asMap() + val explainMetadata = explainResponseMap[transformId] as Map + val metadata = explainMetadata["transform_metadata"] as Map + val continuousStats = metadata["continuous_stats"] as Map + return continuousStats["documents_behind"] as Map + } + protected fun updateTransformStartTime(update: Transform, desiredStartTimeMillis: Long? = null) { // Before updating start time of a job always make sure there are no unassigned shards that could cause the config // index to move to a new node and negate this forced start @@ -216,6 +241,6 @@ abstract class TransformRestTestCase : IndexManagementRestTestCase() { protected fun Transform.toHttpEntity(): HttpEntity = StringEntity(toJsonString(), APPLICATION_JSON) override fun xContentRegistry(): NamedXContentRegistry { - return NamedXContentRegistry(SearchModule(Settings.EMPTY, false, emptyList()).namedXContents) + return NamedXContentRegistry(SearchModule(Settings.EMPTY, emptyList()).namedXContents) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRunnerIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRunnerIT.kt index ec859c023..2834da49c 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRunnerIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRunnerIT.kt @@ -17,21 +17,20 @@ import org.opensearch.indexmanagement.transform.model.Transform import org.opensearch.indexmanagement.transform.model.TransformMetadata import org.opensearch.indexmanagement.waitFor import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule +import org.opensearch.rest.RestStatus import org.opensearch.script.Script import org.opensearch.script.ScriptType import org.opensearch.search.aggregations.AggregationBuilders import org.opensearch.search.aggregations.AggregatorFactories import org.opensearch.search.aggregations.metrics.ScriptedMetricAggregationBuilder +import java.lang.Integer.min import java.time.Instant import java.time.temporal.ChronoUnit class TransformRunnerIT : TransformRestTestCase() { fun `test transform`() { - if (!indexExists("transform-source-index")) { - generateNYCTaxiData("transform-source-index") - assertIndexExists("transform-source-index") - } + validateSourceIndex("transform-source-index") val transform = Transform( id = "id_1", @@ -71,10 +70,7 @@ class TransformRunnerIT : TransformRestTestCase() { } fun `test transform with data filter`() { - if (!indexExists("transform-source-index")) { - generateNYCTaxiData("transform-source-index") - assertIndexExists("transform-source-index") - } + validateSourceIndex("transform-source-index") val transform = Transform( id = "id_2", @@ -132,10 +128,7 @@ class TransformRunnerIT : TransformRestTestCase() { } fun `test transform with aggregations`() { - if (!indexExists("transform-source-index")) { - generateNYCTaxiData("transform-source-index") - assertIndexExists("transform-source-index") - } + validateSourceIndex("transform-source-index") val aggregatorFactories = AggregatorFactories.builder() aggregatorFactories.addAggregator(AggregationBuilders.sum("revenue").field("total_amount")) @@ -210,10 +203,7 @@ class TransformRunnerIT : TransformRestTestCase() { } fun `test transform with failure during indexing`() { - if (!indexExists("transform-source-index")) { - generateNYCTaxiData("transform-source-index") - assertIndexExists("transform-source-index") - } + validateSourceIndex("transform-source-index") // Indexing failure because target index is strictly mapped createIndex("transform-target-strict-index", Settings.EMPTY, getStrictMappings()) @@ -252,10 +242,7 @@ class TransformRunnerIT : TransformRestTestCase() { } fun `test transform with invalid aggregation triggering search failure`() { - if (!indexExists("transform-source-index")) { - generateNYCTaxiData("transform-source-index") - assertIndexExists("transform-source-index") - } + validateSourceIndex("transform-source-index") val aggregatorFactories = AggregatorFactories.builder() aggregatorFactories.addAggregator( @@ -388,6 +375,579 @@ class TransformRunnerIT : TransformRestTestCase() { assertTrue("Didn't capture search time", metadata.stats.searchTimeInMillis > 0) } + fun `test no-op execution when no buckets have been modified`() { + validateSourceIndex("transform-no-op-source-index") + + val transform = Transform( + id = "id_8", + schemaVersion = 1L, + enabled = true, + enabledAt = Instant.now(), + updatedAt = Instant.now(), + jobSchedule = IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES), + description = "test transform", + metadataId = null, + sourceIndex = "transform-no-op-source-index", + targetIndex = "transform-no-op-target-index", + roles = emptyList(), + pageSize = 100, + groups = listOf( + Terms(sourceField = "store_and_fwd_flag", targetField = "flag"), + Histogram(sourceField = "trip_distance", targetField = "distance", interval = 0.1) + ), + continuous = true + ).let { createTransform(it, it.id) } + + updateTransformStartTime(transform) + + waitFor { assertTrue("Target transform index was not created", indexExists(transform.targetIndex)) } + + val firstIterationMetadata = waitFor { + val job = getTransform(transformId = transform.id) + assertNotNull("Transform job doesn't have metadata set", job.metadataId) + val transformMetadata = getTransformMetadata(job.metadataId!!) + assertEquals("Transform did not complete iteration or had incorrect number of documents processed", 5000, transformMetadata.stats.documentsProcessed) + assertEquals("Transform did not complete iteration", null, transformMetadata.afterKey) + assertNotNull("Continuous stats were not updated", transformMetadata.continuousStats) + transformMetadata + } + + assertEquals("Not the expected transform status", TransformMetadata.Status.STARTED, firstIterationMetadata.status) + assertEquals("Not the expected pages processed", 3L, firstIterationMetadata.stats.pagesProcessed) + assertEquals("Not the expected documents indexed", 198L, firstIterationMetadata.stats.documentsIndexed) + assertEquals("Not the expected documents processed", 5000L, firstIterationMetadata.stats.documentsProcessed) + assertTrue("Didn't capture indexed time", firstIterationMetadata.stats.indexTimeInMillis > 0) + assertTrue("Didn't capture search time", firstIterationMetadata.stats.searchTimeInMillis > 0) + + updateTransformStartTime(transform) + + val secondIterationMetadata = waitFor { + val job = getTransform(transformId = transform.id) + assertNotNull("Transform job doesn't have metadata set", job.metadataId) + val transformMetadata = getTransformMetadata(job.metadataId!!) + assertTrue("Transform did not complete iteration or update timestamp", transformMetadata.continuousStats!!.lastTimestamp!! > firstIterationMetadata.continuousStats!!.lastTimestamp) + transformMetadata + } + + assertEquals("Transform did not have null afterKey after iteration", null, secondIterationMetadata.afterKey) + assertEquals("Not the expected transform status", TransformMetadata.Status.STARTED, firstIterationMetadata.status) + assertEquals("Not the expected pages processed", firstIterationMetadata.stats.pagesProcessed, secondIterationMetadata.stats.pagesProcessed) + assertEquals("Not the expected documents indexed", firstIterationMetadata.stats.documentsIndexed, secondIterationMetadata.stats.documentsIndexed) + assertEquals("Not the expected documents processed", firstIterationMetadata.stats.documentsProcessed, secondIterationMetadata.stats.documentsProcessed) + assertEquals("Not the expected index time", firstIterationMetadata.stats.indexTimeInMillis, secondIterationMetadata.stats.indexTimeInMillis) + assertEquals("Not the expected search time", firstIterationMetadata.stats.searchTimeInMillis, secondIterationMetadata.stats.searchTimeInMillis) + + disableTransform(transform.id) + } + + fun `test continuous transform picks up new documents`() { + validateSourceIndex("continuous-transform-source-index") + + val aggregatorFactories = AggregatorFactories.builder() + aggregatorFactories.addAggregator(AggregationBuilders.sum("revenue").field("total_amount")) + + val transform = Transform( + id = "id_9", + schemaVersion = 1L, + enabled = true, + enabledAt = Instant.now(), + updatedAt = Instant.now(), + jobSchedule = IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES), + description = "test transform", + metadataId = null, + sourceIndex = "continuous-transform-source-index", + targetIndex = "continuous-transform-target-index", + roles = emptyList(), + pageSize = 100, + groups = listOf( + Terms(sourceField = "store_and_fwd_flag", targetField = "flag") + ), + continuous = true, + aggregations = aggregatorFactories + ).let { createTransform(it, it.id) } + + updateTransformStartTime(transform) + + waitFor { assertTrue("Target transform index was not created", indexExists(transform.targetIndex)) } + + val firstIterationMetadata = waitFor { + val job = getTransform(transformId = transform.id) + assertNotNull("Transform job doesn't have metadata set", job.metadataId) + val transformMetadata = getTransformMetadata(job.metadataId!!) + assertEquals("Transform did not complete iteration or had incorrect number of documents processed", 5000, transformMetadata.stats.documentsProcessed) + assertEquals("Transform did not complete iteration", null, transformMetadata.afterKey) + transformMetadata + } + + assertEquals("Not the expected transform status", TransformMetadata.Status.STARTED, firstIterationMetadata.status) + assertEquals("Not the expected pages processed", 2L, firstIterationMetadata.stats.pagesProcessed) + assertEquals("Not the expected documents indexed", 2L, firstIterationMetadata.stats.documentsIndexed) + assertEquals("Not the expected documents processed", 5000L, firstIterationMetadata.stats.documentsProcessed) + assertTrue("Doesn't capture indexed time", firstIterationMetadata.stats.indexTimeInMillis > 0) + assertTrue("Didn't capture search time", firstIterationMetadata.stats.searchTimeInMillis > 0) + + var hits = waitFor { + val response = client().makeRequest( + "GET", "continuous-transform-target-index/_search", + StringEntity("{}", ContentType.APPLICATION_JSON) + ) + assertEquals("Request failed", RestStatus.OK, response.restStatus()) + val responseHits = response.asMap().getValue("hits") as Map<*, *> + val totalDocs = (responseHits["hits"] as ArrayList<*>).fold(0) { sum, bucket -> + val docCount = ((bucket as Map<*, *>)["_source"] as Map<*, *>)["transform._doc_count"] as Int + sum + docCount + } + assertEquals("Not all documents included in the transform target index", 5000, totalDocs) + + responseHits["hits"] as ArrayList<*> + } + hits.forEach { + val bucket = ((it as Map<*, *>)["_source"] as Map<*, *>) + if (bucket["flag"] == "N") { + assertEquals("Transform sum not calculated correctly", 76547.9, bucket["revenue"] as Double, 0.1) + } else { + assertEquals("Transform sum not calculated correctly", 359.8, bucket["revenue"] as Double, 0.1) + } + } + + // Add the same 5000 documents again, and start another execution + insertSampleBulkData(transform.sourceIndex, javaClass.classLoader.getResource("data/nyc_5000.ndjson").readText()) + + waitFor { + val documentsBehind = getTransformDocumentsBehind(transform.id)[transform.sourceIndex] + assertEquals("Documents behind not calculated correctly", 5000, documentsBehind) + } + + updateTransformStartTime(transform) + + val secondIterationMetadata = waitFor { + val job = getTransform(transformId = transform.id) + assertNotNull("Transform job doesn't have metadata set", job.metadataId) + val transformMetadata = getTransformMetadata(job.metadataId!!) + // As the new documents all fall into the same buckets as the last, all of the documents are processed again + assertEquals("Transform did not complete iteration or had incorrect number of documents processed", 15000, transformMetadata.stats.documentsProcessed) + assertEquals("Transform did not have null afterKey after iteration", null, transformMetadata.afterKey) + transformMetadata + } + + assertEquals("Not the expected transform status", TransformMetadata.Status.STARTED, secondIterationMetadata.status) + assertEquals("More than expected pages processed", 4L, secondIterationMetadata.stats.pagesProcessed) + assertEquals("More than expected documents indexed", 4L, secondIterationMetadata.stats.documentsIndexed) + assertEquals("Not the expected documents processed", 15000L, secondIterationMetadata.stats.documentsProcessed) + assertTrue("Doesn't capture indexed time", secondIterationMetadata.stats.indexTimeInMillis > firstIterationMetadata.stats.indexTimeInMillis) + assertTrue("Didn't capture search time", secondIterationMetadata.stats.searchTimeInMillis > firstIterationMetadata.stats.searchTimeInMillis) + + hits = waitFor { + val response = client().makeRequest( + "GET", "continuous-transform-target-index/_search", + StringEntity("{}", ContentType.APPLICATION_JSON) + ) + assertEquals("Request failed", RestStatus.OK, response.restStatus()) + val responseHits = response.asMap().getValue("hits") as Map<*, *> + val totalDocs = (responseHits["hits"] as ArrayList<*>).fold(0) { sum, bucket -> + val docCount = ((bucket as Map<*, *>)["_source"] as Map<*, *>)["transform._doc_count"] as Int + sum + docCount + } + assertEquals("Not all documents included in the transform target index", 10000, totalDocs) + + responseHits["hits"] as ArrayList<*> + } + hits.forEach { + val bucket = ((it as Map<*, *>)["_source"] as Map<*, *>) + if (bucket["flag"] == "N") { + assertEquals("Transform sum not calculated correctly", 153095.9, bucket["revenue"] as Double, 0.1) + } else { + assertEquals("Transform sum not calculated correctly", 719.7, bucket["revenue"] as Double, 0.1) + } + } + disableTransform(transform.id) + } + + fun `test continuous transform only transforms modified buckets`() { + val sourceIndex = "modified-bucket-source-index" + createIndex(sourceIndex, Settings.EMPTY, """"properties":{"iterating_id":{"type":"integer"},"twice_id":{"type":"integer"}}""") + for (i in 0..47) { + val jsonString = "{\"iterating_id\": \"$i\",\"twice_id\": \"${i * 2}\"}" + insertSampleData(sourceIndex, 1, jsonString = jsonString) + } + + val aggregatorFactories = AggregatorFactories.builder() + aggregatorFactories.addAggregator(AggregationBuilders.sum("twice_id_sum").field("twice_id")) + + val transform = Transform( + id = "id_10", + schemaVersion = 1L, + enabled = true, + enabledAt = Instant.now(), + updatedAt = Instant.now(), + jobSchedule = IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES), + description = "test transform", + metadataId = null, + sourceIndex = sourceIndex, + targetIndex = "modified-bucket-target-index", + roles = emptyList(), + pageSize = 100, + groups = listOf( + Histogram(sourceField = "iterating_id", targetField = "id_group", interval = 5.0) + ), + continuous = true, + aggregations = aggregatorFactories + ).let { createTransform(it, it.id) } + + updateTransformStartTime(transform) + + waitFor { assertTrue("Target transform index was not created", indexExists(transform.targetIndex)) } + + val firstIterationMetadata = waitFor { + val job = getTransform(transformId = transform.id) + assertNotNull("Transform job doesn't have metadata set", job.metadataId) + val transformMetadata = getTransformMetadata(job.metadataId!!) + assertEquals("Transform did not complete iteration or had incorrect number of documents processed", 48, transformMetadata.stats.documentsProcessed) + assertEquals("Transform did not complete iteration", null, transformMetadata.afterKey) + transformMetadata + } + + assertEquals("Not the expected transform status", TransformMetadata.Status.STARTED, firstIterationMetadata.status) + assertEquals("Not the expected pages processed", 2L, firstIterationMetadata.stats.pagesProcessed) + assertEquals("Not the expected documents indexed", 10L, firstIterationMetadata.stats.documentsIndexed) + assertEquals("Not the expected documents processed", 48L, firstIterationMetadata.stats.documentsProcessed) + assertTrue("Doesn't capture indexed time", firstIterationMetadata.stats.indexTimeInMillis > 0) + assertTrue("Didn't capture search time", firstIterationMetadata.stats.searchTimeInMillis > 0) + + // Get all of the buckets + var hits = waitFor { + val response = client().makeRequest( + "GET", "${transform.targetIndex}/_search", + StringEntity("{\"size\": 25}", ContentType.APPLICATION_JSON) + ) + assertEquals("Request failed", RestStatus.OK, response.restStatus()) + val responseHits = response.asMap().getValue("hits") as Map<*, *> + val totalDocs = (responseHits["hits"] as ArrayList<*>).fold(0) { sum, bucket -> + val docCount = ((bucket as Map<*, *>)["_source"] as Map<*, *>)["transform._doc_count"] as Int + sum + docCount + } + assertEquals("Not all documents included in the transform target index", 48, totalDocs) + + responseHits["hits"] as ArrayList<*> + } + + // Validate the buckets include the correct information + hits.forEach { + val bucket = ((it as Map<*, *>)["_source"] as Map<*, *>) + val idGroup = (bucket["id_group"] as Double).toInt() + if (idGroup != 45) { + val expectedSum = ((idGroup * 2)..((idGroup * 2) + 8) step 2).sum() + assertEquals("ID sum not calculated correctly", expectedSum, (bucket["twice_id_sum"] as Double).toInt()) + } else { + // The last bucket will only be partially filled + assertEquals("ID sum not calculated correctly", 276, (bucket["twice_id_sum"] as Double).toInt()) + } + } + // Add more data + for (i in 48..99) { + val jsonString = "{\"iterating_id\": \"$i\",\"twice_id\": \"${i * 2}\"}" + insertSampleData(sourceIndex, 1, jsonString = jsonString) + } + + waitFor { + val documentsBehind = getTransformDocumentsBehind(transform.id)[transform.sourceIndex] + assertEquals("Documents behind not calculated correctly", 52, documentsBehind) + } + + updateTransformStartTime(transform) + + val secondIterationMetadata = waitFor { + val job = getTransform(transformId = transform.id) + assertNotNull("Transform job doesn't have metadata set", job.metadataId) + val transformMetadata = getTransformMetadata(job.metadataId!!) + // As the ids 45-47 will be processed a second time when the bucket is recalculated, this number is greater than 100 + assertEquals("Transform did not complete iteration or had incorrect number of documents processed", 103L, transformMetadata.stats.documentsProcessed) + assertEquals("Transform did not have null afterKey after iteration", null, transformMetadata.afterKey) + transformMetadata + } + + assertEquals("Not the expected transform status", TransformMetadata.Status.STARTED, secondIterationMetadata.status) + assertEquals("More than expected pages processed", 4L, secondIterationMetadata.stats.pagesProcessed) + assertEquals("More than expected documents indexed", 21L, secondIterationMetadata.stats.documentsIndexed) + assertEquals("Not the expected documents processed", 103L, secondIterationMetadata.stats.documentsProcessed) + assertTrue("Doesn't capture indexed time", secondIterationMetadata.stats.indexTimeInMillis > firstIterationMetadata.stats.indexTimeInMillis) + assertTrue("Didn't capture search time", secondIterationMetadata.stats.searchTimeInMillis > firstIterationMetadata.stats.searchTimeInMillis) + + disableTransform(transform.id) + + hits = waitFor { + val response = client().makeRequest( + "GET", "${transform.targetIndex}/_search", + StringEntity("{\"size\": 25}", ContentType.APPLICATION_JSON) + ) + assertEquals("Request failed", RestStatus.OK, response.restStatus()) + val responseHits = response.asMap().getValue("hits") as Map<*, *> + val totalDocs = (responseHits["hits"] as ArrayList<*>).fold(0) { sum, bucket -> + val docCount = ((bucket as Map<*, *>)["_source"] as Map<*, *>)["transform._doc_count"] as Int + sum + docCount + } + assertEquals("Not all documents included in the transform target index", 100, totalDocs) + + responseHits["hits"] as ArrayList<*> + } + + hits.forEach { + val bucket = ((it as Map<*, *>)["_source"] as Map<*, *>) + val idGroup = (bucket["id_group"] as Double).toInt() + val expectedSum = ((idGroup * 2)..((idGroup * 2) + 8) step 2).sum() + assertEquals("ID sum not calculated correctly", expectedSum, (bucket["twice_id_sum"] as Double).toInt()) + } + } + + fun `test continuous transform with wildcard indices`() { + validateSourceIndex("wildcard-source-1") + validateSourceIndex("wildcard-source-2") + validateSourceIndex("wildcard-source-3") + + val transform = Transform( + id = "id_11", + schemaVersion = 1L, + enabled = true, + enabledAt = Instant.now(), + updatedAt = Instant.now(), + jobSchedule = IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES), + description = "test transform", + metadataId = null, + sourceIndex = "wildcard-s*e-*", + targetIndex = "wildcard-target-index", + roles = emptyList(), + pageSize = 100, + groups = listOf( + Terms(sourceField = "store_and_fwd_flag", targetField = "flag") + ), + continuous = true + ).let { createTransform(it, it.id) } + + updateTransformStartTime(transform) + + waitFor { assertTrue("Target transform index was not created", indexExists(transform.targetIndex)) } + + val firstIterationMetadata = waitFor { + val job = getTransform(transformId = transform.id) + assertNotNull("Transform job doesn't have metadata set", job.metadataId) + val transformMetadata = getTransformMetadata(job.metadataId!!) + assertEquals("Transform did not complete iteration or had incorrect number of documents processed", 15000, transformMetadata.stats.documentsProcessed) + assertEquals("Transform did not complete iteration", null, transformMetadata.afterKey) + assertNotNull("Continuous stats were not updated", transformMetadata.continuousStats) + transformMetadata + } + + assertEquals("Not the expected transform status", TransformMetadata.Status.STARTED, firstIterationMetadata.status) + assertEquals("Not the expected pages processed", 6L, firstIterationMetadata.stats.pagesProcessed) + assertEquals("Not the expected documents indexed", 2L, firstIterationMetadata.stats.documentsIndexed) + assertEquals("Not the expected documents processed", 15000L, firstIterationMetadata.stats.documentsProcessed) + assertTrue("Doesn't capture indexed time", firstIterationMetadata.stats.indexTimeInMillis > 0) + assertTrue("Didn't capture search time", firstIterationMetadata.stats.searchTimeInMillis > 0) + + waitFor { + val documentsBehind = getTransformDocumentsBehind(transform.id) + assertNotNull(documentsBehind) + assertEquals("Not the expected documents behind", 0, documentsBehind.values.sumOf { it as Int }) + } + + // Start the continuous transform again, and make sure it was a no-op + updateTransformStartTime(transform) + + Thread.sleep(5000) + + val secondIterationMetadata = waitFor { + val job = getTransform(transformId = transform.id) + assertNotNull("Transform job doesn't have metadata set", job.metadataId) + val transformMetadata = getTransformMetadata(job.metadataId!!) + assertEquals("Transform did not complete iteration or had incorrect number of documents processed", 15000, transformMetadata.stats.documentsProcessed) + assertEquals("Transform did not have null afterKey after iteration", null, transformMetadata.afterKey) + assertTrue("Timestamp was not updated", transformMetadata.continuousStats!!.lastTimestamp!!.isAfter(firstIterationMetadata.continuousStats!!.lastTimestamp)) + transformMetadata + } + + assertEquals("Not the expected transform status", TransformMetadata.Status.STARTED, secondIterationMetadata.status) + assertEquals("More than expected pages processed", 6, secondIterationMetadata.stats.pagesProcessed) + assertEquals("More than expected documents indexed", 2L, secondIterationMetadata.stats.documentsIndexed) + assertEquals("Not the expected documents processed", 15000L, secondIterationMetadata.stats.documentsProcessed) + assertEquals("Not the expected indexed time", secondIterationMetadata.stats.indexTimeInMillis, firstIterationMetadata.stats.indexTimeInMillis) + assertEquals("Not the expected search time", secondIterationMetadata.stats.searchTimeInMillis, firstIterationMetadata.stats.searchTimeInMillis) + + disableTransform(transform.id) + } + + fun `test continuous transforms with null buckets`() { + val sourceIndex = "null-bucket-source-index" + createIndex(sourceIndex, Settings.EMPTY, """"properties":{"iterating_id":{"type":"integer"},"term_id":{"type":"keyword"},"twice_id":{"type":"integer"}}""") + for (i in 0..12) { + val jsonString = "{\"iterating_id\": \"$i\",\"term_id\": \"${i % 5}\",\"twice_id\": \"${i * 2}\"}" + insertSampleData(sourceIndex, 1, jsonString = jsonString) + val idNullJsonString = "{\"iterating_id\": null,\"term_id\": \"${i % 5}\",\"twice_id\": \"${i * 2}\"}" + insertSampleData(sourceIndex, 1, jsonString = idNullJsonString) + val termNullJsonString = "{\"iterating_id\": \"$i\",\"term_id\": null,\"twice_id\": \"${i * 2}\"}" + insertSampleData(sourceIndex, 1, jsonString = termNullJsonString) + val bothNullJsonString = "{\"iterating_id\": null,\"term_id\": null,\"twice_id\": \"${i * 2}\"}" + insertSampleData(sourceIndex, 1, jsonString = bothNullJsonString) + } + + val aggregatorFactories = AggregatorFactories.builder() + aggregatorFactories.addAggregator(AggregationBuilders.sum("twice_id_sum").field("twice_id")) + + val transform = Transform( + id = "id_12", + schemaVersion = 1L, + enabled = true, + enabledAt = Instant.now(), + updatedAt = Instant.now(), + jobSchedule = IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES), + description = "test transform", + metadataId = null, + sourceIndex = sourceIndex, + targetIndex = "null-bucket-target-index", + roles = emptyList(), + pageSize = 100, + groups = listOf( + Histogram(sourceField = "iterating_id", targetField = "id_group", interval = 5.0), + Terms(sourceField = "term_id", targetField = "id_term") + ), + continuous = true, + aggregations = aggregatorFactories + ).let { createTransform(it, it.id) } + + updateTransformStartTime(transform) + + waitFor { assertTrue("Target transform index was not created", indexExists(transform.targetIndex)) } + + val firstIterationMetadata = waitFor { + val job = getTransform(transformId = transform.id) + assertNotNull("Transform job doesn't have metadata set", job.metadataId) + val transformMetadata = getTransformMetadata(job.metadataId!!) + assertEquals("Transform did not complete iteration or had incorrect number of documents processed", 52, transformMetadata.stats.documentsProcessed) + assertEquals("Transform did not complete iteration", null, transformMetadata.afterKey) + transformMetadata + } + + assertEquals("Not the expected transform status", TransformMetadata.Status.STARTED, firstIterationMetadata.status) + assertEquals("Not the expected pages processed", 2L, firstIterationMetadata.stats.pagesProcessed) + assertEquals("Not the expected documents indexed", 22L, firstIterationMetadata.stats.documentsIndexed) + assertEquals("Not the expected documents processed", 52L, firstIterationMetadata.stats.documentsProcessed) + assertTrue("Doesn't capture indexed time", firstIterationMetadata.stats.indexTimeInMillis > 0) + assertTrue("Didn't capture search time", firstIterationMetadata.stats.searchTimeInMillis > 0) + + // Get all of the buckets + var hits = waitFor { + val response = client().makeRequest( + "GET", "${transform.targetIndex}/_search", + StringEntity("{\"size\": 25}", ContentType.APPLICATION_JSON) + ) + assertEquals("Request failed", RestStatus.OK, response.restStatus()) + val responseHits = response.asMap().getValue("hits") as Map<*, *> + val totalDocs = (responseHits["hits"] as ArrayList<*>).fold(0) { sum, bucket -> + val docCount = ((bucket as Map<*, *>)["_source"] as Map<*, *>)["transform._doc_count"] as Int + sum + docCount + } + assertEquals("Not all documents included in the transform target index", 52, totalDocs) + + responseHits["hits"] as ArrayList<*> + } + + // Validate the buckets include the correct information + hits.forEach { + val bucket = ((it as Map<*, *>)["_source"] as Map<*, *>) + val idGroup = (bucket["id_group"] as Double?)?.toInt() + val idTerm = (bucket["id_term"] as String?)?.toInt() + if (idGroup == null) { + if (idTerm == null) { + val expectedSum = (0..(24) step 2).sum() + assertEquals("ID sum not calculated correctly", expectedSum, (bucket["twice_id_sum"] as Double).toInt()) + } else { + val expectedSum = ((idTerm * 2)..(24) step 10).sum() + assertEquals("ID sum not calculated correctly", expectedSum, (bucket["twice_id_sum"] as Double).toInt()) + } + } else if (idTerm == null) { + // use the min to get the correct sum for the half full top bucket + val expectedSum = ((idGroup * 2)..(min(idGroup * 2 + 8, 24)) step 2).sum() + assertEquals("ID sum not calculated correctly", expectedSum, (bucket["twice_id_sum"] as Double).toInt()) + } else { + val expectedSum = idGroup * 2 + idTerm * 2 + assertEquals("ID sum not calculated correctly", expectedSum, (bucket["twice_id_sum"] as Double).toInt()) + } + } + + // Add more data, don't add any to the (null, null) bucket to check that it won't be updated without new data + for (i in 13..24) { + val jsonString = "{\"iterating_id\": \"$i\",\"term_id\": \"${i % 5}\",\"twice_id\": \"${i * 2}\"}" + insertSampleData(sourceIndex, 1, jsonString = jsonString) + val idNullJsonString = "{\"iterating_id\": null,\"term_id\": \"${i % 5}\",\"twice_id\": \"${i * 2}\"}" + insertSampleData(sourceIndex, 1, jsonString = idNullJsonString) + val termNullJsonString = "{\"iterating_id\": \"$i\",\"term_id\": null,\"twice_id\": \"${i * 2}\"}" + insertSampleData(sourceIndex, 1, jsonString = termNullJsonString) + } + + waitFor { + val documentsBehind = getTransformDocumentsBehind(transform.id)[transform.sourceIndex] + assertEquals("Documents behind not calculated correctly", 36, documentsBehind) + } + + updateTransformStartTime(transform) + + val secondIterationMetadata = waitFor { + val job = getTransform(transformId = transform.id) + assertNotNull("Transform job doesn't have metadata set", job.metadataId) + val transformMetadata = getTransformMetadata(job.metadataId!!) + assertEquals("Transform did not complete iteration or had incorrect number of documents processed", 104L, transformMetadata.stats.documentsProcessed) + assertEquals("Transform did not have null afterKey after iteration", null, transformMetadata.afterKey) + transformMetadata + } + + assertEquals("Not the expected transform status", TransformMetadata.Status.STARTED, secondIterationMetadata.status) + assertEquals("More than expected pages processed", 4L, secondIterationMetadata.stats.pagesProcessed) + assertEquals("More than expected documents indexed", 42L, secondIterationMetadata.stats.documentsIndexed) + assertEquals("Not the expected documents processed", 104L, secondIterationMetadata.stats.documentsProcessed) + assertTrue("Doesn't capture indexed time", secondIterationMetadata.stats.indexTimeInMillis > firstIterationMetadata.stats.indexTimeInMillis) + assertTrue("Didn't capture search time", secondIterationMetadata.stats.searchTimeInMillis > firstIterationMetadata.stats.searchTimeInMillis) + + disableTransform(transform.id) + + hits = waitFor { + val response = client().makeRequest( + "GET", "${transform.targetIndex}/_search", + StringEntity("{\"size\": 40}", ContentType.APPLICATION_JSON) + ) + assertEquals("Request failed", RestStatus.OK, response.restStatus()) + val responseHits = response.asMap().getValue("hits") as Map<*, *> + val totalDocs = (responseHits["hits"] as ArrayList<*>).fold(0) { sum, bucket -> + val docCount = ((bucket as Map<*, *>)["_source"] as Map<*, *>)["transform._doc_count"] as Int + sum + docCount + } + assertEquals("Not all documents included in the transform target index", 88, totalDocs) + + responseHits["hits"] as ArrayList<*> + } + + // Validate the buckets include the correct information + hits.forEach { + val bucket = ((it as Map<*, *>)["_source"] as Map<*, *>) + val idGroup = (bucket["id_group"] as Double?)?.toInt() + val idTerm = (bucket["id_term"] as String?)?.toInt() + if (idGroup == null) { + if (idTerm == null) { + val expectedSum = (0..(24) step 2).sum() + assertEquals("ID sum not calculated correctly", expectedSum, (bucket["twice_id_sum"] as Double).toInt()) + } else { + val expectedSum = ((idTerm * 2)..(48) step 10).sum() + assertEquals("ID sum not calculated correctly", expectedSum, (bucket["twice_id_sum"] as Double).toInt()) + } + } else if (idTerm == null) { + // use the min to get the correct sum for the half full top bucket + val expectedSum = ((idGroup * 2)..(idGroup * 2 + 8) step 2).sum() + assertEquals("ID sum not calculated correctly", expectedSum, (bucket["twice_id_sum"] as Double).toInt()) + } else { + val expectedSum = idGroup * 2 + idTerm * 2 + assertEquals("ID sum not calculated correctly", expectedSum, (bucket["twice_id_sum"] as Double).toInt()) + } + } + } + private fun getStrictMappings(): String { return """ "dynamic": "strict", @@ -398,4 +958,11 @@ class TransformRunnerIT : TransformRestTestCase() { } """.trimIndent() } + + private fun validateSourceIndex(indexName: String) { + if (!indexExists(indexName)) { + generateNYCTaxiData(indexName) + assertIndexExists(indexName) + } + } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/action/ResponseTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/action/ResponseTests.kt index 64dadef1b..a31358333 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/action/ResponseTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/action/ResponseTests.kt @@ -23,7 +23,8 @@ class ResponseTests : OpenSearchTestCase() { fun `test explain transform response`() { val idsToExplain = randomList(10) { randomAlphaOfLength(10) to randomExplainTransform() }.toMap() - val res = ExplainTransformResponse(idsToExplain) + val failedToExplain = randomList(10) { randomAlphaOfLength(10) to randomAlphaOfLength(10) }.toMap() + val res = ExplainTransformResponse(idsToExplain, failedToExplain) val out = BytesStreamOutput().apply { res.writeTo(this) } val streamedRes = ExplainTransformResponse(buildStreamInputForTransforms(out)) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/model/TransformTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/model/TransformTests.kt index 7de07ae79..54292224a 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/model/TransformTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/model/TransformTests.kt @@ -26,17 +26,22 @@ class TransformTests : OpenSearchTestCase() { } } - fun `test transform requires page size to be between 1 and 10K`() { + fun `test transform page size constraints`() { assertFailsWith(IllegalArgumentException::class, "Page size was less than 1") { randomTransform().copy(pageSize = -1) } assertFailsWith(IllegalArgumentException::class, "Page size was greater than 10K") { - randomTransform().copy(pageSize = 10001) + randomTransform().copy(continuous = false, pageSize = 10001) + } + + assertFailsWith(IllegalArgumentException::class, "Page size was greater than 1K") { + randomTransform().copy(continuous = true, pageSize = 1001) } randomTransform().copy(pageSize = 1) - randomTransform().copy(pageSize = 10000) + randomTransform().copy(continuous = false, pageSize = 10000) + randomTransform().copy(continuous = true, pageSize = 1000) randomTransform().copy(pageSize = 500) } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/model/XContentTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/model/XContentTests.kt index 94a590293..f97624486 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/model/XContentTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/model/XContentTests.kt @@ -74,6 +74,6 @@ class XContentTests : OpenSearchTestCase() { } override fun xContentRegistry(): NamedXContentRegistry { - return NamedXContentRegistry(SearchModule(Settings.EMPTY, false, emptyList()).namedXContents) + return NamedXContentRegistry(SearchModule(Settings.EMPTY, emptyList()).namedXContents) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestExplainTransformActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestExplainTransformActionIT.kt index 21ddd7ba2..e8e4019ab 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestExplainTransformActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestExplainTransformActionIT.kt @@ -5,7 +5,9 @@ package org.opensearch.indexmanagement.transform.resthandler +import org.junit.Assert import org.opensearch.client.ResponseException +import org.opensearch.indexmanagement.IndexManagementPlugin import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.TRANSFORM_BASE_URI import org.opensearch.indexmanagement.makeRequest import org.opensearch.indexmanagement.transform.TransformRestTestCase @@ -48,8 +50,19 @@ class RestExplainTransformActionIT : TransformRestTestCase() { assertEquals("Did not have metadata_id in explain response", updatedTransform.metadataId, explainMetadata["metadata_id"]) val metadata = explainMetadata["transform_metadata"] as Map assertNotNull("Did not have metadata in explain response", metadata) - // Not sure if this is true for transforms - assertEquals("Status should be finished", TransformMetadata.Status.FINISHED.type, metadata["status"]) + if (!transform.continuous) { + assertEquals("Status should be finished", TransformMetadata.Status.FINISHED.type, metadata["status"]) + } else { + assertEquals("Status should be started", TransformMetadata.Status.STARTED.type, metadata["status"]) + assertNull("Should not have key for shard id to global checkpoint number map", metadata["shard_id_to_global_checkpoint"]) + assertNotNull("Did not have key for continuous transform stats", metadata["continuous_stats"]) + val continuousStats = metadata["continuous_stats"] as Map + assertNotNull("Did not have last_timestamp in continuous transform stats", continuousStats["last_timestamp"]) + assertNotNull("Did not have documents_behind in continuous transform stats", continuousStats["documents_behind"]) + val documentsBehind = continuousStats["documents_behind"] as Map + Assert.assertTrue("Continuous transform stats documents_behind did not contain the source index", documentsBehind.containsKey(transform.sourceIndex)) + Assert.assertEquals("Continuous transform stats should have 0 documents_behind", 0, documentsBehind[transform.sourceIndex]) + } } } @@ -85,6 +98,48 @@ class RestExplainTransformActionIT : TransformRestTestCase() { assertNotNull("Non null wildcard_some_other_transform_id value wasn't in the response", map["wildcard_some_other_transform_id"]) } + @Throws(Exception::class) + fun `test explain continuous transform with wildcard id`() { + val transform1 = randomTransform().copy( + id = "continuous_wildcard_1", + jobSchedule = IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES), + enabled = true, + enabledAt = Instant.now(), + metadataId = null, + continuous = true, + pageSize = 50 + ).let { createTransform(it, it.id) } + val transform2 = randomTransform().copy( + id = "continuous_wildcard_2", + jobSchedule = IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES), + enabled = true, + enabledAt = Instant.now(), + metadataId = null, + continuous = true, + pageSize = 50 + ).let { createTransform(it, it.id) } + + updateTransformStartTime(transform1) + updateTransformStartTime(transform2) + + waitFor { + val response = client().makeRequest("GET", "$TRANSFORM_BASE_URI/continuous_wildcard*/_explain") + assertEquals(RestStatus.OK, response.restStatus()) + val responseMap = response.asMap() + listOf(transform1, transform2).forEach { transform -> + val transformMetadata = (responseMap[transform.id] as Map)["transform_metadata"] as Map + assertNotNull("Did not have metadata for transform ID", transformMetadata) + val continuousStats = transformMetadata["continuous_stats"] as Map + assertNotNull("Did not have last_timestamp in continuous transform stats", continuousStats["last_timestamp"]) + assertNotNull("Did not have documents_behind in continuous transform stats", continuousStats["documents_behind"]) + val documentsBehind = continuousStats["documents_behind"] as Map + assertTrue("Source index was not contained in the documents behind explain response", documentsBehind.containsKey(transform.sourceIndex)) + assertEquals("Continuous transform stats should have 0 documents_behind", 0, documentsBehind[transform.sourceIndex]) + assertEquals("Documents behind should only display the transform source indices", 1, documentsBehind.size) + } + } + } + @Throws(Exception::class) fun `test explain transform for job that hasnt started`() { createTransform(transform = randomTransform().copy(metadataId = null), transformId = "not_started_some_transform_id") @@ -103,16 +158,21 @@ class RestExplainTransformActionIT : TransformRestTestCase() { @Throws(Exception::class) fun `test explain transform when config doesnt exist`() { + deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX) val responseExplicit = client().makeRequest("GET", "$TRANSFORM_BASE_URI/no_config_some_transform/_explain") - assertEquals("Non-existent transform didn't return null", mapOf("no_config_some_transform" to null), responseExplicit.asMap()) + val expectedResponse = mapOf("no_config_some_transform" to "Failed to search transform metadata") + assertEquals("Non-existent transform didn't return null", expectedResponse, responseExplicit.asMap()) val responseExplicitMultiple = client().makeRequest("GET", "$TRANSFORM_BASE_URI/no_config_some_transform,no_config_another_transform/_explain") - assertEquals("Non-existent transform didn't return null", mapOf("no_config_some_transform" to null, "no_config_another_transform" to null), responseExplicitMultiple.asMap()) + val expectedResponseMultiple = mapOf("no_config_some_transform" to "Failed to search transform metadata", "no_config_another_transform" to "Failed to search transform metadata") + assertEquals("Non-existent transform didn't return null", expectedResponseMultiple, responseExplicitMultiple.asMap()) val responseWildcard = client().makeRequest("GET", "$TRANSFORM_BASE_URI/no_config_another_*/_explain") - assertEquals("Wildcard transform didn't return nothing", mapOf?>(), responseWildcard.asMap()) + val expectedResponseWildcard = mapOf?>() + assertEquals("Wildcard transform didn't return nothing", expectedResponseWildcard, responseWildcard.asMap()) val responseMultipleTypes = client().makeRequest("GET", "$TRANSFORM_BASE_URI/no_config_some_transform,no_config_another_*/_explain") - assertEquals("Non-existent and wildcard transform didn't return only non-existent as null", mapOf("no_config_some_transform" to null), responseMultipleTypes.asMap()) + val expectedResponseMixed = mapOf("no_config_some_transform" to "Failed to search transform metadata") + assertEquals("Non-existent and wildcard transform didn't return only non-existent with error", expectedResponseMixed, responseMultipleTypes.asMap()) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestGetTransformActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestGetTransformActionIT.kt index 96e8f3ec2..0a679a343 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestGetTransformActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestGetTransformActionIT.kt @@ -15,7 +15,6 @@ import org.opensearch.indexmanagement.transform.randomTransform import org.opensearch.rest.RestStatus import org.opensearch.test.OpenSearchTestCase import org.opensearch.test.junit.annotations.TestLogging -import org.opensearch.test.rest.OpenSearchRestTestCase.randomList @TestLogging(value = "level:DEBUG", reason = "Debugging tests") @Suppress("UNCHECKED_CAST") diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestIndexTransformActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestIndexTransformActionIT.kt index bf70147d8..0857c1900 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestIndexTransformActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestIndexTransformActionIT.kt @@ -87,4 +87,32 @@ class RestIndexTransformActionIT : TransformRestTestCase() { assertEquals("Mappings are different", expectedMap, mappingsMap) } + + @Throws(Exception::class) + fun `test updating transform continuous field fails`() { + val transform = createRandomTransform() + try { + client().makeRequest( + "PUT", + "$TRANSFORM_BASE_URI/${transform.id}?refresh=true&if_seq_no=${transform.seqNo}&if_primary_term=${transform.primaryTerm}", + emptyMap(), + transform.copy(continuous = !transform.continuous, pageSize = 50).toHttpEntity() // Lower page size to make sure that doesn't throw an error first + ) + fail("Expected 405 Method Not Allowed response") + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + val actualMessage = e.response.asMap() + val expectedErrorMessage = mapOf( + "error" to mapOf( + "root_cause" to listOf>( + mapOf("type" to "status_exception", "reason" to "Not allowed to modify [continuous]") + ), + "type" to "status_exception", + "reason" to "Not allowed to modify [continuous]" + ), + "status" to 400 + ) + assertEquals(expectedErrorMessage, actualMessage) + } + } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestStartTransformActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestStartTransformActionIT.kt index 978861c91..d4cef5b90 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestStartTransformActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestStartTransformActionIT.kt @@ -133,10 +133,15 @@ class RestStartTransformActionIT : TransformRestTestCase() { // Transform should be able to finish, with actual transformed docs waitFor { val metadata = getTransformMetadata(updatedTransform.metadataId!!) - assertEquals("Status should be finished", TransformMetadata.Status.FINISHED, metadata.status) + if (transform.continuous) { + assertEquals("Status should be started", TransformMetadata.Status.STARTED, metadata.status) + } else { + assertEquals("Status should be finished", TransformMetadata.Status.FINISHED, metadata.status) + } assertEquals("Did not transform documents", 5000, metadata.stats.documentsProcessed) assertTrue("Did not transform documents", metadata.stats.documentsIndexed > 0) } + if (transform.continuous) disableTransform(transform.id) } @Throws(Exception::class) @@ -159,7 +164,8 @@ class RestStartTransformActionIT : TransformRestTestCase() { groups = listOf( Terms(sourceField = "store_and_fwd_flag", targetField = "flag") ), - aggregations = AggregatorFactories.builder() + aggregations = AggregatorFactories.builder(), + continuous = false ).let { createTransform(it, it.id) } updateTransformStartTime(transform) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestStopTransformActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestStopTransformActionIT.kt index c653f5fe9..7e8efea9b 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestStopTransformActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestStopTransformActionIT.kt @@ -57,7 +57,8 @@ class RestStopTransformActionIT : TransformRestTestCase() { jobSchedule = IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES), enabled = true, enabledAt = Instant.now(), - metadataId = null + metadataId = null, + continuous = false ) ) updateTransformStartTime(transform) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/util/IndexUtilsTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/util/IndexUtilsTests.kt index e3b3e8b6c..2e51d1190 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/util/IndexUtilsTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/util/IndexUtilsTests.kt @@ -79,8 +79,9 @@ class IndexUtilsTests : OpenSearchTestCase() { fun `test should update index with same version`() { val indexContent = "{\"testIndex\":{\"settings\":{\"index\":{\"creation_date\":\"1558407515699\"," + "\"number_of_shards\":\"1\",\"number_of_replicas\":\"1\",\"uuid\":\"t-VBBW6aR6KpJ3XP5iISOA\"," + - "\"version\":{\"created\":\"6040399\"},\"provided_name\":\"data_test\"}},\"mappings\":" + - "{\"_doc\":{\"_meta\":{\"schema_version\":1},\"properties\":{\"name\":{\"type\":\"keyword\"}}}}}}" + "\"version\":{\"created\":\"6040399\"},\"provided_name\":\"data_test\"}},\"mapping_version\":123," + + "\"settings_version\":123,\"mappings\":{\"_doc\":{\"_meta\":{\"schema_version\":1},\"properties\":" + + "{\"name\":{\"type\":\"keyword\"}}}}}}" val parser = createParser(XContentType.JSON.xContent(), indexContent) val index: IndexMetadata = IndexMetadata.fromXContent(parser) diff --git a/src/test/resources/bwc/indexmanagement/1.13.2.0/opendistro-indexmanagement-1.13.2.0.zip b/src/test/resources/bwc/indexmanagement/1.13.2.0/opendistro-indexmanagement-1.13.2.0.zip deleted file mode 100644 index 71332e6b7..000000000 Binary files a/src/test/resources/bwc/indexmanagement/1.13.2.0/opendistro-indexmanagement-1.13.2.0.zip and /dev/null differ diff --git a/src/test/resources/bwc/job-scheduler/1.13.0.0/opendistro-job-scheduler-1.13.0.0.zip b/src/test/resources/bwc/job-scheduler/1.13.0.0/opendistro-job-scheduler-1.13.0.0.zip deleted file mode 100644 index daf3b8f73..000000000 Binary files a/src/test/resources/bwc/job-scheduler/1.13.0.0/opendistro-job-scheduler-1.13.0.0.zip and /dev/null differ diff --git a/src/test/resources/job-scheduler/opensearch-job-scheduler-1.3.0.0-SNAPSHOT.zip b/src/test/resources/job-scheduler/opensearch-job-scheduler-1.3.0.0-SNAPSHOT.zip deleted file mode 100644 index d9d9ede0a..000000000 Binary files a/src/test/resources/job-scheduler/opensearch-job-scheduler-1.3.0.0-SNAPSHOT.zip and /dev/null differ diff --git a/src/test/resources/lang-mustache/lang-mustache-1.0.0.zip b/src/test/resources/lang-mustache/lang-mustache-1.0.0.zip deleted file mode 100644 index 83be078eb..000000000 Binary files a/src/test/resources/lang-mustache/lang-mustache-1.0.0.zip and /dev/null differ diff --git a/src/test/resources/mappings/cached-opendistro-ism-config.json b/src/test/resources/mappings/cached-opendistro-ism-config.json index 2144b3ce3..074257bf8 100644 --- a/src/test/resources/mappings/cached-opendistro-ism-config.json +++ b/src/test/resources/mappings/cached-opendistro-ism-config.json @@ -429,6 +429,10 @@ } } } + }, + "custom": { + "enabled": false, + "type": "object" } } }, @@ -658,6 +662,10 @@ "rolled_over": { "type": "boolean" }, + "index_creation_date": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, "transition_to": { "type": "text", "fields": { @@ -1211,6 +1219,9 @@ } } } + }, + "continuous": { + "type": "boolean" } } }, @@ -1242,6 +1253,14 @@ "stats": { "type": "object", "enabled": false + }, + "continuous_stats": { + "type": "object", + "enabled": false + }, + "shard_id_to_global_checkpoint": { + "type": "object", + "enabled": false } } } diff --git a/src/test/resources/mappings/cached-opendistro-ism-history.json b/src/test/resources/mappings/cached-opendistro-ism-history.json index 04d3c46d8..44c7ab896 100644 --- a/src/test/resources/mappings/cached-opendistro-ism-history.json +++ b/src/test/resources/mappings/cached-opendistro-ism-history.json @@ -1,6 +1,6 @@ { "_meta" : { - "schema_version": 3 + "schema_version": 4 }, "dynamic": "strict", "properties": { @@ -37,6 +37,10 @@ "rolled_over": { "type": "boolean" }, + "index_creation_date": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, "transition_to": { "type": "keyword" }, diff --git a/worksheets/ism/create.http b/worksheets/ism/create.http index 712cad027..4203b730d 100644 --- a/worksheets/ism/create.http +++ b/worksheets/ism/create.http @@ -1,3 +1,24 @@ +### +PUT http://localhost:9200/testindex +Content-Type: application/json + +### +POST http://localhost:9200/_opendistro/_ism/add/testindex +Content-Type: application/json + +{ + "policy_id": "example" +} + +### +GET localhost:9200/_opendistro/_ism/explain/testindex?show_applied_policy +Accept: application/json + +### +GET localhost:9200/.opendistro-ism-config/_search +Accept: application/json + +### PUT localhost:9200/_opendistro/_ism/policies/example Content-Type: application/json @@ -46,3 +67,6 @@ Content-Type: application/json ] } } + +### +DELETE localhost:9200/_opendistro/_ism/policies/example \ No newline at end of file