From 2874937d73b18c578786758e2daaf91dbc50bfd4 Mon Sep 17 00:00:00 2001 From: Drew Baugher <46505179+dbbaughe@users.noreply.github.com> Date: Fri, 15 Apr 2022 16:37:13 -0700 Subject: [PATCH] Updates search text field to keyword subfield for policies and managed indices (#267) * Use the keyword subfield instead of the tokenized text field for querying managed indices and policies Signed-off-by: Drew Baugher * Changing the job scheduler distribution path Signed-off-by: Ravi --- build.gradle | 2 +- .../action/explain/TransportExplainAction.kt | 4 +- .../getpolicy/TransportGetPoliciesAction.kt | 2 +- .../util/RestHandlerUtils.kt | 2 +- .../IndexStateManagementRestTestCase.kt | 28 ++++- .../IndexStateManagementRestApiIT.kt | 38 ++++++ .../resthandler/RestExplainActionIT.kt | 111 +++++++++++++++++- 7 files changed, 178 insertions(+), 9 deletions(-) diff --git a/build.gradle b/build.gradle index 0a01e7643..89878f8ea 100644 --- a/build.gradle +++ b/build.gradle @@ -33,7 +33,7 @@ buildscript { common_utils_version = System.getProperty("common_utils.version", opensearch_build) job_scheduler_version = System.getProperty("job_scheduler_version.version", opensearch_build) job_scheduler_build_download = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot + - '/latest/linux/x64/builds/opensearch/plugins/opensearch-job-scheduler-' + job_scheduler_no_snapshot + '.zip' + '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-job-scheduler-' + job_scheduler_no_snapshot + '.zip' kotlin_version = System.getProperty("kotlin.version", "1.6.10") } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt index 3db799895..8219c3799 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt @@ -47,7 +47,7 @@ import org.opensearch.indexmanagement.indexstatemanagement.transport.action.mana import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexRequest import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_INDEX_UUID_FIELD -import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_NAME_FIELD +import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_NAME_KEYWORD_FIELD import org.opensearch.indexmanagement.indexstatemanagement.util.MetadataCheck import org.opensearch.indexmanagement.indexstatemanagement.util.checkMetadata import org.opensearch.indexmanagement.indexstatemanagement.util.managedIndexMetadataID @@ -158,7 +158,7 @@ class TransportExplainAction @Inject constructor( .must( QueryBuilders .queryStringQuery(params.queryString) - .defaultField(MANAGED_INDEX_NAME_FIELD) + .defaultField(MANAGED_INDEX_NAME_KEYWORD_FIELD) .defaultOperator(Operator.AND) ).filter(QueryBuilders.termsQuery(MANAGED_INDEX_INDEX_UUID_FIELD, indexUUIDs)) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/TransportGetPoliciesAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/TransportGetPoliciesAction.kt index 10c0f7c10..1fa0f1130 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/TransportGetPoliciesAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/getpolicy/TransportGetPoliciesAction.kt @@ -81,7 +81,7 @@ class TransportGetPoliciesAction @Inject constructor( QueryBuilders .queryStringQuery(params.queryString) .defaultOperator(Operator.AND) - .field("policy.policy_id") + .field("policy.policy_id.keyword") ) val searchSourceBuilder = SearchSourceBuilder() diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt index c16f0cebe..31b5b4b48 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt @@ -39,7 +39,7 @@ const val TOTAL_MANAGED_INDICES = "total_managed_indices" const val ISM_TEMPLATE_FIELD = "policy.ism_template" const val MANAGED_INDEX_FIELD = "managed_index" -const val MANAGED_INDEX_NAME_FIELD = "$MANAGED_INDEX_FIELD.name" +const val MANAGED_INDEX_NAME_KEYWORD_FIELD = "$MANAGED_INDEX_FIELD.name.keyword" const val MANAGED_INDEX_INDEX_FIELD = "$MANAGED_INDEX_FIELD.index" const val MANAGED_INDEX_INDEX_UUID_FIELD = "$MANAGED_INDEX_FIELD.index_uuid" diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index 5071361a5..be64799ff 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -47,7 +47,6 @@ import org.opensearch.indexmanagement.indexstatemanagement.util.FAILED_INDICES import org.opensearch.indexmanagement.indexstatemanagement.util.FAILURES import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_NUMBER_OF_REPLICAS import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_NUMBER_OF_SHARDS -import org.opensearch.indexmanagement.indexstatemanagement.util.SHOW_POLICY_QUERY_PARAM import org.opensearch.indexmanagement.indexstatemanagement.util.UPDATED_INDICES import org.opensearch.indexmanagement.makeRequest import org.opensearch.indexmanagement.opensearchapi.parseWithType @@ -191,6 +190,29 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() return index to policyID } + protected fun createDataStream( + dataStream: String, + template: StringEntity? = null + ) { + val dataStreamTemplate = template ?: StringEntity( + """ + { + "data_stream": {}, + "index_patterns": ["$dataStream"] + } + """.trimIndent(), + APPLICATION_JSON + ) + val res = client().makeRequest( + "PUT", + "/_index_template/transform-data-stream-template", + dataStreamTemplate + ) + assertEquals("Unexpected RestStatus", RestStatus.OK, res.restStatus()) + val response = client().makeRequest("PUT", "/_data_stream/$dataStream") + assertEquals("Unexpected RestStatus", RestStatus.OK, response.restStatus()) + } + protected fun changeAlias( index: String, alias: String, @@ -533,10 +555,10 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() protected fun getFlatSettings(indexName: String) = (getIndexSettings(indexName) as Map>>)[indexName]!!["settings"] as Map - protected fun getExplainMap(indexName: String?, showPolicy: Boolean = false): Map { + protected fun getExplainMap(indexName: String?, queryParams: String = ""): Map { var endpoint = RestExplainAction.EXPLAIN_BASE_URI if (indexName != null) endpoint += "/$indexName" - if (showPolicy) endpoint += "?$SHOW_POLICY_QUERY_PARAM" + if (queryParams.isNotEmpty()) endpoint += "?$queryParams" val response = client().makeRequest(RestRequest.Method.GET.toString(), endpoint) assertEquals("Unexpected RestStatus", RestStatus.OK, response.restStatus()) return response.asMap() diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt index 67fcd2192..cb50d18ed 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt @@ -317,4 +317,42 @@ class IndexStateManagementRestApiIT : IndexStateManagementRestTestCase() { assertEquals(expectedMessage.toString(), actualMessage.toString()) } + + fun `test get policies with hyphen`() { + val randomPolicy = randomPolicy(id = "testing-hyphens-01") + createPolicy(randomPolicy, policyId = randomPolicy.id, refresh = true) + val policy = getPolicy(randomPolicy.id) + + val response = client().makeRequest(RestRequest.Method.GET.toString(), "$POLICY_BASE_URI?queryString=*testing-hyphens*") + + val actualMessage = response.asMap() + val expectedMessage = mapOf( + "total_policies" to 1, + "policies" to listOf( + mapOf( + _SEQ_NO to policy.seqNo, + _ID to policy.id, + _PRIMARY_TERM to policy.primaryTerm, + Policy.POLICY_TYPE to mapOf( + "schema_version" to policy.schemaVersion, + "policy_id" to policy.id, + "last_updated_time" to policy.lastUpdatedTime.toEpochMilli(), + "default_state" to policy.defaultState, + "ism_template" to null, + "description" to policy.description, + "error_notification" to policy.errorNotification, + "states" to policy.states.map { + mapOf( + "name" to it.name, + "transitions" to it.transitions, + "actions" to it.actions + ) + } + ) + ) + ) + ) + + assertEquals(expectedMessage.toString(), actualMessage.toString()) + } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt index c0f4cd69e..c7a980932 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt @@ -9,6 +9,7 @@ import org.opensearch.common.xcontent.XContentFactory import org.opensearch.indexmanagement.IndexManagementPlugin import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy +import org.opensearch.indexmanagement.indexstatemanagement.util.SHOW_POLICY_QUERY_PARAM import org.opensearch.indexmanagement.indexstatemanagement.util.TOTAL_MANAGED_INDICES import org.opensearch.indexmanagement.indexstatemanagement.util.XCONTENT_WITHOUT_TYPE_AND_USER import org.opensearch.indexmanagement.makeRequest @@ -139,6 +140,114 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { } } + fun `test search query string`() { + val indexName1 = "$testIndexName-search-query-string" + val indexName2 = "$indexName1-testing-2" + val indexName3 = "$indexName1-testing-3" + val indexName4 = "$indexName1-testing-4-2022-02-15" + val indexName5 = "$indexName1-testing-5-15-02-2022" + val dataStreamName = "$indexName1-data-stream" + createDataStream(dataStreamName) + + val policy = createRandomPolicy() + createIndex(indexName1, policyID = policy.id) + createIndex(indexName2, policyID = policy.id) + createIndex(indexName3, null) + createIndex(indexName4, policyID = policy.id) + createIndex(indexName5, policyID = policy.id) + addPolicyToIndex(dataStreamName, policy.id) + val indexName1Map = indexName1 to mapOf( + explainResponseOpendistroPolicyIdSetting to policy.id, + explainResponseOpenSearchPolicyIdSetting to policy.id, + "index" to indexName1, + "index_uuid" to getUuid(indexName1), + "policy_id" to policy.id, + "enabled" to true + ) + val indexName2Map = indexName2 to mapOf( + explainResponseOpendistroPolicyIdSetting to policy.id, + explainResponseOpenSearchPolicyIdSetting to policy.id, + "index" to indexName2, + "index_uuid" to getUuid(indexName2), + "policy_id" to policy.id, + "enabled" to true + ) + val indexName4Map = indexName4 to mapOf( + explainResponseOpendistroPolicyIdSetting to policy.id, + explainResponseOpenSearchPolicyIdSetting to policy.id, + "index" to indexName4, + "index_uuid" to getUuid(indexName4), + "policy_id" to policy.id, + "enabled" to true + ) + val indexName5Map = indexName5 to mapOf( + explainResponseOpendistroPolicyIdSetting to policy.id, + explainResponseOpenSearchPolicyIdSetting to policy.id, + "index" to indexName5, + "index_uuid" to getUuid(indexName5), + "policy_id" to policy.id, + "enabled" to true + ) + val datastreamMap = ".ds-$dataStreamName-000001" to mapOf( + explainResponseOpendistroPolicyIdSetting to policy.id, + explainResponseOpenSearchPolicyIdSetting to policy.id, + "index" to ".ds-$dataStreamName-000001", + "index_uuid" to getUuid(".ds-$dataStreamName-000001"), + "policy_id" to policy.id, + "enabled" to true + ) + + waitFor { + val expected = mapOf( + indexName1Map, + indexName2Map, + indexName4Map, + indexName5Map, + "total_managed_indices" to 4 + ) + // These should match all non datastream managed indices + assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=$testIndexName*")) + assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=$testIndexName-*")) + assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=$testIndexName-search-*")) + } + + waitFor { + val expected = mapOf( + indexName1Map, + indexName2Map, + indexName4Map, + indexName5Map, + datastreamMap, + "total_managed_indices" to 5 + ) + // These should match all managed indices including datastreams + assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=*$testIndexName-*")) + assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=*search*")) + assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=*search-query*")) + } + + waitFor { + val expected = mapOf( + datastreamMap, + "total_managed_indices" to 1 + ) + // These should match all datastream managed indices (and system/hidden indices) + assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=.*")) + assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=.ds-$testIndexName-*")) + } + + waitFor { + val expected = mapOf( + indexName4Map, + "total_managed_indices" to 1 + ) + // These should match all just the single index, and validates that it does not match the 15-02-2022 index + // i.e. if it was still matching on tokens then ["2022", "02", "15"] would match both which we don't want + assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=*2022-02-15")) + assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=*2022-02-15*")) + } + } + fun `test attached policy`() { val indexName = "${testIndexName}_watermelon" val policy = createRandomPolicy() @@ -239,7 +348,7 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { TOTAL_MANAGED_INDICES to 1, ) waitFor { - assertResponseMap(expected, getExplainMap(indexName, true)) + assertResponseMap(expected, getExplainMap(indexName, queryParams = SHOW_POLICY_QUERY_PARAM)) } }