Skip to content

Commit

Permalink
Updates search text field to keyword subfield for policies and manage…
Browse files Browse the repository at this point in the history 
…d indices (#267)

* Use the keyword subfield instead of the tokenized text field for querying managed indices and policies

Signed-off-by: Drew Baugher <[email protected]>

* Changing the job scheduler distribution path

Signed-off-by: Ravi <[email protected]>
  • Loading branch information
@dbbaughe
dbbaughe authored Apr 15, 2022
1 parent 87f13da commit 2874937
Show file tree
Hide file tree
Showing 7 changed files with 178 additions and 9 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ buildscript {
common_utils_version = System.getProperty("common_utils.version", opensearch_build)
job_scheduler_version = System.getProperty("job_scheduler_version.version", opensearch_build)
job_scheduler_build_download = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot +
'/latest/linux/x64/builds/opensearch/plugins/opensearch-job-scheduler-' + job_scheduler_no_snapshot + '.zip'
'/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-job-scheduler-' + job_scheduler_no_snapshot + '.zip'
kotlin_version = System.getProperty("kotlin.version", "1.6.10")
}

Expand Down
4 changes: 2 additions & 2 deletions ...h/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ import org.opensearch.indexmanagement.indexstatemanagement.transport.action.mana
import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexRequest
import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE
import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_INDEX_UUID_FIELD
import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_NAME_FIELD
import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_NAME_KEYWORD_FIELD
import org.opensearch.indexmanagement.indexstatemanagement.util.MetadataCheck
import org.opensearch.indexmanagement.indexstatemanagement.util.checkMetadata
import org.opensearch.indexmanagement.indexstatemanagement.util.managedIndexMetadataID
Expand Down Expand Up @@ -158,7 +158,7 @@ class TransportExplainAction @Inject constructor(
.must(
QueryBuilders
.queryStringQuery(params.queryString)
.defaultField(MANAGED_INDEX_NAME_FIELD)
.defaultField(MANAGED_INDEX_NAME_KEYWORD_FIELD)
.defaultOperator(Operator.AND)
).filter(QueryBuilders.termsQuery(MANAGED_INDEX_INDEX_UUID_FIELD, indexUUIDs))

Expand Down
2 changes: 1 addition & 1 deletion ...xmanagement/indexstatemanagement/transport/action/getpolicy/TransportGetPoliciesAction.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ class TransportGetPoliciesAction @Inject constructor(
QueryBuilders
.queryStringQuery(params.queryString)
.defaultOperator(Operator.AND)
.field("policy.policy_id")
.field("policy.policy_id.keyword")
)

val searchSourceBuilder = SearchSourceBuilder()
Expand Down
2 changes: 1 addition & 1 deletion src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ const val TOTAL_MANAGED_INDICES = "total_managed_indices"

const val ISM_TEMPLATE_FIELD = "policy.ism_template"
const val MANAGED_INDEX_FIELD = "managed_index"
const val MANAGED_INDEX_NAME_FIELD = "$MANAGED_INDEX_FIELD.name"
const val MANAGED_INDEX_NAME_KEYWORD_FIELD = "$MANAGED_INDEX_FIELD.name.keyword"
const val MANAGED_INDEX_INDEX_FIELD = "$MANAGED_INDEX_FIELD.index"
const val MANAGED_INDEX_INDEX_UUID_FIELD = "$MANAGED_INDEX_FIELD.index_uuid"

Expand Down
28 changes: 25 additions & 3 deletions ...n/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ import org.opensearch.indexmanagement.indexstatemanagement.util.FAILED_INDICES
import org.opensearch.indexmanagement.indexstatemanagement.util.FAILURES
import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_NUMBER_OF_REPLICAS
import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_NUMBER_OF_SHARDS
import org.opensearch.indexmanagement.indexstatemanagement.util.SHOW_POLICY_QUERY_PARAM
import org.opensearch.indexmanagement.indexstatemanagement.util.UPDATED_INDICES
import org.opensearch.indexmanagement.makeRequest
import org.opensearch.indexmanagement.opensearchapi.parseWithType
Expand Down Expand Up @@ -191,6 +190,29 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase()
return index to policyID
}

protected fun createDataStream(
dataStream: String,
template: StringEntity? = null
) {
val dataStreamTemplate = template ?: StringEntity(
"""
{
"data_stream": {},
"index_patterns": ["$dataStream"]
}
""".trimIndent(),
APPLICATION_JSON
)
val res = client().makeRequest(
"PUT",
"/_index_template/transform-data-stream-template",
dataStreamTemplate
)
assertEquals("Unexpected RestStatus", RestStatus.OK, res.restStatus())
val response = client().makeRequest("PUT", "/_data_stream/$dataStream")
assertEquals("Unexpected RestStatus", RestStatus.OK, response.restStatus())
}

protected fun changeAlias(
index: String,
alias: String,
Expand Down Expand Up @@ -533,10 +555,10 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase()
protected fun getFlatSettings(indexName: String) =
(getIndexSettings(indexName) as Map<String, Map<String, Map<String, Any?>>>)[indexName]!!["settings"] as Map<String, String>

protected fun getExplainMap(indexName: String?, showPolicy: Boolean = false): Map<String, Any> {
protected fun getExplainMap(indexName: String?, queryParams: String = ""): Map<String, Any> {
var endpoint = RestExplainAction.EXPLAIN_BASE_URI
if (indexName != null) endpoint += "/$indexName"
if (showPolicy) endpoint += "?$SHOW_POLICY_QUERY_PARAM"
if (queryParams.isNotEmpty()) endpoint += "?$queryParams"
val response = client().makeRequest(RestRequest.Method.GET.toString(), endpoint)
assertEquals("Unexpected RestStatus", RestStatus.OK, response.restStatus())
return response.asMap()
Expand Down
38 changes: 38 additions & 0 deletions ...nsearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,4 +317,42 @@ class IndexStateManagementRestApiIT : IndexStateManagementRestTestCase() {

assertEquals(expectedMessage.toString(), actualMessage.toString())
}

fun `test get policies with hyphen`() {
val randomPolicy = randomPolicy(id = "testing-hyphens-01")
createPolicy(randomPolicy, policyId = randomPolicy.id, refresh = true)
val policy = getPolicy(randomPolicy.id)

val response = client().makeRequest(RestRequest.Method.GET.toString(), "$POLICY_BASE_URI?queryString=*testing-hyphens*")

val actualMessage = response.asMap()
val expectedMessage = mapOf(
"total_policies" to 1,
"policies" to listOf(
mapOf(
_SEQ_NO to policy.seqNo,
_ID to policy.id,
_PRIMARY_TERM to policy.primaryTerm,
Policy.POLICY_TYPE to mapOf(
"schema_version" to policy.schemaVersion,
"policy_id" to policy.id,
"last_updated_time" to policy.lastUpdatedTime.toEpochMilli(),
"default_state" to policy.defaultState,
"ism_template" to null,
"description" to policy.description,
"error_notification" to policy.errorNotification,
"states" to policy.states.map {
mapOf(
"name" to it.name,
"transitions" to it.transitions,
"actions" to it.actions
)
}
)
)
)
)

assertEquals(expectedMessage.toString(), actualMessage.toString())
}
}
111 changes: 110 additions & 1 deletion ...in/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import org.opensearch.common.xcontent.XContentFactory
import org.opensearch.indexmanagement.IndexManagementPlugin
import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase
import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy
import org.opensearch.indexmanagement.indexstatemanagement.util.SHOW_POLICY_QUERY_PARAM
import org.opensearch.indexmanagement.indexstatemanagement.util.TOTAL_MANAGED_INDICES
import org.opensearch.indexmanagement.indexstatemanagement.util.XCONTENT_WITHOUT_TYPE_AND_USER
import org.opensearch.indexmanagement.makeRequest
Expand Down Expand Up @@ -139,6 +140,114 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() {
}
}

fun `test search query string`() {
val indexName1 = "$testIndexName-search-query-string"
val indexName2 = "$indexName1-testing-2"
val indexName3 = "$indexName1-testing-3"
val indexName4 = "$indexName1-testing-4-2022-02-15"
val indexName5 = "$indexName1-testing-5-15-02-2022"
val dataStreamName = "$indexName1-data-stream"
createDataStream(dataStreamName)

val policy = createRandomPolicy()
createIndex(indexName1, policyID = policy.id)
createIndex(indexName2, policyID = policy.id)
createIndex(indexName3, null)
createIndex(indexName4, policyID = policy.id)
createIndex(indexName5, policyID = policy.id)
addPolicyToIndex(dataStreamName, policy.id)
val indexName1Map = indexName1 to mapOf<String, Any>(
explainResponseOpendistroPolicyIdSetting to policy.id,
explainResponseOpenSearchPolicyIdSetting to policy.id,
"index" to indexName1,
"index_uuid" to getUuid(indexName1),
"policy_id" to policy.id,
"enabled" to true
)
val indexName2Map = indexName2 to mapOf<String, Any>(
explainResponseOpendistroPolicyIdSetting to policy.id,
explainResponseOpenSearchPolicyIdSetting to policy.id,
"index" to indexName2,
"index_uuid" to getUuid(indexName2),
"policy_id" to policy.id,
"enabled" to true
)
val indexName4Map = indexName4 to mapOf<String, Any>(
explainResponseOpendistroPolicyIdSetting to policy.id,
explainResponseOpenSearchPolicyIdSetting to policy.id,
"index" to indexName4,
"index_uuid" to getUuid(indexName4),
"policy_id" to policy.id,
"enabled" to true
)
val indexName5Map = indexName5 to mapOf<String, Any>(
explainResponseOpendistroPolicyIdSetting to policy.id,
explainResponseOpenSearchPolicyIdSetting to policy.id,
"index" to indexName5,
"index_uuid" to getUuid(indexName5),
"policy_id" to policy.id,
"enabled" to true
)
val datastreamMap = ".ds-$dataStreamName-000001" to mapOf<String, Any>(
explainResponseOpendistroPolicyIdSetting to policy.id,
explainResponseOpenSearchPolicyIdSetting to policy.id,
"index" to ".ds-$dataStreamName-000001",
"index_uuid" to getUuid(".ds-$dataStreamName-000001"),
"policy_id" to policy.id,
"enabled" to true
)

waitFor {
val expected = mapOf(
indexName1Map,
indexName2Map,
indexName4Map,
indexName5Map,
"total_managed_indices" to 4
)
// These should match all non datastream managed indices
assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=$testIndexName*"))
assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=$testIndexName-*"))
assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=$testIndexName-search-*"))
}

waitFor {
val expected = mapOf(
indexName1Map,
indexName2Map,
indexName4Map,
indexName5Map,
datastreamMap,
"total_managed_indices" to 5
)
// These should match all managed indices including datastreams
assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=*$testIndexName-*"))
assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=*search*"))
assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=*search-query*"))
}

waitFor {
val expected = mapOf(
datastreamMap,
"total_managed_indices" to 1
)
// These should match all datastream managed indices (and system/hidden indices)
assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=.*"))
assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=.ds-$testIndexName-*"))
}

waitFor {
val expected = mapOf(
indexName4Map,
"total_managed_indices" to 1
)
// These should match all just the single index, and validates that it does not match the 15-02-2022 index
// i.e. if it was still matching on tokens then ["2022", "02", "15"] would match both which we don't want
assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=*2022-02-15"))
assertResponseMap(expected, getExplainMap(indexName = null, queryParams = "queryString=*2022-02-15*"))
}
}

fun `test attached policy`() {
val indexName = "${testIndexName}_watermelon"
val policy = createRandomPolicy()
Expand Down Expand Up @@ -239,7 +348,7 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() {
TOTAL_MANAGED_INDICES to 1,
)
waitFor {
assertResponseMap(expected, getExplainMap(indexName, true))
assertResponseMap(expected, getExplainMap(indexName, queryParams = SHOW_POLICY_QUERY_PARAM))
}
}

Expand Down

0 comments on commit 2874937

Please sign in to comment.