From 531aa724ce4c3e4e1c5c4e0e36a3db6de93d54ff Mon Sep 17 00:00:00 2001 From: Ashish Agrawal Date: Fri, 8 Apr 2022 18:06:30 -0700 Subject: [PATCH] Finding Search API (#385) * Findings search API based on Annie's work Signed-off-by: Annie Lee * Fix Search API and add IT tests Signed-off-by: Ashish Agrawal Co-authored-by: Annie Lee --- .../org/opensearch/alerting/AlertingPlugin.kt | 11 +- .../DocumentReturningMonitorRunner.kt | 9 +- .../alerting/action/GetFindingsAction.kt | 15 ++ .../alerting/action/GetFindingsRequest.kt | 42 +++++ .../alerting/action/GetFindingsResponse.kt | 63 +++++++ .../org/opensearch/alerting/model/Finding.kt | 35 ++-- .../alerting/model/FindingDocument.kt | 86 +++++++++ .../alerting/model/FindingWithDocs.kt | 80 +++++++++ .../resthandler/RestGetFindingsAction.kt | 67 +++++++ .../transport/TransportGetFindingsAction.kt | 168 ++++++++++++++++++ .../alerting/findings/finding_mapping.json | 54 ++---- .../alerting/AlertingRestTestCase.kt | 65 ++++++- .../alerting/DocumentMonitorRunnerIT.kt | 6 +- .../org/opensearch/alerting/TestHelpers.kt | 12 +- .../model/DocLevelMonitorInputTests.kt | 2 +- .../opensearch/alerting/model/FindingTests.kt | 6 - .../alerting/resthandler/FindingsRestApiIT.kt | 138 ++++++++++++++ .../alerting/resthandler/MonitorRestApiIT.kt | 8 +- .../alerting/core/model/DocLevelQuery.kt | 18 +- 19 files changed, 778 insertions(+), 107 deletions(-) create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/action/GetFindingsAction.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/action/GetFindingsRequest.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/action/GetFindingsResponse.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/model/FindingDocument.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/model/FindingWithDocs.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetFindingsAction.kt create mode 100644 alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetFindingsAction.kt create mode 100644 alerting/src/test/kotlin/org/opensearch/alerting/resthandler/FindingsRestApiIT.kt diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 8c2417658..ace23402b 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -17,6 +17,7 @@ import org.opensearch.alerting.action.GetAlertsAction import org.opensearch.alerting.action.GetDestinationsAction import org.opensearch.alerting.action.GetEmailAccountAction import org.opensearch.alerting.action.GetEmailGroupAction +import org.opensearch.alerting.action.GetFindingsAction import org.opensearch.alerting.action.GetMonitorAction import org.opensearch.alerting.action.IndexDestinationAction import org.opensearch.alerting.action.IndexEmailAccountAction @@ -53,6 +54,7 @@ import org.opensearch.alerting.resthandler.RestGetAlertsAction import org.opensearch.alerting.resthandler.RestGetDestinationsAction import org.opensearch.alerting.resthandler.RestGetEmailAccountAction import org.opensearch.alerting.resthandler.RestGetEmailGroupAction +import org.opensearch.alerting.resthandler.RestGetFindingsAction import org.opensearch.alerting.resthandler.RestGetMonitorAction import org.opensearch.alerting.resthandler.RestIndexDestinationAction import org.opensearch.alerting.resthandler.RestIndexEmailAccountAction @@ -76,6 +78,7 @@ import org.opensearch.alerting.transport.TransportGetAlertsAction import org.opensearch.alerting.transport.TransportGetDestinationsAction import org.opensearch.alerting.transport.TransportGetEmailAccountAction import org.opensearch.alerting.transport.TransportGetEmailGroupAction +import org.opensearch.alerting.transport.TransportGetFindingsSearchAction import org.opensearch.alerting.transport.TransportGetMonitorAction import org.opensearch.alerting.transport.TransportIndexDestinationAction import org.opensearch.alerting.transport.TransportIndexEmailAccountAction @@ -141,6 +144,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R @JvmField val EMAIL_GROUP_BASE_URI = "$DESTINATION_BASE_URI/email_groups" @JvmField val LEGACY_OPENDISTRO_EMAIL_ACCOUNT_BASE_URI = "$LEGACY_OPENDISTRO_DESTINATION_BASE_URI/email_accounts" @JvmField val LEGACY_OPENDISTRO_EMAIL_GROUP_BASE_URI = "$LEGACY_OPENDISTRO_DESTINATION_BASE_URI/email_groups" + @JvmField val FINDING_BASE_URI = "/_plugins/_alerting/findings" @JvmField val ALERTING_JOB_TYPES = listOf("monitor") } @@ -180,7 +184,8 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R RestSearchEmailGroupAction(), RestGetEmailGroupAction(), RestGetDestinationsAction(), - RestGetAlertsAction() + RestGetAlertsAction(), + RestGetFindingsAction() ) } @@ -204,7 +209,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R ActionPlugin.ActionHandler(SearchEmailGroupAction.INSTANCE, TransportSearchEmailGroupAction::class.java), ActionPlugin.ActionHandler(DeleteEmailGroupAction.INSTANCE, TransportDeleteEmailGroupAction::class.java), ActionPlugin.ActionHandler(GetDestinationsAction.INSTANCE, TransportGetDestinationsAction::class.java), - ActionPlugin.ActionHandler(GetAlertsAction.INSTANCE, TransportGetAlertsAction::class.java) + ActionPlugin.ActionHandler(GetAlertsAction.INSTANCE, TransportGetAlertsAction::class.java), + ActionPlugin.ActionHandler(GetFindingsAction.INSTANCE, TransportGetFindingsSearchAction::class.java) + ) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentReturningMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentReturningMonitorRunner.kt index 206d086b4..16bb92b5e 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentReturningMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentReturningMonitorRunner.kt @@ -165,7 +165,7 @@ object DocumentReturningMonitorRunner : MonitorRunner { if (!dryrun && monitor.id != Monitor.NO_ID) { docsToQueries.forEach { val triggeredQueries = it.value.map { queryId -> idQueryMap[queryId]!! } - val findingId = createFindings(monitor, monitorCtx, index, triggeredQueries, listOf(it.key), trigger) + val findingId = createFindings(monitor, monitorCtx, index, triggeredQueries, listOf(it.key)) findings.add(findingId) if (triggerResult.triggeredDocs.contains(it.key)) { @@ -208,8 +208,7 @@ object DocumentReturningMonitorRunner : MonitorRunner { monitorCtx: MonitorRunnerExecutionContext, index: String, docLevelQueries: List, - matchingDocIds: List, - trigger: DocumentLevelTrigger + matchingDocIds: List ): String { val finding = Finding( id = UUID.randomUUID().toString(), @@ -218,9 +217,7 @@ object DocumentReturningMonitorRunner : MonitorRunner { monitorName = monitor.name, index = index, docLevelQueries = docLevelQueries, - timestamp = Instant.now(), - triggerId = trigger.id, - triggerName = trigger.name + timestamp = Instant.now() ) val findingStr = finding.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS).string() diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/action/GetFindingsAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetFindingsAction.kt new file mode 100644 index 000000000..939ef006c --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetFindingsAction.kt @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.action + +import org.opensearch.action.ActionType + +class GetFindingsAction private constructor() : ActionType(NAME, ::GetFindingsResponse) { + companion object { + val INSTANCE = GetFindingsAction() + val NAME = "cluster:admin/opendistro/alerting/findings/get" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/action/GetFindingsRequest.kt b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetFindingsRequest.kt new file mode 100644 index 000000000..15f9a0d41 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetFindingsRequest.kt @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.action + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionRequestValidationException +import org.opensearch.alerting.model.Table +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.io.stream.StreamOutput +import java.io.IOException + +class GetFindingsRequest : ActionRequest { + val findingId: String? + val table: Table + + constructor( + findingId: String?, + table: Table + ) : super() { + this.findingId = findingId + this.table = table + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + findingId = sin.readOptionalString(), + table = Table.readFrom(sin) + ) + + override fun validate(): ActionRequestValidationException? { + return null + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeOptionalString(findingId) + table.writeTo(out) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/action/GetFindingsResponse.kt b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetFindingsResponse.kt new file mode 100644 index 000000000..66943e318 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetFindingsResponse.kt @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.action + +import org.opensearch.action.ActionResponse +import org.opensearch.alerting.model.FindingWithDocs +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.ToXContentObject +import org.opensearch.common.xcontent.XContentBuilder +import org.opensearch.rest.RestStatus +import java.io.IOException + +class GetFindingsResponse : ActionResponse, ToXContentObject { + var status: RestStatus + var totalFindings: Int? + var findings: List + + constructor( + status: RestStatus, + totalFindings: Int?, + findings: List + ) : super() { + this.status = status + this.totalFindings = totalFindings + this.findings = findings + } + + @Throws(IOException::class) + constructor(sin: StreamInput) { + this.status = sin.readEnum(RestStatus::class.java) + val findings = mutableListOf() + this.totalFindings = sin.readOptionalInt() + var currentSize = sin.readInt() + for (i in 0 until currentSize) { + findings.add(FindingWithDocs.readFrom(sin)) + } + this.findings = findings + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeEnum(status) + out.writeOptionalInt(totalFindings) + out.writeInt(findings.size) + for (finding in findings) { + finding.writeTo(out) + } + } + + @Throws(IOException::class) + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject() + .field("total_findings", totalFindings) + .field("findings", findings) + + return builder.endObject() + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/Finding.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/Finding.kt index 683cff32d..929b99103 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/model/Finding.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/model/Finding.kt @@ -27,10 +27,9 @@ class Finding( val monitorName: String, val index: String, val docLevelQueries: List, - val timestamp: Instant, - val triggerId: String?, - val triggerName: String? + val timestamp: Instant ) : Writeable, ToXContent { + @Throws(IOException::class) constructor(sin: StreamInput) : this( id = sin.readString(), @@ -39,9 +38,7 @@ class Finding( monitorName = sin.readString(), index = sin.readString(), docLevelQueries = sin.readList((DocLevelQuery)::readFrom), - timestamp = sin.readInstant(), - triggerId = sin.readOptionalString(), - triggerName = sin.readOptionalString() + timestamp = sin.readInstant() ) fun asTemplateArg(): Map { @@ -52,9 +49,7 @@ class Finding( MONITOR_NAME_FIELD to monitorName, INDEX_FIELD to index, QUERIES_FIELD to docLevelQueries, - TIMESTAMP_FIELD to timestamp.toEpochMilli(), - TRIGGER_ID_FIELD to triggerId, - TRIGGER_NAME_FIELD to triggerName + TIMESTAMP_FIELD to timestamp.toEpochMilli() ) } @@ -67,8 +62,6 @@ class Finding( .field(INDEX_FIELD, index) .field(QUERIES_FIELD, docLevelQueries.toTypedArray()) .field(TIMESTAMP_FIELD, timestamp.toEpochMilli()) - .field(TRIGGER_ID_FIELD, triggerId) - .field(TRIGGER_NAME_FIELD, triggerName) builder.endObject() return builder } @@ -82,8 +75,6 @@ class Finding( out.writeString(index) out.writeCollection(docLevelQueries) out.writeInstant(timestamp) - out.writeOptionalString(triggerId) - out.writeOptionalString(triggerName) } companion object { @@ -94,21 +85,18 @@ class Finding( const val INDEX_FIELD = "index" const val QUERIES_FIELD = "queries" const val TIMESTAMP_FIELD = "timestamp" - const val TRIGGER_ID_FIELD = "trigger_id" - const val TRIGGER_NAME_FIELD = "trigger_name" const val NO_ID = "" @JvmStatic @JvmOverloads @Throws(IOException::class) - fun parse(xcp: XContentParser, id: String = NO_ID): Finding { + fun parse(xcp: XContentParser): Finding { + var id: String = NO_ID lateinit var relatedDocId: String lateinit var monitorId: String lateinit var monitorName: String lateinit var index: String val queries: MutableList = mutableListOf() lateinit var timestamp: Instant - lateinit var triggerId: String - lateinit var triggerName: String ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { @@ -116,6 +104,7 @@ class Finding( xcp.nextToken() when (fieldName) { + FINDING_ID_FIELD -> id = xcp.text() RELATED_DOC_ID_FIELD -> relatedDocId = xcp.text() MONITOR_ID_FIELD -> monitorId = xcp.text() MONITOR_NAME_FIELD -> monitorName = xcp.text() @@ -126,9 +115,9 @@ class Finding( queries.add(DocLevelQuery.parse(xcp)) } } - TIMESTAMP_FIELD -> timestamp = requireNotNull(xcp.instant()) - TRIGGER_ID_FIELD -> triggerId = xcp.text() - TRIGGER_NAME_FIELD -> triggerName = xcp.text() + TIMESTAMP_FIELD -> { + timestamp = requireNotNull(xcp.instant()) + } } } @@ -139,9 +128,7 @@ class Finding( monitorName = monitorName, index = index, docLevelQueries = queries, - timestamp = timestamp, - triggerId = triggerId, - triggerName = triggerName + timestamp = timestamp ) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/FindingDocument.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/FindingDocument.kt new file mode 100644 index 000000000..1e0c20035 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/model/FindingDocument.kt @@ -0,0 +1,86 @@ +package org.opensearch.alerting.model + +import org.apache.logging.log4j.LogManager +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.io.stream.Writeable +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils +import java.io.IOException + +private val log = LogManager.getLogger(FindingDocument::class.java) + +class FindingDocument( + val index: String, + val id: String, + val found: Boolean, + val document: String +) : Writeable, ToXContent { + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + index = sin.readString(), + id = sin.readString(), + found = sin.readBoolean(), + document = sin.readString() + ) + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return builder.startObject() + .field(INDEX_FIELD, index) + .field(FINDING_DOCUMENT_ID_FIELD, id) + .field(FOUND_FIELD, found) + .field(DOCUMENT_FIELD, document) + .endObject() + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(index) + out.writeString(id) + out.writeBoolean(found) + out.writeString(document) + } + + companion object { + const val INDEX_FIELD = "index" + const val FINDING_DOCUMENT_ID_FIELD = "id" + const val FOUND_FIELD = "found" + const val DOCUMENT_FIELD = "document" + const val NO_ID = "" + const val NO_INDEX = "" + + @JvmStatic @JvmOverloads + @Throws(IOException::class) + fun parse(xcp: XContentParser, id: String = NO_ID, index: String = NO_INDEX): FindingDocument { + var found = false + var document: String = "" + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + FOUND_FIELD -> found = xcp.booleanValue() + DOCUMENT_FIELD -> document = xcp.text() + } + } + + return FindingDocument( + index = index, + id = id, + found = found, + document = document + ) + } + + @JvmStatic + @Throws(IOException::class) + fun readFrom(sin: StreamInput): FindingDocument { + return FindingDocument(sin) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/FindingWithDocs.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/FindingWithDocs.kt new file mode 100644 index 000000000..8e997f247 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/model/FindingWithDocs.kt @@ -0,0 +1,80 @@ +package org.opensearch.alerting.model + +import org.apache.logging.log4j.LogManager +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.io.stream.StreamOutput +import org.opensearch.common.io.stream.Writeable +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils +import java.io.IOException + +private val log = LogManager.getLogger(Finding::class.java) + +class FindingWithDocs( + val finding: Finding, + val documents: List +) : Writeable, ToXContent { + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + finding = Finding.readFrom(sin), + documents = sin.readList((FindingDocument)::readFrom) + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + finding.writeTo(out) + documents.forEach { + it.writeTo(out) + } + } + + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject() + .field(FINDING_FIELD, finding) + .field(DOCUMENTS_FIELD, documents) + builder.endObject() + return builder + } + + companion object { + const val FINDING_FIELD = "finding" + const val DOCUMENTS_FIELD = "document_list" + + @JvmStatic + @Throws(IOException::class) + fun parse(xcp: XContentParser): FindingWithDocs { + lateinit var finding: Finding + val documents: MutableList = mutableListOf() + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + FINDING_FIELD -> finding = Finding.parse(xcp) + DOCUMENTS_FIELD -> { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp) + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + documents.add(FindingDocument.parse(xcp)) + } + } + } + } + + return FindingWithDocs( + finding = finding, + documents = documents + ) + } + + @JvmStatic + @Throws(IOException::class) + fun readFrom(sin: StreamInput): FindingWithDocs { + return FindingWithDocs(sin) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetFindingsAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetFindingsAction.kt new file mode 100644 index 000000000..59a0bee1e --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetFindingsAction.kt @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.resthandler + +import org.apache.logging.log4j.LogManager +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.action.GetFindingsAction +import org.opensearch.alerting.action.GetFindingsRequest +import org.opensearch.alerting.model.Table +import org.opensearch.client.node.NodeClient +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.BaseRestHandler.RestChannelConsumer +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.GET +import org.opensearch.rest.action.RestToXContentListener + +/** + * This class consists of the REST handler to search findings . + */ +class RestGetFindingsAction : BaseRestHandler() { + + private val log = LogManager.getLogger(RestGetFindingsAction::class.java) + + override fun getName(): String { + return "get_findings_action" + } + + override fun routes(): List { + return listOf( + Route(GET, "${AlertingPlugin.FINDING_BASE_URI}/_search") + ) + } + + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.info("${request.method()} ${request.path()}") + + val findingID: String? = request.param("findingId") + val sortString = request.param("sortString", "id.keyword") + val sortOrder = request.param("sortOrder", "asc") + val missing: String? = request.param("missing") + val size = request.paramAsInt("size", 20) + val startIndex = request.paramAsInt("startIndex", 0) + val searchString = request.param("searchString", "") + + val table = Table( + sortOrder, + sortString, + missing, + size, + startIndex, + searchString + ) + + val getFindingsSearchRequest = GetFindingsRequest( + findingID, + table + ) + return RestChannelConsumer { + channel -> + client.execute(GetFindingsAction.INSTANCE, getFindingsSearchRequest, RestToXContentListener(channel)) + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetFindingsAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetFindingsAction.kt new file mode 100644 index 000000000..8c76bc14f --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetFindingsAction.kt @@ -0,0 +1,168 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.transport + +import org.apache.logging.log4j.LogManager +import org.opensearch.action.ActionListener +import org.opensearch.action.get.MultiGetRequest +import org.opensearch.action.search.SearchRequest +import org.opensearch.action.search.SearchResponse +import org.opensearch.action.support.ActionFilters +import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.action.GetFindingsAction +import org.opensearch.alerting.action.GetFindingsRequest +import org.opensearch.alerting.action.GetFindingsResponse +import org.opensearch.alerting.model.Finding +import org.opensearch.alerting.model.FindingDocument +import org.opensearch.alerting.model.FindingWithDocs +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.util.AlertingException +import org.opensearch.client.Client +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.Strings +import org.opensearch.common.inject.Inject +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.NamedXContentRegistry +import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParserUtils +import org.opensearch.common.xcontent.XContentType +import org.opensearch.index.query.Operator +import org.opensearch.index.query.QueryBuilders +import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.search.fetch.subphase.FetchSourceContext +import org.opensearch.search.sort.SortBuilders +import org.opensearch.search.sort.SortOrder +import org.opensearch.tasks.Task +import org.opensearch.transport.TransportService + +private val log = LogManager.getLogger(TransportGetFindingsSearchAction::class.java) + +class TransportGetFindingsSearchAction @Inject constructor( + transportService: TransportService, + val client: Client, + clusterService: ClusterService, + actionFilters: ActionFilters, + val settings: Settings, + val xContentRegistry: NamedXContentRegistry +) : HandledTransportAction ( + GetFindingsAction.NAME, transportService, actionFilters, ::GetFindingsRequest +), + SecureTransportAction { + + @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + listenFilterBySettingChange(clusterService) + } + + override fun doExecute( + task: Task, + getFindingsRequest: GetFindingsRequest, + actionListener: ActionListener + ) { + val tableProp = getFindingsRequest.table + + val sortBuilder = SortBuilders + .fieldSort(tableProp.sortString) + .order(SortOrder.fromString(tableProp.sortOrder)) + if (!tableProp.missing.isNullOrBlank()) { + sortBuilder.missing(tableProp.missing) + } + + val searchSourceBuilder = SearchSourceBuilder() + .sort(sortBuilder) + .size(tableProp.size) + .from(tableProp.startIndex) + .fetchSource(FetchSourceContext(true, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY)) + .seqNoAndPrimaryTerm(true) + .version(true) + + val queryBuilder = QueryBuilders.boolQuery() + + if (!getFindingsRequest.findingId.isNullOrBlank()) + queryBuilder.filter(QueryBuilders.termQuery("_id", getFindingsRequest.findingId)) + + if (!tableProp.searchString.isNullOrBlank()) { + queryBuilder + .must( + QueryBuilders + .queryStringQuery(tableProp.searchString) + .defaultOperator(Operator.AND) + .field("queries.tags") + .field("queries.name") + ) + } + + searchSourceBuilder.query(queryBuilder) + + client.threadPool().threadContext.stashContext().use { + search(searchSourceBuilder, actionListener) + } + } + + fun search(searchSourceBuilder: SearchSourceBuilder, actionListener: ActionListener) { + val searchRequest = SearchRequest() + .source(searchSourceBuilder) + .indices(".opensearch-alerting-findings") + client.search( + searchRequest, + object : ActionListener { + override fun onResponse(response: SearchResponse) { + val totalFindingCount = response.hits.totalHits?.value?.toInt() + val mgetRequest = MultiGetRequest() + val findingsWithDocs = mutableListOf() + val findings = mutableListOf() + for (hit in response.hits) { + val xcp = XContentFactory.xContent(XContentType.JSON) + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.sourceAsString) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp) + val finding = Finding.parse(xcp) + findings.add(finding) + val documentIds = finding.relatedDocId.split(",").toTypedArray() + // Add getRequests to mget request + documentIds.forEach { + docId -> + mgetRequest.add(MultiGetRequest.Item(finding.index, docId)) + } + } + val documents = searchDocument(mgetRequest) + findings.forEach { + val documentIds = it.relatedDocId.split(",").toTypedArray() + val relatedDocs = mutableListOf() + for (docId in documentIds) { + val key = "${it.index}|$docId" + documents[key]?.let { document -> relatedDocs.add(document) } + } + findingsWithDocs.add(FindingWithDocs(it, relatedDocs)) + } + actionListener.onResponse(GetFindingsResponse(response.status(), totalFindingCount, findingsWithDocs)) + } + + override fun onFailure(t: Exception) { + actionListener.onFailure(AlertingException.wrap(t)) + } + } + ) + } + + // TODO: Verify what happens if indices are closed/deleted + fun searchDocument( + mgetRequest: MultiGetRequest + ): Map { + val response = client.multiGet(mgetRequest).actionGet() + val documents: MutableMap = mutableMapOf() + response.responses.forEach { + val key = "${it.index}|${it.id}" + val docData = if (it.isFailed) "" else it.response.sourceAsString + val findingDocument = FindingDocument(it.index, it.id, !it.isFailed, docData) + documents[key] = findingDocument + } + + return documents + } +} diff --git a/alerting/src/main/resources/org/opensearch/alerting/findings/finding_mapping.json b/alerting/src/main/resources/org/opensearch/alerting/findings/finding_mapping.json index aeb9d324e..fc52e3945 100644 --- a/alerting/src/main/resources/org/opensearch/alerting/findings/finding_mapping.json +++ b/alerting/src/main/resources/org/opensearch/alerting/findings/finding_mapping.json @@ -30,52 +30,17 @@ "index": { "type": "keyword" }, - "query_id": { - "type": "keyword" - }, - "query_tags": { - "type" : "text", - "fields" : { - "keyword" : { - "type" : "keyword" - } - } - }, - "severity": { - "type": "keyword" - }, - "timestamp": { - "type": "date" - }, - "monitor_user": { + "queries" : { + "type": "nested", "properties": { - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } + "id": { + "type": "keyword" }, - "backend_roles": { - "type" : "text", - "fields" : { - "keyword" : { - "type" : "keyword" - } - } - }, - "roles": { - "type" : "text", - "fields" : { - "keyword" : { - "type" : "keyword" - } - } + "name": { + "type": "text" }, - "custom_attribute_names": { - "type" : "text", + "tags": { + "type": "text", "fields" : { "keyword" : { "type" : "keyword" @@ -83,6 +48,9 @@ } } } + }, + "timestamp": { + "type": "date" } } } \ No newline at end of file diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt index 52cfb5c12..2cfb25d2e 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt @@ -16,7 +16,9 @@ import org.junit.rules.DisableOnDebug import org.opensearch.action.search.SearchResponse import org.opensearch.alerting.AlertingPlugin.Companion.EMAIL_ACCOUNT_BASE_URI import org.opensearch.alerting.AlertingPlugin.Companion.EMAIL_GROUP_BASE_URI +import org.opensearch.alerting.action.GetFindingsResponse import org.opensearch.alerting.alerts.AlertIndices +import org.opensearch.alerting.core.model.DocLevelQuery import org.opensearch.alerting.core.model.ScheduledJob import org.opensearch.alerting.core.model.SearchInput import org.opensearch.alerting.core.settings.ScheduledJobSettings @@ -25,6 +27,7 @@ import org.opensearch.alerting.model.Alert import org.opensearch.alerting.model.BucketLevelTrigger import org.opensearch.alerting.model.DocumentLevelTrigger import org.opensearch.alerting.model.Finding +import org.opensearch.alerting.model.FindingWithDocs import org.opensearch.alerting.model.Monitor import org.opensearch.alerting.model.QueryLevelTrigger import org.opensearch.alerting.model.destination.Destination @@ -43,6 +46,7 @@ import org.opensearch.common.unit.TimeValue import org.opensearch.common.xcontent.LoggingDeprecationHandler import org.opensearch.common.xcontent.NamedXContentRegistry import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentFactory import org.opensearch.common.xcontent.XContentFactory.jsonBuilder import org.opensearch.common.xcontent.XContentParser @@ -59,10 +63,12 @@ import java.time.ZonedDateTime import java.time.format.DateTimeFormatter import java.time.temporal.ChronoUnit import java.util.Locale +import java.util.UUID import javax.management.MBeanServerInvocationHandler import javax.management.ObjectName import javax.management.remote.JMXConnectorFactory import javax.management.remote.JMXServiceURL +import kotlin.collections.HashMap abstract class AlertingRestTestCase : ODFERestTestCase() { @@ -504,6 +510,29 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { }.filter { alert -> alert.monitorId == monitor.id } } + protected fun createFinding( + monitorId: String = "NO_ID", + monitorName: String = "NO_NAME", + index: String = "testIndex", + docLevelQueries: List = listOf(DocLevelQuery(query = "test_field:\"us-west-2\"", name = "testQuery")), + matchingDocIds: Set + ): String { + val finding = Finding( + id = UUID.randomUUID().toString(), + relatedDocId = matchingDocIds.joinToString(","), + monitorId = monitorId, + monitorName = monitorName, + index = index, + docLevelQueries = docLevelQueries, + timestamp = Instant.now() + ) + + val findingStr = finding.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS).string() + + indexDoc(".opensearch-alerting-findings", finding.id, findingStr) + return finding.id + } + protected fun searchFindings( monitor: Monitor, indices: String = ".opensearch-alerting-findings", @@ -522,7 +551,7 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { val searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content)) return searchResponse.hits.hits.map { val xcp = createParser(JsonXContent.jsonXContent, it.sourceRef).also { it.nextToken() } - Finding.parse(xcp, it.id) + Finding.parse(xcp) }.filter { finding -> finding.monitorId == monitor.id } } @@ -610,6 +639,40 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { protected fun executeMonitor(client: RestClient, monitor: Monitor, params: Map = mapOf()): Response = client.makeRequest("POST", "$ALERTING_BASE_URI/_execute", params, monitor.toHttpEntityWithUser()) + protected fun searchFindings(params: Map = mutableMapOf()): GetFindingsResponse { + + var baseEndpoint = "${AlertingPlugin.FINDING_BASE_URI}/_search?" + for (entry in params.entries) { + baseEndpoint += "${entry.key}=${entry.value}&" + } + + val response = client().makeRequest("GET", baseEndpoint) + + assertEquals("Unable to retrieve findings", RestStatus.OK, response.restStatus()) + + val parser = createParser(XContentType.JSON.xContent(), response.entity.content) + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser) + + var totalFindings = 0 + val findings = mutableListOf() + + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + parser.nextToken() + + when (parser.currentName()) { + "total_findings" -> totalFindings = parser.intValue() + "findings" -> { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser) + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + findings.add(FindingWithDocs.parse(parser)) + } + } + } + } + + return GetFindingsResponse(response.restStatus(), totalFindings, findings) + } + protected fun indexDoc(index: String, id: String, doc: String, refresh: Boolean = true): Response { val requestBody = StringEntity(doc, APPLICATION_JSON) val params = if (refresh) mapOf("refresh" to "true") else mapOf() diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt index d65cbd527..baa2a8946 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt @@ -42,7 +42,7 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() { val index = createTestIndex() - val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", severity = "3") + val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3") val docReturningInput = DocLevelMonitorInput("description", listOf(index), listOf(docQuery)) val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id) @@ -82,7 +82,7 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() { "test_field" : "us-west-2" }""" - val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", severity = "3") + val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3") val docReturningInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery)) val trigger = randomDocumentReturningTrigger(condition = ALWAYS_RUN) @@ -113,7 +113,7 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() { "test_field" : "us-west-2" }""" - val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", severity = "3") + val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3") val docReturningInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery)) val trigger = randomDocumentReturningTrigger(condition = ALWAYS_RUN) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt b/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt index 3fa37d4f1..900a8b7a9 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/TestHelpers.kt @@ -369,10 +369,10 @@ fun randomAlert(monitor: Monitor = randomQueryLevelMonitor()): Alert { fun randomDocLevelQuery( id: String = OpenSearchRestTestCase.randomAlphaOfLength(10), query: String = OpenSearchRestTestCase.randomAlphaOfLength(10), - severity: String = "${randomInt(5)}", + name: String = "${randomInt(5)}", tags: List = mutableListOf(0..randomInt(10)).map { OpenSearchRestTestCase.randomAlphaOfLength(10) } ): DocLevelQuery { - return DocLevelQuery(id = id, query = query, severity = severity, tags = tags) + return DocLevelQuery(id = id, query = query, name = name, tags = tags) } fun randomDocLevelMonitorInput( @@ -390,9 +390,7 @@ fun randomFinding( monitorName: String = OpenSearchRestTestCase.randomAlphaOfLength(10), index: String = OpenSearchRestTestCase.randomAlphaOfLength(10), docLevelQueries: List = listOf(randomDocLevelQuery()), - timestamp: Instant = Instant.now(), - triggerId: String = OpenSearchRestTestCase.randomAlphaOfLength(10), - triggerName: String = OpenSearchRestTestCase.randomAlphaOfLength(10) + timestamp: Instant = Instant.now() ): Finding { return Finding( id = id, @@ -401,9 +399,7 @@ fun randomFinding( monitorName = monitorName, index = index, docLevelQueries = docLevelQueries, - timestamp = timestamp, - triggerId = triggerId, - triggerName = triggerName + timestamp = timestamp ) } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/model/DocLevelMonitorInputTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/model/DocLevelMonitorInputTests.kt index 13b2ebb63..2f1da18a8 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/model/DocLevelMonitorInputTests.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/model/DocLevelMonitorInputTests.kt @@ -26,7 +26,7 @@ class DocLevelMonitorInputTests : OpenSearchTestCase() { // THEN assertEquals("Template args 'id' field does not match:", templateArgs[DocLevelQuery.QUERY_ID_FIELD], query.id) assertEquals("Template args 'query' field does not match:", templateArgs[DocLevelQuery.QUERY_FIELD], query.query) - assertEquals("Template args 'severity' field does not match:", templateArgs[DocLevelQuery.SEVERITY_FIELD], query.severity) + assertEquals("Template args 'name' field does not match:", templateArgs[DocLevelQuery.NAME_FIELD], query.name) assertEquals("Template args 'tags' field does not match:", templateArgs[DocLevelQuery.TAGS_FIELD], query.tags) } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/model/FindingTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/model/FindingTests.kt index dd73908d0..ca0169ee9 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/model/FindingTests.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/model/FindingTests.kt @@ -31,11 +31,5 @@ class FindingTests : OpenSearchTestCase() { templateArgs[Finding.TIMESTAMP_FIELD], finding.timestamp.toEpochMilli() ) - assertEquals("Template args 'triggerId' field does not match:", templateArgs[Finding.TRIGGER_ID_FIELD], finding.triggerId) - assertEquals( - "Template args 'triggerName' field does not match:", - templateArgs[Finding.TRIGGER_NAME_FIELD], - finding.triggerName - ) } } diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/FindingsRestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/FindingsRestApiIT.kt new file mode 100644 index 000000000..c8df534ba --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/FindingsRestApiIT.kt @@ -0,0 +1,138 @@ +package org.opensearch.alerting.resthandler + +import org.opensearch.alerting.AlertingRestTestCase +import org.opensearch.alerting.core.model.DocLevelQuery +import org.opensearch.test.junit.annotations.TestLogging + +@TestLogging("level:DEBUG", reason = "Debug for tests.") +@Suppress("UNCHECKED_CAST") +class FindingsRestApiIT : AlertingRestTestCase() { + + fun `test find Finding where doc is not retrieved`() { + + createFinding(matchingDocIds = setOf("someId")) + val response = searchFindings() + assertEquals(1, response.totalFindings) + assertEquals(1, response.findings[0].documents.size) + assertFalse(response.findings[0].documents[0].found) + } + + fun `test find Finding where doc is retrieved`() { + val testIndex = createTestIndex() + val testDoc = """{ + "message" : "This is an error from IAD region", + "test_field" : "us-west-2" + }""" + indexDoc(testIndex, "someId", testDoc) + val testDoc2 = """{ + "message" : "This is an error2 from IAD region", + "test_field" : "us-west-3" + }""" + indexDoc(testIndex, "someId2", testDoc2) + + val findingWith1 = createFinding(matchingDocIds = setOf("someId"), index = testIndex) + val findingWith2 = createFinding(matchingDocIds = setOf("someId", "someId2"), index = testIndex) + val response = searchFindings() + assertEquals(2, response.totalFindings) + for (findingWithDoc in response.findings) { + if (findingWithDoc.finding.id == findingWith1) { + assertEquals(1, findingWithDoc.documents.size) + assertTrue(findingWithDoc.documents[0].found) + assertEquals(testDoc, findingWithDoc.documents[0].document) + } else if (findingWithDoc.finding.id == findingWith2) { + assertEquals(2, findingWithDoc.documents.size) + assertTrue(findingWithDoc.documents[0].found) + assertTrue(findingWithDoc.documents[1].found) + assertEquals(testDoc, findingWithDoc.documents[0].document) + assertEquals(testDoc2, findingWithDoc.documents[1].document) + } else { + fail("Found a finding that should not have been retrieved") + } + } + } + + fun `test find Finding for specific finding by id`() { + val testIndex = createTestIndex() + val testDoc = """{ + "message" : "This is an error from IAD region", + "test_field" : "us-west-2" + }""" + indexDoc(testIndex, "someId", testDoc) + val testDoc2 = """{ + "message" : "This is an error2 from IAD region", + "test_field" : "us-west-3" + }""" + indexDoc(testIndex, "someId2", testDoc2) + + createFinding(matchingDocIds = setOf("someId"), index = testIndex) + val findingId = createFinding(matchingDocIds = setOf("someId", "someId2"), index = testIndex) + val response = searchFindings(mapOf(Pair("findingId", findingId))) + assertEquals(1, response.totalFindings) + assertEquals(findingId, response.findings[0].finding.id) + assertEquals(2, response.findings[0].documents.size) + assertTrue(response.findings[0].documents[0].found) + assertTrue(response.findings[0].documents[1].found) + assertEquals(testDoc, response.findings[0].documents[0].document) + assertEquals(testDoc2, response.findings[0].documents[1].document) + } + + fun `test find Finding by tag`() { + val testIndex = createTestIndex() + val testDoc = """{ + "message" : "This is an error from IAD region", + "test_field" : "us-west-2" + }""" + indexDoc(testIndex, "someId", testDoc) + val testDoc2 = """{ + "message" : "This is an error2 from IAD region", + "test_field" : "us-west-3" + }""" + indexDoc(testIndex, "someId2", testDoc2) + + val docLevelQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "realQuery", tags = listOf("sigma")) + createFinding(matchingDocIds = setOf("someId"), index = testIndex) + val findingId = createFinding( + matchingDocIds = setOf("someId", "someId2"), + index = testIndex, + docLevelQueries = listOf(docLevelQuery) + ) + val response = searchFindings(mapOf(Pair("searchString", "sigma"))) + assertEquals(1, response.totalFindings) + assertEquals(findingId, response.findings[0].finding.id) + assertEquals(2, response.findings[0].documents.size) + assertTrue(response.findings[0].documents[0].found) + assertTrue(response.findings[0].documents[1].found) + assertEquals(testDoc, response.findings[0].documents[0].document) + assertEquals(testDoc2, response.findings[0].documents[1].document) + } + + fun `test find Finding by name`() { + val testIndex = createTestIndex() + val testDoc = """{ + "message" : "This is an error from IAD region", + "test_field" : "us-west-2" + }""" + indexDoc(testIndex, "someId", testDoc) + val testDoc2 = """{ + "message" : "This is an error2 from IAD region", + "test_field" : "us-west-3" + }""" + indexDoc(testIndex, "someId2", testDoc2) + + val docLevelQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "realQuery", tags = listOf("sigma")) + createFinding(matchingDocIds = setOf("someId"), index = testIndex) + val findingId = createFinding( + matchingDocIds = setOf("someId", "someId2"), + index = testIndex, + docLevelQueries = listOf(docLevelQuery) + ) + val response = searchFindings(mapOf(Pair("searchString", "realQuery"))) + assertEquals(1, response.totalFindings) + assertEquals(findingId, response.findings[0].finding.id) + assertEquals(2, response.findings[0].documents.size) + assertTrue(response.findings[0].documents[0].found) + assertTrue(response.findings[0].documents[1].found) + assertEquals(testDoc, response.findings[0].documents[0].document) + assertEquals(testDoc2, response.findings[0].documents[1].document) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt index 9d2337df4..ace1fb9bf 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt @@ -1117,7 +1117,7 @@ class MonitorRestApiIT : AlertingRestTestCase() { @Throws(Exception::class) fun `test creating a document monitor`() { val testIndex = createTestIndex() - val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", severity = "3") + val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3") val docReturningInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery)) val trigger = randomDocumentReturningTrigger(condition = ALWAYS_RUN) @@ -1138,7 +1138,7 @@ class MonitorRestApiIT : AlertingRestTestCase() { @Throws(Exception::class) fun `test getting a document level monitor`() { val testIndex = createTestIndex() - val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", severity = "3") + val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3") val docReturningInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery)) val trigger = randomDocumentReturningTrigger(condition = ALWAYS_RUN) @@ -1154,7 +1154,7 @@ class MonitorRestApiIT : AlertingRestTestCase() { @Throws(Exception::class) fun `test updating conditions for a doc-level monitor`() { val testIndex = createTestIndex() - val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", severity = "3") + val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3") val docReturningInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery)) val trigger = randomDocumentReturningTrigger(condition = ALWAYS_RUN) @@ -1185,7 +1185,7 @@ class MonitorRestApiIT : AlertingRestTestCase() { @Throws(Exception::class) fun `test deleting a document level monitor`() { val testIndex = createTestIndex() - val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", severity = "3") + val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3") val docReturningInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery)) val trigger = randomDocumentReturningTrigger(condition = ALWAYS_RUN) diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/model/DocLevelQuery.kt b/core/src/main/kotlin/org/opensearch/alerting/core/model/DocLevelQuery.kt index 8a4c235a4..2a4d32bca 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/model/DocLevelQuery.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/model/DocLevelQuery.kt @@ -17,24 +17,24 @@ import java.io.IOException data class DocLevelQuery( val id: String = NO_ID, + val name: String, val query: String, - val severity: String, val tags: List = mutableListOf() ) : Writeable, ToXContentObject { @Throws(IOException::class) constructor(sin: StreamInput) : this( sin.readString(), // id + sin.readString(), // name sin.readString(), // query - sin.readString(), // severity sin.readStringList() // tags ) fun asTemplateArg(): Map { return mapOf( QUERY_ID_FIELD to id, + NAME_FIELD to name, QUERY_FIELD to query, - SEVERITY_FIELD to severity, TAGS_FIELD to tags ) } @@ -42,16 +42,16 @@ data class DocLevelQuery( @Throws(IOException::class) override fun writeTo(out: StreamOutput) { out.writeString(id) + out.writeString(name) out.writeString(query) - out.writeString(severity) out.writeStringCollection(tags) } override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { builder.startObject() .field(QUERY_ID_FIELD, id) + .field(NAME_FIELD, name) .field(QUERY_FIELD, query) - .field(SEVERITY_FIELD, severity) .field(TAGS_FIELD, tags.toTypedArray()) .endObject() return builder @@ -59,8 +59,8 @@ data class DocLevelQuery( companion object { const val QUERY_ID_FIELD = "id" + const val NAME_FIELD = "name" const val QUERY_FIELD = "query" - const val SEVERITY_FIELD = "severity" const val TAGS_FIELD = "tags" const val NO_ID = "" @@ -69,7 +69,7 @@ data class DocLevelQuery( fun parse(xcp: XContentParser): DocLevelQuery { var id: String = NO_ID lateinit var query: String - lateinit var severity: String + lateinit var name: String val tags: MutableList = mutableListOf() ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) @@ -79,8 +79,8 @@ data class DocLevelQuery( when (fieldName) { QUERY_ID_FIELD -> id = xcp.text() + NAME_FIELD -> name = xcp.text() QUERY_FIELD -> query = xcp.text() - SEVERITY_FIELD -> severity = xcp.text() TAGS_FIELD -> { ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp) while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { @@ -92,8 +92,8 @@ data class DocLevelQuery( return DocLevelQuery( id = id, + name = name, query = query, - severity = severity, tags = tags ) }