From 3533dd725d31db01c197c7c13ff4330580108415 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Wed, 2 Oct 2024 18:42:27 +0000 Subject: [PATCH 1/4] delete query index only if put mappings throws an exception Signed-off-by: Subhobrata Dey --- .../alerting/DocumentLevelMonitorRunner.kt | 12 +----- .../alerting/util/DocLevelMonitorQueries.kt | 37 ++++++++++++++++--- .../SampleRemoteMonitorRestHandler.java | 6 +-- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt index 6f339fdce..ce9a92708 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt @@ -121,17 +121,6 @@ class DocumentLevelMonitorRunner : MonitorRunner() { throw IndexNotFoundException(docLevelMonitorInput.indices.joinToString(",")) } - if (monitor.deleteQueryIndexInEveryRun == true && - monitorCtx.docLevelMonitorQueries!!.docLevelQueryIndexExists(monitor.dataSources) - ) { - val ack = monitorCtx.docLevelMonitorQueries!!.deleteDocLevelQueryIndex(monitor.dataSources) - if (!ack) { - logger.error( - "Deletion of concrete queryIndex:${monitor.dataSources.queryIndex} is not ack'd! " + - "for monitor ${monitor.id}" - ) - } - } monitorCtx.docLevelMonitorQueries!!.initDocLevelQueryIndex(monitor.dataSources) monitorCtx.docLevelMonitorQueries!!.indexDocLevelQueries( monitor = monitor, @@ -389,6 +378,7 @@ class DocumentLevelMonitorRunner : MonitorRunner() { // TODO: Update the Document as part of the Trigger and return back the trigger action result return monitorResult.copy(triggerResults = triggerResults, inputResults = inputRunResults) } catch (e: Exception) { + e.printStackTrace() val errorMessage = ExceptionsHelper.detailedMessage(e) monitorCtx.alertService!!.upsertMonitorErrorAlert(monitor, errorMessage, executionId, workflowRunContext) logger.error("Failed running Document-level-monitor ${monitor.name}", e) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt index 7ce3309db..ce99916ae 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt @@ -358,7 +358,8 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ monitorMetadata, updatedIndexName, sourceIndexFieldLimit, - updatedProperties + updatedProperties, + indexTimeout ) if (updateMappingResponse.isAcknowledged) { @@ -487,7 +488,8 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ monitorMetadata: MonitorMetadata, sourceIndex: String, sourceIndexFieldLimit: Long, - updatedProperties: MutableMap + updatedProperties: MutableMap, + indexTimeout: TimeValue ): Pair { var targetQueryIndex = monitorMetadata.sourceToQueryIndexMapping[sourceIndex + monitor.id] if ( @@ -551,9 +553,34 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ } } } else { - log.debug("unknown exception during PUT mapping on queryIndex: $targetQueryIndex") - val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception - throw AlertingException.wrap(unwrappedException) + // retry with deleting query index + if (monitor.deleteQueryIndexInEveryRun == true && docLevelQueryIndexExists(monitor.dataSources)) { + try { + log.info("unknown exception during PUT mapping on queryIndex: $targetQueryIndex, retrying with deletion of query index") + val ack = monitorCtx.docLevelMonitorQueries!!.deleteDocLevelQueryIndex(monitor.dataSources) + if (!ack) { + log.error( + "Deletion of concrete queryIndex:${monitor.dataSources.queryIndex} is not ack'd! " + + "for monitor ${monitor.id}" + ) + } + initDocLevelQueryIndex(monitor.dataSources) + indexDocLevelQueries( + monitor = monitor, + monitorId = monitor.id, + monitorMetadata, + indexTimeout = indexTimeout + ) + } catch (e: Exception) { + log.debug("unknown exception during PUT mapping on queryIndex: $targetQueryIndex") + val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception + throw AlertingException.wrap(unwrappedException) + } + } else { + log.debug("unknown exception during PUT mapping on queryIndex: $targetQueryIndex") + val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception + throw AlertingException.wrap(unwrappedException) + } } } // We did rollover, so try to apply mappings again on new targetQueryIndex diff --git a/sample-remote-monitor-plugin/src/main/java/org/opensearch/alerting/SampleRemoteMonitorRestHandler.java b/sample-remote-monitor-plugin/src/main/java/org/opensearch/alerting/SampleRemoteMonitorRestHandler.java index a8f384cb6..9875ef55f 100644 --- a/sample-remote-monitor-plugin/src/main/java/org/opensearch/alerting/SampleRemoteMonitorRestHandler.java +++ b/sample-remote-monitor-plugin/src/main/java/org/opensearch/alerting/SampleRemoteMonitorRestHandler.java @@ -94,7 +94,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient "id", null)), trigger1Serialized)), Map.of(), new DataSources(), - true, + false, "sample-remote-monitor-plugin" ); IndexMonitorRequest indexMonitorRequest1 = new IndexMonitorRequest( @@ -155,7 +155,7 @@ public void onFailure(Exception e) { List.of(), Map.of(), new DataSources(), - true, + false, "sample-remote-monitor-plugin" ); IndexMonitorRequest indexMonitorRequest2 = new IndexMonitorRequest( @@ -239,7 +239,7 @@ public void onFailure(Exception e) { "id", null)), trigger1Serialized)), Map.of(), new DataSources(), - true, + false, "sample-remote-monitor-plugin" ); IndexMonitorRequest indexDocLevelMonitorRequest = new IndexMonitorRequest( From 66b5b1a1ac93196bcf7b65bf19a8375a8c91046b Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Wed, 2 Oct 2024 18:45:22 +0000 Subject: [PATCH 2/4] remove stacktrace Signed-off-by: Subhobrata Dey --- .../kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt index ce9a92708..b5b2c4f49 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt @@ -378,7 +378,6 @@ class DocumentLevelMonitorRunner : MonitorRunner() { // TODO: Update the Document as part of the Trigger and return back the trigger action result return monitorResult.copy(triggerResults = triggerResults, inputResults = inputRunResults) } catch (e: Exception) { - e.printStackTrace() val errorMessage = ExceptionsHelper.detailedMessage(e) monitorCtx.alertService!!.upsertMonitorErrorAlert(monitor, errorMessage, executionId, workflowRunContext) logger.error("Failed running Document-level-monitor ${monitor.name}", e) From f5b416fc11ff4133da9ddf98c56dcc80d2579afa Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Wed, 2 Oct 2024 18:52:31 +0000 Subject: [PATCH 3/4] fix ktlint Signed-off-by: Subhobrata Dey --- .../org/opensearch/alerting/util/DocLevelMonitorQueries.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt index ce99916ae..b84332cd8 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt @@ -556,7 +556,10 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ // retry with deleting query index if (monitor.deleteQueryIndexInEveryRun == true && docLevelQueryIndexExists(monitor.dataSources)) { try { - log.info("unknown exception during PUT mapping on queryIndex: $targetQueryIndex, retrying with deletion of query index") + log.info( + "unknown exception during PUT mapping on queryIndex: $targetQueryIndex, " + + "retrying with deletion of query index" + ) val ack = monitorCtx.docLevelMonitorQueries!!.deleteDocLevelQueryIndex(monitor.dataSources) if (!ack) { log.error( From 6323602d6033b10ae1bcf8e0653ee3a4225693f6 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Wed, 2 Oct 2024 22:29:14 +0000 Subject: [PATCH 4/4] add logs Signed-off-by: Subhobrata Dey --- .../alerting/util/DocLevelMonitorQueries.kt | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt index b84332cd8..d47537c7f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt @@ -554,18 +554,21 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ } } else { // retry with deleting query index - if (monitor.deleteQueryIndexInEveryRun == true && docLevelQueryIndexExists(monitor.dataSources)) { + if (monitor.deleteQueryIndexInEveryRun == true) { try { - log.info( + log.error( "unknown exception during PUT mapping on queryIndex: $targetQueryIndex, " + - "retrying with deletion of query index" + "retrying with deletion of query index", + e ) - val ack = monitorCtx.docLevelMonitorQueries!!.deleteDocLevelQueryIndex(monitor.dataSources) - if (!ack) { - log.error( - "Deletion of concrete queryIndex:${monitor.dataSources.queryIndex} is not ack'd! " + - "for monitor ${monitor.id}" - ) + if (docLevelQueryIndexExists(monitor.dataSources)) { + val ack = monitorCtx.docLevelMonitorQueries!!.deleteDocLevelQueryIndex(monitor.dataSources) + if (!ack) { + log.error( + "Deletion of concrete queryIndex:${monitor.dataSources.queryIndex} is not ack'd! " + + "for monitor ${monitor.id}" + ) + } } initDocLevelQueryIndex(monitor.dataSources) indexDocLevelQueries( @@ -575,12 +578,20 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ indexTimeout = indexTimeout ) } catch (e: Exception) { - log.debug("unknown exception during PUT mapping on queryIndex: $targetQueryIndex") + log.error( + "Doc level monitor ${monitor.id}: unknown exception during " + + "PUT mapping on queryIndex: $targetQueryIndex", + e + ) val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception throw AlertingException.wrap(unwrappedException) } } else { - log.debug("unknown exception during PUT mapping on queryIndex: $targetQueryIndex") + log.error( + "Doc level monitor ${monitor.id}: unknown exception during " + + "PUT mapping on queryIndex: $targetQueryIndex", + e + ) val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception throw AlertingException.wrap(unwrappedException) }