Merge branch 'opensearch-project:main' into clusterMetricsFlakyTest

opensearch-project · Apr 14, 2022 · bfc2b92 · bfc2b92
2 parents 35b5c4a + 14a43cf
commit bfc2b92
Show file tree

Hide file tree

Showing 58 changed files with 1,007 additions and 92 deletions.
diff --git a/.github/workflows/bwc-test-workflow.yml b/.github/workflows/bwc-test-workflow.yml
@@ -12,7 +12,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [ 14 ]
+        java: [ 11 ]
     # Job name
     name: Build and test Alerting
     # This job runs on Linux
@@ -27,11 +27,11 @@ jobs:
       - name: Checkout Branch
         uses: actions/checkout@v2
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK 14
+      - name: Set Up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 14
+          java-version: 11
       - name: Run Alerting Backwards Compatibility Tests
         run: |
           echo "Running backwards compatibility tests..."
-          ./gradlew bwcTestSuite -Dbuild.qualifier=alpha1
+          ./gradlew bwcTestSuite
diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml
@@ -12,7 +12,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [ 11 ]
+        java: [ 11, 17 ]
     # Job name
     name: Build and test Alerting
     # This job runs on Linux
@@ -27,4 +27,4 @@ jobs:
       - name: Checkout Branch
         uses: actions/checkout@v2
       - name: Run integration tests with multi node config
-        run: ./gradlew integTest -PnumNodes=3 -Dbuild.qualifier=alpha1
+        run: ./gradlew integTest -PnumNodes=3
diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml
@@ -12,7 +12,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [ 11 ]
+        java: [ 11, 17 ]
     # Job name
     name: Build and test Alerting
     # This job runs on Linux
@@ -27,38 +27,36 @@ jobs:
       - name: Checkout Branch
         uses: actions/checkout@v2
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK 11
+      - name: Set Up JDK ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:
-          java-version: 11
+          java-version: ${{ matrix.java }}
       - name: Build Alerting
         # Only assembling since the full build is governed by other workflows
-        run: ./gradlew assemble -Dbuild.qualifier=alpha1
+        run: ./gradlew assemble
       - name: Pull and Run Docker
         run: |
-          plugin=`ls alerting/build/distributions/*.zip`
+          plugin=`basename $(ls alerting/build/distributions/*.zip)`
           list_of_files=`ls`
           list_of_all_files=`ls alerting/build/distributions/`
           version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3`
           plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4`
           qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1`
           candidate_version=`echo $plugin|awk -F- '{print $5}'| cut -d. -f 1-1`
-          docker_version=$version-$qualifier-candidate_version
+          docker_version=$version-$qualifier
 
-          if [ -z $candidate_version ]; then
-              candidate_version=$qualifier && qualifier=""
-              docker_version=$version
-          fi
+          [[ -z $candidate_version ]] && candidate_version=$qualifier && qualifier=""
 
-          echo $version $plugin_version $qualifier $candidate_version $docker_version
+          echo plugin version plugin_version qualifier candidate_version docker_version
+          echo "($plugin) ($version) ($plugin_version) ($qualifier) ($candidate_version) ($docker_version)"
           echo $ls $list_of_all_files
 
           if docker pull opensearchstaging/opensearch:$docker_version
           then
             echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
             echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-alerting ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-alerting; fi" >> Dockerfile
-            echo "ADD alerting/build/distributions/opensearch-alerting-$plugin_version-$candidate_version.zip /tmp/" >> Dockerfile
-            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/opensearch-alerting-$plugin_version-$candidate_version.zip" >> Dockerfile
+            echo "ADD alerting/build/distributions/$plugin /tmp/" >> Dockerfile
+            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
 
             docker build -t opensearch-alerting:test .
             echo "imagePresent=true" >> $GITHUB_ENV

diff --git a/.github/workflows/test-workflow.yml b/.github/workflows/test-workflow.yml
@@ -12,7 +12,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [11, 14]
+        java: [11, 17]
     # Job name
     name: Build Alerting with JDK ${{ matrix.java }}
     # This job runs on Linux
@@ -27,7 +27,7 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
       - name: Build and run with Gradle
-        run: ./gradlew build -Dbuild.qualifier=alpha1
+        run: ./gradlew build
       - name: Create Artifact Path
         run: |
           mkdir -p alerting-artifacts

diff --git a/alerting/build.gradle b/alerting/build.gradle
@@ -111,6 +111,20 @@ testClusters.integTest {
             debugPort += 1
         }
     }
+
+    plugin(provider({
+        new RegularFile() {
+            @Override
+            File getAsFile() { fileTree("src/test/resources/notifications-core").getSingleFile() }
+        }
+    }))
+
+    plugin(provider({
+        new RegularFile() {
+            @Override
+            File getAsFile() { fileTree("src/test/resources/notifications").getSingleFile() }
+        }
+    }))
 }
 
 testClusters.integTest.nodes.each { node ->
@@ -187,7 +201,19 @@ task prepareBwcTests {
     dependsOn bundle
     doLast {
         plugins = [
-                project.getObjects().fileProperty().value(bundle.getArchiveFile())
+                project.getObjects().fileProperty().value(bundle.getArchiveFile()),
+                provider({
+                    new RegularFile() {
+                        @Override
+                        File getAsFile() { fileTree("src/test/resources/notifications-core").getSingleFile() }
+                    }
+                }),
+                provider({
+                    new RegularFile() {
+                        @Override
+                        File getAsFile() { fileTree("src/test/resources/notifications").getSingleFile() }
+                    }
+                })
         ]
     }
 }

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt
@@ -17,9 +17,6 @@ import org.opensearch.action.search.SearchRequest
 import org.opensearch.action.search.SearchResponse
 import org.opensearch.alerting.alerts.AlertError
 import org.opensearch.alerting.alerts.AlertIndices
-import org.opensearch.alerting.elasticapi.firstFailureOrNull
-import org.opensearch.alerting.elasticapi.retry
-import org.opensearch.alerting.elasticapi.suspendUntil
 import org.opensearch.alerting.model.ActionExecutionResult
 import org.opensearch.alerting.model.ActionRunResult
 import org.opensearch.alerting.model.AggregationResultBucket
@@ -29,6 +26,9 @@ import org.opensearch.alerting.model.Monitor
 import org.opensearch.alerting.model.QueryLevelTriggerRunResult
 import org.opensearch.alerting.model.Trigger
 import org.opensearch.alerting.model.action.AlertCategory
+import org.opensearch.alerting.opensearchapi.firstFailureOrNull
+import org.opensearch.alerting.opensearchapi.retry
+import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.alerting.script.QueryLevelTriggerExecutionContext
 import org.opensearch.alerting.util.IndexUtils
 import org.opensearch.alerting.util.getBucketKeysHash

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt
@@ -82,6 +82,7 @@ import org.opensearch.alerting.transport.TransportIndexMonitorAction
 import org.opensearch.alerting.transport.TransportSearchEmailAccountAction
 import org.opensearch.alerting.transport.TransportSearchEmailGroupAction
 import org.opensearch.alerting.transport.TransportSearchMonitorAction
+import org.opensearch.alerting.util.destinationmigration.DestinationMigrationCoordinator
 import org.opensearch.client.Client
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver
 import org.opensearch.cluster.node.DiscoveryNodes
@@ -149,6 +150,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
     lateinit var threadPool: ThreadPool
     lateinit var alertIndices: AlertIndices
     lateinit var clusterService: ClusterService
+    lateinit var destinationMigrationCoordinator: DestinationMigrationCoordinator
 
     override fun getRestHandlers(
         settings: Settings,
@@ -248,9 +250,10 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
         scheduledJobIndices = ScheduledJobIndices(client.admin(), clusterService)
         scheduler = JobScheduler(threadPool, runner)
         sweeper = JobSweeper(environment.settings(), client, clusterService, threadPool, xContentRegistry, scheduler, ALERTING_JOB_TYPES)
+        destinationMigrationCoordinator = DestinationMigrationCoordinator(client, clusterService, threadPool)
         this.threadPool = threadPool
         this.clusterService = clusterService
-        return listOf(sweeper, scheduler, runner, scheduledJobIndices)
+        return listOf(sweeper, scheduler, runner, scheduledJobIndices, destinationMigrationCoordinator)
     }
 
     override fun getSettings(): List<Setting<*>> {

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt
@@ -10,11 +10,11 @@ import org.opensearch.action.search.SearchRequest
 import org.opensearch.action.search.SearchResponse
 import org.opensearch.alerting.core.model.ClusterMetricsInput
 import org.opensearch.alerting.core.model.SearchInput
-import org.opensearch.alerting.elasticapi.convertToMap
-import org.opensearch.alerting.elasticapi.suspendUntil
 import org.opensearch.alerting.model.InputRunResults
 import org.opensearch.alerting.model.Monitor
 import org.opensearch.alerting.model.TriggerAfterKey
+import org.opensearch.alerting.opensearchapi.convertToMap
+import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.alerting.util.AggregationQueryRewriter
 import org.opensearch.alerting.util.addUserBackendRolesFilter
 import org.opensearch.alerting.util.executeTransportAction

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt
@@ -17,9 +17,6 @@ import org.opensearch.alerting.alerts.AlertIndices
 import org.opensearch.alerting.alerts.moveAlerts
 import org.opensearch.alerting.core.JobRunner
 import org.opensearch.alerting.core.model.ScheduledJob
-import org.opensearch.alerting.elasticapi.InjectorContextElement
-import org.opensearch.alerting.elasticapi.retry
-import org.opensearch.alerting.elasticapi.withClosableContext
 import org.opensearch.alerting.model.ActionRunResult
 import org.opensearch.alerting.model.Alert
 import org.opensearch.alerting.model.AlertingConfigAccessor
@@ -39,6 +36,9 @@ import org.opensearch.alerting.model.action.AlertCategory
 import org.opensearch.alerting.model.action.PerAlertActionScope
 import org.opensearch.alerting.model.action.PerExecutionActionScope
 import org.opensearch.alerting.model.destination.DestinationContextFactory
+import org.opensearch.alerting.opensearchapi.InjectorContextElement
+import org.opensearch.alerting.opensearchapi.retry
+import org.opensearch.alerting.opensearchapi.withClosableContext
 import org.opensearch.alerting.script.BucketLevelTriggerExecutionContext
 import org.opensearch.alerting.script.QueryLevelTriggerExecutionContext
 import org.opensearch.alerting.script.TriggerExecutionContext

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alerts/AlertError.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alerts/AlertError.kt
@@ -5,8 +5,8 @@
 
 package org.opensearch.alerting.alerts
 
-import org.opensearch.alerting.elasticapi.instant
-import org.opensearch.alerting.elasticapi.optionalTimeField
+import org.opensearch.alerting.opensearchapi.instant
+import org.opensearch.alerting.opensearchapi.optionalTimeField
 import org.opensearch.common.io.stream.StreamInput
 import org.opensearch.common.io.stream.StreamOutput
 import org.opensearch.common.io.stream.Writeable

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alerts/AlertIndices.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alerts/AlertIndices.kt
@@ -23,7 +23,7 @@ import org.opensearch.action.support.IndicesOptions
 import org.opensearch.action.support.master.AcknowledgedResponse
 import org.opensearch.alerting.alerts.AlertIndices.Companion.ALERT_INDEX
 import org.opensearch.alerting.alerts.AlertIndices.Companion.HISTORY_WRITE_INDEX
-import org.opensearch.alerting.elasticapi.suspendUntil
+import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_HISTORY_ENABLED
 import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_HISTORY_INDEX_MAX_AGE

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alerts/AlertMover.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alerts/AlertMover.kt
@@ -13,9 +13,9 @@ import org.opensearch.action.search.SearchRequest
 import org.opensearch.action.search.SearchResponse
 import org.opensearch.alerting.alerts.AlertIndices.Companion.ALERT_INDEX
 import org.opensearch.alerting.alerts.AlertIndices.Companion.HISTORY_WRITE_INDEX
-import org.opensearch.alerting.elasticapi.suspendUntil
 import org.opensearch.alerting.model.Alert
 import org.opensearch.alerting.model.Monitor
+import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.client.Client
 import org.opensearch.common.bytes.BytesReference
 import org.opensearch.common.xcontent.LoggingDeprecationHandler

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/ActionExecutionResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/ActionExecutionResult.kt
@@ -5,8 +5,8 @@
 
 package org.opensearch.alerting.model
 
-import org.opensearch.alerting.elasticapi.instant
-import org.opensearch.alerting.elasticapi.optionalTimeField
+import org.opensearch.alerting.opensearchapi.instant
+import org.opensearch.alerting.opensearchapi.optionalTimeField
 import org.opensearch.common.io.stream.StreamInput
 import org.opensearch.common.io.stream.StreamOutput
 import org.opensearch.common.io.stream.Writeable

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/Alert.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/Alert.kt
@@ -6,9 +6,9 @@
 package org.opensearch.alerting.model
 
 import org.opensearch.alerting.alerts.AlertError
-import org.opensearch.alerting.elasticapi.instant
-import org.opensearch.alerting.elasticapi.optionalTimeField
-import org.opensearch.alerting.elasticapi.optionalUserField
+import org.opensearch.alerting.opensearchapi.instant
+import org.opensearch.alerting.opensearchapi.optionalTimeField
+import org.opensearch.alerting.opensearchapi.optionalUserField
 import org.opensearch.alerting.util.IndexUtils.Companion.NO_SCHEMA_VERSION
 import org.opensearch.common.io.stream.StreamInput
 import org.opensearch.common.io.stream.StreamOutput

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/AlertingConfigAccessor.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/AlertingConfigAccessor.kt
@@ -10,10 +10,10 @@ import kotlinx.coroutines.withContext
 import org.opensearch.action.get.GetRequest
 import org.opensearch.action.get.GetResponse
 import org.opensearch.alerting.core.model.ScheduledJob
-import org.opensearch.alerting.elasticapi.suspendUntil
 import org.opensearch.alerting.model.destination.Destination
 import org.opensearch.alerting.model.destination.email.EmailAccount
 import org.opensearch.alerting.model.destination.email.EmailGroup
+import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.client.Client
 import org.opensearch.common.bytes.BytesReference
 import org.opensearch.common.xcontent.LoggingDeprecationHandler

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/BucketLevelTriggerRunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/BucketLevelTriggerRunResult.kt
@@ -38,7 +38,7 @@ data class BucketLevelTriggerRunResult(
     override fun writeTo(out: StreamOutput) {
         super.writeTo(out)
         out.writeMap(aggregationResultBuckets, StreamOutput::writeString) {
-            valueOut: StreamOutput, aggResultBucket: AggregationResultBucket ->
+                valueOut: StreamOutput, aggResultBucket: AggregationResultBucket ->
             aggResultBucket.writeTo(valueOut)
         }
         out.writeMap(actionResultsMap as Map<String, Any>)

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/Monitor.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/Monitor.kt
@@ -11,9 +11,9 @@ import org.opensearch.alerting.core.model.Input
 import org.opensearch.alerting.core.model.Schedule
 import org.opensearch.alerting.core.model.ScheduledJob
 import org.opensearch.alerting.core.model.SearchInput
-import org.opensearch.alerting.elasticapi.instant
-import org.opensearch.alerting.elasticapi.optionalTimeField
-import org.opensearch.alerting.elasticapi.optionalUserField
+import org.opensearch.alerting.opensearchapi.instant
+import org.opensearch.alerting.opensearchapi.optionalTimeField
+import org.opensearch.alerting.opensearchapi.optionalUserField
 import org.opensearch.alerting.settings.AlertingSettings.Companion.MONITOR_MAX_INPUTS
 import org.opensearch.alerting.settings.AlertingSettings.Companion.MONITOR_MAX_TRIGGERS
 import org.opensearch.alerting.settings.SupportedClusterMetricsSettings
@@ -289,7 +289,7 @@ data class Monitor(
                 requireNotNull(schedule) { "Monitor schedule is null" },
                 lastUpdateTime ?: Instant.now(),
                 enabledTime,
-                MonitorType.valueOf(monitorType.toUpperCase(Locale.ROOT)),
+                MonitorType.valueOf(monitorType.uppercase(Locale.ROOT)),
                 user,
                 schemaVersion,
                 inputs.toList(),

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/MonitorRunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/MonitorRunResult.kt
@@ -8,7 +8,7 @@ package org.opensearch.alerting.model
 import org.apache.logging.log4j.LogManager
 import org.opensearch.OpenSearchException
 import org.opensearch.alerting.alerts.AlertError
-import org.opensearch.alerting.elasticapi.optionalTimeField
+import org.opensearch.alerting.opensearchapi.optionalTimeField
 import org.opensearch.common.io.stream.StreamInput
 import org.opensearch.common.io.stream.StreamOutput
 import org.opensearch.common.io.stream.Writeable

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/action/Throttle.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/action/Throttle.kt
@@ -58,7 +58,7 @@ data class Throttle(
                 xcp.nextToken()
                 when (fieldName) {
                     UNIT_FIELD -> {
-                        val unitString = xcp.text().toUpperCase(Locale.ROOT)
+                        val unitString = xcp.text().uppercase(Locale.ROOT)
                         require(StringUtils.equals(unitString, ChronoUnit.MINUTES.name), { "Only support MINUTES throttle unit currently" })
                         unit = ChronoUnit.valueOf(unitString)
                     }

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/destination/Chime.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/destination/Chime.kt
@@ -5,7 +5,7 @@
 
 package org.opensearch.alerting.model.destination
 
-import org.opensearch.alerting.elasticapi.string
+import org.opensearch.alerting.opensearchapi.string
 import org.opensearch.common.Strings
 import org.opensearch.common.io.stream.StreamInput
 import org.opensearch.common.io.stream.StreamOutput

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/destination/Destination.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/destination/Destination.kt
@@ -12,11 +12,11 @@ import org.opensearch.alerting.destination.message.ChimeMessage
 import org.opensearch.alerting.destination.message.CustomWebhookMessage
 import org.opensearch.alerting.destination.message.EmailMessage
 import org.opensearch.alerting.destination.message.SlackMessage
-import org.opensearch.alerting.elasticapi.convertToMap
-import org.opensearch.alerting.elasticapi.instant
-import org.opensearch.alerting.elasticapi.optionalTimeField
-import org.opensearch.alerting.elasticapi.optionalUserField
 import org.opensearch.alerting.model.destination.email.Email
+import org.opensearch.alerting.opensearchapi.convertToMap
+import org.opensearch.alerting.opensearchapi.instant
+import org.opensearch.alerting.opensearchapi.optionalTimeField
+import org.opensearch.alerting.opensearchapi.optionalUserField
 import org.opensearch.alerting.util.DestinationType
 import org.opensearch.alerting.util.IndexUtils.Companion.NO_SCHEMA_VERSION
 import org.opensearch.alerting.util.isHostInDenylist
@@ -193,7 +193,7 @@ data class Destination(
                 schemaVersion,
                 seqNo,
                 primaryTerm,
-                DestinationType.valueOf(type.toUpperCase(Locale.ROOT)),
+                DestinationType.valueOf(type.uppercase(Locale.ROOT)),
                 requireNotNull(name) { "Destination name is null" },
                 user,
                 lastUpdateTime ?: Instant.now(),

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/destination/Slack.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/destination/Slack.kt
@@ -5,7 +5,7 @@
 
 package org.opensearch.alerting.model.destination
 
-import org.opensearch.alerting.elasticapi.string
+import org.opensearch.alerting.opensearchapi.string
 import org.opensearch.common.Strings
 import org.opensearch.common.io.stream.StreamInput
 import org.opensearch.common.io.stream.StreamOutput