Cherry-picking commits for 1.3 release from main to 1.x (opensearch-p…

…roject#345) * Added support for JDK 8 and 14. (opensearch-project#335) * Added support for JDK 8 and 14. Signed-off-by: AWSHurneyt <[email protected]> * Added support for JDK 8 and 14. Signed-off-by: AWSHurneyt <[email protected]> * Added 1.3 release notes. (opensearch-project#336) * Added 1.3 release notes. Signed-off-by: AWSHurneyt <[email protected]> * Added 1.3 release notes. Signed-off-by: AWSHurneyt <[email protected]> * Add backport and auto delete workflow (opensearch-project#339) Signed-off-by: Annie Lee <[email protected]> * Implemented support for ClusterMetrics monitors (opensearch-project#221) (opensearch-project#342) * Implemented support for LocalUriInput monitors. Signed-off-by: AWSHurneyt <[email protected]> * Refactored feature naming convention from LocalUriInput to ClusterMetricsInput. Added Cluster Metrics as a new monitor type to align with frontend experience. Signed-off-by: AWSHurneyt <[email protected]> * Fixed ktlint errors. Signed-off-by: AWSHurneyt <[email protected]> * Implemented randomClusterMetricsMonitor test helper, and refactored tests accordingly. Signed-off-by: AWSHurneyt <[email protected]> * Renamed some assets to align with the new name for this feature. Signed-off-by: AWSHurneyt <[email protected]> * Refactored cluster metrics feature to remove support for Cat repositories API. Signed-off-by: AWSHurneyt <[email protected]> * Refactored supported JSON payload to return all response fields. Signed-off-by: AWSHurneyt <[email protected]> * Refactored nodes stats request object to return all metrics. Signed-off-by: AWSHurneyt <[email protected]> * Refactored nodes stats request object to return all metrics. Signed-off-by: AWSHurneyt <[email protected]> * Removing unused connectionTimeout and socketTimeout params. Signed-off-by: AWSHurneyt <[email protected]> * Added cluster metrics monitor to release notes. (opensearch-project#343) * Added 1.3 release notes. Signed-off-by: AWSHurneyt <[email protected]> * Added 1.3 release notes. Signed-off-by: AWSHurneyt <[email protected]> * Added cluster metrics monitor to release notes. Signed-off-by: AWSHurneyt <[email protected]> * Updated DEVELOPER_GUIDE.md and workflows to reference changes to the supported JDKs. (opensearch-project#338) * Added support for JDK 8 and 14. Signed-off-by: AWSHurneyt <[email protected]> * Added support for JDK 8 and 14. Signed-off-by: AWSHurneyt <[email protected]> * Updated DEVELOPER_GUIDE.md to reference changes to the supported JDKs. Signed-off-by: AWSHurneyt <[email protected]> * Added additional release note. Signed-off-by: AWSHurneyt <[email protected]> * Updated DEVELOPER_GUIDE.md to reference changes to the supported JDKs. Signed-off-by: AWSHurneyt <[email protected]> * Updated DEVELOPER_GUIDE.md to reference changes to the supported JDKs. Signed-off-by: AWSHurneyt <[email protected]> * Updated workflows to build using JDK 11. Signed-off-by: AWSHurneyt <[email protected]> * Removing redundant JDK setup step. Signed-off-by: AWSHurneyt <[email protected]> * Fixed guide wording. Signed-off-by: AWSHurneyt <[email protected]> * Fixed guide wording. Signed-off-by: AWSHurneyt <[email protected]> * Reverted bwc workflow JDK. Signed-off-by: AWSHurneyt <[email protected]> Co-authored-by: Annie Lee <[email protected]>
AWSHurneyt · Mar 11, 2022 · c0cd108 · c0cd108
1 parent 47c70b8
commit c0cd108
Show file tree

Hide file tree

Showing 20 changed files with 1,392 additions and 19 deletions.
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
@@ -0,0 +1,29 @@
+
+name: Backport
+on:
+  pull_request_target:
+    types:
+      - closed
+      - labeled
+
+jobs:
+  backport:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    name: Backport
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+
+      - name: Backport
+        uses: VachaShah/[email protected]
+        with:
+          github_token: ${{ steps.github_app_token.outputs.token }}
+          branch_name: backport/backport-${{ github.event.number }}
diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml
@@ -0,0 +1,15 @@
+name: Delete merged branch of the backport PRs
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  delete-branch:
+    runs-on: ubuntu-latest
+    if: startsWith(github.event.pull_request.head.ref,'backport/')
+    steps:
+      - name: Delete merged branch
+        uses: SvanBoxel/delete-merged-branch@main
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml
@@ -12,7 +12,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [ 14 ]
+        java: [ 11 ]
     # Job name
     name: Build and test Alerting
     # This job runs on Linux
@@ -26,10 +26,5 @@ jobs:
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
         uses: actions/checkout@v2
-      # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK 14
-        uses: actions/setup-java@v1
-        with:
-          java-version: 14
       - name: Run integration tests with multi node config
         run: ./gradlew integTest -PnumNodes=3 -Dopensearch.version=1.3.0-SNAPSHOT
diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml
@@ -12,7 +12,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [ 14 ]
+        java: [ 11 ]
     # Job name
     name: Build and test Alerting
     # This job runs on Linux
@@ -27,10 +27,10 @@ jobs:
       - name: Checkout Branch
         uses: actions/checkout@v2
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK 14
+      - name: Set Up JDK 11
         uses: actions/setup-java@v1
         with:
-          java-version: 14
+          java-version: 11
       - name: Build Alerting
         # Only assembling since the full build is governed by other workflows
         run: ./gradlew assemble -Dopensearch.version=1.3.0-SNAPSHOT

diff --git a/.github/workflows/test-workflow.yml b/.github/workflows/test-workflow.yml
@@ -12,7 +12,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [14]
+        java: [8, 11, 14]
     # Job name
     name: Build Alerting with JDK ${{ matrix.java }}
     # This job runs on Linux

diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -1,6 +1,7 @@
 - [Developer Guide](#developer-guide)
   - [Forking and Cloning](#forking-and-cloning)
   - [Install Prerequisites](#install-prerequisites)
+    - [JDK 11](#jdk-11)
     - [JDK 14](#jdk-14)
   - [Setup](#setup)
   - [Build](#build)
@@ -18,14 +19,32 @@ Fork this repository on GitHub, and clone locally with `git clone`.
 
 ### Install Prerequisites
 
+#### JDK 11
+
+OpenSearch builds using Java 11 at a minimum, using the Adoptium distribution. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`. This is configured in [buildSrc/build.gradle](buildSrc/build.gradle) and [distribution/tools/java-version-checker/build.gradle](distribution/tools/java-version-checker/build.gradle).
+
+```
+allprojects {
+  targetCompatibility = JavaVersion.VERSION_11
+  sourceCompatibility = JavaVersion.VERSION_11
+}
+```
+
+```
+sourceCompatibility = JavaVersion.VERSION_11
+targetCompatibility = JavaVersion.VERSION_11
+```
+
+Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11).
+
 #### JDK 14
 
-OpenSearch components build using Java 14 at a minimum. This means you must have a JDK 14 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 14 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-14`.
+To run the full suite of tests, download and install [JDK 14](https://jdk.java.net/archive/) and set `JAVA11_HOME`, and `JAVA14_HOME`. They are required by the [backwards compatibility test](./TESTING.md#testing-backwards-compatibility).
 
 ### Setup
 
 1. Clone the repository (see [Forking and Cloning](#forking-and-cloning))
-2. Make sure `JAVA_HOME` is pointing to a Java 14 JDK (see [Install Prerequisites](#install-prerequisites))
+2. Make sure `JAVA_HOME` is pointing to a Java 11 JDK (see [Install Prerequisites](#install-prerequisites))
 3. Launch Intellij IDEA, Choose Import Project and select the settings.gradle file in the root of this package.
 
 ### Build
@@ -105,4 +124,9 @@ You can do this by running `./gradlew :alerting:run -PnumNodes=<numberOfNodesYou
 You can also run the integration tests against a multi-node cluster by running `./gradlew :alerting:integTest -PnumNodes=<numberOfNodesYouWant>`
 
 You can also debug a multi-node cluster, by using a combination of above multi-node and debug steps.
-But, you must set up debugger configurations to listen on each port starting from `5005` and increasing by 1 for each node.  
+But, you must set up debugger configurations to listen on each port starting from `5005` and increasing by 1 for each node.
+
+### Backport
+
+- [Link to backport documentation](https://github.com/opensearch-project/opensearch-plugins/blob/main/BACKPORT.md)
+
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt
@@ -2,6 +2,7 @@
  * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
  */
+
 package org.opensearch.alerting
 
 import org.opensearch.action.ActionRequest
@@ -30,6 +31,7 @@ import org.opensearch.alerting.core.JobSweeper
 import org.opensearch.alerting.core.ScheduledJobIndices
 import org.opensearch.alerting.core.action.node.ScheduledJobsStatsAction
 import org.opensearch.alerting.core.action.node.ScheduledJobsStatsTransportAction
+import org.opensearch.alerting.core.model.ClusterMetricsInput
 import org.opensearch.alerting.core.model.ScheduledJob
 import org.opensearch.alerting.core.model.SearchInput
 import org.opensearch.alerting.core.resthandler.RestScheduledJobStatsHandler
@@ -117,7 +119,7 @@ import java.util.function.Supplier
  * Entry point of the OpenDistro for Elasticsearch alerting plugin
  * This class initializes the [RestGetMonitorAction], [RestDeleteMonitorAction], [RestIndexMonitorAction] rest handlers.
  * It also adds [Monitor.XCONTENT_REGISTRY], [SearchInput.XCONTENT_REGISTRY], [QueryLevelTrigger.XCONTENT_REGISTRY],
- * [BucketLevelTrigger.XCONTENT_REGISTRY] to the [NamedXContentRegistry] so that we are able to deserialize the custom named objects.
+ * [BucketLevelTrigger.XCONTENT_REGISTRY], [ClusterMetricsInput.XCONTENT_REGISTRY] to the [NamedXContentRegistry] so that we are able to deserialize the custom named objects.
  */
 internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, ReloadablePlugin, SearchPlugin, Plugin() {
 
@@ -209,7 +211,8 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
             Monitor.XCONTENT_REGISTRY,
             SearchInput.XCONTENT_REGISTRY,
             QueryLevelTrigger.XCONTENT_REGISTRY,
-            BucketLevelTrigger.XCONTENT_REGISTRY
+            BucketLevelTrigger.XCONTENT_REGISTRY,
+            ClusterMetricsInput.XCONTENT_REGISTRY
         )
     }
 

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt
@@ -8,6 +8,7 @@ package org.opensearch.alerting
 import org.apache.logging.log4j.LogManager
 import org.opensearch.action.search.SearchRequest
 import org.opensearch.action.search.SearchResponse
+import org.opensearch.alerting.core.model.ClusterMetricsInput
 import org.opensearch.alerting.core.model.SearchInput
 import org.opensearch.alerting.elasticapi.convertToMap
 import org.opensearch.alerting.elasticapi.suspendUntil
@@ -16,6 +17,8 @@ import org.opensearch.alerting.model.Monitor
 import org.opensearch.alerting.model.TriggerAfterKey
 import org.opensearch.alerting.util.AggregationQueryRewriter
 import org.opensearch.alerting.util.addUserBackendRolesFilter
+import org.opensearch.alerting.util.executeTransportAction
+import org.opensearch.alerting.util.toMap
 import org.opensearch.client.Client
 import org.opensearch.common.io.stream.BytesStreamOutput
 import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput
@@ -85,6 +88,11 @@ class InputService(
                         )
                         results += searchResponse.convertToMap()
                     }
+                    is ClusterMetricsInput -> {
+                        logger.debug("ClusterMetricsInput clusterMetricType: ${input.clusterMetricType}")
+                        val response = executeTransportAction(input, client)
+                        results += response.toMap()
+                    }
                     else -> {
                         throw IllegalArgumentException("Unsupported input type: ${input.name()}.")
                     }

diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/Monitor.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/Monitor.kt
@@ -5,6 +5,7 @@
 
 package org.opensearch.alerting.model
 
+import org.opensearch.alerting.core.model.ClusterMetricsInput
 import org.opensearch.alerting.core.model.CronSchedule
 import org.opensearch.alerting.core.model.Input
 import org.opensearch.alerting.core.model.Schedule
@@ -15,6 +16,7 @@ import org.opensearch.alerting.elasticapi.optionalTimeField
 import org.opensearch.alerting.elasticapi.optionalUserField
 import org.opensearch.alerting.settings.AlertingSettings.Companion.MONITOR_MAX_INPUTS
 import org.opensearch.alerting.settings.AlertingSettings.Companion.MONITOR_MAX_TRIGGERS
+import org.opensearch.alerting.settings.SupportedClusterMetricsSettings
 import org.opensearch.alerting.util.IndexUtils.Companion.NO_SCHEMA_VERSION
 import org.opensearch.alerting.util._ID
 import org.opensearch.alerting.util._VERSION
@@ -69,6 +71,8 @@ data class Monitor(
                     require(trigger is QueryLevelTrigger) { "Incompatible trigger [$trigger.id] for monitor type [$monitorType]" }
                 MonitorType.BUCKET_LEVEL_MONITOR ->
                     require(trigger is BucketLevelTrigger) { "Incompatible trigger [$trigger.id] for monitor type [$monitorType]" }
+                MonitorType.CLUSTER_METRICS_MONITOR ->
+                    require(trigger is QueryLevelTrigger) { "Incompatible trigger [$trigger.id] for monitor type [$monitorType]" }
             }
         }
         if (enabled) {
@@ -113,7 +117,8 @@ data class Monitor(
     // This is different from 'type' which denotes the Scheduled Job type
     enum class MonitorType(val value: String) {
         QUERY_LEVEL_MONITOR("query_level_monitor"),
-        BUCKET_LEVEL_MONITOR("bucket_level_monitor");
+        BUCKET_LEVEL_MONITOR("bucket_level_monitor"),
+        CLUSTER_METRICS_MONITOR("cluster_metrics_monitor");
 
         override fun toString(): String {
             return value
@@ -250,7 +255,10 @@ data class Monitor(
                     INPUTS_FIELD -> {
                         ensureExpectedToken(Token.START_ARRAY, xcp.currentToken(), xcp)
                         while (xcp.nextToken() != Token.END_ARRAY) {
-                            inputs.add(Input.parse(xcp))
+                            val input = Input.parse(xcp)
+                            if (input is ClusterMetricsInput)
+                                SupportedClusterMetricsSettings.validateApiType(input)
+                            inputs.add(input)
                         }
                     }
                     TRIGGERS_FIELD -> {