Add more metrics for backend plugin (#1323)

* Add more metrics for backend plugin Signed-off-by: Joshua Li <[email protected]> * Fix tests Signed-off-by: Joshua Li <[email protected]> Signed-off-by: Joshua Li <[email protected]>
opensearch-project · Jan 10, 2023 · 3720885 · 3720885
1 parent 84127f6
commit 3720885
Show file tree

Hide file tree

Showing 11 changed files with 167 additions and 14 deletions.
diff --git a/src/main/kotlin/org/opensearch/observability/action/CreateObservabilityObjectRequest.kt b/src/main/kotlin/org/opensearch/observability/action/CreateObservabilityObjectRequest.kt
@@ -17,6 +17,7 @@ import org.opensearch.common.xcontent.XContentParser
 import org.opensearch.common.xcontent.XContentParserUtils
 import org.opensearch.commons.utils.fieldIfNotNull
 import org.opensearch.commons.utils.logger
+import org.opensearch.observability.metrics.Metrics
 import org.opensearch.observability.model.BaseObjectData
 import org.opensearch.observability.model.ObservabilityObjectDataProperties
 import org.opensearch.observability.model.ObservabilityObjectType
@@ -73,8 +74,14 @@ internal class CreateObservabilityObjectRequest : ActionRequest, ToXContentObjec
                     }
                 }
             }
-            type ?: throw IllegalArgumentException("Object type field absent")
-            baseObjectData ?: throw IllegalArgumentException("Object data field absent")
+            try {
+                type ?: throw IllegalArgumentException("Object type field absent")
+                baseObjectData ?: throw IllegalArgumentException("Object data field absent")
+            } catch (e: IllegalArgumentException) {
+                Metrics.OBSERVABILITY_CREATE_USER_ERROR.counter.increment()
+                throw e
+            }
+            Metrics.incrementObservabilityObjectActionCounter(type, Metrics.Action.CREATE)
             return CreateObservabilityObjectRequest(objectId, type, baseObjectData)
         }
     }

diff --git a/src/main/kotlin/org/opensearch/observability/action/CreateObservabilityObjectResponse.kt b/src/main/kotlin/org/opensearch/observability/action/CreateObservabilityObjectResponse.kt
@@ -13,6 +13,7 @@ import org.opensearch.common.xcontent.XContentBuilder
 import org.opensearch.common.xcontent.XContentParser
 import org.opensearch.common.xcontent.XContentParserUtils
 import org.opensearch.commons.utils.logger
+import org.opensearch.observability.metrics.Metrics
 import org.opensearch.observability.model.BaseResponse
 import org.opensearch.observability.model.RestTag.OBJECT_ID_FIELD
 import java.io.IOException
@@ -56,7 +57,10 @@ internal class CreateObservabilityObjectResponse : BaseResponse {
                     }
                 }
             }
-            objectId ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
+            objectId ?: run {
+                Metrics.OBSERVABILITY_CREATE_SYSTEM_ERROR.counter.increment()
+                throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
+            }
             return CreateObservabilityObjectResponse(objectId)
         }
     }

diff --git a/src/main/kotlin/org/opensearch/observability/action/DeleteObservabilityObjectRequest.kt b/src/main/kotlin/org/opensearch/observability/action/DeleteObservabilityObjectRequest.kt
@@ -19,6 +19,7 @@ import org.opensearch.common.xcontent.XContentParserUtils
 import org.opensearch.commons.utils.logger
 import org.opensearch.commons.utils.stringList
 import org.opensearch.observability.ObservabilityPlugin.Companion.LOG_PREFIX
+import org.opensearch.observability.metrics.Metrics
 import org.opensearch.observability.model.RestTag.OBJECT_ID_FIELD
 import org.opensearch.observability.model.RestTag.OBJECT_ID_LIST_FIELD
 import java.io.IOException
@@ -63,7 +64,10 @@ internal class DeleteObservabilityObjectRequest : ActionRequest, ToXContentObjec
                     }
                 }
             }
-            objectIds ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
+            objectIds ?: run {
+                Metrics.OBSERVABILITY_DELETE_USER_ERROR.counter.increment()
+                throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
+            }
             return DeleteObservabilityObjectRequest(objectIds)
         }
     }

diff --git a/src/main/kotlin/org/opensearch/observability/action/DeleteObservabilityObjectResponse.kt b/src/main/kotlin/org/opensearch/observability/action/DeleteObservabilityObjectResponse.kt
@@ -17,6 +17,7 @@ import org.opensearch.commons.utils.STRING_WRITER
 import org.opensearch.commons.utils.enumReader
 import org.opensearch.commons.utils.enumWriter
 import org.opensearch.commons.utils.logger
+import org.opensearch.observability.metrics.Metrics
 import org.opensearch.observability.model.BaseResponse
 import org.opensearch.observability.model.RestTag.DELETE_RESPONSE_LIST_TAG
 import org.opensearch.rest.RestStatus
@@ -61,7 +62,10 @@ internal class DeleteObservabilityObjectResponse : BaseResponse {
                     }
                 }
             }
-            objectIdToStatus ?: throw IllegalArgumentException("$DELETE_RESPONSE_LIST_TAG field absent")
+            objectIdToStatus ?: run {
+                Metrics.OBSERVABILITY_DELETE_SYSTEM_ERROR.counter.increment()
+                throw IllegalArgumentException("$DELETE_RESPONSE_LIST_TAG field absent")
+            }
             return DeleteObservabilityObjectResponse(objectIdToStatus)
         }
 

diff --git a/src/main/kotlin/org/opensearch/observability/action/GetObservabilityObjectRequest.kt b/src/main/kotlin/org/opensearch/observability/action/GetObservabilityObjectRequest.kt
@@ -23,6 +23,7 @@ import org.opensearch.commons.utils.enumSet
 import org.opensearch.commons.utils.fieldIfNotNull
 import org.opensearch.commons.utils.logger
 import org.opensearch.commons.utils.stringList
+import org.opensearch.observability.metrics.Metrics
 import org.opensearch.observability.model.ObservabilityObjectType
 import org.opensearch.observability.model.RestTag.FILTER_PARAM_LIST_FIELD
 import org.opensearch.observability.model.RestTag.FROM_INDEX_FIELD
@@ -188,6 +189,7 @@ class GetObservabilityObjectRequest : ActionRequest, ToXContentObject {
         if (maxItems <= 0) {
             validationException = ValidateActions.addValidationError("maxItems is not +ve", validationException)
         }
+        validationException ?: Metrics.OBSERVABILITY_GET_USER_ERROR.counter.increment()
         return validationException
     }
 }
diff --git a/src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectRequest.kt b/src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectRequest.kt
@@ -19,6 +19,7 @@ import org.opensearch.common.xcontent.XContentParser
 import org.opensearch.common.xcontent.XContentParserUtils
 import org.opensearch.commons.utils.fieldIfNotNull
 import org.opensearch.commons.utils.logger
+import org.opensearch.observability.metrics.Metrics
 import org.opensearch.observability.model.BaseObjectData
 import org.opensearch.observability.model.ObservabilityObjectDataProperties
 import org.opensearch.observability.model.ObservabilityObjectType
@@ -76,9 +77,15 @@ internal class UpdateObservabilityObjectRequest : ActionRequest, ToXContentObjec
                     }
                 }
             }
-            objectId ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
-            type ?: throw IllegalArgumentException("Object type field absent")
-            baseObjectData ?: throw IllegalArgumentException("Object data field absent")
+            try {
+                objectId ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
+                type ?: throw IllegalArgumentException("Object type field absent")
+                baseObjectData ?: throw IllegalArgumentException("Object data field absent")
+            } catch (e: IllegalArgumentException) {
+                Metrics.OBSERVABILITY_UPDATE_USER_ERROR.counter.increment()
+                throw e
+            }
+            Metrics.incrementObservabilityObjectActionCounter(type, Metrics.Action.UPDATE)
             return UpdateObservabilityObjectRequest(baseObjectData, type, objectId)
         }
     }

diff --git a/src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectResponse.kt b/src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectResponse.kt
@@ -13,6 +13,7 @@ import org.opensearch.common.xcontent.XContentParser
 import org.opensearch.common.xcontent.XContentParser.Token
 import org.opensearch.common.xcontent.XContentParserUtils
 import org.opensearch.observability.ObservabilityPlugin.Companion.LOG_PREFIX
+import org.opensearch.observability.metrics.Metrics
 import org.opensearch.observability.model.BaseResponse
 import org.opensearch.observability.model.RestTag.OBJECT_ID_FIELD
 import org.opensearch.observability.util.logger
@@ -58,7 +59,10 @@ internal class UpdateObservabilityObjectResponse(
                     }
                 }
             }
-            objectId ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
+            objectId ?: run {
+                Metrics.OBSERVABILITY_UPDATE_SYSTEM_ERROR.counter.increment()
+                throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
+            }
             return UpdateObservabilityObjectResponse(objectId)
         }
     }

diff --git a/src/main/kotlin/org/opensearch/observability/metrics/Metrics.kt b/src/main/kotlin/org/opensearch/observability/metrics/Metrics.kt
@@ -7,6 +7,9 @@ package org.opensearch.observability.metrics
 
 import com.github.wnameless.json.unflattener.JsonUnflattener
 import org.json.JSONObject
+import org.opensearch.observability.ObservabilityPlugin.Companion.LOG_PREFIX
+import org.opensearch.observability.model.ObservabilityObjectType
+import org.opensearch.observability.util.logger
 
 /**
  * Enum to hold all the metrics that need to be logged into _plugins/_observability/_local/stats API
@@ -42,10 +45,63 @@ enum class Metrics(val metricName: String, val counter: Counter<*>) {
         "exception.internal_server_error", RollingCounter()
     ),
 
+    // ==== REST endpoint metrics ==== //
+
+    // Observability counters
+    OBSERVABILITY_CREATE_TOTAL("observability.create.total", BasicCounter()),
+    OBSERVABILITY_CREATE_INTERVAL_COUNT("observability.create.count", RollingCounter()),
+    OBSERVABILITY_CREATE_USER_ERROR("observability.create.user_error", RollingCounter()),
+    OBSERVABILITY_CREATE_SYSTEM_ERROR("observability.create.system_error", RollingCounter()),
+    OBSERVABILITY_GET_TOTAL("observability.get.total", BasicCounter()),
+    OBSERVABILITY_GET_INTERVAL_COUNT("observability.get.count", RollingCounter()),
+    OBSERVABILITY_GET_USER_ERROR("observability.get.user_error", RollingCounter()),
+    OBSERVABILITY_GET_SYSTEM_ERROR("observability.get.system_error", RollingCounter()),
+    OBSERVABILITY_UPDATE_TOTAL("observability.update.total", BasicCounter()),
+    OBSERVABILITY_UPDATE_INTERVAL_COUNT("observability.update.count", RollingCounter()),
+    OBSERVABILITY_UPDATE_USER_ERROR("observability.update.user_error", RollingCounter()),
+    OBSERVABILITY_UPDATE_SYSTEM_ERROR("observability.update.system_error", RollingCounter()),
+    OBSERVABILITY_DELETE_TOTAL("observability.delete.total", BasicCounter()),
+    OBSERVABILITY_DELETE_INTERVAL_COUNT("observability.delete.count", RollingCounter()),
+    OBSERVABILITY_DELETE_USER_ERROR("observability.delete.user_error", RollingCounter()),
+    OBSERVABILITY_DELETE_SYSTEM_ERROR("observability.delete.system_error", RollingCounter()),
+
+    // Per object type action counters, object type is only known for CREATE and UPDATE
+    NOTEBOOK_CREATE_TOTAL("notebook.create.total", BasicCounter()),
+    NOTEBOOK_CREATE_INTERVAL_COUNT("notebook.create.count", RollingCounter()),
+    NOTEBOOK_UPDATE_TOTAL("notebook.update.total", BasicCounter()),
+    NOTEBOOK_UPDATE_INTERVAL_COUNT("notebook.update.count", RollingCounter()),
+
+    SAVED_QUERY_CREATE_TOTAL("saved_query.create.total", BasicCounter()),
+    SAVED_QUERY_CREATE_INTERVAL_COUNT("saved_query.create.count", RollingCounter()),
+    SAVED_QUERY_UPDATE_TOTAL("saved_query.update.total", BasicCounter()),
+    SAVED_QUERY_UPDATE_INTERVAL_COUNT("saved_query.update.count", RollingCounter()),
+
+    SAVED_VISUALIZATION_CREATE_TOTAL("saved_visualization.create.total", BasicCounter()),
+    SAVED_VISUALIZATION_CREATE_INTERVAL_COUNT("saved_visualization.create.count", RollingCounter()),
+    SAVED_VISUALIZATION_UPDATE_TOTAL("saved_visualization.update.total", BasicCounter()),
+    SAVED_VISUALIZATION_UPDATE_INTERVAL_COUNT("saved_visualization.update.count", RollingCounter()),
+
+    OPERATIONAL_PANEL_CREATE_TOTAL("operational_panel.create.total", BasicCounter()),
+    OPERATIONAL_PANEL_CREATE_INTERVAL_COUNT("operational_panel.create.count", RollingCounter()),
+    OPERATIONAL_PANEL_UPDATE_TOTAL("operational_panel.update.total", BasicCounter()),
+    OPERATIONAL_PANEL_UPDATE_INTERVAL_COUNT("operational_panel.update.count", RollingCounter()),
+
+    APPLICATION_CREATE_TOTAL("application.create.total", BasicCounter()),
+    APPLICATION_CREATE_INTERVAL_COUNT("application.create.count", RollingCounter()),
+    APPLICATION_UPDATE_TOTAL("application.update.total", BasicCounter()),
+    APPLICATION_UPDATE_INTERVAL_COUNT("application.update.count", RollingCounter()),
+
+    TIMESTAMP_CREATE_TOTAL("timestamp.create.total", BasicCounter()),
+    TIMESTAMP_CREATE_INTERVAL_COUNT("timestamp.create.count", RollingCounter()),
+    TIMESTAMP_UPDATE_TOTAL("timestamp.update.total", BasicCounter()),
+    TIMESTAMP_UPDATE_INTERVAL_COUNT("timestamp.update.count", RollingCounter()),
+
+    // Permission errors
     OBSERVABILITY_SECURITY_PERMISSION_ERROR("security_permission_error", RollingCounter()),
     OBSERVABILITY_PERMISSION_USER_ERROR("permission_user_error", RollingCounter());
 
     companion object {
+        private val log by logger(Metrics::class.java)
         private val values: Array<Metrics> = Metrics.values()
 
         /**
@@ -75,5 +131,30 @@ enum class Metrics(val metricName: String, val counter: Counter<*>) {
         fun collectToFlattenedJSON(): String {
             return JsonUnflattener.unflatten(collectToJSON())
         }
+
+        /**
+         * Increment observability object action counter
+         *
+         * @param type object type
+         * @param action create or update
+         */
+        fun incrementObservabilityObjectActionCounter(type: ObservabilityObjectType, action: Action) {
+            try {
+                Metrics.valueOf("${type.name}_${action.name}_TOTAL").counter.increment()
+                Metrics.valueOf("${type.name}_${action.name}_INTERVAL_COUNT").counter.increment()
+            } catch (e: IllegalArgumentException) {
+                log.warn("$LOG_PREFIX:IllegalArgumentException invalid type or action for counter metric", e)
+            }
+        }
+    }
+
+    /**
+     * Metrics action for per object type counters. Type is only known for CREATE and UPDATE.
+     *
+     * @constructor Action
+     */
+    enum class Action {
+        CREATE,
+        UPDATE,
     }
 }
diff --git a/src/main/kotlin/org/opensearch/observability/model/SearchResults.kt b/src/main/kotlin/org/opensearch/observability/model/SearchResults.kt
@@ -16,6 +16,7 @@ import org.opensearch.common.xcontent.ToXContent.Params
 import org.opensearch.common.xcontent.XContentBuilder
 import org.opensearch.common.xcontent.XContentParser
 import org.opensearch.common.xcontent.XContentParserUtils
+import org.opensearch.observability.metrics.Metrics
 import org.opensearch.search.SearchHit
 
 internal abstract class SearchResults<ItemClass : BaseModel> : BaseModel {
@@ -127,7 +128,10 @@ internal abstract class SearchResults<ItemClass : BaseModel> : BaseModel {
                 }
             }
         }
-        objectList ?: throw IllegalArgumentException("$objectListFieldName field absent")
+        objectList ?: run {
+            Metrics.OBSERVABILITY_GET_SYSTEM_ERROR.counter.increment()
+            throw IllegalArgumentException("$objectListFieldName field absent")
+        }
         if (totalHits == 0L) {
             totalHits = objectList.size.toLong()
         }

diff --git a/src/main/kotlin/org/opensearch/observability/resthandler/ObservabilityRestHandler.kt b/src/main/kotlin/org/opensearch/observability/resthandler/ObservabilityRestHandler.kt
@@ -18,6 +18,7 @@ import org.opensearch.observability.action.ObservabilityActions
 import org.opensearch.observability.action.UpdateObservabilityObjectAction
 import org.opensearch.observability.action.UpdateObservabilityObjectRequest
 import org.opensearch.observability.index.ObservabilityQueryHelper
+import org.opensearch.observability.metrics.Metrics
 import org.opensearch.observability.model.ObservabilityObjectType
 import org.opensearch.observability.model.RestTag.FROM_INDEX_FIELD
 import org.opensearch.observability.model.RestTag.MAX_ITEMS_FIELD
@@ -200,10 +201,26 @@ internal class ObservabilityRestHandler : BaseRestHandler() {
      */
     override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer {
         return when (request.method()) {
-            POST -> executePostRequest(request, client)
-            PUT -> executePutRequest(request, client)
-            GET -> executeGetRequest(request, client)
-            DELETE -> executeDeleteRequest(request, client)
+            POST -> {
+                Metrics.OBSERVABILITY_CREATE_TOTAL.counter.increment()
+                Metrics.OBSERVABILITY_CREATE_INTERVAL_COUNT.counter.increment()
+                executePostRequest(request, client)
+            }
+            PUT -> {
+                Metrics.OBSERVABILITY_UPDATE_TOTAL.counter.increment()
+                Metrics.OBSERVABILITY_UPDATE_INTERVAL_COUNT.counter.increment()
+                executePutRequest(request, client)
+            }
+            GET -> {
+                Metrics.OBSERVABILITY_GET_TOTAL.counter.increment()
+                Metrics.OBSERVABILITY_GET_INTERVAL_COUNT.counter.increment()
+                executeGetRequest(request, client)
+            }
+            DELETE -> {
+                Metrics.OBSERVABILITY_DELETE_TOTAL.counter.increment()
+                Metrics.OBSERVABILITY_DELETE_INTERVAL_COUNT.counter.increment()
+                executeDeleteRequest(request, client)
+            }
             else -> RestChannelConsumer {
                 it.sendResponse(BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, "${request.method()} is not allowed"))
             }

diff --git a/src/test/kotlin/org/opensearch/observability/metrics/MetricsTests.kt b/src/test/kotlin/org/opensearch/observability/metrics/MetricsTests.kt
@@ -0,0 +1,19 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.observability.metrics
+
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertDoesNotThrow
+import org.opensearch.observability.model.ObservabilityObjectType
+
+internal class MetricsTests {
+    @Test
+    fun testInvalidArguments() {
+        assertDoesNotThrow {
+            Metrics.incrementObservabilityObjectActionCounter(ObservabilityObjectType.NONE, Metrics.Action.UPDATE)
+        }
+    }
+}