diff --git a/src/main/kotlin/org/opensearch/observability/action/CreateObservabilityObjectRequest.kt b/src/main/kotlin/org/opensearch/observability/action/CreateObservabilityObjectRequest.kt index 8ce247d13..cdbb370f8 100644 --- a/src/main/kotlin/org/opensearch/observability/action/CreateObservabilityObjectRequest.kt +++ b/src/main/kotlin/org/opensearch/observability/action/CreateObservabilityObjectRequest.kt @@ -17,6 +17,7 @@ import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentParserUtils import org.opensearch.commons.utils.fieldIfNotNull import org.opensearch.commons.utils.logger +import org.opensearch.observability.metrics.Metrics import org.opensearch.observability.model.BaseObjectData import org.opensearch.observability.model.ObservabilityObjectDataProperties import org.opensearch.observability.model.ObservabilityObjectType @@ -73,8 +74,14 @@ internal class CreateObservabilityObjectRequest : ActionRequest, ToXContentObjec } } } - type ?: throw IllegalArgumentException("Object type field absent") - baseObjectData ?: throw IllegalArgumentException("Object data field absent") + try { + type ?: throw IllegalArgumentException("Object type field absent") + baseObjectData ?: throw IllegalArgumentException("Object data field absent") + } catch (e: IllegalArgumentException) { + Metrics.OBSERVABILITY_CREATE_USER_ERROR.counter.increment() + throw e + } + Metrics.incrementObservabilityObjectActionCounter(type, Metrics.Action.CREATE) return CreateObservabilityObjectRequest(objectId, type, baseObjectData) } } diff --git a/src/main/kotlin/org/opensearch/observability/action/CreateObservabilityObjectResponse.kt b/src/main/kotlin/org/opensearch/observability/action/CreateObservabilityObjectResponse.kt index f9cb1bd35..f86c51e44 100644 --- a/src/main/kotlin/org/opensearch/observability/action/CreateObservabilityObjectResponse.kt +++ b/src/main/kotlin/org/opensearch/observability/action/CreateObservabilityObjectResponse.kt @@ -13,6 +13,7 @@ import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentParserUtils import org.opensearch.commons.utils.logger +import org.opensearch.observability.metrics.Metrics import org.opensearch.observability.model.BaseResponse import org.opensearch.observability.model.RestTag.OBJECT_ID_FIELD import java.io.IOException @@ -56,7 +57,10 @@ internal class CreateObservabilityObjectResponse : BaseResponse { } } } - objectId ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent") + objectId ?: run { + Metrics.OBSERVABILITY_CREATE_SYSTEM_ERROR.counter.increment() + throw IllegalArgumentException("$OBJECT_ID_FIELD field absent") + } return CreateObservabilityObjectResponse(objectId) } } diff --git a/src/main/kotlin/org/opensearch/observability/action/DeleteObservabilityObjectRequest.kt b/src/main/kotlin/org/opensearch/observability/action/DeleteObservabilityObjectRequest.kt index ace60136f..6eaf45765 100644 --- a/src/main/kotlin/org/opensearch/observability/action/DeleteObservabilityObjectRequest.kt +++ b/src/main/kotlin/org/opensearch/observability/action/DeleteObservabilityObjectRequest.kt @@ -19,6 +19,7 @@ import org.opensearch.common.xcontent.XContentParserUtils import org.opensearch.commons.utils.logger import org.opensearch.commons.utils.stringList import org.opensearch.observability.ObservabilityPlugin.Companion.LOG_PREFIX +import org.opensearch.observability.metrics.Metrics import org.opensearch.observability.model.RestTag.OBJECT_ID_FIELD import org.opensearch.observability.model.RestTag.OBJECT_ID_LIST_FIELD import java.io.IOException @@ -63,7 +64,10 @@ internal class DeleteObservabilityObjectRequest : ActionRequest, ToXContentObjec } } } - objectIds ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent") + objectIds ?: run { + Metrics.OBSERVABILITY_DELETE_USER_ERROR.counter.increment() + throw IllegalArgumentException("$OBJECT_ID_FIELD field absent") + } return DeleteObservabilityObjectRequest(objectIds) } } diff --git a/src/main/kotlin/org/opensearch/observability/action/DeleteObservabilityObjectResponse.kt b/src/main/kotlin/org/opensearch/observability/action/DeleteObservabilityObjectResponse.kt index cc4a4644a..f23cba883 100644 --- a/src/main/kotlin/org/opensearch/observability/action/DeleteObservabilityObjectResponse.kt +++ b/src/main/kotlin/org/opensearch/observability/action/DeleteObservabilityObjectResponse.kt @@ -17,6 +17,7 @@ import org.opensearch.commons.utils.STRING_WRITER import org.opensearch.commons.utils.enumReader import org.opensearch.commons.utils.enumWriter import org.opensearch.commons.utils.logger +import org.opensearch.observability.metrics.Metrics import org.opensearch.observability.model.BaseResponse import org.opensearch.observability.model.RestTag.DELETE_RESPONSE_LIST_TAG import org.opensearch.rest.RestStatus @@ -61,7 +62,10 @@ internal class DeleteObservabilityObjectResponse : BaseResponse { } } } - objectIdToStatus ?: throw IllegalArgumentException("$DELETE_RESPONSE_LIST_TAG field absent") + objectIdToStatus ?: run { + Metrics.OBSERVABILITY_DELETE_SYSTEM_ERROR.counter.increment() + throw IllegalArgumentException("$DELETE_RESPONSE_LIST_TAG field absent") + } return DeleteObservabilityObjectResponse(objectIdToStatus) } diff --git a/src/main/kotlin/org/opensearch/observability/action/GetObservabilityObjectRequest.kt b/src/main/kotlin/org/opensearch/observability/action/GetObservabilityObjectRequest.kt index 5a050f083..569db0d12 100644 --- a/src/main/kotlin/org/opensearch/observability/action/GetObservabilityObjectRequest.kt +++ b/src/main/kotlin/org/opensearch/observability/action/GetObservabilityObjectRequest.kt @@ -23,6 +23,7 @@ import org.opensearch.commons.utils.enumSet import org.opensearch.commons.utils.fieldIfNotNull import org.opensearch.commons.utils.logger import org.opensearch.commons.utils.stringList +import org.opensearch.observability.metrics.Metrics import org.opensearch.observability.model.ObservabilityObjectType import org.opensearch.observability.model.RestTag.FILTER_PARAM_LIST_FIELD import org.opensearch.observability.model.RestTag.FROM_INDEX_FIELD @@ -188,6 +189,7 @@ class GetObservabilityObjectRequest : ActionRequest, ToXContentObject { if (maxItems <= 0) { validationException = ValidateActions.addValidationError("maxItems is not +ve", validationException) } + validationException ?: Metrics.OBSERVABILITY_GET_USER_ERROR.counter.increment() return validationException } } diff --git a/src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectRequest.kt b/src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectRequest.kt index b6229b310..e988ce8b7 100644 --- a/src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectRequest.kt +++ b/src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectRequest.kt @@ -19,6 +19,7 @@ import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentParserUtils import org.opensearch.commons.utils.fieldIfNotNull import org.opensearch.commons.utils.logger +import org.opensearch.observability.metrics.Metrics import org.opensearch.observability.model.BaseObjectData import org.opensearch.observability.model.ObservabilityObjectDataProperties import org.opensearch.observability.model.ObservabilityObjectType @@ -76,9 +77,15 @@ internal class UpdateObservabilityObjectRequest : ActionRequest, ToXContentObjec } } } - objectId ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent") - type ?: throw IllegalArgumentException("Object type field absent") - baseObjectData ?: throw IllegalArgumentException("Object data field absent") + try { + objectId ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent") + type ?: throw IllegalArgumentException("Object type field absent") + baseObjectData ?: throw IllegalArgumentException("Object data field absent") + } catch (e: IllegalArgumentException) { + Metrics.OBSERVABILITY_UPDATE_USER_ERROR.counter.increment() + throw e + } + Metrics.incrementObservabilityObjectActionCounter(type, Metrics.Action.UPDATE) return UpdateObservabilityObjectRequest(baseObjectData, type, objectId) } } diff --git a/src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectResponse.kt b/src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectResponse.kt index 6ccf5f3d8..a1f58531d 100644 --- a/src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectResponse.kt +++ b/src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectResponse.kt @@ -13,6 +13,7 @@ import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentParser.Token import org.opensearch.common.xcontent.XContentParserUtils import org.opensearch.observability.ObservabilityPlugin.Companion.LOG_PREFIX +import org.opensearch.observability.metrics.Metrics import org.opensearch.observability.model.BaseResponse import org.opensearch.observability.model.RestTag.OBJECT_ID_FIELD import org.opensearch.observability.util.logger @@ -58,7 +59,10 @@ internal class UpdateObservabilityObjectResponse( } } } - objectId ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent") + objectId ?: run { + Metrics.OBSERVABILITY_UPDATE_SYSTEM_ERROR.counter.increment() + throw IllegalArgumentException("$OBJECT_ID_FIELD field absent") + } return UpdateObservabilityObjectResponse(objectId) } } diff --git a/src/main/kotlin/org/opensearch/observability/metrics/Metrics.kt b/src/main/kotlin/org/opensearch/observability/metrics/Metrics.kt index 07b310e2e..584ca8300 100644 --- a/src/main/kotlin/org/opensearch/observability/metrics/Metrics.kt +++ b/src/main/kotlin/org/opensearch/observability/metrics/Metrics.kt @@ -7,6 +7,9 @@ package org.opensearch.observability.metrics import com.github.wnameless.json.unflattener.JsonUnflattener import org.json.JSONObject +import org.opensearch.observability.ObservabilityPlugin.Companion.LOG_PREFIX +import org.opensearch.observability.model.ObservabilityObjectType +import org.opensearch.observability.util.logger /** * Enum to hold all the metrics that need to be logged into _plugins/_observability/_local/stats API @@ -42,10 +45,63 @@ enum class Metrics(val metricName: String, val counter: Counter<*>) { "exception.internal_server_error", RollingCounter() ), + // ==== REST endpoint metrics ==== // + + // Observability counters + OBSERVABILITY_CREATE_TOTAL("observability.create.total", BasicCounter()), + OBSERVABILITY_CREATE_INTERVAL_COUNT("observability.create.count", RollingCounter()), + OBSERVABILITY_CREATE_USER_ERROR("observability.create.user_error", RollingCounter()), + OBSERVABILITY_CREATE_SYSTEM_ERROR("observability.create.system_error", RollingCounter()), + OBSERVABILITY_GET_TOTAL("observability.get.total", BasicCounter()), + OBSERVABILITY_GET_INTERVAL_COUNT("observability.get.count", RollingCounter()), + OBSERVABILITY_GET_USER_ERROR("observability.get.user_error", RollingCounter()), + OBSERVABILITY_GET_SYSTEM_ERROR("observability.get.system_error", RollingCounter()), + OBSERVABILITY_UPDATE_TOTAL("observability.update.total", BasicCounter()), + OBSERVABILITY_UPDATE_INTERVAL_COUNT("observability.update.count", RollingCounter()), + OBSERVABILITY_UPDATE_USER_ERROR("observability.update.user_error", RollingCounter()), + OBSERVABILITY_UPDATE_SYSTEM_ERROR("observability.update.system_error", RollingCounter()), + OBSERVABILITY_DELETE_TOTAL("observability.delete.total", BasicCounter()), + OBSERVABILITY_DELETE_INTERVAL_COUNT("observability.delete.count", RollingCounter()), + OBSERVABILITY_DELETE_USER_ERROR("observability.delete.user_error", RollingCounter()), + OBSERVABILITY_DELETE_SYSTEM_ERROR("observability.delete.system_error", RollingCounter()), + + // Per object type action counters, object type is only known for CREATE and UPDATE + NOTEBOOK_CREATE_TOTAL("notebook.create.total", BasicCounter()), + NOTEBOOK_CREATE_INTERVAL_COUNT("notebook.create.count", RollingCounter()), + NOTEBOOK_UPDATE_TOTAL("notebook.update.total", BasicCounter()), + NOTEBOOK_UPDATE_INTERVAL_COUNT("notebook.update.count", RollingCounter()), + + SAVED_QUERY_CREATE_TOTAL("saved_query.create.total", BasicCounter()), + SAVED_QUERY_CREATE_INTERVAL_COUNT("saved_query.create.count", RollingCounter()), + SAVED_QUERY_UPDATE_TOTAL("saved_query.update.total", BasicCounter()), + SAVED_QUERY_UPDATE_INTERVAL_COUNT("saved_query.update.count", RollingCounter()), + + SAVED_VISUALIZATION_CREATE_TOTAL("saved_visualization.create.total", BasicCounter()), + SAVED_VISUALIZATION_CREATE_INTERVAL_COUNT("saved_visualization.create.count", RollingCounter()), + SAVED_VISUALIZATION_UPDATE_TOTAL("saved_visualization.update.total", BasicCounter()), + SAVED_VISUALIZATION_UPDATE_INTERVAL_COUNT("saved_visualization.update.count", RollingCounter()), + + OPERATIONAL_PANEL_CREATE_TOTAL("operational_panel.create.total", BasicCounter()), + OPERATIONAL_PANEL_CREATE_INTERVAL_COUNT("operational_panel.create.count", RollingCounter()), + OPERATIONAL_PANEL_UPDATE_TOTAL("operational_panel.update.total", BasicCounter()), + OPERATIONAL_PANEL_UPDATE_INTERVAL_COUNT("operational_panel.update.count", RollingCounter()), + + APPLICATION_CREATE_TOTAL("application.create.total", BasicCounter()), + APPLICATION_CREATE_INTERVAL_COUNT("application.create.count", RollingCounter()), + APPLICATION_UPDATE_TOTAL("application.update.total", BasicCounter()), + APPLICATION_UPDATE_INTERVAL_COUNT("application.update.count", RollingCounter()), + + TIMESTAMP_CREATE_TOTAL("timestamp.create.total", BasicCounter()), + TIMESTAMP_CREATE_INTERVAL_COUNT("timestamp.create.count", RollingCounter()), + TIMESTAMP_UPDATE_TOTAL("timestamp.update.total", BasicCounter()), + TIMESTAMP_UPDATE_INTERVAL_COUNT("timestamp.update.count", RollingCounter()), + + // Permission errors OBSERVABILITY_SECURITY_PERMISSION_ERROR("security_permission_error", RollingCounter()), OBSERVABILITY_PERMISSION_USER_ERROR("permission_user_error", RollingCounter()); companion object { + private val log by logger(Metrics::class.java) private val values: Array = Metrics.values() /** @@ -75,5 +131,30 @@ enum class Metrics(val metricName: String, val counter: Counter<*>) { fun collectToFlattenedJSON(): String { return JsonUnflattener.unflatten(collectToJSON()) } + + /** + * Increment observability object action counter + * + * @param type object type + * @param action create or update + */ + fun incrementObservabilityObjectActionCounter(type: ObservabilityObjectType, action: Action) { + try { + Metrics.valueOf("${type.name}_${action.name}_TOTAL").counter.increment() + Metrics.valueOf("${type.name}_${action.name}_INTERVAL_COUNT").counter.increment() + } catch (e: IllegalArgumentException) { + log.warn("$LOG_PREFIX:IllegalArgumentException invalid type or action for counter metric", e) + } + } + } + + /** + * Metrics action for per object type counters. Type is only known for CREATE and UPDATE. + * + * @constructor Action + */ + enum class Action { + CREATE, + UPDATE, } } diff --git a/src/main/kotlin/org/opensearch/observability/model/SearchResults.kt b/src/main/kotlin/org/opensearch/observability/model/SearchResults.kt index 4ce5ae143..081699e2f 100644 --- a/src/main/kotlin/org/opensearch/observability/model/SearchResults.kt +++ b/src/main/kotlin/org/opensearch/observability/model/SearchResults.kt @@ -16,6 +16,7 @@ import org.opensearch.common.xcontent.ToXContent.Params import org.opensearch.common.xcontent.XContentBuilder import org.opensearch.common.xcontent.XContentParser import org.opensearch.common.xcontent.XContentParserUtils +import org.opensearch.observability.metrics.Metrics import org.opensearch.search.SearchHit internal abstract class SearchResults : BaseModel { @@ -127,7 +128,10 @@ internal abstract class SearchResults : BaseModel { } } } - objectList ?: throw IllegalArgumentException("$objectListFieldName field absent") + objectList ?: run { + Metrics.OBSERVABILITY_GET_SYSTEM_ERROR.counter.increment() + throw IllegalArgumentException("$objectListFieldName field absent") + } if (totalHits == 0L) { totalHits = objectList.size.toLong() } diff --git a/src/main/kotlin/org/opensearch/observability/resthandler/ObservabilityRestHandler.kt b/src/main/kotlin/org/opensearch/observability/resthandler/ObservabilityRestHandler.kt index c38a4b074..add6077a2 100644 --- a/src/main/kotlin/org/opensearch/observability/resthandler/ObservabilityRestHandler.kt +++ b/src/main/kotlin/org/opensearch/observability/resthandler/ObservabilityRestHandler.kt @@ -18,6 +18,7 @@ import org.opensearch.observability.action.ObservabilityActions import org.opensearch.observability.action.UpdateObservabilityObjectAction import org.opensearch.observability.action.UpdateObservabilityObjectRequest import org.opensearch.observability.index.ObservabilityQueryHelper +import org.opensearch.observability.metrics.Metrics import org.opensearch.observability.model.ObservabilityObjectType import org.opensearch.observability.model.RestTag.FROM_INDEX_FIELD import org.opensearch.observability.model.RestTag.MAX_ITEMS_FIELD @@ -200,10 +201,26 @@ internal class ObservabilityRestHandler : BaseRestHandler() { */ override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { return when (request.method()) { - POST -> executePostRequest(request, client) - PUT -> executePutRequest(request, client) - GET -> executeGetRequest(request, client) - DELETE -> executeDeleteRequest(request, client) + POST -> { + Metrics.OBSERVABILITY_CREATE_TOTAL.counter.increment() + Metrics.OBSERVABILITY_CREATE_INTERVAL_COUNT.counter.increment() + executePostRequest(request, client) + } + PUT -> { + Metrics.OBSERVABILITY_UPDATE_TOTAL.counter.increment() + Metrics.OBSERVABILITY_UPDATE_INTERVAL_COUNT.counter.increment() + executePutRequest(request, client) + } + GET -> { + Metrics.OBSERVABILITY_GET_TOTAL.counter.increment() + Metrics.OBSERVABILITY_GET_INTERVAL_COUNT.counter.increment() + executeGetRequest(request, client) + } + DELETE -> { + Metrics.OBSERVABILITY_DELETE_TOTAL.counter.increment() + Metrics.OBSERVABILITY_DELETE_INTERVAL_COUNT.counter.increment() + executeDeleteRequest(request, client) + } else -> RestChannelConsumer { it.sendResponse(BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, "${request.method()} is not allowed")) } diff --git a/src/test/kotlin/org/opensearch/observability/metrics/MetricsTests.kt b/src/test/kotlin/org/opensearch/observability/metrics/MetricsTests.kt new file mode 100644 index 000000000..805e9bc9a --- /dev/null +++ b/src/test/kotlin/org/opensearch/observability/metrics/MetricsTests.kt @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.observability.metrics + +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertDoesNotThrow +import org.opensearch.observability.model.ObservabilityObjectType + +internal class MetricsTests { + @Test + fun testInvalidArguments() { + assertDoesNotThrow { + Metrics.incrementObservabilityObjectActionCounter(ObservabilityObjectType.NONE, Metrics.Action.UPDATE) + } + } +}