Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 2.5] Add more metrics for backend plugin #1378

Merged
merged 1 commit into from
Jan 10, 2023
Merged

[Backport 2.5] Add more metrics for backend plugin #1378

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions src/main/kotlin/org/opensearch/observability/action/CreateObservabilityObjectRequest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import org.opensearch.common.xcontent.XContentParser
import org.opensearch.common.xcontent.XContentParserUtils
import org.opensearch.commons.utils.fieldIfNotNull
import org.opensearch.commons.utils.logger
import org.opensearch.observability.metrics.Metrics
import org.opensearch.observability.model.BaseObjectData
import org.opensearch.observability.model.ObservabilityObjectDataProperties
import org.opensearch.observability.model.ObservabilityObjectType
Expand Down Expand Up @@ -73,8 +74,14 @@ internal class CreateObservabilityObjectRequest : ActionRequest, ToXContentObjec
}
}
}
type ?: throw IllegalArgumentException("Object type field absent")
baseObjectData ?: throw IllegalArgumentException("Object data field absent")
try {
type ?: throw IllegalArgumentException("Object type field absent")
baseObjectData ?: throw IllegalArgumentException("Object data field absent")
} catch (e: IllegalArgumentException) {
Metrics.OBSERVABILITY_CREATE_USER_ERROR.counter.increment()
throw e
}
Metrics.incrementObservabilityObjectActionCounter(type, Metrics.Action.CREATE)
return CreateObservabilityObjectRequest(objectId, type, baseObjectData)
}
}
Expand Down
6 changes: 5 additions & 1 deletion src/main/kotlin/org/opensearch/observability/action/CreateObservabilityObjectResponse.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import org.opensearch.common.xcontent.XContentBuilder
import org.opensearch.common.xcontent.XContentParser
import org.opensearch.common.xcontent.XContentParserUtils
import org.opensearch.commons.utils.logger
import org.opensearch.observability.metrics.Metrics
import org.opensearch.observability.model.BaseResponse
import org.opensearch.observability.model.RestTag.OBJECT_ID_FIELD
import java.io.IOException
Expand Down Expand Up @@ -56,7 +57,10 @@ internal class CreateObservabilityObjectResponse : BaseResponse {
}
}
}
objectId ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
objectId ?: run {
Metrics.OBSERVABILITY_CREATE_SYSTEM_ERROR.counter.increment()
throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
}
return CreateObservabilityObjectResponse(objectId)
}
}
Expand Down
6 changes: 5 additions & 1 deletion src/main/kotlin/org/opensearch/observability/action/DeleteObservabilityObjectRequest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import org.opensearch.common.xcontent.XContentParserUtils
import org.opensearch.commons.utils.logger
import org.opensearch.commons.utils.stringList
import org.opensearch.observability.ObservabilityPlugin.Companion.LOG_PREFIX
import org.opensearch.observability.metrics.Metrics
import org.opensearch.observability.model.RestTag.OBJECT_ID_FIELD
import org.opensearch.observability.model.RestTag.OBJECT_ID_LIST_FIELD
import java.io.IOException
Expand Down Expand Up @@ -63,7 +64,10 @@ internal class DeleteObservabilityObjectRequest : ActionRequest, ToXContentObjec
}
}
}
objectIds ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
objectIds ?: run {
Metrics.OBSERVABILITY_DELETE_USER_ERROR.counter.increment()
throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
}
return DeleteObservabilityObjectRequest(objectIds)
}
}
Expand Down
6 changes: 5 additions & 1 deletion src/main/kotlin/org/opensearch/observability/action/DeleteObservabilityObjectResponse.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import org.opensearch.commons.utils.STRING_WRITER
import org.opensearch.commons.utils.enumReader
import org.opensearch.commons.utils.enumWriter
import org.opensearch.commons.utils.logger
import org.opensearch.observability.metrics.Metrics
import org.opensearch.observability.model.BaseResponse
import org.opensearch.observability.model.RestTag.DELETE_RESPONSE_LIST_TAG
import org.opensearch.rest.RestStatus
Expand Down Expand Up @@ -61,7 +62,10 @@ internal class DeleteObservabilityObjectResponse : BaseResponse {
}
}
}
objectIdToStatus ?: throw IllegalArgumentException("$DELETE_RESPONSE_LIST_TAG field absent")
objectIdToStatus ?: run {
Metrics.OBSERVABILITY_DELETE_SYSTEM_ERROR.counter.increment()
throw IllegalArgumentException("$DELETE_RESPONSE_LIST_TAG field absent")
}
return DeleteObservabilityObjectResponse(objectIdToStatus)
}

Expand Down
2 changes: 2 additions & 0 deletions src/main/kotlin/org/opensearch/observability/action/GetObservabilityObjectRequest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import org.opensearch.commons.utils.enumSet
import org.opensearch.commons.utils.fieldIfNotNull
import org.opensearch.commons.utils.logger
import org.opensearch.commons.utils.stringList
import org.opensearch.observability.metrics.Metrics
import org.opensearch.observability.model.ObservabilityObjectType
import org.opensearch.observability.model.RestTag.FILTER_PARAM_LIST_FIELD
import org.opensearch.observability.model.RestTag.FROM_INDEX_FIELD
Expand Down Expand Up @@ -188,6 +189,7 @@ class GetObservabilityObjectRequest : ActionRequest, ToXContentObject {
if (maxItems <= 0) {
validationException = ValidateActions.addValidationError("maxItems is not +ve", validationException)
}
validationException ?: Metrics.OBSERVABILITY_GET_USER_ERROR.counter.increment()
return validationException
}
}
13 changes: 10 additions & 3 deletions src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectRequest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import org.opensearch.common.xcontent.XContentParser
import org.opensearch.common.xcontent.XContentParserUtils
import org.opensearch.commons.utils.fieldIfNotNull
import org.opensearch.commons.utils.logger
import org.opensearch.observability.metrics.Metrics
import org.opensearch.observability.model.BaseObjectData
import org.opensearch.observability.model.ObservabilityObjectDataProperties
import org.opensearch.observability.model.ObservabilityObjectType
Expand Down Expand Up @@ -76,9 +77,15 @@ internal class UpdateObservabilityObjectRequest : ActionRequest, ToXContentObjec
}
}
}
objectId ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
type ?: throw IllegalArgumentException("Object type field absent")
baseObjectData ?: throw IllegalArgumentException("Object data field absent")
try {
objectId ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
type ?: throw IllegalArgumentException("Object type field absent")
baseObjectData ?: throw IllegalArgumentException("Object data field absent")
} catch (e: IllegalArgumentException) {
Metrics.OBSERVABILITY_UPDATE_USER_ERROR.counter.increment()
throw e
}
Metrics.incrementObservabilityObjectActionCounter(type, Metrics.Action.UPDATE)
return UpdateObservabilityObjectRequest(baseObjectData, type, objectId)
}
}
Expand Down
6 changes: 5 additions & 1 deletion src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectResponse.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import org.opensearch.common.xcontent.XContentParser
import org.opensearch.common.xcontent.XContentParser.Token
import org.opensearch.common.xcontent.XContentParserUtils
import org.opensearch.observability.ObservabilityPlugin.Companion.LOG_PREFIX
import org.opensearch.observability.metrics.Metrics
import org.opensearch.observability.model.BaseResponse
import org.opensearch.observability.model.RestTag.OBJECT_ID_FIELD
import org.opensearch.observability.util.logger
Expand Down Expand Up @@ -58,7 +59,10 @@ internal class UpdateObservabilityObjectResponse(
}
}
}
objectId ?: throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
objectId ?: run {
Metrics.OBSERVABILITY_UPDATE_SYSTEM_ERROR.counter.increment()
throw IllegalArgumentException("$OBJECT_ID_FIELD field absent")
}
return UpdateObservabilityObjectResponse(objectId)
}
}
Expand Down
81 changes: 81 additions & 0 deletions src/main/kotlin/org/opensearch/observability/metrics/Metrics.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ package org.opensearch.observability.metrics

import com.github.wnameless.json.unflattener.JsonUnflattener
import org.json.JSONObject
import org.opensearch.observability.ObservabilityPlugin.Companion.LOG_PREFIX
import org.opensearch.observability.model.ObservabilityObjectType
import org.opensearch.observability.util.logger

/**
* Enum to hold all the metrics that need to be logged into _plugins/_observability/_local/stats API
Expand Down Expand Up @@ -42,10 +45,63 @@ enum class Metrics(val metricName: String, val counter: Counter<*>) {
"exception.internal_server_error", RollingCounter()
),

// ==== REST endpoint metrics ==== //

// Observability counters
OBSERVABILITY_CREATE_TOTAL("observability.create.total", BasicCounter()),
OBSERVABILITY_CREATE_INTERVAL_COUNT("observability.create.count", RollingCounter()),
OBSERVABILITY_CREATE_USER_ERROR("observability.create.user_error", RollingCounter()),
OBSERVABILITY_CREATE_SYSTEM_ERROR("observability.create.system_error", RollingCounter()),
OBSERVABILITY_GET_TOTAL("observability.get.total", BasicCounter()),
OBSERVABILITY_GET_INTERVAL_COUNT("observability.get.count", RollingCounter()),
OBSERVABILITY_GET_USER_ERROR("observability.get.user_error", RollingCounter()),
OBSERVABILITY_GET_SYSTEM_ERROR("observability.get.system_error", RollingCounter()),
OBSERVABILITY_UPDATE_TOTAL("observability.update.total", BasicCounter()),
OBSERVABILITY_UPDATE_INTERVAL_COUNT("observability.update.count", RollingCounter()),
OBSERVABILITY_UPDATE_USER_ERROR("observability.update.user_error", RollingCounter()),
OBSERVABILITY_UPDATE_SYSTEM_ERROR("observability.update.system_error", RollingCounter()),
OBSERVABILITY_DELETE_TOTAL("observability.delete.total", BasicCounter()),
OBSERVABILITY_DELETE_INTERVAL_COUNT("observability.delete.count", RollingCounter()),
OBSERVABILITY_DELETE_USER_ERROR("observability.delete.user_error", RollingCounter()),
OBSERVABILITY_DELETE_SYSTEM_ERROR("observability.delete.system_error", RollingCounter()),

// Per object type action counters, object type is only known for CREATE and UPDATE
NOTEBOOK_CREATE_TOTAL("notebook.create.total", BasicCounter()),
NOTEBOOK_CREATE_INTERVAL_COUNT("notebook.create.count", RollingCounter()),
NOTEBOOK_UPDATE_TOTAL("notebook.update.total", BasicCounter()),
NOTEBOOK_UPDATE_INTERVAL_COUNT("notebook.update.count", RollingCounter()),

SAVED_QUERY_CREATE_TOTAL("saved_query.create.total", BasicCounter()),
SAVED_QUERY_CREATE_INTERVAL_COUNT("saved_query.create.count", RollingCounter()),
SAVED_QUERY_UPDATE_TOTAL("saved_query.update.total", BasicCounter()),
SAVED_QUERY_UPDATE_INTERVAL_COUNT("saved_query.update.count", RollingCounter()),

SAVED_VISUALIZATION_CREATE_TOTAL("saved_visualization.create.total", BasicCounter()),
SAVED_VISUALIZATION_CREATE_INTERVAL_COUNT("saved_visualization.create.count", RollingCounter()),
SAVED_VISUALIZATION_UPDATE_TOTAL("saved_visualization.update.total", BasicCounter()),
SAVED_VISUALIZATION_UPDATE_INTERVAL_COUNT("saved_visualization.update.count", RollingCounter()),

OPERATIONAL_PANEL_CREATE_TOTAL("operational_panel.create.total", BasicCounter()),
OPERATIONAL_PANEL_CREATE_INTERVAL_COUNT("operational_panel.create.count", RollingCounter()),
OPERATIONAL_PANEL_UPDATE_TOTAL("operational_panel.update.total", BasicCounter()),
OPERATIONAL_PANEL_UPDATE_INTERVAL_COUNT("operational_panel.update.count", RollingCounter()),

APPLICATION_CREATE_TOTAL("application.create.total", BasicCounter()),
APPLICATION_CREATE_INTERVAL_COUNT("application.create.count", RollingCounter()),
APPLICATION_UPDATE_TOTAL("application.update.total", BasicCounter()),
APPLICATION_UPDATE_INTERVAL_COUNT("application.update.count", RollingCounter()),

TIMESTAMP_CREATE_TOTAL("timestamp.create.total", BasicCounter()),
TIMESTAMP_CREATE_INTERVAL_COUNT("timestamp.create.count", RollingCounter()),
TIMESTAMP_UPDATE_TOTAL("timestamp.update.total", BasicCounter()),
TIMESTAMP_UPDATE_INTERVAL_COUNT("timestamp.update.count", RollingCounter()),

// Permission errors
OBSERVABILITY_SECURITY_PERMISSION_ERROR("security_permission_error", RollingCounter()),
OBSERVABILITY_PERMISSION_USER_ERROR("permission_user_error", RollingCounter());

companion object {
private val log by logger(Metrics::class.java)
private val values: Array<Metrics> = Metrics.values()

/**
Expand Down Expand Up @@ -75,5 +131,30 @@ enum class Metrics(val metricName: String, val counter: Counter<*>) {
fun collectToFlattenedJSON(): String {
return JsonUnflattener.unflatten(collectToJSON())
}

/**
* Increment observability object action counter
*
* @param type object type
* @param action create or update
*/
fun incrementObservabilityObjectActionCounter(type: ObservabilityObjectType, action: Action) {
try {
Metrics.valueOf("${type.name}_${action.name}_TOTAL").counter.increment()
Metrics.valueOf("${type.name}_${action.name}_INTERVAL_COUNT").counter.increment()
} catch (e: IllegalArgumentException) {
log.warn("$LOG_PREFIX:IllegalArgumentException invalid type or action for counter metric", e)
}
}
}

/**
* Metrics action for per object type counters. Type is only known for CREATE and UPDATE.
*
* @constructor Action
*/
enum class Action {
CREATE,
UPDATE,
}
}
6 changes: 5 additions & 1 deletion src/main/kotlin/org/opensearch/observability/model/SearchResults.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import org.opensearch.common.xcontent.ToXContent.Params
import org.opensearch.common.xcontent.XContentBuilder
import org.opensearch.common.xcontent.XContentParser
import org.opensearch.common.xcontent.XContentParserUtils
import org.opensearch.observability.metrics.Metrics
import org.opensearch.search.SearchHit

internal abstract class SearchResults<ItemClass : BaseModel> : BaseModel {
Expand Down Expand Up @@ -127,7 +128,10 @@ internal abstract class SearchResults<ItemClass : BaseModel> : BaseModel {
}
}
}
objectList ?: throw IllegalArgumentException("$objectListFieldName field absent")
objectList ?: run {
Metrics.OBSERVABILITY_GET_SYSTEM_ERROR.counter.increment()
throw IllegalArgumentException("$objectListFieldName field absent")
}
if (totalHits == 0L) {
totalHits = objectList.size.toLong()
}
Expand Down
25 changes: 21 additions & 4 deletions src/main/kotlin/org/opensearch/observability/resthandler/ObservabilityRestHandler.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import org.opensearch.observability.action.ObservabilityActions
import org.opensearch.observability.action.UpdateObservabilityObjectAction
import org.opensearch.observability.action.UpdateObservabilityObjectRequest
import org.opensearch.observability.index.ObservabilityQueryHelper
import org.opensearch.observability.metrics.Metrics
import org.opensearch.observability.model.ObservabilityObjectType
import org.opensearch.observability.model.RestTag.FROM_INDEX_FIELD
import org.opensearch.observability.model.RestTag.MAX_ITEMS_FIELD
Expand Down Expand Up @@ -200,10 +201,26 @@ internal class ObservabilityRestHandler : BaseRestHandler() {
*/
override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer {
return when (request.method()) {
POST -> executePostRequest(request, client)
PUT -> executePutRequest(request, client)
GET -> executeGetRequest(request, client)
DELETE -> executeDeleteRequest(request, client)
POST -> {
Metrics.OBSERVABILITY_CREATE_TOTAL.counter.increment()
Metrics.OBSERVABILITY_CREATE_INTERVAL_COUNT.counter.increment()
executePostRequest(request, client)
}
PUT -> {
Metrics.OBSERVABILITY_UPDATE_TOTAL.counter.increment()
Metrics.OBSERVABILITY_UPDATE_INTERVAL_COUNT.counter.increment()
executePutRequest(request, client)
}
GET -> {
Metrics.OBSERVABILITY_GET_TOTAL.counter.increment()
Metrics.OBSERVABILITY_GET_INTERVAL_COUNT.counter.increment()
executeGetRequest(request, client)
}
DELETE -> {
Metrics.OBSERVABILITY_DELETE_TOTAL.counter.increment()
Metrics.OBSERVABILITY_DELETE_INTERVAL_COUNT.counter.increment()
executeDeleteRequest(request, client)
}
else -> RestChannelConsumer {
it.sendResponse(BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, "${request.method()} is not allowed"))
}
Expand Down
19 changes: 19 additions & 0 deletions src/test/kotlin/org/opensearch/observability/metrics/MetricsTests.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.observability.metrics

import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertDoesNotThrow
import org.opensearch.observability.model.ObservabilityObjectType

internal class MetricsTests {
@Test
fun testInvalidArguments() {
assertDoesNotThrow {
Metrics.incrementObservabilityObjectActionCounter(ObservabilityObjectType.NONE, Metrics.Action.UPDATE)
}
}
}