Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fixing metrics #1194

Merged
merged 7 commits into from
Aug 10, 2023
Merged

fixing metrics #1194

Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
52abf14
fixing metrics
dhrubo-os Aug 9, 2023
fed820e
addressing comments
dhrubo-os Aug 9, 2023
c4b3c35
addressing comments
dhrubo-os Aug 9, 2023
b27d655
updating test
dhrubo-os Aug 9, 2023
d236cd6
added IllegalArgumentException in the if statement
dhrubo-os Aug 10, 2023
de0a996
addressing comments
dhrubo-os Aug 10, 2023
6e08ffa
fixing spotless
dhrubo-os Aug 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions plugin/src/main/java/org/opensearch/ml/model/MLModelManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,9 +318,10 @@ private void uploadMLModelMeta(MLRegisterModelMetaInput mlRegisterModelMetaInput
* @param mlTask ML task
*/
public void registerMLModel(MLRegisterModelInput registerModelInput, MLTask mlTask) {
mlStats.getStat(MLNodeLevelStat.ML_NODE_TOTAL_REQUEST_COUNT).increment();

checkAndAddRunningTask(mlTask, maxRegisterTasksPerNode);
try {
mlStats.getStat(MLNodeLevelStat.ML_NODE_TOTAL_REQUEST_COUNT).increment();
mlStats.getStat(MLNodeLevelStat.ML_NODE_EXECUTING_TASK_COUNT).increment();
mlStats.createCounterStatIfAbsent(mlTask.getFunctionName(), REGISTER, ML_ACTION_REQUEST_COUNT).increment();

Expand Down Expand Up @@ -380,7 +381,6 @@ public void registerMLModel(MLRegisterModelInput registerModelInput, MLTask mlTa
handleException(registerModelInput.getFunctionName(), mlTask.getTaskId(), e);
}
} catch (Exception e) {
mlStats.createCounterStatIfAbsent(mlTask.getFunctionName(), REGISTER, MLActionLevelStat.ML_ACTION_FAILURE_COUNT).increment();
handleException(registerModelInput.getFunctionName(), mlTask.getTaskId(), e);
} finally {
mlStats.getStat(MLNodeLevelStat.ML_NODE_EXECUTING_TASK_COUNT).increment();
Expand All @@ -392,9 +392,9 @@ private void indexRemoteModel(MLRegisterModelInput registerModelInput, MLTask ml
FunctionName functionName = mlTask.getFunctionName();
try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) {
mlStats.getStat(MLNodeLevelStat.ML_NODE_TOTAL_REQUEST_COUNT).increment();

mlStats.createCounterStatIfAbsent(functionName, REGISTER, ML_ACTION_REQUEST_COUNT).increment();
mlStats.getStat(MLNodeLevelStat.ML_NODE_EXECUTING_TASK_COUNT).increment();

String modelName = registerModelInput.getModelName();
String version = modelVersion == null ? registerModelInput.getVersion() : modelVersion;
Instant now = Instant.now();
Expand Down Expand Up @@ -462,7 +462,6 @@ private void registerModelFromUrl(MLRegisterModelInput registerModelInput, MLTas
FunctionName functionName = mlTask.getFunctionName();
try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) {
mlStats.getStat(MLNodeLevelStat.ML_NODE_TOTAL_REQUEST_COUNT).increment();

mlStats.createCounterStatIfAbsent(functionName, REGISTER, ML_ACTION_REQUEST_COUNT).increment();
mlStats.getStat(MLNodeLevelStat.ML_NODE_EXECUTING_TASK_COUNT).increment();
String modelName = registerModelInput.getModelName();
Expand Down Expand Up @@ -690,6 +689,7 @@ private void deleteModel(String modelId) {

private void handleException(FunctionName functionName, String taskId, Exception e) {
mlStats.createCounterStatIfAbsent(functionName, REGISTER, MLActionLevelStat.ML_ACTION_FAILURE_COUNT).increment();
mlStats.getStat(MLNodeLevelStat.ML_NODE_TOTAL_FAILURE_COUNT).increment();
Map<String, Object> updated = ImmutableMap.of(ERROR_FIELD, MLExceptionUtils.getRootCauseMessage(e), STATE_FIELD, FAILED);
mlTaskManager.updateMLTask(taskId, updated, TIMEOUT_IN_MILLIS, true);
}
Expand All @@ -713,6 +713,7 @@ public void deployModel(
ActionListener<String> listener
) {
mlStats.createCounterStatIfAbsent(functionName, ActionName.DEPLOY, ML_ACTION_REQUEST_COUNT).increment();
mlStats.getStat(MLNodeLevelStat.ML_NODE_TOTAL_REQUEST_COUNT).increment();
List<String> workerNodes = mlTask.getWorkerNodes();
if (modelCacheHelper.isModelDeployed(modelId)) {
if (workerNodes != null && workerNodes.size() > 0) {
Expand Down Expand Up @@ -836,6 +837,7 @@ public void deployModel(

private void handleDeployModelException(String modelId, FunctionName functionName, ActionListener<String> listener, Exception e) {
mlStats.createCounterStatIfAbsent(functionName, ActionName.DEPLOY, MLActionLevelStat.ML_ACTION_FAILURE_COUNT).increment();
mlStats.getStat(MLNodeLevelStat.ML_NODE_TOTAL_FAILURE_COUNT).increment();
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should only track the real internal server failure. For example, some exception are expected (which means that's not a code bug) like illegal input parameter. You can refer to other place

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agree, addressed.

removeModel(modelId);
listener.onFailure(e);
}
Expand Down Expand Up @@ -1045,6 +1047,7 @@ public synchronized Map<String, String> undeployModel(String[] modelIds) {
if (modelCacheHelper.isModelDeployed(modelId)) {
modelUndeployStatus.put(modelId, UNDEPLOYED);
mlStats.getStat(MLNodeLevelStat.ML_NODE_TOTAL_MODEL_COUNT).decrement();
mlStats.getStat(MLNodeLevelStat.ML_NODE_TOTAL_REQUEST_COUNT).increment();
mlStats
.createCounterStatIfAbsent(getModelFunctionName(modelId), ActionName.UNDEPLOY, ML_ACTION_REQUEST_COUNT)
.increment();
Expand Down