[Collector] Adding Latency and Error service metrics (#442)

* [Collector] Adding Latency and Error service metrics Signed-off-by: Khushboo Rajput <[email protected]> * Removing fastdoubleparser-LICENSE.tx * Removing fastdoubleparser-NOTICE.txt Signed-off-by: Khushboo Rajput <[email protected]> --------- Signed-off-by: Khushboo Rajput <[email protected]>
opensearch-project · May 17, 2023 · 9524c08 · 9524c08
1 parent f65f8f9
commit 9524c08
Show file tree

Hide file tree

Showing 53 changed files with 146 additions and 106 deletions.
diff --git a/licenses/fastdoubleparser-0.8.0.jar.sha1 b/licenses/fastdoubleparser-0.8.0.jar.sha1
@@ -0,0 +1 @@
+85c25540369921659556ead85e02c99ef0d24280
diff --git a/licenses/fastdoubleparser-LICENSE.txt b/licenses/fastdoubleparser-LICENSE.txt
diff --git a/licenses/fastdoubleparser-NOTICE.txt b/licenses/fastdoubleparser-NOTICE.txt
diff --git a/licenses/jackson-annotations-2.14.1.jar.sha1 b/licenses/jackson-annotations-2.14.1.jar.sha1
diff --git a/licenses/jackson-annotations-2.15.0.jar.sha1 b/licenses/jackson-annotations-2.15.0.jar.sha1
@@ -0,0 +1 @@
+89b0fd554928425a776a6e97ed010034312af21d
diff --git a/licenses/jackson-databind-2.14.1.jar.sha1 b/licenses/jackson-databind-2.14.1.jar.sha1
diff --git a/licenses/jackson-databind-2.15.0.jar.sha1 b/licenses/jackson-databind-2.15.0.jar.sha1
@@ -0,0 +1 @@
+0d41caa3a4e9f85382702a059a65c512f85ac230
diff --git a/licenses/jackson-module-paranamer-2.14.1.jar.sha1 b/licenses/jackson-module-paranamer-2.14.1.jar.sha1
diff --git a/licenses/jackson-module-paranamer-2.15.0.jar.sha1 b/licenses/jackson-module-paranamer-2.15.0.jar.sha1
@@ -0,0 +1 @@
+1d92b098577555a9a9e6c8783ce03ae24d49c9d2
diff --git a/licenses/netty-buffer-4.1.90.Final.jar.sha1 b/licenses/netty-buffer-4.1.90.Final.jar.sha1
diff --git a/licenses/netty-buffer-4.1.91.Final.jar.sha1 b/licenses/netty-buffer-4.1.91.Final.jar.sha1
@@ -0,0 +1 @@
+d8f180291c3501e931968ca7e40ae0323c4eacee
diff --git a/licenses/netty-codec-4.1.90.Final.jar.sha1 b/licenses/netty-codec-4.1.90.Final.jar.sha1
diff --git a/licenses/netty-codec-4.1.91.Final.jar.sha1 b/licenses/netty-codec-4.1.91.Final.jar.sha1
@@ -0,0 +1 @@
+3044b8e325e33f72c96ac1ea51dda85bef090cc0
diff --git a/licenses/netty-codec-http-4.1.90.Final.jar.sha1 b/licenses/netty-codec-http-4.1.90.Final.jar.sha1
diff --git a/licenses/netty-codec-http-4.1.91.Final.jar.sha1 b/licenses/netty-codec-http-4.1.91.Final.jar.sha1
@@ -0,0 +1 @@
+4519d2ff470941f0086214b19c9acf992868112f
diff --git a/licenses/netty-codec-http2-4.1.90.Final.jar.sha1 b/licenses/netty-codec-http2-4.1.90.Final.jar.sha1
diff --git a/licenses/netty-codec-http2-4.1.91.Final.jar.sha1 b/licenses/netty-codec-http2-4.1.91.Final.jar.sha1
@@ -0,0 +1 @@
+4ee7027e1653c6ee3f843191e0d932f29e8e14e1
diff --git a/licenses/netty-codec-socks-4.1.90.Final.jar.sha1 b/licenses/netty-codec-socks-4.1.90.Final.jar.sha1
diff --git a/licenses/netty-codec-socks-4.1.91.Final.jar.sha1 b/licenses/netty-codec-socks-4.1.91.Final.jar.sha1
@@ -0,0 +1 @@
+8f0a52677da411a8ab762c426d723c7f54471504
diff --git a/licenses/netty-common-4.1.90.Final.jar.sha1 b/licenses/netty-common-4.1.90.Final.jar.sha1
diff --git a/licenses/netty-common-4.1.91.Final.jar.sha1 b/licenses/netty-common-4.1.91.Final.jar.sha1
@@ -0,0 +1 @@
+93e5056462a242718e7689d81180d125c79d7723
diff --git a/licenses/netty-handler-4.1.90.Final.jar.sha1 b/licenses/netty-handler-4.1.90.Final.jar.sha1
diff --git a/licenses/netty-handler-4.1.91.Final.jar.sha1 b/licenses/netty-handler-4.1.91.Final.jar.sha1
@@ -0,0 +1 @@
+444cf41e4fe28c47ffebba5e77b9458a12f938a1
diff --git a/licenses/netty-handler-proxy-4.1.90.Final.jar.sha1 b/licenses/netty-handler-proxy-4.1.90.Final.jar.sha1
diff --git a/licenses/netty-handler-proxy-4.1.91.Final.jar.sha1 b/licenses/netty-handler-proxy-4.1.91.Final.jar.sha1
@@ -0,0 +1 @@
+2e3e57eae1a61e4e5f558e39619186fec6c424d3
diff --git a/licenses/netty-resolver-4.1.90.Final.jar.sha1 b/licenses/netty-resolver-4.1.90.Final.jar.sha1
diff --git a/licenses/netty-resolver-4.1.91.Final.jar.sha1 b/licenses/netty-resolver-4.1.91.Final.jar.sha1
@@ -0,0 +1 @@
+04725d117d4b71ef0e743aa79062489b45472b26
diff --git a/licenses/netty-transport-4.1.90.Final.jar.sha1 b/licenses/netty-transport-4.1.90.Final.jar.sha1
diff --git a/licenses/netty-transport-4.1.91.Final.jar.sha1 b/licenses/netty-transport-4.1.91.Final.jar.sha1
@@ -0,0 +1 @@
+c2f6bd7143194ca842b535546a405c06aa993934
diff --git a/licenses/netty-transport-native-unix-common-4.1.90.Final.jar.sha1 b/licenses/netty-transport-native-unix-common-4.1.90.Final.jar.sha1
diff --git a/licenses/netty-transport-native-unix-common-4.1.91.Final.jar.sha1 b/licenses/netty-transport-native-unix-common-4.1.91.Final.jar.sha1
@@ -0,0 +1 @@
+80990b5885b8b67be096d7090cba18f05c67120e
diff --git a/licenses/performanceanalyzer-rca-3.0.0.0-SNAPSHOT.jar.sha1 b/licenses/performanceanalyzer-rca-3.0.0.0-SNAPSHOT.jar.sha1
@@ -1 +1 @@
-763fed1fb35a016fdba53d3c66bd611d823090c0
+1328e960851fa7948368c94fbd092216b2df752b
diff --git a/licenses/protobuf-java-3.21.12.jar.sha1 b/licenses/protobuf-java-3.21.12.jar.sha1
diff --git a/.../java/org/opensearch/performanceanalyzer/collectors/AdmissionControlMetricsCollector.java b/.../java/org/opensearch/performanceanalyzer/collectors/AdmissionControlMetricsCollector.java
@@ -60,7 +60,7 @@ public void collectMetrics(long startTime) {
             return;
         }
 
-        long startTimeMillis = System.currentTimeMillis();
+        long mCurrT = System.currentTimeMillis();
         try {
 
             Method getAdmissionController =
@@ -110,17 +110,15 @@ public void collectMetrics(long startTime) {
             PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
                     WriterMetrics.ADMISSION_CONTROL_COLLECTOR_EXECUTION_TIME,
                     "",
-                    System.currentTimeMillis() - startTimeMillis);
+                    System.currentTimeMillis() - mCurrT);
 
         } catch (Exception ex) {
-            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
-                    ExceptionsAndErrors.ADMISSION_CONTROL_COLLECTOR_ERROR,
-                    getCollectorName(),
-                    System.currentTimeMillis() - startTimeMillis);
             LOG.debug(
                     "Exception in collecting AdmissionControl Metrics: {} for startTime {}",
                     ex::toString,
                     () -> startTime);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.ADMISSION_CONTROL_COLLECTOR_ERROR, "", 1);
         }
     }
 
@@ -183,6 +181,8 @@ private boolean canLoadAdmissionControllerClasses() {
                     Class.forName(ADMISSION_CONTROL_SERVICE, false, admissionControlClassLoader);
         } catch (Exception e) {
             LOG.debug("Failed to load AdmissionControllerService classes : {}", e::toString);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.ADMISSION_CONTROL_COLLECTOR_ERROR, "", 1);
             return false;
         }
         return true;

diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/CacheConfigMetricsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/CacheConfigMetricsCollector.java
@@ -18,11 +18,13 @@
 import org.opensearch.common.cache.Cache;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.performanceanalyzer.OpenSearchResources;
+import org.opensearch.performanceanalyzer.PerformanceAnalyzerApp;
 import org.opensearch.performanceanalyzer.metrics.AllMetrics.CacheConfigDimension;
 import org.opensearch.performanceanalyzer.metrics.AllMetrics.CacheConfigValue;
 import org.opensearch.performanceanalyzer.metrics.MetricsConfiguration;
 import org.opensearch.performanceanalyzer.metrics.MetricsProcessor;
 import org.opensearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
+import org.opensearch.performanceanalyzer.rca.framework.metrics.WriterMetrics;
 
 /*
  * Unlike Cache Hit, Miss, Eviction Count and Size, which is tracked on a per shard basis,
@@ -54,6 +56,7 @@ public void collectMetrics(long startTime) {
             return;
         }
 
+        long mCurrT = System.currentTimeMillis();
         value.setLength(0);
         value.append(PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds());
         // This is for backward compatibility. Core OpenSearch may or may not emit maxWeight metric.
@@ -117,6 +120,10 @@ public void collectMetrics(long startTime) {
         value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor)
                 .append(shardRequestCacheMaxSizeStatus.serialize());
         saveMetricValues(value.toString(), startTime);
+        PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
+                WriterMetrics.CACHE_CONFIG_METRICS_COLLECTOR_EXECUTION_TIME,
+                "",
+                System.currentTimeMillis() - mCurrT);
     }
 
     @Override

diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/CircuitBreakerCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/CircuitBreakerCollector.java
@@ -25,8 +25,7 @@ public class CircuitBreakerCollector extends PerformanceAnalyzerMetricsCollector
     public static final int SAMPLING_TIME_INTERVAL =
             MetricsConfiguration.CONFIG_MAP.get(CircuitBreakerCollector.class).samplingInterval;
 
-    private static final Logger LOG =
-            LogManager.getLogger(CircuitBreakerCollector.class);
+    private static final Logger LOG = LogManager.getLogger(CircuitBreakerCollector.class);
     private static final int KEYS_PATH_LENGTH = 0;
     private StringBuilder value;
 
@@ -54,10 +53,10 @@ public void collectMetrics(long startTime) {
                 value.append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor)
                         .append(
                                 new CircuitBreakerStatus(
-                                        stats.getName(),
-                                        stats.getEstimated(),
-                                        stats.getTrippedCount(),
-                                        stats.getLimit())
+                                                stats.getName(),
+                                                stats.getEstimated(),
+                                                stats.getTrippedCount(),
+                                                stats.getLimit())
                                         .serialize());
             }
 
@@ -69,14 +68,13 @@ public void collectMetrics(long startTime) {
                     System.currentTimeMillis() - mCurrT);
 
         } catch (Exception ex) {
-            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
-                    ExceptionsAndErrors.CIRCUIT_BREAKER_COLLECTOR_ERROR, "", 1);
             LOG.debug(
                     "Exception in Collecting CircuitBreaker Metrics: {} for startTime {}",
                     () -> ex.toString(),
                     () -> startTime);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.CIRCUIT_BREAKER_COLLECTOR_ERROR, "", 1);
         }
-
     }
 
     @Override

diff --git a/...va/org/opensearch/performanceanalyzer/collectors/ClusterApplierServiceStatsCollector.java b/...va/org/opensearch/performanceanalyzer/collectors/ClusterApplierServiceStatsCollector.java
@@ -104,12 +104,12 @@ public void collectMetrics(long startTime) {
                     "",
                     System.currentTimeMillis() - mCurrT);
         } catch (Exception ex) {
-            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
-                    ExceptionsAndErrors.CLUSTER_APPLIER_SERVICE_STATS_COLLECTOR_ERROR, "", 1);
             LOG.debug(
                     "Exception in Collecting Cluster Applier Service Metrics: {} for startTime {}",
                     () -> ex.toString(),
                     () -> startTime);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.CLUSTER_APPLIER_SERVICE_STATS_COLLECTOR_ERROR, "", 1);
         }
     }
 

diff --git a/...java/org/opensearch/performanceanalyzer/collectors/ClusterManagerServiceEventMetrics.java b/...java/org/opensearch/performanceanalyzer/collectors/ClusterManagerServiceEventMetrics.java
@@ -27,6 +27,7 @@
 import org.opensearch.performanceanalyzer.metrics.MetricsProcessor;
 import org.opensearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
 import org.opensearch.performanceanalyzer.metrics.ThreadIDUtil;
+import org.opensearch.performanceanalyzer.rca.framework.metrics.ExceptionsAndErrors;
 import org.opensearch.performanceanalyzer.rca.framework.metrics.WriterMetrics;
 
 @SuppressWarnings("unchecked")
@@ -83,6 +84,8 @@ public void collectMetrics(long startTime) {
                 return;
             }
 
+            long mCurrT = System.currentTimeMillis();
+
             value.setLength(0);
             Queue<Runnable> current = getClusterManagerServiceCurrentQueue();
 
@@ -145,19 +148,24 @@ public void collectMetrics(long startTime) {
                             PerformanceAnalyzerMetrics.START_FILE_NAME);
 
                     value.setLength(0);
+                    PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
+                            WriterMetrics
+                                    .CLUSTER_MANAGER_SERVICE_EVENTS_METRICS_COLLECTOR_EXECUTION_TIME,
+                            "",
+                            System.currentTimeMillis() - mCurrT);
                 }
             } else {
                 generateFinishMetrics(startTime);
             }
             LOG.debug(() -> "Successfully collected ClusterManager Event Metrics.");
         } catch (Exception ex) {
-            PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
-                    WriterMetrics.CLUSTER_MANAGER_METRICS_ERROR, "", 1);
             LOG.debug(
                     "Exception in Collecting ClusterManager Metrics: {} for startTime {} with ExceptionCode: {}",
                     () -> ex.toString(),
                     () -> startTime,
                     () -> StatExceptionCode.CLUSTER_MANAGER_METRICS_ERROR.toString());
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.CLUSTER_MANAGER_METRICS_ERROR, "", 1);
         }
     }
 
@@ -235,8 +243,8 @@ Queue<Runnable> getClusterManagerServiceCurrentQueue() throws Exception {
                                         getPrioritizedTPExecutorCurrentField()
                                                 .get(prioritizedOpenSearchThreadPoolExecutor);
                     } else {
-                        PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
-                                WriterMetrics.CLUSTER_MANAGER_NODE_NOT_UP, "", 1);
+                        PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                                ExceptionsAndErrors.CLUSTER_MANAGER_NODE_NOT_UP, "", 1);
                     }
                 }
             }

diff --git a/...main/java/org/opensearch/performanceanalyzer/collectors/ClusterManagerServiceMetrics.java b/...main/java/org/opensearch/performanceanalyzer/collectors/ClusterManagerServiceMetrics.java
@@ -19,6 +19,7 @@
 import org.opensearch.performanceanalyzer.metrics.MetricsConfiguration;
 import org.opensearch.performanceanalyzer.metrics.MetricsProcessor;
 import org.opensearch.performanceanalyzer.metrics.PerformanceAnalyzerMetrics;
+import org.opensearch.performanceanalyzer.rca.framework.metrics.ExceptionsAndErrors;
 import org.opensearch.performanceanalyzer.rca.framework.metrics.WriterMetrics;
 
 @SuppressWarnings("unchecked")
@@ -57,6 +58,7 @@ public void collectMetrics(long startTime) {
                 return;
             }
 
+            long mCurrT = System.currentTimeMillis();
             /*
             pendingTasks API returns object of PendingClusterTask which contains insertOrder, priority, source, timeInQueue.
                 Example :
@@ -98,13 +100,17 @@ public void collectMetrics(long startTime) {
                     startTime,
                     PerformanceAnalyzerMetrics.CLUSTER_MANAGER_CURRENT,
                     PerformanceAnalyzerMetrics.CLUSTER_MANAGER_META_DATA);
-        } catch (Exception ex) {
             PerformanceAnalyzerApp.WRITER_METRICS_AGGREGATOR.updateStat(
-                    WriterMetrics.CLUSTER_MANAGER_METRICS_ERROR, "", 1);
+                    WriterMetrics.CLUSTER_MANAGER_SERVICE_METRICS_COLLECTOR_EXECUTION_TIME,
+                    "",
+                    System.currentTimeMillis() - mCurrT);
+        } catch (Exception ex) {
             LOG.debug(
                     "Exception in Collecting ClusterManager Metrics: {} for startTime {}",
                     () -> ex.toString(),
                     () -> startTime);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.CLUSTER_MANAGER_METRICS_ERROR, "", 1);
         }
     }
 

diff --git a/...g/opensearch/performanceanalyzer/collectors/ClusterManagerThrottlingMetricsCollector.java b/...g/opensearch/performanceanalyzer/collectors/ClusterManagerThrottlingMetricsCollector.java
@@ -85,12 +85,12 @@ public void collectMetrics(long startTime) {
                     System.currentTimeMillis() - mCurrT);
 
         } catch (Exception ex) {
-            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
-                    ExceptionsAndErrors.CLUSTER_MANAGER_THROTTLING_COLLECTOR_ERROR, "", 1);
             LOG.debug(
                     "Exception in Collecting ClusterManager Throttling Metrics: {} for startTime {}",
                     () -> ex.toString(),
                     () -> startTime);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.CLUSTER_MANAGER_THROTTLING_COLLECTOR_ERROR, "", 1);
         }
     }
 

diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/ElectionTermCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/ElectionTermCollector.java
@@ -80,14 +80,12 @@ public void collectMetrics(long startTime) {
                     System.currentTimeMillis() - mCurrT);
 
         } catch (Exception ex) {
-            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
-                    ExceptionsAndErrors.ELECTION_TERM_COLLECTOR_ERROR,
-                    "",
-                    System.currentTimeMillis() - mCurrT);
             LOG.debug(
                     "Exception in Collecting Election term Metrics: {} for startTime {}",
                     () -> ex.toString(),
                     () -> startTime);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.ELECTION_TERM_COLLECTOR_ERROR, "", 1);
         }
     }
 

diff --git a/...in/java/org/opensearch/performanceanalyzer/collectors/FaultDetectionMetricsCollector.java b/...in/java/org/opensearch/performanceanalyzer/collectors/FaultDetectionMetricsCollector.java
@@ -123,14 +123,12 @@ public void collectMetrics(long startTime) {
                     "",
                     System.currentTimeMillis() - mCurrT);
         } catch (Exception ex) {
-            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
-                    ExceptionsAndErrors.FAULT_DETECTION_COLLECTOR_ERROR,
-                    "",
-                    System.currentTimeMillis() - mCurrT);
             LOG.debug(
                     "Exception in Collecting FaultDetection Metrics: {} for startTime {}",
                     () -> ex.toString(),
                     () -> startTime);
+            PerformanceAnalyzerApp.ERRORS_AND_EXCEPTIONS_AGGREGATOR.updateStat(
+                    ExceptionsAndErrors.FAULT_DETECTION_COLLECTOR_ERROR, "", 1);
         }
     }
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		763fed1fb35a016fdba53d3c66bd611d823090c0
		1328e960851fa7948368c94fbd092216b2df752b