From be7e02d293bce7fc77d4da820588769a8fa44f6c Mon Sep 17 00:00:00 2001 From: Ruizhen Date: Fri, 7 Aug 2020 11:31:20 -0700 Subject: [PATCH 1/3] Expose queue rejection time period setting in rca.conf --- pa_config/rca.conf | 4 ++ pa_config/rca_idle_master.conf | 4 ++ pa_config/rca_master.conf | 4 ++ .../rca/configs/QueueRejectionRcaConfig.java | 40 +++++++++++++++++++ .../rca/framework/core/RcaConf.java | 6 +++ .../rca/threadpool/QueueRejectionRca.java | 35 ++++++++++++---- 6 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/QueueRejectionRcaConfig.java diff --git a/pa_config/rca.conf b/pa_config/rca.conf index deb26e682..b75b1aebe 100644 --- a/pa_config/rca.conf +++ b/pa_config/rca.conf @@ -49,6 +49,10 @@ "promotion-rate-mb-per-second" : 500, "young-gen-gc-time-ms-per-second" : 400 }, + //queue rejection rca + "queue-rejection-rca": { + "rejection-time-period-in-seconds" : 300 + }, //hot shard rca "hot-shard-rca": { "cpu-utilization" : 0.01, diff --git a/pa_config/rca_idle_master.conf b/pa_config/rca_idle_master.conf index 67a90aef1..3fb3012f8 100644 --- a/pa_config/rca_idle_master.conf +++ b/pa_config/rca_idle_master.conf @@ -49,6 +49,10 @@ "promotion-rate-mb-per-second" : 500, "young-gen-gc-time-ms-per-second" : 400 }, + //queue rejection rca + "queue-rejection-rca": { + "rejection-time-period-in-seconds" : 300 + }, //unbalanced node rca "hot-node-cluster-rca": { "unbalanced-resource-percentage" : 0.3, diff --git a/pa_config/rca_master.conf b/pa_config/rca_master.conf index c604ffdcc..e4f2a04e7 100644 --- a/pa_config/rca_master.conf +++ b/pa_config/rca_master.conf @@ -49,6 +49,10 @@ "promotion-rate-mb-per-second" : 500, "young-gen-gc-time-ms-per-second" : 400 }, + //queue rejection rca + "queue-rejection-rca": { + "rejection-time-period-in-seconds" : 300 + }, //unbalanced node rca "hot-node-cluster-rca": { "unbalanced-resource-percentage" : 0.3, diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/QueueRejectionRcaConfig.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/QueueRejectionRcaConfig.java new file mode 100644 index 000000000..9189e5ed0 --- /dev/null +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/QueueRejectionRcaConfig.java @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf; + +public class QueueRejectionRcaConfig { + private Integer rejectionTimePeriodInSeconds; + public static final int DEFAULT_REJECTION_TIME_PERIOD_IN_SECONDS = 300; + public static final String CONFIG_NAME = "queue-rejection-rca"; + + public QueueRejectionRcaConfig(final RcaConf rcaConf) { + rejectionTimePeriodInSeconds = rcaConf.readRcaConfig( + CONFIG_NAME, RCA_CONF_KEY_CONSTANTS.REJECTION_TIME_PERIOD_IN_SECONDS, Integer.class); + if (rejectionTimePeriodInSeconds == null) { + rejectionTimePeriodInSeconds = DEFAULT_REJECTION_TIME_PERIOD_IN_SECONDS; + } + } + + public int getRejectionTimePeriodInSeconds() { + return rejectionTimePeriodInSeconds; + } + + public static class RCA_CONF_KEY_CONSTANTS { + public static final String REJECTION_TIME_PERIOD_IN_SECONDS = "rejection-time-period-in-seconds"; + } +} diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConf.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConf.java index 3d9a8a68e..fc9b7fffa 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConf.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConf.java @@ -23,6 +23,8 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HotNodeClusterRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HotShardClusterRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HotShardRcaConfig; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.QueueRejectionRcaConfig; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.store.rca.threadpool.QueueRejectionRca; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationFeature; @@ -123,6 +125,10 @@ public HighHeapUsageYoungGenRcaConfig getHighHeapUsageYoungGenRcaConfig() { return new HighHeapUsageYoungGenRcaConfig(this); } + public QueueRejectionRcaConfig getQueueRejectionRcaConfig() { + return new QueueRejectionRcaConfig(this); + } + public HotNodeClusterRcaConfig getHotNodeClusterRcaConfig() { return new HotNodeClusterRcaConfig(this); } diff --git a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/threadpool/QueueRejectionRca.java b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/threadpool/QueueRejectionRca.java index 8f52c7847..daeaa7e0a 100644 --- a/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/threadpool/QueueRejectionRca.java +++ b/src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/store/rca/threadpool/QueueRejectionRca.java @@ -21,6 +21,8 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.grpc.Resource; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metrics.AllMetrics.ThreadPoolType; import com.amazon.opendistro.elasticsearch.performanceanalyzer.metricsdb.MetricsDB; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HighHeapUsageOldGenRcaConfig; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.QueueRejectionRcaConfig; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Metric; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Rca; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.Resources; @@ -31,6 +33,7 @@ import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotNodeSummary; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.HotResourceSummary; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.api.summaries.ResourceUtil; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.util.InstanceDetails; import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.scheduler.FlowUnitOperationArgWrapper; import com.google.common.annotations.VisibleForTesting; @@ -51,7 +54,6 @@ */ public class QueueRejectionRca extends Rca> { private static final Logger LOG = LogManager.getLogger(QueueRejectionRca.class); - private static final long REJECTION_TIME_PERIOD_IN_MILLISECOND = TimeUnit.SECONDS.toMillis(300); private final int rcaPeriod; private final List queueRejectionCollectors; private int counter; @@ -64,9 +66,9 @@ public QueueRejectionRca(final int rcaPeriod, M threadPool_Re clock = Clock.systemUTC(); queueRejectionCollectors = Collections.unmodifiableList(Arrays.asList( new QueueRejectionCollector(ResourceUtil.WRITE_QUEUE_REJECTION, ThreadPoolType.WRITE, - threadPool_RejectedReqs, REJECTION_TIME_PERIOD_IN_MILLISECOND), + threadPool_RejectedReqs), new QueueRejectionCollector(ResourceUtil.SEARCH_QUEUE_REJECTION, ThreadPoolType.SEARCH, - threadPool_RejectedReqs, REJECTION_TIME_PERIOD_IN_MILLISECOND) + threadPool_RejectedReqs) )); } @@ -123,6 +125,18 @@ public void generateFlowUnitListFromWire(FlowUnitOperationArgWrapper args) { setFlowUnits(flowUnitList); } + /** + * read rejection-time-period-in-seconds from rca.conf + * @param conf RcaConf object + */ + @Override + public void readRcaConf(RcaConf conf) { + QueueRejectionRcaConfig configObj = conf.getQueueRejectionRcaConfig(); + long rejectedTimePeriod = TimeUnit.SECONDS.toMillis(configObj.getRejectionTimePeriodInSeconds()); + queueRejectionCollectors.forEach( + collector -> collector.setRejectionTimePeriod(rejectedTimePeriod)); + } + /** * A collector class to collect rejection from each queue type */ @@ -132,16 +146,21 @@ private static class QueueRejectionCollector { private final Metric threadPool_RejectedReqs; private boolean hasRejection; private long rejectionTimestamp; - private long rejectionTimePeriodThreshold; + private long rejectionTimePeriodInMillis; public QueueRejectionCollector(final Resource threadPool, final ThreadPoolType threadPoolMetric, - final Metric threadPool_RejectedReqs, final long threshold) { + final Metric threadPool_RejectedReqs) { this.threadPool = threadPool; this.threadPoolMetric = threadPoolMetric; this.threadPool_RejectedReqs = threadPool_RejectedReqs; this.hasRejection = false; this.rejectionTimestamp = 0; - this.rejectionTimePeriodThreshold = threshold; + this.rejectionTimePeriodInMillis = + TimeUnit.SECONDS.toMillis(QueueRejectionRcaConfig.DEFAULT_REJECTION_TIME_PERIOD_IN_SECONDS); + } + + public void setRejectionTimePeriod(long rejectionTimePeriodInMillis) { + this.rejectionTimePeriodInMillis = rejectionTimePeriodInMillis; } public void collect(final long currTimestamp) { @@ -169,14 +188,14 @@ public void collect(final long currTimestamp) { } public boolean isUnhealthy(final long currTimestamp) { - return hasRejection && (currTimestamp - rejectionTimestamp) >= rejectionTimePeriodThreshold; + return hasRejection && (currTimestamp - rejectionTimestamp) >= rejectionTimePeriodInMillis; } public HotResourceSummary generateSummary(final long currTimestamp) { HotResourceSummary resourceSummary = null; if (isUnhealthy(currTimestamp)) { resourceSummary = new HotResourceSummary(threadPool, - TimeUnit.MILLISECONDS.toSeconds(rejectionTimePeriodThreshold), + TimeUnit.MILLISECONDS.toSeconds(rejectionTimePeriodInMillis), TimeUnit.MILLISECONDS.toSeconds(currTimestamp - rejectionTimestamp), 0); } From d37d595012298bcce6bbfa26082c0bb0f4697fd3 Mon Sep 17 00:00:00 2001 From: Ruizhen Date: Fri, 7 Aug 2020 14:31:56 -0700 Subject: [PATCH 2/3] Add unit test --- .../configs/QueueRejectionRcaConfigTest.java | 47 +++++++++++++++++++ src/test/resources/rca/rca.conf | 4 ++ .../resources/rca/rca_elected_master.conf | 4 ++ 3 files changed, 55 insertions(+) create mode 100644 src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/QueueRejectionRcaConfigTest.java diff --git a/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/QueueRejectionRcaConfigTest.java b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/QueueRejectionRcaConfigTest.java new file mode 100644 index 000000000..00c2e3c11 --- /dev/null +++ b/src/test/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/configs/QueueRejectionRcaConfigTest.java @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs; + +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core.RcaConf; +import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.util.RcaConsts; +import java.nio.file.Paths; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class QueueRejectionRcaConfigTest { + + private RcaConf rcaConf; + private RcaConf rcaOldConf; + + @Before + public void init() { + String rcaConfPath = Paths.get(RcaConsts.TEST_CONFIG_PATH, "rca_elected_master.conf").toString(); + String rcaOldConfPath = Paths.get(RcaConsts.TEST_CONFIG_PATH, "rca_master.conf").toString(); + rcaConf = new RcaConf(rcaConfPath); + rcaOldConf = new RcaConf(rcaOldConfPath); + } + + @Test + public void testReadConfig() { + QueueRejectionRcaConfig config = new QueueRejectionRcaConfig(rcaConf); + Assert.assertEquals(400, config.getRejectionTimePeriodInSeconds()); + + QueueRejectionRcaConfig oldConfig = new QueueRejectionRcaConfig(rcaOldConf); + Assert.assertEquals(QueueRejectionRcaConfig.DEFAULT_REJECTION_TIME_PERIOD_IN_SECONDS, + oldConfig.getRejectionTimePeriodInSeconds()); + } +} diff --git a/src/test/resources/rca/rca.conf b/src/test/resources/rca/rca.conf index 4210d3c8d..93e4ae3e8 100644 --- a/src/test/resources/rca/rca.conf +++ b/src/test/resources/rca/rca.conf @@ -54,6 +54,10 @@ "promotion-rate-mb-per-second" : 500, "young-gen-gc-time-ms-per-second" : 400 }, + //queue rejection rca + "queue-rejection-rca": { + "rejection-time-period-in-seconds" : 300 + }, //hot shard rca "hot-shard-rca": { "cpu-utilization" : 0.01, diff --git a/src/test/resources/rca/rca_elected_master.conf b/src/test/resources/rca/rca_elected_master.conf index 9e2ef7c6a..c22da8532 100644 --- a/src/test/resources/rca/rca_elected_master.conf +++ b/src/test/resources/rca/rca_elected_master.conf @@ -53,6 +53,10 @@ "promotion-rate-mb-per-second" : 500, "young-gen-gc-time-ms-per-second" : 400 }, + //queue rejection rca + "queue-rejection-rca": { + "rejection-time-period-in-seconds" : 400 + }, //unbalanced node rca "hot-node-cluster-rca": { "unbalanced-resource-percentage" : 0.3, From cbb2e4c606b998947bda977961d321c8e7babc74 Mon Sep 17 00:00:00 2001 From: Ruizhen Date: Fri, 7 Aug 2020 14:49:01 -0700 Subject: [PATCH 3/3] Address more PR comments --- pa_config/rca.conf | 1 - pa_config/rca_idle_master.conf | 1 - pa_config/rca_master.conf | 1 - src/test/resources/rca/rca.conf | 1 - src/test/resources/rca/rca_elected_master.conf | 1 - 5 files changed, 5 deletions(-) diff --git a/pa_config/rca.conf b/pa_config/rca.conf index ec8000847..c716cf1fe 100644 --- a/pa_config/rca.conf +++ b/pa_config/rca.conf @@ -49,7 +49,6 @@ "promotion-rate-mb-per-second" : 500, "young-gen-gc-time-ms-per-second" : 400 }, - //queue rejection rca "queue-rejection-rca": { "rejection-time-period-in-seconds" : 300 }, diff --git a/pa_config/rca_idle_master.conf b/pa_config/rca_idle_master.conf index 1e287b19c..212bff943 100644 --- a/pa_config/rca_idle_master.conf +++ b/pa_config/rca_idle_master.conf @@ -49,7 +49,6 @@ "promotion-rate-mb-per-second" : 500, "young-gen-gc-time-ms-per-second" : 400 }, - //queue rejection rca "queue-rejection-rca": { "rejection-time-period-in-seconds" : 300 }, diff --git a/pa_config/rca_master.conf b/pa_config/rca_master.conf index 01bb96f69..474573238 100644 --- a/pa_config/rca_master.conf +++ b/pa_config/rca_master.conf @@ -49,7 +49,6 @@ "promotion-rate-mb-per-second" : 500, "young-gen-gc-time-ms-per-second" : 400 }, - //queue rejection rca "queue-rejection-rca": { "rejection-time-period-in-seconds" : 300 }, diff --git a/src/test/resources/rca/rca.conf b/src/test/resources/rca/rca.conf index 93e4ae3e8..cef29b640 100644 --- a/src/test/resources/rca/rca.conf +++ b/src/test/resources/rca/rca.conf @@ -54,7 +54,6 @@ "promotion-rate-mb-per-second" : 500, "young-gen-gc-time-ms-per-second" : 400 }, - //queue rejection rca "queue-rejection-rca": { "rejection-time-period-in-seconds" : 300 }, diff --git a/src/test/resources/rca/rca_elected_master.conf b/src/test/resources/rca/rca_elected_master.conf index c22da8532..7d5987680 100644 --- a/src/test/resources/rca/rca_elected_master.conf +++ b/src/test/resources/rca/rca_elected_master.conf @@ -53,7 +53,6 @@ "promotion-rate-mb-per-second" : 500, "young-gen-gc-time-ms-per-second" : 400 }, - //queue rejection rca "queue-rejection-rca": { "rejection-time-period-in-seconds" : 400 },