Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Reader changes for dynamic enable/disable of RCA graph components #325

Merged
merged 18 commits into from
Aug 7, 2020
Merged

Reader changes for dynamic enable/disable of RCA graph components #325

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
d787724
Common changes needed to support dynamic en/disabling of config overr…
ktkrg Jul 19, 2020
59bfc48
Add ability to apply the config overrides
ktkrg Jul 22, 2020
23d833e
Merge from master
ktkrg Jul 31, 2020
b0ed031
Refactor muting logic for actions and graph nodes
ktkrg Jul 31, 2020
771f1c9
Add unit tests
ktkrg Jul 31, 2020
274009c
Remove merge conflict markers in comments
ktkrg Jul 31, 2020
3188ef2
Add licence header to new files
ktkrg Jul 31, 2020
e631aa8
Filter at the decider instead of collator
ktkrg Aug 4, 2020
eb77ae7
Merge from master
ktkrg Aug 4, 2020
27189e0
Delete collator test as it is not used to filter actions
ktkrg Aug 4, 2020
77473dc
Merge from master
ktkrg Aug 6, 2020
b4c9f28
Use the right log levels
ktkrg Aug 6, 2020
6282932
Add abstract SuppressibleAction class to handle muted actions
ktkrg Aug 6, 2020
a914d68
Remove unwanted import
ktkrg Aug 6, 2020
35c416d
Merge from master
ktkrg Aug 6, 2020
2b16890
Address PR comments
ktkrg Aug 7, 2020
300dcc5
Address PR comments
ktkrg Aug 7, 2020
556b527
Address PR comments
ktkrg Aug 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/main/java/com/amazon/opendistro/elasticsearch/performanceanalyzer/AppContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public class AppContext {
// initiate a node config cache within each AppContext space
// to store node config settings from ES
private final NodeConfigCache nodeConfigCache;
private Set<String> mutedActions;
private volatile Set<String> mutedActions;

public AppContext() {
this.clusterDetailsEventProcessor = null;
Expand Down
13 changes: 8 additions & 5 deletions ...opendistro/elasticsearch/performanceanalyzer/config/overrides/ConfigOverridesApplier.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,14 @@ private void apply(final ConfigOverrides overrides) {
}
}

LOG.debug("New set of muted rcas: {}", currentMutedRcaSet);
LOG.debug("New set of muted deciders: {}", currentMutedDeciderSet);
LOG.debug("New set of muted actions: {}", currentMutedActionSet);
rcaConf.updateAllRcaConfFiles(currentMutedRcaSet, currentMutedDeciderSet, currentMutedActionSet);
this.lastAppliedTimestamp = System.currentTimeMillis();
LOG.info("New set of muted rcas: {}", currentMutedRcaSet);
LOG.info("New set of muted deciders: {}", currentMutedDeciderSet);
LOG.info("New set of muted actions: {}", currentMutedActionSet);
boolean updateSuccess = rcaConf.updateAllRcaConfFiles(currentMutedRcaSet,
currentMutedDeciderSet, currentMutedActionSet);
if (updateSuccess) {
this.lastAppliedTimestamp = System.currentTimeMillis();
}
}
}
}
Expand Down
40 changes: 25 additions & 15 deletions ...a/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/core/RcaConf.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.core;

import com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.collectors.StatsCollector;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.RcaControllerHelper;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.CacheConfig;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.CacheDeciderConfig;
Expand All @@ -24,6 +25,7 @@
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HotNodeClusterRcaConfig;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HotShardClusterRcaConfig;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.configs.HotShardRcaConfig;
import com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.framework.util.RcaConsts;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationFeature;
Expand All @@ -42,7 +44,6 @@
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
Expand Down Expand Up @@ -202,17 +203,24 @@ public <T> T readRcaConfig(String rcaName, String key, Class<? extends T> clazz)
return setting;
}

public void updateAllRcaConfFiles(final Set<String> mutedRcas, final Set<String> mutedDeciders,
public boolean updateAllRcaConfFiles(final Set<String> mutedRcas, final Set<String> mutedDeciders,
final Set<String> mutedActions) {
boolean updateStatus = true;
// update all rca.conf files
List<String> rcaConfFiles = RcaControllerHelper.getAllConfFilePaths();

for (String confFilePath : rcaConfFiles) {
updateRcaConf(confFilePath, mutedRcas, mutedDeciders, mutedActions);
updateStatus = updateRcaConf(confFilePath, mutedRcas, mutedDeciders,
mutedActions);
if (!updateStatus) {
LOG.error("Failed to update the conf file at path: {}", confFilePath);
StatsCollector.instance().logMetric(RcaConsts.WRITE_UPDATED_RCA_CONF_ERROR);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can use the new metric framework. There your metric can also tell you which file erred out ?

break;
}
}
return updateStatus;
}

private void updateRcaConf(String originalFilePath, final Set<String> mutedRcas,
private boolean updateRcaConf(String originalFilePath, final Set<String> mutedRcas,
final Set<String> mutedDeciders, final Set<String> mutedActions) {

String updatedPath = originalFilePath + ".updated";
Expand All @@ -236,16 +244,19 @@ private void updateRcaConf(String originalFilePath, final Set<String> mutedRcas,
mapper.writeValue(updatedFileOutputStream, configObject);
} catch (IOException e) {
LOG.error("Unable to copy rca conf to a temp file", e);
return;
return false;
}

try {
LOG.info("Writing new file: {}", Paths.get(updatedPath));
Files.move(Paths.get(updatedPath), Paths.get(originalFilePath), StandardCopyOption.ATOMIC_MOVE,
StandardCopyOption.REPLACE_EXISTING);
Files
.move(Paths.get(updatedPath), Paths.get(originalFilePath), StandardCopyOption.ATOMIC_MOVE,
StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
LOG.error("Unable to move and replace the old conf file with updated conf file.", e);
return false;
}
return true;
}

@SuppressWarnings("unchecked")
Expand All @@ -254,18 +265,17 @@ public <T> T readDeciderConfig(String deciderName, String key, Class<? extends T
try {
Map<String, Object> deciderObj = null;
if (conf.getDeciderConfigSettings() != null
&& conf.getDeciderConfigSettings().containsKey(deciderName)
&& conf.getDeciderConfigSettings().get(deciderName) != null) {
deciderObj = (Map<String, Object>)conf.getDeciderConfigSettings().get(deciderName);
&& conf.getDeciderConfigSettings().containsKey(deciderName)
&& conf.getDeciderConfigSettings().get(deciderName) != null) {
deciderObj = (Map<String, Object>) conf.getDeciderConfigSettings().get(deciderName);
}

if (deciderObj != null
&& deciderObj.containsKey(key)
&& deciderObj.get(key) != null) {
&& deciderObj.containsKey(key)
&& deciderObj.get(key) != null) {
setting = clazz.cast(deciderObj.get(key));
}
}
catch (ClassCastException ne) {
} catch (ClassCastException ne) {
LOG.error("rca.conf contains value in invalid format, trace : {}", ne.getMessage());
}
return setting;
Expand Down
1 change: 1 addition & 0 deletions ...com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/util/RcaConsts.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ public class RcaConsts {
Paths.get(CONFIG_DIR_PATH, THRESHOLDS_DIR_NAME).toString();
public static final String MUTE_ERROR_METRIC = "MuteError";
public static final String HOT_SHARD_RCA_ERROR_METRIC = "HotShardError";
public static final String WRITE_UPDATED_RCA_CONF_ERROR = "WriteUpdatedRcaConfError";

static final String dir = System.getProperty("user.dir");
public static final String TEST_CONFIG_PATH =
Expand Down
14 changes: 14 additions & 0 deletions ...distro/elasticsearch/performanceanalyzer/config/overrides/ConfigOverridesApplierTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package com.amazon.opendistro.elasticsearch.performanceanalyzer.config.overrides;

import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.initMocks;
Expand All @@ -27,6 +28,7 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableSet;
import java.io.IOException;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collections;
Expand Down Expand Up @@ -161,6 +163,18 @@ public void testApplyValidOverrides() throws JsonProcessingException {
assertEquals(ImmutableSet.of(ACT2), combinedMutedActions);
}

@Test
public void testUpdateRcaConfFailure() throws IOException {
long lastAppliedTimestamp = System.currentTimeMillis();
testOverridesApplier.setLastAppliedTimestamp(lastAppliedTimestamp);
String overridesJson = new ObjectMapper().writeValueAsString(buildTestOverrides());
when(mockRcaConf.updateAllRcaConfFiles(any(), any(), any())).thenReturn(false);

testOverridesApplier.applyOverride(overridesJson, Long.toString(lastAppliedTimestamp + 300));

assertEquals(lastAppliedTimestamp, testOverridesApplier.getLastAppliedTimestamp());
}

private ConfigOverrides buildTestOverrides() {
final ConfigOverrides overrides = new ConfigOverrides();
Overrides disabled = new Overrides();
Expand Down