Skip to content

Commit

Permalink
Closes #806 - Added branch option to agent mappings (#809)
Browse files Browse the repository at this point in the history
  • Loading branch information
Jonas Kunz authored Jul 9, 2020
1 parent 5ffae3c commit 7fe6b87
Show file tree
Hide file tree
Showing 27 changed files with 935 additions and 697 deletions.
12 changes: 6 additions & 6 deletions ...er/src/main/java/rocks/inspectit/ocelot/agentconfiguration/AgentConfigurationManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@
import org.springframework.stereotype.Component;
import rocks.inspectit.ocelot.config.model.InspectitServerSettings;
import rocks.inspectit.ocelot.events.ConfigurationPromotionEvent;
import rocks.inspectit.ocelot.events.WorkspaceChangedEvent;
import rocks.inspectit.ocelot.file.FileManager;
import rocks.inspectit.ocelot.mappings.AgentMappingManager;
import rocks.inspectit.ocelot.mappings.AgentMappingsChangedEvent;
import rocks.inspectit.ocelot.mappings.AgentMappingSerializer;
import rocks.inspectit.ocelot.mappings.model.AgentMapping;

import javax.annotation.PostConstruct;
Expand All @@ -39,7 +39,7 @@ public class AgentConfigurationManager {
InspectitServerSettings config;

@Autowired
private AgentMappingManager mappingManager;
private AgentMappingSerializer mappingsSerializer;

@Autowired
private ExecutorService executorService;
Expand All @@ -60,25 +60,25 @@ public class AgentConfigurationManager {
private AgentConfigurationReloadTask reloadTask;

@PostConstruct
@VisibleForTesting
void init() {
replaceConfigurations(Collections.emptyList());
reloadConfigurationAsync();
}

@EventListener({ConfigurationPromotionEvent.class, AgentMappingsChangedEvent.class})
@EventListener({ConfigurationPromotionEvent.class, WorkspaceChangedEvent.class})
private synchronized void reloadConfigurationAsync() {
if (reloadTask != null) {
reloadTask.cancel();
}
reloadTask = new AgentConfigurationReloadTask(mappingManager.getAgentMappings(), fileManager, this::replaceConfigurations);
reloadTask = new AgentConfigurationReloadTask(mappingsSerializer, fileManager, this::replaceConfigurations);
executorService.submit(reloadTask);
}

/**
* Fetches the configuration as yaml string given a set of attributes describing the target agent.
*
* @param agentAttributes the attributes of the agent for which the configuration shall be queried
*
* @return the configuration for this agent or null if the attributes match no mapping
*/
public AgentConfiguration getConfiguration(Map<String, String> agentAttributes) {
Expand Down
89 changes: 47 additions & 42 deletions ...src/main/java/rocks/inspectit/ocelot/agentconfiguration/AgentConfigurationReloadTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@
import rocks.inspectit.ocelot.file.FileInfo;
import rocks.inspectit.ocelot.file.FileManager;
import rocks.inspectit.ocelot.file.accessor.AbstractFileAccessor;
import rocks.inspectit.ocelot.file.accessor.git.RevisionAccess;
import rocks.inspectit.ocelot.mappings.AgentMappingSerializer;
import rocks.inspectit.ocelot.mappings.model.AgentMapping;
import rocks.inspectit.ocelot.utils.CancellableTask;

import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.stream.Collectors;
Expand All @@ -21,47 +23,30 @@
* A task for asynchronously loading the configurations based on a given list of mappings.
*/
@Slf4j
class AgentConfigurationReloadTask implements Runnable {
class AgentConfigurationReloadTask extends CancellableTask<List<AgentConfiguration>> {

/**
* Predicate to check if a given file path ends with .yml or .yaml
*/
private static final Predicate<String> HAS_YAML_ENDING = filePath -> filePath.toLowerCase().endsWith(".yml") || filePath.toLowerCase().endsWith(".yaml");

/**
* Internal flag to check if cancel has been called.
*/
private AtomicBoolean cancelFlag = new AtomicBoolean(false);

/**
* Callback which is invoked when this task has finished.
*/
private Consumer<List<AgentConfiguration>> onLoadCallback;
private static final Predicate<String> HAS_YAML_ENDING = filePath -> filePath.toLowerCase()
.endsWith(".yml") || filePath.toLowerCase().endsWith(".yaml");

private FileManager fileManager;

private List<AgentMapping> mappingsToLoad;
private AgentMappingSerializer mappingsSerializer;

/**
* Creates a new reload task, but does NOT start it.
* The loading process is done in {@link #run()}.
*
* @param mappingsToLoad the mappings to load the configurations for
* @param fileManager the FileManager used to read the configuration files
* @param onLoadCallback invoked when the loading has finished successfully. Will not be invoked if the loading failed or was canceled.
* @param mappingsSerializer the serializer responsible for extracting the mappings from the current revision
* @param fileManager the FileManager used to read the configuration files
* @param onLoadCallback invoked when the loading has finished successfully. Will not be invoked if the loading failed or was canceled.
*/
public AgentConfigurationReloadTask(List<AgentMapping> mappingsToLoad, FileManager fileManager, Consumer<List<AgentConfiguration>> onLoadCallback) {
this.mappingsToLoad = mappingsToLoad;
public AgentConfigurationReloadTask(AgentMappingSerializer mappingsSerializer, FileManager fileManager, Consumer<List<AgentConfiguration>> onLoadCallback) {
super(onLoadCallback);
this.mappingsSerializer = mappingsSerializer;
this.fileManager = fileManager;
this.onLoadCallback = onLoadCallback;
}

/**
* Can be invoked to cancel this task.
* As soon as this method returns, it is guaranteed that the configured onLoad-callback will not be invoked anymore.
*/
public synchronized void cancel() {
cancelFlag.set(true);
}

/**
Expand All @@ -70,66 +55,84 @@ public synchronized void cancel() {
@Override
public void run() {
log.info("Starting configuration reloading...");
RevisionAccess fileAccess = fileManager.getWorkspaceRevision();
if (!fileAccess.agentMappingsExist()) {
onTaskSuccess(Collections.emptyList());
return;
}
List<AgentMapping> mappingsToLoad = mappingsSerializer.readAgentMappings(fileAccess);
List<AgentConfiguration> newConfigurations = new ArrayList<>();
for (AgentMapping mapping : mappingsToLoad) {
try {
String configYaml = loadConfigForMapping(mapping);
if (cancelFlag.get()) {
if (isCanceled()) {
log.debug("Configuration reloading canceled");
return;
}
AgentConfiguration agentConfiguration = AgentConfiguration.builder().mapping(mapping).configYaml(configYaml).build();
AgentConfiguration agentConfiguration = AgentConfiguration.builder()
.mapping(mapping)
.configYaml(configYaml)
.build();
newConfigurations.add(agentConfiguration);
} catch (Exception e) {
log.error("Could not load agent mapping '{}'.", mapping.getName(), e);
}
}
synchronized (this) {
if (cancelFlag.get()) {
log.debug("Configuration reloading canceled");
return;
}
onLoadCallback.accept(newConfigurations);
log.info("Configurations successfully reloaded");
}
onTaskSuccess(newConfigurations);
}


/**
* Loads the given mapping as yaml string.
*
* @param mapping the mapping to load
*
* @return the merged yaml for the given mapping or an empty string if the mapping does not contain any existing files
* If this task has been canceled, null is returned.
*/
@VisibleForTesting
String loadConfigForMapping(AgentMapping mapping) {
AbstractFileAccessor fileAccessor = fileManager.getLiveRevision();
AbstractFileAccessor fileAccessor = getFileAccessorForMapping(mapping);

LinkedHashSet<String> allYamlFiles = new LinkedHashSet<>();
for (String path : mapping.getSources()) {
if (cancelFlag.get()) {
if (isCanceled()) {
return null;
}
allYamlFiles.addAll(getAllYamlFiles(fileAccessor, path));
}

Object result = null;
for (String path : allYamlFiles) {
if (cancelFlag.get()) {
if (isCanceled()) {
return null;
}
result = loadAndMergeYaml(fileAccessor, result, path);
}
return result == null ? "" : new Yaml().dump(result);
}

private AbstractFileAccessor getFileAccessorForMapping(AgentMapping mapping) {
AbstractFileAccessor fileAccessor;
switch (mapping.getSourceBranch()) {
case LIVE:
fileAccessor = fileManager.getLiveRevision();
break;
case WORKSPACE:
fileAccessor = fileManager.getWorkspaceRevision();
break;
default:
throw new UnsupportedOperationException("Unhandled branch: " + mapping.getSourceBranch());
}
return fileAccessor;
}

/**
* If the given path is a yaml file, a list containing only it is returned.
* If the path is a directory, the absolute path of all contained yaml files is returned in alphabetical order.
* If it is neither, an empty list is returned.
*
* @param path the path to check for yaml files, can start with a slash which will be ignored
*
* @return a list of absolute paths of contained YAML files
*/
private List<String> getAllYamlFiles(AbstractFileAccessor fileAccessor, String path) {
Expand Down Expand Up @@ -161,6 +164,7 @@ private List<String> getAllYamlFiles(AbstractFileAccessor fileAccessor, String p
*
* @param toMerge the existing structure of nested maps / lists with which the loaded yaml will be merged.
* @param path the path of the yaml file to load
*
* @return the merged structure
*/
private Object loadAndMergeYaml(AbstractFileAccessor fileAccessor, Object toMerge, String path) {
Expand All @@ -183,6 +187,7 @@ private Object loadAndMergeYaml(AbstractFileAccessor fileAccessor, Object toMerg
* This exception will be thrown if a configuration file cannot be parsed, e.g. it contains invalid characters.
*/
static class InvalidConfigurationFileException extends RuntimeException {

public InvalidConfigurationFileException(String path, Exception e) {
super(String.format("The configuration file '%s' is invalid and cannot be parsed.", path), e);
}
Expand Down
109 changes: 29 additions & 80 deletions ...erver/src/main/java/rocks/inspectit/ocelot/autocomplete/util/ConfigurationFilesCache.java
View file
Original file line number Diff line number Diff line change
@@ -1,36 +1,40 @@
package rocks.inspectit.ocelot.autocomplete.util;

import com.google.common.annotations.VisibleForTesting;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import org.yaml.snakeyaml.Yaml;
import rocks.inspectit.ocelot.config.loaders.ConfigFileLoader;
import rocks.inspectit.ocelot.file.FileChangedEvent;
import rocks.inspectit.ocelot.file.FileInfo;
import rocks.inspectit.ocelot.events.WorkspaceChangedEvent;
import rocks.inspectit.ocelot.file.FileManager;
import rocks.inspectit.ocelot.file.accessor.git.RevisionAccess;

import javax.annotation.PostConstruct;
import java.io.IOException;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.Collection;
import java.util.Collections;
import java.util.concurrent.ExecutorService;

/**
* Caches the workspace revision.
*/
@Slf4j
@Component
public class ConfigurationFilesCache {

/**
* Predicate to check if a given file path ends with .yml or .yaml.
*/
private static final Predicate<String> HAS_YAML_ENDING = filePath -> filePath.toLowerCase().endsWith(".yml") || filePath.toLowerCase().endsWith(".yaml");

@Autowired
private FileManager fileManager;

private Collection<Object> yamlContents;
@Autowired
private ExecutorService executor;

/**
* The currently active task for reloading the configuration.
*/
private ConfigurationFilesCacheReloadTask activeReloadTask;

/**
* The current parsed contents of all configuration files.
*/
private Collection<Object> currentParsedFiles = Collections.emptyList();

/**
* Returns the most recently loaded .yaml and .yml files as a list of Objects. Each Object resembles the corresponding
Expand All @@ -50,77 +54,22 @@ public class ConfigurationFilesCache {
* @return A Collection containing all loaded .yaml and .yml files root elements as Maps or Lists.
*/
public Collection<Object> getParsedConfigurationFiles() {
return yamlContents;
return currentParsedFiles;
}

/**
* Loads all .yaml and .yml files. The files are loaded from the "configuration" folder of the server and from the
* "files" folder of the working directory. The files contents are parsed into either nested Lists or Maps.
*/
@PostConstruct
@EventListener(FileChangedEvent.class)
public void loadFiles() throws IOException {
List<String> filePaths = getAllPaths();
yamlContents = Stream.concat(
filePaths.stream()
.map(this::loadYamlFile)
.filter(Objects::nonNull),
ConfigFileLoader.getDefaultConfigFiles().values().stream()
.map(this::parseYaml)
).collect(Collectors.toList());
}

/**
* Takes as String and parses it either into a nested List or Maps.
* Literals such as "name:" are parsed as keys for Maps. The respectively following literals are then added as values to this
* key.
* Literals such as "- a \n - b" are parsed as lists.
* All other literals are parsed as scalars and are added as values.
*
* @param content The String to be parsed.
* @return The String parsed into a nested Lists or Map.
*/
private Object parseYaml(String content) {
Yaml yaml = new Yaml();
return yaml.load(content);
}

/**
* This method loads a .yaml or .yml file found in a given path and returns it either as nested List or Map.
* The Map/List can either contain a terminal value such as Strings, Lists of elements or Maps. The latter two
* of which can each again contain Lists, Maps or terminal values as values.
*
* @param path path of the file which should be loaded.
* @return the file as an Object parsed as described above.
*/
@VisibleForTesting
Object loadYamlFile(String path) {
Optional<String> src = fileManager.getWorkingDirectory().readConfigurationFile(path);

return src.map(this::parseYaml).orElseGet(() -> {
log.warn("Unable to load file with path {}", path);
return null;
});
}

/**
* Searches in the current directory for files with .yml or .yaml ending. Returns all paths to those files as a
* lexicographically ordered List of Strings.
*
* @return A list of all found paths to .yml or .yaml files.
*/
@VisibleForTesting
List<String> getAllPaths() {
try {
List<FileInfo> fileInfos = fileManager.getWorkingDirectory().listConfigurationFiles("");

return fileInfos.stream()
.flatMap(file -> file.getAbsoluteFilePaths(""))
.filter(HAS_YAML_ENDING)
.sorted()
.collect(Collectors.toList());
} catch (Exception e) {
return new ArrayList<>();
@EventListener(WorkspaceChangedEvent.class)
public synchronized void loadFiles() {
RevisionAccess fileAccess = fileManager.getWorkspaceRevision();
if (activeReloadTask != null) {
activeReloadTask.cancel();
}
activeReloadTask = new ConfigurationFilesCacheReloadTask(fileAccess, (configs) -> currentParsedFiles = configs);
executor.submit(activeReloadTask);
}

}
Loading

0 comments on commit 7fe6b87

Please sign in to comment.