Skip to content

Commit

Permalink
[backport 2.x] Removed create/delete queryIndex (#215) (#255)
Browse files Browse the repository at this point in the history 
Signed-off-by: Subhobrata Dey <[email protected]>
Co-authored-by: Petar Dzepina <[email protected]>
  • Loading branch information
@sbcd90 @petardz
sbcd90 and petardz authored Jan 10, 2023
1 parent 70f5a09 commit 153ad3d
Show file tree
Hide file tree
Showing 15 changed files with 441 additions and 146 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import org.opensearch.gradle.test.RestIntegTestTask

buildscript {
ext {
opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT")
opensearch_version = System.getProperty("opensearch.version", "2.5.0-SNAPSHOT")
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
buildVersionQualifier = System.getProperty("build.version_qualifier", "")
version_tokens = opensearch_version.tokenize('-')
Expand Down
12 changes: 10 additions & 2 deletions src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ public class DetectorMonitorConfig {
public static final String OPENSEARCH_DEFAULT_ALL_FINDINGS_INDICES_PATTERN = ".opensearch-sap-findings-default*";
public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN = "<.opensearch-sap-findings-default-{now/d}-1>";

public static final String OPENSEARCH_SAP_RULE_INDEX_TEMPLATE = ".opensearch-sap-detectors-queries-index-template";

private static Map<String, MonitorConfig> detectorTypeToIndicesMapping;

static {
Expand Down Expand Up @@ -113,6 +115,13 @@ public static List<String> getAllFindingsIndicesPatternForAllTypes() {
.collect(Collectors.toList());
}

public static List<String> getAllRuleIndices() {
return detectorTypeToIndicesMapping.entrySet()
.stream()
.map(e -> e.getValue().getRuleIndex())
.collect(Collectors.toList());
}

public static String getFindingsIndexPattern(String detectorType) {
return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ?
detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getFindingsIndexPattern() :
Expand Down Expand Up @@ -145,8 +154,7 @@ private MonitorConfig(
String findingsIndex,
String findingsIndexPattern,
String allFindingsIndicesPattern,
String ruleIndex
) {
String ruleIndex) {
this.alertsIndex = alertsIndex;
this.alertsHistoryIndex = alertsHistoryIndex;
this.alertsHistoryIndexPattern = alertsHistoryIndexPattern;
Expand Down
50 changes: 46 additions & 4 deletions src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,17 @@

package org.opensearch.securityanalytics.mapper;

import java.util.Locale;
import java.util.Collection;
import java.util.Optional;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.OpenSearchStatusException;
import org.opensearch.ResourceNotFoundException;
import org.opensearch.action.ActionListener;
import org.opensearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.opensearch.action.support.GroupedActionListener;
import org.opensearch.action.support.master.AcknowledgedResponse;
import org.opensearch.client.IndicesAdminClient;
import org.opensearch.cluster.metadata.MappingMetadata;
Expand Down Expand Up @@ -74,10 +75,51 @@ public void onFailure(Exception e) {

private void createMappingActionContinuation(ImmutableOpenMap<String, MappingMetadata> indexMappings, String ruleTopic, String aliasMappings, boolean partial, ActionListener<AcknowledgedResponse> actionListener) {

int numOfIndices = indexMappings.size();

GroupedActionListener doCreateMappingActionsListener = new GroupedActionListener(new ActionListener<Collection<AcknowledgedResponse>>() {
@Override
public void onResponse(Collection<AcknowledgedResponse> response) {
// We will return ack==false if one of the requests returned that
// else return ack==true
Optional<AcknowledgedResponse> notAckd = response.stream().filter(e -> e.isAcknowledged() == false).findFirst();
AcknowledgedResponse ack = new AcknowledgedResponse(
notAckd.isPresent() ? false : true
);
actionListener.onResponse(ack);
}

@Override
public void onFailure(Exception e) {
actionListener.onFailure(
new SecurityAnalyticsException(
"Failed applying mappings to index", RestStatus.INTERNAL_SERVER_ERROR, e)
);
}
}, numOfIndices);

indexMappings.forEach(iter -> {
String indexName = iter.key;
MappingMetadata mappingMetadata = iter.value;
// Try to apply mapping to index
doCreateMapping(indexName, mappingMetadata, ruleTopic, aliasMappings, partial, doCreateMappingActionsListener);
});
}

/**
* Applies alias mappings to index.
* @param indexName Index name
* @param mappingMetadata Index mappings
* @param ruleTopic Rule topic spcifying specific alias templates
* @param aliasMappings User-supplied alias mappings
* @param partial Partial flag indicating if we should apply mappings partially, in case source index doesn't have all paths specified in alias mappings
* @param actionListener actionListener used to return response/error
*/
private void doCreateMapping(String indexName, MappingMetadata mappingMetadata, String ruleTopic, String aliasMappings, boolean partial, ActionListener<AcknowledgedResponse> actionListener) {

PutMappingRequest request;
try {

String indexName = indexMappings.iterator().next().key;
String aliasMappingsJSON;
// aliasMappings parameter has higher priority then ruleTopic
if (aliasMappings != null) {
Expand All @@ -86,7 +128,7 @@ private void createMappingActionContinuation(ImmutableOpenMap<String, MappingMet
aliasMappingsJSON = MapperTopicStore.aliasMappings(ruleTopic);
}

List<String> missingPathsInIndex = MapperUtils.validateIndexMappings(indexMappings, aliasMappingsJSON);
List<String> missingPathsInIndex = MapperUtils.validateIndexMappings(indexName, mappingMetadata, aliasMappingsJSON);

if(missingPathsInIndex.size() > 0) {
// If user didn't allow partial apply, we should error out here
Expand Down
22 changes: 9 additions & 13 deletions src/main/java/org/opensearch/securityanalytics/mapper/MapperUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package org.opensearch.securityanalytics.mapper;

import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import org.apache.commons.lang3.tuple.Pair;
import org.opensearch.cluster.metadata.MappingMetadata;
Expand Down Expand Up @@ -89,24 +90,22 @@ public void onError(String error) {
* <li>Alias mappings have to have property type=alias and path property has to exist
* <li>Paths from alias mappings should exists in index mappings
* </ul>
* @param indexMappings Index Mappings to which alias mappings will be applied
* @param aliasMappingsJSON Alias Mappings as JSON string
* @param indexName Source index name
* @param mappingMetadata Source index mapping to which alias mappings will be applied
* @param aliasMappingsJSON Alias mappings as JSON string
* @return list of alias mappings paths which are missing in index mappings
* */
public static List<String> validateIndexMappings(ImmutableOpenMap<String, MappingMetadata> indexMappings, String aliasMappingsJSON) throws IOException {
public static List<String> validateIndexMappings(String indexName, MappingMetadata mappingMetadata, String aliasMappingsJSON) throws IOException {

// Check if index's mapping is empty
if (isIndexMappingsEmpty(indexMappings)) {
throw new IllegalArgumentException("Index mappings are empty");
if (isIndexMappingsEmpty(mappingMetadata)) {
throw new IllegalArgumentException(String.format(Locale.ROOT, "Mappings for index [%s] are empty", indexName));
}

// Get all paths (field names) to which we're going to apply aliases
List<String> paths = getAllPathsFromAliasMappings(aliasMappingsJSON);

// Traverse Index Mappings and extract all fields(paths)
String indexName = indexMappings.iterator().next().key;
MappingMetadata mappingMetadata = indexMappings.get(indexName);

List<String> flatFields = getAllNonAliasFieldsFromIndex(mappingMetadata);
// Return list of paths from Alias Mappings which are missing in Index Mappings
return paths.stream()
Expand Down Expand Up @@ -164,11 +163,8 @@ public static List<String> getAllNonAliasFieldsFromIndex(MappingMetadata mapping
return mappingsTraverser.extractFlatNonAliasFields();
}

public static boolean isIndexMappingsEmpty(ImmutableOpenMap<String, MappingMetadata> indexMappings) {
if (indexMappings.iterator().hasNext()) {
return indexMappings.iterator().next().value.getSourceAsMap().size() == 0;
}
throw new IllegalArgumentException("Invalid Index Mappings");
public static boolean isIndexMappingsEmpty(MappingMetadata mappingMetadata) {
return mappingMetadata.getSourceAsMap().size() == 0;
}

public static Map<String, Object> getAliasMappingsWithFilter(
Expand Down
5 changes: 0 additions & 5 deletions ...n/java/org/opensearch/securityanalytics/transport/TransportCreateIndexMappingsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,6 @@ public TransportCreateIndexMappingsAction(
protected void doExecute(Task task, CreateIndexMappingsRequest request, ActionListener<AcknowledgedResponse> actionListener) {
this.threadPool.getThreadContext().stashContext();

IndexMetadata index = clusterService.state().metadata().index(request.getIndexName());
if (index == null) {
actionListener.onFailure(new IllegalStateException("Could not find index [" + request.getIndexName() + "]"));
return;
}
mapperService.createMappingAction(
request.getIndexName(),
request.getRuleTopic(),
Expand Down
43 changes: 1 addition & 42 deletions src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteDetectorAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,48 +152,7 @@ public void onResponse(Collection<DeleteMonitorResponse> responses) {
}).count() > 0) {
onFailures(new OpenSearchStatusException("Monitor associated with detected could not be deleted", errorStatusSupplier.get()));
}
ruleTopicIndices.countQueries(ruleIndex, new ActionListener<>() {
@Override
public void onResponse(SearchResponse response) {
if (response.isTimedOut()) {
log.info("Count response timed out");
deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy());
} else {
long count = response.getHits().getTotalHits().value;

if (count == 0) {
try {
ruleTopicIndices.deleteRuleTopicIndex(ruleIndex,
new ActionListener<>() {
@Override
public void onResponse(AcknowledgedResponse response) {
deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy());
}

@Override
public void onFailure(Exception e) {
// error is suppressed as it is not a critical deletion
log.info(e.getMessage());
deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy());
}
});
} catch (IOException e) {
deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy());
}
} else {
deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy());
}
}
}

@Override
public void onFailure(Exception e) {
// error is suppressed as it is not a critical deletion
log.info(e.getMessage());


}
});
deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy());
}

@Override
Expand Down
8 changes: 4 additions & 4 deletions src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -696,9 +696,9 @@ void createDetector() {

if (!detector.getInputs().isEmpty()) {
try {
ruleTopicIndices.initRuleTopicIndex(detector.getRuleIndex(), new ActionListener<>() {
ruleTopicIndices.initRuleTopicIndexTemplate(new ActionListener<>() {
@Override
public void onResponse(CreateIndexResponse createIndexResponse) {
public void onResponse(AcknowledgedResponse acknowledgedResponse) {

initRuleIndexAndImportRules(request, new ActionListener<>() {
@Override
Expand Down Expand Up @@ -802,9 +802,9 @@ void onGetResponse(Detector currentDetector, User user) {

if (!detector.getInputs().isEmpty()) {
try {
ruleTopicIndices.initRuleTopicIndex(detector.getRuleIndex(), new ActionListener<>() {
ruleTopicIndices.initRuleTopicIndexTemplate(new ActionListener<>() {
@Override
public void onResponse(CreateIndexResponse createIndexResponse) {
public void onResponse(AcknowledgedResponse acknowledgedResponse) {
initRuleIndexAndImportRules(request, new ActionListener<>() {
@Override
public void onResponse(List<IndexMonitorResponse> monitorResponses) {
Expand Down
58 changes: 22 additions & 36 deletions src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,22 @@
*/
package org.opensearch.securityanalytics.util;

import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.action.ActionListener;
import org.opensearch.action.admin.indices.create.CreateIndexRequest;
import org.opensearch.action.admin.indices.create.CreateIndexResponse;
import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.opensearch.action.support.master.AcknowledgedResponse;
import org.opensearch.client.AdminClient;
import org.opensearch.client.Client;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.search.builder.SearchSourceBuilder;

import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Objects;
import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig;

public class RuleTopicIndices {
private static final Logger log = LogManager.getLogger(DetectorIndices.class);
Expand All @@ -37,40 +33,30 @@ public RuleTopicIndices(Client client, ClusterService clusterService) {
this.clusterService = clusterService;
}

public static String ruleTopicIndexMappings() throws IOException {
return new String(Objects.requireNonNull(DetectorIndices.class.getClassLoader().getResourceAsStream("mappings/detector-queries.json")).readAllBytes(), Charset.defaultCharset());
}

public static String ruleTopicIndexSettings() throws IOException {
return new String(Objects.requireNonNull(DetectorIndices.class.getClassLoader().getResourceAsStream("mappings/detector-settings.json")).readAllBytes(), Charset.defaultCharset());
}

public void initRuleTopicIndex(String ruleTopicIndex, ActionListener<CreateIndexResponse> actionListener) throws IOException {
if (!ruleTopicIndexExists(ruleTopicIndex)) {
CreateIndexRequest indexRequest = new CreateIndexRequest(ruleTopicIndex)
.mapping(ruleTopicIndexMappings())
public void initRuleTopicIndexTemplate(ActionListener<AcknowledgedResponse> actionListener) throws IOException {
if (!ruleTopicIndexTemplateExists()) {
// Compose list of all patterns to cover all query indices
List<String> indexPatterns = new ArrayList<>();
for(String ruleIndex : DetectorMonitorConfig.getAllRuleIndices()) {
indexPatterns.add(ruleIndex + "*");
}
PutIndexTemplateRequest indexRequest =
new PutIndexTemplateRequest(DetectorMonitorConfig.OPENSEARCH_SAP_RULE_INDEX_TEMPLATE)
.patterns(indexPatterns)
.settings(Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build());
client.admin().indices().create(indexRequest, actionListener);
client.admin().indices().putTemplate(indexRequest, actionListener);
} else {
actionListener.onResponse(new CreateIndexResponse(true, true, ruleTopicIndex));
actionListener.onResponse(new AcknowledgedResponse(true));
}
}

public void deleteRuleTopicIndex(String ruleTopicIndex, ActionListener<AcknowledgedResponse> actionListener) throws IOException {
if (ruleTopicIndexExists(ruleTopicIndex)) {
DeleteIndexRequest request = new DeleteIndexRequest(ruleTopicIndex);
client.admin().indices().delete(request, actionListener);
}
}

public void countQueries(String ruleTopicIndex, ActionListener<SearchResponse> listener) {
SearchRequest request = new SearchRequest(ruleTopicIndex)
.source(new SearchSourceBuilder().size(0));
client.search(request, listener);
}

public boolean ruleTopicIndexExists(String ruleTopicIndex) {
public boolean ruleTopicIndexTemplateExists() {
ClusterState clusterState = clusterService.state();
return clusterState.getRoutingTable().hasIndex(ruleTopicIndex);
return clusterState.metadata().templates()
.get(DetectorMonitorConfig.OPENSEARCH_SAP_RULE_INDEX_TEMPLATE) != null;
}
}
16 changes: 0 additions & 16 deletions src/main/resources/mappings/detector-queries.json
View file

This file was deleted.

Loading

0 comments on commit 153ad3d

Please sign in to comment.