fix e2e workflow till finding generation (#8)

Signed-off-by: Subhobrata Dey <[email protected]>
petardz · Jul 8, 2023 · bad292b · bad292b
1 parent e940c16
commit bad292b
Show file tree

Hide file tree

Showing 52 changed files with 135 additions and 1,402 deletions.
diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java
@@ -19,6 +19,7 @@
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.client.Client;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
+import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.component.LifecycleComponent;
@@ -33,8 +34,6 @@
 import org.opensearch.env.Environment;
 import org.opensearch.env.NodeEnvironment;
 import org.opensearch.index.IndexSettings;
-import org.opensearch.index.codec.CodecService;
-import org.opensearch.index.codec.CodecServiceConfig;
 import org.opensearch.index.codec.CodecServiceFactory;
 import org.opensearch.index.engine.EngineFactory;
 import org.opensearch.index.mapper.Mapper;
@@ -264,7 +263,7 @@ public List<Setting<?>> getSettings() {
     }
 
     @Override
-    public void onNodeStarted() {
+    public void onNodeStarted(DiscoveryNode localNode) {
 //      Trigger initialization of log types
         logTypeService.ensureConfigIndexIsInitialized(new ActionListener<>() {
             @Override

diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java
@@ -32,12 +32,14 @@
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.metadata.MappingMetadata;
 import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.Strings;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.rest.RestStatus;
 import org.opensearch.securityanalytics.action.GetIndexMappingsResponse;
 import org.opensearch.securityanalytics.action.GetMappingsViewResponse;
 import org.opensearch.securityanalytics.logtype.LogTypeService;
 import org.opensearch.securityanalytics.model.CreateMappingResult;
-import org.opensearch.securityanalytics.model.LogType;
 import org.opensearch.securityanalytics.util.IndexUtils;
 import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 
@@ -165,56 +167,128 @@ private void doCreateMapping(
     ) {
 
         try {
+            if (aliasMappings != null) {
+                Pair<List<String>, List<String>> validationResult = MapperUtils.validateIndexMappings(indexName, mappingMetadata, aliasMappings);
+                List<String> missingPathsInIndex = validationResult.getLeft();
+                List<String> presentPathsInIndex = validationResult.getRight();
+
+                if (missingPathsInIndex.size() > 0) {
+                    // If user didn't allow partial apply, we should error out here
+                    if (!partial) {
+                        actionListener.onFailure(
+                                new IllegalArgumentException("Not all paths were found in index mappings: " +
+                                        missingPathsInIndex.stream()
+                                                .collect(Collectors.joining(", ", "[", "]")))
+                        );
+                    }
+                }
 
-            Pair<List<String>, List<String>> validationResult = MapperUtils.validateIndexMappings(indexName, mappingMetadata, aliasMappings);
-            List<String> missingPathsInIndex = validationResult.getLeft();
-            List<String> presentPathsInIndex = validationResult.getRight();
+                // Filter out mappings of sourceIndex fields to which we're applying alias mappings
+                Map<String, Object> presentPathsMappings = MapperUtils.getFieldMappingsFlat(mappingMetadata, presentPathsInIndex);
+                // Filtered alias mappings -- contains only aliases which are applicable to index:
+                //      1. fields in path params exists in index
+                //      2. alias isn't named as one of existing fields in index
+                Map<String, Object> filteredAliasMappings = filterNonApplicableAliases(
+                        mappingMetadata,
+                        missingPathsInIndex,
+                        aliasMappings
+                );
+                Map<String, Object> allMappings = new HashMap<>(presentPathsMappings);
+                allMappings.putAll((Map<String, ?>) filteredAliasMappings.get(PROPERTIES));
+
+                Map<String, Object> mappingsRoot = new HashMap<>();
+                mappingsRoot.put(PROPERTIES, allMappings);
+                // Apply mappings to sourceIndex
+                PutMappingRequest request = new PutMappingRequest(indexName).source(filteredAliasMappings);
+                indicesClient.putMapping(request, new ActionListener<>() {
+                    @Override
+                    public void onResponse(AcknowledgedResponse acknowledgedResponse) {
+                        CreateMappingResult result = new CreateMappingResult(
+                                acknowledgedResponse,
+                                indexName,
+                                mappingsRoot
+                        );
+                        actionListener.onResponse(result);
+                    }
 
-            if(missingPathsInIndex.size() > 0) {
-                // If user didn't allow partial apply, we should error out here
-                if (!partial) {
-                    actionListener.onFailure(
-                            new IllegalArgumentException("Not all paths were found in index mappings: " +
-                                    missingPathsInIndex.stream()
-                                            .collect(Collectors.joining(", ", "[", "]")))
-                    );
-                }
-            }
+                    @Override
+                    public void onFailure(Exception e) {
+                        actionListener.onFailure(e);
+                    }
+                });
+            } else {
+                logTypeService.getRuleFieldMappings(logType, new ActionListener<>() {
+                    @Override
+                    public void onResponse(Map<String, String> mappings) {
+                        try {
+                            Map<String, Map<String, String>> aliasMappingFields = new HashMap<>();
+                            XContentBuilder aliasMappingsObj = XContentFactory.jsonBuilder().startObject();
+                            for (Map.Entry<String, String> mapping: mappings.entrySet()) {
+                                aliasMappingFields.put(mapping.getValue(), Map.of("type", "alias", "path", mapping.getKey()));
+                            }
+                            aliasMappingsObj.field("properties", aliasMappingFields);
+                            String aliasMappings = Strings.toString(aliasMappingsObj.endObject());
+
+                            Pair<List<String>, List<String>> validationResult = MapperUtils.validateIndexMappings(indexName, mappingMetadata, aliasMappings);
+                            List<String> missingPathsInIndex = validationResult.getLeft();
+                            List<String> presentPathsInIndex = validationResult.getRight();
+
+                            if (missingPathsInIndex.size() > 0) {
+                                // If user didn't allow partial apply, we should error out here
+                                if (!partial) {
+                                    actionListener.onFailure(
+                                            new IllegalArgumentException("Not all paths were found in index mappings: " +
+                                                    missingPathsInIndex.stream()
+                                                            .collect(Collectors.joining(", ", "[", "]")))
+                                    );
+                                }
+                            }
 
-            // Filter out mappings of sourceIndex fields to which we're applying alias mappings
-            Map<String, Object> presentPathsMappings = MapperUtils.getFieldMappingsFlat(mappingMetadata, presentPathsInIndex);
-            // Filtered alias mappings -- contains only aliases which are applicable to index:
-            //      1. fields in path params exists in index
-            //      2. alias isn't named as one of existing fields in index
-            Map<String, Object> filteredAliasMappings = filterNonApplicableAliases(
-                    mappingMetadata,
-                    missingPathsInIndex,
-                    aliasMappings
-            );
-            Map<String, Object> allMappings = new HashMap<>(presentPathsMappings);
-            allMappings.putAll((Map<String, ?>) filteredAliasMappings.get(PROPERTIES));
-
-            Map<String, Object> mappingsRoot = new HashMap<>();
-            mappingsRoot.put(PROPERTIES, allMappings);
-            // Apply mappings to sourceIndex
-            PutMappingRequest request = new PutMappingRequest(indexName).source(filteredAliasMappings);
-            indicesClient.putMapping(request, new ActionListener<>() {
-                @Override
-                public void onResponse(AcknowledgedResponse acknowledgedResponse) {
-                    CreateMappingResult result = new CreateMappingResult(
-                            acknowledgedResponse,
-                            indexName,
-                            mappingsRoot
-                    );
-                    actionListener.onResponse(result);
-                }
+                            // Filter out mappings of sourceIndex fields to which we're applying alias mappings
+                            Map<String, Object> presentPathsMappings = MapperUtils.getFieldMappingsFlat(mappingMetadata, presentPathsInIndex);
+                            // Filtered alias mappings -- contains only aliases which are applicable to index:
+                            //      1. fields in path params exists in index
+                            //      2. alias isn't named as one of existing fields in index
+                            Map<String, Object> filteredAliasMappings = filterNonApplicableAliases(
+                                    mappingMetadata,
+                                    missingPathsInIndex,
+                                    aliasMappings
+                            );
+                            Map<String, Object> allMappings = new HashMap<>(presentPathsMappings);
+                            allMappings.putAll((Map<String, ?>) filteredAliasMappings.get(PROPERTIES));
+
+                            Map<String, Object> mappingsRoot = new HashMap<>();
+                            mappingsRoot.put(PROPERTIES, allMappings);
+                            // Apply mappings to sourceIndex
+                            PutMappingRequest request = new PutMappingRequest(indexName).source(filteredAliasMappings);
+                            indicesClient.putMapping(request, new ActionListener<>() {
+                                @Override
+                                public void onResponse(AcknowledgedResponse acknowledgedResponse) {
+                                    CreateMappingResult result = new CreateMappingResult(
+                                            acknowledgedResponse,
+                                            indexName,
+                                            mappingsRoot
+                                    );
+                                    actionListener.onResponse(result);
+                                }
 
-                @Override
-                public void onFailure(Exception e) {
-                    actionListener.onFailure(e);
-                }
-            });
-        } catch (IOException | IllegalArgumentException e) {
+                                @Override
+                                public void onFailure(Exception e) {
+                                    actionListener.onFailure(e);
+                                }
+                            });
+                        } catch (IOException ex) {
+                            actionListener.onFailure(ex);
+                        }
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        actionListener.onFailure(e);
+                    }
+                });
+            }
+        } catch(IOException | IllegalArgumentException e){
             actionListener.onFailure(e);
         }
     }

diff --git a/src/main/java/org/opensearch/securityanalytics/model/Detector.java b/src/main/java/org/opensearch/securityanalytics/model/Detector.java
@@ -507,7 +507,7 @@ public Instant getEnabledTime() {
     }
 
     public String getDetectorType() {
-        return logType;
+        return logType.toLowerCase(Locale.ROOT);
     }
 
     public User getUser() {

diff --git a/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml b/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml
diff --git a/src/main/resources/OSMapping/ad_ldap/mappings.json b/src/main/resources/OSMapping/ad_ldap/mappings.json
diff --git a/src/main/resources/OSMapping/apache_access/fieldmappings.yml b/src/main/resources/OSMapping/apache_access/fieldmappings.yml
diff --git a/src/main/resources/OSMapping/apache_access/mappings.json b/src/main/resources/OSMapping/apache_access/mappings.json
diff --git a/src/main/resources/OSMapping/azure/fieldmappings.yml b/src/main/resources/OSMapping/azure/fieldmappings.yml