Merge branch 'main' of https://github.com/opensearch-project/security…

…-analytics
opensearch-project · Jan 6, 2023 · 8931b26 · 8931b26
2 parents 6731e8b + 51a176b
commit 8931b26
Show file tree

Hide file tree

Showing 49 changed files with 2,388 additions and 1,350 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -18,11 +18,11 @@ jobs:
         os: [ ubuntu-latest, windows-latest, macos-latest ]
         include:
           - os: windows-latest
-            os_build_args: -x integTest -x jacocoTestReport
+            os_build_args: -x jacocoTestReport
             working_directory: X:\
             os_java_options: -Xmx4096M
           - os: macos-latest
-            os_build_args: -x integTest -x jacocoTestReport
+            os_build_args: -x jacocoTestReport
 
     name: Build and Test security-analytics with JDK ${{ matrix.java }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
@@ -59,6 +59,27 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() && matrix.os == 'ubuntu-latest' }}
+        with:
+          name: logs-ubuntu
+          path: build/testclusters/integTest-*/logs/*
+
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() && matrix.os == 'macos-latest' }}
+        with:
+          name: logs-mac
+          path: build/testclusters/integTest-*/logs/*
+
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() && matrix.os == 'windows-latest' }}
+        with:
+          name: logs-windows
+          path: build\testclusters\integTest-*\logs\*
+
       - name: Upload Artifacts
         uses: actions/upload-artifact@v1
         with:

diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml
@@ -28,3 +28,9 @@ jobs:
         uses: actions/checkout@v2
       - name: Run integration tests with multi node config
         run: ./gradlew integTest -PnumNodes=3
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: logs
+          path: build/testclusters/integTest-*/logs/*
diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml
@@ -82,7 +82,7 @@ jobs:
           if [ $security -gt 0 ]
           then
             echo "Security plugin is available"
-            ./gradlew :integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity=true -Dhttps=true -Duser=admin -Dpassword=admin          
+            ./gradlew :integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dhttps=true -Duser=admin -Dpassword=admin          
           else
             echo "Security plugin is NOT available skipping this run as tests without security have already been run"
           fi
diff --git a/build.gradle b/build.gradle
@@ -165,7 +165,6 @@ integTest {
     systemProperty 'java.io.tmpdir', es_tmp_dir.absolutePath
 
     systemProperty "https", System.getProperty("https")
-    systemProperty "security", System.getProperty("security")
     systemProperty "user", System.getProperty("user")
     systemProperty "password", System.getProperty("password")
 
@@ -309,7 +308,6 @@ task integTestRemote(type: RestIntegTestTask) {
     systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath
 
     systemProperty "https", System.getProperty("https")
-    systemProperty "security", System.getProperty("security")
     systemProperty "user", System.getProperty("user")
     systemProperty "password", System.getProperty("password")
 

diff --git a/release-notes/opensearch-security-analytics.release-notes-2.4.1.0.md b/release-notes/opensearch-security-analytics.release-notes-2.4.1.0.md
@@ -0,0 +1,7 @@
+## Version 2.4.1.0 Release Notes
+
+Compatible with OpenSearch 2.4.1
+
+### Bug Fixes
+
+* fix for running windows integration tests ([#176](https://github.com/opensearch-project/security-analytics/pull/176))
diff --git a/src/main/java/org/opensearch/securityanalytics/action/IndexRuleRequest.java b/src/main/java/org/opensearch/securityanalytics/action/IndexRuleRequest.java
@@ -64,7 +64,7 @@ public IndexRuleRequest(
         super();
         this.ruleId = ruleId;
         this.refreshPolicy = refreshPolicy;
-        this.logType = logType;
+        this.logType = logType.toLowerCase(Locale.ROOT);
         this.method = method;
         this.rule = rule;
         this.forced = forced;

diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java
@@ -58,32 +58,32 @@ public class DetectorMonitorConfig {
     }
 
     public static String getRuleIndex(String detectorType) {
-        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
-                detectorTypeToIndicesMapping.get(detectorType).getRuleIndex() :
+        return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ?
+                detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getRuleIndex() :
                 OPENSEARCH_DEFAULT_RULE_INDEX;
     }
 
     public static String getAlertsIndex(String detectorType) {
-        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
-                detectorTypeToIndicesMapping.get(detectorType).getAlertsIndex() :
+        return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ?
+                detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAlertsIndex() :
                 OPENSEARCH_DEFAULT_ALERT_INDEX;
     }
 
     public static String getAlertsHistoryIndex(String detectorType) {
-        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
-                detectorTypeToIndicesMapping.get(detectorType).getAlertsHistoryIndex() :
+        return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ?
+                detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAlertsHistoryIndex() :
                 OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX;
     }
 
     public static String getAlertsHistoryIndexPattern(String detectorType) {
-        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
-                detectorTypeToIndicesMapping.get(detectorType).getAlertsHistoryIndexPattern() :
+        return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ?
+                detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAlertsHistoryIndexPattern() :
                 OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX_PATTERN;
     }
 
     public static String getAllAlertsIndicesPattern(String detectorType) {
-        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
-                detectorTypeToIndicesMapping.get(detectorType).getAllAlertsIndicesPattern() :
+        return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ?
+                detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAllAlertsIndicesPattern() :
                 OPENSEARCH_DEFAULT_ALL_ALERT_INDICES_PATTERN;
     }
 
@@ -95,14 +95,14 @@ public static List<String> getAllAlertsIndicesPatternForAllTypes() {
     }
 
     public static String getFindingsIndex(String detectorType) {
-        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
-                detectorTypeToIndicesMapping.get(detectorType).getFindingsIndex() :
+        return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ?
+                detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getFindingsIndex() :
                 OPENSEARCH_DEFAULT_FINDINGS_INDEX;
     }
 
     public static String getAllFindingsIndicesPattern(String detectorType) {
-        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
-                detectorTypeToIndicesMapping.get(detectorType).getAllFindingsIndicesPattern() :
+        return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ?
+                detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getAllFindingsIndicesPattern() :
                 OPENSEARCH_DEFAULT_ALL_FINDINGS_INDICES_PATTERN;
     }
 
@@ -114,8 +114,8 @@ public static List<String> getAllFindingsIndicesPatternForAllTypes() {
     }
 
     public static String getFindingsIndexPattern(String detectorType) {
-        return detectorTypeToIndicesMapping.containsKey(detectorType) ?
-                detectorTypeToIndicesMapping.get(detectorType).getFindingsIndexPattern() :
+        return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ?
+                detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getFindingsIndexPattern() :
                 OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN;
     }
 

diff --git a/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java b/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java
@@ -5,6 +5,7 @@
 package org.opensearch.securityanalytics.findings;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -16,6 +17,7 @@
 import org.opensearch.client.Client;
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.commons.alerting.AlertingPluginInterface;
+import org.opensearch.commons.alerting.model.DocLevelQuery;
 import org.opensearch.commons.alerting.model.FindingWithDocs;
 import org.opensearch.commons.alerting.model.Table;
 import org.opensearch.rest.RestStatus;
@@ -83,9 +85,9 @@ public void onFailure(Exception e) {
                 };
 
                 // monitor --> detectorId mapping
-                Map<String, String> monitorToDetectorMapping = new HashMap<>();
+                Map<String, Detector> monitorToDetectorMapping = new HashMap<>();
                 detector.getMonitorIds().forEach(
-                        monitorId -> monitorToDetectorMapping.put(monitorId, detector.getId())
+                        monitorId -> monitorToDetectorMapping.put(monitorId, detector)
                 );
                 // Get findings for all monitor ids
                 FindingsService.this.getFindingsByMonitorIds(
@@ -112,7 +114,7 @@ public void onFailure(Exception e) {
      * @param listener ActionListener to get notified on response or error
      */
     public void getFindingsByMonitorIds(
-            Map<String, String> monitorToDetectorMapping,
+            Map<String, Detector> monitorToDetectorMapping,
             List<String> monitorIds,
             String findingIndexName,
             Table table,
@@ -169,11 +171,11 @@ public void getFindings(
 
         List<String> allMonitorIds = new ArrayList<>();
         // Used to convert monitorId back to detectorId to store in result FindingDto
-        Map<String, String> monitorToDetectorMapping = new HashMap<>();
+        Map<String, Detector> monitorToDetectorMapping = new HashMap<>();
         detectors.forEach(detector -> {
             // monitor --> detector map
             detector.getMonitorIds().forEach(
-                monitorId -> monitorToDetectorMapping.put(monitorId, detector.getId())
+                monitorId -> monitorToDetectorMapping.put(monitorId, detector)
             );
             // all monitorIds
             allMonitorIds.addAll(detector.getMonitorIds());
@@ -201,13 +203,21 @@ public void onFailure(Exception e) {
         );
     }
 
-    public FindingDto mapFindingWithDocsToFindingDto(FindingWithDocs findingWithDocs, String detectorId) {
+    public FindingDto mapFindingWithDocsToFindingDto(FindingWithDocs findingWithDocs, Detector detector) {
+        List<DocLevelQuery> docLevelQueries = findingWithDocs.getFinding().getDocLevelQueries();
+        if (docLevelQueries.isEmpty()) { // this is finding generated by a bucket level monitor
+            for (Map.Entry<String, String> entry : detector.getRuleIdMonitorIdMap().entrySet()) {
+                if(entry.getValue().equals(findingWithDocs.getFinding().getMonitorId())) {
+                    docLevelQueries = Collections.singletonList(new DocLevelQuery(entry.getKey(),"","",Collections.emptyList()));
+                }
+            }
+        }
         return new FindingDto(
-                detectorId,
+                detector.getId(),
                 findingWithDocs.getFinding().getId(),
                 findingWithDocs.getFinding().getRelatedDocIds(),
                 findingWithDocs.getFinding().getIndex(),
-                findingWithDocs.getFinding().getDocLevelQueries(),
+                docLevelQueries,
                 findingWithDocs.getFinding().getTimestamp(),
                 findingWithDocs.getDocuments()
         );

diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperTopicStore.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperTopicStore.java
@@ -9,6 +9,7 @@
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.stream.Collectors;
@@ -54,11 +55,11 @@ private MapperTopicStore() {
     }
 
     public static String aliasMappings(String mapperTopic) throws IOException {
-        if (INSTANCE.mapperMap.containsKey(mapperTopic)) {
+        if (INSTANCE.mapperMap.containsKey(mapperTopic.toLowerCase(Locale.ROOT))) {
             return new String(Objects.requireNonNull(
 
                     INSTANCE.getClass().getClassLoader().getResourceAsStream(INSTANCE.
-                            mapperMap.get(mapperTopic))).readAllBytes(),
+                            mapperMap.get(mapperTopic.toLowerCase(Locale.ROOT)))).readAllBytes(),
                     StandardCharsets.UTF_8);
         }
         throw new IllegalArgumentException("Mapper not found: [" + mapperTopic + "]");

diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MappingsTraverser.java b/src/main/java/org/opensearch/securityanalytics/mapper/MappingsTraverser.java
@@ -120,14 +120,6 @@ public void addListener(MappingsTraverserListener l) {
         this.mappingsTraverserListeners.add(l);
     }
 
-    /**
-     * Sets set of property "type" values to skip during traversal.
-     * @param types Set of strings representing property "type"
-     */
-    public void setTypesToSkip(Set<String> types) {
-        this.typesToSkip = types;
-    }
-
     /**
      * Traverses mappings tree and collects all fields that are not of type "alias".
      * Nested fields are flattened.
@@ -136,7 +128,7 @@ public void setTypesToSkip(Set<String> types) {
     public List<String> extractFlatNonAliasFields() {
         List<String> flatProperties = new ArrayList<>();
         // Setup
-        this.typesToSkip.add(ALIAS);
+        this.propertiesToSkip.add(Pair.of(TYPE, ALIAS));
         this.mappingsTraverserListeners.add(new MappingsTraverserListener() {
             @Override
             public void onLeafVisited(Node node) {

diff --git a/src/main/java/org/opensearch/securityanalytics/model/Detector.java b/src/main/java/org/opensearch/securityanalytics/model/Detector.java
@@ -248,7 +248,7 @@ private XContentBuilder createXContentBuilder(XContentBuilder builder, ToXConten
         }
         builder.field(TYPE_FIELD, type)
                 .field(NAME_FIELD, name)
-                .field(DETECTOR_TYPE_FIELD, detectorType);
+                .field(DETECTOR_TYPE_FIELD, detectorType.getDetectorType());
 
         if (!secure) {
             if (user == null) {

diff --git a/src/main/java/org/opensearch/securityanalytics/model/DetectorTrigger.java b/src/main/java/org/opensearch/securityanalytics/model/DetectorTrigger.java
@@ -27,6 +27,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 public class DetectorTrigger implements Writeable, ToXContentObject {
 
@@ -66,7 +67,9 @@ public DetectorTrigger(String id, String name, String severity, List<String> rul
         this.id = id == null? UUIDs.base64UUID(): id;
         this.name = name;
         this.severity = severity;
-        this.ruleTypes = ruleTypes;
+        this.ruleTypes = ruleTypes.stream()
+                .map( e -> e.toLowerCase(Locale.ROOT))
+                .collect(Collectors.toList());
         this.ruleIds = ruleIds;
         this.ruleSeverityLevels = ruleSeverityLevels;
         this.tags = tags;

diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java
@@ -32,6 +32,7 @@
 import org.opensearch.securityanalytics.rules.utils.Either;
 import org.apache.commons.lang3.tuple.Pair;
 import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.SafeConstructor;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -70,7 +71,7 @@ public QueryBackend(String ruleCategory, boolean convertAndAsIn, boolean enableF
             assert is != null;
             String content = new String(is.readAllBytes(), Charset.defaultCharset());
 
-            Yaml yaml = new Yaml();
+            Yaml yaml = new Yaml(new SafeConstructor());
             Map<String, Object> fieldMappingsObj = yaml.load(content);
             this.fieldMappings = (Map<String, String>) fieldMappingsObj.get("fieldmappings");
 

diff --git a/src/main/java/org/opensearch/securityanalytics/rules/objects/SigmaRule.java b/src/main/java/org/opensearch/securityanalytics/rules/objects/SigmaRule.java
@@ -12,6 +12,7 @@
 import org.opensearch.securityanalytics.rules.exceptions.SigmaLogsourceError;
 import org.opensearch.securityanalytics.rules.exceptions.SigmaStatusError;
 import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.SafeConstructor;
 
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -167,7 +168,7 @@ protected static SigmaRule fromDict(Map<String, Object> rule, boolean collectErr
     }
 
     public static SigmaRule fromYaml(String rule, boolean collectErrors) throws SigmaError {
-        Yaml yaml = new Yaml();
+        Yaml yaml = new Yaml(new SafeConstructor());
         Map<String, Object> ruleMap = yaml.load(rule);
         return fromDict(ruleMap, collectErrors);
     }