fix mapper it tests

Signed-off-by: Surya Sashank Nistala <[email protected]>
opensearch-project · Jan 11, 2023 · 55fffbc · 55fffbc
1 parent 0d1c796
commit 55fffbc
Show file tree

Hide file tree

Showing 30 changed files with 69 additions and 68 deletions.
diff --git a/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml b/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml
@@ -1,3 +1,3 @@
 fieldmappings:
     TargetUserName: winlog-event_data-TargetUserName
-    @timestamp: timestamp
+    creationTime: timestamp
diff --git a/src/main/resources/OSMapping/ad_ldap/mappings.json b/src/main/resources/OSMapping/ad_ldap/mappings.json
@@ -5,7 +5,7 @@
       "type": "alias"
     },
     "timestamp": {
-      "path": "@timestamp",
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/main/resources/OSMapping/apache_access/fieldmappings.yml b/src/main/resources/OSMapping/apache_access/fieldmappings.yml
@@ -5,4 +5,4 @@ fieldmappings:
     fieldB: mappedB
     fieldA1: mappedA
     CommandLine: windows-event_data-CommandLine
-    @timestamp: timestamp
+    creationTime: timestamp
diff --git a/src/main/resources/OSMapping/apache_access/mappings.json b/src/main/resources/OSMapping/apache_access/mappings.json
@@ -9,7 +9,7 @@
       "path": "EventID"
     },
     "timestamp": {
-      "path": "@timestamp",
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/main/resources/OSMapping/dns/fieldmappings.yml b/src/main/resources/OSMapping/dns/fieldmappings.yml
@@ -2,4 +2,4 @@ fieldmappings:
     record_type: dns-answers-type
     query: dns-question-name
     parent_domain: dns-question-registered_domain
-    @timestamp: timestamp
+    creationTime: timestamp
diff --git a/src/main/resources/OSMapping/dns/mappings.json b/src/main/resources/OSMapping/dns/mappings.json
@@ -13,7 +13,7 @@
       "path": "dns.question.registered_domain"
     },
     "timestamp": {
-      "path": "@timestamp",
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/main/resources/OSMapping/linux/fieldmappings.yml b/src/main/resources/OSMapping/linux/fieldmappings.yml
@@ -12,4 +12,4 @@ fieldmappings:
   ParentImage: process-parent-executable
   CurrentDirectory: process-working_directory
   LogonId: process-real_user-id
-  @timestamp: timestamp
+  creationTime: timestamp
diff --git a/src/main/resources/OSMapping/linux/mappings.json b/src/main/resources/OSMapping/linux/mappings.json
@@ -49,7 +49,7 @@
       "type": "alias"
     },
     "timestamp": {
-      "path": "@timestamp",
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/main/resources/OSMapping/network/NetFlowMapping.json b/src/main/resources/OSMapping/network/NetFlowMapping.json
@@ -25,7 +25,7 @@
       "path": "netflow.http_status_code"
     },
     "timestamp": {
-      "path": "@timestamp",
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/main/resources/OSMapping/network/fieldmappings.yml b/src/main/resources/OSMapping/network/fieldmappings.yml
@@ -15,4 +15,4 @@ fieldmappings:
     resp_mime_types: zeek-http-resp_mime_types
     cipher: zeek-kerberos-cipher
     request_type: zeek-kerberos-request_type
-    @timestamp: timestamp
+    creationTime: timestamp
diff --git a/src/main/resources/OSMapping/network/mappings.json b/src/main/resources/OSMapping/network/mappings.json
@@ -61,7 +61,7 @@
       "type": "alias"
     },
     "timestamp": {
-      "path": "@timestamp",
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/main/resources/OSMapping/others_application/fieldmappings.yml b/src/main/resources/OSMapping/others_application/fieldmappings.yml
@@ -4,4 +4,4 @@ fieldmappings:
   HiveName: unmapped.HiveName
   fieldB: mappedB
   fieldA1: mappedA
-  @timestamp: timestamp
+  creationTime: timestamp
diff --git a/src/main/resources/OSMapping/others_application/mappings.json b/src/main/resources/OSMapping/others_application/mappings.json
@@ -25,7 +25,7 @@
       "path": "ServiceName"
     },
     "timestamp": {
-      "path": "@timestamp",
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/main/resources/OSMapping/others_apt/fieldmappings.yml b/src/main/resources/OSMapping/others_apt/fieldmappings.yml
@@ -4,4 +4,4 @@ fieldmappings:
   HiveName: unmapped.HiveName
   fieldB: mappedB
   fieldA1: mappedA
-  @timestamp: timestamp
+  creationTime: timestamp
diff --git a/src/main/resources/OSMapping/others_apt/mappings.json b/src/main/resources/OSMapping/others_apt/mappings.json
@@ -25,7 +25,7 @@
       "path": "ServiceName"
     },
     "timestamp": {
-      "path": "@timestamp",
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/main/resources/OSMapping/others_cloud/fieldmappings.yml b/src/main/resources/OSMapping/others_cloud/fieldmappings.yml
@@ -4,4 +4,4 @@ fieldmappings:
   HiveName: unmapped.HiveName
   fieldB: mappedB
   fieldA1: mappedA
-  @timestamp: timestamp
+  creationTime: timestamp
diff --git a/src/main/resources/OSMapping/others_cloud/mappings.json b/src/main/resources/OSMapping/others_cloud/mappings.json
@@ -24,8 +24,8 @@
       "type": "alias",
       "path": "ServiceName"
     },
-    "@timestamp": {
-      "path": "@timestamp",
+    "creationTime": {
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/main/resources/OSMapping/others_compliance/fieldmappings.yml b/src/main/resources/OSMapping/others_compliance/fieldmappings.yml
@@ -4,4 +4,4 @@ fieldmappings:
   HiveName: unmapped.HiveName
   fieldB: mappedB
   fieldA1: mappedA
-  @timestamp: timestamp
+  creationTime: timestamp
diff --git a/src/main/resources/OSMapping/others_compliance/mappings.json b/src/main/resources/OSMapping/others_compliance/mappings.json
@@ -25,7 +25,7 @@
       "path": "ServiceName"
     },
     "timestamp": {
-      "path": "@timestamp",
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/main/resources/OSMapping/others_macos/fieldmappings.yml b/src/main/resources/OSMapping/others_macos/fieldmappings.yml
@@ -4,4 +4,4 @@ fieldmappings:
     HiveName: unmapped.HiveName
     fieldB: mappedB
     fieldA1: mappedA
-    @timestamp: timestamp
+    creationTime: timestamp
diff --git a/src/main/resources/OSMapping/others_macos/mappings.json b/src/main/resources/OSMapping/others_macos/mappings.json
@@ -5,7 +5,7 @@
       "path": "CommandLine"
     },
     "timestamp": {
-      "path": "@timestamp",
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/main/resources/OSMapping/others_proxy/fieldmappings.yml b/src/main/resources/OSMapping/others_proxy/fieldmappings.yml
@@ -4,4 +4,4 @@ fieldmappings:
   HiveName: unmapped.HiveName
   fieldB: mappedB
   fieldA1: mappedA
-  @timestamp: timestamp
+  creationTime: timestamp
diff --git a/src/main/resources/OSMapping/others_proxy/mappings.json b/src/main/resources/OSMapping/others_proxy/mappings.json
@@ -25,7 +25,7 @@
       "path": "ServiceName"
     },
     "timestamp": {
-      "path": "@timestamp",
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/main/resources/OSMapping/others_web/fieldmappings.yml b/src/main/resources/OSMapping/others_web/fieldmappings.yml
@@ -4,5 +4,5 @@ fieldmappings:
   HiveName: unmapped.HiveName
   fieldB: mappedB
   fieldA1: mappedA
-  @timestamp: timestamp
+  creationTime: timestamp
 
diff --git a/src/main/resources/OSMapping/others_web/mappings.json b/src/main/resources/OSMapping/others_web/mappings.json
@@ -25,7 +25,7 @@
       "path": "ServiceName"
     },
     "timestamp": {
-      "path": "@timestamp",
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/main/resources/OSMapping/test_windows/fieldmappings.yml b/src/main/resources/OSMapping/test_windows/fieldmappings.yml
@@ -9,4 +9,4 @@ fieldmappings:
     Message: windows-message
     Provider_Name: windows-provider-name
     ServiceName: windows-servicename
-    @timestamp: timestamp
+    creationTime: timestamp
diff --git a/src/main/resources/OSMapping/test_windows/mappings.json b/src/main/resources/OSMapping/test_windows/mappings.json
@@ -25,7 +25,7 @@
       "path": "ServiceName"
     },
     "timestamp": {
-      "path": "@timestamp",
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/main/resources/OSMapping/windows/fieldmappings.yml b/src/main/resources/OSMapping/windows/fieldmappings.yml
@@ -61,5 +61,5 @@ fieldmappings:
     windows-hostname: winlog-computer_name
     windows-provider-name: winlog-provider_name
     windows-servicename: winlog-event_data-ServiceName
-    @timestamp: timestamp
+    creationTime: timestamp
 
diff --git a/src/main/resources/OSMapping/windows/mappings.json b/src/main/resources/OSMapping/windows/mappings.json
@@ -189,7 +189,7 @@
       "type": "alias"
     },
     "timestamp": {
-      "path": "@timestamp",
+      "path": "creationTime",
       "type": "alias"
     }
   }

diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java
@@ -49,9 +49,9 @@ public void testCreateMappingSuccess() throws IOException {
         // both req params and req body are supported
         request.setJsonEntity(
                 "{ \"index_name\":\"" + testIndexName + "\"," +
-                "  \"rule_topic\":\"netflow\", " +
-                "  \"partial\":true" +
-                "}"
+                        "  \"rule_topic\":\"netflow\", " +
+                        "  \"partial\":true" +
+                        "}"
         );
         Response response = client().performRequest(request);
         assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
@@ -88,22 +88,22 @@ public void testCreateMappingWithAliasesSuccess() throws IOException {
         // both req params and req body are supported
         request.setJsonEntity(
                 "{\n" +
-                "   \"index_name\": \"my_index\",\n" +
-                "  \"rule_topic\":\"netflow\", " +
-                "  \"partial\":true," +
-                "   \"alias_mappings\": {\n" +
-                "        \"properties\": {\n" +
-                "           \"source.ip\": {\n" +
-                "              \"type\": \"alias\",\n" +
-                "              \"path\": \"netflow.source_ipv4_address\"\n" +
-                "           },\n" +
-                "           \"source.port\": {\n" +
-                "              \"type\": \"alias\",\n" +
-                "              \"path\": \"netflow.source_transport_port\"\n" +
-                "           }\n" +
-                "       }\n" +
-                "   }\n" +
-                "}"
+                        "   \"index_name\": \"my_index\",\n" +
+                        "  \"rule_topic\":\"netflow\", " +
+                        "  \"partial\":true," +
+                        "   \"alias_mappings\": {\n" +
+                        "        \"properties\": {\n" +
+                        "           \"source.ip\": {\n" +
+                        "              \"type\": \"alias\",\n" +
+                        "              \"path\": \"netflow.source_ipv4_address\"\n" +
+                        "           },\n" +
+                        "           \"source.port\": {\n" +
+                        "              \"type\": \"alias\",\n" +
+                        "              \"path\": \"netflow.source_transport_port\"\n" +
+                        "           }\n" +
+                        "       }\n" +
+                        "   }\n" +
+                        "}"
         );
         // request.addParameter("indexName", testIndexName);
         // request.addParameter("ruleTopic", "netflow");
@@ -224,7 +224,7 @@ public void testExistingMappingsAreUntouched() throws IOException {
         GetMappingsResponse getMappingsResponse = SecurityAnalyticsClientUtils.executeGetMappingsRequest(testIndexName);
         Map<String, Object> properties =
                 (Map<String, Object>) getMappingsResponse.getMappings().get(testIndexName)
-                .getSourceAsMap().get("properties");
+                        .getSourceAsMap().get("properties");
         // Verify that there is still mapping for integer field "plain1"
         assertTrue(((Map<String, Object>)properties.get("plain1")).get("type").equals("integer"));
     }
@@ -291,7 +291,7 @@ public void testGetMappingsViewSuccess() throws IOException {
         assertEquals(6, unmappedIndexFields.size());
         // Verify unmapped field aliases
         List<String> unmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
-        assertEquals(2, unmappedFieldAliases.size());
+        assertEquals(3, unmappedFieldAliases.size());
     }
 
     public void testCreateMappings_withIndexPattern_indexTemplate_createAndUpdate_success() throws IOException {
@@ -621,7 +621,7 @@ public void testGetMappingsView_index_pattern_two_indices_Success() throws IOExc
         assertTrue(extraField.isPresent());
         // Verify unmapped field aliases
         List<String> unmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
-        assertEquals(2, unmappedFieldAliases.size());
+        assertEquals(3, unmappedFieldAliases.size());
     }
 
     public void testGetMappingsView_alias_without_writeindex_Success() throws IOException {
@@ -656,7 +656,7 @@ public void testGetMappingsView_alias_without_writeindex_Success() throws IOExce
         assertTrue(extraField.isPresent());
         // Verify unmapped field aliases
         List<String> unmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
-        assertEquals(2, unmappedFieldAliases.size());
+        assertEquals(3, unmappedFieldAliases.size());
     }
 
     public void testGetMappingsView_alias_with_writeindex_Success() throws IOException {
@@ -694,7 +694,7 @@ public void testGetMappingsView_alias_with_writeindex_Success() throws IOExcepti
         assertTrue(extraField.isPresent());
         // Verify unmapped field aliases
         List<String> unmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
-        assertEquals(2, unmappedFieldAliases.size());
+        assertEquals(3, unmappedFieldAliases.size());
     }
 
     public void testGetMappingsView_datastream_one_backing_index_Success() throws IOException {
@@ -721,7 +721,7 @@ public void testGetMappingsView_datastream_one_backing_index_Success() throws IO
         assertEquals(7, unmappedIndexFields.size());
         // Verify unmapped field aliases
         List<String> unmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
-        assertEquals(2, unmappedFieldAliases.size());
+        assertEquals(3, unmappedFieldAliases.size());
 
         deleteDatastream(datastreamName);
     }
@@ -771,11 +771,10 @@ public void testGetMappingsView_datastream_two_backing_index_Success() throws IO
         assertEquals(1, props.size());
         assertTrue(props.containsKey("source.ip"));
         // Verify unmapped index fields
-        List<String> unmappedIndexFields = (List<String>) respMap.get("unmapped_index_fields");
-        assertEquals(1, unmappedIndexFields.size());
+
         // Verify unmapped field aliases
         List<String> unmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
-        assertEquals(5, unmappedFieldAliases.size());
+        assertEquals(6, unmappedFieldAliases.size());
 
         deleteDatastream(datastreamName);
     }
@@ -856,20 +855,20 @@ private void createSampleIndex(String indexName, Settings settings, String alias
                         "                    \"keyword\":{" +
                         "                      \"type\":\"keyword\"," +
                         "                      \"ignore_above\":256" +
-                                              "}" +
-                                            "}" +
-                                        "}," +
+                        "}" +
+                        "}" +
+                        "}," +
                         "              \"last\":{" +
-                                          "\"type\":\"text\"," +
-                                            "\"fields\":{" +
+                        "\"type\":\"text\"," +
+                        "\"fields\":{" +
                         "                      \"keyword\":{" +
                         "                           \"type\":\"keyword\"," +
                         "                           \"ignore_above\":256" +
-                                                "}" +
-                                            "}" +
-                                        "}" +
-                                    "}" +
-                                "}" +
+                        "}" +
+                        "}" +
+                        "}" +
+                        "}" +
+                        "}" +
                         "    }";
 
         createIndex(indexName, settings, indexMapping, aliases);
@@ -1152,6 +1151,8 @@ public void testCreateDNSMapping() throws IOException{
         //Loop over the mappings and run update request for each one specifying the index to be updated
         mappings.entrySet().forEach(entry -> {
             String key = entry.getKey();
+            if("timestamp".equals(key))
+                return;
             String path = ((Map<String, Object>) entry.getValue()).get("path").toString();
             try {
                 Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI);