[ML] Adds new security module to detect anomalous activity in host-based logs

x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/logo.json

@@ -0,0 +1,3 @@
+{
+  "icon": "logoSecurity"
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/manifest.json

@@ -0,0 +1,60 @@
+{
+  "id": "security_host",
+  "title": "Security: Host",
+  "description": "Detect anomalous activity in your ECS-compatible host-based logs.",
+  "type": "Host data",
+  "logoFile": "logo.json",
+  "defaultIndexPattern": "auditbeat-*,logs-*,filebeat-*,winlogbeat-*",
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "exists": {
+            "field": "event.category"
+          }
+        },
+        {
+          "exists": {
+            "field": "host.name"
+          }
+        },
+        {
+          "exists": {
+            "field": "event.dataset"
+          }
+        },
+        {
+          "term": {
+            "event.outcome": "success"
+          }
+        }
+      ],
+      "must_not": { "terms": { "_tier": ["data_frozen", "data_cold"] } }
+    }
+  },
+  "jobs": [
+    {
+      "id": "high_count_events_for_a_host_name",
+      "file": "high_count_events_for_a_host_name.json"
+    },
+    {
+      "id": "low_count_events_for_a_host_name",
+      "file": "low_count_events_for_a_host_name.json"
+    }
+  ],
+  "datafeeds": [
+    {
+      "id": "datafeed-high_count_events_for_a_host_name",
+      "file": "datafeed_high_count_events_for_a_host_name.json",
+      "job_id": "high_count_events_for_a_host_name"
+    },
+    {
+      "id": "datafeed-low_count_events_for_a_host_name",
+      "file": "datafeed_low_count_events_for_a_host_name.json",
+      "job_id": "low_count_events_for_a_host_name"
+    }
+  ],
+  "tags": [
+    "security"
+  ]
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/datafeed_high_count_events_for_a_host_name.json

@@ -0,0 +1,33 @@
+{
+  "job_id": "JOB_ID",
+  "indices": [
+    "INDEX_PATTERN_NAME"
+  ],
+  "max_empty_searches": 10,
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "exists": {
+            "field": "event.category"
+          }
+        },
+        {
+          "exists": {
+            "field": "host.name"
+          }
+        },
+        {
+          "exists": {
+            "field": "event.dataset"
+          }
+        },
+        {
+          "term": {
+            "event.outcome": "success"
+          }
+        }
+      ]
+    }
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/datafeed_low_count_events_for_a_host_name.json

@@ -0,0 +1,33 @@
+{
+  "job_id": "JOB_ID",
+  "indices": [
+    "INDEX_PATTERN_NAME"
+  ],
+  "max_empty_searches": 10,
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "exists": {
+            "field": "event.category"
+          }
+        },
+        {
+          "exists": {
+            "field": "host.name"
+          }
+        },
+        {
+          "exists": {
+            "field": "event.dataset"
+          }
+        },
+        {
+          "term": {
+            "event.outcome": "success"
+          }
+        }
+      ]
+    }
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/high_count_events_for_a_host_name.json

@@ -0,0 +1,29 @@
+{
+  "description": "Security: Host - Looks for a sudden spike in host based traffic. This can be due to a range of security issues, such as a compromised system, DDoS attacks, malware infections, privilege escalation, or data exfiltration.",
+  "groups": ["security", "host"],
+  "analysis_config": {
+    "bucket_span": "3h",
+    "detectors": [
+      {
+        "detector_description": "high count of host based events",
+        "function": "high_count",
+        "partition_field_name": "host.name",
+        "detector_index": 0
+      }
+    ],
+    "influencers": ["host.name", "host.ip", "event.dataset", "event.action", "event.category"]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "128mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-security-host",
+    "security_app_display_name": "Spike in the Host Traffic",
+    "managed": true,
+    "job_revision": 1
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/low_count_events_for_a_host_name.json

@@ -0,0 +1,29 @@
+{
+  "description": "Security: Host - Looks for a sudden drop in host based traffic. This can be due to a range of security issues, such as a compromised system, a failed service, or a network misconfiguration.",
+  "groups": ["security", "host"],
+  "analysis_config": {
+    "bucket_span": "3h",
+    "detectors": [
+      {
+        "detector_description": "low count of host based events",
+        "function": "low_count",
+        "partition_field_name": "host.name",
+        "detector_index": 0
+      }
+    ],
+    "influencers": ["host.name", "host.ip", "event.dataset", "event.action", "event.category"]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "128mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-security-host",
+    "security_app_display_name": "Decrease in the Host Traffic",
+    "managed": true,
+    "job_revision": 1
+  }
+}

x-pack/test/api_integration/apis/ml/modules/get_module.ts

@@ -28,6 +28,7 @@ const moduleIds = [
   'sample_data_weblogs',
   'security_auth',
   'security_cloudtrail',
+  'security_host',
   'security_linux_v3',
   'security_network',
   'security_packetbeat',
@@ -41,6 +42,7 @@ const securityModuleIds = [
   'logs_ui_categories',
   'security_auth',
   'security_cloudtrail',
+  'security_host',
   'security_linux_v3',
   'security_network',
   'security_packetbeat',

x-pack/test/functional/services/ml/supplied_configurations.ts

@@ -27,7 +27,7 @@ export function MachineLearningSuppliedConfigurationsProvider({ getService }: Ft
       );
     },
     async assertAllConfigurationsAreLoaded() {
-      const expectedLength = 18;
+      const expectedLength = 19;
       await retry.tryForTime(10 * 1000, async () => {
         const cards = await testSubjects.findAll('mlSuppliedConfigurationsCard');
         expect(cards.length).to.eql(