[8.x] [ML] Adds new security module to detect anomalous activity in h…

…ost-based logs (#195582) (#201898)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[ML] Adds new security module to detect anomalous activity in
host-based logs
(#195582)](#195582)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Kirti
Sodhi","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-11-27T00:24:35Z","message":"[ML]
Adds new security module to detect anomalous activity in host-based logs
(#195582)\n\n## Summary\r\n\r\nAdds a new security module `Security:
Host` to the prebuilt security\r\njobs. The module has the following
jobs:\r\n- Detect Spike in Host based traffic\r\n- Detect Decrease in
Host based
traffic","sha":"5ed42978f8f4471f1f1852ffe90e04bcbb0713e6","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":[":ml","Feature:Anomaly
Detection","v9.0.0","release_note:feature","backport:version","v8.18.0"],"title":"[ML]
Adds new security module to detect anomalous activity in host-based
logs","number":195582,"url":"https://github.com/elastic/kibana/pull/195582","mergeCommit":{"message":"[ML]
Adds new security module to detect anomalous activity in host-based logs
(#195582)\n\n## Summary\r\n\r\nAdds a new security module `Security:
Host` to the prebuilt security\r\njobs. The module has the following
jobs:\r\n- Detect Spike in Host based traffic\r\n- Detect Decrease in
Host based
traffic","sha":"5ed42978f8f4471f1f1852ffe90e04bcbb0713e6"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/195582","number":195582,"mergeCommit":{"message":"[ML]
Adds new security module to detect anomalous activity in host-based logs
(#195582)\n\n## Summary\r\n\r\nAdds a new security module `Security:
Host` to the prebuilt security\r\njobs. The module has the following
jobs:\r\n- Detect Spike in Host based traffic\r\n- Detect Decrease in
Host based
traffic","sha":"5ed42978f8f4471f1f1852ffe90e04bcbb0713e6"}},{"branch":"8.x","label":"v8.18.0","branchLabelMappingKey":"^v8.18.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Kirti Sodhi <[email protected]>

x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/logo.json

@@ -0,0 +1,3 @@
+{
+  "icon": "logoSecurity"
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/manifest.json

@@ -0,0 +1,60 @@
+{
+  "id": "security_host",
+  "title": "Security: Host",
+  "description": "Detect anomalous activity in your ECS-compatible host-based logs.",
+  "type": "Host data",
+  "logoFile": "logo.json",
+  "defaultIndexPattern": "auditbeat-*,logs-*,filebeat-*,winlogbeat-*",
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "exists": {
+            "field": "event.category"
+          }
+        },
+        {
+          "exists": {
+            "field": "host.name"
+          }
+        },
+        {
+          "exists": {
+            "field": "event.dataset"
+          }
+        },
+        {
+          "term": {
+            "event.outcome": "success"
+          }
+        }
+      ],
+      "must_not": { "terms": { "_tier": ["data_frozen", "data_cold"] } }
+    }
+  },
+  "jobs": [
+    {
+      "id": "high_count_events_for_a_host_name",
+      "file": "high_count_events_for_a_host_name.json"
+    },
+    {
+      "id": "low_count_events_for_a_host_name",
+      "file": "low_count_events_for_a_host_name.json"
+    }
+  ],
+  "datafeeds": [
+    {
+      "id": "datafeed-high_count_events_for_a_host_name",
+      "file": "datafeed_high_count_events_for_a_host_name.json",
+      "job_id": "high_count_events_for_a_host_name"
+    },
+    {
+      "id": "datafeed-low_count_events_for_a_host_name",
+      "file": "datafeed_low_count_events_for_a_host_name.json",
+      "job_id": "low_count_events_for_a_host_name"
+    }
+  ],
+  "tags": [
+    "security"
+  ]
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/datafeed_high_count_events_for_a_host_name.json

@@ -0,0 +1,33 @@
+{
+  "job_id": "JOB_ID",
+  "indices": [
+    "INDEX_PATTERN_NAME"
+  ],
+  "max_empty_searches": 10,
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "exists": {
+            "field": "event.category"
+          }
+        },
+        {
+          "exists": {
+            "field": "host.name"
+          }
+        },
+        {
+          "exists": {
+            "field": "event.dataset"
+          }
+        },
+        {
+          "term": {
+            "event.outcome": "success"
+          }
+        }
+      ]
+    }
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/datafeed_low_count_events_for_a_host_name.json

@@ -0,0 +1,33 @@
+{
+  "job_id": "JOB_ID",
+  "indices": [
+    "INDEX_PATTERN_NAME"
+  ],
+  "max_empty_searches": 10,
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "exists": {
+            "field": "event.category"
+          }
+        },
+        {
+          "exists": {
+            "field": "host.name"
+          }
+        },
+        {
+          "exists": {
+            "field": "event.dataset"
+          }
+        },
+        {
+          "term": {
+            "event.outcome": "success"
+          }
+        }
+      ]
+    }
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/high_count_events_for_a_host_name.json

@@ -0,0 +1,29 @@
+{
+  "description": "Security: Host - Looks for a sudden spike in host based traffic. This can be due to a range of security issues, such as a compromised system, DDoS attacks, malware infections, privilege escalation, or data exfiltration.",
+  "groups": ["security", "host"],
+  "analysis_config": {
+    "bucket_span": "3h",
+    "detectors": [
+      {
+        "detector_description": "high count of host based events",
+        "function": "high_count",
+        "partition_field_name": "host.name",
+        "detector_index": 0
+      }
+    ],
+    "influencers": ["host.name", "host.ip", "event.dataset", "event.action", "event.category"]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "128mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-security-host",
+    "security_app_display_name": "Spike in the Host Traffic",
+    "managed": true,
+    "job_revision": 1
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/low_count_events_for_a_host_name.json

@@ -0,0 +1,29 @@
+{
+  "description": "Security: Host - Looks for a sudden drop in host based traffic. This can be due to a range of security issues, such as a compromised system, a failed service, or a network misconfiguration.",
+  "groups": ["security", "host"],
+  "analysis_config": {
+    "bucket_span": "3h",
+    "detectors": [
+      {
+        "detector_description": "low count of host based events",
+        "function": "low_count",
+        "partition_field_name": "host.name",
+        "detector_index": 0
+      }
+    ],
+    "influencers": ["host.name", "host.ip", "event.dataset", "event.action", "event.category"]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "128mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-security-host",
+    "security_app_display_name": "Decrease in the Host Traffic",
+    "managed": true,
+    "job_revision": 1
+  }
+}

x-pack/test/api_integration/apis/ml/modules/get_module.ts

@@ -28,6 +28,7 @@ const moduleIds = [
   'sample_data_weblogs',
   'security_auth',
   'security_cloudtrail',
+  'security_host',
   'security_linux_v3',
   'security_network',
   'security_packetbeat',
@@ -41,6 +42,7 @@ const securityModuleIds = [
   'logs_ui_categories',
   'security_auth',
   'security_cloudtrail',
+  'security_host',
   'security_linux_v3',
   'security_network',
   'security_packetbeat',

x-pack/test/api_integration/apis/ml/modules/recognize_module.ts

@@ -135,6 +135,7 @@ export default ({ getService }: FtrProviderContext) => {
         responseCode: 200,
         moduleIds: [
           'security_auth',
+          'security_host',
           'security_linux_v3',
           'security_network',
           'security_windows_v3',

x-pack/test/functional/services/ml/supplied_configurations.ts

@@ -27,7 +27,7 @@ export function MachineLearningSuppliedConfigurationsProvider({ getService }: Ft
       );
     },
     async assertAllConfigurationsAreLoaded() {
-      const expectedLength = 18;
+      const expectedLength = 19;
       await retry.tryForTime(10 * 1000, async () => {
         const cards = await testSubjects.findAll('mlSuppliedConfigurationsCard');
         expect(cards.length).to.eql(