From 4135920bbdaa22d3540d44680732185f953eae2b Mon Sep 17 00:00:00 2001 From: Kirti Kirti Date: Tue, 8 Oct 2024 12:08:34 -0400 Subject: [PATCH 1/5] Add new prebuilt security module --- .../modules/security_host/logo.json | 3 + .../modules/security_host/manifest.json | 60 +++++++++++++++++++ ...eed_high_count_events_for_a_host_name.json | 33 ++++++++++ ...feed_low_count_events_for_a_host_name.json | 33 ++++++++++ .../ml/high_count_events_for_a_host_name.json | 29 +++++++++ .../ml/low_count_events_for_a_host_name.json | 29 +++++++++ 6 files changed, 187 insertions(+) create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/logo.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/manifest.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/datafeed_high_count_events_for_a_host_name.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/datafeed_low_count_events_for_a_host_name.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/high_count_events_for_a_host_name.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/low_count_events_for_a_host_name.json diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/logo.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/logo.json new file mode 100644 index 0000000000000..862f970b7405d --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/logo.json @@ -0,0 +1,3 @@ +{ + "icon": "logoSecurity" +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/manifest.json new file mode 100644 index 0000000000000..46d35c3761b6e --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/manifest.json @@ -0,0 +1,60 @@ +{ + "id": "security_host", + "title": "Security: Host", + "description": "Detect anomalous activity in your ECS-compatible host-based logs.", + "type": "Host data", + "logoFile": "logo.json", + "defaultIndexPattern": "auditbeat-*,logs-*,filebeat-*,winlogbeat-*", + "query": { + "bool": { + "filter": [ + { + "exists": { + "field": "event.category" + } + }, + { + "exists": { + "field": "host.name" + } + }, + { + "exists": { + "field": "event.dataset" + } + }, + { + "term": { + "event.outcome": "success" + } + } + ], + "must_not": { "terms": { "_tier": ["data_frozen", "data_cold"] } } + } + }, + "jobs": [ + { + "id": "high_count_events_for_a_host_name", + "file": "high_count_events_for_a_host_name.json" + }, + { + "id": "low_count_events_for_a_host_name", + "file": "low_count_events_for_a_host_name.json" + } + ], + "datafeeds": [ + { + "id": "datafeed-high_count_events_for_a_host_name", + "file": "datafeed_high_count_events_for_a_host_name.json", + "job_id": "high_count_events_for_a_host_name" + }, + { + "id": "datafeed-low_count_events_for_a_host_name", + "file": "datafeed_low_count_events_for_a_host_name.json", + "job_id": "low_count_events_for_a_host_name" + } + ], + "tags": [ + "security" + ] +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/datafeed_high_count_events_for_a_host_name.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/datafeed_high_count_events_for_a_host_name.json new file mode 100644 index 0000000000000..2e6792469a2cd --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/datafeed_high_count_events_for_a_host_name.json @@ -0,0 +1,33 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "exists": { + "field": "event.category" + } + }, + { + "exists": { + "field": "host.name" + } + }, + { + "exists": { + "field": "event.dataset" + } + }, + { + "term": { + "event.outcome": "success" + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/datafeed_low_count_events_for_a_host_name.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/datafeed_low_count_events_for_a_host_name.json new file mode 100644 index 0000000000000..2e6792469a2cd --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/datafeed_low_count_events_for_a_host_name.json @@ -0,0 +1,33 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "exists": { + "field": "event.category" + } + }, + { + "exists": { + "field": "host.name" + } + }, + { + "exists": { + "field": "event.dataset" + } + }, + { + "term": { + "event.outcome": "success" + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/high_count_events_for_a_host_name.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/high_count_events_for_a_host_name.json new file mode 100644 index 0000000000000..6942adf56fb45 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/high_count_events_for_a_host_name.json @@ -0,0 +1,29 @@ +{ + "description": "Security: Host - Looks for a sudden spike in host based traffic. This can be due to a range of security issues, such as a compromised system, DDoS attacks, malware infections, privilege escalation, or data exfiltration.", + "groups": ["security", "host"], + "analysis_config": { + "bucket_span": "3h", + "detectors": [ + { + "detector_description": "Detects high count of host based events.", + "function": "high_count", + "partition_field_name": "host.name", + "detector_index": 0 + } + ], + "influencers": ["host.name", "host.ip", "event.dataset", "event.action", "event.category"] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "128mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-host", + "security_app_display_name": "Spike in the Host Traffic", + "managed": true, + "job_revision": 1 + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/low_count_events_for_a_host_name.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/low_count_events_for_a_host_name.json new file mode 100644 index 0000000000000..6efa6226c554a --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/low_count_events_for_a_host_name.json @@ -0,0 +1,29 @@ +{ + "description": "Security: Host - Looks for a sudden drop in host based traffic. This can be due to a range of security issues, such as a compromised system, a failed service, or a network misconfiguration.", + "groups": ["security", "host"], + "analysis_config": { + "bucket_span": "3h", + "detectors": [ + { + "detector_description": "Detects low count of host based events.", + "function": "low_count", + "partition_field_name": "host.name", + "detector_index": 0 + } + ], + "influencers": ["host.name", "host.ip", "event.dataset", "event.action", "event.category"] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "128mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-host", + "security_app_display_name": "Decrease in the Host Traffic", + "managed": true, + "job_revision": 1 + } +} From ed39312e64780074b734222d5724a77e31817a2f Mon Sep 17 00:00:00 2001 From: Kirti Kirti Date: Fri, 11 Oct 2024 08:37:56 -0400 Subject: [PATCH 2/5] Add security_host to module list --- x-pack/test/api_integration/apis/ml/modules/get_module.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/test/api_integration/apis/ml/modules/get_module.ts b/x-pack/test/api_integration/apis/ml/modules/get_module.ts index ee223eb0de3eb..d8f3ce7365260 100644 --- a/x-pack/test/api_integration/apis/ml/modules/get_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/get_module.ts @@ -28,6 +28,7 @@ const moduleIds = [ 'sample_data_weblogs', 'security_auth', 'security_cloudtrail', + 'security_host', 'security_linux_v3', 'security_network', 'security_packetbeat', @@ -41,6 +42,7 @@ const securityModuleIds = [ 'logs_ui_categories', 'security_auth', 'security_cloudtrail', + 'security_host', 'security_linux_v3', 'security_network', 'security_packetbeat', From 2f203c808074ef771728fa00af1365ee5fd45088 Mon Sep 17 00:00:00 2001 From: Kirti Kirti Date: Fri, 11 Oct 2024 08:38:49 -0400 Subject: [PATCH 3/5] change expected length of config to 19 --- x-pack/test/functional/services/ml/supplied_configurations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/functional/services/ml/supplied_configurations.ts b/x-pack/test/functional/services/ml/supplied_configurations.ts index 622d662994533..ff92877fcc4b3 100644 --- a/x-pack/test/functional/services/ml/supplied_configurations.ts +++ b/x-pack/test/functional/services/ml/supplied_configurations.ts @@ -27,7 +27,7 @@ export function MachineLearningSuppliedConfigurationsProvider({ getService }: Ft ); }, async assertAllConfigurationsAreLoaded() { - const expectedLength = 18; + const expectedLength = 19; await retry.tryForTime(10 * 1000, async () => { const cards = await testSubjects.findAll('mlSuppliedConfigurationsCard'); expect(cards.length).to.eql( From f7ae3ecba3440b4e5bfb1beba489969eec77da76 Mon Sep 17 00:00:00 2001 From: Kirti Kirti Date: Fri, 11 Oct 2024 08:39:46 -0400 Subject: [PATCH 4/5] Update detector description of the jobs --- .../security_host/ml/high_count_events_for_a_host_name.json | 2 +- .../security_host/ml/low_count_events_for_a_host_name.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/high_count_events_for_a_host_name.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/high_count_events_for_a_host_name.json index 6942adf56fb45..f103d2b34c5ad 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/high_count_events_for_a_host_name.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/high_count_events_for_a_host_name.json @@ -5,7 +5,7 @@ "bucket_span": "3h", "detectors": [ { - "detector_description": "Detects high count of host based events.", + "detector_description": "high count of host based events", "function": "high_count", "partition_field_name": "host.name", "detector_index": 0 diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/low_count_events_for_a_host_name.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/low_count_events_for_a_host_name.json index 6efa6226c554a..ae8bfd163826b 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/low_count_events_for_a_host_name.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_host/ml/low_count_events_for_a_host_name.json @@ -5,7 +5,7 @@ "bucket_span": "3h", "detectors": [ { - "detector_description": "Detects low count of host based events.", + "detector_description": "low count of host based events", "function": "low_count", "partition_field_name": "host.name", "detector_index": 0 From ccbfe6525de6bf1419dba87ce3c2c51ce3e957c1 Mon Sep 17 00:00:00 2001 From: Kirti Kirti Date: Fri, 11 Oct 2024 11:57:57 -0400 Subject: [PATCH 5/5] add security_host to recognize_module.ts --- x-pack/test/api_integration/apis/ml/modules/recognize_module.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts b/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts index 31dd5f3354635..3daa5e73f308a 100644 --- a/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts @@ -135,6 +135,7 @@ export default ({ getService }: FtrProviderContext) => { responseCode: 200, moduleIds: [ 'security_auth', + 'security_host', 'security_linux_v3', 'security_network', 'security_windows_v3',