diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1f12c8ea9a50..c1370e06e3ed 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -93,6 +93,17 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif *Filebeat* +- Add `text/csv` decoder to `httpjson` input {pull}28564[28564] +- Update `aws-s3` input to connect to non AWS S3 buckets {issue}28222[28222] {pull}28234[28234] +- Add support for '/var/log/pods/' path for add_kubernetes_metadata processor with `resource_type: pod`. {pull}28868[28868] +- Add documentation for add_kubernetes_metadata processors `log_path` matcher. {pull}28868[28868] +- Add support for parsers on journald input {pull}29070[29070] +- Add support in httpjson input for oAuth2ProviderDefault of password grant_type. {pull}29087[29087] +- Add support for filtering in journald input with `unit`, `kernel`, `identifiers` and `include_matches`. {pull}29294[29294] +- Add new `userAgent` and `beatInfo` template functions for httpjson input {pull}29528[29528] +- Add extraction of `related.hosts` to Microsoft 365 Defender ingest pipeline {issue}29859[29859] {pull}29863[29863] +- threatintel module: Add new Recorded Future integration. {pull}30030[30030] +- Add pipeline in FB's supported hints. {pull}30212[30212] *Auditbeat* diff --git a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml index c6b5fe5495d3..c1edf50171ba 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml @@ -235,6 +235,7 @@ processors: field: url.full ignore_failure: true if: ctx?.url?.full != null + ###################### ## ECS User Mapping ## ###################### @@ -270,10 +271,15 @@ processors: field: related.hash value: '{{file.hash.sha256}}' if: ctx?.file?.hash?.sha256 != null -- append: - field: related.hosts - value: '{{host.hostname}}' - if: ctx?.host?.hostname != null +- foreach: + field: json.alerts.devices + ignore_missing: true + processor: + append: + field: related.hosts + value: '{{ _ingest._value.deviceDnsName }}' + allow_duplicates: false + ignore_failure: true ############# ## Cleanup ## diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log index 0fd241be2e3f..b07d38465142 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log +++ b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log @@ -1,4 +1,4 @@ -{"status":"Resolved","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","determination":"NotAvailable","incidentId":12,"incidentName":"12","redirectIncidentId":null,"severity":"Low","alerts":{"creationTime":"2020-06-30T09:32:31.4579225Z","detectionSource":"WindowsDefenderAv","firstActivity":"2020-06-30T09:31:22.5729558Z","incidentId":12,"serviceSource":"MicrosoftDefenderATP","actorName":null,"alertId":"da637291063515066999_-2102938302","determination":null,"lastActivity":"2020-06-30T09:46:15.0876676Z","assignedTo":"Automation","devices":[{"osBuild":17763,"osProcessor":"x64","rbacGroupId":0,"aadDeviceId":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"riskScore":"High","version":"Other","deviceDnsName":"TestServer4","healthStatus":"Inactive","osPlatform":"Other"}],"investigationId":9,"threatFamilyName":null,"title":"'Mountsi' malware was detected","category":"Malware","classification":null,"description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"File","fileName":"amsistream-1D89ECED25A52AB98B76FF619B7BA07A","sha1":"ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281","sha256":"fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356"},"investigationState":"Benign","lastUpdatedTime":"2020-08-26T09:41:27.7233333Z","mitreTechniques":[],"resolvedTime":"2020-06-30T11:13:12.2680434Z","severity":"Informational","status":"Resolved"},"assignedTo":"elastic@elasticuser.com","lastUpdateTime":"2020-09-23T19:44:36.29Z","tags":[]} +{"status":"Resolved","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","determination":"NotAvailable","incidentId":12,"incidentName":"12","redirectIncidentId":null,"severity":"Low","alerts":{"creationTime":"2020-06-30T09:32:31.4579225Z","detectionSource":"WindowsDefenderAv","firstActivity":"2020-06-30T09:31:22.5729558Z","incidentId":12,"serviceSource":"MicrosoftDefenderATP","actorName":null,"alertId":"da637291063515066999_-2102938302","determination":null,"lastActivity":"2020-06-30T09:46:15.0876676Z","assignedTo":"Automation","devices":[{"osBuild":17763,"osProcessor":"x64","rbacGroupId":0,"aadDeviceId":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"riskScore":"High","version":"Other","deviceDnsName":"TestServer4","healthStatus":"Inactive","osPlatform":"Other"},{"osBuild":17763,"osProcessor":"x64","rbacGroupId":0,"aadDeviceId":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"riskScore":"High","version":"Other","deviceDnsName":"TestServer5","healthStatus":"Inactive","osPlatform":"Other"}],"investigationId":9,"threatFamilyName":null,"title":"'Mountsi' malware was detected","category":"Malware","classification":null,"description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"File","fileName":"amsistream-1D89ECED25A52AB98B76FF619B7BA07A","sha1":"ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281","sha256":"fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356"},"investigationState":"Benign","lastUpdatedTime":"2020-08-26T09:41:27.7233333Z","mitreTechniques":[],"resolvedTime":"2020-06-30T11:13:12.2680434Z","severity":"Informational","status":"Resolved"},"assignedTo":"elastic@elasticuser.com","lastUpdateTime":"2020-09-23T19:44:36.29Z","tags":[]} {"incidentName":"12","redirectIncidentId":null,"severity":"Low","status":"Resolved","tags":[],"alerts":{"devices":[{"aadDeviceId":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","healthStatus":"Inactive","osPlatform":"Other","rbacGroupId":0,"rbacGroupName":null,"deviceDnsName":"TestServer4","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osBuild":17763,"osProcessor":"x64","riskScore":"High","version":"Other"}],"entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"File","fileName":"amsistream-B103C1A335BDB049E617D1AC4A41FCDC","sha1":"ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281","sha256":"fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356"},"firstActivity":"2020-06-30T09:31:22.5729558Z","lastUpdatedTime":"2020-08-26T09:41:27.7233333Z","alertId":"da637291063515066999_-2102938302","category":"Malware","classification":null,"determination":null,"mitreTechniques":[],"serviceSource":"MicrosoftDefenderATP","status":"Resolved","assignedTo":"Automation","detectionSource":"WindowsDefenderAv","incidentId":12,"investigationId":9,"severity":"Informational","title":"'Mountsi' malware was detected","actorName":null,"description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","lastActivity":"2020-06-30T09:46:15.0876676Z","resolvedTime":"2020-06-30T11:13:12.2680434Z","creationTime":"2020-06-30T09:32:31.4579225Z","investigationState":"Benign","threatFamilyName":null},"assignedTo":"elastic@elasticuser.com","determination":"NotAvailable","incidentId":12,"lastUpdateTime":"2020-09-23T19:44:36.29Z","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z"} {"alerts":{"mitreTechniques":[],"status":"Resolved","title":"'Exeselrun' malware was detected","threatFamilyName":null,"alertId":"da637291085389812387_-1296910232","category":"Malware","detectionSource":"WindowsDefenderAv","lastUpdatedTime":"2020-08-26T09:41:27.7233333Z","serviceSource":"MicrosoftDefenderATP","severity":"Informational","resolvedTime":"2020-06-30T11:13:12.2680434Z","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","devices":[{"deviceDnsName":"testserver4","healthStatus":"Inactive","osProcessor":"x64","rbacGroupId":0,"riskScore":"High","version":"Other","aadDeviceId":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osBuild":17763,"osPlatform":"Other","rbacGroupName":null}],"firstActivity":"2020-06-30T10:07:44.3144099Z","incidentId":12,"investigationId":9,"investigationState":"Benign","lastActivity":"2020-06-30T10:07:44.3144099Z","actorName":null,"assignedTo":"Automation","classification":null,"creationTime":"2020-06-30T10:08:58.9655663Z","determination":null,"entities":{"fileName":"SB.xsl","filePath":"C:\\Windows\\Temp\\sb-sim-temp-ikyxqi\\sb_10554_bs_h4qpk5","sha1":"d1bb29ce3d01d01451e3623132545d5f577a1bd6","sha256":"ce8d3a3811a3bf923902d6924532308506fe5d024435ddee0cabf90ad9b29f6a","deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"File"}},"createdTime":"2020-06-30T09:32:31.85Z","determination":"NotAvailable","incidentId":12,"lastUpdateTime":"2020-09-23T19:44:36.29Z","redirectIncidentId":null,"assignedTo":"elastic@elasticuser.com","classification":"Unknown","incidentName":"12","severity":"Low","status":"Resolved","tags":[]} {"classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","determination":"NotAvailable","incidentName":"12","lastUpdateTime":"2020-09-23T19:44:36.29Z","redirectIncidentId":null,"severity":"Low","assignedTo":"elastic@elasticuser.com","status":"Resolved","incidentId":12,"tags":[],"alerts":{"assignedTo":"elastic@elasticuser.com","firstActivity":"2020-06-30T10:07:44.333733Z","investigationId":9,"mitreTechniques":[],"resolvedTime":"2020-06-30T11:13:12.2680434Z","title":"An active 'Exeselrun' malware was detected","alertId":"da637291085411733957_-1043898914","category":"Malware","classification":null,"detectionSource":"WindowsDefenderAv","determination":null,"threatFamilyName":null,"actorName":null,"serviceSource":"MicrosoftDefenderATP","status":"Resolved","creationTime":"2020-06-30T10:09:01.1569718Z","devices":[{"deviceDnsName":"TestServer4","healthStatus":"Inactive","osBuild":17763,"osProcessor":"x64","rbacGroupName":null,"riskScore":"High","version":"Other","aadDeviceId":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osPlatform":"Other","rbacGroupId":0}],"entities":{"filePath":"C:\\Windows\\Temp\\sb-sim-temp-ikyxqi\\sb_10554_bs_h4qpk5","deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"File","fileName":"SB.xsl"},"incidentId":12,"investigationState":"Benign","lastActivity":"2020-06-30T10:07:44.333733Z","lastUpdatedTime":"2020-08-26T09:41:27.7233333Z","severity":"Low","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection."}} diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json index 8a6c6766cedb..e5e7070d1ae4 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json +++ b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json @@ -41,6 +41,18 @@ "rbacGroupId": 0, "riskScore": "High", "version": "Other" + }, + { + "deviceDnsName": "TestServer5", + "firstSeen": "2020-06-30T08:55:08.8320449Z", + "healthStatus": "Inactive", + "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "osBuild": 17763, + "osPlatform": "Other", + "osProcessor": "x64", + "rbacGroupId": 0, + "riskScore": "High", + "version": "Other" } ], "microsoft.m365_defender.alerts.entities.deviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", @@ -65,6 +77,10 @@ "fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356", "ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281" ], + "related.hosts": [ + "TestServer4", + "TestServer5" + ], "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.", "service.type": "microsoft", "tags": [ @@ -99,7 +115,7 @@ "file.name": "amsistream-B103C1A335BDB049E617D1AC4A41FCDC", "fileset.name": "m365_defender", "input.type": "log", - "log.offset": 2071, + "log.offset": 2381, "message": "'Mountsi' malware was detected", "microsoft.m365_defender.alerts.assignedTo": "Automation", "microsoft.m365_defender.alerts.creationTime": "2020-06-30T09:32:31.4579225Z", @@ -140,6 +156,9 @@ "fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356", "ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281" ], + "related.hosts": [ + "TestServer4" + ], "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.", "service.type": "microsoft", "tags": [ @@ -175,7 +194,7 @@ "file.path": "C:\\Windows\\Temp\\sb-sim-temp-ikyxqi\\sb_10554_bs_h4qpk5", "fileset.name": "m365_defender", "input.type": "log", - "log.offset": 4142, + "log.offset": 4452, "message": "'Exeselrun' malware was detected", "microsoft.m365_defender.alerts.assignedTo": "Automation", "microsoft.m365_defender.alerts.creationTime": "2020-06-30T10:08:58.9655663Z", @@ -216,6 +235,9 @@ "ce8d3a3811a3bf923902d6924532308506fe5d024435ddee0cabf90ad9b29f6a", "d1bb29ce3d01d01451e3623132545d5f577a1bd6" ], + "related.hosts": [ + "testserver4" + ], "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.", "service.type": "microsoft", "tags": [ @@ -249,7 +271,7 @@ "file.path": "C:\\Windows\\Temp\\sb-sim-temp-ikyxqi\\sb_10554_bs_h4qpk5", "fileset.name": "m365_defender", "input.type": "log", - "log.offset": 6249, + "log.offset": 6559, "message": "An active 'Exeselrun' malware was detected", "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com", "microsoft.m365_defender.alerts.creationTime": "2020-06-30T10:09:01.1569718Z", @@ -286,6 +308,9 @@ "observer.name": "MicrosoftDefenderATP", "observer.product": "365 Defender", "observer.vendor": "Microsoft", + "related.hosts": [ + "TestServer4" + ], "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.", "service.type": "microsoft", "tags": [ @@ -317,7 +342,7 @@ ], "fileset.name": "m365_defender", "input.type": "log", - "log.offset": 8376, + "log.offset": 8686, "message": "Suspicious 'AccessibilityEscalation' behavior was detected", "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com", "microsoft.m365_defender.alerts.classification": "FalsePositive", @@ -356,6 +381,9 @@ "observer.vendor": "Microsoft", "process.pid": 6720, "process.start": "2020-06-30T10:31:04.1092404Z", + "related.hosts": [ + "testserver4" + ], "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.", "service.type": "microsoft", "tags": [ @@ -387,7 +415,7 @@ ], "fileset.name": "m365_defender", "input.type": "log", - "log.offset": 10542, + "log.offset": 10852, "message": "Suspicious 'AccessibilityEscalation' behavior was detected", "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com", "microsoft.m365_defender.alerts.classification": "FalsePositive", @@ -424,6 +452,9 @@ "observer.name": "MicrosoftDefenderATP", "observer.product": "365 Defender", "observer.vendor": "Microsoft", + "related.hosts": [ + "testserver4" + ], "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.", "service.type": "microsoft", "tags": [ @@ -455,7 +486,7 @@ ], "fileset.name": "m365_defender", "input.type": "log", - "log.offset": 12598, + "log.offset": 12908, "message": "Suspicious 'AccessibilityEscalation' behavior was detected", "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com", "microsoft.m365_defender.alerts.classification": "FalsePositive", @@ -494,6 +525,9 @@ "observer.vendor": "Microsoft", "process.pid": 1324, "process.start": "2020-06-30T10:09:10.5747992Z", + "related.hosts": [ + "testserver4" + ], "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.", "service.type": "microsoft", "tags": [ @@ -522,7 +556,7 @@ "event.timezone": "UTC", "fileset.name": "m365_defender", "input.type": "log", - "log.offset": 14764, + "log.offset": 15074, "message": "Activity from infrequent country", "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com", "microsoft.m365_defender.alerts.classification": "FalsePositive", @@ -575,7 +609,7 @@ "event.timezone": "UTC", "fileset.name": "m365_defender", "input.type": "log", - "log.offset": 15990, + "log.offset": 16300, "message": "Activity from infrequent country", "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com", "microsoft.m365_defender.alerts.classification": "FalsePositive", @@ -611,4 +645,4 @@ "user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3", "user.name": "brent@elasticbv.onmicrosoft.com" } -] \ No newline at end of file +]